diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxproj index a84aa41e3..88d9b8814 100644 --- a/Fluid.xcodeproj/project.pbxproj +++ b/Fluid.xcodeproj/project.pbxproj @@ -29,6 +29,7 @@ 803000000000000000000003 /* MediaRemoteAdapter in Frameworks */ = {isa = PBXBuildFile; productRef = 803000000000000000000004 /* MediaRemoteAdapter */; }; B51800000000000000000002 /* SpokenSendTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B51800000000000000000001 /* SpokenSendTests.swift */; }; B51900000000000000000002 /* PrivateAIProviderPromptFormatTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B51900000000000000000001 /* PrivateAIProviderPromptFormatTests.swift */; }; + FEED67600000000000000002 /* PasteDeliveryCoordinatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = FEED67600000000000000001 /* PasteDeliveryCoordinatorTests.swift */; }; 7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; }; 7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; }; 7CE006BD2E80EBE600DDCCD6 /* AppUpdater in Frameworks */ = {isa = PBXBuildFile; productRef = 7CE006BC2E80EBE600DDCCD6 /* AppUpdater */; }; @@ -70,6 +71,7 @@ DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectAudioReliabilityTests.swift; sourceTree = ""; }; B51800000000000000000001 /* SpokenSendTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpokenSendTests.swift; sourceTree = ""; }; B51900000000000000000001 /* PrivateAIProviderPromptFormatTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PrivateAIProviderPromptFormatTests.swift; sourceTree = ""; }; + FEED67600000000000000001 /* PasteDeliveryCoordinatorTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PasteDeliveryCoordinatorTests.swift; sourceTree = ""; }; 7C078D8F2E3B339200FB7CAC /* FluidVoice Debug.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "FluidVoice Debug.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HotkeyShortcutTests.swift; sourceTree = ""; }; 7CDB0A202F3C4D5600FB7CAD /* FluidDictationIntegrationTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = FluidDictationIntegrationTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -161,6 +163,7 @@ 803000000000000000000001 /* MediaPlaybackServiceTests.swift */, B51800000000000000000001 /* SpokenSendTests.swift */, B51900000000000000000001 /* PrivateAIProviderPromptFormatTests.swift */, + FEED67600000000000000001 /* PasteDeliveryCoordinatorTests.swift */, ); path = FluidDictationIntegrationTests; sourceTree = ""; @@ -332,6 +335,7 @@ 803000000000000000000002 /* MediaPlaybackServiceTests.swift in Sources */, B51800000000000000000002 /* SpokenSendTests.swift in Sources */, B51900000000000000000002 /* PrivateAIProviderPromptFormatTests.swift in Sources */, + FEED67600000000000000002 /* PasteDeliveryCoordinatorTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Sources/Fluid/Analytics/AnalyticsDatabase.swift b/Sources/Fluid/Analytics/AnalyticsDatabase.swift index e9ba54a86..742403687 100644 --- a/Sources/Fluid/Analytics/AnalyticsDatabase.swift +++ b/Sources/Fluid/Analytics/AnalyticsDatabase.swift @@ -111,6 +111,72 @@ final class AnalyticsDatabase { } } + func recordInsertionLatency( + path: AnalyticsInsertionPath, + outcome: AnalyticsInsertionOutcome, + requestMilliseconds: Int, + readyMilliseconds: Int?, + toggleStopMilliseconds: Int? = nil, + at date: Date + ) throws { + try self.finalizeDays(before: date) + let requestMilliseconds = max(0, requestMilliseconds) + let readyMilliseconds = readyMilliseconds.map { max(0, $0) } + let readyCount = readyMilliseconds == nil ? 0 : 1 + let readyValue = readyMilliseconds ?? 0 + let day = self.dayString(date) + + try self.transaction { + try self.run( + "INSERT INTO daily_insertion_latency (" + + "day, delivery_path, outcome, request_count, request_total_ms, request_min_ms, request_max_ms, " + + "ready_count, ready_total_ms, ready_min_ms, ready_max_ms" + + ") VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?) " + + "ON CONFLICT(day, delivery_path, outcome) DO UPDATE SET " + + "request_count = request_count + 1, " + + "request_total_ms = request_total_ms + excluded.request_total_ms, " + + "request_min_ms = MIN(request_min_ms, excluded.request_min_ms), " + + "request_max_ms = MAX(request_max_ms, excluded.request_max_ms), " + + "ready_count = ready_count + excluded.ready_count, " + + "ready_total_ms = ready_total_ms + excluded.ready_total_ms, " + + "ready_min_ms = CASE " + + "WHEN excluded.ready_count = 0 THEN ready_min_ms " + + "WHEN ready_count = 0 THEN excluded.ready_min_ms " + + "ELSE MIN(ready_min_ms, excluded.ready_min_ms) END, " + + "ready_max_ms = CASE " + + "WHEN excluded.ready_count = 0 THEN ready_max_ms " + + "WHEN ready_count = 0 THEN excluded.ready_max_ms " + + "ELSE MAX(ready_max_ms, excluded.ready_max_ms) END", + bindings: [ + .text(day), .text(path.rawValue), .text(outcome.rawValue), + .integer(requestMilliseconds), .integer(requestMilliseconds), .integer(requestMilliseconds), + .integer(readyCount), .integer(readyValue), .integer(readyValue), .integer(readyValue), + ] + ) + + if let toggleStopMilliseconds, + outcome == .dispatched, + path == .clipboard || path == .clipboardFallback + { + let milliseconds = max(0, toggleStopMilliseconds) + try self.run( + "INSERT INTO daily_clipboard_toggle_latency " + + "(day, delivery_path, sample_count, total_ms, min_ms, max_ms) " + + "VALUES (?, ?, 1, ?, ?, ?) " + + "ON CONFLICT(day, delivery_path) DO UPDATE SET " + + "sample_count = sample_count + 1, " + + "total_ms = total_ms + excluded.total_ms, " + + "min_ms = MIN(min_ms, excluded.min_ms), " + + "max_ms = MAX(max_ms, excluded.max_ms)", + bindings: [ + .text(day), .text(path.rawValue), + .integer(milliseconds), .integer(milliseconds), .integer(milliseconds), + ] + ) + } + } + } + func recordOnboardingStarted(origin: AnalyticsOnboardingOrigin, at date: Date) throws { try self.finalizeDays(before: date) try self.transaction { @@ -273,8 +339,20 @@ final class AnalyticsDatabase { "WHERE day < ? ORDER BY day, role, mode, provider, model", bindings: [.text(today)] ) + let insertionRows = try self.query( + "SELECT day, delivery_path, outcome, request_count, request_total_ms, request_min_ms, " + + "request_max_ms, ready_count, ready_total_ms, ready_min_ms, ready_max_ms " + + "FROM daily_insertion_latency WHERE day < ? ORDER BY day, delivery_path, outcome", + bindings: [.text(today)] + ) + let clipboardToggleRows = try self.query( + "SELECT day, delivery_path, sample_count, total_ms, min_ms, max_ms " + + "FROM daily_clipboard_toggle_latency WHERE day < ? ORDER BY day, delivery_path", + bindings: [.text(today)] + ) - guard !usageRows.isEmpty || !modelRows.isEmpty else { return } + guard !usageRows.isEmpty || !modelRows.isEmpty || !insertionRows.isEmpty || !clipboardToggleRows.isEmpty + else { return } try self.transaction { for row in usageRows where row.count == 5 { try self.enqueue(.usageDailySummary, at: date, properties: [ @@ -295,8 +373,49 @@ final class AnalyticsDatabase { "use_count": Int(row[5]) ?? 0, ]) } + for row in insertionRows where row.count == 11 { + let requestCount = Int(row[3]) ?? 0 + let requestTotal = Int(row[4]) ?? 0 + let readyCount = Int(row[7]) ?? 0 + let readyTotal = Int(row[8]) ?? 0 + let toggleRow = clipboardToggleRows.first { + $0.count == 6 && + row[2] == AnalyticsInsertionOutcome.dispatched.rawValue && + $0[0] == row[0] && + $0[1] == row[1] + } + var properties: [String: Any] = [ + "latency_date": row[0], + "delivery_path": row[1], + "outcome": row[2], + "request_count": requestCount, + "request_total_ms": requestTotal, + "request_average_ms": Self.average(total: requestTotal, count: requestCount), + "request_min_ms": Int(row[5]) ?? 0, + "request_max_ms": Int(row[6]) ?? 0, + "ready_count": readyCount, + ] + if readyCount > 0 { + properties["ready_total_ms"] = readyTotal + properties["ready_average_ms"] = Self.average(total: readyTotal, count: readyCount) + properties["ready_min_ms"] = Int(row[9]) ?? 0 + properties["ready_max_ms"] = Int(row[10]) ?? 0 + } + if let toggleRow { + let count = Int(toggleRow[2]) ?? 0 + let total = Int(toggleRow[3]) ?? 0 + properties["toggle_stop_to_dispatch_count"] = count + properties["toggle_stop_to_dispatch_total_ms"] = total + properties["toggle_stop_to_dispatch_average_ms"] = Self.average(total: total, count: count) + properties["toggle_stop_to_dispatch_min_ms"] = Int(toggleRow[4]) ?? 0 + properties["toggle_stop_to_dispatch_max_ms"] = Int(toggleRow[5]) ?? 0 + } + try self.enqueue(.insertionLatencyDailySummary, at: date, properties: properties) + } try self.run("DELETE FROM daily_usage WHERE day < ?", bindings: [.text(today)]) try self.run("DELETE FROM daily_model_usage WHERE day < ?", bindings: [.text(today)]) + try self.run("DELETE FROM daily_insertion_latency WHERE day < ?", bindings: [.text(today)]) + try self.run("DELETE FROM daily_clipboard_toggle_latency WHERE day < ?", bindings: [.text(today)]) } } @@ -364,6 +483,8 @@ final class AnalyticsDatabase { try self.execute("DELETE FROM outbox") try self.execute("DELETE FROM daily_usage") try self.execute("DELETE FROM daily_model_usage") + try self.execute("DELETE FROM daily_insertion_latency") + try self.execute("DELETE FROM daily_clipboard_toggle_latency") try self.execute("DELETE FROM event_dedupe") try self.execute("DELETE FROM onboarding_flows") try self.execute("DELETE FROM model_download_attempts") @@ -398,6 +519,29 @@ final class AnalyticsDatabase { use_count INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(day, role, mode, provider, model) ); + CREATE TABLE IF NOT EXISTS daily_insertion_latency ( + day TEXT NOT NULL, + delivery_path TEXT NOT NULL, + outcome TEXT NOT NULL, + request_count INTEGER NOT NULL DEFAULT 0, + request_total_ms INTEGER NOT NULL DEFAULT 0, + request_min_ms INTEGER NOT NULL DEFAULT 0, + request_max_ms INTEGER NOT NULL DEFAULT 0, + ready_count INTEGER NOT NULL DEFAULT 0, + ready_total_ms INTEGER NOT NULL DEFAULT 0, + ready_min_ms INTEGER NOT NULL DEFAULT 0, + ready_max_ms INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(day, delivery_path, outcome) + ); + CREATE TABLE IF NOT EXISTS daily_clipboard_toggle_latency ( + day TEXT NOT NULL, + delivery_path TEXT NOT NULL, + sample_count INTEGER NOT NULL DEFAULT 0, + total_ms INTEGER NOT NULL DEFAULT 0, + min_ms INTEGER NOT NULL DEFAULT 0, + max_ms INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(day, delivery_path) + ); CREATE TABLE IF NOT EXISTS event_dedupe ( dedupe_key TEXT PRIMARY KEY, created_at REAL NOT NULL @@ -526,6 +670,11 @@ final class AnalyticsDatabase { return String(format: "%04d-%02d-%02d", components.year ?? 0, components.month ?? 0, components.day ?? 0) } + private static func average(total: Int, count: Int) -> Double { + guard count > 0 else { return 0 } + return (Double(total) / Double(count) * 10).rounded() / 10 + } + private func purgeDeletedPages() throws { try self.execute("PRAGMA incremental_vacuum") try self.execute("PRAGMA wal_checkpoint(TRUNCATE)") diff --git a/Sources/Fluid/Analytics/AnalyticsEvent.swift b/Sources/Fluid/Analytics/AnalyticsEvent.swift index 3e8e52ee0..bbee4ec3e 100644 --- a/Sources/Fluid/Analytics/AnalyticsEvent.swift +++ b/Sources/Fluid/Analytics/AnalyticsEvent.swift @@ -5,6 +5,7 @@ enum AnalyticsEvent: String { case activeUser = "active_user" case usageDailySummary = "usage_daily_summary" case modelUsageDailySummary = "model_usage_daily_summary" + case insertionLatencyDailySummary = "insertion_latency_daily_summary" case onboardingStarted = "onboarding_started" case onboardingStepViewed = "onboarding_step_viewed" case onboardingStepCompleted = "onboarding_step_completed" @@ -30,6 +31,24 @@ enum AnalyticsModelRole: String { case aiPostProcessing = "ai_post_processing" } +enum AnalyticsInsertionPath: String { + case clipboard + case direct + case clipboardFallback = "clipboard_fallback" + case notAttempted = "not_attempted" +} + +enum AnalyticsInsertionOutcome: String { + case dispatched + case emptyText = "empty_text" + case accessibilityNotTrusted = "accessibility_not_trusted" + case clipboardSnapshotFailed = "clipboard_snapshot_failed" + case clipboardWriteFailed = "clipboard_write_failed" + case pasteCommandFailed = "paste_command_failed" + case targetUnavailable = "target_unavailable" + case targetRestoreFailed = "target_restore_failed" +} + struct AnalyticsModelDescriptor: Equatable { let provider: String let model: String diff --git a/Sources/Fluid/Analytics/AnalyticsService.swift b/Sources/Fluid/Analytics/AnalyticsService.swift index a4282af7c..dfaa28f21 100644 --- a/Sources/Fluid/Analytics/AnalyticsService.swift +++ b/Sources/Fluid/Analytics/AnalyticsService.swift @@ -60,6 +60,25 @@ final class AnalyticsService { } } + func recordInsertionLatency( + path: AnalyticsInsertionPath, + outcome: AnalyticsInsertionOutcome, + requestMilliseconds: Int, + readyMilliseconds: Int?, + toggleStopMilliseconds: Int? + ) { + self.submit { core, context in + await core.recordInsertionLatency( + path: path, + outcome: outcome, + requestMilliseconds: requestMilliseconds, + readyMilliseconds: readyMilliseconds, + toggleStopMilliseconds: toggleStopMilliseconds, + context: context + ) + } + } + func recordOnboardingStarted(origin: AnalyticsOnboardingOrigin) { self.submit { core, context in await core.recordOnboardingStarted(origin: origin, context: context) @@ -251,6 +270,26 @@ private actor AnalyticsCore { } } + func recordInsertionLatency( + path: AnalyticsInsertionPath, + outcome: AnalyticsInsertionOutcome, + requestMilliseconds: Int, + readyMilliseconds: Int?, + toggleStopMilliseconds: Int?, + context: AnalyticsContext + ) async { + await self.write(context: context) { database, date in + try database.recordInsertionLatency( + path: path, + outcome: outcome, + requestMilliseconds: requestMilliseconds, + readyMilliseconds: readyMilliseconds, + toggleStopMilliseconds: toggleStopMilliseconds, + at: date + ) + } + } + func recordOnboardingStarted(origin: AnalyticsOnboardingOrigin, context: AnalyticsContext) async { await self.write(context: context) { database, date in try database.recordOnboardingStarted(origin: origin, at: date) diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index bba0d6793..b9c88319f 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -1634,15 +1634,19 @@ struct ContentView: View { private func deliverSpokenSend( _ outputPlan: DictationLiteralOutputPlan, targetPID: pid_t?, - textReadyAt: TimeInterval + textReadyAt: TimeInterval, + toggleStopRequestedAt: TimeInterval?, + preserveTranscriptOnClipboard: Bool ) async -> TypingService.DeliveryOutcome { let sendsExistingDraft = outputPlan.plainText.isEmpty let outcome = await self.asr.typeOutputPlanToActiveFieldAndWait( outputPlan, preferredTargetPID: targetPID, textReadyAt: textReadyAt, + toggleStopRequestedAt: toggleStopRequestedAt, postInsertionKey: self.settings.spokenSendKey, - requiredFocusTarget: self.recordingFocusTarget + requiredFocusTarget: self.recordingFocusTarget, + preserveTranscriptOnClipboard: preserveTranscriptOnClipboard ) if outcome.didDispatchAction { NotchContentState.shared.setSpokenSendIndicatorState(.sent) @@ -1681,11 +1685,20 @@ struct ContentView: View { private func captureRecordingTargetContext() { // Capture the focused target PID BEFORE any overlay/UI changes. // Used to restore focus when the user interacts with overlay dropdowns. - let focusTarget = TypingService.captureSystemFocusTarget() - self.recordingFocusTarget = focusTarget - let focusedPID = focusTarget?.pid + let targetContext = TypingService.captureRecordingTargetContext() + if let targetContext, let element = targetContext.element { + self.recordingFocusTarget = TypingService.CapturedFocusTarget( + pid: targetContext.pid, + window: targetContext.window, + element: element + ) + } else { + self.recordingFocusTarget = TypingService.captureSystemFocusTarget() + } + NotchContentState.shared.recordingTargetContext = targetContext + NotchContentState.shared.recordingTargetPID = targetContext?.pid + ?? self.recordingFocusTarget?.pid ?? NSWorkspace.shared.frontmostApplication?.processIdentifier - NotchContentState.shared.recordingTargetPID = focusedPID let info = self.getCurrentAppInfo() self.recordingAppInfo = info @@ -1715,19 +1728,12 @@ struct ContentView: View { } private func resolveTypingTargetPID() -> (pid: pid_t?, shouldRestoreOriginalFocus: Bool) { - let originalPID = NotchContentState.shared.recordingTargetPID - let currentFocusedPID = TypingService.captureSystemFocusedPID() - ?? NSWorkspace.shared.frontmostApplication?.processIdentifier - - let selfBundleID = Bundle.main.bundleIdentifier - if let currentFocusedPID, - let app = NSRunningApplication(processIdentifier: currentFocusedPID), - app.bundleIdentifier != selfBundleID - { - return (currentFocusedPID, currentFocusedPID == originalPID) + guard let context = NotchContentState.shared.recordingTargetContext else { + return (NotchContentState.shared.recordingTargetPID, true) } - - return (originalPID, true) + let isStillFocused = context.pid == TypingService.currentFocusedPID() && + (context.element == nil || TypingService.isCapturedFocusStillActive(context)) + return (context.pid, !isStillFocused) } // MARK: - Commented out app-specific prompts - using general processing only @@ -2086,7 +2092,10 @@ struct ContentView: View { // MARK: - Stop and Process Transcription - private func stopAndProcessTranscription(route: DictationOutputRoute = .normal) async { + private func stopAndProcessTranscription( + route: DictationOutputRoute = .normal, + toggleStopRequestedAt: TimeInterval? = nil + ) async { DebugLogger.shared.debug("stopAndProcessTranscription called", source: "ContentView") DebugLogger.shared.info("Output route selected: \(route.rawValue)", source: "ContentView") self.appBench("stop_path_enter route=\(route.rawValue)") @@ -2120,15 +2129,11 @@ struct ContentView: View { DebugLogger.shared.debug("Hiding dictation overlay at stop path", source: "ContentView") self.hideOverlayAsync(reason: "stop_path") } else { - // Show "Transcribing" state before calling stop() when the overlay needs - // to remain available for prompt, command, rewrite, or AI feedback. DebugLogger.shared.debug("Showing transcription processing state", source: "ContentView") self.appBench("processing_ui_request status=Transcribing") self.menuBarManager.setProcessing(true) NotchOverlayManager.shared.updateTranscriptionText("Transcribing") self.appBench("processing_ui_requested status=Transcribing") - - // Give SwiftUI a chance to render the processing state before heavier work. await Task.yield() } @@ -2154,10 +2159,7 @@ struct ContentView: View { guard transcribedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false else { DebugLogger.shared.debug("Transcription returned empty text", source: "ContentView") - // Finish the same short exit transition even when no text is emitted. - if !didRequestOverlayHideOnStop { - await self.menuBarManager.finishProcessingAndHideOverlay() - } + await self.menuBarManager.finishProcessingAndHideOverlay() return } @@ -2415,18 +2417,17 @@ struct ContentView: View { ) } // When FluidVoice itself is frontmost, the bound editor already receives `finalText`. - // Avoid re-inserting or overwriting the clipboard in that self-target case. let shouldCopyToClipboard = shouldPersistOutputs && !sendsExistingDraft && SettingsStore.shared.copyTranscriptionToClipboard && !isFluidFrontmost - - if shouldCopyToClipboard { + let shouldTypeExternally = shouldPersistOutputs && !isFluidFrontmost + if shouldCopyToClipboard, !shouldTypeExternally { ClipboardService.copyToClipboard(finalText) } var didTypeExternally = false - let shouldTypeExternally = shouldPersistOutputs && !isFluidFrontmost + var didFailTextDelivery = false DebugLogger.shared.debug( "Typing decision → frontmost: \(frontmostName), fluidFrontmost: \(isFluidFrontmost), editorFocused: \(self.isTranscriptionFocused), willTypeExternally: \(shouldTypeExternally)", @@ -2445,32 +2446,51 @@ struct ContentView: View { && !self.isSpokenSendBlockedApp(appInfo) // Dispatch insertion as soon as the destination app is ready; the // overlay hides asynchronously after output so it cannot delay paste. + let focusReady: Bool if typingTarget.shouldRestoreOriginalFocus { - await self.restoreFocusToRecordingTarget() + focusReady = await self.restoreFocusToRecordingTarget() + } else { + focusReady = true } + if spokenSendAllowed { NotchContentState.shared.setSpokenSendIndicatorState(.sending) NotchOverlayManager.shared.updateTranscriptionText("Sending") } - self.appBench( - "text_ready_to_type_request elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - finalTextReadyAt) * 1000).rounded()))" - ) - if spokenSendAllowed { + + let deliveryResult: TextDeliveryResult + if !focusReady { + deliveryResult = .recoverableFailure(.targetRestoreFailed) + } else if spokenSendAllowed { + self.appBench( + "text_ready_to_type_request elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - finalTextReadyAt) * 1000).rounded()))" + ) let deliveryOutcome = await self.deliverSpokenSend( finalOutputPlan, targetPID: typingTarget.pid, - textReadyAt: finalTextReadyAt + textReadyAt: finalTextReadyAt, + toggleStopRequestedAt: toggleStopRequestedAt, + preserveTranscriptOnClipboard: shouldCopyToClipboard ) didTypeExternally = deliveryOutcome.didInsert + deliveryResult = deliveryOutcome.didInsert || deliveryOutcome.didDispatchAction + ? .commandPosted + : .recoverableFailure(.pasteCommandFailed) } else { - self.asr.typeOutputPlanToActiveField( + self.appBench( + "text_ready_to_type_request elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - finalTextReadyAt) * 1000).rounded()))" + ) + deliveryResult = await self.asr.typeOutputPlanToActiveField( finalOutputPlan, preferredTargetPID: typingTarget.pid, textReadyAt: finalTextReadyAt, - tracksDictionaryCorrections: true + toggleStopRequestedAt: toggleStopRequestedAt, + tracksDictionaryCorrections: true, + preserveTranscriptOnClipboard: shouldCopyToClipboard ) - didTypeExternally = true + didTypeExternally = deliveryResult.wasDispatched } + if spokenSendRequested, !spokenSendAllowed { NotchContentState.shared.setSpokenSendIndicatorState(.failed) DebugLogger.shared.warning( @@ -2484,12 +2504,20 @@ struct ContentView: View { } NotchOverlayManager.shared.updateTranscriptionText("") NotchContentState.shared.setSpokenSendIndicatorState(.hidden) - if !shouldShowAIProcessingFailure, !didRequestOverlayHideOnStop { + + if case let .recoverableFailure(failure) = deliveryResult { + didFailTextDelivery = true + self.showTextDeliveryFailure(failure, transcript: finalText) + } else if !shouldShowAIProcessingFailure, !didRequestOverlayHideOnStop { self.hideOverlayAfterOutput() } } - if !didTypeExternally, !shouldShowAIProcessingFailure, !didRequestOverlayHideOnStop { + if !didTypeExternally, + !shouldShowAIProcessingFailure, + !didFailTextDelivery, + !didRequestOverlayHideOnStop + { self.hideOverlayAfterOutput() } } @@ -2505,6 +2533,62 @@ struct ContentView: View { NotchContentState.shared.setSpokenSendIndicatorState(shouldSend ? .detected : .hidden) } + private func showTextDeliveryFailure(_ failure: TextDeliveryFailure, transcript: String) { + let message = switch failure { + case .emptyText: + "There was no text to insert" + case .accessibilityNotTrusted: + "Enable Accessibility to insert text" + case .targetUnavailable, .targetRestoreFailed: + "Could not restore the target text field" + case .clipboardSnapshotFailed: + "Clipboard contents could not be preserved exactly" + case .clipboardWriteFailed: + "Could not prepare the clipboard" + case .pasteCommandFailed: + "Could not send the paste command" + } + NotchContentState.shared.showTextDeliveryFailure(message: message, transcript: transcript) + self.menuBarManager.finishProcessingKeepingOverlayVisible() + } + + private func showPrivateAIEditModeUnavailableIfNeeded() -> Bool { + let settings = SettingsStore.shared + let providerID = settings.rewriteModeLinkedToGlobal + ? settings.selectedProviderID + : settings.rewriteModeSelectedProviderID + guard PrivateFeatures.privateAIProvider, + providerID.trimmingCharacters(in: .whitespacesAndNewlines) == + PrivateAIProviderFeature.shared.providerID + else { + return false + } + + guard !self.asr.isRunningOrStarting, + !NotchContentState.shared.isProcessing + else { + return true + } + + self.menuBarManager.setOverlayMode(.edit) + self.advanceOverlayLifecycle() + let expectedOverlayLifecycleID = self.overlayLifecycleID + self.menuBarManager.showRecordingOverlayImmediately() + NotchContentState.shared.showAIProcessingFailure( + message: "Edit Mode cannot be used with Fluid-1", + canRetry: false + ) + self.menuBarManager.finishProcessingKeepingOverlayVisible() + + Task { @MainActor in + try? await Task.sleep(nanoseconds: 6_000_000_000) + guard self.overlayLifecycleID == expectedOverlayLifecycleID else { return } + NotchContentState.shared.clearAIProcessingFailure() + await self.menuBarManager.finishProcessingAndHideOverlay() + } + return true + } + private func advanceOverlayLifecycle() { self.spokenSendAutoStopTask?.cancel() self.spokenSendAutoStopTask = nil @@ -2514,6 +2598,7 @@ struct ContentView: View { self.spokenSendLastVoiceActivityAt = ProcessInfo.processInfo.systemUptime self.overlayLifecycleID &+= 1 NotchContentState.shared.clearAIProcessingFailure() + NotchContentState.shared.clearTextDeliveryFailure() } private func handleSpokenSendPartialTranscription(_ text: String) { @@ -2805,7 +2890,10 @@ struct ContentView: View { return } if typingTarget.shouldRestoreOriginalFocus { - await self.restoreFocusToRecordingTarget() + guard await self.restoreFocusToRecordingTarget() else { + self.showTextDeliveryFailure(.targetRestoreFailed, transcript: text) + return + } } let appInfo = self.getCurrentAppInfo() let outputPlan = ASRService.makeDictationLiteralOutputPlan( @@ -2814,8 +2902,41 @@ struct ContentView: View { bundleID: appInfo.bundleId, windowTitle: appInfo.windowTitle ) - self.asr.typeOutputPlanToActiveField(outputPlan, preferredTargetPID: typingTarget.pid) - DebugLogger.shared.info("Actions: Pasted latest transcription into focused field", source: "ContentView") + let result = await self.asr.typeOutputPlanToActiveField( + outputPlan, + preferredTargetPID: typingTarget.pid + ) + if case let .recoverableFailure(failure) = result { + self.showTextDeliveryFailure(failure, transcript: text) + } else { + DebugLogger.shared.info("Actions: Pasted latest transcription into focused field", source: "ContentView") + } + } + } + + @MainActor + private func retryTextDelivery(_ transcript: String) async { + guard !transcript.isEmpty else { return } + + self.menuBarManager.setProcessing(true) + NotchOverlayManager.shared.updateTranscriptionText("Inserting") + let typingTarget = self.resolveTypingTargetPID() + if typingTarget.shouldRestoreOriginalFocus, + !(await self.restoreFocusToRecordingTarget()) + { + self.showTextDeliveryFailure(.targetRestoreFailed, transcript: transcript) + return + } + + let result = await self.asr.typeTextToActiveField( + transcript, + preferredTargetPID: typingTarget.pid, + preserveTranscriptOnClipboard: SettingsStore.shared.copyTranscriptionToClipboard + ) + if case let .recoverableFailure(failure) = result { + self.showTextDeliveryFailure(failure, transcript: transcript) + } else { + self.hideOverlayAfterOutput() } } @@ -2910,24 +3031,29 @@ struct ContentView: View { let frontmostApp = NSWorkspace.shared.frontmostApplication let isFluidFrontmost = frontmostApp?.bundleIdentifier == Bundle.main.bundleIdentifier - if SettingsStore.shared.copyTranscriptionToClipboard, !isFluidFrontmost { - ClipboardService.copyToClipboard(finalText) - } - - let focusedPID = TypingService.captureSystemFocusedPID() - ?? NSWorkspace.shared.frontmostApplication?.processIdentifier - NotchContentState.shared.recordingTargetPID = focusedPID + let targetContext = TypingService.captureRecordingTargetContext() + NotchContentState.shared.recordingTargetContext = targetContext + NotchContentState.shared.recordingTargetPID = targetContext?.pid let shouldTypeExternally = !isFluidFrontmost if shouldTypeExternally { let typingTarget = self.resolveTypingTargetPID() if typingTarget.shouldRestoreOriginalFocus { - await self.restoreFocusToRecordingTarget() + guard await self.restoreFocusToRecordingTarget() else { + self.showTextDeliveryFailure(.targetRestoreFailed, transcript: finalText) + return + } } - self.asr.typeOutputPlanToActiveField( + let result = await self.asr.typeOutputPlanToActiveField( outputPlan, - preferredTargetPID: typingTarget.pid + preferredTargetPID: typingTarget.pid, + preserveTranscriptOnClipboard: SettingsStore.shared.copyTranscriptionToClipboard ) + if case let .recoverableFailure(failure) = result { + self.showTextDeliveryFailure(failure, transcript: finalText) + } + } else if SettingsStore.shared.copyTranscriptionToClipboard { + ClipboardService.copyToClipboard(finalText) } } @@ -3024,13 +3150,9 @@ struct ContentView: View { self.pendingAIReprocessText = nil } - if SettingsStore.shared.copyTranscriptionToClipboard { - ClipboardService.copyToClipboard(finalText) - } - - let focusedPID = TypingService.captureSystemFocusedPID() - ?? NSWorkspace.shared.frontmostApplication?.processIdentifier - NotchContentState.shared.recordingTargetPID = focusedPID + let targetContext = TypingService.captureRecordingTargetContext() + NotchContentState.shared.recordingTargetContext = targetContext + NotchContentState.shared.recordingTargetPID = targetContext?.pid let frontmostApp = NSWorkspace.shared.frontmostApplication let isFluidFrontmost = frontmostApp?.bundleIdentifier?.contains("fluid") == true @@ -3038,12 +3160,22 @@ struct ContentView: View { if shouldTypeExternally { let typingTarget = self.resolveTypingTargetPID() if typingTarget.shouldRestoreOriginalFocus { - await self.restoreFocusToRecordingTarget() + guard await self.restoreFocusToRecordingTarget() else { + self.showTextDeliveryFailure(.targetRestoreFailed, transcript: finalText) + return + } } - self.asr.typeOutputPlanToActiveField( + let result = await self.asr.typeOutputPlanToActiveField( outputPlan, - preferredTargetPID: typingTarget.pid + preferredTargetPID: typingTarget.pid, + preserveTranscriptOnClipboard: SettingsStore.shared.copyTranscriptionToClipboard ) + if case let .recoverableFailure(failure) = result { + self.showTextDeliveryFailure(failure, transcript: finalText) + return + } + } else if SettingsStore.shared.copyTranscriptionToClipboard { + ClipboardService.copyToClipboard(finalText) } if aiFallbackReason == nil { @@ -3075,20 +3207,27 @@ struct ContentView: View { if !self.rewriteModeService.rewrittenText.isEmpty { DebugLogger.shared.info("Rewrite successful, typing result (chars: \(self.rewriteModeService.rewrittenText.count))", source: "ContentView") - // Copy to clipboard as backup - if SettingsStore.shared.copyTranscriptionToClipboard { - ClipboardService.copyToClipboard(self.rewriteModeService.rewrittenText) - } - // Type the rewritten text let typingTarget = self.resolveTypingTargetPID() if typingTarget.shouldRestoreOriginalFocus { - await self.restoreFocusToRecordingTarget() + guard await self.restoreFocusToRecordingTarget() else { + self.showTextDeliveryFailure( + .targetRestoreFailed, + transcript: self.rewriteModeService.rewrittenText + ) + return + } } - self.asr.typeTextToActiveField( + let deliveryResult = await self.asr.typeTextToActiveField( self.rewriteModeService.rewrittenText, - preferredTargetPID: typingTarget.pid + preferredTargetPID: typingTarget.pid, + preserveTranscriptOnClipboard: SettingsStore.shared.copyTranscriptionToClipboard ) + if case let .recoverableFailure(failure) = deliveryResult { + self.showTextDeliveryFailure(failure, transcript: self.rewriteModeService.rewrittenText) + return + } + // Clear the rewrite service state for next use self.rewriteModeService.clearState() self.hideOverlayAfterOutput() @@ -3285,43 +3424,22 @@ struct ContentView: View { } } - /// Best-effort: re-activate the app that was focused when recording started. - /// Skips the AX restore work when the captured text element is already focused. - private func restoreFocusToRecordingTarget() async { - guard let pid = NotchContentState.shared.recordingTargetPID else { return } + /// Restores only the window and element captured when recording started. + private func restoreFocusToRecordingTarget() async -> Bool { + guard let context = NotchContentState.shared.recordingTargetContext else { return false } + let pid = context.pid let startedAt = ProcessInfo.processInfo.systemUptime self.appBench("focus_restore_start targetPID=\(pid)") - if let focusTarget = self.recordingFocusTarget, focusTarget.pid == pid { - if TypingService.isExactFocusTargetActive(focusTarget) { - self.appBench("focus_restore_result activated=false element=true elapsedMs=0 reason=already_focused") - return - } - let activated = TypingService.activateApp(pid: pid) - let focusedElementRestored = TypingService.restoreFocusTarget(focusTarget) - self.appBench( - "focus_restore_result activated=\(activated) element=\(focusedElementRestored) elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - startedAt) * 1000).rounded()))" - ) - return - } - if TypingService.isCapturedFocusStillActive(for: pid) { - self.appBench("focus_restore_result activated=false element=true elapsedMs=0 reason=already_focused") - DebugLogger.shared.debug( - "Restore focus skipped; captured element still focused, targetPID: \(pid)", - source: "ContentView" - ) - self.appBench("focus_restore_settle_done delayMs=0") - return - } - let activated = TypingService.activateApp(pid: pid) - let focusedElementRestored = TypingService.restoreCapturedFocus(in: pid) + let result = await TypingService.prepareTargetForDelivery(context) self.appBench( - "focus_restore_result activated=\(activated) element=\(focusedElementRestored) elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - startedAt) * 1000).rounded()))" + "focus_restore_result result=\(result.rawValue) elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - startedAt) * 1000).rounded()))" ) DebugLogger.shared.debug( - "Restore focus -> appActivated: \(activated), elementFocusRestored: \(focusedElementRestored), targetPID: \(pid)", + "Restore focus result: \(result.rawValue), targetPID: \(pid)", source: "ContentView" ) self.appBench("focus_restore_settle_done delayMs=0") + return result.isReady } // MARK: - ASR Model Management @@ -3419,6 +3537,11 @@ struct ContentView: View { NotchContentState.shared.onPasteLastRequested = { self.pasteLastDictationFromHistory() } + NotchContentState.shared.onRetryTextDeliveryRequested = { transcript in + Task { @MainActor in + await self.retryTextDelivery(transcript) + } + } NotchContentState.shared.onUndoLastAIRequested = { self.undoLastAIProcessingFromHistory() } @@ -3467,10 +3590,13 @@ struct ContentView: View { ) self.beginDictationRecording(for: .primary, mode: .dictate) }, - stopAndProcessCallback: { + stopAndProcessCallback: { toggleStopRequestedAt in let route = self.currentDictationOutputRouteForHotkeyStop() DebugLogger.shared.info("Hotkey stop callback using route: \(route.rawValue)", source: "ContentView") - await self.stopAndProcessTranscription(route: route) + await self.stopAndProcessTranscription( + route: route, + toggleStopRequestedAt: toggleStopRequestedAt + ) }, promptModeCallback: { DebugLogger.shared.info("Prompt mode triggered", source: "ContentView") diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index 596250356..0ed4090ec 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -46,6 +46,7 @@ final class SettingsStore: ObservableObject { self.repairForcedOnboardingResetIfNeeded() self.migrateOverlayBottomOffsetTo50IfNeeded() self.migratePrivateAIContextDefaultTo4KIfNeeded() + Self.migrateTextInsertionModeToReliablePasteIfNeeded(defaults: self.defaults) self.refreshLaunchAtStartupStatus(clearError: true, logMismatch: false) } @@ -78,7 +79,9 @@ final class SettingsStore: ObservableObject { case llama case mlx - var id: String { self.rawValue } + var id: String { + self.rawValue + } /// Default backend when no preference is stored. /// Apple Silicon → MLX (fastest Fluid-1 path). Intel → llama.cpp. @@ -203,7 +206,9 @@ final class SettingsStore: ObservableObject { let uid: String var name: String - var id: String { self.uid } + var id: String { + self.uid + } } enum DictationPromptSelection: Equatable { @@ -1820,7 +1825,9 @@ final class SettingsStore: ObservableObject { /// Direct Core Audio is the required capture backend. Legacy persisted /// preferences are intentionally ignored because AVAudioEngine can block or /// crash while audio devices are changing. - var experimentalDirectAudioCaptureEnabled: Bool { true } + var experimentalDirectAudioCaptureEnabled: Bool { + true + } var copyTranscriptionToClipboard: Bool { get { self.defaults.bool(forKey: Keys.copyTranscriptionToClipboard) } @@ -4091,7 +4098,9 @@ final class SettingsStore: ObservableObject { case tab case space - var id: Self { self } + var id: Self { + self + } var title: String { switch self { @@ -4126,7 +4135,9 @@ final class SettingsStore: ObservableObject { var aliases: [String] var isEnabled: Bool - var id: SpokenFormattingAction { self.action } + var id: SpokenFormattingAction { + self.action + } init(action: SpokenFormattingAction, aliases: [String], isEnabled: Bool = true) { self.action = action @@ -5320,6 +5331,7 @@ private extension SettingsStore { static let spokenSendImmediatelyEnabled = "SpokenSendImmediatelyEnabled" static let spokenSendPhrase = "SpokenSendPhrase" static let spokenSendKey = "SpokenSendKey" + static let reliablePasteMigrationV1 = "TextInsertionModeMigratedToReliablePasteV1" static let autoUpdateCheckEnabled = "AutoUpdateCheckEnabled" static let betaReleasesEnabled = "BetaReleasesEnabled" static let lastUpdateCheckDate = "LastUpdateCheckDate" @@ -5520,9 +5532,15 @@ extension SettingsStore { } } + static func migrateTextInsertionModeToReliablePasteIfNeeded(defaults: UserDefaults) { + guard !defaults.bool(forKey: Keys.reliablePasteMigrationV1) else { return } + defaults.set(TextInsertionMode.reliablePaste.rawValue, forKey: Keys.textInsertionMode) + defaults.set(true, forKey: Keys.reliablePasteMigrationV1) + } + enum TextInsertionMode: String, CaseIterable, Identifiable, Codable { - case standard case reliablePaste + case standard var id: String { self.rawValue @@ -5531,18 +5549,18 @@ extension SettingsStore { var displayName: String { switch self { case .standard: - return "Clipboard Free Insert" + return "Direct Paste" case .reliablePaste: - return "Clipboard Paste" + return "Clipboard Paste (Recommended)" } } var description: String { switch self { case .standard: - return "Fastest path. Inserts text without changing the clipboard, with paste fallback if direct insertion is unavailable." + return "Posts text directly without changing the clipboard. Some editors may reject or truncate it." case .reliablePaste: - return "Compatibility path. Uses a temporary clipboard paste and restores your previous clipboard after insertion." + return "Fast, compatible insertion using a temporary clipboard entry that is restored after paste." } } } @@ -5552,7 +5570,7 @@ extension SettingsStore { guard let raw = self.defaults.string(forKey: Keys.textInsertionMode), let mode = TextInsertionMode(rawValue: raw) else { - return .standard + return .reliablePaste } return mode } diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index 35fd26d84..c61d8b535 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -3953,10 +3953,14 @@ final class ASRService: ObservableObject { ) let priorityUIDs = SettingsStore.shared.microphonePriority.map(\.uid) let livenessRequiresRecovery = - (self.isRunning && - resolvedInput?.uid != microphonePreferenceCoordinator.confirmedActiveInputUID) || - (self.hasPreparedAudioCapture && - resolvedInput?.id != self.directAudioLifecycleController.snapshot.deviceID) + ( + self.isRunning && + resolvedInput?.uid != microphonePreferenceCoordinator.confirmedActiveInputUID + ) || + ( + self.hasPreparedAudioCapture && + resolvedInput?.id != self.directAudioLifecycleController.snapshot.deviceID + ) let shouldReconcileInputSelection = AudioCaptureIdlePolicy.shouldReconcileInputSelection( priorityInputUIDs: priorityUIDs, migrationPending: migrationPending, @@ -4837,20 +4841,35 @@ final class ASRService: ObservableObject { private let typingService = TypingService() // Reuse instance to avoid conflicts - func typeTextToActiveField(_ text: String) { - self.typeTextToActiveField(text, preferredTargetPID: nil, textReadyAt: nil) + @MainActor + func typeTextToActiveField(_ text: String) async -> TextDeliveryResult { + await self.typeTextToActiveField(text, preferredTargetPID: nil, textReadyAt: nil) } - func typeTextToActiveField(_ text: String, preferredTargetPID: pid_t?, textReadyAt: TimeInterval? = nil) { - self.typeOutputPlanToActiveField(.plain(text), preferredTargetPID: preferredTargetPID, textReadyAt: textReadyAt) + @MainActor + func typeTextToActiveField( + _ text: String, + preferredTargetPID: pid_t?, + textReadyAt: TimeInterval? = nil, + preserveTranscriptOnClipboard: Bool = false + ) async -> TextDeliveryResult { + await self.typeOutputPlanToActiveField( + .plain(text), + preferredTargetPID: preferredTargetPID, + textReadyAt: textReadyAt, + preserveTranscriptOnClipboard: preserveTranscriptOnClipboard + ) } + @MainActor func typeOutputPlanToActiveField( _ plan: DictationLiteralOutputPlan, preferredTargetPID: pid_t?, textReadyAt: TimeInterval? = nil, - tracksDictionaryCorrections: Bool = false - ) { + toggleStopRequestedAt: TimeInterval? = nil, + tracksDictionaryCorrections: Bool = false, + preserveTranscriptOnClipboard: Bool = false + ) async -> TextDeliveryResult { let requestedAt = ProcessInfo.processInfo.systemUptime let textReadyAge = textReadyAt.map { Int(((requestedAt - $0) * 1000).rounded()) } let text = plan.plainText @@ -4859,11 +4878,13 @@ final class ASRService: ObservableObject { message: "asr_type_request chars=\(text.count) preferredPID=\(preferredTargetPID.map { String($0) } ?? "nil") textReadyAgeMs=\(textReadyAge.map { String($0) } ?? "nil")", source: "TypingBenchmark" ) - self.typingService.typeOutputPlanInstantly( + let result = await self.typingService.typeOutputPlanInstantly( plan, preferredTargetPID: preferredTargetPID, textReadyAt: textReadyAt, - tracksDictionaryCorrections: tracksDictionaryCorrections + toggleStopRequestedAt: toggleStopRequestedAt, + tracksDictionaryCorrections: tracksDictionaryCorrections, + preserveTranscriptOnClipboard: preserveTranscriptOnClipboard ) let dispatchedAt = ProcessInfo.processInfo.systemUptime let textReadyToDispatchMs = textReadyAt.map { @@ -4871,18 +4892,21 @@ final class ASRService: ObservableObject { } ?? "nil" DebugLogger.shared.benchmark( "TYPING_BENCH", - message: "asr_type_dispatched chars=\(text.count) preferredPID=\(preferredTargetPID.map { String($0) } ?? "nil") textReadyToDispatchMs=\(textReadyToDispatchMs)", + message: "asr_type_dispatched chars=\(text.count) preferredPID=\(preferredTargetPID.map { String($0) } ?? "nil") result=\(String(describing: result)) textReadyToDispatchMs=\(textReadyToDispatchMs)", source: "TypingBenchmark" ) + return result } func typeOutputPlanToActiveFieldAndWait( _ plan: DictationLiteralOutputPlan, preferredTargetPID: pid_t?, textReadyAt: TimeInterval? = nil, + toggleStopRequestedAt: TimeInterval? = nil, tracksDictionaryCorrections: Bool = false, postInsertionKey: SettingsStore.SpokenSendKey? = nil, - requiredFocusTarget: TypingService.CapturedFocusTarget? = nil + requiredFocusTarget: TypingService.CapturedFocusTarget? = nil, + preserveTranscriptOnClipboard: Bool = false ) async -> TypingService.DeliveryOutcome { let requestedAt = ProcessInfo.processInfo.systemUptime let textReadyAge = textReadyAt.map { Int(((requestedAt - $0) * 1000).rounded()) } @@ -4897,9 +4921,11 @@ final class ASRService: ObservableObject { plan, preferredTargetPID: preferredTargetPID, textReadyAt: textReadyAt, + toggleStopRequestedAt: toggleStopRequestedAt, tracksDictionaryCorrections: tracksDictionaryCorrections, postInsertionKey: postInsertionKey, - requiredFocusTarget: requiredFocusTarget + requiredFocusTarget: requiredFocusTarget, + preserveTranscriptOnClipboard: preserveTranscriptOnClipboard ) { outcome in continuation.resume(returning: outcome) } diff --git a/Sources/Fluid/Services/GlobalHotkeyManager.swift b/Sources/Fluid/Services/GlobalHotkeyManager.swift index a857f385d..aa036c2cb 100644 --- a/Sources/Fluid/Services/GlobalHotkeyManager.swift +++ b/Sources/Fluid/Services/GlobalHotkeyManager.swift @@ -242,7 +242,7 @@ final class GlobalHotkeyManager: NSObject { private var rewriteModeShortcutEnabled: Bool private var startRecordingCallback: (() async -> Void)? private var dictationModeCallback: (() async -> Void)? - private var stopAndProcessCallback: (() async -> Void)? + private var stopAndProcessCallback: ((TimeInterval?) async -> Void)? private var promptModeCallback: (() async -> Void)? private var promptSelectionCallback: ((SettingsStore.DictationPromptSelection) async -> Void)? private var commandModeCallback: (() async -> Void)? @@ -470,7 +470,7 @@ final class GlobalHotkeyManager: NSObject { rewriteModeShortcutEnabled: Bool, startRecordingCallback: (() async -> Void)? = nil, dictationModeCallback: (() async -> Void)? = nil, - stopAndProcessCallback: (() async -> Void)? = nil, + stopAndProcessCallback: ((TimeInterval?) async -> Void)? = nil, promptModeCallback: (() async -> Void)? = nil, promptSelectionCallback: ((SettingsStore.DictationPromptSelection) async -> Void)? = nil, commandModeCallback: (() async -> Void)? = nil, @@ -519,7 +519,7 @@ final class GlobalHotkeyManager: NSObject { } } - func setStopAndProcessCallback(_ callback: @escaping () async -> Void) { + func setStopAndProcessCallback(_ callback: @escaping (TimeInterval?) async -> Void) { self.stopAndProcessCallback = callback } @@ -761,7 +761,7 @@ final class GlobalHotkeyManager: NSObject { source: "GlobalHotkeyManager" ) if isSameMode { - self.stopRecordingIfNeeded() + self.stopRecordingAfterToggle() } else { self.triggerDictationMode() } @@ -874,7 +874,7 @@ final class GlobalHotkeyManager: NSObject { if self.asrService.isRunningOrStarting { if self.isPromptModeRecordingProvider?() ?? false { DebugLogger.shared.info("Prompt shortcut pressed in Prompt mode - stopping", source: "GlobalHotkeyManager") - self.stopRecordingIfNeeded() + self.stopRecordingAfterToggle() } else { DebugLogger.shared.info("Prompt shortcut pressed while recording - switching mode", source: "GlobalHotkeyManager") self.triggerPromptSelection(assignment.selection) @@ -930,7 +930,7 @@ final class GlobalHotkeyManager: NSObject { if self.asrService.isRunningOrStarting { if self.isCommandRecordingProvider?() ?? false { DebugLogger.shared.info("Command mode shortcut pressed in Command mode - stopping", source: "GlobalHotkeyManager") - self.stopRecordingIfNeeded() + self.stopRecordingAfterToggle() } else { DebugLogger.shared.info("Command mode shortcut pressed while recording - switching mode", source: "GlobalHotkeyManager") self.triggerCommandMode() @@ -981,7 +981,7 @@ final class GlobalHotkeyManager: NSObject { if self.asrService.isRunningOrStarting { if self.isRewriteRecordingProvider?() ?? false { DebugLogger.shared.info("Rewrite mode shortcut pressed in Edit mode - stopping", source: "GlobalHotkeyManager") - self.stopRecordingIfNeeded() + self.stopRecordingAfterToggle() } else { DebugLogger.shared.info("Rewrite mode shortcut pressed while recording - switching mode", source: "GlobalHotkeyManager") self.triggerRewriteMode() @@ -1121,7 +1121,7 @@ final class GlobalHotkeyManager: NSObject { if self.asrService.isRunningOrStarting { if self.isCommandRecordingProvider?() ?? false { DebugLogger.shared.info("Command mode modifier released (toggle, same mode) - stopping", source: "GlobalHotkeyManager") - self.stopRecordingIfNeeded() + self.stopRecordingAfterToggle() } else { DebugLogger.shared.info("Command mode modifier released (toggle, switch mode) - switching", source: "GlobalHotkeyManager") self.triggerCommandMode() @@ -1153,7 +1153,7 @@ final class GlobalHotkeyManager: NSObject { if self.asrService.isRunningOrStarting { if self.isRewriteRecordingProvider?() ?? false { DebugLogger.shared.info("Rewrite mode modifier released (toggle, same mode) - stopping", source: "GlobalHotkeyManager") - self.stopRecordingIfNeeded() + self.stopRecordingAfterToggle() } else { DebugLogger.shared.info("Rewrite mode modifier released (toggle, switch mode) - switching", source: "GlobalHotkeyManager") self.triggerRewriteMode() @@ -1349,7 +1349,7 @@ final class GlobalHotkeyManager: NSObject { source: "GlobalHotkeyManager" ) if isSameMode { - self.stopRecordingIfNeeded() + self.stopRecordingAfterToggle() } else { self.triggerDictationMode() } @@ -1588,7 +1588,7 @@ final class GlobalHotkeyManager: NSObject { if self.asrService.isRunningOrStarting { if self.isPromptModeRecordingProvider?() ?? false { DebugLogger.shared.info("Prompt mode shortcut pressed in Prompt mode - stopping", source: "GlobalHotkeyManager") - self.stopRecordingIfNeeded() + self.stopRecordingAfterToggle() } else { DebugLogger.shared.info("Prompt mode shortcut pressed while recording - switching mode", source: "GlobalHotkeyManager") self.triggerPromptMode() @@ -1635,7 +1635,7 @@ final class GlobalHotkeyManager: NSObject { if self.asrService.isRunningOrStarting { if self.isPromptModeRecordingProvider?() ?? false { DebugLogger.shared.info("Prompt mode modifier released (toggle, same mode) - stopping", source: "GlobalHotkeyManager") - self.stopRecordingIfNeeded() + self.stopRecordingAfterToggle() } else { DebugLogger.shared.info("Prompt mode modifier released (toggle, switch mode) - switching", source: "GlobalHotkeyManager") self.triggerPromptMode() @@ -1669,7 +1669,7 @@ final class GlobalHotkeyManager: NSObject { if self.asrService.isRunningOrStarting { if self.isPromptModeRecordingProvider?() ?? false { DebugLogger.shared.info("Prompt shortcut modifier released (toggle, same mode) - stopping", source: "GlobalHotkeyManager") - self.stopRecordingIfNeeded() + self.stopRecordingAfterToggle() } else { DebugLogger.shared.info("Prompt shortcut modifier released (toggle, switch mode) - switching", source: "GlobalHotkeyManager") self.triggerPromptSelection(assignment.selection) @@ -1909,6 +1909,7 @@ final class GlobalHotkeyManager: NSObject { } private func toggleRecording() { + let toggleStopRequestedAt = ProcessInfo.processInfo.systemUptime Task { @MainActor [weak self] in guard let self = self else { return } @@ -1916,7 +1917,7 @@ final class GlobalHotkeyManager: NSObject { guard self.canTriggerRecordingAction("toggle") else { return } if self.asrService.isRunningOrStarting { - await self.stopRecordingInternal() + await self.stopRecordingInternal(toggleStopRequestedAt: toggleStopRequestedAt) } else { // Use callback if available, otherwise fallback to direct start if let callback = self.startRecordingCallback { @@ -1946,7 +1947,11 @@ final class GlobalHotkeyManager: NSObject { } } - private func stopRecordingIfNeeded() { + private func stopRecordingAfterToggle() { + self.stopRecordingIfNeeded(toggleStopRequestedAt: ProcessInfo.processInfo.systemUptime) + } + + private func stopRecordingIfNeeded(toggleStopRequestedAt: TimeInterval? = nil) { Task { @MainActor [weak self] in guard let self = self else { return } @@ -1963,12 +1968,12 @@ final class GlobalHotkeyManager: NSObject { return } - await self.stopRecordingInternal() + await self.stopRecordingInternal(toggleStopRequestedAt: toggleStopRequestedAt) } } @MainActor - private func stopRecordingInternal() async { + private func stopRecordingInternal(toggleStopRequestedAt: TimeInterval? = nil) async { if self.asrService.isStarting, self.asrService.isRunning == false { DebugLogger.shared.debug("Cancelling pending audio capture start", source: "GlobalHotkeyManager") await self.asrService.cancelPendingAudioCaptureStart(reason: "hotkey_released") @@ -1987,7 +1992,7 @@ final class GlobalHotkeyManager: NSObject { defer { isProcessingStop = false } if let callback = stopAndProcessCallback { - await callback() + await callback(toggleStopRequestedAt) } else { await self.asrService.stopWithoutTranscription() } diff --git a/Sources/Fluid/Services/PasteDeliveryCoordinator.swift b/Sources/Fluid/Services/PasteDeliveryCoordinator.swift new file mode 100644 index 000000000..6974955d8 --- /dev/null +++ b/Sources/Fluid/Services/PasteDeliveryCoordinator.swift @@ -0,0 +1,712 @@ +import AppKit +import Carbon +import Foundation +import UniformTypeIdentifiers + +enum TextDeliveryFailure: String, Equatable { + case emptyText = "empty_text" + case accessibilityNotTrusted = "accessibility_not_trusted" + case clipboardSnapshotFailed = "clipboard_snapshot_failed" + case clipboardWriteFailed = "clipboard_write_failed" + case pasteCommandFailed = "paste_command_failed" + case targetUnavailable = "target_unavailable" + case targetRestoreFailed = "target_restore_failed" +} + +enum TextDeliveryResult: Equatable { + case commandPosted + case recoverableFailure(TextDeliveryFailure) + + var wasDispatched: Bool { + self == .commandPosted + } +} + +struct PasteboardSnapshot: Equatable { + struct Item: Equatable { + struct Representation: Equatable { + let type: NSPasteboard.PasteboardType + let data: Data + } + + /// AppKit uses source order as representation preference, so this must remain ordered. + let representations: [Representation] + /// Materialized while the original owner still provides file access or promised image data. + let portableImage: Representation? + + init(representations: [Representation], portableImage: Representation? = nil) { + self.representations = representations + self.portableImage = portableImage + } + } + + let items: [Item] +} + +@MainActor +protocol PasteboardManaging: AnyObject { + func captureSnapshot() -> PasteboardSnapshot? + func writeTemporaryText(_ text: String, sessionID: String) -> Bool + func writeIntentionalText(_ text: String) -> Bool + func isOwned(sessionID: String, expectedText: String) -> Bool + func restore(_ snapshot: PasteboardSnapshot) -> Bool +} + +@MainActor +protocol PasteCommandPosting: AnyObject { + func postGlobalPasteCommand() async -> Bool +} + +@MainActor +final class SystemPasteboardManager: PasteboardManaging { + private static let sourceType = NSPasteboard.PasteboardType("org.nspasteboard.source") + private static let transientType = NSPasteboard.PasteboardType("org.nspasteboard.TransientType") + private static let autoGeneratedType = NSPasteboard.PasteboardType("org.nspasteboard.AutoGeneratedType") + private static let sessionType = NSPasteboard.PasteboardType( + "\(Bundle.main.bundleIdentifier ?? "com.FluidApp.Fluid").PasteSession" + ) + private static let fileSemanticTypes: Set = [ + .fileURL, + NSPasteboard.PasteboardType("NSFilenamesPboardType"), + NSPasteboard.PasteboardType("com.apple.pasteboard.promised-file-url"), + NSPasteboard.PasteboardType("com.apple.pasteboard.promised-file-content-type"), + NSPasteboard.PasteboardType("com.apple.filepromise"), + ] + + private let pasteboard: NSPasteboard + + init(pasteboard: NSPasteboard = .general) { + self.pasteboard = pasteboard + } + + func captureSnapshot() -> PasteboardSnapshot? { + let startedAt = ProcessInfo.processInfo.systemUptime + for attempt in 1...2 { + let startingChangeCount = self.pasteboard.changeCount + guard let sourceItems = self.pasteboard.pasteboardItems else { + let endingChangeCount = self.pasteboard.changeCount + if endingChangeCount != startingChangeCount { + self.logSnapshotRetry( + attempt: attempt, + startingChangeCount: startingChangeCount, + endingChangeCount: endingChangeCount + ) + continue + } + self.logSnapshotFailure( + reason: "pasteboard_items_unavailable", + itemCount: 0, + itemIndex: -1, + totalBytes: 0 + ) + return nil + } + var items: [PasteboardSnapshot.Item] = [] + var totalBytes = 0 + var totalPortableImageBytes = 0 + var totalRepresentations = 0 + var failure: (reason: String, itemIndex: Int, type: NSPasteboard.PasteboardType?)? + + itemLoop: for (itemIndex, sourceItem) in sourceItems.enumerated() { + let sourceTypes = sourceItem.types + guard !sourceTypes.isEmpty else { + failure = ("empty_item", itemIndex, nil) + break + } + + var representations: [PasteboardSnapshot.Item.Representation] = [] + representations.reserveCapacity(sourceTypes.count) + for type in sourceTypes { + guard let data = sourceItem.data(forType: type) else { + failure = ("unavailable_representation", itemIndex, type) + break itemLoop + } + representations.append(.init(type: type, data: data)) + totalBytes += data.count + totalRepresentations += 1 + } + let item = PasteboardSnapshot.Item(representations: representations) + let portableImage = Self.portableImageRepresentation(for: item) + totalPortableImageBytes += portableImage?.data.count ?? 0 + items.append( + .init( + representations: representations, + portableImage: portableImage + ) + ) + } + + let endingChangeCount = self.pasteboard.changeCount + guard endingChangeCount == startingChangeCount else { + self.logSnapshotRetry( + attempt: attempt, + startingChangeCount: startingChangeCount, + endingChangeCount: endingChangeCount + ) + continue + } + + if let failure { + self.logSnapshotFailure( + reason: failure.reason, + itemCount: sourceItems.count, + itemIndex: failure.itemIndex, + totalBytes: totalBytes, + type: failure.type + ) + return nil + } + + self.log( + "clipboard_snapshot_complete items=\(items.count) representations=\(totalRepresentations) " + + "bytes=\(totalBytes) portableImageBytes=\(totalPortableImageBytes) " + + "elapsedMs=\(Self.elapsedMs(since: startedAt)) firstItem=\(Self.representationSummary(items.first))" + ) + return PasteboardSnapshot(items: items) + } + + self.log("clipboard_snapshot_failed reason=clipboard_changed_during_snapshot") + return nil + } + + func writeTemporaryText(_ text: String, sessionID: String) -> Bool { + let item = NSPasteboardItem() + guard item.setString(text, forType: .string), + item.setString(Bundle.main.bundleIdentifier ?? "com.FluidApp.Fluid", forType: Self.sourceType), + item.setData(Data(), forType: Self.transientType), + item.setData(Data(), forType: Self.autoGeneratedType), + item.setString(sessionID, forType: Self.sessionType) + else { + return false + } + + self.pasteboard.clearContents() + return self.pasteboard.writeObjects([item]) && self.isOwned(sessionID: sessionID, expectedText: text) + } + + func writeIntentionalText(_ text: String) -> Bool { + let item = NSPasteboardItem() + guard item.setString(text, forType: .string) else { return false } + + self.pasteboard.clearContents() + return self.pasteboard.writeObjects([item]) && self.pasteboard.string(forType: .string) == text + } + + func isOwned(sessionID: String, expectedText: String) -> Bool { + self.pasteboard.string(forType: .string) == expectedText && + self.pasteboard.string(forType: Self.sessionType) == sessionID + } + + func restore(_ snapshot: PasteboardSnapshot) -> Bool { + var items: [NSPasteboardItem] = [] + for snapshotItem in snapshot.items { + let item = NSPasteboardItem() + let portableImage = snapshotItem.portableImage + let hasFileURL = snapshotItem.representations.contains { $0.type == .fileURL } + + if !hasFileURL, + let portableImage, + !item.setData(portableImage.data, forType: portableImage.type) + { + self.log("clipboard_restore_failed reason=portable_image_write_failed type=\(portableImage.type.rawValue)") + return false + } + for representation in snapshotItem.representations { + guard Self.writeRestoredRepresentation(representation, to: item) else { + self.log("clipboard_restore_failed reason=representation_write_failed type=\(representation.type.rawValue)") + return false + } + if representation.type == .fileURL, + let portableImage, + !item.setData(portableImage.data, forType: portableImage.type) + { + self.log("clipboard_restore_failed reason=portable_image_write_failed type=\(portableImage.type.rawValue)") + return false + } + } + guard Self.addRestorationMarkers(to: item) else { + self.log("clipboard_restore_failed reason=marker_write_failed") + return false + } + items.append(item) + } + + self.pasteboard.clearContents() + guard items.isEmpty || self.pasteboard.writeObjects(items) else { + self.log("clipboard_restore_failed reason=pasteboard_write_failed") + return false + } + guard self.containsRestoredSnapshot(snapshot) else { + self.log("clipboard_restore_failed reason=post_write_verification_failed") + return false + } + + let fileURLFallbacks = zip(self.pasteboard.pasteboardItems ?? [], snapshot.items).reduce(into: 0) { count, pair in + let (restoredItem, snapshotItem) = pair + if snapshotItem.representations.contains(where: { $0.type == .fileURL }), + restoredItem.data(forType: .fileURL) == nil, + snapshotItem.portableImage != nil + { + count += 1 + } + } + self.log( + "clipboard_restore_verified items=\(snapshot.items.count) changeCount=\(self.pasteboard.changeCount) fileURLFallbacks=\(fileURLFallbacks) firstItem=\(Self.representationSummary(snapshot.items.first))" + ) + if let portableImage = snapshot.items.first?.portableImage { + self.log("clipboard_restore_portable_image type=\(portableImage.type.rawValue) bytes=\(portableImage.data.count)") + } + return true + } + + private func containsRestoredSnapshot(_ snapshot: PasteboardSnapshot) -> Bool { + let restoredItems = self.pasteboard.pasteboardItems ?? [] + guard restoredItems.count == snapshot.items.count else { return false } + + for (restoredItem, snapshotItem) in zip(restoredItems, snapshot.items) { + for representation in snapshotItem.representations { + if representation.type == .fileURL, + restoredItem.data(forType: .fileURL) == nil, + snapshotItem.portableImage != nil + { + continue + } + guard Self.restoredData(restoredItem, matches: representation) else { + return false + } + } + if let portableImage = snapshotItem.portableImage, + restoredItem.data(forType: portableImage.type) != portableImage.data + { + return false + } + guard restoredItem.types.contains(Self.transientType), + restoredItem.types.contains(Self.autoGeneratedType) + else { + return false + } + } + return true + } + + private static func restoredData( + _ restoredItem: NSPasteboardItem, + matches representation: PasteboardSnapshot.Item.Representation + ) -> Bool { + if representation.type == .fileURL, + let expectedURL = URL(dataRepresentation: representation.data, relativeTo: nil, isAbsolute: true), + let restoredURL = restoredItem.string(forType: .fileURL).flatMap(URL.init(string:)) + { + return restoredURL == expectedURL + } + return restoredItem.data(forType: representation.type) == representation.data + } + + private static func portableImageRepresentation( + for item: PasteboardSnapshot.Item + ) -> PasteboardSnapshot.Item.Representation? { + let sourceTypes = Set(item.representations.map(\.type)) + guard !sourceTypes.contains(.png), + !sourceTypes.contains(.tiff) + else { + return nil + } + + if !sourceTypes.isDisjoint(with: Self.fileSemanticTypes) { + guard Self.fileItemContainsImage(item) else { return nil } + + // Preserve an image file's encoded bytes directly. Decoding and re-encoding it + // here would put work proportional to its pixel count on the paste critical path. + if let fileURL = Self.fileURL(in: item), + let fileType = UTType(filenameExtension: fileURL.pathExtension), + fileType.conforms(to: .image), + let imageData = try? Data(contentsOf: fileURL, options: .mappedIfSafe) + { + return .init( + type: NSPasteboard.PasteboardType(fileType.identifier), + data: imageData + ) + } + } + + for representation in item.representations { + guard UTType(representation.type.rawValue)?.conforms(to: .image) == true, + let pngData = Self.pngData(from: representation.data) + else { + continue + } + return .init(type: .png, data: pngData) + } + return nil + } + + private static func fileItemContainsImage(_ item: PasteboardSnapshot.Item) -> Bool { + if let fileURL = fileURL(in: item), + isImageFilename(fileURL.lastPathComponent) + { + return true + } + + guard let textRepresentation = item.representations.first(where: { $0.type == .string }), + let filename = String(data: textRepresentation.data, encoding: .utf8) + else { + return false + } + return Self.isImageFilename(filename) + } + + private static func fileURL(in item: PasteboardSnapshot.Item) -> URL? { + guard let representation = item.representations.first(where: { $0.type == .fileURL }) else { + return nil + } + return URL(dataRepresentation: representation.data, relativeTo: nil, isAbsolute: true) + } + + private static func isImageFilename(_ filename: String) -> Bool { + let pathExtension = (filename as NSString).pathExtension + return !pathExtension.isEmpty && UTType(filenameExtension: pathExtension)?.conforms(to: .image) == true + } + + private static func writeRestoredRepresentation( + _ representation: PasteboardSnapshot.Item.Representation, + to item: NSPasteboardItem + ) -> Bool { + if representation.type == .fileURL, + let fileURL = URL(dataRepresentation: representation.data, relativeTo: nil, isAbsolute: true) + { + // A file URL is a semantic pasteboard property-list value. Re-emitting its + // opaque bytes can appear valid to this process while disappearing for readers. + return item.setString(fileURL.absoluteString, forType: .fileURL) + } + return item.setData(representation.data, forType: representation.type) + } + + private static func pngData(from sourceData: Data) -> Data? { + if let bitmap = NSBitmapImageRep(data: sourceData) { + return bitmap.representation(using: .png, properties: [:]) + } + + guard let image = NSImage(data: sourceData) else { return nil } + return Self.pngData(from: image) + } + + private static func pngData(from image: NSImage) -> Data? { + var proposedRect = NSRect(origin: .zero, size: image.size) + guard let cgImage = image.cgImage( + forProposedRect: &proposedRect, + context: nil, + hints: nil + ) else { + return nil + } + return NSBitmapImageRep(cgImage: cgImage).representation(using: .png, properties: [:]) + } + + private static func addRestorationMarkers(to item: NSPasteboardItem) -> Bool { + let transientWasWritten = item.types.contains(Self.transientType) || + item.setData(Data(), forType: Self.transientType) + let autoGeneratedWasWritten = item.types.contains(Self.autoGeneratedType) || + item.setData(Data(), forType: Self.autoGeneratedType) + return transientWasWritten && autoGeneratedWasWritten + } + + private static func representationSummary(_ item: PasteboardSnapshot.Item?) -> String { + guard let item else { return "none" } + return item.representations + .map { "\($0.type.rawValue):\($0.data.count)" } + .joined(separator: ",") + } + + private func logSnapshotFailure( + reason: String, + itemCount: Int, + itemIndex: Int, + totalBytes: Int, + type: NSPasteboard.PasteboardType? = nil + ) { + self.log( + "clipboard_snapshot_failed reason=\(reason) items=\(itemCount) itemIndex=\(itemIndex) bytes=\(totalBytes) type=\(type?.rawValue ?? "nil")" + ) + } + + private func logSnapshotRetry( + attempt: Int, + startingChangeCount: Int, + endingChangeCount: Int + ) { + self.log( + "clipboard_snapshot_retry attempt=\(attempt) startChangeCount=\(startingChangeCount) endChangeCount=\(endingChangeCount)" + ) + } + + private func log(_ message: String) { + DebugLogger.shared.benchmark("TYPING_BENCH", message: message, source: "TypingBenchmark") + } + + private static func elapsedMs(since start: TimeInterval) -> Int { + Int(((ProcessInfo.processInfo.systemUptime - start) * 1000).rounded()) + } +} + +@MainActor +enum KeyboardLayoutKeyCodeResolver { + static func keyCode(for character: Character, qwertyFallback: CGKeyCode) -> CGKeyCode { + guard let source = TISCopyCurrentKeyboardLayoutInputSource()?.takeRetainedValue(), + let rawLayoutData = TISGetInputSourceProperty(source, kTISPropertyUnicodeKeyLayoutData) + else { + return qwertyFallback + } + + let layoutData = Unmanaged.fromOpaque(rawLayoutData).takeUnretainedValue() as Data + return layoutData.withUnsafeBytes { buffer in + guard let layout = buffer.baseAddress?.assumingMemoryBound(to: UCKeyboardLayout.self) else { + return qwertyFallback + } + return self.keyCode(for: character, qwertyFallback: qwertyFallback) { keyCode in + self.unmodifiedScalar(for: keyCode, layout: layout) + } + } + } + + static func keyCode( + for character: Character, + qwertyFallback: CGKeyCode, + translatedScalar: (CGKeyCode) -> Unicode.Scalar? + ) -> CGKeyCode { + guard let targetScalar = character.unicodeScalars.first else { return qwertyFallback } + + for keyCode: CGKeyCode in 0..<128 where translatedScalar(keyCode) == targetScalar { + return keyCode + } + return qwertyFallback + } + + private static func unmodifiedScalar( + for keyCode: CGKeyCode, + layout: UnsafePointer + ) -> Unicode.Scalar? { + var deadKeyState: UInt32 = 0 + var characters = [UniChar](repeating: 0, count: 4) + var length = 0 + let status = UCKeyTranslate( + layout, + keyCode, + UInt16(kUCKeyActionDisplay), + 0, + UInt32(LMGetKbdType()), + UInt32(kUCKeyTranslateNoDeadKeysMask), + &deadKeyState, + characters.count, + &length, + &characters + ) + guard status == noErr, length > 0 else { return nil } + return Unicode.Scalar(characters[0]) + } +} + +@MainActor +final class SystemPasteCommandPoster: PasteCommandPosting { + func postGlobalPasteCommand() async -> Bool { + guard AXIsProcessTrusted() else { return false } + + let pasteKeyCode = KeyboardLayoutKeyCodeResolver.keyCode(for: "v", qwertyFallback: 0x09) + let source = CGEventSource(stateID: .combinedSessionState) + guard let vDown = CGEvent( + keyboardEventSource: source, + virtualKey: pasteKeyCode, + keyDown: true + ), let vUp = CGEvent( + keyboardEventSource: source, + virtualKey: pasteKeyCode, + keyDown: false + ) else { + return false + } + + vDown.flags = .maskCommand + vUp.flags = .maskCommand + + // The coordinator verifies pasteboard ownership before reaching this point, + // so the temporary item is ready to consume without an additional delay. + vDown.post(tap: .cghidEventTap) + vUp.post(tap: .cghidEventTap) + return true + } +} + +@MainActor +final class PasteDeliveryCoordinator { + static let shared = PasteDeliveryCoordinator() + nonisolated static let defaultSettlementDelayNanoseconds: UInt64 = 500_000_000 + + private struct ClipboardLease { + let originalSnapshot: PasteboardSnapshot + let sessionID: String + let text: String + let generation: UInt64 + let shouldKeepTranscript: Bool + var settlementTask: Task? + } + + private let pasteboard: PasteboardManaging + private let commandPoster: PasteCommandPosting + private let settlementDelayNanoseconds: UInt64 + private var generation: UInt64 = 0 + private var lease: ClipboardLease? + private var isDelivering = false + private var deliveryWaiters: [CheckedContinuation] = [] + + init( + pasteboard: PasteboardManaging? = nil, + commandPoster: PasteCommandPosting? = nil, + settlementDelayNanoseconds: UInt64 = PasteDeliveryCoordinator.defaultSettlementDelayNanoseconds + ) { + self.pasteboard = pasteboard ?? SystemPasteboardManager() + self.commandPoster = commandPoster ?? SystemPasteCommandPoster() + self.settlementDelayNanoseconds = settlementDelayNanoseconds + } + + func deliver( + _ text: String, + preserveTranscriptOnClipboard: Bool, + onCommandPosted: ((TimeInterval) -> Void)? = nil + ) async -> TextDeliveryResult { + await self.acquireDeliverySlot() + defer { self.releaseDeliverySlot() } + + let startedAt = ProcessInfo.processInfo.systemUptime + self.generation &+= 1 + let generation = self.generation + self.lease?.settlementTask?.cancel() + + let originalSnapshot = self.captureOriginalSnapshotForNextDelivery() + self.lease = nil + + guard let originalSnapshot else { + self.log("delivery_failed generation=\(generation) reason=clipboard_snapshot_failed") + return .recoverableFailure(.clipboardSnapshotFailed) + } + + let sessionID = UUID().uuidString + let writeStartedAt = ProcessInfo.processInfo.systemUptime + guard self.pasteboard.writeTemporaryText(text, sessionID: sessionID) else { + self.restoreAfterFailure(originalSnapshot, generation: generation, reason: "clipboard_write_failed") + self.log("delivery_failed generation=\(generation) reason=clipboard_write_failed") + return .recoverableFailure(.clipboardWriteFailed) + } + self.log( + "clipboard_write generation=\(generation) elapsedMs=\(Self.elapsedMs(since: writeStartedAt))" + ) + + guard await self.commandPoster.postGlobalPasteCommand() else { + self.restoreAfterFailure(originalSnapshot, generation: generation, reason: "paste_command_failed") + self.log("delivery_failed generation=\(generation) reason=paste_command_failed") + return .recoverableFailure(.pasteCommandFailed) + } + let commandPostedAt = ProcessInfo.processInfo.systemUptime + onCommandPosted?(commandPostedAt) + + self.log( + "command_posted generation=\(generation) totalMs=\(Self.elapsedMs(since: startedAt))" + ) + + self.lease = ClipboardLease( + originalSnapshot: originalSnapshot, + sessionID: sessionID, + text: text, + generation: generation, + shouldKeepTranscript: preserveTranscriptOnClipboard, + settlementTask: nil + ) + self.scheduleSettlement(sessionID: sessionID, generation: generation) + return .commandPosted + } + + private func acquireDeliverySlot() async { + guard self.isDelivering else { + self.isDelivering = true + return + } + + await withCheckedContinuation { continuation in + self.deliveryWaiters.append(continuation) + } + } + + private func releaseDeliverySlot() { + guard !self.deliveryWaiters.isEmpty else { + self.isDelivering = false + return + } + + self.deliveryWaiters.removeFirst().resume() + } + + func runPendingSettlementForTesting() { + guard let lease = self.lease else { return } + lease.settlementTask?.cancel() + self.settleIfCurrent(sessionID: lease.sessionID, generation: lease.generation) + } + + private func scheduleSettlement(sessionID: String, generation: UInt64) { + let settlementDelayNanoseconds = self.settlementDelayNanoseconds + let task = Task { @MainActor [weak self] in + try? await Task.sleep(nanoseconds: settlementDelayNanoseconds) + guard !Task.isCancelled else { return } + self?.settleIfCurrent(sessionID: sessionID, generation: generation) + } + self.lease?.settlementTask = task + self.log("settlement_scheduled generation=\(generation) delayMs=\(settlementDelayNanoseconds / 1_000_000)") + } + + private func settleIfCurrent(sessionID: String, generation: UInt64) { + guard let lease = self.lease, + lease.generation == generation, + lease.sessionID == sessionID + else { + self.log("settlement_skipped generation=\(generation) reason=stale_generation") + return + } + + self.lease = nil + guard self.pasteboard.isOwned(sessionID: sessionID, expectedText: lease.text) else { + self.log("settlement_skipped generation=\(generation) reason=clipboard_changed") + return + } + + if lease.shouldKeepTranscript { + let didWrite = self.pasteboard.writeIntentionalText(lease.text) + self.log("intentional_copy_settled generation=\(generation) success=\(didWrite)") + } else { + let didRestore = self.pasteboard.restore(lease.originalSnapshot) + self.log("restore_completed generation=\(generation) success=\(didRestore)") + } + } + + private func captureOriginalSnapshotForNextDelivery() -> PasteboardSnapshot? { + if let lease = self.lease, + self.pasteboard.isOwned(sessionID: lease.sessionID, expectedText: lease.text) + { + return lease.originalSnapshot + } + return self.pasteboard.captureSnapshot() + } + + private func restoreAfterFailure( + _ snapshot: PasteboardSnapshot, + generation: UInt64, + reason: String + ) { + let didRestore = self.pasteboard.restore(snapshot) + self.log("failure_restore generation=\(generation) reason=\(reason) success=\(didRestore)") + } + + private func log(_ message: String) { + DebugLogger.shared.benchmark("TYPING_BENCH", message: message, source: "TypingBenchmark") + } + + private static func elapsedMs(since start: TimeInterval) -> Int { + Int(((ProcessInfo.processInfo.systemUptime - start) * 1000).rounded()) + } +} diff --git a/Sources/Fluid/Services/RewriteModeService.swift b/Sources/Fluid/Services/RewriteModeService.swift index 41650cc0b..a23cf4a9a 100644 --- a/Sources/Fluid/Services/RewriteModeService.swift +++ b/Sources/Fluid/Services/RewriteModeService.swift @@ -146,10 +146,11 @@ final class RewriteModeService: ObservableObject { } } - func acceptRewrite() { + @MainActor + func acceptRewrite() async { guard !self.rewrittenText.isEmpty else { return } NSApp.hide(nil) // Restore focus to the previous app - self.typingService.typeTextInstantly(self.rewrittenText) + _ = await self.typingService.typeTextInstantly(self.rewrittenText) } func clearState() { diff --git a/Sources/Fluid/Services/TypingService.swift b/Sources/Fluid/Services/TypingService.swift index 4ee2ac383..e0383b5ad 100644 --- a/Sources/Fluid/Services/TypingService.swift +++ b/Sources/Fluid/Services/TypingService.swift @@ -73,32 +73,35 @@ final class TypingService { return !isSecureTextField && exactFocusIsActive } - // Logging toggle (off by default). Enable by setting env FLUID_TYPING_LOGS=1 - // or UserDefaults bool for key "enableTypingLogs". - private static var isLoggingEnabled: Bool { + /// Logging toggle (off by default). Enable by setting env FLUID_TYPING_LOGS=1 + /// or UserDefaults bool for key "enableTypingLogs". + private nonisolated static var isLoggingEnabled: Bool { if let env = ProcessInfo.processInfo.environment["FLUID_TYPING_LOGS"], env == "1" { return true } return UserDefaults.standard.bool(forKey: "enableTypingLogs") } - private func log(_ message: @autoclosure () -> String) { + private nonisolated func log(_ message: @autoclosure () -> String) { guard TypingService.isLoggingEnabled else { return } - DebugLogger.shared.debug(message(), source: "TypingService") + Self.emitDebugLog(message()) } - private var isCurrentlyTyping = false - - private struct FocusSnapshot { + struct RecordingTargetContext { + let id: UUID let pid: pid_t + let bundleIdentifier: String? let window: AXUIElement? let element: AXUIElement? } - private struct PasteboardItemSnapshot { - let dataByType: [NSPasteboard.PasteboardType: Data] - } + enum FocusPreparationResult: String { + case alreadyFocused = "already_focused" + case restoredExactTarget = "restored_exact_target" + case activatedForRecovery = "activated_for_recovery" + case failed - private struct PasteboardSnapshot { - let items: [PasteboardItemSnapshot] + var isReady: Bool { + self != .failed + } } private struct FocusedTextSnapshot { @@ -110,101 +113,50 @@ final class TypingService { let appScriptSelectedRange: CFRange? } - private enum PasteVerificationResult: String { - case appScriptContainsText = "appscript_contains_text" - case appScriptCaretMovedExpectedDistance = "appscript_caret_moved_expected_distance" - case fieldContainsText = "field_contains_text" - case caretMovedExpectedDistance = "caret_moved_expected_distance" - case timeout - case unavailable - } - - private static let focusSnapshotQueue = DispatchQueue(label: "TypingService.FocusSnapshot") - private static let pasteboardSessionSemaphore = DispatchSemaphore(value: 1) - private static let pasteboardRestoreQueue = DispatchQueue(label: "TypingService.PasteboardRestore", qos: .utility) - private static var focusSnapshot: FocusSnapshot? private static let ghosttyBundleIdentifier = "com.mitchellh.ghostty" + private static let directInsertionQueue = DispatchQueue( + label: "com.FluidApp.Fluid.direct-text-insertion", + qos: .userInitiated + ) + static let recoveryActivationOptions: NSApplication.ActivationOptions = [.activateIgnoringOtherApps] private var textInsertionMode: SettingsStore.TextInsertionMode { SettingsStore.shared.textInsertionMode } - // MARK: - Layout-aware key code lookup - - /// Returns the virtual key code that produces `character` under the current keyboard layout. - /// Uses the TIS (Text Input Services) API which must run on the main thread, so the lookup - /// is dispatched there when called from a background thread. Falls back to `qwertyFallback` - /// if the layout data is unavailable. - private static func virtualKeyCode(for character: Character, qwertyFallback: CGKeyCode) -> CGKeyCode { - if Thread.isMainThread { - return self.tisLookup(for: character, qwertyFallback: qwertyFallback) - } - var result = qwertyFallback - DispatchQueue.main.sync { - result = self.tisLookup(for: character, qwertyFallback: qwertyFallback) - } - return result - } - - /// Performs the actual TIS + UCKeyTranslate scan. Must be called on the main thread. - private static func tisLookup(for character: Character, qwertyFallback: CGKeyCode) -> CGKeyCode { - guard let targetScalar = character.unicodeScalars.first else { return qwertyFallback } - - guard let sourceRef = TISCopyCurrentKeyboardLayoutInputSource()?.takeRetainedValue(), - let rawPtr = TISGetInputSourceProperty(sourceRef, kTISPropertyUnicodeKeyLayoutData) - else { - return qwertyFallback - } - let layoutData = Unmanaged.fromOpaque(rawPtr).takeUnretainedValue() as Data - - return layoutData.withUnsafeBytes { buffer -> CGKeyCode in - guard let layoutPtr = buffer.baseAddress?.assumingMemoryBound(to: UCKeyboardLayout.self) else { - return qwertyFallback - } - var deadKeyState: UInt32 = 0 - var chars = [UniChar](repeating: 0, count: 4) - var length = 0 - let kbType = UInt32(LMGetKbdType()) - - for keyCode: UInt16 in 0..<128 { - deadKeyState = 0 - length = 0 - let status = UCKeyTranslate( - layoutPtr, - keyCode, - UInt16(kUCKeyActionDisplay), - 0, - kbType, - UInt32(kUCKeyTranslateNoDeadKeysMask), - &deadKeyState, - chars.count, - &length, - &chars - ) - guard status == noErr, length > 0 else { continue } - if Unicode.Scalar(chars[0]) == targetScalar { - return CGKeyCode(keyCode) - } - } - return qwertyFallback - } - } - - /// The virtual key code for "v" in the current keyboard layout (used for Cmd+V paste). - /// Re-evaluated on every call so runtime keyboard layout switches are picked up immediately. - private static var pasteVirtualKeyCode: CGKeyCode { - virtualKeyCode(for: "v", qwertyFallback: 9) - } - // MARK: - Focus helpers (shared) /// Best-effort: returns the PID owning the currently focused accessibility element. /// This is more reliable than NSWorkspace.frontmostApplication for floating overlays/launchers. static func captureSystemFocusTarget() -> CapturedFocusTarget? { - // Accessibility is required to query system-focused AX element. + guard AXIsProcessTrusted() else { return nil } + + let systemWideElement = AXUIElementCreateSystemWide() + var focusedElementRef: CFTypeRef? + let result = AXUIElementCopyAttributeValue( + systemWideElement, + kAXFocusedUIElementAttribute as CFString, + &focusedElementRef + ) + guard result == .success, let focusedElementRef, + CFGetTypeID(focusedElementRef) == AXUIElementGetTypeID() + else { return nil } + + let element = unsafeBitCast(focusedElementRef, to: AXUIElement.self) + var pid: pid_t = 0 + AXUIElementGetPid(element, &pid) + guard pid > 0 else { return nil } + let appElement = AXUIElementCreateApplication(pid) + let window = Self.copyAXElementAttribute(from: appElement, attribute: kAXFocusedWindowAttribute as CFString) + ?? Self.copyAXElementAttribute(from: appElement, attribute: kAXMainWindowAttribute as CFString) + Self.logFocusState("[TypingService] Captured focus snapshot") + return CapturedFocusTarget(pid: pid, window: window, element: element) + } + + /// Captures the immutable destination window and element before any FluidVoice UI interaction. + static func captureRecordingTargetContext() -> RecordingTargetContext? { guard AXIsProcessTrusted() else { - self.storeFocusSnapshot(nil) - return nil + return self.fallbackRecordingTargetContext() } let systemWideElement = AXUIElementCreateSystemWide() @@ -215,28 +167,50 @@ final class TypingService { kAXFocusedUIElementAttribute as CFString, &focusedElementRef ) - guard result == .success, let focusedElementRef else { - Self.storeFocusSnapshot(nil) - return nil - } - guard CFGetTypeID(focusedElementRef) == AXUIElementGetTypeID() else { - Self.storeFocusSnapshot(nil) - return nil - } + guard result == .success, let focusedElementRef, + CFGetTypeID(focusedElementRef) == AXUIElementGetTypeID() + else { return self.fallbackRecordingTargetContext() } let element = unsafeBitCast(focusedElementRef, to: AXUIElement.self) var pid: pid_t = 0 AXUIElementGetPid(element, &pid) - guard pid > 0 else { - Self.storeFocusSnapshot(nil) - return nil - } + guard pid > 0 else { return self.fallbackRecordingTargetContext() } let appElement = AXUIElementCreateApplication(pid) let window = Self.copyAXElementAttribute(from: appElement, attribute: kAXFocusedWindowAttribute as CFString) ?? Self.copyAXElementAttribute(from: appElement, attribute: kAXMainWindowAttribute as CFString) - Self.storeFocusSnapshot(FocusSnapshot(pid: pid, window: window, element: element)) Self.logFocusState("[TypingService] Captured focus snapshot") - return CapturedFocusTarget(pid: pid, window: window, element: element) + return RecordingTargetContext( + id: UUID(), + pid: pid, + bundleIdentifier: NSRunningApplication(processIdentifier: pid)?.bundleIdentifier, + window: window, + element: element + ) + } + + /// Read-only focus observation. It must never replace the recording target snapshot. + static func currentFocusedPID() -> pid_t? { + guard AXIsProcessTrusted() else { + return NSWorkspace.shared.frontmostApplication?.processIdentifier + } + + let systemWideElement = AXUIElementCreateSystemWide() + var focusedElementRef: CFTypeRef? + let result = AXUIElementCopyAttributeValue( + systemWideElement, + kAXFocusedUIElementAttribute as CFString, + &focusedElementRef + ) + guard result == .success, let focusedElementRef, + CFGetTypeID(focusedElementRef) == AXUIElementGetTypeID() + else { + return NSWorkspace.shared.frontmostApplication?.processIdentifier + } + + let element = unsafeBitCast(focusedElementRef, to: AXUIElement.self) + var pid: pid_t = 0 + AXUIElementGetPid(element, &pid) + return pid > 0 ? pid : NSWorkspace.shared.frontmostApplication?.processIdentifier } static func captureSystemFocusedPID() -> pid_t? { @@ -289,6 +263,17 @@ final class TypingService { return self.isExactFocusTargetActive(target) } + private static func fallbackRecordingTargetContext() -> RecordingTargetContext? { + guard let app = NSWorkspace.shared.frontmostApplication else { return nil } + return RecordingTargetContext( + id: UUID(), + pid: app.processIdentifier, + bundleIdentifier: app.bundleIdentifier, + window: nil, + element: nil + ) + } + /// Best-effort: returns the text immediately before the caret in the currently focused /// text field. Used by Continuous Dictation Mode to decide capitalization when chaining /// transcribed segments. Returns "" when the focused field/context is unavailable. @@ -296,52 +281,70 @@ final class TypingService { TypingService().captureTextBeforeCursorInFocusedField() } - @discardableResult - static func restoreCapturedFocus(in pid: pid_t) -> Bool { - guard AXIsProcessTrusted() else { return false } - guard let snapshot = loadFocusSnapshot(), - snapshot.pid == pid else { return false } + static func prepareTargetForDelivery(_ context: RecordingTargetContext) async -> FocusPreparationResult { + if context.pid == self.currentFocusedPID(), + context.element == nil || self.isCapturedFocusStillActive(context) + { + return .alreadyFocused + } - Self.logFocusState("[TypingService] Before restoreCapturedFocus") - let appElement = AXUIElementCreateApplication(pid) + if await self.restoreExactTarget(context) { + return .restoredExactTarget + } + + guard context.window != nil, context.element != nil, + self.activateAppForRecovery(pid: context.pid) + else { + return .failed + } + + try? await Task.sleep(nanoseconds: 25_000_000) + return await self.restoreExactTarget(context) ? .activatedForRecovery : .failed + } - if let window = snapshot.window { + private static func restoreExactTarget(_ context: RecordingTargetContext) async -> Bool { + guard AXIsProcessTrusted(), context.window != nil, context.element != nil else { return false } + + self.logFocusState("[TypingService] Before restoreExactTarget") + let appElement = AXUIElementCreateApplication(context.pid) + + if let window = context.window { _ = AXUIElementPerformAction(window, kAXRaiseAction as CFString) _ = AXUIElementSetAttributeValue(appElement, kAXMainWindowAttribute as CFString, window) _ = AXUIElementSetAttributeValue(appElement, kAXFocusedWindowAttribute as CFString, window) - usleep(40_000) + try? await Task.sleep(nanoseconds: 25_000_000) } - guard let element = snapshot.element else { return false } + guard let element = context.element else { return false } - for _ in 0..<3 { + for attempt in 0..<3 { let result = AXUIElementSetAttributeValue( element, kAXFocusedAttribute as CFString, kCFBooleanTrue ) - if result == .success, Self.isCurrentlyFocusedElement(element, expectedPID: pid) { - Self.logFocusState("[TypingService] After restoreCapturedFocus success") + if result == .success, Self.isCurrentlyFocusedElement(element, expectedPID: context.pid) { + Self.logFocusState("[TypingService] After restoreExactTarget success") return true } - usleep(50_000) + if attempt < 2 { + try? await Task.sleep(nanoseconds: 25_000_000) + } } - let isFocused = Self.isCurrentlyFocusedElement(element, expectedPID: pid) - Self.logFocusState("[TypingService] After restoreCapturedFocus final result=\(isFocused)") + let isFocused = Self.isCurrentlyFocusedElement(element, expectedPID: context.pid) + Self.logFocusState("[TypingService] After restoreExactTarget final result=\(isFocused)") return isFocused } - static func isCapturedFocusStillActive(for pid: pid_t) -> Bool { + static func isCapturedFocusStillActive(_ context: RecordingTargetContext) -> Bool { guard AXIsProcessTrusted(), - let snapshot = loadFocusSnapshot(), - snapshot.pid == pid, - let element = snapshot.element + let element = context.element else { return false } - return Self.isCurrentlyFocusedElement(element, expectedPID: pid) + return Self.isCurrentlyFocusedElement(element, expectedPID: context.pid) } private func isGhosttyApplication(pid: pid_t) -> Bool { @@ -374,16 +377,9 @@ final class TypingService { return nil } - /// Activation options used to restore focus to the external target app after dictation. - /// `.activateAllWindows` is intentionally omitted: raising every window of a multi-window - /// app (e.g. WebStorm) destroys the user's window layout on each dictation (issue #748). - static let focusRestoreActivationOptions: NSApplication.ActivationOptions = [ - .activateIgnoringOtherApps, - ] - - /// Best-effort: activates the app with the given PID, unless it's Fluid itself. + /// Recovery-only activation. Normal dictation must preserve the destination's focus. @discardableResult - static func activateApp(pid: pid_t) -> Bool { + static func activateAppForRecovery(pid: pid_t) -> Bool { guard pid > 0 else { return false } guard let app = NSRunningApplication(processIdentifier: pid) else { return false } @@ -395,45 +391,54 @@ final class TypingService { return false } - return app.activate(options: Self.focusRestoreActivationOptions) + return app.activate(options: Self.recoveryActivationOptions) } // MARK: - Public API - func typeTextInstantly(_ text: String) { - self.typeTextInstantly(text, preferredTargetPID: nil, textReadyAt: nil) + @MainActor + func typeTextInstantly(_ text: String) async -> TextDeliveryResult { + await self.typeTextInstantly(text, preferredTargetPID: nil, textReadyAt: nil) } /// Types/inserts text, optionally preferring a specific target PID for CGEvent posting. /// This helps when our overlay temporarily has focus; we can still target the original app. - func typeTextInstantly(_ text: String, preferredTargetPID: pid_t?) { - self.typeTextInstantly(text, preferredTargetPID: preferredTargetPID, textReadyAt: nil) + @MainActor + func typeTextInstantly(_ text: String, preferredTargetPID: pid_t?) async -> TextDeliveryResult { + await self.typeTextInstantly(text, preferredTargetPID: preferredTargetPID, textReadyAt: nil) } /// Types/inserts text, optionally preferring a specific target PID for CGEvent posting. /// This helps when our overlay temporarily has focus; we can still target the original app. - func typeTextInstantly(_ text: String, preferredTargetPID: pid_t?, textReadyAt: TimeInterval?) { - self.typeOutputPlanInstantly(.plain(text), preferredTargetPID: preferredTargetPID, textReadyAt: textReadyAt) + @MainActor + func typeTextInstantly( + _ text: String, + preferredTargetPID: pid_t?, + textReadyAt: TimeInterval?, + toggleStopRequestedAt: TimeInterval? = nil, + preserveTranscriptOnClipboard: Bool = false + ) async -> TextDeliveryResult { + await self.typeOutputPlanInstantly( + .plain(text), + preferredTargetPID: preferredTargetPID, + textReadyAt: textReadyAt, + toggleStopRequestedAt: toggleStopRequestedAt, + preserveTranscriptOnClipboard: preserveTranscriptOnClipboard + ) } + @MainActor func typeOutputPlanInstantly( _ plan: DictationLiteralOutputPlan, preferredTargetPID: pid_t?, textReadyAt: TimeInterval?, + toggleStopRequestedAt: TimeInterval? = nil, tracksDictionaryCorrections: Bool = false, - postInsertionKey: SettingsStore.SpokenSendKey? = nil, - requiredFocusTarget: CapturedFocusTarget? = nil, - completion: ((DeliveryOutcome) -> Void)? = nil - ) { + preserveTranscriptOnClipboard: Bool = false + ) async -> TextDeliveryResult { let requestedAt = ProcessInfo.processInfo.systemUptime let text = plan.plainText let mode = self.textInsertionMode - let settleDelayMs: Int = { - if mode == .reliablePaste { - return preferredTargetPID == nil ? 80 : 0 - } - return preferredTargetPID == nil ? 200 : 0 - }() let textReadyAge = textReadyAt.map { Self.elapsedMs(from: $0, to: requestedAt) } self.bench( "request chars=\(text.count) mode=\(mode.rawValue) autocompleteSteps=\(plan.steps.count) preferredPID=\(preferredTargetPID.map { String($0) } ?? "nil") textReadyAgeMs=\(textReadyAge.map { String($0) } ?? "nil")" @@ -441,19 +446,18 @@ final class TypingService { self.log("[TypingService] ENTRY: typeTextInstantly called with text length: \(text.count)") self.log("[TypingService] Text preview: \"\(String(text.prefix(100)))\"") - guard text.isEmpty == false || postInsertionKey != nil else { + guard !text.isEmpty else { self.bench("request_return reason=empty_text") self.log("[TypingService] ERROR: Empty text provided, aborting") - completion?(.rejected) - return - } - - // Prevent concurrent typing operations - guard !self.isCurrentlyTyping else { - self.bench("request_return reason=already_typing") - self.log("[TypingService] WARNING: Skipping text injection - already in progress") - completion?(.rejected) - return + let result = TextDeliveryResult.recoverableFailure(.emptyText) + self.recordInsertionLatency( + path: .notAttempted, + result: result, + requestedAt: requestedAt, + textReadyAt: textReadyAt, + toggleStopRequestedAt: toggleStopRequestedAt + ) + return result } // Check accessibility permissions first @@ -461,103 +465,144 @@ final class TypingService { self.bench("request_return reason=accessibility_not_trusted") self.log("[TypingService] ERROR: Accessibility permissions required for text injection") self.log("[TypingService] Current accessibility status: \(AXIsProcessTrusted())") - completion?(.rejected) - return + let result = TextDeliveryResult.recoverableFailure(.accessibilityNotTrusted) + self.recordInsertionLatency( + path: .notAttempted, + result: result, + requestedAt: requestedAt, + textReadyAt: textReadyAt, + toggleStopRequestedAt: toggleStopRequestedAt + ) + return result + } + + let usesClipboard = mode == .reliablePaste || + self.ghosttyTargetPID(preferredTargetPID: preferredTargetPID) != nil + let result: TextDeliveryResult + let deliveryPath: AnalyticsInsertionPath + var dispatchedAt: TimeInterval? + if usesClipboard { + deliveryPath = .clipboard + result = await PasteDeliveryCoordinator.shared.deliver( + text, + preserveTranscriptOnClipboard: preserveTranscriptOnClipboard, + onCommandPosted: { dispatchedAt = $0 } + ) + } else if await self.insertTextDirectlyOffMain(text, preferredTargetPID: preferredTargetPID) { + deliveryPath = .direct + if preserveTranscriptOnClipboard { + _ = ClipboardService.copyToClipboard(text) + } + result = .commandPosted + } else { + deliveryPath = .clipboardFallback + self.log("[TypingService] Direct insertion failed; using non-blocking clipboard fallback") + result = await PasteDeliveryCoordinator.shared.deliver( + text, + preserveTranscriptOnClipboard: preserveTranscriptOnClipboard, + onCommandPosted: { dispatchedAt = $0 } + ) } - self.log("[TypingService] Accessibility check passed, proceeding with text injection") - self.isCurrentlyTyping = true - - DispatchQueue.global(qos: .userInitiated).async { - var outcome: DeliveryOutcome = .insertionFailed - let workerStartedAt = ProcessInfo.processInfo.systemUptime - self.bench("worker_start queueDelayMs=\(Self.elapsedMs(from: requestedAt, to: workerStartedAt))") + let completedAt = dispatchedAt ?? ProcessInfo.processInfo.systemUptime + self.bench( + "complete result=\(String(describing: result)) totalMs=\(Self.elapsedMs(from: requestedAt, to: completedAt)) textReadyToCompleteMs=\(textReadyAt.map { String(Self.elapsedMs(from: $0, to: completedAt)) } ?? "nil")" + ) + self.recordInsertionLatency( + path: deliveryPath, + result: result, + requestedAt: requestedAt, + textReadyAt: textReadyAt, + toggleStopRequestedAt: toggleStopRequestedAt, + completedAt: completedAt + ) + if result.wasDispatched, tracksDictionaryCorrections { + AutomaticDictionaryCorrectionTracker.shared.beginObservingInsertion( + text, + targetPID: preferredTargetPID + ) + } + return result + } - defer { - let completedAt = ProcessInfo.processInfo.systemUptime - self.isCurrentlyTyping = false - self.bench( - "complete totalMs=\(Self.elapsedMs(from: requestedAt, to: completedAt)) textReadyToCompleteMs=\(textReadyAt.map { String(Self.elapsedMs(from: $0, to: completedAt)) } ?? "nil")" - ) - self.log("[TypingService] Typing operation completed, isCurrentlyTyping set to false") - completion?(outcome) + func typeOutputPlanInstantly( + _ plan: DictationLiteralOutputPlan, + preferredTargetPID: pid_t?, + textReadyAt: TimeInterval?, + toggleStopRequestedAt: TimeInterval? = nil, + tracksDictionaryCorrections: Bool = false, + postInsertionKey: SettingsStore.SpokenSendKey? = nil, + requiredFocusTarget: CapturedFocusTarget? = nil, + preserveTranscriptOnClipboard: Bool = false, + completion: ((DeliveryOutcome) -> Void)? = nil + ) { + Task { @MainActor in + let hasTextToInsert = !plan.plainText.isEmpty + guard hasTextToInsert || postInsertionKey != nil else { + completion?(.rejected) + return } - self.log("[TypingService] Starting async text insertion process") - if settleDelayMs > 0 { - usleep(useconds_t(settleDelayMs * 1000)) - } - self.bench("settle_delay_done delayMs=\(settleDelayMs) elapsedMs=\(Self.elapsedMs(since: requestedAt))") - let hasTextToInsert = !text.isEmpty if postInsertionKey != nil { - guard let preferredTargetPID, let requiredFocusTarget else { - outcome = .actionSuppressed - return - } - // A held dictation modifier must suppress only the key action, - // not the dictated text. The longer check below waits for - // modifiers after insertion before deciding whether to send. - guard Self.canInsertBeforePostInsertionAction( - preferredTargetPID: preferredTargetPID, - requiredTargetPID: requiredFocusTarget.pid, - isSecureTextField: requiredFocusTarget.isSecureTextField, - exactFocusIsActive: Self.isExactFocusTargetActive(requiredFocusTarget) - ) else { - outcome = .actionSuppressed + guard let preferredTargetPID, let requiredFocusTarget, + Self.canInsertBeforePostInsertionAction( + preferredTargetPID: preferredTargetPID, + requiredTargetPID: requiredFocusTarget.pid, + isSecureTextField: requiredFocusTarget.isSecureTextField, + exactFocusIsActive: Self.isExactFocusTargetActive(requiredFocusTarget) + ) + else { + completion?(.actionSuppressed) return } } + var outcome: DeliveryOutcome = .actionSuppressed if hasTextToInsert { - self.log("[TypingService] Delay completed, calling insertTextInstantly") - let insertStartedAt = ProcessInfo.processInfo.systemUptime - self.bench("insert_call") - let inserted = self.insertTextInstantly(text, preferredTargetPID: preferredTargetPID) - self.bench( - "insert_return elapsedMs=\(Self.elapsedMs(since: insertStartedAt)) totalMs=\(Self.elapsedMs(since: requestedAt))" + let result = await self.typeOutputPlanInstantly( + plan, + preferredTargetPID: preferredTargetPID, + textReadyAt: textReadyAt, + toggleStopRequestedAt: toggleStopRequestedAt, + tracksDictionaryCorrections: tracksDictionaryCorrections, + preserveTranscriptOnClipboard: preserveTranscriptOnClipboard ) - guard inserted else { - outcome = .insertionFailed + guard result.wasDispatched else { + completion?(.insertionFailed) return } - outcome = .inserted - if tracksDictionaryCorrections, postInsertionKey == nil { - Task { @MainActor in - AutomaticDictionaryCorrectionTracker.shared.beginObservingInsertion( - text, - targetPID: preferredTargetPID - ) - } - } } - guard let postInsertionKey else { return } + guard let postInsertionKey else { + completion?(outcome) + return + } guard let preferredTargetPID, let requiredFocusTarget else { - outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed + completion?(hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed) return } - let modifiersReleased = self.waitForPhysicalModifiersToRelease(timeout: 2) - let exactFocusIsActive = Self.isExactFocusTargetActive(requiredFocusTarget) + + let modifiersReleased = await self.waitForPhysicalModifiersToRelease(timeout: 2) guard Self.canDispatchPostInsertionAction( preferredTargetPID: preferredTargetPID, requiredTargetPID: requiredFocusTarget.pid, isSecureTextField: requiredFocusTarget.isSecureTextField, modifiersReleased: modifiersReleased, - exactFocusIsActive: exactFocusIsActive + exactFocusIsActive: Self.isExactFocusTargetActive(requiredFocusTarget) ) else { - outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed + completion?(hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed) return } - usleep(50_000) + try? await Task.sleep(nanoseconds: 50_000_000) guard Self.isExactFocusTargetActive(requiredFocusTarget), - self.postReturnKey(postInsertionKey, targetPID: preferredTargetPID) + await self.postReturnKey(postInsertionKey, targetPID: preferredTargetPID) else { - outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed + completion?(hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed) return } - outcome = hasTextToInsert ? .insertedAndActionDispatched : .actionDispatched + completion?(hasTextToInsert ? .insertedAndActionDispatched : .actionDispatched) } } @@ -565,6 +610,47 @@ final class TypingService { DebugLogger.shared.benchmark("TYPING_BENCH", message: message, source: "TypingBenchmark") } + private func recordInsertionLatency( + path: AnalyticsInsertionPath, + result: TextDeliveryResult, + requestedAt: TimeInterval, + textReadyAt: TimeInterval?, + toggleStopRequestedAt: TimeInterval?, + completedAt: TimeInterval = ProcessInfo.processInfo.systemUptime + ) { + let outcome = Self.analyticsOutcome(for: result) + let isClipboardDispatch = outcome == .dispatched && + (path == .clipboard || path == .clipboardFallback) + AnalyticsService.shared.recordInsertionLatency( + path: path, + outcome: outcome, + requestMilliseconds: max(0, Self.elapsedMs(from: requestedAt, to: completedAt)), + readyMilliseconds: textReadyAt.map { + max(0, Self.elapsedMs(from: $0, to: completedAt)) + }, + toggleStopMilliseconds: isClipboardDispatch ? toggleStopRequestedAt.map { + max(0, Self.elapsedMs(from: $0, to: completedAt)) + } : nil + ) + } + + private static func analyticsOutcome(for result: TextDeliveryResult) -> AnalyticsInsertionOutcome { + switch result { + case .commandPosted: + .dispatched + case let .recoverableFailure(failure): + switch failure { + case .emptyText: .emptyText + case .accessibilityNotTrusted: .accessibilityNotTrusted + case .clipboardSnapshotFailed: .clipboardSnapshotFailed + case .clipboardWriteFailed: .clipboardWriteFailed + case .pasteCommandFailed: .pasteCommandFailed + case .targetUnavailable: .targetUnavailable + case .targetRestoreFailed: .targetRestoreFailed + } + } + } + private static func elapsedMs(since start: TimeInterval) -> Int { Int(((ProcessInfo.processInfo.systemUptime - start) * 1000).rounded()) } @@ -575,29 +661,21 @@ final class TypingService { // MARK: - Internal insertion pipeline - private func insertTextInstantly(_ text: String, preferredTargetPID: pid_t?) -> Bool { - self.log("[TypingService] insertTextInstantly called with \(text.count) characters") - self.log("[TypingService] Attempting to type text: \"\(text.prefix(50))\(text.count > 50 ? "..." : "")\"") - - if self.textInsertionMode == .standard, - let ghosttyTargetPID = self.ghosttyTargetPID(preferredTargetPID: preferredTargetPID) - { - self.log("[TypingService] Ghostty target detected in standard mode (PID \(ghosttyTargetPID)); forcing Reliable Paste path") - if self.tryReliablePasteInsertion(text, preferredTargetPID: ghosttyTargetPID) { - self.log("[TypingService] SUCCESS: Ghostty Reliable Paste path completed") - return true + private func insertTextDirectlyOffMain(_ text: String, preferredTargetPID: pid_t?) async -> Bool { + await withCheckedContinuation { continuation in + Self.directInsertionQueue.async { + continuation.resume( + returning: self.insertTextDirectly(text, preferredTargetPID: preferredTargetPID) + ) } - self.log("[TypingService] Ghostty Reliable Paste path fell through to direct-typing fallbacks") } + } - if self.textInsertionMode == .reliablePaste { - self.log("[TypingService] Reliable Paste mode enabled") - if self.tryReliablePasteInsertion(text, preferredTargetPID: preferredTargetPID) { - self.log("[TypingService] SUCCESS: Reliable Paste mode completed") - return true - } - self.log("[TypingService] Reliable Paste mode fell through to direct-typing fallbacks") - } else if let preferredTargetPID, preferredTargetPID > 0 { + private nonisolated func insertTextDirectly(_ text: String, preferredTargetPID: pid_t?) -> Bool { + self.log("[TypingService] insertTextInstantly called with \(text.count) characters") + self.log("[TypingService] Attempting to type text: \"\(text.prefix(50))\(text.count > 50 ? "..." : "")\"") + + if let preferredTargetPID, preferredTargetPID > 0 { self.log("[TypingService] Experimental Direct Typing mode: trying preferred PID unicode insertion first") if self.insertTextBulkInstant(text, targetPID: preferredTargetPID) { self.log("[TypingService] SUCCESS: Preferred PID CGEvent insertion completed") @@ -655,39 +733,22 @@ final class TypingService { } } - // Fallback: Use clipboard-based insertion (more reliable) - self.log("[TypingService] CGEvent failed, trying clipboard fallback") - if self.insertTextViaClipboard(text) { - self.log("[TypingService] SUCCESS: Clipboard insertion completed") - return true - } - - // Last resort: Character-by-character - self.log("[TypingService] WARNING: All methods failed, trying character-by-character") - for (index, char) in text.enumerated() { - if index % 10 == 0 { - self.log("[TypingService] Typing character \(index + 1)/\(text.count)") - } - self.typeCharacter(char) - usleep(1000) - } - self.log("[TypingService] Character-by-character typing completed") - return true + return false } - private func waitForPhysicalModifiersToRelease(timeout: TimeInterval) -> Bool { + private func waitForPhysicalModifiersToRelease(timeout: TimeInterval) async -> Bool { let relevant: CGEventFlags = [.maskCommand, .maskControl, .maskAlternate, .maskShift, .maskSecondaryFn] let startedAt = ProcessInfo.processInfo.systemUptime while ProcessInfo.processInfo.systemUptime - startedAt < timeout { if CGEventSource.flagsState(.combinedSessionState).isDisjoint(with: relevant) { return true } - usleep(15_000) + try? await Task.sleep(nanoseconds: 15_000_000) } return false } - private func postReturnKey(_ key: SettingsStore.SpokenSendKey, targetPID: pid_t) -> Bool { + private func postReturnKey(_ key: SettingsStore.SpokenSendKey, targetPID: pid_t) async -> Bool { let returnKeyCode = CGKeyCode(kVK_Return) guard let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: returnKeyCode, keyDown: true), let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: returnKeyCode, keyDown: false) @@ -700,48 +761,14 @@ final class TypingService { keyDown.setIntegerValueField(.eventSourceUserData, value: Self.synthesizedEventUserData) keyUp.setIntegerValueField(.eventSourceUserData, value: Self.synthesizedEventUserData) keyDown.postToPid(targetPID) - usleep(10_000) + try? await Task.sleep(nanoseconds: 10_000_000) keyUp.postToPid(targetPID) return true } - private func tryReliablePasteInsertion(_ text: String, preferredTargetPID: pid_t?) -> Bool { - if let preferredTargetPID, preferredTargetPID > 0 { - self.log("[TypingService] Trying clipboard-to-PID insertion first") - if self.insertTextViaClipboardToPid(text, targetPID: preferredTargetPID) { - self.log("[TypingService] Reliable Paste dispatched via clipboard-to-PID") - return true - } - } - - self.log("[TypingService] Trying global clipboard insertion") - if self.insertTextViaClipboard(text) { - self.log("[TypingService] Reliable Paste dispatched via global clipboard paste") - return true - } - - self.log("[TypingService] Global clipboard insertion failed, trying menu paste") - if self.insertTextViaMenuPaste(text) { - self.log("[TypingService] Reliable Paste dispatched via menu paste") - return true - } - - return false - } - - private static let cgEventUnicodeChunkSize = 200 - - private static func storeFocusSnapshot(_ snapshot: FocusSnapshot?) { - self.focusSnapshotQueue.sync { - Self.focusSnapshot = snapshot - } - } - - private static func loadFocusSnapshot() -> FocusSnapshot? { - self.focusSnapshotQueue.sync { Self.focusSnapshot } - } + private nonisolated static let cgEventUnicodeChunkSize = 200 - private static func copyAXElementAttribute(from element: AXUIElement, attribute: CFString) -> AXUIElement? { + private nonisolated static func copyAXElementAttribute(from element: AXUIElement, attribute: CFString) -> AXUIElement? { var value: CFTypeRef? let result = AXUIElementCopyAttributeValue(element, attribute, &value) guard result == .success, let value else { return nil } @@ -749,14 +776,14 @@ final class TypingService { return unsafeBitCast(value, to: AXUIElement.self) } - private static func stringAXAttribute(from element: AXUIElement, attribute: CFString) -> String? { + private nonisolated static func stringAXAttribute(from element: AXUIElement, attribute: CFString) -> String? { var value: CFTypeRef? let result = AXUIElementCopyAttributeValue(element, attribute, &value) guard result == .success else { return nil } return value as? String } - private static func currentFocusDebugDescription() -> String { + private nonisolated static func currentFocusDebugDescription() -> String { let systemWideElement = AXUIElementCreateSystemWide() var focusedElementRef: CFTypeRef? let result = AXUIElementCopyAttributeValue( @@ -781,12 +808,24 @@ final class TypingService { return "focusedPID=\(pid) role=\(role) subrole=\(subrole) title=\(title) description=\(description)" } - private static func logFocusState(_ prefix: String) { + private nonisolated static func logFocusState(_ prefix: String) { guard self.isLoggingEnabled else { return } - DebugLogger.shared.debug("\(prefix) | \(self.currentFocusDebugDescription())", source: "TypingService") + self.emitDebugLog("\(prefix) | \(self.currentFocusDebugDescription())") } - private static func isCurrentlyFocusedElement(_ expectedElement: AXUIElement, expectedPID: pid_t) -> Bool { + private nonisolated static func emitDebugLog(_ message: String) { + if Thread.isMainThread { + MainActor.assumeIsolated { + DebugLogger.shared.debug(message, source: "TypingService") + } + } else { + DispatchQueue.main.async { + DebugLogger.shared.debug(message, source: "TypingService") + } + } + } + + private nonisolated static func isCurrentlyFocusedElement(_ expectedElement: AXUIElement, expectedPID: pid_t) -> Bool { let systemWideElement = AXUIElementCreateSystemWide() var focusedElementRef: CFTypeRef? let result = AXUIElementCopyAttributeValue( @@ -814,134 +853,7 @@ final class TypingService { return ["AXTextField", "AXTextArea", "AXSearchField", "AXComboBox", "AXWebArea", "AXGroup"].contains(currentRole) } - private func capturePasteboardSnapshot(_ pasteboard: NSPasteboard) -> PasteboardSnapshot { - let items: [PasteboardItemSnapshot] = pasteboard.pasteboardItems?.map { item in - var dataByType: [NSPasteboard.PasteboardType: Data] = [:] - for type in item.types { - if let data = item.data(forType: type) { - dataByType[type] = data - } - } - return PasteboardItemSnapshot(dataByType: dataByType) - } ?? [] - return PasteboardSnapshot(items: items) - } - - private func restorePasteboardSnapshot(_ snapshot: PasteboardSnapshot, to pasteboard: NSPasteboard) { - pasteboard.clearContents() - guard !snapshot.items.isEmpty else { return } - - let restoredItems = snapshot.items.map { snap -> NSPasteboardItem in - let item = NSPasteboardItem() - for (type, data) in snap.dataByType { - item.setData(data, forType: type) - } - return item - } - _ = pasteboard.writeObjects(restoredItems) - } - - /// Builds the pasteboard item for a temporary paste write, tagged with the nspasteboard.org - /// Transient and AutoGenerated marker types so clipboard managers exclude it from history. - /// `ConcealedType` is deliberately not used: it signals sensitive/password content, which - /// would be misleading for a dictation transcript. - static func makeTransientPasteboardItem(_ text: String) -> NSPasteboardItem { - let item = NSPasteboardItem() - item.setString(text, forType: .string) - item.setData(Data(), forType: NSPasteboard.PasteboardType("org.nspasteboard.TransientType")) - item.setData(Data(), forType: NSPasteboard.PasteboardType("org.nspasteboard.AutoGeneratedType")) - return item - } - - private func withTemporaryPasteboardString( - _ text: String, - restoreDelayMicros: useconds_t, - action: () -> Bool - ) -> Bool { - Self.pasteboardSessionSemaphore.wait() - var releasesPasteboardSessionOnReturn = true - defer { - if releasesPasteboardSessionOnReturn { - Self.pasteboardSessionSemaphore.signal() - } - } - - let pasteboard = NSPasteboard.general - let snapshot = self.capturePasteboardSnapshot(pasteboard) - - pasteboard.clearContents() - guard pasteboard.writeObjects([Self.makeTransientPasteboardItem(text)]) else { - self.log("[TypingService] ERROR: Failed to set temporary clipboard string") - self.restorePasteboardSnapshot(snapshot, to: pasteboard) - return false - } - let temporaryChangeCount = pasteboard.changeCount - let focusedTextSnapshot = self.captureFocusedTextSnapshot() - let actionResult = action() - guard actionResult else { - self.restorePasteboardSnapshot(snapshot, to: pasteboard) - self.log("[TypingService] Restored previous clipboard snapshot after paste dispatch failure") - return false - } - - releasesPasteboardSessionOnReturn = false - Self.pasteboardRestoreQueue.async { - defer { Self.pasteboardSessionSemaphore.signal() } - _ = self.waitForFocusedTextVerification( - from: focusedTextSnapshot, - expectedText: text, - timeoutMicros: restoreDelayMicros - ) - let pasteboard = NSPasteboard.general - - // Avoid clobbering user clipboard changes that happened after our insertion. - if pasteboard.changeCount == temporaryChangeCount || pasteboard.string(forType: .string) == text { - self.restorePasteboardSnapshot(snapshot, to: pasteboard) - self.log("[TypingService] Restored previous clipboard snapshot") - } else { - self.log("[TypingService] Skipped clipboard restore because clipboard changed externally") - } - } - - return true - } - - /// Clipboard-paste insertion targeted at a specific PID. - /// Uses postToPid for Cmd+V while preserving the full previous pasteboard payload. - private func insertTextViaClipboardToPid(_ text: String, targetPID: pid_t, activateTargetFirst: Bool = true) -> Bool { - self.log("[TypingService] Starting clipboard-to-PID insertion to PID \(targetPID)") - - guard targetPID > 0 else { - self.log("[TypingService] ERROR: Invalid target PID \(targetPID)") - return false - } - - if activateTargetFirst, NSWorkspace.shared.frontmostApplication?.processIdentifier != targetPID { - _ = Self.activateApp(pid: targetPID) - usleep(80_000) - } - - return self.withTemporaryPasteboardString(text, restoreDelayMicros: 5_000_000) { - let vKey = Self.pasteVirtualKeyCode - guard let cmdVDown = CGEvent(keyboardEventSource: nil, virtualKey: vKey, keyDown: true), - let cmdVUp = CGEvent(keyboardEventSource: nil, virtualKey: vKey, keyDown: false) - else { - self.log("[TypingService] ERROR: Failed to create Cmd+V events for PID insertion") - return false - } - - cmdVDown.flags = .maskCommand - cmdVUp.flags = .maskCommand - - cmdVDown.postToPid(targetPID) - usleep(10_000) - cmdVUp.postToPid(targetPID) - self.log("[TypingService] Cmd+V posted to PID \(targetPID)") - return true - } - } - - private func insertTextBulkInstant(_ text: String, targetPID: pid_t) -> Bool { + private nonisolated func insertTextBulkInstant(_ text: String, targetPID: pid_t) -> Bool { self.log("[TypingService] Starting chunked bulk CGEvent insertion (NO CLIPBOARD) to PID \(targetPID)") guard targetPID > 0 else { @@ -957,7 +869,7 @@ final class TypingService { } } - private func insertTextBulkHIDInstant(_ text: String) -> Bool { + private nonisolated func insertTextBulkHIDInstant(_ text: String) -> Bool { self.log("[TypingService] Starting chunked bulk CGEvent insertion via HID (NO PID)") let utf16Array = Array(text.utf16) @@ -967,7 +879,7 @@ final class TypingService { } } - private func postUnicodeChunks( + private nonisolated func postUnicodeChunks( _ utf16Array: [UInt16], destinationDescription: String, post: (CGEvent) -> Void @@ -1009,7 +921,7 @@ final class TypingService { return true } - private static func unicodeChunkEnd(in utf16Array: [UInt16], start: Int) -> Int { + private nonisolated static func unicodeChunkEnd(in utf16Array: [UInt16], start: Int) -> Int { var end = min(start + Self.cgEventUnicodeChunkSize, utf16Array.count) if end < utf16Array.count, end > start, @@ -1021,73 +933,15 @@ final class TypingService { return max(end, start + 1) } - private static func isHighSurrogate(_ value: UInt16) -> Bool { + private nonisolated static func isHighSurrogate(_ value: UInt16) -> Bool { (0xd800...0xdbff).contains(value) } - private static func isLowSurrogate(_ value: UInt16) -> Bool { + private nonisolated static func isLowSurrogate(_ value: UInt16) -> Bool { (0xdc00...0xdfff).contains(value) } - /// Clipboard-based text insertion as fallback - /// More reliable but slightly slower - copies text to clipboard then pastes - private func insertTextViaClipboard(_ text: String) -> Bool { - self.log("[TypingService] Starting clipboard-based insertion") - return self.withTemporaryPasteboardString(text, restoreDelayMicros: 5_000_000) { - let vKey = Self.pasteVirtualKeyCode - guard let cmdVDown = CGEvent(keyboardEventSource: nil, virtualKey: vKey, keyDown: true), - let cmdVUp = CGEvent(keyboardEventSource: nil, virtualKey: vKey, keyDown: false) - else { - self.log("[TypingService] ERROR: Failed to create Cmd+V events") - return false - } - - cmdVDown.flags = .maskCommand - cmdVUp.flags = .maskCommand - - cmdVDown.post(tap: .cghidEventTap) - usleep(10_000) - cmdVUp.post(tap: .cghidEventTap) - self.log("[TypingService] Cmd+V sent via clipboard insertion") - return true - } - } - - private func insertTextViaMenuPaste(_ text: String) -> Bool { - self.log("[TypingService] Starting menu-based paste insertion") - guard let appName = NSWorkspace.shared.frontmostApplication?.localizedName, !appName.isEmpty else { - self.log("[TypingService] ERROR: No frontmost app name available for menu paste") - return false - } - - return self.withTemporaryPasteboardString(text, restoreDelayMicros: 5_000_000) { - let escapedAppName = appName.replacingOccurrences(of: "\"", with: "\\\"") - let script = """ - tell application "System Events" - tell process "\(escapedAppName)" - click menu item "Paste" of menu "Edit" of menu bar 1 - end tell - end tell - """ - - guard let appleScript = NSAppleScript(source: script) else { - self.log("[TypingService] ERROR: Failed to create AppleScript for menu paste") - return false - } - - var errorInfo: NSDictionary? - let result = appleScript.executeAndReturnError(&errorInfo) - if let errorInfo { - self.log("[TypingService] ERROR: Menu paste AppleScript failed: \(errorInfo)") - return false - } - - self.log("[TypingService] Menu paste executed for app \(appName), result: \(result.stringValue ?? "ok")") - return true - } - } - - private func insertTextViaAccessibility(_ text: String) -> Bool { + private nonisolated func insertTextViaAccessibility(_ text: String) -> Bool { self.log("[TypingService] Starting Accessibility API insertion") // Try multiple strategies to find text input element @@ -1123,7 +977,7 @@ final class TypingService { return false } - private func getFocusedTextElement() -> AXUIElement? { + private nonisolated func getFocusedTextElement() -> AXUIElement? { let systemWideElement = AXUIElementCreateSystemWide() var focusedElement: CFTypeRef? @@ -1143,7 +997,7 @@ final class TypingService { return nil } - private func findTextElementInFrontmostApp() -> AXUIElement? { + private nonisolated func findTextElementInFrontmostApp() -> AXUIElement? { guard let frontmostApp = NSWorkspace.shared.frontmostApplication else { self.log("[TypingService] Could not get frontmost app") return nil @@ -1153,7 +1007,7 @@ final class TypingService { return self.findTextElementRecursively(appElement, depth: 0, maxDepth: 8) } - private func findTextElementRecursively(_ element: AXUIElement, depth: Int, maxDepth: Int) -> AXUIElement? { + private nonisolated func findTextElementRecursively(_ element: AXUIElement, depth: Int, maxDepth: Int) -> AXUIElement? { if depth > maxDepth { return nil } // Check if this element is a text input element @@ -1180,7 +1034,7 @@ final class TypingService { return nil } - private func findKeyboardFocusedElement() -> AXUIElement? { + private nonisolated func findKeyboardFocusedElement() -> AXUIElement? { guard let frontmostApp = NSWorkspace.shared.frontmostApplication else { return nil } let appElement = AXUIElementCreateApplication(frontmostApp.processIdentifier) @@ -1200,7 +1054,7 @@ final class TypingService { return nil } - private func tryAllTextInsertionMethods(_ element: AXUIElement, _ text: String) -> Bool { + private nonisolated func tryAllTextInsertionMethods(_ element: AXUIElement, _ text: String) -> Bool { // Get element info for debugging if let role = getElementAttribute(element, kAXRoleAttribute as CFString) { self.log("[TypingService] Trying insertion on element with role: \(role)") @@ -1234,7 +1088,7 @@ final class TypingService { return false } - private func getElementAttribute(_ element: AXUIElement, _ attribute: CFString) -> String? { + private nonisolated func getElementAttribute(_ element: AXUIElement, _ attribute: CFString) -> String? { var value: CFTypeRef? let result = AXUIElementCopyAttributeValue(element, attribute, &value) if result == .success, let stringValue = value as? String { @@ -1243,7 +1097,7 @@ final class TypingService { return nil } - private func getSystemFocusedElementAndPID() -> (element: AXUIElement, pid: pid_t)? { + private nonisolated func getSystemFocusedElementAndPID() -> (element: AXUIElement, pid: pid_t)? { let systemWideElement = AXUIElementCreateSystemWide() var focusedElementRef: CFTypeRef? @@ -1258,14 +1112,14 @@ final class TypingService { return (element: element, pid: pid) } - private func getElementStringValue(_ element: AXUIElement) -> String? { + private nonisolated func getElementStringValue(_ element: AXUIElement) -> String? { var value: CFTypeRef? let result = AXUIElementCopyAttributeValue(element, kAXValueAttribute as CFString, &value) guard result == .success, let str = value as? String else { return nil } return str } - private func getSelectedTextRange(_ element: AXUIElement) -> CFRange? { + private nonisolated func getSelectedTextRange(_ element: AXUIElement) -> CFRange? { var value: CFTypeRef? let result = AXUIElementCopyAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, &value) guard result == .success, let axValue = value else { return nil } @@ -1320,71 +1174,6 @@ final class TypingService { let selectedRange: CFRange? } - private func waitForFocusedTextVerification( - from snapshot: FocusedTextSnapshot?, - expectedText: String, - timeoutMicros: useconds_t - ) -> PasteVerificationResult { - guard let snapshot else { - usleep(timeoutMicros) - return .unavailable - } - - let pollMicros: useconds_t = 50_000 - let expectedLength = max(1, (expectedText as NSString).length) - let tolerance = max(2, expectedLength / 5) - var waited: useconds_t = 0 - - while waited < timeoutMicros { - usleep(pollMicros) - waited += pollMicros - - guard let current = self.captureFocusedTextSnapshot(), - current.pid == snapshot.pid - else { - continue - } - - if let currentValue = current.appScriptValue, - currentValue.contains(expectedText), - currentValue != snapshot.appScriptValue - { - return .appScriptContainsText - } - - if let before = snapshot.appScriptSelectedRange, - let after = current.appScriptSelectedRange, - after.length == 0 - { - let expectedCaretLocation = before.location + expectedLength - let caretDelta = abs(after.location - expectedCaretLocation) - if caretDelta <= tolerance { - return .appScriptCaretMovedExpectedDistance - } - } - - if let currentValue = current.value, - currentValue.contains(expectedText), - currentValue != snapshot.value - { - return .fieldContainsText - } - - if let before = snapshot.selectedRange, - let after = current.selectedRange, - after.length == 0 - { - let expectedCaretLocation = before.location + expectedLength - let caretDelta = abs(after.location - expectedCaretLocation) - if caretDelta <= tolerance { - return .caretMovedExpectedDistance - } - } - } - - return .timeout - } - private func captureAppScriptTextSnapshot(forBundleIdentifier bundleIdentifier: String?) -> AppScriptTextSnapshot? { switch bundleIdentifier { case "com.apple.dt.Xcode": @@ -1451,7 +1240,7 @@ final class TypingService { return CFRange(location: start, length: end - start) } - private func insertTextAtCursorUsingSelectedRange(_ element: AXUIElement, _ text: String) -> Bool { + private nonisolated func insertTextAtCursorUsingSelectedRange(_ element: AXUIElement, _ text: String) -> Bool { guard let currentValue = self.getElementStringValue(element) else { self.log("[TypingService] Cursor insert failed: could not read kAXValueAttribute") return false @@ -1490,8 +1279,8 @@ final class TypingService { return true } - // Why is it working now? And why is it not working now? - private func setTextViaValue(_ element: AXUIElement, _ text: String) -> Bool { + /// Why is it working now? And why is it not working now? + private nonisolated func setTextViaValue(_ element: AXUIElement, _ text: String) -> Bool { let cfText = text as CFString let result = AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, cfText) @@ -1504,7 +1293,7 @@ final class TypingService { } } - private func setTextViaSelection(_ element: AXUIElement, _ text: String) -> Bool { + private nonisolated func setTextViaSelection(_ element: AXUIElement, _ text: String) -> Bool { // First, select all existing text let selectAllResult = AXUIElementSetAttributeValue(element, kAXSelectedTextAttribute as CFString, "" as CFString) self.log("[TypingService] Select all result: \(selectAllResult.rawValue)") @@ -1522,7 +1311,7 @@ final class TypingService { } } - private func insertTextAtInsertionPoint(_ element: AXUIElement, _ text: String) -> Bool { + private nonisolated func insertTextAtInsertionPoint(_ element: AXUIElement, _ text: String) -> Bool { // Try to get the insertion point var insertionPoint: CFTypeRef? let getResult = AXUIElementCopyAttributeValue(element, kAXInsertionPointLineNumberAttribute as CFString, &insertionPoint) @@ -1540,26 +1329,4 @@ final class TypingService { return false } } - - private func typeCharacter(_ char: Character) { - let charString = String(char) - let utf16Array = Array(charString.utf16) - - // Create keyboard events for this character - guard let keyDownEvent = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true), - let keyUpEvent = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) - else { - self.log("[TypingService] ERROR: Failed to create CGEvents for character: \(char)") - return - } - - // Set the unicode string for both events - keyDownEvent.keyboardSetUnicodeString(stringLength: utf16Array.count, unicodeString: utf16Array) - keyUpEvent.keyboardSetUnicodeString(stringLength: utf16Array.count, unicodeString: utf16Array) - - // Post the events - keyDownEvent.post(tap: .cghidEventTap) - usleep(2000) // Short delay between key down and up (2ms) - keyUpEvent.post(tap: .cghidEventTap) - } } diff --git a/Sources/Fluid/UI/AnalyticsPrivacyView.swift b/Sources/Fluid/UI/AnalyticsPrivacyView.swift index 08dad9dc9..6414983ce 100644 --- a/Sources/Fluid/UI/AnalyticsPrivacyView.swift +++ b/Sources/Fluid/UI/AnalyticsPrivacyView.swift @@ -36,16 +36,17 @@ struct AnalyticsPrivacyView: View { self.bullet("Onboarding steps viewed and completed, including skips.") self.bullet("Transcription and AI provider/model identifiers, counted as daily totals.") self.bullet("Model download source, outcome, and download duration.") + self.bullet("Daily aggregated insertion timing statistics—including toggle-stop-to-paste-dispatch—split by delivery path and outcome.") self.sectionTitle("We do NOT collect") self.bullet("Any transcription text or audio.") self.bullet("Selected text, rewrite prompts, or AI responses.") self.bullet("Terminal commands or outputs from Command Mode.") self.bullet("Window titles, app names, file names/paths, clipboard contents, or anything you type.") - self.bullet("Hardware identifiers, CPU/chip details, performance timings, or individual transcription events.") + self.bullet("Hardware identifiers, CPU/chip details, target-app identity, or individual insertion timings/events.") self.sectionTitle("How it’s used") - self.bullet("To understand feature adoption, onboarding completion, model usage, active installations, and retention without requiring accounts.") + self.bullet("To understand feature adoption, insertion performance, onboarding completion, model usage, active installations, and retention without requiring accounts.") self.sectionTitle("Control") self.bullet("You can disable analytics anytime in Settings → Share Anonymous Analytics.") diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift index 80e0d2330..455351536 100644 --- a/Sources/Fluid/UI/SettingsView.swift +++ b/Sources/Fluid/UI/SettingsView.swift @@ -943,7 +943,7 @@ struct SettingsView: View { self.optionToggleRow( title: "Share Anonymous Analytics", - description: "Send lean, anonymous daily usage, onboarding, retention, and model metrics. Never includes transcription text or prompts.", + description: "Send lean, anonymous daily usage, insertion performance, onboarding, retention, and model metrics. Never includes transcription text or prompts.", isOn: self.analyticsToggleBinding ) diff --git a/Sources/Fluid/Views/BottomOverlayView.swift b/Sources/Fluid/Views/BottomOverlayView.swift index 607b9f0e2..eb4944263 100644 --- a/Sources/Fluid/Views/BottomOverlayView.swift +++ b/Sources/Fluid/Views/BottomOverlayView.swift @@ -1661,9 +1661,10 @@ private struct BottomOverlayPromptMenuView: View { } private func restoreTypingTargetApp() { - let pid = NotchContentState.shared.recordingTargetPID - DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { - if let pid { _ = TypingService.activateApp(pid: pid) } + guard let context = NotchContentState.shared.recordingTargetContext else { return } + Task { @MainActor in + try? await Task.sleep(nanoseconds: 50_000_000) + _ = await TypingService.prepareTargetForDelivery(context) } } } @@ -2287,7 +2288,11 @@ struct BottomOverlayView: View { private var shouldReservePreviewArea: Bool { self.layout.showsPreview && - (self.settings.enableStreamingPreview || self.contentState.isAIProcessingFailureVisible) + ( + self.settings.enableStreamingPreview || + self.contentState.isAIProcessingFailureVisible || + self.contentState.isTextDeliveryFailureVisible + ) } private var overlayFrameHeight: CGFloat? { @@ -2341,6 +2346,10 @@ struct BottomOverlayView: View { self.shouldReservePreviewArea && self.contentState.isAIProcessingFailureVisible && !self.contentState.isProcessing } + private var shouldShowTextDeliveryFailure: Bool { + self.shouldReservePreviewArea && self.contentState.isTextDeliveryFailureVisible && !self.contentState.isProcessing + } + private var shouldSuppressPreviewDuringRelease: Bool { if self.shouldShowProcessingPreview { return false @@ -2350,7 +2359,7 @@ struct BottomOverlayView: View { private func previewResizeBucket(for previewText: String) -> Int { guard self.shouldReservePreviewArea else { return 0 } - if self.shouldShowAIProcessingFailure { return 1 } + if self.shouldShowAIProcessingFailure || self.shouldShowTextDeliveryFailure { return 1 } let trimmed = previewText.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return self.shouldShowProcessingStatus ? 1 : 0 } @@ -2816,6 +2825,32 @@ struct BottomOverlayView: View { .frame(maxWidth: self.previewMaxWidth, alignment: .leading) } + private var textDeliveryFailureView: some View { + HStack(spacing: 8) { + Text(self.contentState.textDeliveryFailureMessage) + .font(.system(size: self.layout.transFontSize, weight: .semibold)) + .foregroundStyle(Color.orange.opacity(0.9)) + .lineLimit(1) + .truncationMode(.tail) + + Spacer(minLength: 4) + + self.failureIconButton(systemName: "doc.on.doc", help: "Copy transcript") { + _ = ClipboardService.copyToClipboard(self.contentState.textDeliveryFailureTranscript) + } + self.failureIconButton(systemName: "arrow.down.doc", help: "Paste last transcript") { + let transcript = self.contentState.textDeliveryFailureTranscript + self.contentState.clearTextDeliveryFailure() + self.contentState.onRetryTextDeliveryRequested?(transcript) + } + self.failureIconButton(systemName: "xmark", help: "Dismiss") { + self.contentState.clearTextDeliveryFailure() + NotchOverlayManager.shared.hide() + } + } + .frame(maxWidth: self.previewMaxWidth, alignment: .leading) + } + private func scrollablePreviewText(_ previewText: String) -> some View { ScrollViewReader { proxy in ScrollView(.vertical, showsIndicators: false) { @@ -2882,6 +2917,8 @@ struct BottomOverlayView: View { Group { if self.shouldSuppressPreviewDuringRelease { Color.clear + } else if self.shouldShowTextDeliveryFailure { + self.textDeliveryFailureView } else if self.shouldShowAIProcessingFailure { self.aiProcessingFailureView } else if self.shouldShowProcessingPreview { @@ -2942,6 +2979,8 @@ struct BottomOverlayView: View { Group { if self.shouldSuppressPreviewDuringRelease { Color.clear + } else if self.shouldShowTextDeliveryFailure { + self.textDeliveryFailureView } else if self.shouldShowAIProcessingFailure { self.aiProcessingFailureView } else if self.shouldShowProcessingPreview { @@ -3239,6 +3278,10 @@ struct BottomOverlayView: View { guard !self.layout.usesFixedCanvas else { return } self.refreshDynamicPreviewSizeIfNeeded(for: self.currentPreviewSizingText) } + .onChange(of: self.contentState.isTextDeliveryFailureVisible) { _, _ in + guard !self.layout.usesFixedCanvas else { return } + self.refreshDynamicPreviewSizeIfNeeded(for: self.currentPreviewSizingText) + } .onChange(of: self.processingStatusVisible) { _, _ in guard !self.layout.usesFixedCanvas else { return } self.refreshDynamicPreviewSizeIfNeeded(for: self.currentPreviewSizingText) diff --git a/Sources/Fluid/Views/NotchContentViews.swift b/Sources/Fluid/Views/NotchContentViews.swift index 101846872..a2c28101f 100644 --- a/Sources/Fluid/Views/NotchContentViews.swift +++ b/Sources/Fluid/Views/NotchContentViews.swift @@ -106,6 +106,9 @@ class NotchContentState: ObservableObject { @Published private(set) var canRetryAIProcessingFailure: Bool = true @Published private(set) var spokenSendIndicatorState: SpokenSendIndicatorState = .hidden @Published private(set) var spokenSendCountdownID: UInt64 = 0 + @Published var isTextDeliveryFailureVisible: Bool = false + @Published private(set) var textDeliveryFailureMessage: String = "Text could not be inserted" + private(set) var textDeliveryFailureTranscript: String = "" @Published var activeDictationShortcutSlot: SettingsStore.DictationShortcutSlot? = nil @Published var promptModeOverrideProfileName: String? = nil // Name shown in overlay when prompt mode hotkey is active @Published var promptModeOverrideProfileID: String? = nil // ID of the active override profile (for checkmark in menu) @@ -120,6 +123,7 @@ class NotchContentState: ObservableObject { /// The PID of the app we should restore focus to after interacting with overlays. /// Captured at recording start to keep the target stable for the session. @Published var recordingTargetPID: pid_t? = nil + var recordingTargetContext: TypingService.RecordingTargetContext? /// Cached transcription preview text to avoid recomputing on every render @Published private(set) var cachedPreviewText: String = "" @@ -175,6 +179,7 @@ class NotchContentState: ObservableObject { func setProcessing(_ processing: Bool) { if processing { self.clearAIProcessingFailure() + self.clearTextDeliveryFailure() } self.isProcessing = processing } @@ -204,6 +209,16 @@ class NotchContentState: ObservableObject { return self.spokenSendCountdownID } + func showTextDeliveryFailure(message: String, transcript: String) { + self.textDeliveryFailureMessage = message + self.textDeliveryFailureTranscript = transcript + self.isTextDeliveryFailureVisible = true + } + + func clearTextDeliveryFailure() { + self.isTextDeliveryFailureVisible = false + } + /// Update transcription and recompute cached lines func updateTranscription(_ text: String) { let boundedText = Self.tailCharacters(in: text, maxCharacters: Self.maxStoredTranscriptionCharacters) @@ -260,6 +275,8 @@ class NotchContentState: ObservableObject { var onCopyLastRequested: (() -> Void)? /// Called when the user requests re-pasting the latest saved transcription entry. var onPasteLastRequested: (() -> Void)? + /// Called when the user retries the exact transcript from a delivery failure. + var onRetryTextDeliveryRequested: ((String) -> Void)? /// Called when the user requests undoing AI processing for the latest entry. var onUndoLastAIRequested: (() -> Void)? /// Called when the user requests opening Preferences. @@ -722,9 +739,10 @@ struct NotchExpandedView: View { } private func restoreRecordingTargetFocus() { - let pid = NotchContentState.shared.recordingTargetPID - DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { - if let pid { _ = TypingService.activateApp(pid: pid) } + guard let context = NotchContentState.shared.recordingTargetContext else { return } + Task { @MainActor in + try? await Task.sleep(nanoseconds: 50_000_000) + _ = await TypingService.prepareTargetForDelivery(context) } } @@ -756,7 +774,6 @@ struct NotchExpandedView: View { ) } - @ViewBuilder private func promptMenuRow( _ title: String, rowID: String, @@ -1020,7 +1037,53 @@ struct NotchExpandedView: View { self.promptHoverMenuRow - if self.contentState.isAIProcessingFailureVisible && !self.contentState.isProcessing { + if self.contentState.isTextDeliveryFailureVisible && !self.contentState.isProcessing { + HStack(spacing: 6) { + Text(self.contentState.textDeliveryFailureMessage) + .font(.system(size: 10, weight: .semibold)) + .foregroundStyle(Color.orange.opacity(0.9)) + .lineLimit(1) + .truncationMode(.tail) + + Spacer(minLength: 2) + + Button { + _ = ClipboardService.copyToClipboard(self.contentState.textDeliveryFailureTranscript) + } label: { + Image(systemName: "doc.on.doc") + .font(.system(size: 9, weight: .bold)) + .frame(width: 16, height: 16) + } + .buttonStyle(.plain) + .help("Copy transcript") + + Button { + let transcript = self.contentState.textDeliveryFailureTranscript + self.contentState.clearTextDeliveryFailure() + self.contentState.onRetryTextDeliveryRequested?(transcript) + } label: { + Image(systemName: "arrow.down.doc") + .font(.system(size: 9, weight: .bold)) + .frame(width: 16, height: 16) + } + .buttonStyle(.plain) + .help("Paste last transcript") + + Button { + self.contentState.clearTextDeliveryFailure() + NotchOverlayManager.shared.hide() + } label: { + Image(systemName: "xmark") + .font(.system(size: 9, weight: .bold)) + .frame(width: 16, height: 16) + } + .buttonStyle(.plain) + .help("Dismiss") + } + .foregroundStyle(.white.opacity(0.9)) + .frame(width: self.previewMaxWidth, alignment: .leading) + .transition(.opacity.combined(with: .scale(scale: 0.95))) + } else if self.contentState.isAIProcessingFailureVisible && !self.contentState.isProcessing { HStack(spacing: 6) { Text(self.contentState.aiProcessingFailureMessage) .font(.system(size: 10, weight: .semibold)) @@ -1187,7 +1250,6 @@ struct NotchWaveformView: View { } } - @ViewBuilder private func barsView(using height: @escaping (Int) -> CGFloat) -> some View { HStack(spacing: self.barSpacing) { ForEach(0.. CGFloat) -> some View { HStack(spacing: self.barSpacing) { ForEach(0.. Bool) async { + for _ in 0..<100 where !condition() { + await Task.yield() + } + XCTAssertTrue(condition()) + } +} + +final class TextInsertionModeMigrationTests: XCTestCase { + private let modeKey = "TextInsertionMode" + private let migrationKey = "TextInsertionModeMigratedToReliablePasteV1" + + func testMigrationOverridesExistingDirectModeOnce() throws { + let defaults = try self.makeDefaults() + defer { defaults.removePersistentDomain(forName: self.name) } + defaults.set(SettingsStore.TextInsertionMode.standard.rawValue, forKey: self.modeKey) + + SettingsStore.migrateTextInsertionModeToReliablePasteIfNeeded(defaults: defaults) + + XCTAssertEqual(defaults.string(forKey: self.modeKey), SettingsStore.TextInsertionMode.reliablePaste.rawValue) + XCTAssertTrue(defaults.bool(forKey: self.migrationKey)) + + defaults.set(SettingsStore.TextInsertionMode.standard.rawValue, forKey: self.modeKey) + SettingsStore.migrateTextInsertionModeToReliablePasteIfNeeded(defaults: defaults) + + XCTAssertEqual(defaults.string(forKey: self.modeKey), SettingsStore.TextInsertionMode.standard.rawValue) + } + + func testMigrationDefaultsUnsetModeToReliablePaste() throws { + let defaults = try self.makeDefaults() + defer { defaults.removePersistentDomain(forName: self.name) } + + SettingsStore.migrateTextInsertionModeToReliablePasteIfNeeded(defaults: defaults) + + XCTAssertEqual(defaults.string(forKey: self.modeKey), SettingsStore.TextInsertionMode.reliablePaste.rawValue) + XCTAssertTrue(defaults.bool(forKey: self.migrationKey)) + } + + func testRecommendedModeAppearsFirst() { + XCTAssertEqual(SettingsStore.TextInsertionMode.allCases.first, .reliablePaste) + XCTAssertEqual(SettingsStore.TextInsertionMode.reliablePaste.displayName, "Clipboard Paste (Recommended)") + XCTAssertEqual(SettingsStore.TextInsertionMode.standard.displayName, "Direct Paste") + } + + private func makeDefaults() throws -> UserDefaults { + let defaults = try XCTUnwrap(UserDefaults(suiteName: self.name)) + defaults.removePersistentDomain(forName: self.name) + return defaults + } +} + +@MainActor +private final class FakePasteboardManager: PasteboardManaging { + private(set) var text: String + private(set) var restoreCount = 0 + private(set) var intentionalWriteCount = 0 + private(set) var temporaryWriteCount = 0 + private(set) var isTemporary = false + private let snapshotSucceeds: Bool + private let temporaryWriteSucceeds: Bool + private var sessionID: String? + + init( + text: String, + snapshotSucceeds: Bool = true, + temporaryWriteSucceeds: Bool = true + ) { + self.text = text + self.snapshotSucceeds = snapshotSucceeds + self.temporaryWriteSucceeds = temporaryWriteSucceeds + } + + func captureSnapshot() -> PasteboardSnapshot? { + guard self.snapshotSucceeds else { return nil } + return PasteboardSnapshot(items: [ + .init(representations: [ + .init(type: .string, data: Data(self.text.utf8)), + ]), + ]) + } + + func writeTemporaryText(_ text: String, sessionID: String) -> Bool { + self.temporaryWriteCount += 1 + self.text = text + self.sessionID = sessionID + self.isTemporary = true + return self.temporaryWriteSucceeds + } + + func writeIntentionalText(_ text: String) -> Bool { + self.intentionalWriteCount += 1 + self.text = text + self.sessionID = nil + self.isTemporary = false + return true + } + + func isOwned(sessionID: String, expectedText: String) -> Bool { + self.sessionID == sessionID && self.text == expectedText + } + + func restore(_ snapshot: PasteboardSnapshot) -> Bool { + self.restoreCount += 1 + self.sessionID = nil + self.isTemporary = false + guard let data = snapshot.items.first?.representations.first(where: { $0.type == .string })?.data, + let restoredText = String(data: data, encoding: .utf8) + else { + self.text = "" + return snapshot.items.isEmpty + } + self.text = restoredText + return true + } + + func simulateExternalCopy(_ text: String) { + self.text = text + self.sessionID = nil + self.isTemporary = false + } +} + +@MainActor +private final class FakePasteCommandPoster: PasteCommandPosting { + private let succeeds: Bool + private(set) var postCount = 0 + + init(succeeds: Bool = true) { + self.succeeds = succeeds + } + + func postGlobalPasteCommand() async -> Bool { + self.postCount += 1 + return self.succeeds + } +} + +@MainActor +private final class SuspendingFakePasteCommandPoster: PasteCommandPosting { + private(set) var postCount = 0 + private(set) var firstPostIsSuspended = false + private var firstPostContinuation: CheckedContinuation? + + func postGlobalPasteCommand() async -> Bool { + self.postCount += 1 + guard self.postCount == 1 else { return true } + + self.firstPostIsSuspended = true + await withCheckedContinuation { continuation in + self.firstPostContinuation = continuation + } + return true + } + + func resumeFirstPost() { + let continuation = self.firstPostContinuation + self.firstPostContinuation = nil + continuation?.resume() + } +} diff --git a/Tests/FluidDictationIntegrationTests/TypingServiceTransientPasteboardTests.swift b/Tests/FluidDictationIntegrationTests/TypingServiceTransientPasteboardTests.swift index aeedc61ef..36b479fa5 100644 --- a/Tests/FluidDictationIntegrationTests/TypingServiceTransientPasteboardTests.swift +++ b/Tests/FluidDictationIntegrationTests/TypingServiceTransientPasteboardTests.swift @@ -2,21 +2,30 @@ import AppKit @testable import FluidVoice_Debug import XCTest -// The temporary clipboard write used to drive synthetic Cmd+V must be tagged so clipboard -// managers exclude it from history. It is restored immediately and is not user-copied content. +/// Verifies that temporary Cmd+V payloads are tagged so clipboard managers exclude them +/// from history while the plain-text representation remains available to the paste target. +@MainActor final class TypingServiceTransientPasteboardTests: XCTestCase { + func testTemporaryPasteboardWriteCarriesText() { + let pasteboard = self.makePasteboard() + defer { pasteboard.releaseGlobally() } - func testTransientPasteboardItem_carriesText() { - let item = TypingService.makeTransientPasteboardItem("hello world") - XCTAssertEqual( - item.string(forType: .string), - "hello world", - "the plain string must survive so paste and clipboard restore behave unchanged" + XCTAssertTrue( + SystemPasteboardManager(pasteboard: pasteboard) + .writeTemporaryText("hello world", sessionID: "session") ) + XCTAssertEqual(pasteboard.string(forType: .string), "hello world") } - func testTransientPasteboardItem_isMarkedForClipboardManagers() { - let item = TypingService.makeTransientPasteboardItem("hello world") + func testTemporaryPasteboardWriteIsMarkedForClipboardManagers() throws { + let pasteboard = self.makePasteboard() + defer { pasteboard.releaseGlobally() } + + XCTAssertTrue( + SystemPasteboardManager(pasteboard: pasteboard) + .writeTemporaryText("hello world", sessionID: "session") + ) + let item = try XCTUnwrap(pasteboard.pasteboardItems?.first) let transientType = NSPasteboard.PasteboardType("org.nspasteboard.TransientType") let autoGeneratedType = NSPasteboard.PasteboardType("org.nspasteboard.AutoGeneratedType") let types = item.types.map(\.rawValue) @@ -38,25 +47,32 @@ final class TypingServiceTransientPasteboardTests: XCTestCase { ) } - func testTransientMarkersSurvivePasteboardWrite() throws { - let name = NSPasteboard.Name("com.fluidvoice.tests.transient.\(UUID().uuidString)") - let pasteboard = NSPasteboard(name: name) + func testTemporaryPasteboardWriteCarriesSourceAndSession() throws { + let pasteboard = self.makePasteboard() defer { pasteboard.releaseGlobally() } - pasteboard.clearContents() XCTAssertTrue( - pasteboard.writeObjects([TypingService.makeTransientPasteboardItem("hello world")]) + SystemPasteboardManager(pasteboard: pasteboard) + .writeTemporaryText("hello world", sessionID: "session") ) - let item = try XCTUnwrap(pasteboard.pasteboardItems?.first) - let types = item.types.map(\.rawValue) - XCTAssertEqual(item.string(forType: .string), "hello world") - XCTAssertTrue(types.contains("org.nspasteboard.TransientType")) - XCTAssertTrue(types.contains("org.nspasteboard.AutoGeneratedType")) + let sourceType = NSPasteboard.PasteboardType("org.nspasteboard.source") + let sessionType = NSPasteboard.PasteboardType( + "\(Bundle.main.bundleIdentifier ?? "com.FluidApp.Fluid").PasteSession" + ) + XCTAssertEqual(item.string(forType: sourceType), Bundle.main.bundleIdentifier ?? "com.FluidApp.Fluid") + XCTAssertEqual(item.string(forType: sessionType), "session") } - func testTransientPasteboardItem_isNotMarkedConcealed() { - let item = TypingService.makeTransientPasteboardItem("hello world") + func testTemporaryPasteboardWriteIsNotMarkedConcealed() throws { + let pasteboard = self.makePasteboard() + defer { pasteboard.releaseGlobally() } + + XCTAssertTrue( + SystemPasteboardManager(pasteboard: pasteboard) + .writeTemporaryText("hello world", sessionID: "session") + ) + let item = try XCTUnwrap(pasteboard.pasteboardItems?.first) let types = item.types.map(\.rawValue) XCTAssertFalse( types.contains("org.nspasteboard.ConcealedType"), @@ -64,19 +80,346 @@ final class TypingServiceTransientPasteboardTests: XCTestCase { ) } - func testFocusRestoreDoesNotRaiseAllWindowsOfTargetApp() { - // Issue #748: restoring focus after dictation must NOT raise every window of the - // target app. `.activateAllWindows` brings forward all windows of the process and - // destroys a multi-window layout (e.g. WebStorm) on every dictation. - let options = TypingService.focusRestoreActivationOptions + func testRecoveryActivationDoesNotRaiseAllWindowsOfTargetApp() { + // Issue #748: recovery must NOT raise every window of the target app. + let options = TypingService.recoveryActivationOptions XCTAssertTrue( options.contains(.activateIgnoringOtherApps), - "focus restore must still activate the target app and bring it forward" + "recovery must still activate the target app and bring it forward" ) XCTAssertFalse( options.contains(.activateAllWindows), - "Restoring focus must not raise every window of the target app (issue #748)" + "Recovery must not raise every window of the target app (issue #748)" + ) + } + + func testIntentionalClipboardWriteIsHistoryVisible() throws { + let pasteboard = self.makePasteboard() + defer { pasteboard.releaseGlobally() } + + XCTAssertTrue(SystemPasteboardManager(pasteboard: pasteboard).writeIntentionalText("keep me")) + + let item = try XCTUnwrap(pasteboard.pasteboardItems?.first) + let types = item.types.map(\.rawValue) + XCTAssertEqual(item.string(forType: .string), "keep me") + XCTAssertFalse(types.contains("org.nspasteboard.TransientType")) + XCTAssertFalse(types.contains("org.nspasteboard.AutoGeneratedType")) + XCTAssertFalse(types.contains("org.nspasteboard.ConcealedType")) + XCTAssertFalse(types.contains { $0.hasSuffix(".PasteSession") }) + } + + func testSnapshotRestoresEveryRepresentation() throws { + let pasteboard = self.makePasteboard() + defer { pasteboard.releaseGlobally() } + let customType = NSPasteboard.PasteboardType("com.fluidvoice.tests.custom") + let customData = Data([0x01, 0x02, 0x03]) + let originalItem = NSPasteboardItem() + XCTAssertTrue(originalItem.setString("original", forType: .string)) + XCTAssertTrue(originalItem.setData(customData, forType: customType)) + let secondItem = NSPasteboardItem() + XCTAssertTrue(secondItem.setString("second", forType: .string)) + XCTAssertTrue(pasteboard.writeObjects([originalItem, secondItem])) + let originalItems = try XCTUnwrap(pasteboard.pasteboardItems) + let originalTypes = originalItems.map(\.types) + let originalRepresentations = originalItems.map { item in + item.types.map { type in item.data(forType: type) } + } + + let manager = SystemPasteboardManager(pasteboard: pasteboard) + let snapshot = try XCTUnwrap(manager.captureSnapshot()) + XCTAssertTrue(manager.writeTemporaryText("temporary", sessionID: "session")) + XCTAssertTrue(manager.restore(snapshot)) + + let restoredItems = try XCTUnwrap(pasteboard.pasteboardItems) + XCTAssertEqual(restoredItems.count, 2) + for (restoredItem, sourceTypes) in zip(restoredItems, originalTypes) { + XCTAssertEqual(Array(restoredItem.types.prefix(sourceTypes.count)), sourceTypes) + XCTAssertTrue(restoredItem.types.contains(Self.transientType)) + XCTAssertTrue(restoredItem.types.contains(Self.autoGeneratedType)) + } + XCTAssertEqual( + zip(restoredItems, originalTypes).map { item, types in + types.map { type in item.data(forType: type) } + }, + originalRepresentations + ) + let restoredItem = restoredItems[0] + XCTAssertEqual(restoredItem.string(forType: .string), "original") + XCTAssertEqual(restoredItem.data(forType: customType), customData) + XCTAssertEqual(restoredItems[1].string(forType: .string), "second") + } + + func testSnapshotRestoresImageBeforeTextInOriginalPreferenceOrder() throws { + let pasteboard = self.makePasteboard() + defer { pasteboard.releaseGlobally() } + + let bitmap = try XCTUnwrap( + NSBitmapImageRep( + bitmapDataPlanes: nil, + pixelsWide: 2, + pixelsHigh: 2, + bitsPerSample: 8, + samplesPerPixel: 4, + hasAlpha: true, + isPlanar: false, + colorSpaceName: .deviceRGB, + bytesPerRow: 0, + bitsPerPixel: 0 + ) + ) + bitmap.setColor(NSColor(deviceRed: 0, green: 0, blue: 1, alpha: 1), atX: 0, y: 0) + bitmap.setColor(NSColor(deviceRed: 1, green: 0, blue: 0, alpha: 1), atX: 1, y: 0) + bitmap.setColor(NSColor(deviceRed: 0, green: 1, blue: 0, alpha: 1), atX: 0, y: 1) + bitmap.setColor(NSColor(deviceRed: 1, green: 1, blue: 1, alpha: 1), atX: 1, y: 1) + let imageData = try XCTUnwrap(bitmap.representation(using: .tiff, properties: [:])) + let originalItem = NSPasteboardItem() + XCTAssertTrue(originalItem.setData(imageData, forType: .tiff)) + XCTAssertTrue(originalItem.setString("image text fallback", forType: .string)) + XCTAssertTrue(pasteboard.writeObjects([originalItem])) + let originalTypes = try XCTUnwrap(pasteboard.pasteboardItems?.first?.types) + XCTAssertEqual(originalTypes.first, .tiff) + + let manager = SystemPasteboardManager(pasteboard: pasteboard) + let snapshot = try XCTUnwrap(manager.captureSnapshot()) + XCTAssertEqual(snapshot.items.first?.representations.map(\.type), originalTypes) + XCTAssertTrue(manager.writeTemporaryText("temporary", sessionID: "session")) + XCTAssertTrue(manager.restore(snapshot)) + + let restoredItem = try XCTUnwrap(pasteboard.pasteboardItems?.first) + XCTAssertEqual(Array(restoredItem.types.prefix(originalTypes.count)), originalTypes) + XCTAssertEqual(restoredItem.data(forType: .tiff), imageData) + XCTAssertEqual(restoredItem.string(forType: .string), "image text fallback") + let restoredImageData = try XCTUnwrap(restoredItem.data(forType: .tiff)) + XCTAssertNotNil(NSImage(data: restoredImageData)) + } + + func testSnapshotAddsPortableImageBeforeFilenameForICNSOnlyClipboard() throws { + let pasteboard = self.makePasteboard() + defer { pasteboard.releaseGlobally() } + + let bitmap = try XCTUnwrap( + NSBitmapImageRep( + bitmapDataPlanes: nil, + pixelsWide: 2, + pixelsHigh: 2, + bitsPerSample: 8, + samplesPerPixel: 4, + hasAlpha: true, + isPlanar: false, + colorSpaceName: .deviceRGB, + bytesPerRow: 0, + bitsPerPixel: 0 + ) + ) + bitmap.setColor(NSColor(deviceRed: 0, green: 0, blue: 1, alpha: 1), atX: 0, y: 0) + let sourceImageData = try XCTUnwrap(bitmap.representation(using: .png, properties: [:])) + let icnsType = NSPasteboard.PasteboardType("com.apple.icns") + let originalItem = NSPasteboardItem() + XCTAssertTrue(originalItem.setString("Example.icns", forType: .string)) + XCTAssertTrue(originalItem.setData(sourceImageData, forType: icnsType)) + XCTAssertTrue(pasteboard.writeObjects([originalItem])) + + let manager = SystemPasteboardManager(pasteboard: pasteboard) + let snapshot = try XCTUnwrap(manager.captureSnapshot()) + XCTAssertNotNil(snapshot.items.first?.portableImage) + XCTAssertTrue(manager.writeTemporaryText("temporary", sessionID: "session")) + XCTAssertTrue(manager.restore(snapshot)) + + let externalReader = NSPasteboard(name: pasteboard.name) + let restoredItem = try XCTUnwrap(externalReader.pasteboardItems?.first) + XCTAssertEqual(restoredItem.types.first, .png) + XCTAssertEqual(restoredItem.string(forType: .string), "Example.icns") + XCTAssertEqual(restoredItem.data(forType: icnsType), sourceImageData) + XCTAssertNotNil(restoredItem.data(forType: .png).flatMap(NSImage.init(data:))) + XCTAssertTrue(restoredItem.types.contains(Self.transientType)) + XCTAssertTrue(restoredItem.types.contains(Self.autoGeneratedType)) + } + + func testSnapshotAddsPortableImageForImageFileURL() throws { + let pasteboard = self.makePasteboard() + defer { pasteboard.releaseGlobally() } + + let temporaryDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("fluidvoice-image-pasteboard-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: temporaryDirectory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: temporaryDirectory) } + + let bitmap = try XCTUnwrap( + NSBitmapImageRep( + bitmapDataPlanes: nil, + pixelsWide: 1, + pixelsHigh: 1, + bitsPerSample: 8, + samplesPerPixel: 4, + hasAlpha: true, + isPlanar: false, + colorSpaceName: .deviceRGB, + bytesPerRow: 0, + bitsPerPixel: 0 + ) + ) + bitmap.setColor(NSColor(deviceRed: 1, green: 0, blue: 0, alpha: 1), atX: 0, y: 0) + let sourceImageData = try XCTUnwrap(bitmap.representation(using: .png, properties: [:])) + let imageURL = temporaryDirectory.appendingPathComponent("Example.png") + try sourceImageData.write(to: imageURL) + + bitmap.setColor(NSColor(deviceRed: 0, green: 0, blue: 1, alpha: 1), atX: 0, y: 0) + let iconData = try XCTUnwrap(bitmap.representation(using: .png, properties: [:])) + let icnsType = NSPasteboard.PasteboardType("com.apple.icns") + let originalItem = NSPasteboardItem() + XCTAssertTrue(originalItem.setString(imageURL.absoluteString, forType: .fileURL)) + XCTAssertTrue(originalItem.setString("Example.png", forType: .string)) + XCTAssertTrue(originalItem.setData(iconData, forType: icnsType)) + XCTAssertTrue(pasteboard.writeObjects([originalItem])) + + let manager = SystemPasteboardManager(pasteboard: pasteboard) + let snapshot = try XCTUnwrap(manager.captureSnapshot()) + XCTAssertNotNil(snapshot.items.first?.portableImage) + XCTAssertTrue(manager.writeTemporaryText("temporary", sessionID: "session")) + XCTAssertTrue(manager.restore(snapshot)) + + let restoredItem = try XCTUnwrap(pasteboard.pasteboardItems?.first) + XCTAssertEqual(Array(restoredItem.types.prefix(2)), [.fileURL, .png]) + XCTAssertEqual(restoredItem.string(forType: .fileURL), imageURL.absoluteString) + XCTAssertEqual(restoredItem.data(forType: icnsType), iconData) + XCTAssertNotEqual(restoredItem.data(forType: .png), iconData) + XCTAssertEqual(restoredItem.data(forType: .png), sourceImageData) + + let restoredPNG = try XCTUnwrap(restoredItem.data(forType: .png)) + let restoredBitmap = try XCTUnwrap(NSBitmapImageRep(data: restoredPNG)) + let restoredColor = try XCTUnwrap(restoredBitmap.colorAt(x: 0, y: 0)?.usingColorSpace(.deviceRGB)) + XCTAssertGreaterThan(restoredColor.redComponent, 0.8) + XCTAssertGreaterThan(restoredColor.redComponent, restoredColor.greenComponent + 0.5) + XCTAssertGreaterThan(restoredColor.redComponent, restoredColor.blueComponent + 0.5) + } + + func testSnapshotUsesEmbeddedImageWhenImageFileURLCannotBeRead() throws { + let pasteboard = self.makePasteboard() + defer { pasteboard.releaseGlobally() } + + let bitmap = try XCTUnwrap( + NSBitmapImageRep( + bitmapDataPlanes: nil, + pixelsWide: 1, + pixelsHigh: 1, + bitsPerSample: 8, + samplesPerPixel: 4, + hasAlpha: true, + isPlanar: false, + colorSpaceName: .deviceRGB, + bytesPerRow: 0, + bitsPerPixel: 0 + ) + ) + let iconData = try XCTUnwrap(bitmap.representation(using: .png, properties: [:])) + let icnsType = NSPasteboard.PasteboardType("com.apple.icns") + let originalItem = NSPasteboardItem() + XCTAssertTrue(originalItem.setString("file:///.file/id=123456.789", forType: .fileURL)) + XCTAssertTrue(originalItem.setString("Example.png", forType: .string)) + XCTAssertTrue(originalItem.setData(iconData, forType: icnsType)) + XCTAssertTrue(pasteboard.writeObjects([originalItem])) + + let manager = SystemPasteboardManager(pasteboard: pasteboard) + let snapshot = try XCTUnwrap(manager.captureSnapshot()) + XCTAssertNotNil(snapshot.items.first?.portableImage) + XCTAssertTrue(manager.writeTemporaryText("temporary", sessionID: "session")) + XCTAssertTrue(manager.restore(snapshot)) + + let externalReader = NSPasteboard(name: pasteboard.name) + let restoredItem = try XCTUnwrap(externalReader.pasteboardItems?.first) + XCTAssertTrue(restoredItem.types.contains(.png)) + XCTAssertLessThan( + try XCTUnwrap(restoredItem.types.firstIndex(of: .png)), + try XCTUnwrap(restoredItem.types.firstIndex(of: .string)) + ) + XCTAssertNotNil(restoredItem.data(forType: .png).flatMap(NSImage.init(data:))) + } + + func testSnapshotDoesNotTurnNonImageFileIconIntoImage() throws { + let pasteboard = self.makePasteboard() + defer { pasteboard.releaseGlobally() } + + let bitmap = try XCTUnwrap( + NSBitmapImageRep( + bitmapDataPlanes: nil, + pixelsWide: 1, + pixelsHigh: 1, + bitsPerSample: 8, + samplesPerPixel: 4, + hasAlpha: true, + isPlanar: false, + colorSpaceName: .deviceRGB, + bytesPerRow: 0, + bitsPerPixel: 0 + ) ) + let iconData = try XCTUnwrap(bitmap.representation(using: .png, properties: [:])) + let icnsType = NSPasteboard.PasteboardType("com.apple.icns") + let originalItem = NSPasteboardItem() + XCTAssertTrue(originalItem.setString("file:///tmp/Example.txt", forType: .fileURL)) + XCTAssertTrue(originalItem.setString("Example.txt", forType: .string)) + XCTAssertTrue(originalItem.setData(iconData, forType: icnsType)) + XCTAssertTrue(pasteboard.writeObjects([originalItem])) + + let manager = SystemPasteboardManager(pasteboard: pasteboard) + let snapshot = try XCTUnwrap(manager.captureSnapshot()) + XCTAssertTrue(manager.writeTemporaryText("temporary", sessionID: "session")) + XCTAssertTrue(manager.restore(snapshot)) + + let restoredItem = try XCTUnwrap(pasteboard.pasteboardItems?.first) + XCTAssertFalse(restoredItem.types.contains(.png)) + XCTAssertEqual(restoredItem.string(forType: .fileURL), "file:///tmp/Example.txt") + XCTAssertEqual(restoredItem.data(forType: icnsType), iconData) + } + + func testLargeRepresentationIsSnapshottedAndRestoredWithoutSizeLimit() throws { + let pasteboard = self.makePasteboard() + defer { pasteboard.releaseGlobally() } + let customType = NSPasteboard.PasteboardType("com.fluidvoice.tests.large") + let originalData = Data(repeating: 0xab, count: 20 * 1024 * 1024) + let originalItem = NSPasteboardItem() + XCTAssertTrue(originalItem.setData(originalData, forType: customType)) + XCTAssertTrue(pasteboard.writeObjects([originalItem])) + + let manager = SystemPasteboardManager(pasteboard: pasteboard) + let snapshot = try XCTUnwrap(manager.captureSnapshot()) + XCTAssertEqual(snapshot.items.first?.representations.first?.data, originalData) + XCTAssertTrue(manager.writeTemporaryText("temporary", sessionID: "session")) + XCTAssertTrue(manager.restore(snapshot)) + + XCTAssertEqual(pasteboard.data(forType: customType), originalData) } + + func testMoreThanOneHundredItemsAreSnapshottedAndRestored() throws { + let pasteboard = self.makePasteboard() + defer { pasteboard.releaseGlobally() } + let originalItems = (0..<101).map { index in + let item = NSPasteboardItem() + XCTAssertTrue(item.setString("item \(index)", forType: .string)) + return item + } + XCTAssertTrue(pasteboard.writeObjects(originalItems)) + + let manager = SystemPasteboardManager(pasteboard: pasteboard) + let snapshot = try XCTUnwrap(manager.captureSnapshot()) + XCTAssertEqual(snapshot.items.count, 101) + XCTAssertTrue(manager.writeTemporaryText("temporary", sessionID: "session")) + XCTAssertTrue(manager.restore(snapshot)) + + let restoredItems = try XCTUnwrap(pasteboard.pasteboardItems) + XCTAssertEqual(restoredItems.count, 101) + XCTAssertEqual(restoredItems.first?.string(forType: .string), "item 0") + XCTAssertEqual(restoredItems.last?.string(forType: .string), "item 100") + } + + private func makePasteboard() -> NSPasteboard { + let name = NSPasteboard.Name("com.fluidvoice.tests.transient.\(UUID().uuidString)") + let pasteboard = NSPasteboard(name: name) + pasteboard.clearContents() + return pasteboard + } + + private static let transientType = NSPasteboard.PasteboardType("org.nspasteboard.TransientType") + private static let autoGeneratedType = NSPasteboard.PasteboardType("org.nspasteboard.AutoGeneratedType") }