diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml index b51c1e06..5132f8d0 100644 --- a/.github/workflows/Release.yml +++ b/.github/workflows/Release.yml @@ -31,6 +31,10 @@ env: ARTIFACT_SIGNING_ENDPOINT: 'https://eus.codesigning.azure.net/' ARTIFACT_SIGNING_ACCOUNT_NAME: 'JoeFinAppsSigningCerts' ARTIFACT_SIGNING_CERTIFICATE_PROFILE_NAME: 'JoeFinApps' + # Unlock token for the Windows AI language model (a Limited Access Feature). Absent secrets + # leave the properties empty, which just builds without on-device text AI rather than failing. + LAF_TOKEN: ${{ secrets.LAF_TOKEN }} + LAF_PUBLISHER_ID: ${{ secrets.LAF_PUBLISHER_ID }} jobs: build: @@ -47,7 +51,7 @@ jobs: run: dotnet restore ${{ env.PROJECT_PATH }} - name: Run tests - run: dotnet test ${{ env.TEST_PATH }} -r win-x64 + run: dotnet test --project ${{ env.TEST_PATH }} -r win-x64 - name: Compute build version, archive paths, and release metadata id: compute @@ -89,6 +93,8 @@ jobs: -p:PublishReadyToRun=false -p:PublishSingleFile=true -p:CopyOutputSymbolsToPublishDirectory=false + -p:LafToken=$env:LAF_TOKEN + -p:LafPublisherId=$env:LAF_PUBLISHER_ID --nologo - name: Build x64 self-contained @@ -103,6 +109,8 @@ jobs: -p:PublishReadyToRun=true -p:PublishSingleFile=true -p:CopyOutputSymbolsToPublishDirectory=false + -p:LafToken=$env:LAF_TOKEN + -p:LafPublisherId=$env:LAF_PUBLISHER_ID --nologo - name: Build ARM64 framework-dependent @@ -116,6 +124,8 @@ jobs: -p:PublishSingleFile=true -p:EnableMsixTooling=true -p:CopyOutputSymbolsToPublishDirectory=false + -p:LafToken=$env:LAF_TOKEN + -p:LafPublisherId=$env:LAF_PUBLISHER_ID --nologo - name: Build ARM64 self-contained @@ -129,6 +139,8 @@ jobs: -p:PublishSingleFile=true -p:EnableMsixTooling=true -p:CopyOutputSymbolsToPublishDirectory=false + -p:LafToken=$env:LAF_TOKEN + -p:LafPublisherId=$env:LAF_PUBLISHER_ID --nologo - name: Rename ARM64 executables diff --git a/.github/workflows/buildDev.yml b/.github/workflows/buildDev.yml index a72cfec5..fee1c509 100644 --- a/.github/workflows/buildDev.yml +++ b/.github/workflows/buildDev.yml @@ -12,6 +12,10 @@ concurrency: env: PROJECT_PATH: "Text-Grab/Text-Grab.csproj" TEST_PATH: "Tests/Tests.csproj" + # Unlock token for the Windows AI language model (a Limited Access Feature). Absent secrets + # leave the properties empty, which just builds without on-device text AI rather than failing. + LAF_TOKEN: ${{ secrets.LAF_TOKEN }} + LAF_PUBLISHER_ID: ${{ secrets.LAF_PUBLISHER_ID }} jobs: build: @@ -27,10 +31,10 @@ jobs: - name: Build run: dotnet build ${{ env.PROJECT_PATH }} -p:EnableMsixTooling=true - name: Test - run: dotnet test ${{ env.TEST_PATH }} -r win-x64 + run: dotnet test --project ${{ env.TEST_PATH }} -r win-x64 - name: Build for release and Publish - run: dotnet publish ${{ env.PROJECT_PATH }} -c Release --self-contained -r win-x64 -p:PublishSingleFile=true -p:EnableMsixTooling=true -o publish + run: dotnet publish ${{ env.PROJECT_PATH }} -c Release --self-contained -r win-x64 -p:PublishSingleFile=true -p:EnableMsixTooling=true -p:LafToken=$env:LAF_TOKEN -p:LafPublisherId=$env:LAF_PUBLISHER_ID -o publish - name: Upload artifact uses: actions/upload-artifact@v7 diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index b59cb94e..95827918 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -2,6 +2,7 @@ net10.0-windows10.0.22621 + Exe enable x64;x86;ARM64 win-x86;win-x64;win-arm64 @@ -13,16 +14,7 @@ - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - + diff --git a/Tests/WinAiMeetingNotesTests.cs b/Tests/WinAiMeetingNotesTests.cs new file mode 100644 index 00000000..a5551b5e --- /dev/null +++ b/Tests/WinAiMeetingNotesTests.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Text_Grab.Utilities; + +namespace Tests; + +/// +/// Tests for the text splitting behind "Summarize as Meeting Notes". The model calls themselves need +/// a Copilot+ device, but the chunking that decides what gets sent to the model does not. +/// +public class WinAiMeetingNotesTests +{ + private const int Target = 100; + + /// Text that already fits is sent to the model in one piece. + [Fact] + public void SplitIntoParts_ShortText_ReturnsSinglePart() + { + string input = "Standup notes: shipped the OCR fix, starting on the settings page next."; + + List parts = WinAiMeetingNotes.SplitIntoParts(input, Target); + + Assert.Single(parts); + Assert.Equal(input, parts[0]); + } + + [Fact] + public void SplitIntoParts_TextExactlyAtTarget_ReturnsSinglePart() + { + string input = new('a', Target); + + List parts = WinAiMeetingNotes.SplitIntoParts(input, Target); + + Assert.Single(parts); + } + + [Fact] + public void SplitIntoParts_LongText_EveryPartWithinTarget() + { + string input = string.Join(" ", Enumerable.Repeat("discussed the roadmap and agreed on dates", 40)); + + List parts = WinAiMeetingNotes.SplitIntoParts(input, Target); + + Assert.True(parts.Count > 1); + Assert.All(parts, part => Assert.True(part.Length <= Target, $"Part was {part.Length} characters.")); + } + + /// Splitting must not lose or reorder any of the meeting text. + [Fact] + public void SplitIntoParts_LongText_PreservesAllWords() + { + string input = string.Join("\n", Enumerable.Range(0, 60).Select(index => $"Speaker {index}: point number {index}")); + + List parts = WinAiMeetingNotes.SplitIntoParts(input, Target); + + string[] originalWords = input.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + string[] splitWords = string.Join(" ", parts).Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + + Assert.Equal(originalWords, splitWords); + } + + /// A blank line is the most natural place to break a transcript. + [Fact] + public void SplitIntoParts_ParagraphBreaks_PrefersBlankLines() + { + string paragraph = new('a', 60); + string input = string.Join("\n\n", paragraph, paragraph, paragraph); + + List parts = WinAiMeetingNotes.SplitIntoParts(input, Target); + + Assert.All(parts, part => Assert.Equal(paragraph, part)); + } + + /// Text with nowhere good to break still terminates, splitting mid-word as a last resort. + [Fact] + public void SplitIntoParts_NoBreakCharacters_StillSplits() + { + string input = new('x', Target * 3); + + List parts = WinAiMeetingNotes.SplitIntoParts(input, Target); + + Assert.Equal(3, parts.Count); + Assert.All(parts, part => Assert.Equal(Target, part.Length)); + } + + [Fact] + public void SplitIntoParts_EmptyText_ReturnsSinglePart() + { + List parts = WinAiMeetingNotes.SplitIntoParts(string.Empty, Target); + + Assert.Single(parts); + Assert.Equal(string.Empty, parts[0]); + } +} diff --git a/Text-Grab-Package/Package.appxmanifest b/Text-Grab-Package/Package.appxmanifest index c6e38862..4aff676c 100644 --- a/Text-Grab-Package/Package.appxmanifest +++ b/Text-Grab-Package/Package.appxmanifest @@ -157,5 +157,7 @@ + + diff --git a/Text-Grab/App.config b/Text-Grab/App.config index 9b3197a4..9b6a6b58 100644 --- a/Text-Grab/App.config +++ b/Text-Grab/App.config @@ -121,6 +121,9 @@ System + + Color + False @@ -193,9 +196,18 @@ False + + True + + + False + True + + BaseMultilingual + Auto @@ -292,6 +304,12 @@ False + + True + + + False + diff --git a/Text-Grab/App.xaml.cs b/Text-Grab/App.xaml.cs index a1b96440..48407841 100644 --- a/Text-Grab/App.xaml.cs +++ b/Text-Grab/App.xaml.cs @@ -162,6 +162,8 @@ public static void SetTheme(object? sender = null, EventArgs? e = null) // for now this is best but... not ideal ApplicationAccentColorManager.ApplySystemAccent(); + NotifyIconUtilities.RefreshTrayIconStyle(); + // TODO: try to apply the teal color again, maybe something in WPFUI is broken // Color teal = (Color)ColorConverter.ConvertFromString("#308E98"); diff --git a/Text-Grab/Controls/BottomBarSettings.xaml b/Text-Grab/Controls/BottomBarSettings.xaml index 6a041e8c..5a6c4cdb 100644 --- a/Text-Grab/Controls/BottomBarSettings.xaml +++ b/Text-Grab/Controls/BottomBarSettings.xaml @@ -223,6 +223,14 @@ Show Cursor/Selection Text + + + Show Transcribe + + (DefaultSettings.DefaultLaunch, true); @@ -91,6 +93,18 @@ private void Window_Loaded(object sender, RoutedEventArgs e) NotifyIcon.TooltipText = toolTipText; } + public void ApplyTrayIconStyle() + { + bool isMonochrome = Enum.TryParse(DefaultSettings.TrayIconStyle, true, out TrayIconStyle style) + && style == TrayIconStyle.Monochrome; + + string iconPath = isMonochrome + ? (SystemThemeUtility.IsLightTheme() ? "/Images/Select-Black.ico" : "/Images/Select-White.ico") + : "/Images/TealSelect40.png"; + + NotifyIcon.Icon = new BitmapImage(new Uri($"pack://application:,,,{iconPath}")); + } + private void EditWindowMenuItem_Click(object sender, RoutedEventArgs e) { EditTextWindow etw = new(); diff --git a/Text-Grab/Controls/WordBorder.xaml.cs b/Text-Grab/Controls/WordBorder.xaml.cs index db089d22..bbfb9bbc 100644 --- a/Text-Grab/Controls/WordBorder.xaml.cs +++ b/Text-Grab/Controls/WordBorder.xaml.cs @@ -491,7 +491,7 @@ private void EditWordTextBox_ContextMenuOpening(object sender, ContextMenuEventA translateSeparator = separator; } - if (WindowsAiUtilities.CanDeviceUseWinAI()) + if (WinAiTranslator.IsAvailable()) { if (translateMenuItem != null) { @@ -680,12 +680,13 @@ private async void TranslateWordMenuItem_Click(object sender, RoutedEventArgs e) if (string.IsNullOrWhiteSpace(Word)) return; - if (!WindowsAiUtilities.CanDeviceUseWinAI()) + (bool available, string? reason) = WinAiTranslator.CheckAvailability(); + if (!available) { await new Wpf.Ui.Controls.MessageBox { Title = "Translation Not Available", - Content = "Windows AI is not available on this device.", + Content = reason ?? "Windows AI is not available on this device.", CloseButtonText = "OK" }.ShowDialogAsync(); return; @@ -700,7 +701,20 @@ private async void TranslateWordMenuItem_Click(object sender, RoutedEventArgs e) string targetLanguage = GetSystemLanguageName(); // Translate the word - string translatedText = await WindowsAiUtilities.TranslateText(originalWord, targetLanguage); + TranslationResult result = await WinAiTranslator.TranslateAsync(originalWord, targetLanguage); + + if (!result.Succeeded) + { + await new Wpf.Ui.Controls.MessageBox + { + Title = result.Failure is TranslationFailure.NotNeeded ? "Nothing to Translate" : "Translation Failed", + Content = result.Message ?? "The word could not be translated.", + CloseButtonText = "OK" + }.ShowDialogAsync(); + return; + } + + string translatedText = result.Text; // Update the word with translation if (!string.IsNullOrWhiteSpace(translatedText) && translatedText != originalWord) diff --git a/Text-Grab/Enums.cs b/Text-Grab/Enums.cs index d02308ab..d88a3648 100644 --- a/Text-Grab/Enums.cs +++ b/Text-Grab/Enums.cs @@ -13,6 +13,12 @@ public enum AppTheme Light = 2 } +public enum TrayIconStyle +{ + Color = 0, + Monochrome = 1, +} + public enum CurrentCase { Lower = 0, diff --git a/Text-Grab/Images/Select-Black.ico b/Text-Grab/Images/Select-Black.ico new file mode 100644 index 00000000..91007d4d Binary files /dev/null and b/Text-Grab/Images/Select-Black.ico differ diff --git a/Text-Grab/Images/Select-White.ico b/Text-Grab/Images/Select-White.ico new file mode 100644 index 00000000..008710ef Binary files /dev/null and b/Text-Grab/Images/Select-White.ico differ diff --git a/Text-Grab/Models/ButtonInfo.cs b/Text-Grab/Models/ButtonInfo.cs index 38eaf846..33fd9e10 100644 --- a/Text-Grab/Models/ButtonInfo.cs +++ b/Text-Grab/Models/ButtonInfo.cs @@ -794,6 +794,14 @@ public static List AllButtons RequiresCopilotPlus = true }, new() + { + OrderNumber = 8.15, + ButtonText = "Summarize as Meeting Notes", + ClickEvent = "MeetingNotesMenuItem_Click", + SymbolIcon = SymbolRegular.NotepadPerson20, + RequiresCopilotPlus = true + }, + new() { OrderNumber = 8.2, ButtonText = "Rewrite with Local AI", diff --git a/Text-Grab/Pages/GeneralSettings.xaml b/Text-Grab/Pages/GeneralSettings.xaml index f470b199..b9f287ac 100644 --- a/Text-Grab/Pages/GeneralSettings.xaml +++ b/Text-Grab/Pages/GeneralSettings.xaml @@ -107,6 +107,26 @@ + + + + + + + (DefaultSettings.DefaultLaunch, true); switch (defaultLaunchSetting) { @@ -255,6 +269,26 @@ private void DarkThemeRdBtn_Checked(object sender, RoutedEventArgs e) App.SetTheme(); } + private void ColorTrayIconRdBtn_Checked(object sender, RoutedEventArgs e) + { + if (!settingsSet) + return; + + DefaultSettings.TrayIconStyle = TrayIconStyle.Color.ToString(); + DefaultSettings.Save(); + NotifyIconUtilities.RefreshTrayIconStyle(); + } + + private void MonochromeTrayIconRdBtn_Checked(object sender, RoutedEventArgs e) + { + if (!settingsSet) + return; + + DefaultSettings.TrayIconStyle = TrayIconStyle.Monochrome.ToString(); + DefaultSettings.Save(); + NotifyIconUtilities.RefreshTrayIconStyle(); + } + private void ReadBarcodesBarcode_Checked(object sender, RoutedEventArgs e) { if (!settingsSet) diff --git a/Text-Grab/Properties/Settings.Designer.cs b/Text-Grab/Properties/Settings.Designer.cs index 0376e9ea..c9967880 100644 --- a/Text-Grab/Properties/Settings.Designer.cs +++ b/Text-Grab/Properties/Settings.Designer.cs @@ -12,7 +12,7 @@ namespace Text_Grab.Properties { [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "18.9.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "18.10.0.0")] internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase { private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings()))); @@ -479,6 +479,18 @@ public string AppTheme { } } + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("Color")] + public string TrayIconStyle { + get { + return ((string)(this["TrayIconStyle"])); + } + set { + this["TrayIconStyle"] = value; + } + } + [global::System.Configuration.UserScopedSettingAttribute()] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Configuration.DefaultSettingValueAttribute("False")] @@ -767,6 +779,30 @@ public bool EtwShowSimilarMatches { } } + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("True")] + public bool EtwShowTranscribe { + get { + return ((bool)(this["EtwShowTranscribe"])); + } + set { + this["EtwShowTranscribe"] = value; + } + } + + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("False")] + public bool TranscribeButtonJustIcon { + get { + return ((bool)(this["TranscribeButtonJustIcon"])); + } + set { + this["TranscribeButtonJustIcon"] = value; + } + } + [global::System.Configuration.UserScopedSettingAttribute()] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Configuration.DefaultSettingValueAttribute("True")] @@ -779,6 +815,18 @@ public bool EtwNormalizeLineEndingsOnPaste { } } + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("BaseMultilingual")] + public string AudioTranscriptionModel { + get { + return ((string)(this["AudioTranscriptionModel"])); + } + set { + this["AudioTranscriptionModel"] = value; + } + } + [global::System.Configuration.UserScopedSettingAttribute()] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Configuration.DefaultSettingValueAttribute("Auto")] @@ -1162,5 +1210,29 @@ public bool HdrBorderlessGranted { this["HdrBorderlessGranted"] = value; } } + + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("True")] + public bool NotifyOnTranscriptionComplete { + get { + return ((bool)(this["NotifyOnTranscriptionComplete"])); + } + set { + this["NotifyOnTranscriptionComplete"] = value; + } + } + + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("False")] + public bool IncludeTimecodesInTranscription { + get { + return ((bool)(this["IncludeTimecodesInTranscription"])); + } + set { + this["IncludeTimecodesInTranscription"] = value; + } + } } } diff --git a/Text-Grab/Properties/Settings.settings b/Text-Grab/Properties/Settings.settings index d6b32b78..d4ece6f8 100644 --- a/Text-Grab/Properties/Settings.settings +++ b/Text-Grab/Properties/Settings.settings @@ -116,6 +116,9 @@ System + + Color + False @@ -188,9 +191,18 @@ False + + True + + + False + True + + BaseMultilingual + Auto @@ -287,5 +299,11 @@ False + + True + + + False + \ No newline at end of file diff --git a/Text-Grab/Styles/ButtonStyles.xaml b/Text-Grab/Styles/ButtonStyles.xaml index a61bc9b0..eef7507a 100644 --- a/Text-Grab/Styles/ButtonStyles.xaml +++ b/Text-Grab/Styles/ButtonStyles.xaml @@ -317,8 +317,7 @@ x:Name="PART_Popup" AllowsTransparency="true" Focusable="false" - IsOpen="{Binding IsSubmenuOpen, - RelativeSource={RelativeSource TemplatedParent}}" + IsOpen="{Binding IsSubmenuOpen, RelativeSource={RelativeSource TemplatedParent}}" Placement="Bottom" PlacementTarget="{Binding ElementName=templateRoot}" PopupAnimation="{DynamicResource {x:Static SystemParameters.MenuPopupAnimationKey}}"> @@ -337,12 +336,9 @@ VerticalAlignment="Top"> + Width="{Binding ActualWidth, ElementName=SubMenuBorder}" + Height="{Binding ActualHeight, ElementName=SubMenuBorder}" + Fill="{Binding Background, ElementName=SubMenuBorder}" /> @@ -573,12 +568,9 @@ VerticalAlignment="Top"> + Width="{Binding ActualWidth, ElementName=SubMenuBorder}" + Height="{Binding ActualHeight, ElementName=SubMenuBorder}" + Fill="{Binding Background, ElementName=SubMenuBorder}" /> + Visibility="{Binding HeadersVisibility, ConverterParameter={x:Static DataGridHeadersVisibility.Row}, Converter={x:Static DataGrid.HeadersVisibilityConverter}, RelativeSource={RelativeSource AncestorType={x:Type DataGrid}}}" /> diff --git a/Text-Grab/Styles/ListViewScrollFix.xaml b/Text-Grab/Styles/ListViewScrollFix.xaml index aa605b6f..3dad6541 100644 --- a/Text-Grab/Styles/ListViewScrollFix.xaml +++ b/Text-Grab/Styles/ListViewScrollFix.xaml @@ -26,18 +26,12 @@ VerticalScrollBarVisibility="Hidden"> + Value="{Binding Path=HorizontalOffset, RelativeSource={RelativeSource TemplatedParent}, Mode=OneWay}" /> + Value="{Binding Path=VerticalOffset, RelativeSource={RelativeSource TemplatedParent}, Mode=OneWay}" /> + Data="{Binding Content, RelativeSource={RelativeSource TemplatedParent}}"> diff --git a/Text-Grab/Text-Grab.csproj b/Text-Grab/Text-Grab.csproj index 02af482a..c50823fb 100644 --- a/Text-Grab/Text-Grab.csproj +++ b/Text-Grab/Text-Grab.csproj @@ -28,12 +28,39 @@ $(NoWarn);WFO0003 + + + $(LAF_TOKEN) + $(LAF_PUBLISHER_ID) + + + + + + + + + + + + @@ -65,7 +92,7 @@ - + @@ -78,15 +105,28 @@ - - - - - + + + + + + + + + + + - - + + none + + @@ -112,6 +152,12 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/Text-Grab/Utilities/AudioTranscriptionUtilities.cs b/Text-Grab/Utilities/AudioTranscriptionUtilities.cs new file mode 100644 index 00000000..73aef626 --- /dev/null +++ b/Text-Grab/Utilities/AudioTranscriptionUtilities.cs @@ -0,0 +1,1037 @@ +using NAudio.CoreAudioApi; +using NAudio.MediaFoundation; +using NAudio.Utils; +using NAudio.Wave; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Whisper.net; +using Whisper.net.Ggml; + +namespace Text_Grab.Utilities; + +/// +/// Lightweight, always-on file logger for the audio transcription path. Writes timestamped lines +/// (with current process working set) to a stable, easy-to-find location so a run can be diagnosed +/// after the fact. Also mirrors to . +/// +public static class AudioDebugLog +{ + private static readonly object _lock = new(); + + /// Rolled over into audio-debug.prev.log once the live file passes this size. + private const long MaxLogBytes = 1024 * 1024; + + private static long _writtenBytes = -1; // -1 until the size of an existing log is read once + + private static string LogDirectory => Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + "Text-Grab", "Logs"); + + /// Stable per-user log path so a run can be found and collected after the fact. + public static string LogPath { get; } = Path.Combine(LogDirectory, "audio-debug.log"); + + /// The previous log, kept so a rollover mid-run doesn't lose the start of the session. + private static string PreviousLogPath { get; } = Path.Combine(LogDirectory, "audio-debug.prev.log"); + + public static void Write(string message) + { + // Environment.WorkingSet, not a Process object: a cached Process reports whatever it last + // refreshed, and a fresh Process.GetCurrentProcess() per logged line is not free. + long workingSetMb = 0; + try { workingSetMb = Environment.WorkingSet / (1024 * 1024); } catch { } + + string line = $"{DateTime.Now:HH:mm:ss.fff} [WS {workingSetMb,6} MB] {message}"; + Debug.WriteLine("[AudioTranscription] " + line); + try + { + lock (_lock) + { + if (_writtenBytes < 0) + { + Directory.CreateDirectory(LogDirectory); + _writtenBytes = File.Exists(LogPath) ? new FileInfo(LogPath).Length : 0; + } + + // Roll over instead of growing without bound: this log is always on. + if (_writtenBytes > MaxLogBytes) + { + File.Move(LogPath, PreviousLogPath, overwrite: true); + _writtenBytes = 0; + } + + string text = line + Environment.NewLine; + File.AppendAllText(LogPath, text); + _writtenBytes += Encoding.UTF8.GetByteCount(text); + } + } + catch { /* logging must never throw */ } + } +} + +/// +/// One loaded plus the number of leases still using it. Disposing a +/// factory frees the native model, so a factory that is superseded (the user picks another model) +/// while a transcription is still decoding against it is d instead: it is +/// disposed only once the last lease is returned. +/// +internal sealed class WhisperFactoryHandle +{ + private readonly object _lock = new(); + private int _users; + private bool _retired; + private bool _disposed; + + internal WhisperFactory Factory { get; } + internal WhisperModelChoice Choice { get; } + + internal WhisperFactoryHandle(WhisperFactory factory, WhisperModelChoice choice) + { + Factory = factory; + Choice = choice; + } + + internal WhisperFactoryLease Lease() + { + lock (_lock) + _users++; + + return new WhisperFactoryLease(this); + } + + /// Marks the factory superseded; it is disposed as soon as the last lease is returned. + internal void Retire() + { + lock (_lock) + { + _retired = true; + DisposeIfIdle(); + } + } + + internal void Return() + { + lock (_lock) + { + _users--; + DisposeIfIdle(); + } + } + + /// Caller must hold . + private void DisposeIfIdle() + { + if (_disposed || !_retired || _users > 0) + return; + + _disposed = true; + AudioDebugLog.Write($"WhisperFactoryHandle: disposing retired factory for {Choice}"); + try { Factory.Dispose(); } catch { } + } +} + +/// +/// A borrowed reference to a shared . The factory owns the native model +/// that every built from it decodes against, so it must outlive them: +/// hold the lease for as long as any such processor lives, and dispose it after the processor. +/// +internal sealed class WhisperFactoryLease : IDisposable +{ + private WhisperFactoryHandle? _handle; + + internal WhisperFactoryLease(WhisperFactoryHandle handle) => _handle = handle; + + internal WhisperFactory Factory => + (_handle ?? throw new ObjectDisposedException(nameof(WhisperFactoryLease))).Factory; + + public void Dispose() => Interlocked.Exchange(ref _handle, null)?.Return(); +} + +/// +/// On-device audio transcription backed by local Whisper (whisper.cpp) models via Whisper.net. +/// Runs entirely on the CPU, works packaged or unpackaged on x64 and arm64, and does not depend on +/// any experimental OS runtime. Arbitrary audio is decoded/resampled to the 16 kHz mono WAV that +/// Whisper requires using NAudio's Media Foundation reader/resampler. +/// +public static class AudioTranscriptionUtilities +{ + private static readonly HashSet AudioExtensions = new(StringComparer.OrdinalIgnoreCase) + { + ".wav", ".mp3", ".m4a", ".aac", ".flac", ".ogg", ".oga", ".opus", ".wma", ".mp4", ".mov", + }; + + private static WhisperFactoryHandle? _factoryHandle; + private static readonly SemaphoreSlim _factoryLock = new(1, 1); + + // Silero VAD (voice activity detection) lets live transcription skip silence and cut on natural + // speech boundaries instead of fixed time windows. The factory is small and shared. + private static WhisperVadFactory? _vadFactory; + private static readonly SemaphoreSlim _vadFactoryLock = new(1, 1); + + private static string ModelDirectory => Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + "Text-Grab", "WhisperModels"); + + /// The transcription model currently selected in settings (defaults to multilingual base). + public static WhisperModelChoice CurrentModelChoice => WhisperModelInfo.Parse(AppUtilities.TextGrabSettings.AudioTranscriptionModel); + + private static string ModelPathFor(WhisperModelChoice choice) + { + string typeName = WhisperModelInfo.GgmlTypeFor(choice).ToString().ToLowerInvariant(); + QuantizationType quantization = WhisperModelInfo.QuantizationFor(choice); + string suffix = quantization == QuantizationType.NoQuantization ? string.Empty : $"-{quantization.ToString().ToLowerInvariant()}"; + return Path.Combine(ModelDirectory, $"ggml-{typeName}{suffix}.bin"); + } + + private static string VadModelPath => Path.Combine(ModelDirectory, "ggml-silero-vad-v5.bin"); + + /// + /// Returns true when the given path points to a file with a recognized audio (or A/V) extension. + /// + public static bool IsAudioFile(string path) + { + if (string.IsNullOrWhiteSpace(path)) + return false; + + return AudioExtensions.Contains(Path.GetExtension(path)); + } + + /// An -compatible filter string for the supported audio/A-V extensions. + public static string GetAudioFileFilter() + { + string extensions = string.Join(";", AudioExtensions.Select(ext => $"*{ext}")); + return $"Audio/video files|{extensions}|All files (*.*)|*.*"; + } + + /// Basic file info (name, size, duration) for the "what to expect" panel — no decoding. + internal readonly record struct AudioFileInfo(string FileName, long FileSizeBytes, TimeSpan Duration); + + /// + /// Reads the file size and duration of an audio/A-V file without decoding it, for upfront user + /// feedback before transcription starts. Throws if the file doesn't exist or can't be opened by + /// Media Foundation (e.g. an unsupported or corrupt format) — callers should show that inline. + /// + internal static AudioFileInfo GetAudioFileInfo(string audioFilePath) + { + if (!File.Exists(audioFilePath)) + throw new FileNotFoundException("Audio file not found.", audioFilePath); + + long fileSizeBytes = new FileInfo(audioFilePath).Length; + + MediaFoundationApi.Startup(); + using MediaFoundationReader reader = new(audioFilePath); + TimeSpan duration = reader.TotalTime; + + return new AudioFileInfo(Path.GetFileName(audioFilePath), fileSizeBytes, duration); + } + + /// + /// Whisper runs on the CPU on every supported Windows build (x64 / arm64, packaged or not), so + /// audio transcription is always available. The model is fetched on first use. + /// + public static bool IsAudioTranscriptionSupported() => true; + + /// True once the selected Whisper model has been downloaded and is available locally. + public static bool IsModelDownloaded() => File.Exists(ModelPathFor(CurrentModelChoice)); + + /// + /// Downloads a GGML model to LocalAppData if it isn't already present, returning its path. The + /// download is written to a temp file first, then moved into place so a cancelled or failed + /// download never leaves a corrupt model behind. + /// + private static async Task EnsureModelDownloadedAsync(WhisperModelChoice choice, IProgress? progress, CancellationToken cancellationToken) + { + string modelPath = ModelPathFor(choice); + if (File.Exists(modelPath)) + return modelPath; + + Directory.CreateDirectory(ModelDirectory); + GgmlType ggmlType = WhisperModelInfo.GgmlTypeFor(choice); + QuantizationType quantization = WhisperModelInfo.QuantizationFor(choice); + AudioDebugLog.Write($"EnsureModelDownloadedAsync: downloading Whisper '{ggmlType}' ({quantization}) model to {modelPath}"); + progress?.Report($"Downloading speech model ({WhisperModelInfo.DisplayName(choice)}, first run)…"); + + string tempPath = modelPath + ".download"; + try + { + using (Stream modelStream = await WhisperGgmlDownloader.Default.GetGgmlModelAsync(ggmlType, quantization, cancellationToken).ConfigureAwait(false)) + using (FileStream fileWriter = File.Create(tempPath)) + await modelStream.CopyToAsync(fileWriter, cancellationToken).ConfigureAwait(false); + + if (File.Exists(modelPath)) + File.Delete(modelPath); + File.Move(tempPath, modelPath); + AudioDebugLog.Write("EnsureModelDownloadedAsync: download complete"); + return modelPath; + } + catch + { + try { if (File.Exists(tempPath)) File.Delete(tempPath); } catch { } + throw; + } + } + + /// + /// Borrows the shared, cached for the currently selected model, + /// downloading the model if needed. The factory is expensive to create (it loads the model), so + /// it is created once and reused. If the model choice changes, the old factory is retired and a + /// new one is loaded — see for why the caller must hold the + /// lease for as long as it uses processors built from the factory. + /// + internal static async Task AcquireFactoryAsync(IProgress? progress, CancellationToken cancellationToken) + { + WhisperModelChoice choice = CurrentModelChoice; + + await _factoryLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_factoryHandle is not null && _factoryHandle.Choice == choice) + return _factoryHandle.Lease(); + + if (_factoryHandle is not null) + { + AudioDebugLog.Write($"AcquireFactoryAsync: model changed {_factoryHandle.Choice} -> {choice}, reloading"); + _factoryHandle.Retire(); + _factoryHandle = null; + } + + string modelPath = await EnsureModelDownloadedAsync(choice, progress, cancellationToken).ConfigureAwait(false); + AudioDebugLog.Write($"AcquireFactoryAsync: loading WhisperFactory for {choice} ({WhisperModelInfo.GgmlTypeFor(choice)})"); + _factoryHandle = new WhisperFactoryHandle(WhisperFactory.FromPath(modelPath), choice); + AudioDebugLog.Write("AcquireFactoryAsync: WhisperFactory ready"); + return _factoryHandle.Lease(); + } + finally + { + _factoryLock.Release(); + } + } + + /// + /// Downloads the Silero VAD model to LocalAppData if needed (same temp-then-move pattern), then + /// returns the shared, cached . The VAD model is tiny (~a few MB). + /// + internal static async Task GetVadFactoryAsync(IProgress? progress, CancellationToken cancellationToken) + { + if (_vadFactory is not null) + return _vadFactory; + + await _vadFactoryLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_vadFactory is not null) + return _vadFactory; + + if (!File.Exists(VadModelPath)) + { + Directory.CreateDirectory(ModelDirectory); + AudioDebugLog.Write("GetVadFactoryAsync: downloading Silero VAD model"); + progress?.Report("Downloading voice-activity model…"); + + string tempPath = VadModelPath + ".download"; + try + { + using (Stream vadStream = await WhisperGgmlDownloader.Default.GetGgmlSileroVadModelAsync(SileroVadType.V5_1_2, cancellationToken).ConfigureAwait(false)) + using (FileStream fileWriter = File.Create(tempPath)) + await vadStream.CopyToAsync(fileWriter, cancellationToken).ConfigureAwait(false); + + if (File.Exists(VadModelPath)) + File.Delete(VadModelPath); + File.Move(tempPath, VadModelPath); + } + catch + { + try { if (File.Exists(tempPath)) File.Delete(tempPath); } catch { } + throw; + } + } + + _vadFactory = WhisperVadFactory.FromPath(VadModelPath); + AudioDebugLog.Write("GetVadFactoryAsync: WhisperVadFactory ready"); + return _vadFactory; + } + finally + { + _vadFactoryLock.Release(); + } + } + + /// + /// Transcribes a complete audio file on-device with Whisper and returns the recognized text. + /// Whisper.net's ProcessAsync yields incrementally as whisper.cpp + /// finishes each segment of the audio, so receives each + /// segment's text as it becomes available (giving "text as it comes in" for long files). The full + /// transcript is still returned. Cancellation stops after the current segment; every segment + /// already surfaced via is preserved by the caller. + /// , if given, is passed to Whisper as an initial prompt so it's biased + /// toward names/jargon it might otherwise mishear; it applies only to this call, nothing persists. + /// When is true, each segment is prefixed with its start time + /// (e.g. [01:23]) and placed on its own line. + /// , if given, reports how far playback has reached through the + /// clip (0.0-1.0) after each segment, based on that segment's end time versus the clip's total + /// duration — lets callers show a real progress bar instead of an indeterminate spinner. + /// + public static async Task TranscribeAudioFileAsync(string audioFilePath, string? hotWords = null, IProgress? statusProgress = null, IProgress? segmentProgress = null, CancellationToken cancellationToken = default, bool includeTimecodes = false, IProgress? clipProgress = null) + { + AudioDebugLog.Write($"TranscribeAudioFileAsync: START path='{audioFilePath}'"); + + if (!File.Exists(audioFilePath)) + throw new FileNotFoundException("Audio file not found.", audioFilePath); + + long fileSizeKb = new FileInfo(audioFilePath).Length / 1024; + AudioDebugLog.Write($"TranscribeAudioFileAsync: file exists, size={fileSizeKb} KB, ext={Path.GetExtension(audioFilePath)}"); + + // Whisper + audio decoding are CPU-bound; run off the UI thread. + return await Task.Run(async () => + { + // The lease is held for the whole decode: a model change (or a live session starting) part + // way through must not free the native model this processor is still reading. + using WhisperFactoryLease factoryLease = await AcquireFactoryAsync(statusProgress, cancellationToken).ConfigureAwait(false); + + statusProgress?.Report("Transcribing audio…"); + AudioDebugLog.Write("TranscribeAudioFileAsync: decoding audio to 16 kHz mono WAV"); + using MemoryStream wavStream = DecodeToWav16kMono(audioFilePath); + AudioDebugLog.Write($"TranscribeAudioFileAsync: decoded WAV bytes={wavStream.Length}"); + + // 16 kHz mono 16-bit PCM, 44-byte WAV header: 32,000 bytes/second of audio. + double clipTotalSeconds = Math.Max(0, wavStream.Length - 44) / 32000.0; + + Stopwatch stopwatch = Stopwatch.StartNew(); + WhisperProcessorBuilder processorBuilder = factoryLease.Factory.CreateBuilder() + .WithLanguage(WhisperModelInfo.LanguageFor(CurrentModelChoice)) + .WithThreads(Math.Max(1, Environment.ProcessorCount - 1)); + + // CarryInitialPrompt re-applies the hot words to every decode window (not just the first), + // so the bias holds across long files instead of fading out after the first segment. + if (!string.IsNullOrWhiteSpace(hotWords)) + processorBuilder = processorBuilder.WithPrompt(hotWords.Trim()).WithCarryInitialPrompt(true); + + await using WhisperProcessor processor = processorBuilder.Build(); + + StringBuilder builder = new(); + int segmentCount = 0; + await foreach (SegmentData segment in processor.ProcessAsync(wavStream, cancellationToken).ConfigureAwait(false)) + { + string segmentText = includeTimecodes + ? $"[{FormatTimecode(segment.Start)}]{segment.Text}{Environment.NewLine}" + : segment.Text; + + builder.Append(segmentText); + segmentProgress?.Report(segmentText); + segmentCount++; + + if (clipTotalSeconds > 0) + clipProgress?.Report(Math.Clamp(segment.End.TotalSeconds / clipTotalSeconds, 0.0, 1.0)); + } + + clipProgress?.Report(1.0); + + stopwatch.Stop(); + string text = CleanTranscript(builder.ToString()); + AudioDebugLog.Write($"TranscribeAudioFileAsync: DONE in {stopwatch.ElapsedMilliseconds} ms, {segmentCount} segments, result length={text.Length}"); + return text; + }, cancellationToken).ConfigureAwait(false); + } + + /// Formats a segment's start time as mm:ss, or h:mm:ss once past an hour. + internal static string FormatTimecode(TimeSpan t) => + t.TotalHours >= 1 ? t.ToString(@"h\:mm\:ss") : t.ToString(@"mm\:ss"); + + /// Collapses whisper's leading spaces / stray whitespace into a tidy transcript. + internal static string CleanTranscript(string raw) + { + if (string.IsNullOrWhiteSpace(raw)) + return string.Empty; + + return raw.Replace("\r\n", "\n").Trim(); + } + + /// + /// Decodes any Media Foundation-supported audio (wav, mp3, m4a, aac, wma, mp4, …) to a 16 kHz + /// mono 16-bit PCM WAV in memory — the format Whisper expects. + /// + internal static MemoryStream DecodeToWav16kMono(string audioFilePath) + { + MediaFoundationApi.Startup(); + + using MediaFoundationReader reader = new(audioFilePath); + WaveFormat targetFormat = new(16000, 16, 1); + + MemoryStream memoryStream = new(); + using (MediaFoundationResampler resampler = new(reader, targetFormat) { ResamplerQuality = 60 }) + { + // WriteWavFileToStream wraps the stream in an IgnoreDisposeStream, so memoryStream stays open. + WaveFileWriter.WriteWavFileToStream(memoryStream, resampler); + } + + memoryStream.Position = 0; + return memoryStream; + } + + /// Wraps raw 16 kHz mono 16-bit PCM bytes in an in-memory WAV stream for Whisper. + internal static MemoryStream PcmToWav16kMono(byte[] pcm, int count) + { + MemoryStream memoryStream = new(); + using (WaveFileWriter writer = new(new IgnoreDisposeStream(memoryStream), new WaveFormat(16000, 16, 1))) + writer.Write(pcm, 0, count); + + memoryStream.Position = 0; + return memoryStream; + } + + /// + /// Converts a raw captured buffer in an arbitrary (e.g. the + /// 32-bit float stereo mix from WASAPI loopback, or a mic's PCM) to a 16 kHz mono 16-bit WAV + /// stream for Whisper. Uses a fast path when the buffer is already in Whisper's format. + /// + internal static MemoryStream ConvertToWav16kMono(byte[] raw, int count, WaveFormat sourceFormat) + { + if (sourceFormat.Encoding == WaveFormatEncoding.Pcm + && sourceFormat.SampleRate == 16000 + && sourceFormat.Channels == 1 + && sourceFormat.BitsPerSample == 16) + { + return PcmToWav16kMono(raw, count); + } + + MediaFoundationApi.Startup(); + using RawSourceWaveStream rawStream = new(new MemoryStream(raw, 0, count), sourceFormat); + WaveFormat targetFormat = new(16000, 16, 1); + + MemoryStream memoryStream = new(); + using (MediaFoundationResampler resampler = new(rawStream, targetFormat) { ResamplerQuality = 60 }) + WaveFileWriter.WriteWavFileToStream(memoryStream, resampler); + + memoryStream.Position = 0; + return memoryStream; + } + + /// + /// Converts a raw captured buffer in an arbitrary to normalized + /// 16 kHz mono float samples in [-1, 1] — the shape both Silero VAD and Whisper consume directly, + /// avoiding a WAV round-trip. Uses a fast path when the buffer is already 16 kHz mono 16-bit PCM. + /// + internal static float[] ConvertToSamples16kMono(byte[] raw, int count, WaveFormat sourceFormat) + { + if (sourceFormat.Encoding == WaveFormatEncoding.Pcm + && sourceFormat.SampleRate == 16000 + && sourceFormat.Channels == 1 + && sourceFormat.BitsPerSample == 16) + { + int sampleCount = count / 2; + float[] fast = new float[sampleCount]; + for (int i = 0; i < sampleCount; i++) + { + short sample = (short)(raw[i * 2] | (raw[i * 2 + 1] << 8)); + fast[i] = sample / 32768f; + } + return fast; + } + + MediaFoundationApi.Startup(); + using RawSourceWaveStream rawStream = new(new MemoryStream(raw, 0, count), sourceFormat); + WaveFormat targetFormat = new(16000, 16, 1); + using MediaFoundationResampler resampler = new(rawStream, targetFormat) { ResamplerQuality = 60 }; + + List samples = new(count / 4); + byte[] buffer = new byte[16000 * 2]; // ~1 second of 16-bit mono + int read; + while ((read = resampler.Read(buffer)) > 0) + { + for (int i = 0; i + 1 < read; i += 2) + { + short sample = (short)(buffer[i] | (buffer[i + 1] << 8)); + samples.Add(sample / 32768f); + } + } + return samples.ToArray(); + } +} + +/// The Whisper model a user can pick, trading speed for accuracy / language coverage. +public enum WhisperModelChoice +{ + /// tiny.en — fastest, English only. + TinyEnglish, + + /// base.en — fast, English only. + BaseEnglish, + + /// base — balanced, multilingual with auto language detection (default). + BaseMultilingual, + + /// small — most accurate offered here, multilingual, noticeably slower. + SmallMultilingual, +} + +/// Maps to its GGML model, language, and display name. +internal static class WhisperModelInfo +{ + public static WhisperModelChoice Parse(string? value) => value switch + { + "TinyEnglish" => WhisperModelChoice.TinyEnglish, + "BaseEnglish" => WhisperModelChoice.BaseEnglish, + "SmallMultilingual" => WhisperModelChoice.SmallMultilingual, + _ => WhisperModelChoice.BaseMultilingual, + }; + + public static GgmlType GgmlTypeFor(WhisperModelChoice choice) => choice switch + { + WhisperModelChoice.TinyEnglish => GgmlType.TinyEn, + WhisperModelChoice.BaseEnglish => GgmlType.BaseEn, + WhisperModelChoice.SmallMultilingual => GgmlType.Small, + _ => GgmlType.Base, + }; + + // English-only models can't language-detect, so force English; multilingual models auto-detect. + public static string LanguageFor(WhisperModelChoice choice) => choice switch + { + WhisperModelChoice.TinyEnglish or WhisperModelChoice.BaseEnglish => "en", + _ => "auto", + }; + + // Q5_0 keeps the small English-only models fast and cheap to load with negligible WER impact. + // Multilingual models use the near-lossless Q8_0 instead, since quantization hurts accuracy more + // on the less-represented languages those models exist to cover. + public static QuantizationType QuantizationFor(WhisperModelChoice choice) => choice switch + { + WhisperModelChoice.TinyEnglish or WhisperModelChoice.BaseEnglish => QuantizationType.Q5_0, + _ => QuantizationType.Q8_0, + }; + + public static string DisplayName(WhisperModelChoice choice) => choice switch + { + WhisperModelChoice.TinyEnglish => "Fastest — English", + WhisperModelChoice.BaseEnglish => "Fast — English", + WhisperModelChoice.SmallMultilingual => "Most accurate — multilingual", + _ => "Balanced — multilingual", + }; +} + +/// Where pulls audio from. +public enum LiveCaptureSource +{ + /// The default microphone / recording device. + Microphone, + + /// System output ("what you hear") via WASAPI loopback on the default render device. + SystemAudio, + + /// Both the microphone and system output, mixed into a single stream before transcription. + MicrophoneAndSystemAudio, +} + +/// +/// Near-live transcription with Whisper from the microphone, system output (WASAPI loopback), or +/// both at once (mixed into a single stream), gated by Silero voice-activity detection. Instead of +/// transcribing fixed time windows (which waste compute on silence and cut words mid-phrase), it +/// buffers audio, runs cheap VAD on a short cadence to find speech regions, and only sends a region +/// to Whisper once it's complete (trailing silence detected). Each completed utterance raises +/// . Events fire on background threads; subscribers must marshal to +/// their UI thread. +/// +public sealed class LiveAudioTranscriber : IDisposable +{ + private const int SampleRate = 16000; + private const int TimerIntervalMs = 200; + private const double MinAudioSeconds = 0.4; // don't bother running VAD on less than this + private const double CompletionSilenceSeconds = 0.25; // trailing silence that marks an utterance done + private const double MaxUtteranceSeconds = 20.0; // hard cap so a long monologue still flushes + + /// + /// A single capture device feeding this transcriber (microphone or system-audio loopback), with + /// its own raw-PCM buffer so concurrent sources never share captured bytes. + /// + private sealed class CaptureChannel + { + public IWaveIn Capture { get; } + public WaveFormat SourceFormat { get; } + public MemoryStream PcmBuffer { get; } = new(); + public object BufferLock { get; } = new(); + private readonly EventHandler _dataAvailableHandler; + + public CaptureChannel(IWaveIn capture) + { + Capture = capture; + SourceFormat = capture.WaveFormat; + _dataAvailableHandler = (_, e) => + { + lock (BufferLock) + PcmBuffer.Write(e.Buffer, 0, e.BytesRecorded); + }; + Capture.DataAvailable += _dataAvailableHandler; + } + + public void Dispose() + { + Capture.DataAvailable -= _dataAvailableHandler; + try { Capture.Dispose(); } catch { } + } + } + + private readonly List _channels = new(); + private WhisperFactoryLease? _factoryLease; + private WhisperProcessor? _processor; + private WhisperVadProcessor? _vadProcessor; + private readonly SemaphoreSlim _processingGate = new(1, 1); + + // Serializes StartAsync/StopAsync so a restart (e.g. changing the model or capture source while a + // session is live) can never overlap a start with an in-flight stop. Without this, a caller that + // fires Stop() (fire-and-forget) and then immediately awaits StartAsync() — as the source/model + // menu handlers used to — could race: the new session's capture channels get added to the same + // list the old session's Cleanup() is disposing/clearing on a background thread, corrupting shared + // state and crashing the app. With the lock, StartAsync simply waits for the prior StopAsync to + // finish flushing and cleaning up before building the new session. + private readonly SemaphoreSlim _lifecycleLock = new(1, 1); + private System.Timers.Timer? _chunkTimer; + private volatile bool _isRunning; + + // Timer.Stop() does not cancel Elapsed callbacks already queued to the thread pool, so one can + // still take the processing gate after StopAsync's flush releases it and run against state + // Cleanup() is tearing down. Teardown sets this, and a timed pass that sees it bails out. + private volatile bool _stopping; + private bool _disposed; + + /// Raised with recognized text for each completed (VAD-delimited) utterance. + public event EventHandler? PhraseRecognized; + + public bool IsRunning => _isRunning; + + /// The source the current (or most recent) session is capturing from. + public LiveCaptureSource Source { get; private set; } = LiveCaptureSource.Microphone; + + /// + /// Starts capturing from the requested source (microphone or system loopback) and transcribing + /// VAD-delimited utterances. Returns false when the device isn't available or startup otherwise + /// fails. The Whisper and VAD models are downloaded on first use, so the first call may take a while. + /// + public async Task StartAsync(LiveCaptureSource source = LiveCaptureSource.Microphone) + { + ObjectDisposedException.ThrowIf(_disposed, this); + + // Waits out any in-flight StopAsync (e.g. a restart triggered by a model/source change) so a + // new session never starts while the previous one is still flushing/cleaning up. + await _lifecycleLock.WaitAsync().ConfigureAwait(false); + try + { + if (_isRunning) + return true; + + _stopping = false; + Source = source; + + bool wantsMic = source is LiveCaptureSource.Microphone or LiveCaptureSource.MicrophoneAndSystemAudio; + bool wantsSystem = source is LiveCaptureSource.SystemAudio or LiveCaptureSource.MicrophoneAndSystemAudio; + + if (wantsMic && WaveInEvent.DeviceCount <= 0) + { + AudioDebugLog.Write("LiveAudioTranscriber: no microphone capture device found"); + return false; + } + + WhisperModelChoice choice = AudioTranscriptionUtilities.CurrentModelChoice; + + // Held for the life of the session (released in Cleanup, after the processor): the shared + // factory owns the native model _processor decodes against. + _factoryLease = await AudioTranscriptionUtilities.AcquireFactoryAsync(null, CancellationToken.None).ConfigureAwait(false); + _processor = _factoryLease.Factory.CreateBuilder() + .WithLanguage(WhisperModelInfo.LanguageFor(choice)) + .WithNoContext() // each utterance stands alone: faster and avoids cross-phrase drift + .WithThreads(Math.Max(1, Environment.ProcessorCount - 1)) + .Build(); + + WhisperVadFactory vadFactory = await AudioTranscriptionUtilities.GetVadFactoryAsync(null, CancellationToken.None).ConfigureAwait(false); + _vadProcessor = vadFactory.CreateBuilder() + .WithThreshold(0.5f) + .WithMinSpeechDuration(TimeSpan.FromMilliseconds(250)) + .WithMinSilenceDuration(TimeSpan.FromMilliseconds(300)) + .WithSpeechPadding(TimeSpan.FromMilliseconds(64)) + .WithThreads(Math.Max(1, Environment.ProcessorCount - 1)) + .Build(); + + // Microphone can be forced to 16 kHz mono (WinMM converts); loopback yields the render + // device's mix format (usually 32-bit float stereo) which we resample when processing. + // Each source gets its own channel/buffer; when both are requested they're captured + // independently and mixed down to one stream per processing pass (see MixChannels). + if (wantsMic) + _channels.Add(new CaptureChannel(new WaveInEvent { WaveFormat = new WaveFormat(16000, 16, 1), BufferMilliseconds = 100 })); + if (wantsSystem) + _channels.Add(new CaptureChannel(new WasapiLoopbackCapture())); + + foreach (CaptureChannel channel in _channels) + AudioDebugLog.Write($"LiveAudioTranscriber: source={source} model={choice} format={channel.SourceFormat.Encoding} {channel.SourceFormat.SampleRate}Hz {channel.SourceFormat.Channels}ch {channel.SourceFormat.BitsPerSample}bit"); + + foreach (CaptureChannel channel in _channels) + channel.Capture.StartRecording(); + + _chunkTimer = new System.Timers.Timer(TimerIntervalMs) { AutoReset = true }; + _chunkTimer.Elapsed += async (_, _) => await ProcessBufferedChunkAsync().ConfigureAwait(false); + _chunkTimer.Start(); + + _isRunning = true; + AudioDebugLog.Write("LiveAudioTranscriber: started"); + return true; + } + catch (Exception ex) + { + AudioDebugLog.Write($"LiveAudioTranscriber: failed to start ({source}): {ex.Message}"); + Cleanup(); + return false; + } + finally + { + _lifecycleLock.Release(); + } + } + + /// + /// Stops capturing, flushes and transcribes any remaining buffered speech, then releases + /// resources. Awaitable so a restart (source/model change) can wait for a clean teardown. Holds + /// the same lifecycle lock as , so a start already waiting on this stop + /// resumes only once the flush/cleanup below has fully finished. + /// + public async Task StopAsync() + { + await _lifecycleLock.WaitAsync().ConfigureAwait(false); + try + { + if (!_isRunning && _channels.Count == 0) + return; + + _isRunning = false; + _stopping = true; + _chunkTimer?.Stop(); + + foreach (CaptureChannel channel in _channels) + try { channel.Capture.StopRecording(); } catch { } + + // Flush whatever remains (this waits for any in-flight pass), then clean up. + try { await ProcessBufferedChunkAsync(flush: true).ConfigureAwait(false); } catch { } + + // Tear down under the same gate the timed passes take, so a callback queued before + // Stop() can never be inside ProcessBufferedChunkAsync while state is being disposed. + await _processingGate.WaitAsync().ConfigureAwait(false); + try + { + Cleanup(); + } + finally + { + _processingGate.Release(); + } + + AudioDebugLog.Write("LiveAudioTranscriber: stopped"); + } + finally + { + _lifecycleLock.Release(); + } + } + + /// Fire-and-forget stop for callers that can't await (see ). + public void Stop() => _ = StopAsync(); + + /// + /// Sums same-length-padded per-channel samples into one stream, clamping to [-1, 1] so two loud + /// sources can't clip beyond Whisper's expected range. A single channel is returned unchanged. + /// + private static float[] MixChannels(List perChannelSamples) + { + if (perChannelSamples.Count == 1) + return perChannelSamples[0]; + + int length = 0; + foreach (float[] samples in perChannelSamples) + length = Math.Max(length, samples.Length); + + float[] mixed = new float[length]; + foreach (float[] samples in perChannelSamples) + for (int i = 0; i < samples.Length; i++) + mixed[i] += samples[i]; + + for (int i = 0; i < mixed.Length; i++) + mixed[i] = Math.Clamp(mixed[i], -1f, 1f); + + return mixed; + } + + /// + /// Runs VAD over the buffered audio and transcribes any completed speech regions. Normal (timed) + /// passes skip if one is already running; a pass waits its turn and + /// forces transcription of whatever speech remains. + /// + private async Task ProcessBufferedChunkAsync(bool flush = false) + { + if (flush) + await _processingGate.WaitAsync().ConfigureAwait(false); + else if (!await _processingGate.WaitAsync(0).ConfigureAwait(false)) + return; + + try + { + // A pass queued before Stop() must not start once teardown has begun. + if (_stopping && !flush) + return; + + WhisperProcessor? processor = _processor; + WhisperVadProcessor? vad = _vadProcessor; + if (processor is null || vad is null || _channels.Count == 0) + return; + + // Snapshot without clearing — audio keeps arriving while we work; we trim precisely later. + List perChannelSamples = new(_channels.Count); + bool anyData = false; + foreach (CaptureChannel channel in _channels) + { + byte[] raw; + lock (channel.BufferLock) + { + if (channel.PcmBuffer.Length == 0) + { + perChannelSamples.Add(Array.Empty()); + continue; + } + raw = channel.PcmBuffer.ToArray(); + } + anyData = true; + perChannelSamples.Add(AudioTranscriptionUtilities.ConvertToSamples16kMono(raw, raw.Length, channel.SourceFormat)); + } + if (!anyData) + return; + + float[] samples = MixChannels(perChannelSamples); + double totalSeconds = samples.Length / (double)SampleRate; + if (!flush && totalSeconds < MinAudioSeconds) + return; + + IReadOnlyList speech = await vad.DetectSpeechAsync(samples).ConfigureAwait(false); + if (speech.Count == 0) + { + // Only silence so far: keep just the tail so the buffer doesn't grow during quiet. + TrimAllChannelsFront(totalSeconds - 0.5); + return; + } + + bool forced = flush || totalSeconds >= MaxUtteranceSeconds; + double cutSeconds = 0; + StringBuilder phrase = new(); + + for (int i = 0; i < speech.Count; i++) + { + VadSegmentData seg = speech[i]; + bool complete = forced || (totalSeconds - seg.End.TotalSeconds) >= CompletionSilenceSeconds; + if (!complete) + break; // speech still in progress: leave this and later regions buffered + + int start = Math.Max(0, (int)(seg.Start.TotalSeconds * SampleRate) - SampleRate / 20); + int end = Math.Min(samples.Length, (int)(seg.End.TotalSeconds * SampleRate) + SampleRate / 20); + cutSeconds = seg.End.TotalSeconds; + if (end <= start) + continue; + + ReadOnlyMemory slice = new(samples, start, end - start); + StringBuilder segmentText = new(); + await foreach (SegmentData s in processor.ProcessAsync(slice, CancellationToken.None).ConfigureAwait(false)) + segmentText.Append(s.Text); + + string cleaned = AudioTranscriptionUtilities.CleanTranscript(segmentText.ToString()); + if (cleaned.Length > 0) + { + if (phrase.Length > 0) + phrase.Append(' '); + phrase.Append(cleaned); + } + } + + if (cutSeconds > 0) + TrimAllChannelsFront(cutSeconds); + + if (phrase.Length > 0) + PhraseRecognized?.Invoke(this, phrase.ToString()); + } + catch (Exception ex) + { + AudioDebugLog.Write($"LiveAudioTranscriber: chunk processing error: {ex.Message}"); + } + finally + { + _processingGate.Release(); + } + } + + /// + /// Drops the first of buffered audio (rounded to a whole sample frame) + /// from every capture channel. Only the consumed prefix is removed, so audio captured during + /// processing is preserved. Each channel is trimmed using its own format, but by the same real-time + /// duration, so mixed channels stay in sync. + /// + private void TrimAllChannelsFront(double seconds) + { + if (seconds <= 0) + return; + + foreach (CaptureChannel channel in _channels) + { + int bytesToRemove = (int)(seconds * channel.SourceFormat.AverageBytesPerSecond); + int blockAlign = channel.SourceFormat.BlockAlign; + if (blockAlign > 0) + bytesToRemove -= bytesToRemove % blockAlign; + if (bytesToRemove <= 0) + continue; + + lock (channel.BufferLock) + { + byte[] current = channel.PcmBuffer.ToArray(); + int remove = Math.Min(bytesToRemove, current.Length); + channel.PcmBuffer.SetLength(0); + if (current.Length > remove) + channel.PcmBuffer.Write(current, remove, current.Length - remove); + } + } + } + + private void Cleanup() + { + if (_chunkTimer is not null) + { + _chunkTimer.Stop(); + _chunkTimer.Dispose(); + _chunkTimer = null; + } + + foreach (CaptureChannel channel in _channels) + channel.Dispose(); + _channels.Clear(); + + if (_processor is not null) + { + try { _processor.Dispose(); } catch { } + _processor = null; + } + + if (_vadProcessor is not null) + { + try { _vadProcessor.Dispose(); } catch { } + _vadProcessor = null; + } + + // Return the factory only after the processor built from it is disposed. The VAD factory is + // shared and owned by AudioTranscriptionUtilities; nothing to release there. + _factoryLease?.Dispose(); + _factoryLease = null; + } + + public void Dispose() + { + if (_disposed) + return; + + _disposed = true; + Stop(); + } +} diff --git a/Text-Grab/Utilities/CaptureLanguageUtilities.cs b/Text-Grab/Utilities/CaptureLanguageUtilities.cs index c5e96756..bed65194 100644 --- a/Text-Grab/Utilities/CaptureLanguageUtilities.cs +++ b/Text-Grab/Utilities/CaptureLanguageUtilities.cs @@ -4,37 +4,33 @@ using System.Threading.Tasks; using Text_Grab.Interfaces; using Text_Grab.Models; -using Windows.Media.Ocr; namespace Text_Grab.Utilities; internal static class CaptureLanguageUtilities { + /// + /// Builds the language list for capture menus. The UI-automation / Windows AI / plain-OCR + /// portion comes from , which caches those + /// checks (each of which can be a genuinely slow WinRT/WinAppSDK probe) instead of redoing + /// them on every call — this used to duplicate that work uncached, which is what made menus + /// like EditTextWindow's "Capture" menu slow to open, especially in new windows. + /// public static async Task> GetCaptureLanguagesAsync(bool includeTesseract) { - List languages = []; - - if (AppUtilities.TextGrabSettings.UiAutomationEnabled) - languages.Add(new UiAutomationLang()); - - if (WindowsAiUtilities.CanDeviceUseWinAI()) - languages.Add(new WindowsAiLang()); - - if (AppUtilities.TextGrabSettings.WindowsAiDescriptionEnabled - && WindowsAiUtilities.CanDeviceDescribeImagesWithWinAI()) - { - languages.Add(new WindowsAiDescriptionLang()); - } + List languages = [.. LanguageUtilities.GetAllLanguages()]; if (includeTesseract && AppUtilities.TextGrabSettings.UseTesseract && TesseractHelper.CanLocateTesseractExe()) { - languages.AddRange(await TesseractHelper.TesseractLanguages()); - } + List tesseractLanguages = await TesseractHelper.TesseractLanguages(); - foreach (Windows.Globalization.Language language in OcrEngine.AvailableRecognizerLanguages) - languages.Add(new GlobalLang(language)); + // Insert before the plain OCR languages (GlobalLang), after the UiAutomation/WindowsAi + // pseudo-languages, to preserve the original ordering. + int insertIndex = languages.FindIndex(l => l is GlobalLang); + languages.InsertRange(insertIndex < 0 ? languages.Count : insertIndex, tesseractLanguages); + } return languages; } diff --git a/Text-Grab/Utilities/GrabTemplateManager.cs b/Text-Grab/Utilities/GrabTemplateManager.cs index 500e9f3b..7d927b1f 100644 --- a/Text-Grab/Utilities/GrabTemplateManager.cs +++ b/Text-Grab/Utilities/GrabTemplateManager.cs @@ -17,9 +17,9 @@ namespace Text_Grab.Utilities; /// the transition release. Pattern follows . /// /// -/// TODO: This class has no thread-safety guards. All current callers are UI-thread -/// methods so this is safe today, but if templates are ever read/written from -/// background threads a lock (like SettingsService._managedJsonLock) should be added. +/// Only the in-memory cache is guarded (see _cacheLock). The underlying storage — the +/// settings string and the JSON file — is not, so concurrent writes from background threads would +/// still race; all current callers are UI-thread methods. /// public static class GrabTemplateManager { @@ -33,12 +33,41 @@ public static class GrabTemplateManager private const string TemplatesFileName = "GrabTemplates.json"; + // In-memory cache of the resolved template list. GetAllTemplates() used to hit disk + // (and potentially Settings.Save(), which is slow) on every call — including every time + // a menu that lists templates (e.g. the EditTextWindow "Capture" menu) was opened. Cached + // here instead and only refreshed by writes that go through this class. + private static List? _cachedTemplates; + + // Guards _cachedTemplates: it is process-wide static state and `??=` is not atomic, so two + // threads could each load and publish a different list. + private static readonly object _cacheLock = new(); + // Allow tests to override the file path. // TODO: If more test seams are needed, consider consolidating these into a small // options/config object instead of individual static properties. - internal static string? TestFilePath { get; set; } + private static string? _testFilePath; + internal static string? TestFilePath + { + get => _testFilePath; + set { _testFilePath = value; InvalidateCache(); } + } + internal static string? TestImagesFolderPath { get; set; } - internal static bool? TestPreferFileBackedMode { get; set; } + + private static bool? _testPreferFileBackedMode; + internal static bool? TestPreferFileBackedMode + { + get => _testPreferFileBackedMode; + set { _testPreferFileBackedMode = value; InvalidateCache(); } + } + + /// Drops the in-memory template cache so the next read re-resolves from disk/settings. + internal static void InvalidateCache() + { + lock (_cacheLock) + _cachedTemplates = null; + } private static bool PreferFileBackedTemplates => TestPreferFileBackedMode ?? AppUtilities.TextGrabSettingsService.IsFileBackedManagedSettingsEnabled; @@ -131,8 +160,27 @@ public static string GetTemplateImagesFolder() // ── Read ────────────────────────────────────────────────────────────────── - /// Returns all saved templates, or an empty list if none exist. + /// + /// Returns all saved templates, or an empty list if none exist. Resolved once per process + /// (or since the last write/) and cached; callers get a fresh list + /// of fresh copies each time, so neither a structural edit (add/remove) nor an edit to a template + /// itself can leak into another caller's list — or into the cache — before being persisted. + /// public static List GetAllTemplates() + { + lock (_cacheLock) + { + _cachedTemplates ??= LoadTemplatesFromStorage(); + return [.. _cachedTemplates.Select(CloneTemplate)]; + } + } + + /// Deep copy via the same JSON shape these are persisted in. + private static GrabTemplate CloneTemplate(GrabTemplate template) => + JsonSerializer.Deserialize(JsonSerializer.Serialize(template, JsonOptions), JsonOptions) + ?? template; + + private static List LoadTemplatesFromStorage() { try { @@ -173,6 +221,10 @@ public static void SaveTemplates(List templates) { string json = JsonSerializer.Serialize(templates, JsonOptions); SaveTemplatesJson(json); + + // Copies here too: the caller keeps its list and may go on editing it. + lock (_cacheLock) + _cachedTemplates = [.. templates.Select(CloneTemplate)]; } internal static string GetTemplatesJsonForExport() @@ -219,10 +271,7 @@ public static void DeleteTemplate(string id) if (original is null) return null; - string json = JsonSerializer.Serialize(original, JsonOptions); - GrabTemplate? copy = JsonSerializer.Deserialize(json, JsonOptions); - if (copy is null) - return null; + GrabTemplate copy = CloneTemplate(original); copy.Id = Guid.NewGuid().ToString(); copy.Name = $"{original.Name} (copy)"; diff --git a/Text-Grab/Utilities/LanguageHeuristics.cs b/Text-Grab/Utilities/LanguageHeuristics.cs new file mode 100644 index 00000000..98385a7b --- /dev/null +++ b/Text-Grab/Utilities/LanguageHeuristics.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Text_Grab.Utilities; + +/// +/// Cheap script-based guesses about what language a string is already in, used to skip +/// translation work that would be a no-op. +/// +internal static class LanguageHeuristics +{ + + // Language code mapping for quick lookup + private static readonly Dictionary LanguageCodeMap = new(StringComparer.OrdinalIgnoreCase) + { + { "English", "en" }, + { "Spanish", "es" }, + { "French", "fr" }, + { "German", "de" }, + { "Italian", "it" }, + { "Portuguese", "pt" }, + { "Russian", "ru" }, + { "Japanese", "ja" }, + { "Chinese (Simplified)", "zh-Hans" }, + { "Chinese", "zh-Hans" }, + { "Korean", "ko" }, + { "Arabic", "ar" }, + { "Hindi", "hi" }, + }; + + /// + /// Quickly detects if text is likely in the target language using simple heuristics. + /// This is a fast check to avoid expensive translation calls. + /// + /// Text to analyze + /// Target language name (e.g., "English", "Spanish") + /// True if text appears to already be in target language + internal static bool IsLikelyInTargetLanguage(string text, string targetLanguage) + { + if (string.IsNullOrWhiteSpace(text) || text.Length < 3) + return false; + + // Get language code for target + if (!LanguageCodeMap.TryGetValue(targetLanguage, out string? targetCode)) + return false; // Unknown language, proceed with translation + + // Character range detection + bool hasCJK = text.Any(c => c is >= (char)0x4E00 and <= (char)0x9FFF or // CJK Unified Ideographs + >= (char)0x3040 and <= (char)0x309F or // Hiragana + >= (char)0x30A0 and <= (char)0x30FF or // Katakana + >= (char)0xAC00 and <= (char)0xD7AF); // Hangul + + bool hasArabic = text.Any(c => c is >= (char)0x0600 and <= (char)0x06FF); + bool hasCyrillic = text.Any(c => c is >= (char)0x0400 and <= (char)0x04FF); + bool hasDevanagari = text.Any(c => c is >= (char)0x0900 and <= (char)0x097F); + bool hasLatin = text.Any(c => c is >= 'A' and <= 'Z' or >= 'a' and <= 'z'); + + // Quick script-based checks + switch (targetCode) + { + case "en": + case "es": + case "fr": + case "de": + case "it": + case "pt": + // Latin script languages - if mostly CJK/Arabic/Cyrillic, definitely not in target + if (hasCJK || hasArabic || hasCyrillic || hasDevanagari) + return false; + // If has Latin characters, might be in target language + if (hasLatin && text.Length > 10 && targetCode == "en") + { + // Check for common English words as additional heuristic + string lowerText = text.ToLowerInvariant(); + string[] commonEnglishWords = [" the ", " and ", " or ", " is ", " are ", " was ", " were ", " in ", " on ", " at ", " to ", " of ", " for ", " with "]; + int englishWordCount = commonEnglishWords.Count(w => lowerText.Contains(w)); + // If text contains multiple common English words, likely already English + if (englishWordCount >= 2) + return true; + } + break; + + case "ru": + // Russian - should have Cyrillic + return hasCyrillic && !hasCJK && !hasArabic; + + case "ja": + // Japanese - should have Hiragana/Katakana/Kanji + return hasCJK && !hasArabic && !hasCyrillic; + + case "zh-Hans": + // Chinese - should have CJK + return hasCJK && !hasArabic && !hasCyrillic; + + case "ko": + // Korean - should have Hangul + return text.Any(c => c is >= (char)0xAC00 and <= (char)0xD7AF) && !hasArabic && !hasCyrillic; + + case "ar": + // Arabic - should have Arabic script + return hasArabic && !hasCJK && !hasCyrillic; + + case "hi": + // Hindi - should have Devanagari + return hasDevanagari && !hasCJK && !hasArabic; + } + + return false; + } + +} diff --git a/Text-Grab/Utilities/LimitedAccessFeatureUtilities.cs b/Text-Grab/Utilities/LimitedAccessFeatureUtilities.cs new file mode 100644 index 00000000..a1336980 --- /dev/null +++ b/Text-Grab/Utilities/LimitedAccessFeatureUtilities.cs @@ -0,0 +1,156 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Reflection; +using Windows.ApplicationModel; + +namespace Text_Grab.Utilities; + +/// +/// Unlocks the Windows AI language model, which Microsoft ships as a Limited Access Feature. +/// +/// An app must call with a token issued for +/// its own publisher ID before Microsoft.Windows.AI.Text.LanguageModel will do anything; +/// without it every call fails with "Access is denied. Limited Access Feature is not available: +/// com.microsoft.windows.ai.languagemodel." +/// +/// Tokens are requested from Microsoft at https://aka.ms/laffeatures and must not be committed to +/// source control, so the token and publisher ID are read at runtime from (in order): +/// 1. AssemblyMetadata baked in at build time — set the MSBuild properties +/// LafToken and LafPublisherId (see Text-Grab.csproj). +/// 2. The LAF_TOKEN and LAF_PUBLISHER_ID environment variables, for local development. +/// +/// This mirrors how microsoft/ai-dev-gallery handles the same feature. +/// +internal static class LimitedAccessFeatureUtilities +{ + internal const string LanguageModelFeatureId = "com.microsoft.windows.ai.languagemodel"; + + private const string TokenKey = "LAF_TOKEN"; + private const string PublisherIdKey = "LAF_PUBLISHER_ID"; + + /// The unlock is process-wide and only needs to happen once. + private static (bool Unlocked, string? Reason)? _languageModelUnlock; + + private static readonly Lock _unlockLock = new(); + + /// + /// Attempts to unlock the language model feature, returning why it is unavailable when it is. + /// The result is cached until forgets a failed one. + /// + internal static (bool Unlocked, string? Reason) TryUnlockLanguageModel() + { + lock (_unlockLock) + { + _languageModelUnlock ??= UnlockLanguageModel(); + return _languageModelUnlock.Value; + } + } + + /// + /// Forgets a cached unlock *failure* so the next request asks Windows again. Called when the + /// Windows AI runtime is restarted after a dropped connection: TryUnlockFeature talks to that + /// runtime too, so a failure cached while it was away would keep the feature off for the life + /// of the process. A successful unlock is kept, since it never needs redoing. + /// + internal static void ResetUnlockCache() + { + lock (_unlockLock) + { + if (_languageModelUnlock is { Unlocked: false }) + _languageModelUnlock = null; + } + } + + private static (bool Unlocked, string? Reason) UnlockLanguageModel() + { + string publisherId = GetSetting(PublisherIdKey); + + // The publisher ID is the hash half of the package family name. Falling back to it means a + // build with only a token configured still forms the correct usage string. + if (string.IsNullOrWhiteSpace(publisherId)) + publisherId = GetPublisherHash(); + + string token = GetSetting(TokenKey); + string usage = $"{publisherId} has registered their use of {LanguageModelFeatureId} with Microsoft and agrees to the terms of use."; + + try + { + LimitedAccessFeatureRequestResult result = + LimitedAccessFeatures.TryUnlockFeature(LanguageModelFeatureId, token, usage); + + if (result.Status is LimitedAccessFeatureStatus.Available or LimitedAccessFeatureStatus.AvailableWithoutToken) + { + Debug.WriteLine($"Windows AI language model unlocked: {result.Status}"); + return (true, null); + } + + Debug.WriteLine($"Windows AI language model not unlocked: {result.Status}"); + return (false, DescribeFailure(result.Status, token, publisherId)); + } + catch (Exception ex) + { + Debug.WriteLine($"TryUnlockFeature failed: {ex.Message}"); + return (false, $"Windows could not unlock the AI language model feature: {ex.Message}"); + } + } + + private static string DescribeFailure(LimitedAccessFeatureStatus status, string token, string publisherId) + { + string statusText = status switch + { + LimitedAccessFeatureStatus.Unavailable => "Windows reports the feature as unavailable", + LimitedAccessFeatureStatus.Unknown => "Windows does not recognize this app as registered for the feature", + _ => $"Windows returned {status}", + }; + + string tokenText = string.IsNullOrWhiteSpace(token) + ? "This build of Text-Grab has no unlock token configured." + : "The configured unlock token was rejected."; + + return $""" + Windows AI's language model is a Limited Access Feature and must be unlocked before it can be used. + + {tokenText} {statusText}. + + A token has to be requested from Microsoft at https://aka.ms/laffeatures for publisher ID '{publisherId}', then supplied at build time via the LafToken and LafPublisherId MSBuild properties (or the LAF_TOKEN and LAF_PUBLISHER_ID environment variables). + """; + } + + /// + /// Reads a value from build-time assembly metadata, falling back to an environment variable. + /// + private static string GetSetting(string key) + { + foreach (AssemblyMetadataAttribute attribute in typeof(LimitedAccessFeatureUtilities).Assembly + .GetCustomAttributes()) + { + if (string.Equals(attribute.Key, key, StringComparison.Ordinal) && !string.IsNullOrWhiteSpace(attribute.Value)) + return attribute.Value; + } + + return Environment.GetEnvironmentVariable(key) ?? string.Empty; + } + + /// + /// The publisher hash from the package family name ("Name_hash" -> "hash"), which is the + /// publisher ID a Limited Access Feature token is issued against. + /// + internal static string GetPublisherHash() + { + try + { + string familyName = Package.Current.Id.FamilyName; + if (string.IsNullOrWhiteSpace(familyName)) + return string.Empty; + + string[] parts = familyName.Split('_'); + return parts.Length >= 2 ? parts[1] : string.Empty; + } + catch (Exception ex) + { + Debug.WriteLine($"Could not read the package family name: {ex.Message}"); + return string.Empty; + } + } +} diff --git a/Text-Grab/Utilities/NotificationUtilities.cs b/Text-Grab/Utilities/NotificationUtilities.cs index cab4437b..5331eaf5 100644 --- a/Text-Grab/Utilities/NotificationUtilities.cs +++ b/Text-Grab/Utilities/NotificationUtilities.cs @@ -61,4 +61,12 @@ internal static void ShowToast(string copiedText) toast.Show(); } + + internal static void ShowTranscriptionCompleteToast(string fileDescription) + { + new ToastContentBuilder() + .AddText("Text Grab") + .AddText($"Transcription complete: {fileDescription}") + .Show(); + } } diff --git a/Text-Grab/Utilities/NotifyIconUtilities.cs b/Text-Grab/Utilities/NotifyIconUtilities.cs index 9bc10ba8..eadf9cac 100644 --- a/Text-Grab/Utilities/NotifyIconUtilities.cs +++ b/Text-Grab/Utilities/NotifyIconUtilities.cs @@ -232,6 +232,20 @@ private static NotifyIconWindow CreateNotifyIconWindow() return notifyIconWindow; } + public static void RefreshTrayIconStyle() + { + // Windows theme changes are observed via a registry watcher that raises its event on a + // background thread, but NotifyIcon.Icon is a DependencyProperty owned by the UI thread. + System.Windows.Threading.Dispatcher dispatcher = Application.Current.Dispatcher; + if (!dispatcher.CheckAccess()) + { + dispatcher.BeginInvoke(RefreshTrayIconStyle); + return; + } + + GetExistingNotifyIconWindow()?.ApplyTrayIconStyle(); + } + private static NotifyIconWindow? GetExistingNotifyIconWindow() { return Application.Current.Windows.OfType().FirstOrDefault(); diff --git a/Text-Grab/Utilities/PostGrabActionManager.cs b/Text-Grab/Utilities/PostGrabActionManager.cs index 723dc416..00475c53 100644 --- a/Text-Grab/Utilities/PostGrabActionManager.cs +++ b/Text-Grab/Utilities/PostGrabActionManager.cs @@ -220,11 +220,23 @@ public static async Task ExecutePostGrabAction(ButtonInfo action, PostGr break; case "Translate_Click": - if (WindowsAiUtilities.CanDeviceUseWinAI()) - { - string systemLanguage = LanguageUtilities.GetSystemLanguageForTranslation(); - result = await WindowsAiUtilities.TranslateText(text, systemLanguage); - } + string systemLanguage = LanguageUtilities.GetSystemLanguageForTranslation(); + TranslationResult translation = await WinAiTranslator.TranslateAsync(text, systemLanguage); + result = translation.Text; + + // The grab already happened, so a real failure must be reported or the user just + // sees untranslated text with no explanation. NotNeeded (the text is already in the + // target language) and Unavailable (no Windows AI on this device) are the normal + // case for most users on every single grab, and the text is unchanged either way — + // a modal there would fire after every grab and tell the user nothing. + if (!translation.Succeeded + && translation.Failure is not TranslationFailure.NotNeeded and not TranslationFailure.Unavailable) + await new Wpf.Ui.Controls.MessageBox + { + Title = "Translation Failed", + Content = translation.Message ?? "The text could not be translated.", + CloseButtonText = "OK" + }.ShowDialogAsync(); break; case "ApplyTemplate_Click": diff --git a/Text-Grab/Utilities/WinAiLanguageModel.cs b/Text-Grab/Utilities/WinAiLanguageModel.cs new file mode 100644 index 00000000..fe7118f5 --- /dev/null +++ b/Text-Grab/Utilities/WinAiLanguageModel.cs @@ -0,0 +1,606 @@ +using Microsoft.Windows.AI; +using Microsoft.Windows.AI.ContentSafety; +using Microsoft.Windows.AI.Text; +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Windows.Foundation; + +namespace Text_Grab.Utilities; + +/// Why a request to the Windows AI language model did not produce text. +internal enum WinAiFailure +{ + /// The model answered. + None, + + /// This device cannot run the Windows AI language model at all. + Unavailable, + + /// The model exists but could not be prepared or created. + ModelNotReady, + + /// + /// The connection to the out-of-process Windows AI runtime dropped, which surfaces as + /// "The RPC server is unavailable". Recoverable by restarting the model. + /// + Disconnected, + + /// The prompt did not fit the model's context. + PromptTooLong, + + /// Content moderation or policy rejected the prompt or the response. + Blocked, + + /// The model reported an error, or returned nothing usable. + ModelError, +} + +/// The result of one generation: either text, or a reason there is none. +internal readonly record struct WinAiGenerationResult(string? Text, WinAiFailure Failure, string? Message) +{ + internal bool Succeeded => Failure is WinAiFailure.None && Text is not null; + + internal static WinAiGenerationResult Ok(string text) => new(text, WinAiFailure.None, null); + + internal static WinAiGenerationResult Failed(WinAiFailure failure, string message) => + new(null, failure, message); +} + +/// +/// Shared access to the Windows AI Foundry (Phi Silica), following the +/// pattern used by microsoft/ai-dev-gallery's PhiSilicaClient. +/// +/// Everything in Text-Grab that prompts the language model — translation, meeting notes, regex +/// extraction — goes through here so they all get the same three things: +/// 1. The Limited Access Feature unlock, checked once and reported with a message a user can act +/// on rather than an "Access is denied" exception from deep inside the model call. +/// 2. One created and reused across features; only the lightweight +/// per-request is rebuilt, so turns never accumulate and +/// the second feature to run does not pay the model creation cost again. +/// 3. Prompting the model directly through GenerateResponseAsync with a purpose-built system +/// prompt, instead of bending a skill such as TextRewriter into the job (whose own system +/// prompt fights the instruction and leaves instruction echoes in the output). +/// 4. Recovery from a dropped connection to the Windows AI runtime: the model runs out of +/// process, so when that process is recycled, updated or crashes, every object this process +/// holds becomes a dead proxy and keeps failing with "The RPC server is unavailable" until +/// it is thrown away and remade. That is done here rather than by restarting Text-Grab. +/// +/// Every failure path returns a and a human-readable message so callers +/// can tell the user why nothing came back. +/// +internal static class WinAiLanguageModel +{ + private static LanguageModel? _languageModel; + private static readonly SemaphoreSlim _modelLock = new(1, 1); + + // Phi Silica serves one generation at a time; queueing here keeps concurrent callers from + // interleaving requests on the shared model, which previously showed up as long stalls. + private static readonly SemaphoreSlim _inferenceLock = new(1, 1); + private static bool _disposed; + + #region availability + + /// + /// Whether this device can run the Windows AI language model, and why not when it cannot. + /// Unlike the OCR checks this asks directly rather + /// than assuming ARM64, so Intel/AMD Copilot+ PCs are included and unsupported hardware is + /// excluded properly. + /// + internal static (bool Available, string? Reason) CheckAvailability() + { + if (!AppUtilities.IsPackaged()) + return (false, "Windows AI is only available when Text-Grab runs as an installed (packaged) app."); + + if (OSInterop.IsWindows10()) + return (false, "The on-device Windows AI language model requires Windows 11."); + + try + { + AIFeatureReadyState readyState = LanguageModel.GetReadyState(); + + if (readyState is AIFeatureReadyState.NotSupportedOnCurrentSystem) + return (false, "This device does not support the on-device Windows AI language model. It requires a Copilot+ PC."); + + if (readyState is AIFeatureReadyState.DisabledByUser) + return (false, "The Windows AI language model is turned off in Windows Settings."); + + // Microsoft ships the language model as a Limited Access Feature, so it must be + // unlocked before any call to it will succeed. Do it here, ahead of CreateAsync and + // CreateContext, so the failure is reported once and clearly. + (bool unlocked, string? unlockReason) = LimitedAccessFeatureUtilities.TryUnlockLanguageModel(); + + return unlocked ? (true, null) : (false, unlockReason); + } + catch (Exception ex) + { + Debug.WriteLine($"LanguageModel.GetReadyState failed: {ex.Message}"); + return (false, $"Windows AI could not be reached on this device: {ex.Message}"); + } + } + + /// True when this device can run the Windows AI language model. + internal static bool IsAvailable() => CheckAvailability().Available; + + /// Returns the shared model, creating (and preparing, if needed) it on first use. + /// + /// Private on purpose: a caller that held on to the returned model would keep using it after a + /// dropped connection forced a restart. Features call to check + /// the model can be started, then , which always uses the current one. + /// + private static async Task<(LanguageModel? Model, string? Error)> GetModelAsync(CancellationToken cancellationToken) + { + if (_languageModel is not null) + return (_languageModel, null); + + await _modelLock.WaitAsync(cancellationToken); + try + { + if (_languageModel is not null) + return (_languageModel, null); + + (bool available, string? reason) = CheckAvailability(); + if (!available) + return (null, reason); + + if (LanguageModel.GetReadyState() is AIFeatureReadyState.NotReady) + { + // First run may download the model; the token lets the user back out of the wait. + AIFeatureReadyResult readyResult = await LanguageModel.EnsureReadyAsync().AsTask(cancellationToken); + if (readyResult.Status != AIFeatureReadyResultState.Success) + { + string detail = readyResult.ExtendedError?.Message ?? readyResult.Status.ToString(); + return (null, $"The Windows AI language model could not be prepared ({detail}). " + + "It may still be downloading — try again in a few minutes."); + } + } + + _languageModel = await LanguageModel.CreateAsync().AsTask(cancellationToken); + return (_languageModel, null); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + Debug.WriteLine($"LanguageModel creation failed: {ex.Message}"); + return (null, $"The Windows AI language model could not be started: {ex.Message}"); + } + finally + { + _modelLock.Release(); + } + } + + /// + /// Makes sure the shared model exists and is ready, without handing it out. Features call this + /// up front so "the model could not be started" is reported before any work begins. + /// + internal static async Task<(bool Ready, string? Error)> EnsureModelAsync(CancellationToken cancellationToken) + { + (LanguageModel? model, string? error) = await GetModelAsync(cancellationToken); + return (model is not null, error); + } + + /// + /// HRESULTs that mean this process is holding a proxy to a Windows AI runtime that is no longer + /// there. The model runs out of process, so a service restart, a model update or a crash leaves + /// every object created before it permanently dead: the next call comes back as "The RPC server + /// is unavailable" (0x800706BA) and every call after it does too, until new objects are made. + /// + private static readonly int[] _connectionLostHResults = + [ + unchecked((int)0x800706BA), // RPC_S_SERVER_UNAVAILABLE — "The RPC server is unavailable." + unchecked((int)0x800706BB), // RPC_S_SERVER_TOO_BUSY + unchecked((int)0x800706BE), // RPC_S_CALL_FAILED + unchecked((int)0x800706BF), // RPC_S_CALL_FAILED_DNE + unchecked((int)0x800706B5), // RPC_S_UNKNOWN_IF + unchecked((int)0x80010108), // RPC_E_DISCONNECTED — the object invoked has disconnected + unchecked((int)0x80010105), // RPC_E_SERVERFAULT + unchecked((int)0x800401FD), // CO_E_OBJNOTCONNECTED + ]; + + /// Whether means the Windows AI runtime went away. + internal static bool IsConnectionLost(Exception? exception) + { + for (Exception? ex = exception; ex is not null; ex = ex.InnerException) + if (Array.IndexOf(_connectionLostHResults, ex.HResult) >= 0) + return true; + + return false; + } + + /// + /// Throws away the shared model, and any cached reason it could not be unlocked, so the next + /// request builds a fresh connection to the Windows AI runtime. and + /// do this by themselves when a request finds the runtime gone; + /// call it directly to offer the user a manual "try again" without restarting Text-Grab. + /// + internal static async Task RestartModelAsync(CancellationToken cancellationToken = default) + { + if (_disposed) + return; + + await ReleaseModelAsync(cancellationToken); + + // A failure to unlock the Limited Access Feature is cached for the life of the process, so + // forget it too: a transient failure there would otherwise fail the retry before it starts. + LimitedAccessFeatureUtilities.ResetUnlockCache(); + } + + /// + /// Drops the cached language model to free the memory it holds. The next request recreates it, + /// so call this when a feature is switched off rather than between requests. + /// + /// + /// Takes the same lock creates the model under, so a release from one + /// window (the GrabFrame translate toggle, its cleanup) cannot dispose the model while another + /// window is still building it. + /// + internal static async Task ReleaseModelAsync(CancellationToken cancellationToken = default) + { + if (_disposed) + return; + + await _modelLock.WaitAsync(cancellationToken); + try + { + _languageModel?.Dispose(); + } + catch (Exception ex) + { + // Disposing a dead proxy can throw; the reference is dropped either way. + Debug.WriteLine($"Disposing the language model failed: {ex.Message}"); + } + finally + { + _languageModel = null; + _modelLock.Release(); + } + } + + /// + /// Fire-and-forget for callers that cannot await (window cleanup, + /// a toggle handler). The model is freed once any in-flight creation finishes. + /// + internal static void ReleaseModel() => _ = ReleaseModelAsync(); + + /// Releases the shared language model. Call once during application shutdown. + internal static void Cleanup() + { + if (_disposed) + return; + + _disposed = true; + + // The process is exiting, so dispose the model directly instead of waiting on the model lock. + try + { + _languageModel?.Dispose(); + } + catch (Exception ex) + { + Debug.WriteLine($"Disposing the language model failed: {ex.Message}"); + } + + _languageModel = null; + + // The semaphores are deliberately left undisposed: they hold nothing worth reclaiming at + // exit, and disposing them throws ObjectDisposedException into any pending WaitAsync — after + // which InferenceLease.Dispose() skips its Release() and the queue is wedged for good. + } + + #endregion availability + + #region generation + + /// + /// Runs a single prompt against the shared model, taking care of availability, model creation + /// and the inference queue. Callers issuing several related requests should take a lease from + /// and call themselves, so + /// their requests are not interleaved with another feature's. + /// + internal static async Task PromptAsync( + string systemPrompt, + string prompt, + float temperature = 0.2f, + Action? onPartial = null, + CancellationToken cancellationToken = default) + { + (bool available, string? reason) = CheckAvailability(); + if (!available) + return WinAiGenerationResult.Failed( + WinAiFailure.Unavailable, reason ?? "Windows AI is not available on this device."); + + try + { + using IDisposable lease = await AcquireInferenceAsync(cancellationToken); + return await GenerateAsync(systemPrompt, prompt, temperature, onPartial, cancellationToken); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + Debug.WriteLine($"Language model request failed: {ex.Message}"); + return WinAiGenerationResult.Failed(WinAiFailure.ModelError, $"The language model failed: {ex.Message}"); + } + } + + /// + /// Waits for exclusive use of the shared model. Dispose the returned lease to let the next + /// caller in. + /// + internal static async Task AcquireInferenceAsync(CancellationToken cancellationToken) + { + await _inferenceLock.WaitAsync(cancellationToken); + return new InferenceLease(); + } + + private sealed class InferenceLease : IDisposable + { + private bool _released; + + public void Dispose() + { + if (_released || _disposed) + return; + + _released = true; + _inferenceLock.Release(); + } + } + + /// + /// Runs one generation against the shared model. The caller holds the inference lease. + /// + /// When the request fails because the connection to the Windows AI runtime dropped — the + /// service was restarted, updated or recycled while Text-Grab held a proxy to it, reported as + /// "The RPC server is unavailable" — the model is thrown away, remade, and the request is tried + /// once more. Without that, every later request fails on the same dead proxy and the feature + /// only comes back when Text-Grab is restarted. + /// + /// Receives generated text as it streams in, for live UI updates. + internal static async Task GenerateAsync( + string systemPrompt, + string prompt, + float temperature, + Action? onDelta, + CancellationToken cancellationToken) + { + (LanguageModel? model, string? error) = await GetModelAsync(cancellationToken); + if (model is null) + return WinAiGenerationResult.Failed( + WinAiFailure.ModelNotReady, error ?? "The Windows AI language model could not be started."); + + // Records whether the failed attempt already pushed text at the UI, so the retry does not + // stream a second copy of the response into it. + bool alreadyStreamed = false; + Action? firstDelta = onDelta is null + ? null + : delta => { alreadyStreamed = true; onDelta(delta); }; + + WinAiGenerationResult result = + await GenerateOnceAsync(model, systemPrompt, prompt, temperature, firstDelta, cancellationToken); + + if (result.Failure is not WinAiFailure.Disconnected) + return result; + + Debug.WriteLine($"Windows AI connection lost, restarting the language model: {result.Message}"); + await RestartModelAsync(cancellationToken); + + (model, error) = await GetModelAsync(cancellationToken); + if (model is null) + return WinAiGenerationResult.Failed( + WinAiFailure.Disconnected, + $"Windows AI stopped responding and could not be restarted: {error ?? result.Message}"); + + WinAiGenerationResult retry = await GenerateOnceAsync( + model, systemPrompt, prompt, temperature, alreadyStreamed ? null : onDelta, cancellationToken); + + return retry.Failure is WinAiFailure.Disconnected + ? WinAiGenerationResult.Failed( + WinAiFailure.Disconnected, + "Windows AI stopped responding, and restarting the model did not bring it back. " + + "Try again in a moment, or restart Text-Grab.") + : retry; + } + + /// + /// Runs against the shared model, for the WinAppSDK text skills + /// (summarize, rewrite, text-to-table) which take a of their own + /// instead of going through . Takes the inference lease, and + /// restarts the model and tries once more when the Windows AI runtime connection has dropped. + /// + /// The skill's result, or null and a message saying why there is none. + internal static async Task<(T? Value, string? Error)> RunWithModelAsync( + Func> work, + CancellationToken cancellationToken = default) where T : class + { + (bool available, string? reason) = CheckAvailability(); + if (!available) + return (null, reason ?? "Windows AI is not available on this device."); + + try + { + using IDisposable lease = await AcquireInferenceAsync(cancellationToken); + + (LanguageModel? model, string? error) = await GetModelAsync(cancellationToken); + if (model is null) + return (null, error ?? "The Windows AI language model could not be started."); + + try + { + return (await work(model, cancellationToken), null); + } + catch (Exception ex) when (IsConnectionLost(ex)) + { + Debug.WriteLine($"Windows AI connection lost, restarting the language model: {ex.Message}"); + await RestartModelAsync(cancellationToken); + + (model, error) = await GetModelAsync(cancellationToken); + if (model is null) + return (null, $"Windows AI stopped responding and could not be restarted: {error ?? ex.Message}"); + + return (await work(model, cancellationToken), null); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + Debug.WriteLine($"Language model request failed: {ex.Message}"); + + return (null, IsConnectionLost(ex) + ? "Windows AI stopped responding, and restarting the model did not bring it back. " + + "Try again in a moment, or restart Text-Grab." + : ex.Message); + } + } + + /// One attempt at a generation, with no recovery of its own. + private static async Task GenerateOnceAsync( + LanguageModel model, + string systemPrompt, + string prompt, + float temperature, + Action? onDelta, + CancellationToken cancellationToken) + { + LanguageModelContext context; + try + { + // A fresh context per request keeps the system prompt in force without carrying previous + // turns forward, which is what kept growing the prompt (and the latency) before. + context = model.CreateContext(systemPrompt, new ContentFilterOptions()); + } + catch (Exception ex) + { + return WinAiGenerationResult.Failed( + Classify(ex), $"The language model could not accept the request: {ex.Message}"); + } + + // Advisory pre-check only. If it cannot answer, send the prompt anyway and let the model + // report PromptLargerThanContext — treating an unknown length as "too long" would turn + // every request into a silent no-op. + try + { + ulong usableLength = model.GetUsablePromptLength(context, prompt); + if (usableLength > 0 && (ulong)prompt.Length > usableLength) + return WinAiGenerationResult.Failed( + WinAiFailure.PromptTooLong, "The text is longer than the language model's context."); + } + catch (Exception ex) + { + Debug.WriteLine($"GetUsablePromptLength failed, sending prompt anyway: {ex.Message}"); + } + + LanguageModelResponseResult result; + try + { + LanguageModelOptions options = new() + { + Temperature = temperature, + ContentFilterOptions = new ContentFilterOptions(), + }; + + IAsyncOperationWithProgress operation = + model.GenerateResponseAsync(context, prompt, options); + + if (onDelta is not null) + operation.Progress = (_, delta) => + { + if (!string.IsNullOrEmpty(delta)) + onDelta(delta); + }; + + result = await operation.AsTask(cancellationToken); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + return WinAiGenerationResult.Failed( + Classify(ex), $"The language model failed to respond: {ex.Message}"); + } + + switch (result.Status) + { + case LanguageModelResponseStatus.Complete: + break; + + case LanguageModelResponseStatus.PromptLargerThanContext: + return WinAiGenerationResult.Failed( + WinAiFailure.PromptTooLong, "The text is longer than the language model's context."); + + case LanguageModelResponseStatus.BlockedByPolicy: + case LanguageModelResponseStatus.PromptBlockedByContentModeration: + case LanguageModelResponseStatus.ResponseBlockedByContentModeration: + return WinAiGenerationResult.Failed( + WinAiFailure.Blocked, + $"Windows AI blocked this text ({result.Status}). Content moderation rejected the request."); + + default: + Exception? extendedError = result.ExtendedError; + string detail = extendedError?.Message ?? result.Status.ToString(); + return WinAiGenerationResult.Failed( + Classify(extendedError), $"The language model returned an error: {detail}"); + } + + string text = result.Text ?? string.Empty; + + return string.IsNullOrWhiteSpace(text) + ? WinAiGenerationResult.Failed(WinAiFailure.ModelError, "The language model returned an empty response.") + : WinAiGenerationResult.Ok(text); + } + + /// A lost connection is worth retrying after a restart; anything else is not. + private static WinAiFailure Classify(Exception? exception) => + IsConnectionLost(exception) ? WinAiFailure.Disconnected : WinAiFailure.ModelError; + + /// + /// Light tidy-up of a model response. Deliberately conservative: an earlier translation + /// implementation tried to detect and strip "instruction echoes" and would silently hand back + /// the untranslated input whenever the guess misfired. Prompting the model directly removes the + /// echoes at the source, so all that is left is trimming stray fences and wrapping quotes. + /// + internal static string CleanResponse(string text) + { + if (string.IsNullOrWhiteSpace(text)) + return string.Empty; + + string cleaned = text.Trim(); + + if (cleaned.StartsWith("```", StringComparison.Ordinal)) + { + int firstNewline = cleaned.IndexOf('\n'); + if (firstNewline > 0) + cleaned = cleaned[(firstNewline + 1)..]; + + if (cleaned.EndsWith("```", StringComparison.Ordinal)) + cleaned = cleaned[..^3]; + + cleaned = cleaned.Trim(); + } + + // Models often wrap a short answer in quotes even when told not to. + if (cleaned.Length > 1 && + ((cleaned[0] == '"' && cleaned[^1] == '"') || + (cleaned[0] == '\'' && cleaned[^1] == '\'') || + (cleaned[0] == '“' && cleaned[^1] == '”'))) + { + cleaned = cleaned[1..^1].Trim(); + } + + return cleaned; + } + + #endregion generation +} diff --git a/Text-Grab/Utilities/WinAiMeetingNotes.cs b/Text-Grab/Utilities/WinAiMeetingNotes.cs new file mode 100644 index 00000000..3eb47b9a --- /dev/null +++ b/Text-Grab/Utilities/WinAiMeetingNotes.cs @@ -0,0 +1,302 @@ +using Microsoft.Windows.AI.Text; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Text_Grab.Utilities; + +/// +/// Turns a long stretch of text — a transcript, a wall of captured chat, scratch notes — into +/// meeting notes: what was discussed, what was decided, and what happens next. +/// +/// Windows AI has no meeting-notes skill, so like translation and regex extraction this prompts the +/// shared directly with a purpose-built system prompt rather than +/// bending TextSummarizer or TextRewriter into the job. +/// +/// The wrinkle is length: meeting text routinely runs past Phi Silica's context, and summarizing +/// half a transcript twice does not make notes. So a long input is handled map-then-reduce — each +/// part is reduced to plain bullets, and those bullets are written up as one set of notes in a final +/// pass. Short input skips straight to that final pass, which is the common case. +/// +internal static class WinAiMeetingNotes +{ + /// Notes should follow the text, with just enough room to phrase an action item. + private const float Temperature = 0.3f; + + /// + /// Characters per part in the first pass. Deliberately well inside the model's context so the + /// common case is one request per part with no splitting. + /// + private const int PartChars = 2000; + + /// Never split a part more than this many times chasing a context that will not fit. + private const int MaxSplitDepth = 3; + + /// The shape of the finished notes, shared by the two prompts that produce them. + private const string NotesFormat = + "Write the notes in Markdown with exactly these sections, in this order:\n" + + "## Summary — two or three sentences on what the meeting was about.\n" + + "## Topics Discussed — one bullet per topic, each with the points made about it.\n" + + "## Decisions — one bullet per decision reached. Write 'None recorded' if there were none.\n" + + "## Next Steps — one bullet per action item, written as '- [ ] Owner — action (due date)', " + + "leaving out the owner or the date when the text does not give one. " + + "Write 'None recorded' if there were none.\n" + + "Use only what is in the text: never invent attendees, decisions, owners or dates. " + + "Keep names, numbers and dates exactly as they appear. " + + "Reply with the notes only: no preamble, no commentary, and never repeat these instructions."; + + private const string NotesSystemPrompt = + "You are a meeting notes writer. The user sends the transcript or raw notes from a meeting. " + + NotesFormat; + + private const string PartSystemPrompt = + "You are taking notes on one part of a longer meeting. The user sends that part of the " + + "transcript. List what was discussed, and anything that was decided or assigned, as short " + + "Markdown bullets — one point per bullet, keeping names, numbers and dates exactly as they " + + "appear. Do not add headings, do not summarize the meeting as a whole, and do not invent " + + "anything that is not in this part. Reply with the bullets only."; + + private const string MergeSystemPrompt = + "You are a meeting notes writer. The user sends rough bullets taken from consecutive parts " + + "of one meeting, in order. Combine them into a single set of notes, merging points that " + + "repeat. " + NotesFormat; + + /// + /// Writes up as meeting notes. The notes are in + /// ; on failure that is null and + /// says why. + /// + /// + /// Optional callback describing the stage in progress, for a loading label. It is raised on a + /// background thread; marshal to the UI thread before touching controls. + /// + internal static async Task SummarizeAsync( + string text, + Action? onProgress = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(text)) + return WinAiGenerationResult.Failed(WinAiFailure.ModelError, "There was no text to turn into meeting notes."); + + (bool available, string? reason) = WinAiLanguageModel.CheckAvailability(); + if (!available) + return WinAiGenerationResult.Failed( + WinAiFailure.Unavailable, reason ?? "Windows AI is not available on this device."); + + try + { + (bool ready, string? error) = await WinAiLanguageModel.EnsureModelAsync(cancellationToken); + if (!ready) + return WinAiGenerationResult.Failed( + WinAiFailure.ModelNotReady, error ?? "The Windows AI language model could not be started."); + + // One lease for the whole write-up: a set of notes is many requests, and letting another + // feature interleave with them on the single-threaded model would stall both. + using (await WinAiLanguageModel.AcquireInferenceAsync(cancellationToken)) + { + List parts = SplitIntoParts(text, PartChars); + + // Short enough to write up in one go, which is most captures. + if (parts.Count == 1) + { + WinAiGenerationResult single = await WinAiLanguageModel.GenerateAsync( + NotesSystemPrompt, parts[0], Temperature, null, cancellationToken); + + if (single.Text is not null) + return Finish(single.Text); + + if (single.Failure is not WinAiFailure.PromptTooLong) + return single; + + // The text fit the character budget but not the model's context, so go through + // the map-then-reduce path with smaller parts. + parts = SplitIntoParts(text, PartChars / 2); + if (parts.Count == 1) + return single; + } + + List bullets = []; + WinAiGenerationResult lastFailure = default; + + for (int index = 0; index < parts.Count; index++) + { + cancellationToken.ThrowIfCancellationRequested(); + onProgress?.Invoke($"Reading part {index + 1} of {parts.Count}..."); + + WinAiGenerationResult part = await SummarizePartAsync(parts[index], 0, cancellationToken); + + if (part.Text is null) + { + // One unreadable part should not lose the rest of the meeting. + Debug.WriteLine($"Meeting notes part {index + 1} failed ({part.Failure}): {part.Message}"); + lastFailure = part; + continue; + } + + bullets.Add(part.Text.Trim()); + } + + if (bullets.Count == 0) + return lastFailure.Message is not null + ? lastFailure + : WinAiGenerationResult.Failed(WinAiFailure.ModelError, "The language model returned no notes."); + + onProgress?.Invoke("Writing up the notes..."); + + WinAiGenerationResult merged = await MergeAsync([.. bullets], cancellationToken); + + return merged.Text is null ? merged : Finish(merged.Text); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + Debug.WriteLine($"Meeting notes exception: {ex.Message}"); + return WinAiGenerationResult.Failed(WinAiFailure.ModelError, $"Meeting notes failed: {ex.Message}"); + } + } + + private static WinAiGenerationResult Finish(string text) + { + string cleaned = WinAiLanguageModel.CleanResponse(text); + + return string.IsNullOrWhiteSpace(cleaned) + ? WinAiGenerationResult.Failed(WinAiFailure.ModelError, "The notes came back empty.") + : WinAiGenerationResult.Ok(cleaned); + } + + /// + /// Reduces one part of the text to bullets, halving it and recursing when — and only when — the + /// model reports the prompt does not fit the context. + /// + private static async Task SummarizePartAsync( + string part, + int depth, + CancellationToken cancellationToken) + { + WinAiGenerationResult outcome = await WinAiLanguageModel.GenerateAsync( + PartSystemPrompt, part, Temperature, null, cancellationToken); + + if (outcome.Text is not null || outcome.Failure is not WinAiFailure.PromptTooLong || depth >= MaxSplitDepth) + return outcome; + + string[] halves = SplitInHalf(part); + if (halves.Length < 2) + return outcome; + + StringBuilder combined = new(); + foreach (string half in halves) + { + WinAiGenerationResult piece = await SummarizePartAsync(half, depth + 1, cancellationToken); + if (piece.Text is null) + return piece; + + combined.AppendLine(piece.Text.Trim()); + } + + return WinAiGenerationResult.Ok(combined.ToString()); + } + + /// + /// Writes the per-part bullets up as one set of notes. When they do not all fit at once, each + /// half is written up separately and the two write-ups are merged. + /// + private static async Task MergeAsync( + string[] sections, + CancellationToken cancellationToken) + { + string joined = string.Join("\n\n", sections); + + WinAiGenerationResult merged = await WinAiLanguageModel.GenerateAsync( + MergeSystemPrompt, joined, Temperature, null, cancellationToken); + + if (merged.Text is not null || merged.Failure is not WinAiFailure.PromptTooLong) + return merged; + + // Two sections that still will not fit together cannot be merged by splitting again, so + // hand back the sections themselves rather than nothing at all. + if (sections.Length < 3) + return WinAiGenerationResult.Ok(joined); + + int middle = sections.Length / 2; + + WinAiGenerationResult first = await MergeAsync(sections[..middle], cancellationToken); + if (first.Text is null) + return first; + + WinAiGenerationResult second = await MergeAsync(sections[middle..], cancellationToken); + if (second.Text is null) + return second; + + return await MergeAsync([first.Text, second.Text], cancellationToken); + } + + /// + /// Splits text into parts of roughly , breaking at a blank line + /// where possible and at a line break otherwise, so a part rarely stops mid-thought. + /// + internal static List SplitIntoParts(string text, int targetChars) + { + if (text.Length <= targetChars) + return [text]; + + List parts = []; + int start = 0; + + while (start < text.Length) + { + if (text.Length - start <= targetChars) + { + parts.Add(text[start..]); + break; + } + + int limit = start + targetChars; + + // Prefer a paragraph break, then any line break, then a space, and only cut mid-word + // when the text offers nothing else. + int breakAt = text.LastIndexOf("\n\n", limit, targetChars, StringComparison.Ordinal); + if (breakAt <= start) + breakAt = text.LastIndexOf('\n', limit, targetChars); + if (breakAt <= start) + breakAt = text.LastIndexOf(' ', limit, targetChars); + if (breakAt <= start) + breakAt = limit; + + parts.Add(text[start..breakAt]); + start = breakAt; + + while (start < text.Length && (text[start] == '\n' || text[start] == '\r' || text[start] == ' ')) + start++; + } + + return parts; + } + + private static string[] SplitInHalf(string text) + { + if (text.Length < 200) + return [text]; + + int middle = text.Length / 2; + int splitAt = text.LastIndexOf('\n', middle); + + if (splitAt <= 0) + splitAt = text.IndexOf('\n', middle); + + if (splitAt <= 0) + { + splitAt = text.LastIndexOf(' ', middle); + if (splitAt <= 0) + return [text]; + } + + return [text[..(splitAt + 1)], text[(splitAt + 1)..]]; + } +} diff --git a/Text-Grab/Utilities/WinAiTranslator.cs b/Text-Grab/Utilities/WinAiTranslator.cs new file mode 100644 index 00000000..0d364697 --- /dev/null +++ b/Text-Grab/Utilities/WinAiTranslator.cs @@ -0,0 +1,498 @@ +using Microsoft.Windows.AI.Text; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace Text_Grab.Utilities; + +/// Why a translation did not produce new text. +internal enum TranslationFailure +{ + /// The text was translated. + None, + + /// This device cannot run the Windows AI language model at all. + Unavailable, + + /// The model exists but could not be prepared or created. + ModelNotReady, + + /// The text already looks like it is in the target language, so nothing was sent. + NotNeeded, + + /// The prompt did not fit the model's context, even after splitting. + PromptTooLong, + + /// Content moderation or policy rejected the prompt or the response. + Blocked, + + /// The model reported an error, or returned nothing usable. + ModelError, +} + +/// +/// The outcome of a translation. is always safe to use: on failure it is the +/// original, untranslated input. +/// +internal readonly record struct TranslationResult(string Text, TranslationFailure Failure, string? Message) +{ + internal bool Succeeded => Failure is TranslationFailure.None; + + internal static TranslationResult Success(string text) => new(text, TranslationFailure.None, null); +} + +/// +/// The outcome of a batched translation. always matches the requested list in +/// length and order; entries that could not be translated keep their original value. +/// +internal readonly record struct BatchTranslationResult( + IReadOnlyList Items, + int TranslatedCount, + TranslationFailure Failure, + string? Message) +{ + internal bool Succeeded => Failure is TranslationFailure.None; +} + +/// +/// On-device translation built on , the shared Windows AI Foundry +/// (Phi Silica), following the pattern used by microsoft/ai-dev-gallery's +/// PhiSilicaClient and Translate samples. +/// +/// The three things that make this fast compared to the previous implementation: +/// 1. The model is prompted directly through GenerateResponseAsync with a translation system +/// prompt, instead of being funneled through the TextRewriter skill (whose own "rewrite this +/// text" system prompt fought the translation instruction and produced instruction echoes in +/// the output). +/// 2. The is created once and reused across features; only the +/// lightweight per-request context is rebuilt so turns never accumulate. +/// 3. Many short strings are translated in one batched inference instead of one inference each, +/// and partial results stream back through the operation's Progress callback so the UI can +/// fill in while the model is still generating. +/// +/// Every failure path returns a and a human-readable message so +/// callers can tell the user why nothing changed, rather than silently handing back the input. +/// +internal static partial class WinAiTranslator +{ + /// Max items packed into a single batched request. + private const int MaxBatchItems = 40; + + /// Approximate character budget for the numbered list in a single batched request. + private const int MaxBatchChars = 1200; + + /// Translation wants the most likely wording, not a creative one. + private const float Temperature = 0.2f; + + [GeneratedRegex(@"^\s*(\d+)\s*[.):\]]\s*(.*)$")] + private static partial Regex NumberedItemRegex(); + + #region availability + + /// + /// Whether this device can run the Windows AI language model, and why not when it cannot. + /// + internal static (bool Available, string? Reason) CheckAvailability() => WinAiLanguageModel.CheckAvailability(); + + /// True when this device can run the Windows AI language model. + internal static bool IsAvailable() => WinAiLanguageModel.IsAvailable(); + + /// + /// Drops the cached language model to free the memory it holds. The next translation recreates + /// it, so call this when translation is switched off rather than between translations. + /// + internal static void ReleaseModel() => WinAiLanguageModel.ReleaseModel(); + + /// Releases the shared language model. Call once during application shutdown. + internal static void Cleanup() => WinAiLanguageModel.Cleanup(); + + #endregion availability + + #region generation + + /// Maps a shared model failure onto the translation-facing reason for it. + private static TranslationFailure ToTranslationFailure(WinAiFailure failure) => failure switch + { + WinAiFailure.None => TranslationFailure.None, + WinAiFailure.Unavailable => TranslationFailure.Unavailable, + WinAiFailure.ModelNotReady => TranslationFailure.ModelNotReady, + WinAiFailure.PromptTooLong => TranslationFailure.PromptTooLong, + WinAiFailure.Blocked => TranslationFailure.Blocked, + _ => TranslationFailure.ModelError, + }; + + private static string SystemPromptFor(string targetLanguage) => + $"You are a translation engine. Translate everything the user sends into {targetLanguage}, " + + $"written in the native script and characters of {targetLanguage}. " + + "Preserve the original line breaks, numbers, punctuation and formatting. " + + "Reply with the translation only: no notes, no explanations, no quotation marks around it, " + + "and never repeat these instructions."; + + #endregion generation + + #region single text + + /// + /// Translates a block of text. The returned is the original + /// input whenever translation did not happen, and then + /// explains why so the caller can tell the user. + /// + /// + /// Optional callback receiving generated text as it arrives. It is raised on a background + /// thread; marshal to the UI thread before touching controls. + /// + internal static async Task TranslateAsync( + string textToTranslate, + string targetLanguage, + Action? onPartial = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(textToTranslate)) + return TranslationResult.Success(textToTranslate); + + if (string.IsNullOrWhiteSpace(targetLanguage)) + return new TranslationResult(textToTranslate, TranslationFailure.Unavailable, "No target language was set."); + + (bool available, string? reason) = CheckAvailability(); + if (!available) + return new TranslationResult(textToTranslate, TranslationFailure.Unavailable, reason); + + if (LanguageHeuristics.IsLikelyInTargetLanguage(textToTranslate, targetLanguage)) + return new TranslationResult( + textToTranslate, TranslationFailure.NotNeeded, $"The text already appears to be in {targetLanguage}."); + + try + { + (bool ready, string? error) = await WinAiLanguageModel.EnsureModelAsync(cancellationToken); + if (!ready) + return new TranslationResult(textToTranslate, TranslationFailure.ModelNotReady, error); + + string systemPrompt = SystemPromptFor(targetLanguage); + + // One lease for the whole translation so another feature's request cannot interleave + // with it on the single-threaded model. + using (await WinAiLanguageModel.AcquireInferenceAsync(cancellationToken)) + { + WinAiGenerationResult outcome = await TranslateBlockAsync( + systemPrompt, textToTranslate, onPartial, cancellationToken); + + if (outcome.Text is null) + return new TranslationResult( + textToTranslate, ToTranslationFailure(outcome.Failure), outcome.Message); + + string cleaned = CleanResult(outcome.Text); + + return string.IsNullOrWhiteSpace(cleaned) + ? new TranslationResult(textToTranslate, TranslationFailure.ModelError, "The translation came back empty.") + : TranslationResult.Success(cleaned); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + Debug.WriteLine($"Translation exception: {ex.Message}"); + return new TranslationResult(textToTranslate, TranslationFailure.ModelError, $"Translation failed: {ex.Message}"); + } + } + + /// + /// Translates one block, splitting it on line boundaries and recursing when — and only when — + /// the model reports the prompt does not fit the context. Any other failure is passed straight + /// back so the caller can report it. + /// + private static async Task TranslateBlockAsync( + string systemPrompt, + string text, + Action? onPartial, + CancellationToken cancellationToken) + { + WinAiGenerationResult outcome = await WinAiLanguageModel.GenerateAsync( + systemPrompt, text, Temperature, onPartial, cancellationToken); + if (outcome.Text is not null || outcome.Failure is not WinAiFailure.PromptTooLong) + return outcome; + + // Too long for one pass: split roughly in half on a line break and translate each side. + string[] pieces = SplitInHalf(text); + if (pieces.Length < 2) + return outcome; + + StringBuilder combined = new(); + foreach (string piece in pieces) + { + WinAiGenerationResult part = await TranslateBlockAsync(systemPrompt, piece, onPartial, cancellationToken); + if (part.Text is null) + return part; + + combined.Append(part.Text); + } + + return WinAiGenerationResult.Ok(combined.ToString()); + } + + private static string[] SplitInHalf(string text) + { + if (text.Length < 200) + return [text]; + + int middle = text.Length / 2; + int splitAt = text.LastIndexOf('\n', middle); + + if (splitAt <= 0) + splitAt = text.IndexOf('\n', middle); + + if (splitAt <= 0) + { + splitAt = text.LastIndexOf(' ', middle); + if (splitAt <= 0) + return [text]; + } + + return [text[..(splitAt + 1)], text[(splitAt + 1)..]]; + } + + #endregion single text + + #region batched text + + /// + /// Translates many short strings (for example every word box in a Grab Frame) using as few + /// inferences as possible. Items are de-duplicated and packed into numbered batches; results + /// are reported through as each line streams in so the UI + /// fills in progressively. + /// + internal static async Task TranslateBatchAsync( + IReadOnlyList items, + string targetLanguage, + Action? onItemTranslated = null, + CancellationToken cancellationToken = default) + { + string[] results = [.. items]; + + if (items.Count == 0) + return new BatchTranslationResult(results, 0, TranslationFailure.None, null); + + if (string.IsNullOrWhiteSpace(targetLanguage)) + return new BatchTranslationResult(results, 0, TranslationFailure.Unavailable, "No target language was set."); + + (bool available, string? reason) = CheckAvailability(); + if (!available) + return new BatchTranslationResult(results, 0, TranslationFailure.Unavailable, reason); + + // De-duplicate: a Grab Frame usually repeats plenty of short words. + Dictionary> byText = []; + for (int index = 0; index < items.Count; index++) + { + string item = items[index]; + if (string.IsNullOrWhiteSpace(item)) + continue; + + if (!byText.TryGetValue(item, out List? indices)) + byText[item] = indices = []; + + indices.Add(index); + } + + if (byText.Count == 0) + return new BatchTranslationResult(results, 0, TranslationFailure.None, null); + + List distinct = [.. byText.Keys]; + int translatedCount = 0; + + void CountAndReport(int index, string translated) + { + translatedCount++; + onItemTranslated?.Invoke(index, translated); + } + + try + { + (bool ready, string? error) = await WinAiLanguageModel.EnsureModelAsync(cancellationToken); + if (!ready) + return new BatchTranslationResult(results, 0, TranslationFailure.ModelNotReady, error); + + string systemPrompt = + "You are a translation engine. The user sends a numbered list. Translate each item into " + + $"{targetLanguage}, written in the native script and characters of {targetLanguage}. " + + "Reply with the same numbers in the same order, one item per line, in the form '1. translation'. " + + "Keep exactly one output line per input line. Do not merge, reorder, add or drop items, " + + "and do not add any commentary."; + + WinAiGenerationResult lastFailure = default; + + // One lease for the whole translation so another feature's request cannot interleave + // with it on the single-threaded model. + using (await WinAiLanguageModel.AcquireInferenceAsync(cancellationToken)) + { + foreach (List batch in BuildBatches(distinct)) + { + cancellationToken.ThrowIfCancellationRequested(); + + WinAiGenerationResult outcome = await TranslateBatchChunkAsync( + systemPrompt, distinct, batch, byText, results, CountAndReport, cancellationToken); + + if (outcome.Text is null) + lastFailure = outcome; + } + } + + // Report a failure only when nothing at all came back; a partial batch failure still + // leaves the frame better off than before. + if (translatedCount == 0 && lastFailure.Message is not null) + return new BatchTranslationResult( + results, 0, ToTranslationFailure(lastFailure.Failure), lastFailure.Message); + + if (translatedCount == 0) + return new BatchTranslationResult( + results, 0, TranslationFailure.ModelError, "The language model did not return any translations."); + + return new BatchTranslationResult(results, translatedCount, TranslationFailure.None, null); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + Debug.WriteLine($"Batch translation exception: {ex.Message}"); + return new BatchTranslationResult( + results, translatedCount, TranslationFailure.ModelError, $"Translation failed: {ex.Message}"); + } + } + + /// Packs distinct item indices into batches bounded by item count and characters. + private static List> BuildBatches(List distinct) + { + List> batches = []; + List current = []; + int currentChars = 0; + + for (int i = 0; i < distinct.Count; i++) + { + int itemChars = distinct[i].Length + 6; // "NN. " plus newline + + if (current.Count > 0 && (current.Count >= MaxBatchItems || currentChars + itemChars > MaxBatchChars)) + { + batches.Add(current); + current = []; + currentChars = 0; + } + + current.Add(i); + currentChars += itemChars; + } + + if (current.Count > 0) + batches.Add(current); + + return batches; + } + + private static async Task TranslateBatchChunkAsync( + string systemPrompt, + List distinct, + List batch, + Dictionary> byText, + string[] results, + Action onItemTranslated, + CancellationToken cancellationToken) + { + StringBuilder promptBuilder = new(); + for (int position = 0; position < batch.Count; position++) + promptBuilder.Append(position + 1).Append(". ").AppendLine(distinct[batch[position]]); + + // Streaming: apply each numbered line the moment the model finishes generating it. + StringBuilder streamed = new(); + int appliedLines = 0; + Lock streamLock = new(); + + void OnDelta(string delta) + { + lock (streamLock) + { + streamed.Append(delta); + string[] lines = streamed.ToString().Split('\n'); + + // The last element is still being generated, so stop one short of it. + for (; appliedLines < lines.Length - 1; appliedLines++) + ApplyLine(lines[appliedLines], distinct, batch, byText, results, onItemTranslated); + } + } + + WinAiGenerationResult outcome = await WinAiLanguageModel.GenerateAsync( + systemPrompt, promptBuilder.ToString(), Temperature, OnDelta, cancellationToken); + + if (outcome.Text is null) + { + // Split the batch and retry each half, but only when the prompt was the problem. + if (outcome.Failure is not WinAiFailure.PromptTooLong || batch.Count < 2) + return outcome; + + int middle = batch.Count / 2; + + WinAiGenerationResult first = await TranslateBatchChunkAsync( + systemPrompt, distinct, [.. batch[..middle]], byText, results, onItemTranslated, cancellationToken); + WinAiGenerationResult second = await TranslateBatchChunkAsync( + systemPrompt, distinct, [.. batch[middle..]], byText, results, onItemTranslated, cancellationToken); + + return first.Text is null ? first : second; + } + + // Authoritative pass over the completed response; the streamed pass above is only a preview. + foreach (string line in outcome.Text.Split('\n')) + ApplyLine(line, distinct, batch, byText, results, onItemTranslated); + + return outcome; + } + + private static void ApplyLine( + string line, + List distinct, + List batch, + Dictionary> byText, + string[] results, + Action onItemTranslated) + { + Match match = NumberedItemRegex().Match(line.TrimEnd('\r')); + if (!match.Success) + return; + + if (!int.TryParse(match.Groups[1].Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int position)) + return; + + position--; // the prompt numbers from 1 + if (position < 0 || position >= batch.Count) + return; + + string translated = CleanResult(match.Groups[2].Value); + if (string.IsNullOrWhiteSpace(translated)) + return; + + string source = distinct[batch[position]]; + if (!byText.TryGetValue(source, out List? indices)) + return; + + foreach (int index in indices) + { + if (string.Equals(results[index], translated, StringComparison.Ordinal)) + continue; + + results[index] = translated; + onItemTranslated(index, translated); + } + } + + #endregion batched text + + /// + /// Light tidy-up of a model response, shared with the other language model features. + /// + internal static string CleanResult(string text) => WinAiLanguageModel.CleanResponse(text); +} diff --git a/Text-Grab/Utilities/WindowsAiUtilities.cs b/Text-Grab/Utilities/WindowsAiUtilities.cs index 5c737127..9ae51088 100644 --- a/Text-Grab/Utilities/WindowsAiUtilities.cs +++ b/Text-Grab/Utilities/WindowsAiUtilities.cs @@ -21,109 +21,6 @@ namespace Text_Grab.Utilities; public static class WindowsAiUtilities { - private const string TranslationPromptTemplate = "Translate to {0} using local alphabet and characters of that langauage:\n\n{1}"; - private static LanguageModel? _translationLanguageModel; - private static readonly SemaphoreSlim _modelInitializationLock = new(1, 1); - private static bool _disposed; - - // Language code mapping for quick lookup - private static readonly Dictionary LanguageCodeMap = new(StringComparer.OrdinalIgnoreCase) - { - { "English", "en" }, - { "Spanish", "es" }, - { "French", "fr" }, - { "German", "de" }, - { "Italian", "it" }, - { "Portuguese", "pt" }, - { "Russian", "ru" }, - { "Japanese", "ja" }, - { "Chinese (Simplified)", "zh-Hans" }, - { "Chinese", "zh-Hans" }, - { "Korean", "ko" }, - { "Arabic", "ar" }, - { "Hindi", "hi" }, - }; - - /// - /// Quickly detects if text is likely in the target language using simple heuristics. - /// This is a fast check to avoid expensive translation calls. - /// - /// Text to analyze - /// Target language name (e.g., "English", "Spanish") - /// True if text appears to already be in target language - private static bool IsLikelyInTargetLanguage(string text, string targetLanguage) - { - if (string.IsNullOrWhiteSpace(text) || text.Length < 3) - return false; - - // Get language code for target - if (!LanguageCodeMap.TryGetValue(targetLanguage, out string? targetCode)) - return false; // Unknown language, proceed with translation - - // Character range detection - bool hasCJK = text.Any(c => c is >= (char)0x4E00 and <= (char)0x9FFF or // CJK Unified Ideographs - >= (char)0x3040 and <= (char)0x309F or // Hiragana - >= (char)0x30A0 and <= (char)0x30FF or // Katakana - >= (char)0xAC00 and <= (char)0xD7AF); // Hangul - - bool hasArabic = text.Any(c => c is >= (char)0x0600 and <= (char)0x06FF); - bool hasCyrillic = text.Any(c => c is >= (char)0x0400 and <= (char)0x04FF); - bool hasDevanagari = text.Any(c => c is >= (char)0x0900 and <= (char)0x097F); - bool hasLatin = text.Any(c => c is >= 'A' and <= 'Z' or >= 'a' and <= 'z'); - - // Quick script-based checks - switch (targetCode) - { - case "en": - case "es": - case "fr": - case "de": - case "it": - case "pt": - // Latin script languages - if mostly CJK/Arabic/Cyrillic, definitely not in target - if (hasCJK || hasArabic || hasCyrillic || hasDevanagari) - return false; - // If has Latin characters, might be in target language - if (hasLatin && text.Length > 10 && targetCode == "en") - { - // Check for common English words as additional heuristic - string lowerText = text.ToLowerInvariant(); - string[] commonEnglishWords = [" the ", " and ", " or ", " is ", " are ", " was ", " were ", " in ", " on ", " at ", " to ", " of ", " for ", " with "]; - int englishWordCount = commonEnglishWords.Count(w => lowerText.Contains(w)); - // If text contains multiple common English words, likely already English - if (englishWordCount >= 2) - return true; - } - break; - - case "ru": - // Russian - should have Cyrillic - return hasCyrillic && !hasCJK && !hasArabic; - - case "ja": - // Japanese - should have Hiragana/Katakana/Kanji - return hasCJK && !hasArabic && !hasCyrillic; - - case "zh-Hans": - // Chinese - should have CJK - return hasCJK && !hasArabic && !hasCyrillic; - - case "ko": - // Korean - should have Hangul - return text.Any(c => c is >= (char)0xAC00 and <= (char)0xD7AF) && !hasArabic && !hasCyrillic; - - case "ar": - // Arabic - should have Arabic script - return hasArabic && !hasCJK && !hasCyrillic; - - case "hi": - // Hindi - should have Devanagari - return hasDevanagari && !hasCJK && !hasArabic; - } - - return false; - } - public static bool CanDeviceUseWinAI() { return CanDeviceUseWinAiFeature(TextRecognizer.GetReadyState); @@ -322,12 +219,16 @@ private static async Task GetTextDescriptionWithWinAI(ImageDescriptionGe return result; } - internal static async Task SummarizeParagraph(string textToSummarize) + /// + /// Summarizes text with the shared Windows AI language model. + /// + /// + /// The summary in , or a and a + /// human-readable message. A failure must never be shown as if it were a summary, which is why + /// this reports one rather than returning an "ERROR: …" string. + /// + internal static async Task SummarizeParagraph(string textToSummarize) { - using LanguageModel languageModel = await LanguageModel.CreateAsync(); - - TextSummarizer textSummarizer = new(languageModel); - bool wasTruncated = false; // TODO: in WinAppSDK 1.8+ we can use this API when the GitHub Actions runner passes @@ -337,316 +238,113 @@ internal static async Task SummarizeParagraph(string textToSummarize) // wasTruncated = true; // } - try - { - LanguageModelResponseResult result = await textSummarizer.SummarizeParagraphAsync(textToSummarize); + // Going through the shared model reuses the one LanguageModel the other AI features hold, + // and means a dropped connection to the Windows AI runtime ("The RPC server is unavailable") + // restarts the model and tries again instead of failing until Text-Grab is restarted. + (LanguageModelResponseResult? result, string? error) = await WinAiLanguageModel.RunWithModelAsync( + (model, token) => new TextSummarizer(model).SummarizeParagraphAsync(textToSummarize).AsTask(token)); - if (result.Status == LanguageModelResponseStatus.Complete) - { - if (wasTruncated) - return $"NOTE: The input text was too long and had to be truncated.\n\nSummary:\n{result.Text}"; - else - return result.Text; - } - else - return $"ERROR: Unable to summarize text. {result.ExtendedError.Message}"; - } - catch (Exception ex) - { - return $"ERROR: Unable to summarize text. {ex.Message}"; - } - } + if (result is null) + return WinAiGenerationResult.Failed(WinAiFailure.ModelNotReady, $"Unable to summarize text. {error}"); - internal static async Task Rewrite(string textToRewrite) - { - using LanguageModel languageModel = await LanguageModel.CreateAsync(); + if (result.Status != LanguageModelResponseStatus.Complete) + return WinAiGenerationResult.Failed( + WinAiFailure.ModelError, + $"Unable to summarize text. {result.ExtendedError?.Message ?? result.Status.ToString()}"); - TextRewriter textRewriter = new(languageModel); - try - { - // TODO: in WinAppSDK 1.8+ we can use this API when the GitHub Actions runner passes - //LanguageModelResponseResult result = await textRewriter.RewriteAsync(textToRewrite, TextRewriteTone.Concise); - LanguageModelResponseResult result = await textRewriter.RewriteAsync(textToRewrite); - if (result.Status == LanguageModelResponseStatus.Complete) - { - return result.Text; - } - else - return $"ERROR: Unable to rewrite text. {result.ExtendedError.Message}"; - } - catch (Exception ex) - { - return $"ERROR: Failed to Rewrite: {ex.Message}"; - } + return WinAiGenerationResult.Ok(wasTruncated + ? $"NOTE: The input text was too long and had to be truncated.\n\nSummary:\n{result.Text}" + : result.Text); } - internal static async Task TextToTable(string textToTable) + internal static async Task Rewrite(string textToRewrite) { - using LanguageModel languageModel = await LanguageModel.CreateAsync(); + // TODO: in WinAppSDK 1.8+ we can pass TextRewriteTone.Concise when the GitHub Actions runner passes + (LanguageModelResponseResult? result, string? error) = await WinAiLanguageModel.RunWithModelAsync( + (model, token) => new TextRewriter(model).RewriteAsync(textToRewrite).AsTask(token)); - TextToTableConverter toTableConverter = new(languageModel); - try - { - TextToTableResponseResult result = await toTableConverter.ConvertAsync(textToTable); - if (result.Status == LanguageModelResponseStatus.Complete) - { - TextToTableRow[] rows = result.GetRows(); - StringBuilder sb = new(); - foreach (TextToTableRow row in rows) - { - string[] columns = row.GetColumns(); - sb.AppendLine(string.Join("\t", columns)); - } - return sb.ToString(); - } - else - return $"ERROR: Unable to rewrite text. {result.ExtendedError.Message}"; - } - catch (Exception ex) - { - return $"ERROR: Failed to Rewrite: {ex.Message}"; - } + if (result is null) + return $"ERROR: Failed to Rewrite: {error}"; + + return result.Status == LanguageModelResponseStatus.Complete + ? result.Text + : $"ERROR: Unable to rewrite text. {result.ExtendedError?.Message ?? result.Status.ToString()}"; } - /// - /// Cleans up translation result by removing instruction echoes and unwanted prefixes. - /// - private static string CleanTranslationResult(string translatedText, string originalText) + internal static async Task TextToTable(string textToTable) { - if (string.IsNullOrWhiteSpace(translatedText)) - return originalText; - - string cleaned = translatedText.Trim(); - - // Remove common instruction echoes (case-insensitive) - string[] instructionPhrases = - [ - "translate", - "translation", - "translated", - "do not reply", - "do not respond", - "extraneous content", - "besides the translated text", - "other than the translated text", - "here is the translation", - "here's the translation", - "the translation is", - ]; - - string lowerCleaned = cleaned.ToLowerInvariant(); - - // If the result contains instruction-like phrases, try to extract just the translation - if (instructionPhrases.Any(phrase => lowerCleaned.Contains(phrase))) - { - // Split by common delimiters and take the longest non-instruction part - string[] parts = cleaned.Split(['\n', '.', ':', '"'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - - string? bestPart = null; - int maxLength = 0; - - foreach (string part in parts) - { - string lowerPart = part.ToLowerInvariant(); - bool hasInstructions = instructionPhrases.Any(phrase => lowerPart.Contains(phrase)); - - if (!hasInstructions && part.Length > maxLength && part.Length >= 3) - { - bestPart = part; - maxLength = part.Length; - } - } + (TextToTableResponseResult? result, string? error) = await WinAiLanguageModel.RunWithModelAsync( + (model, token) => new TextToTableConverter(model).ConvertAsync(textToTable).AsTask(token)); - if (bestPart != null && bestPart.Length > originalText.Length / 3) - { - cleaned = bestPart.Trim(); - } - else - { - // Couldn't extract clean translation, return original - Debug.WriteLine($"Translation contained instructions, returning original text"); - return originalText; - } - } + if (result is null) + return $"ERROR: Failed to convert the text to a table. {error}"; - // Remove common prefixes that might leak through - string[] commonPrefixes = - [ - "translation: ", - "translated: ", - "result: ", - "output: ", - ]; + if (result.Status != LanguageModelResponseStatus.Complete) + return $"ERROR: Unable to convert the text to a table. {result.ExtendedError?.Message ?? result.Status.ToString()}"; - foreach (string prefix in commonPrefixes.Where(prefix => cleaned.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))) - { - cleaned = cleaned[prefix.Length..].Trim(); - } + StringBuilder sb = new(); + foreach (TextToTableRow row in result.GetRows()) + sb.AppendLine(string.Join("\t", row.GetColumns())); - // If cleaned result is suspiciously short or empty, return original - if (string.IsNullOrWhiteSpace(cleaned) || cleaned.Length < 2) - { - Debug.WriteLine($"Translation result too short, returning original text"); - return originalText; - } - - return cleaned; - } - - /// - /// Initializes the shared LanguageModel for translation if not already created. - /// Thread-safe initialization using SemaphoreSlim. - /// - private static async Task EnsureTranslationModelInitializedAsync() - { - if (_translationLanguageModel is not null) - return; - - await _modelInitializationLock.WaitAsync(); - try - { - _translationLanguageModel ??= await LanguageModel.CreateAsync(); - } - finally - { - _modelInitializationLock.Release(); - } - } - - /// - /// Disposes the shared LanguageModel to free resources. - /// Should be called when translation is no longer needed. - /// - public static void DisposeTranslationModel() - { - _translationLanguageModel?.Dispose(); - _translationLanguageModel = null; + return sb.ToString(); } /// /// Releases resources held by static members of . /// Should be called once during application shutdown. /// - public static void Cleanup() - { - if (_disposed) - return; - - DisposeTranslationModel(); - _modelInitializationLock.Dispose(); - _disposed = true; - } - + public static void Cleanup() => WinAiLanguageModel.Cleanup(); /// - /// Translates text to a target language using Windows AI LanguageModel. - /// Reuses a shared LanguageModel instance for improved performance. - /// Includes fast language detection to skip translation if text is already in target language. - /// Filters out instruction echoes from AI responses. + /// Drops the language model this process is holding so the next AI request builds a fresh + /// connection to the Windows AI runtime. The AI features already do this by themselves when a + /// request finds the runtime gone; this is for offering the user a manual reconnect. /// - /// The text to translate - /// The target language (e.g., "English", "Spanish") - /// The translated text, or the original text if translation fails or is unnecessary - /// - /// This implementation uses TextRewriter with a custom prompt as a workaround - /// since Microsoft.Windows.AI.Text doesn't include a dedicated translation API. - /// Translation quality may vary compared to dedicated translation services. - /// The LanguageModel is reused across calls for better performance. - /// Fast language detection is performed first to avoid unnecessary API calls. - /// Result is cleaned to remove any instruction echoes from the AI response. - /// - internal static async Task TranslateText(string textToTranslate, string targetLanguage) - { - if (!CanDeviceUseWinAI()) - return textToTranslate; // Return original text if Windows AI is not available - - // Quick check: if text appears to already be in target language, skip translation - if (IsLikelyInTargetLanguage(textToTranslate, targetLanguage)) - { - Debug.WriteLine($"Skipping translation - text appears to already be in {targetLanguage}"); - return textToTranslate; - } - - try - { - await EnsureTranslationModelInitializedAsync(); - - if (_translationLanguageModel is null) - return textToTranslate; - - // Note: This uses TextRewriter with a simple prompt - // We use a minimal prompt to reduce the chance of instruction echoes - TextRewriter textRewriter = new(_translationLanguageModel); - string translationPrompt = string.Format(TranslationPromptTemplate, targetLanguage, textToTranslate); - - LanguageModelResponseResult result = await textRewriter.RewriteAsync(translationPrompt); - - if (result.Status == LanguageModelResponseStatus.Complete) - { - // Clean the result to remove any instruction echoes - string cleanedResult = CleanTranslationResult(result.Text, textToTranslate); - return cleanedResult; - } - else - { - // Log the error if debugging is enabled - Debug.WriteLine($"Translation failed with status: {result.Status}"); - if (result.ExtendedError != null) - Debug.WriteLine($"Translation error: {result.ExtendedError.Message}"); - return textToTranslate; // Return original text on error - } - } - catch (Exception ex) - { - // Log the exception for debugging - Debug.WriteLine($"Translation exception: {ex.Message}"); - return textToTranslate; // Return original text on error - } - } + public static Task RestartWindowsAiAsync() => WinAiLanguageModel.RestartModelAsync(); /// - /// Extracts a regular expression pattern from text using Windows AI LanguageModel. + /// Extracts a regular expression pattern from text using the shared Windows AI language model. /// - /// The text describing what to match or containing example text to match - /// A regular expression pattern string, or empty string if extraction fails + /// The text describing what to match, or example text to match + /// Aborts the on-device inference. + /// + /// The pattern in , or a and + /// a human-readable message explaining why there is none. + /// /// - /// This method uses the LanguageModel to generate a regex pattern based on the input text. - /// The result is cleaned to contain only the regex pattern without explanations or formatting. + /// This goes through like translation does, so it shares the + /// Limited Access Feature unlock and the one cached LanguageModel, and prompts the model + /// directly with a regex system prompt rather than bending the TextRewriter skill into the job. /// - internal static async Task ExtractRegex(string textDescription) + internal static async Task ExtractRegex( + string textDescription, + CancellationToken cancellationToken = default) { - if (!CanDeviceUseWinAI()) - return string.Empty; - if (string.IsNullOrWhiteSpace(textDescription)) - return string.Empty; - - try - { - using LanguageModel languageModel = await LanguageModel.CreateAsync(); - TextRewriter textRewriter = new(languageModel); + return WinAiGenerationResult.Failed(WinAiFailure.ModelError, "There was no text to build a pattern from."); - string regexPrompt = $"Generate a general regular expression pattern (regex) for: {textDescription}\n\nDo not make it overly constrained on the exact text.\n\nReturn ONLY the regex pattern, nothing else."; + const string systemPrompt = + "You are a regular expression generator. The user describes what to match, or gives an example " + + "of the text they want to match. Reply with a single .NET regular expression pattern that matches " + + "it. Generalize: match text of that kind, not only the exact sample. " + + "Reply with the pattern only: no delimiters, no code fences, no flags, no explanation, " + + "and never repeat these instructions."; - LanguageModelResponseResult result = await textRewriter.RewriteAsync(regexPrompt); + // Pattern generation should be as deterministic as the model allows. + WinAiGenerationResult result = await WinAiLanguageModel.PromptAsync( + systemPrompt, textDescription, temperature: 0.1f, cancellationToken: cancellationToken); - if (result.Status == LanguageModelResponseStatus.Complete) - { - return CleanRegexResult(result.Text); - } - else - { - Debug.WriteLine($"Regex extraction failed with status: {result.Status}"); - if (result.ExtendedError != null) - Debug.WriteLine($"Regex extraction error: {result.ExtendedError.Message}"); - return string.Empty; - } - } - catch (Exception ex) + if (result.Text is null) { - Debug.WriteLine($"Regex extraction exception: {ex.Message}"); - return string.Empty; + Debug.WriteLine($"Regex extraction failed ({result.Failure}): {result.Message}"); + return result; } + + string pattern = CleanRegexResult(result.Text); + + return string.IsNullOrWhiteSpace(pattern) + ? WinAiGenerationResult.Failed(WinAiFailure.ModelError, "The language model did not return a usable pattern.") + : WinAiGenerationResult.Ok(pattern); } /// diff --git a/Text-Grab/Views/EditTextWindow.xaml b/Text-Grab/Views/EditTextWindow.xaml index 156b3435..0bee15ab 100644 --- a/Text-Grab/Views/EditTextWindow.xaml +++ b/Text-Grab/Views/EditTextWindow.xaml @@ -443,6 +443,10 @@ x:Name="SummarizeMenuItem" Click="SummarizeMenuItem_Click" Header="_Summarize Paragraph" /> + + + + + + + + + + + + + + + - + + Header="New Window with Selected _Text" + InputGestureText="Ctrl + N" /> + + + + + + + + + + + + + + + + + + + + + + + private LiveAudioTranscriber? _liveTranscriber; + + /// Which source live transcription captures from (microphone or system loopback). + private LiveCaptureSource _liveCaptureSource = LiveCaptureSource.Microphone; + + /// Cancels an in-progress audio-file transcription; non-null only while one is running. + private CancellationTokenSource? _transcriptionCts; private CancellationTokenSource? cancellationTokenForDirOCR; private readonly string historyId = string.Empty; private int numberOfContextMenuItems; @@ -429,6 +438,11 @@ public void SetBottomBarButtons() else BottomBarText.Visibility = Visibility.Collapsed; + LiveTranscriptionToggleButton.Visibility = + DefaultSettings.EtwShowTranscribe && AudioTranscriptionUtilities.IsAudioTranscriptionSupported() + ? Visibility.Visible + : Visibility.Collapsed; + foreach (CollapsibleButton collapsibleButton in buttons) BottomBarButtons.Children.Add(collapsibleButton); @@ -2433,6 +2447,15 @@ internal void LimitNumberOfCharsPerLine(int numberOfChars, SpotInLine spotInLine internal async void OpenPath(string pathOfFileToOpen, bool isMultipleFiles = false) { + // Audio files are transcribed on-device rather than opened as text. Routing this here means + // CLI arguments, File > Open, and drag/drop all reach the same transcription path. + if (AudioTranscriptionUtilities.IsAudioFile(pathOfFileToOpen)) + { + AudioDebugLog.Write($"OpenPath: audio file detected, routing to transcription: {pathOfFileToOpen}"); + await TranscribeAudioFilesAsync([pathOfFileToOpen]); + return; + } + ResetSpreadsheetUndoHistory(); (string TextContent, OpenContentKind KindOpened) = await IoUtilities.GetContentFromPath(pathOfFileToOpen, isMultipleFiles, selectedILanguage); bool shouldTrackOpenedFile = KindOpened == OpenContentKind.TextFile && !isMultipleFiles; @@ -2662,6 +2685,10 @@ private void UpdateCachedClipboardOcrState() private void CaptureMenuItem_SubmenuOpened(object sender, RoutedEventArgs e) { LoadLanguageMenuItems(LanguageMenuItem); + } + + private void GrabTemplateMenuItem_SubmenuOpened(object sender, RoutedEventArgs e) + { LoadGrabTemplateMenuItems(GrabTemplateMenuItem); } @@ -2862,7 +2889,7 @@ private void ETWindow_DragOver(object sender, System.Windows.DragEventArgs e) e.Handled = true; } - private void ETWindow_Drop(object sender, System.Windows.DragEventArgs e) + private async void ETWindow_Drop(object sender, System.Windows.DragEventArgs e) { if (e.Data.GetDataPresent("Text")) return; @@ -2871,26 +2898,256 @@ private void ETWindow_Drop(object sender, System.Windows.DragEventArgs e) e.Handled = true; Mouse.OverrideCursor = System.Windows.Input.Cursors.Wait; - if (e.Data.GetDataPresent(System.Windows.DataFormats.FileDrop, true)) + try { - string[]? fileNames = e.Data.GetData(System.Windows.DataFormats.FileDrop, true) as string[]; - // Check for a single file or folder. - if (fileNames?.Length is 1) + if (!e.Data.GetDataPresent(System.Windows.DataFormats.FileDrop, true)) + return; + + if (e.Data.GetData(System.Windows.DataFormats.FileDrop, true) is not string[] fileNames) + return; + + List existingFiles = [.. fileNames.Where(File.Exists)]; + if (existingFiles.Count == 0) + return; + + // Audio files are transcribed on-device rather than opened as text. + List audioFiles = [.. existingFiles.Where(AudioTranscriptionUtilities.IsAudioFile)]; + List otherFiles = [.. existingFiles.Where(path => !AudioTranscriptionUtilities.IsAudioFile(path))]; + + bool openAsMultiple = existingFiles.Count > 1; + foreach (string possibleFilePath in otherFiles) + OpenPath(possibleFilePath, openAsMultiple); + + if (audioFiles.Count > 0) { - // Check for a file (a directory will return false). - if (File.Exists(fileNames[0])) - OpenPath(fileNames[0], false); + // Drop the wait cursor; TranscribeAudioFilesAsync shows the loading overlay instead. + Mouse.OverrideCursor = null; + await TranscribeAudioFilesAsync(audioFiles); } - else if (fileNames?.Length > 1) + } + finally + { + Mouse.OverrideCursor = null; + } + } + + /// + /// Transcribes one or more dropped audio files on-device, streaming each Whisper segment into the + /// editor as it is recognized. The status bar stays non-blocking so the user can cancel; because + /// every segment is inserted as it arrives, cancelling keeps all text transcribed so far. + /// (if provided) biases Whisper toward names/jargon it might otherwise + /// mishear; it applies only to this call, nothing is persisted. + /// (if provided) reports progress across all files combined + /// (0.0-1.0), so a caller such as can drive its own progress bar. + /// + internal async Task TranscribeAudioFilesAsync(IList audioFiles, string? hotWords = null, IProgress? overallProgress = null) + { + AudioDebugLog.Write($"TranscribeAudioFilesAsync: START with {audioFiles.Count} file(s). Log: {AudioDebugLog.LogPath}"); + + if (!AudioTranscriptionUtilities.IsAudioTranscriptionSupported()) + { + await new Wpf.Ui.Controls.MessageBox { - foreach (string possibleFilePath in fileNames) + Title = "Audio Transcription Unavailable", + Content = "Audio transcription isn't available on this device.", + CloseButtonText = "OK" + }.ShowDialogAsync(); + return; + } + + // Guard against a second transcription starting while one is already running. + if (_transcriptionCts is not null) + { + AudioDebugLog.Write("TranscribeAudioFilesAsync: ignored — a transcription is already in progress"); + return; + } + + bool multiple = audioFiles.Count > 1; + bool previousIsReadOnly = PassedTextControl.IsReadOnly; + string? errorMessage = null; + bool cancelled = false; + + // Everything from here runs inside the try: the guard above keys off _transcriptionCts, so a + // throw during setup that left it assigned would block every later transcription in this window. + try + { + _transcriptionCts = new CancellationTokenSource(); + CancellationToken cancellationToken = _transcriptionCts.Token; + + // Make the editor read-only (not disabled) so segments can stream in but the user can't type + // in the middle of the stream. AppendText still works while read-only. + PassedTextControl.IsReadOnly = true; + TranscriptionCancelButton.IsEnabled = true; + TranscriptionStatusText.Text = multiple ? "Transcribing audio files…" : "Transcribing audio…"; + TranscriptionProgressBar.Visibility = Visibility.Collapsed; + TranscriptionProgressBar.Value = 0; + TranscriptionStatusBar.Visibility = Visibility.Visible; + + // Status text (model download, "Transcribing…") updates the status bar; each recognized + // segment is appended straight into the editor as it arrives. + Progress statusProgress = new(message => TranscriptionStatusText.Text = message); + Progress segmentProgress = new(AppendTranscriptionText); + + // Give the caret a clean starting line so streamed text doesn't run into existing content. + EnsureTranscriptionInsertionPoint(); + + for (int i = 0; i < audioFiles.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + string audioFile = audioFiles[i]; + + if (multiple) + AppendTranscriptionText($"# {Path.GetFileName(audioFile)}{Environment.NewLine}"); + + int fileIndex = i; + Progress clipProgress = new(fraction => { - if (File.Exists(possibleFilePath)) - OpenPath(possibleFilePath, true); - } + double overall = (fileIndex + fraction) / audioFiles.Count; + TranscriptionProgressBar.Visibility = Visibility.Visible; + TranscriptionProgressBar.Value = overall * 100; + TranscriptionStatusText.Text = multiple + ? $"Transcribing audio files… ({fileIndex + 1}/{audioFiles.Count}) {fraction:P0}" + : $"Transcribing audio… {fraction:P0}"; + overallProgress?.Report(overall); + }); + + string transcription = await AudioTranscriptionUtilities.TranscribeAudioFileAsync( + audioFile, hotWords, statusProgress, segmentProgress, cancellationToken, + includeTimecodes: DefaultSettings.IncludeTimecodesInTranscription, + clipProgress: clipProgress); + + if (string.IsNullOrWhiteSpace(transcription)) + AppendTranscriptionText("(no speech recognized)"); + + // Blank line between files (and after the last, trimmed on sync). + AppendTranscriptionText(Environment.NewLine + Environment.NewLine); } } - Mouse.OverrideCursor = null; + catch (OperationCanceledException) + { + // User cancelled: everything already streamed into the editor is kept — no error dialog. + cancelled = true; + AudioDebugLog.Write("TranscribeAudioFilesAsync: cancelled by user; keeping transcribed-so-far text"); + } + catch (Exception ex) + { + // Surface the real reason (model not ready, unsupported format, etc.) instead of failing silently. + Debug.WriteLine($"Audio transcription error: {ex}"); + AudioDebugLog.Write($"TranscribeAudioFilesAsync: ERROR {ex}"); + errorMessage = ex.Message; + } + finally + { + PassedTextControl.IsReadOnly = previousIsReadOnly; + TranscriptionStatusBar.Visibility = Visibility.Collapsed; + _transcriptionCts?.Dispose(); + _transcriptionCts = null; + + // The transcript was streamed into the raw text box, so the active editor is what needs + // updating. Syncing the other way (as this used to) pushed the markdown document / table + // back over PassedTextControl.Text, round-tripping the whole transcript through the + // serializer for nothing. + SyncActiveEditorFromText(); + } + + AudioDebugLog.Write($"TranscribeAudioFilesAsync: END (cancelled={cancelled}, error={errorMessage is not null})"); + + if (errorMessage is not null) + { + // A failure raised as the window closes has nowhere to show: the dialog would be owned by + // a torn-down window. + if (IsLoaded) + await new Wpf.Ui.Controls.MessageBox + { + Title = "Audio Transcription Failed", + Content = errorMessage, + CloseButtonText = "OK" + }.ShowDialogAsync(); + } + else if (!cancelled && DefaultSettings.NotifyOnTranscriptionComplete) + { + string fileDescription = multiple + ? $"{audioFiles.Count} files" + : Path.GetFileName(audioFiles[0]); + NotificationUtilities.ShowTranscriptionCompleteToast(fileDescription); + } + } + + private void TranscriptionCancelButton_Click(object sender, RoutedEventArgs e) => CancelAudioTranscription(); + + /// + /// Requests cancellation of the running audio-file transcription. Text already streamed into the + /// editor is preserved. Called from the status bar's own Cancel button, and externally by + /// so its Cancel button can stop a transcription it started. + /// + internal void CancelAudioTranscription() + { + if (_transcriptionCts is null) + return; + + TranscriptionCancelButton.IsEnabled = false; + TranscriptionStatusText.Text = "Cancelling…"; + _transcriptionCts.Cancel(); + } + + /// + /// Ensures the caret sits on a fresh line at the end of the document before streaming begins, so + /// the first transcribed segment doesn't run into whatever text is already there. + /// + private void EnsureTranscriptionInsertionPoint() + { + string existingText = PassedTextControl.Text; + if (!string.IsNullOrEmpty(existingText) && existingText[^1] is not '\n' and not '\r') + PassedTextControl.AppendText(Environment.NewLine); + + PassedTextControl.CaretIndex = PassedTextControl.Text.Length; + PassedTextControl.ScrollToEnd(); + } + + /// + /// Appends a streamed transcription fragment to the end of the editor. Works while the editor is + /// read-only (AppendText bypasses the read-only guard), keeping the caret and view at the end. + /// + private void AppendTranscriptionText(string text) + { + if (string.IsNullOrEmpty(text)) + return; + + PassedTextControl.AppendText(text); + PassedTextControl.CaretIndex = PassedTextControl.Text.Length; + PassedTextControl.ScrollToEnd(); + } + + /// + /// Inserts transcribed text into the raw-text editor at the caret, adding a leading separator + /// when appending to existing content. + /// + private void InsertTranscribedText(string text, bool separateWithSpace = false) + { + if (string.IsNullOrEmpty(text)) + return; + + string existingText = PassedTextControl.Text; + if (!string.IsNullOrEmpty(existingText) && PassedTextControl.SelectionStart == existingText.Length) + { + char lastChar = existingText[^1]; + if (separateWithSpace) + { + if (!char.IsWhiteSpace(lastChar)) + text = " " + text; + } + else if (lastChar is not '\n' and not '\r') + { + text = Environment.NewLine + text; + } + } + + AddCopiedTextToTextBox(text); + + // Same as the file path: the phrase went into the raw text box, so the sync has to run in that + // direction. Pulling from the active editor instead round-tripped every phrase through the + // markdown/table serializer. + SyncActiveEditorFromText(); } private void FeedbackMenuItem_Click(object sender, RoutedEventArgs ev) @@ -4721,6 +4978,20 @@ private void SyncTextFromActiveEditor() SyncMarkdownTextFromDocument(); } + /// + /// The mirror of : pushes PassedTextControl.Text out + /// to whichever editor is showing, the same way does. + /// Used by anything that writes straight into the raw text box while another editor is active — + /// streamed audio transcription, most of all. A no-op in raw-text mode. + /// + private void SyncActiveEditorFromText() + { + if (editorMode == EtwEditorMode.Spreadsheet) + RefreshSpreadsheetFromText(); + else if (editorMode == EtwEditorMode.Markdown) + RefreshMarkdownFromText(); + } + private bool SaveCurrentDocument(bool saveAs = false) { SyncTextFromActiveEditor(); @@ -5123,6 +5394,10 @@ private void SetupRoutedCommands() _ = findAndReplaceCommand.InputGestures.Add(new KeyGesture(Key.F, ModifierKeys.Control | ModifierKeys.Shift)); _ = CommandBindings.Add(new CommandBinding(findAndReplaceCommand, FindAndReplaceMenuItem_Click)); + RoutedCommand newWindowWithSelectionCommand = new(); + _ = newWindowWithSelectionCommand.InputGestures.Add(new KeyGesture(Key.N, ModifierKeys.Control)); + _ = CommandBindings.Add(new CommandBinding(newWindowWithSelectionCommand, NewWindowWithText_Clicked)); + List searchers = Singleton.Instance.WebSearchers; foreach (WebSearchUrlModel searcher in searchers) @@ -5750,6 +6025,16 @@ private void Window_Activated(object sender, EventArgs e) private void Window_Closed(object sender, EventArgs e) { + // Stop any in-progress audio-file transcription so it doesn't touch a torn-down window. + _transcriptionCts?.Cancel(); + + if (_liveTranscriber is not null) + { + _liveTranscriber.PhraseRecognized -= LiveTranscriber_PhraseRecognized; + _liveTranscriber.Dispose(); + _liveTranscriber = null; + } + DetachSpreadsheetColumnWidthTracking(); System.Windows.DataObject.RemovePastingHandler(MarkdownEditorControl, MarkdownEditorControl_Pasting); @@ -5904,6 +6189,18 @@ private void Window_Loaded(object sender, RoutedEventArgs e) TranslateToSystemLanguageMenuItem.Header = $"Translate to {systemLanguage}"; } + // Audio transcription runs locally on the CPU via Whisper (whisper.cpp), so it's available + // on every supported device. The Whisper model is downloaded on first use. + if (AudioTranscriptionUtilities.IsAudioTranscriptionSupported()) + { + CaptureTranscribeAudioMenuItem.Visibility = Visibility.Visible; + TranscriptionOptionsMenuItem.Visibility = Visibility.Visible; + OpenAudioVideoMenuItem.Visibility = Visibility.Visible; + SyncTranscriptionModelMenu(); + TranscribeJustIconMenuItem.IsChecked = DefaultSettings.TranscribeButtonJustIcon; + LiveTranscriptionLabel.Visibility = DefaultSettings.TranscribeButtonJustIcon ? Visibility.Collapsed : Visibility.Visible; + } + // Initialize selectedILanguage with the last used OCR language from settings // This ensures that when images are dropped or pasted, the correct language is used selectedILanguage = LanguageUtilities.GetOCRLanguage(); @@ -6458,14 +6755,100 @@ private async void SummarizeMenuItem_Click(object sender, RoutedEventArgs e) { SetToLoading("Summarizing..."); + WinAiGenerationResult? failure = null; + try { - await ApplySelectedTextOrAllTextTransformAsync(text => WindowsAiUtilities.SummarizeParagraph(text)); + string sourceText = GetSelectedTextOrAllText(); + WinAiGenerationResult result = await WindowsAiUtilities.SummarizeParagraph(sourceText); + + // Only open a window on success: a failure message shown as document text reads like a + // real summary and can be saved as one. + if (result.Text is null) + failure = result; + else + OpenTextInNewEditTextWindow(result.Text); + } + catch (Exception ex) + { + Debug.WriteLine($"Summarize exception: {ex.Message}"); + failure = WinAiGenerationResult.Failed(WinAiFailure.ModelError, $"Summarizing failed: {ex.Message}"); } finally { SetToLoaded(); } + + if (failure is { } summaryFailure) + { + await new Wpf.Ui.Controls.MessageBox + { + Title = "Summarize Failed", + Content = summaryFailure.Message ?? "The text could not be summarized.", + CloseButtonText = "OK" + }.ShowDialogAsync(); + } + } + + private async void MeetingNotesMenuItem_Click(object sender, RoutedEventArgs e) + { + SetToLoading("Writing meeting notes..."); + + WinAiGenerationResult? failure = null; + + try + { + string sourceText = GetSelectedTextOrAllText(); + + // A long transcript is summarized part by part, so say which part is being read. + void OnProgress(string stage) => Dispatcher.Invoke(() => SetToLoading(stage)); + + WinAiGenerationResult result = await WinAiMeetingNotes.SummarizeAsync(sourceText, OnProgress); + + if (result.Text is null) + failure = result; + else + OpenTextInNewEditTextWindow(result.Text); + } + catch (Exception ex) + { + Debug.WriteLine($"Meeting notes exception: {ex.Message}"); + failure = WinAiGenerationResult.Failed(WinAiFailure.ModelError, $"Meeting notes failed: {ex.Message}"); + } + finally + { + SetToLoaded(); + } + + if (failure is { } notesFailure) + { + await new Wpf.Ui.Controls.MessageBox + { + Title = "Meeting Notes Failed", + Content = notesFailure.Message ?? "The text could not be written up as meeting notes.", + CloseButtonText = "OK" + }.ShowDialogAsync(); + } + } + + // Summarize and meeting-notes results open in a new window, leaving the source text untouched. + private static void OpenTextInNewEditTextWindow(string text) + { + EditTextWindow resultWindow = new(text, isEncoded: false); + + try + { + resultWindow.Show(); + } + catch (Exception ex) + { + _ = new Wpf.Ui.Controls.MessageBox + { + Title = ex.Message, + Content = "An error occurred while trying to open a new window. Please try again.", + CloseButtonText = "OK" + }.ShowDialogAsync(); + } } private void LearnAiMenuItem_Click(object sender, RoutedEventArgs e) @@ -6547,23 +6930,40 @@ private async Task PerformTranslationAsync(string targetLanguage) { SetToLoading($"Translating to {targetLanguage}..."); + // Captured from inside the transform so a failure can be reported after the text is applied + // instead of silently putting the original text back. + TranslationResult? failedResult = null; + try { - await ApplySelectedTextOrAllTextTransformAsync(text => WindowsAiUtilities.TranslateText(text, targetLanguage)); + await ApplySelectedTextOrAllTextTransformAsync(async text => + { + TranslationResult result = await WinAiTranslator.TranslateAsync(text, targetLanguage); + + if (!result.Succeeded) + failedResult ??= result; + + return result.Text; + }); } catch (Exception ex) { - await new Wpf.Ui.Controls.MessageBox - { - Title = "Translation Error", - Content = $"Translation failed: {ex.Message}", - CloseButtonText = "OK" - }.ShowDialogAsync(); + failedResult = new TranslationResult(string.Empty, TranslationFailure.ModelError, $"Translation failed: {ex.Message}"); } finally { SetToLoaded(); } + + if (failedResult is { } failure) + { + await new Wpf.Ui.Controls.MessageBox + { + Title = failure.Failure is TranslationFailure.NotNeeded ? "Nothing to Translate" : "Translation Failed", + Content = failure.Message ?? "The text could not be translated.", + CloseButtonText = "OK" + }.ShowDialogAsync(); + } } private async void ExtractRegexMenuItem_Click(object sender, RoutedEventArgs e) @@ -6583,10 +6983,10 @@ private async void ExtractRegexMenuItem_Click(object sender, RoutedEventArgs e) SetToLoading("Extracting RegEx pattern..."); - string regexPattern; + WinAiGenerationResult extraction; try { - regexPattern = await WindowsAiUtilities.ExtractRegex(textDescription); + extraction = await WindowsAiUtilities.ExtractRegex(textDescription); } catch (Exception ex) { @@ -6603,12 +7003,14 @@ private async void ExtractRegexMenuItem_Click(object sender, RoutedEventArgs e) SetToLoaded(); - if (string.IsNullOrWhiteSpace(regexPattern)) + if (extraction.Text is not string regexPattern) { + // The shared language model reports why it could not answer, so show that instead of a + // guess about what went wrong. await new Wpf.Ui.Controls.MessageBox { Title = "Extraction Failed", - Content = "Failed to extract a regex pattern. The AI service may not be available or could not generate a pattern.", + Content = extraction.Message ?? "Failed to extract a regex pattern.", CloseButtonText = "OK" }.ShowDialogAsync(); return; @@ -6661,6 +7063,191 @@ private async void ExtractRegexMenuItem_Click(object sender, RoutedEventArgs e) } } + private async void LiveTranscriptionToggleButton_Checked(object sender, RoutedEventArgs e) + { + _liveTranscriber ??= new LiveAudioTranscriber(); + _liveTranscriber.PhraseRecognized -= LiveTranscriber_PhraseRecognized; + _liveTranscriber.PhraseRecognized += LiveTranscriber_PhraseRecognized; + + SetLiveTranscriptionUi(true, _liveCaptureSource switch + { + LiveCaptureSource.SystemAudio => "Starting (system)…", + LiveCaptureSource.MicrophoneAndSystemAudio => "Starting (mic + system)…", + _ => "Starting…", + }); + + bool started; + try + { + started = await _liveTranscriber.StartAsync(_liveCaptureSource); + } + catch (Exception ex) + { + Debug.WriteLine($"Live transcription failed to start: {ex.Message}"); + started = false; + } + + if (!started) + { + _liveTranscriber.PhraseRecognized -= LiveTranscriber_PhraseRecognized; + SetLiveTranscriptionUi(false); + + // Setting IsChecked=false re-enters Unchecked, which is a no-op safe path here. + if (LiveTranscriptionToggleButton.IsChecked is true) + LiveTranscriptionToggleButton.IsChecked = false; + + string reason = _liveCaptureSource switch + { + LiveCaptureSource.SystemAudio => "Couldn't capture system audio. Make sure a playback device is active.", + LiveCaptureSource.MicrophoneAndSystemAudio => "Couldn't start microphone and system audio capture. Make sure a microphone is connected, Text Grab has microphone access in Windows privacy settings, and a playback device is active.", + _ => "Couldn't start microphone capture. Make sure a microphone is connected and that Text Grab has microphone access in Windows privacy settings.", + }; + await new Wpf.Ui.Controls.MessageBox + { + Title = "Couldn't Start Transcription", + Content = reason, + CloseButtonText = "OK" + }.ShowDialogAsync(); + return; + } + + SetLiveTranscriptionUi(true); + } + + private void LiveTranscriptionToggleButton_Unchecked(object sender, RoutedEventArgs e) + { + if (_liveTranscriber is not null) + { + _liveTranscriber.PhraseRecognized -= LiveTranscriber_PhraseRecognized; + _liveTranscriber.Stop(); + } + + SetLiveTranscriptionUi(false); + } + + private void CaptureTranscribeAudioMenuItem_Click(object sender, RoutedEventArgs e) + { + bool transcribe = CaptureTranscribeAudioMenuItem.IsChecked; + if (LiveTranscriptionToggleButton.IsChecked != transcribe) + LiveTranscriptionToggleButton.IsChecked = transcribe; + } + + private void OpenAudioVideoMenuItem_Click(object sender, RoutedEventArgs e) + { + OpenMediaWindow openMediaWindow = new() { Owner = this }; + openMediaWindow.Show(); + } + + private void LiveSourceMenuItem_Click(object sender, RoutedEventArgs e) + { + if (sender is not MenuItem menuItem || menuItem.Tag is not string tag + || !Enum.TryParse(tag, out LiveCaptureSource selectedSource)) + return; + + _liveCaptureSource = selectedSource; + LiveSourceMicMenuItem.IsChecked = selectedSource == LiveCaptureSource.Microphone; + LiveSourceSystemMenuItem.IsChecked = selectedSource == LiveCaptureSource.SystemAudio; + LiveSourceBothMenuItem.IsChecked = selectedSource == LiveCaptureSource.MicrophoneAndSystemAudio; + CaptureLiveSourceMicMenuItem.IsChecked = selectedSource == LiveCaptureSource.Microphone; + CaptureLiveSourceSystemMenuItem.IsChecked = selectedSource == LiveCaptureSource.SystemAudio; + CaptureLiveSourceBothMenuItem.IsChecked = selectedSource == LiveCaptureSource.MicrophoneAndSystemAudio; + + // If a session is already running, restart it on the newly chosen source. + if (LiveTranscriptionToggleButton.IsChecked is true) + { + LiveTranscriptionToggleButton.IsChecked = false; // stops via Unchecked + LiveTranscriptionToggleButton.IsChecked = true; // restarts via Checked with new source + } + else + { + SetLiveTranscriptionUi(false); + } + } + + private void TranscriptionModelMenuItem_Click(object sender, RoutedEventArgs e) + { + if (sender is not MenuItem menuItem || menuItem.Tag is not string tag) + return; + + DefaultSettings.AudioTranscriptionModel = tag; + DefaultSettings.Save(); + SyncTranscriptionModelMenu(); + + // If a session is running, restart it so the newly selected model is loaded. + if (LiveTranscriptionToggleButton.IsChecked is true) + { + LiveTranscriptionToggleButton.IsChecked = false; // stops via Unchecked + LiveTranscriptionToggleButton.IsChecked = true; // restarts via Checked with new model + } + } + + private void TranscribeJustIconMenuItem_Click(object sender, RoutedEventArgs e) + { + bool justIcon = TranscribeJustIconMenuItem.IsChecked; + DefaultSettings.TranscribeButtonJustIcon = justIcon; + DefaultSettings.Save(); + LiveTranscriptionLabel.Visibility = justIcon ? Visibility.Collapsed : Visibility.Visible; + } + + /// Reflects the persisted transcription-model choice in the context-menu check marks. + private void SyncTranscriptionModelMenu() + { + string current = DefaultSettings.AudioTranscriptionModel; + ModelTinyEnglishMenuItem.IsChecked = current == "TinyEnglish"; + ModelBaseEnglishMenuItem.IsChecked = current == "BaseEnglish"; + ModelSmallMultilingualMenuItem.IsChecked = current == "SmallMultilingual"; + CaptureModelTinyEnglishMenuItem.IsChecked = ModelTinyEnglishMenuItem.IsChecked; + CaptureModelBaseEnglishMenuItem.IsChecked = ModelBaseEnglishMenuItem.IsChecked; + CaptureModelSmallMultilingualMenuItem.IsChecked = ModelSmallMultilingualMenuItem.IsChecked; + + // Anything else (including the default) falls back to balanced multilingual. + ModelBaseMultilingualMenuItem.IsChecked = + !ModelTinyEnglishMenuItem.IsChecked + && !ModelBaseEnglishMenuItem.IsChecked + && !ModelSmallMultilingualMenuItem.IsChecked; + CaptureModelBaseMultilingualMenuItem.IsChecked = ModelBaseMultilingualMenuItem.IsChecked; + } + + private void LiveTranscriber_PhraseRecognized(object? sender, string recognizedText) + { + // Recognition events arrive on a background thread; marshal to the UI thread without + // blocking the recognizer (BeginInvoke, not Invoke). + Dispatcher.BeginInvoke(() => + { + string phrase = recognizedText.Trim(); + if (phrase.Length == 0) + return; + + // Insert at the end of the document, separating phrases with a single space. + PassedTextControl.Select(PassedTextControl.Text.Length, 0); + InsertTranscribedText(phrase, separateWithSpace: true); + }); + } + + private void SetLiveTranscriptionUi(bool active, string? label = null) + { + bool systemAudio = _liveCaptureSource == LiveCaptureSource.SystemAudio; + bool both = _liveCaptureSource == LiveCaptureSource.MicrophoneAndSystemAudio; + CaptureTranscribeAudioMenuItem.IsChecked = active; + + if (label is not null) + LiveTranscriptionLabel.Text = label; + else if (active) + LiveTranscriptionLabel.Text = both ? "Listening (mic + system)…" : systemAudio ? "Listening (system)…" : "Listening…"; + else + LiveTranscriptionLabel.Text = both ? "Transcribe (mic + system)" : systemAudio ? "Transcribe (system)" : "Transcribe"; + + // Speaker icon for system audio, mic icon for microphone (and mic+system); pulse variant while active. + LiveTranscriptionIcon.Symbol = systemAudio && !both + ? SymbolRegular.Speaker224 + : (active ? SymbolRegular.MicPulse24 : SymbolRegular.Mic24); + + if (active) + LiveTranscriptionIcon.Foreground = System.Windows.Media.Brushes.OrangeRed; + else + LiveTranscriptionIcon.ClearValue(ForegroundProperty); + } + private void SetToLoading(string message = "") { IsEnabled = false; diff --git a/Text-Grab/Views/GrabFrame.xaml.cs b/Text-Grab/Views/GrabFrame.xaml.cs index 989920e8..0f31ad2f 100644 --- a/Text-Grab/Views/GrabFrame.xaml.cs +++ b/Text-Grab/Views/GrabFrame.xaml.cs @@ -1,4 +1,4 @@ -using Dapplo.Windows.User32; +using Dapplo.Windows.User32; using Fasetto.Word; using System; using System.Collections.Generic; @@ -104,9 +104,9 @@ public partial class GrabFrame : Window private string translationTargetLanguage = "English"; private readonly DispatcherTimer translationTimer = new(); private readonly Dictionary originalTexts = []; - private readonly SemaphoreSlim translationSemaphore = new(3); // Limit to 3 concurrent translations private int totalWordsToTranslate = 0; private int translatedWordsCount = 0; + private bool isTranslating = false; private CancellationTokenSource? translationCancellationTokenSource; private readonly List pdfTextLineOverlays = []; private CancellationTokenSource? _pdfPageNavCts; @@ -1435,12 +1435,11 @@ private void CleanupGrabFrame() translationTimer.Stop(); translationTimer.Tick -= TranslationTimer_Tick; - translationSemaphore.Dispose(); translationCancellationTokenSource?.Cancel(); translationCancellationTokenSource?.Dispose(); // Dispose the shared translation model during cleanup to prevent resource leaks - WindowsAiUtilities.DisposeTranslationModel(); + WinAiTranslator.ReleaseModel(); MinimizeButton.Click -= OnMinimizeButtonClick; RestoreButton.Click -= OnRestoreButtonClick; @@ -2427,7 +2426,7 @@ private async Task DrawOcrRectanglesAsync(string searchWord = "") reSearchTimer.Start(); // Trigger translation if enabled - if (isTranslationEnabled && WindowsAiUtilities.CanDeviceUseWinAI()) + if (isTranslationEnabled && WinAiTranslator.IsAvailable()) { translationTimer.Stop(); translationTimer.Start(); @@ -2492,7 +2491,7 @@ private async Task DrawPdfRectanglesAsync(string searchWord = "") isDrawing = false; reSearchTimer.Start(); - if (isTranslationEnabled && WindowsAiUtilities.CanDeviceUseWinAI()) + if (isTranslationEnabled && WinAiTranslator.IsAvailable()) { translationTimer.Stop(); translationTimer.Start(); @@ -2606,7 +2605,7 @@ private async Task DrawUiAutomationRectanglesAsync(string searchWord = "") reSearchTimer.Start(); - if (isTranslationEnabled && WindowsAiUtilities.CanDeviceUseWinAI()) + if (isTranslationEnabled && WinAiTranslator.IsAvailable()) { translationTimer.Stop(); translationTimer.Start(); @@ -6076,12 +6075,13 @@ private async void TranslateToggleButton_Click(object sender, RoutedEventArgs e) if (isChecked) { - if (!WindowsAiUtilities.CanDeviceUseWinAI()) + (bool available, string? reason) = WinAiTranslator.CheckAvailability(); + if (!available) { await new Wpf.Ui.Controls.MessageBox { Title = "Translation Not Available", - Content = "Windows AI is not available on this device. Translation requires Windows AI support.", + Content = reason ?? "Windows AI is not available on this device.", CloseButtonText = "OK" }.ShowDialogAsync(); TranslateToggleButton.IsChecked = false; @@ -6125,7 +6125,7 @@ private async void TranslateToggleButton_Click(object sender, RoutedEventArgs e) originalTexts.Clear(); // Dispose the translation model to free resources when not in use - WindowsAiUtilities.DisposeTranslationModel(); + WinAiTranslator.ReleaseModel(); } } } @@ -6173,7 +6173,7 @@ private async void TranslationTimer_Tick(object? sender, EventArgs e) { translationTimer.Stop(); - if (!isTranslationEnabled || !WindowsAiUtilities.CanDeviceUseWinAI()) + if (!isTranslationEnabled || !WinAiTranslator.IsAvailable()) return; await PerformTranslationAsync(); @@ -6184,6 +6184,13 @@ private async Task PerformTranslationAsync() if (translationCancellationTokenSource == null || translationCancellationTokenSource.IsCancellationRequested) return; + // The timer restarts on every draw / resize / OCR refresh, so a second pass can be kicked off + // while this one is still awaiting the model. Two passes share the progress counters and the + // streamed callbacks index into their own bordersToTranslate list, so the first run's results + // would land on the second run's word borders. One at a time. + if (isTranslating) + return; + ShowTranslationProgress(); totalWordsToTranslate = wordBorders.Count; @@ -6191,43 +6198,54 @@ private async Task PerformTranslationAsync() CancellationToken cancellationToken = translationCancellationTokenSource.Token; - // Translate all word borders with controlled concurrency (max 3 at a time) - List translationTasks = []; + // Every word box goes through the model together. Translating them one at a time meant one + // full on-device inference per word, which is why this used to take minutes on a busy frame + // and produced worse wording (each word was translated with no surrounding context). + List bordersToTranslate = []; + List textsToTranslate = []; - try + foreach (WordBorder wb in wordBorders) { - foreach (WordBorder wb in wordBorders) - { - if (cancellationToken.IsCancellationRequested) - break; + // Store original text if not already stored + if (!originalTexts.ContainsKey(wb)) + originalTexts[wb] = wb.Word; - // Store original text if not already stored - if (!originalTexts.ContainsKey(wb)) - originalTexts[wb] = wb.Word; + string originalText = originalTexts[wb]; + if (string.IsNullOrWhiteSpace(originalText)) + continue; - string originalText = originalTexts[wb]; - if (!string.IsNullOrWhiteSpace(originalText)) - { - translationTasks.Add(TranslateWordBorderAsync(wb, originalText, cancellationToken)); - } - else + bordersToTranslate.Add(wb); + textsToTranslate.Add(originalText); + } + + totalWordsToTranslate = bordersToTranslate.Count; + UpdateTranslationProgress(); + + string? failureMessage = null; + + // Set as late as possible — nothing above awaits, so no second pass can slip in before here, + // and a throw in the setup above can't leave the flag stuck on. + isTranslating = true; + + try + { + // Results stream back as the model generates them, so boxes fill in progressively. + BatchTranslationResult result = await WinAiTranslator.TranslateBatchAsync( + textsToTranslate, + translationTargetLanguage, + (index, translated) => Dispatcher.InvokeAsync(() => { + if (cancellationToken.IsCancellationRequested) + return; + + bordersToTranslate[index].Word = translated; translatedWordsCount++; UpdateTranslationProgress(); - } - } + }), + cancellationToken); - // Wait for all translations to complete or cancellation - // Use WhenAll with exception handling to gracefully handle cancellations - try - { - await Task.WhenAll(translationTasks); - } - catch (OperationCanceledException) - { - // Expected when cancellation is requested - Debug.WriteLine("Translation tasks cancelled during WhenAll"); - } + if (!result.Succeeded) + failureMessage = result.Message; if (!cancellationToken.IsCancellationRequested) { @@ -6241,11 +6259,29 @@ private async Task PerformTranslationAsync() catch (Exception ex) { Debug.WriteLine($"Translation error: {ex.Message}"); + failureMessage = $"Translation failed: {ex.Message}"; } finally { + isTranslating = false; HideTranslationProgress(); } + + // Turn translation back off on failure so the frame does not silently retry on every + // redraw, and tell the user what went wrong. + if (failureMessage is not null && !cancellationToken.IsCancellationRequested) + { + isTranslationEnabled = false; + TranslateToggleButton.IsChecked = false; + EnableTranslationMenuItem.IsChecked = false; + + await new Wpf.Ui.Controls.MessageBox + { + Title = "Translation Failed", + Content = failureMessage, + CloseButtonText = "OK" + }.ShowDialogAsync(); + } } private void ShowTranslationProgress() @@ -6266,52 +6302,9 @@ private void UpdateTranslationProgress() if (totalWordsToTranslate == 0) return; - double progress = (double)translatedWordsCount / totalWordsToTranslate * 100; - TranslationProgressBar.Value = progress; - TranslationCountText.Text = $"{translatedWordsCount}/{totalWordsToTranslate}"; - } - - private async Task TranslateWordBorderAsync(WordBorder wordBorder, string originalText, CancellationToken cancellationToken) - { - try - { - await translationSemaphore.WaitAsync(cancellationToken); - } - catch (OperationCanceledException) - { - // Semaphore wait was cancelled - exit gracefully - return; - } - - try - { - // Ensure cancellation is honored immediately before starting translation - cancellationToken.ThrowIfCancellationRequested(); - - string translatedText = await WindowsAiUtilities.TranslateText(originalText, translationTargetLanguage); - - // If cancellation was requested during translation, abort before updating UI state - cancellationToken.ThrowIfCancellationRequested(); - - wordBorder.Word = translatedText; - - translatedWordsCount++; - await Dispatcher.InvokeAsync(() => UpdateTranslationProgress()); - } - catch (OperationCanceledException) - { - // Expected during cancellation - don't propagate - Debug.WriteLine($"Translation cancelled for word: {originalText}"); - } - catch (Exception ex) - { - Debug.WriteLine($"Translation failed for '{originalText}': {ex.Message}"); - // On error, keep original text (don't update word border) - } - finally - { - translationSemaphore.Release(); - } + int completed = Math.Min(translatedWordsCount, totalWordsToTranslate); + TranslationProgressBar.Value = (double)completed / totalWordsToTranslate * 100; + TranslationCountText.Text = $"{completed}/{totalWordsToTranslate}"; } private void GetGrabFrameTranslationSettings() @@ -6320,7 +6313,7 @@ private void GetGrabFrameTranslationSettings() translationTargetLanguage = DefaultSettings.GrabFrameTranslationLanguage; // Hide translation button if Windows AI is not available - bool canUseWinAI = WindowsAiUtilities.CanDeviceUseWinAI(); + bool canUseWinAI = WinAiTranslator.IsAvailable(); translateToolAvailable = canUseWinAI; SetToolButtonVisibility(TranslateToggleButton, "Translate", canUseWinAI); TranslationMenuItem.Visibility = canUseWinAI ? Visibility.Visible : Visibility.Collapsed; diff --git a/Text-Grab/Views/OpenMediaWindow.xaml b/Text-Grab/Views/OpenMediaWindow.xaml new file mode 100644 index 00000000..3b7fb63e --- /dev/null +++ b/Text-Grab/Views/OpenMediaWindow.xaml @@ -0,0 +1,212 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Text-Grab/Views/OpenMediaWindow.xaml.cs b/Text-Grab/Views/OpenMediaWindow.xaml.cs new file mode 100644 index 00000000..827e6ae9 --- /dev/null +++ b/Text-Grab/Views/OpenMediaWindow.xaml.cs @@ -0,0 +1,153 @@ +using System; +using System.Collections.Generic; +using System.Windows; +using Text_Grab.Utilities; +using Wpf.Ui.Controls; + +namespace Text_Grab.Views; + +public partial class OpenMediaWindow : FluentWindow +{ + private string? selectedFilePath; + private EditTextWindow? transcribingOwner; + + public OpenMediaWindow() + { + InitializeComponent(); + App.SetTheme(); + + NotifyOnCompleteToggle.IsChecked = AppUtilities.TextGrabSettings.NotifyOnTranscriptionComplete; + IncludeTimecodesToggle.IsChecked = AppUtilities.TextGrabSettings.IncludeTimecodesInTranscription; + } + + private void BrowseButton_Click(object sender, RoutedEventArgs e) + { + Microsoft.Win32.OpenFileDialog dlg = new() + { + Filter = AudioTranscriptionUtilities.GetAudioFileFilter(), + DefaultDirectory = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments) + }; + + bool? result = dlg.ShowDialog(); + if (result is true) + UpdateFileInfo(dlg.FileName); + } + + private void UpdateFileInfo(string path) + { + FilePathTextBox.Text = path; + FileErrorText.Visibility = Visibility.Collapsed; + FileInfoPanel.Visibility = Visibility.Collapsed; + selectedFilePath = null; + StartTranscriptionButton.IsEnabled = false; + + try + { + AudioTranscriptionUtilities.AudioFileInfo info = AudioTranscriptionUtilities.GetAudioFileInfo(path); + + FileNameText.Text = info.FileName; + FileSizeText.Text = $"Size: {info.FileSizeBytes / (1024.0 * 1024.0):0.#} MB"; + FileDurationText.Text = $"Duration: {AudioTranscriptionUtilities.FormatTimecode(info.Duration)}"; + FileModelText.Text = $"Model: {WhisperModelInfo.DisplayName(AudioTranscriptionUtilities.CurrentModelChoice)}"; + FileInfoPanel.Visibility = Visibility.Visible; + + selectedFilePath = path; + StartTranscriptionButton.IsEnabled = true; + } + catch (Exception ex) + { + FileErrorText.Text = $"⚠ Couldn't read this file: {ex.Message}"; + FileErrorText.Visibility = Visibility.Visible; + } + } + + private void HotWordsLookupButton_Click(object sender, RoutedEventArgs e) + { + QuickSimpleLookup qsl = new() + { + DestinationTextBox = HotWordsTextBox, + IsPickerMode = true, + }; + qsl.Owner = this; + qsl.Show(); + } + + private void CancelButton_Click(object sender, RoutedEventArgs e) + { + if (transcribingOwner is not null) + { + // A transcription is running: stop it instead of just closing over it. The window + // closes itself once StartTranscriptionButton_Click's await returns. + transcribingOwner.CancelAudioTranscription(); + CancelButton.IsEnabled = false; + TranscribingStatusText.Text = "Cancelling…"; + return; + } + + Close(); + } + + private void NotifyOnCompleteToggle_Checked(object sender, RoutedEventArgs e) + { + AppUtilities.TextGrabSettings.NotifyOnTranscriptionComplete = true; + AppUtilities.TextGrabSettings.Save(); + } + + private void NotifyOnCompleteToggle_Unchecked(object sender, RoutedEventArgs e) + { + AppUtilities.TextGrabSettings.NotifyOnTranscriptionComplete = false; + AppUtilities.TextGrabSettings.Save(); + } + + private void IncludeTimecodesToggle_Checked(object sender, RoutedEventArgs e) + { + AppUtilities.TextGrabSettings.IncludeTimecodesInTranscription = true; + AppUtilities.TextGrabSettings.Save(); + } + + private void IncludeTimecodesToggle_Unchecked(object sender, RoutedEventArgs e) + { + AppUtilities.TextGrabSettings.IncludeTimecodesInTranscription = false; + AppUtilities.TextGrabSettings.Save(); + } + + private async void StartTranscriptionButton_Click(object sender, RoutedEventArgs e) + { + if (Owner is EditTextWindow etw && selectedFilePath is not null) + { + transcribingOwner = etw; + etw.Activate(); + SetTranscribingState(true); + + Progress progress = new(fraction => + { + TranscribingProgressBar.Value = fraction * 100; + TranscribingStatusText.Text = $"Transcribing… {fraction:P0}"; + }); + + await etw.TranscribeAudioFilesAsync([selectedFilePath], HotWordsTextBox.Text.Trim(), progress); + } + + Close(); + } + + /// + /// Toggles this window between "pick a file" and "transcription in progress": inputs and the + /// Start button are disabled/hidden, and the Cancel button switches to cancelling the running + /// transcription (owned by the main editor window) rather than just closing over it. + /// + private void SetTranscribingState(bool transcribing) + { + BrowseButton.IsEnabled = !transcribing; + HotWordsTextBox.IsEnabled = !transcribing; + HotWordsLookupButton.IsEnabled = !transcribing; + NotifyOnCompleteToggle.IsEnabled = !transcribing; + IncludeTimecodesToggle.IsEnabled = !transcribing; + + StartTranscriptionButton.Visibility = transcribing ? Visibility.Collapsed : Visibility.Visible; + TranscribingPanel.Visibility = transcribing ? Visibility.Visible : Visibility.Collapsed; + TranscribingProgressBar.Value = 0; + TranscribingStatusText.Text = "Transcribing…"; + CancelButton.IsEnabled = true; + } +} diff --git a/Text-Grab/Views/QuickSimpleLookup.xaml.cs b/Text-Grab/Views/QuickSimpleLookup.xaml.cs index c28adf32..63b903f3 100644 --- a/Text-Grab/Views/QuickSimpleLookup.xaml.cs +++ b/Text-Grab/Views/QuickSimpleLookup.xaml.cs @@ -59,6 +59,14 @@ private async void SearchBar_SearchChanged(object? sender, EventArgs e) public bool IsEditingDataGrid { get; set; } = false; public bool IsFromETW { get; set; } = false; + + /// + /// When set, a pick always writes straight into and closes the + /// window, regardless of EditWindowToggleButton — for callers (like the audio hot-words + /// picker) that want QSL purely as a value picker, not the ETW insert/clipboard flow. + /// + public bool IsPickerMode { get; set; } = false; + public List ItemsDictionary { get; set; } = []; #endregion Properties @@ -88,9 +96,15 @@ private static LookupItem ParseStringToLookupItem(char splitChar, string row) if (cells.FirstOrDefault() is string firstCell) newRow.ShortValue = firstCell; + // CSV rows are written as "ShortValue,LongValue" with no quoting/escaping (see + // LookupItem.ToCSVString), so a LongValue that itself contains commas splits into more than + // two cells here. Rejoin with the same delimiter to reconstitute the original value instead of + // losing the commas (space-joining is still correct for tab-split rows: typed/pasted multi-cell + // entries are meant to read as one space-separated phrase, not regain literal tab characters). + string joinSeparator = splitChar == ',' ? "," : " "; newRow.LongValue = ""; if (cells.Count > 1 && cells[1] is not null) - newRow.LongValue = string.Join(" ", cells.Skip(1).ToArray()); + newRow.LongValue = string.Join(joinSeparator, cells.Skip(1).ToArray()); newRow.Kind = kind; return newRow; @@ -569,7 +583,7 @@ private async void PutValueIntoClipboard(KeyboardModifiersDown? keysDown = null) if (stringBuilder.Length > 3 && stringBuilder.ToString().EndsWith("\r\n")) stringBuilder.Remove(stringBuilder.Length - 2, 2); - if (DestinationTextBox is not null && EditWindowToggleButton.IsChecked is true) + if (DestinationTextBox is not null && (IsPickerMode || EditWindowToggleButton.IsChecked is true)) { // Do it this way instead of append text because it inserts the text at the cursor // Then puts the cursor at the end of the newly added text diff --git a/docs/Configuring-LAF-Environment-Variables.md b/docs/Configuring-LAF-Environment-Variables.md new file mode 100644 index 00000000..41608fc6 --- /dev/null +++ b/docs/Configuring-LAF-Environment-Variables.md @@ -0,0 +1,73 @@ +# Configuring LAF environment variables + +Text-Grab's on-device text AI — translation, summarize, meeting notes, rewrite, text-to-table, extract RegEx — +runs on Phi Silica through `Microsoft.Windows.AI.Text.LanguageModel`. Microsoft ships that model as +a **Limited Access Feature (LAF)**, so an app has to unlock it before any call will succeed: + +```csharp +LimitedAccessFeatures.TryUnlockFeature( + "com.microsoft.windows.ai.languagemodel", + token, + $"{publisherId} has registered their use of com.microsoft.windows.ai.languagemodel with Microsoft and agrees to the terms of use."); +``` + +Without a valid token the call returns `LimitedAccessFeatureStatus.Unknown` and every AI request +fails with *"Access is denied. Limited Access Feature is not available: +com.microsoft.windows.ai.languagemodel."* + +Tokens are issued per publisher ID by Microsoft at . The publisher ID is +the hash half of the package family name (`40087JoeFinApps.TextGrab_`). + +**The token is a secret. It must never be committed to this repository.** + +## The two values + +| Name | Meaning | +| --- | --- | +| `LAF_TOKEN` | The unlock token Microsoft issued for `com.microsoft.windows.ai.languagemodel`. | +| `LAF_PUBLISHER_ID` | The publisher ID the token was issued against, used to build the usage string. | + +Set `LAF_PUBLISHER_ID` explicitly rather than relying on the fallback. `LimitedAccessFeatureUtilities` +derives it from `Package.Current.Id.FamilyName` when it is unset, and a locally sideloaded MSIX +signed with a development certificate has a *different* publisher hash than the Store package — so +the fallback would build a usage string the token was not issued for. + +## Local development + +Persist both values for your user account once, then restart Visual Studio or your shell so it picks +them up: + +```powershell +setx LAF_TOKEN "" +setx LAF_PUBLISHER_ID "" +``` + +`Text-Grab.csproj` defaults the `LafToken` / `LafPublisherId` MSBuild properties from these +environment variables, so ordinary `dotnet build` and Visual Studio builds bake the token in without +any extra flags. + +## Explicit build-time injection + +The properties can also be passed directly, which is what CI does: + +```powershell +dotnet build Text-Grab/Text-Grab.csproj -p:LafToken="" -p:LafPublisherId="" +``` + +The project maps them into assembly metadata: + +| MSBuild property | `AssemblyMetadata` key | +| --- | --- | +| `LafToken` | `LAF_TOKEN` | +| `LafPublisherId` | `LAF_PUBLISHER_ID` | + +At runtime `LimitedAccessFeatureUtilities.GetSetting` reads the assembly metadata first and falls +back to the environment variables, so a build with neither still runs — it just reports the feature +as unavailable instead of crashing. + +## CI + +`Release.yml` and `buildDev.yml` read the repository secrets `LAF_TOKEN` and `LAF_PUBLISHER_ID` and +forward them to every `dotnet publish` step. Add them under **Settings → Secrets and variables → +Actions**. If they are missing the build still succeeds; the published binaries simply ship without +working on-device text AI. diff --git a/global.json b/global.json index a5bf76b6..7a33100e 100644 --- a/global.json +++ b/global.json @@ -3,5 +3,8 @@ "version": "10.0.100", "allowPrerelease": false, "rollForward": "latestFeature" + }, + "test": { + "runner": "Microsoft.Testing.Platform" } } \ No newline at end of file