From d97350f0b08471f822a0b6fee462a0aa4c95eb5c Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Thu, 20 Aug 2026 23:13:04 +0200 Subject: [PATCH 1/4] feat(ai): add multimodal message input components Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8305953e-ce14-4a55-b291-77beaac47b0a --- .../AI/src/Blocks/DataContentBlock.cs | 17 + .../AI/src/Components/AudioCaptureButton.cs | 628 ++++++++++++++++++ .../Components/ConversationTurnRenderer.cs | 18 + .../AI/src/Components/LiveSpeechButton.cs | 488 ++++++++++++++ .../AI/src/Components/MediaContent.cs | 124 ++++ .../AI/src/Components/MessageAttachButton.cs | 384 +++++++++++ .../src/Components/MessageAttachmentList.cs | 136 ++++ .../AI/src/Components/MessageInput.cs | 582 +++++++++++++--- .../AI/src/Components/MessageInputContext.cs | 178 +++++ .../AI/src/Components/MessageList.cs | 50 +- .../AI/src/Components/MessageListContext.cs | 33 + .../AI/src/Components/MessageSendButton.cs | 102 +++ .../AI/src/Components/MessageStopButton.cs | 95 +++ src/Components/AI/src/Engine/AgentContext.cs | 129 +++- .../AI/src/Engine/ConversationTurn.cs | 20 +- src/Components/AI/src/Engine/UIAgent.cs | 119 ++-- .../Microsoft.AspNetCore.Components.AI.csproj | 3 + .../AI/src/Pipeline/BlockMappingPipeline.cs | 2 + .../AI/src/Pipeline/DataContentHandler.cs | 32 + src/Components/AI/src/PublicAPI.Unshipped.txt | 172 +++++ src/Components/AI/src/wwwroot/ai-chat.css | 242 +++++++ src/Components/AI/src/wwwroot/ai-chat.js | 349 ++++++++++ src/Components/Media/src/Audio.cs | 59 ++ ...crosoft.AspNetCore.Components.Media.csproj | 2 +- .../Media/src/PublicAPI.Unshipped.txt | 4 + 25 files changed, 3788 insertions(+), 180 deletions(-) create mode 100644 src/Components/AI/src/Blocks/DataContentBlock.cs create mode 100644 src/Components/AI/src/Components/AudioCaptureButton.cs create mode 100644 src/Components/AI/src/Components/LiveSpeechButton.cs create mode 100644 src/Components/AI/src/Components/MediaContent.cs create mode 100644 src/Components/AI/src/Components/MessageAttachButton.cs create mode 100644 src/Components/AI/src/Components/MessageAttachmentList.cs create mode 100644 src/Components/AI/src/Components/MessageInputContext.cs create mode 100644 src/Components/AI/src/Components/MessageSendButton.cs create mode 100644 src/Components/AI/src/Components/MessageStopButton.cs create mode 100644 src/Components/AI/src/Pipeline/DataContentHandler.cs create mode 100644 src/Components/AI/src/wwwroot/ai-chat.js create mode 100644 src/Components/Media/src/Audio.cs diff --git a/src/Components/AI/src/Blocks/DataContentBlock.cs b/src/Components/AI/src/Blocks/DataContentBlock.cs new file mode 100644 index 000000000000..86666bfe36da --- /dev/null +++ b/src/Components/AI/src/Blocks/DataContentBlock.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; + +namespace Microsoft.AspNetCore.Components.AI; + +/// +/// Represents binary content, such as an image, audio clip, video, or file, in a conversation. +/// +public class DataContentBlock : ContentBlock +{ + /// + /// Gets or sets the binary content represented by this block. + /// + public DataContent Content { get; set; } = default!; +} diff --git a/src/Components/AI/src/Components/AudioCaptureButton.cs b/src/Components/AI/src/Components/AudioCaptureButton.cs new file mode 100644 index 000000000000..88a6e57c10e4 --- /dev/null +++ b/src/Components/AI/src/Components/AudioCaptureButton.cs @@ -0,0 +1,628 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.AspNetCore.Components.Rendering; +using Microsoft.Extensions.AI; +using Microsoft.JSInterop; +using System.Linq; + +namespace Microsoft.AspNetCore.Components.AI; + +/// +/// Records audio in the browser and transcribes or attaches it to the nearest +/// . +/// +public sealed class AudioCaptureButton : ComponentBase, IAsyncDisposable +{ + private const string ModulePath = + "./_content/Microsoft.AspNetCore.Components.AI/ai-chat.js"; + + private readonly SpeechCallbacks _speechCallbacks; + private DotNetObjectReference? _speechCallbackReference; + private IJSObjectReference? _module; + private IJSObjectReference? _recorder; + private IJSObjectReference? _speechRecognizer; + private MessageInputContext? _subscribedContext; + private IDisposable? _changeSubscription; + private CancellationTokenSource? _operationCts; + private string _dictationPrefix = string.Empty; + private string _committedTranscript = string.Empty; + private bool _isRecording; + private bool _isTranscribing; + private bool _isDictating; + private bool _isSupported = true; + private bool _isDisposed; + + /// + /// Initializes a new instance of . + /// + public AudioCaptureButton() + { + _speechCallbacks = new SpeechCallbacks(this); + } + + /// + /// Gets or sets the nearest message input. + /// + [CascadingParameter] + public MessageInputContext Context { get; set; } = default!; + + /// + /// Gets or sets the JavaScript runtime used to access browser recording APIs. + /// + [Inject] + internal IJSRuntime JSRuntime { get; set; } = default!; + + /// + /// Gets or sets the maximum recording size in bytes. + /// + [Parameter] + public long MaximumBytes { get; set; } = 10 * 1024 * 1024; + + /// + /// Gets or sets whether a new recording replaces existing audio attachments. + /// + [Parameter] + public bool ReplaceExistingAudio { get; set; } = true; + + /// + /// Gets or sets whether captured audio is added to the outgoing message. + /// + [Parameter] + public bool AttachRecording { get; set; } = true; + + /// + /// Gets or sets whether browser speech recognition updates the composer while recording. + /// + [Parameter] + public bool ShowInterimTranscript { get; set; } + + /// + /// Gets or sets the browser speech-recognition language. The browser default is used when omitted. + /// + [Parameter] + public string? SpeechRecognitionLanguage { get; set; } + + /// + /// Gets or sets the accessible label shown before recording starts. + /// + [Parameter] + public string StartLabel { get; set; } = "Record audio"; + + /// + /// Gets or sets the accessible label shown while recording. + /// + [Parameter] + public string StopLabel { get; set; } = "Stop recording"; + + /// + /// Gets or sets custom button content based on whether recording is active. + /// + [Parameter] + public RenderFragment? ChildContent { get; set; } + + /// + /// Gets or sets a callback invoked when audio has been captured. + /// + [Parameter] + public EventCallback OnRecorded { get; set; } + + /// + /// Gets or sets an optional callback that transcribes captured audio into composer text. + /// + [Parameter] + public Func>? Transcribe { get; set; } + + /// + /// Gets or sets a callback invoked after captured audio has been transcribed. + /// + [Parameter] + public EventCallback OnTranscribed { get; set; } + + /// + /// Gets or sets additional attributes applied to the recording button. + /// + [Parameter(CaptureUnmatchedValues = true)] + public Dictionary? AdditionalAttributes { get; set; } + + /// + protected override void OnParametersSet() + { + if (ReferenceEquals(_subscribedContext, Context)) + { + return; + } + + _changeSubscription?.Dispose(); + _subscribedContext = Context; + _changeSubscription = Context.RegisterOnChanged( + () => _ = InvokeAsync(StateHasChanged)); + } + + /// + protected override void BuildRenderTree(RenderTreeBuilder builder) + { + var isActive = _isRecording || _isTranscribing; + var disabled = !_isSupported || + (!isActive && (Context.IsConversationBusy || Context.IsComposing)); + var label = isActive ? StopLabel : StartLabel; + + builder.OpenElement(0, "button"); + builder.AddMultipleAttributes(1, AdditionalAttributes); + builder.AddAttribute(2, "type", "button"); + builder.AddAttribute(3, "class", CssClass()); + builder.AddAttribute(4, "disabled", disabled); + builder.AddAttribute(5, "aria-label", label); + builder.AddAttribute(6, "aria-pressed", isActive ? "true" : "false"); + builder.AddAttribute( + 7, + "onclick", + EventCallback.Factory.Create(this, ToggleRecordingAsync)); + + if (ChildContent is not null) + { + builder.AddContent(8, ChildContent(isActive)); + } + else + { + builder.AddContent(9, label); + } + + builder.CloseElement(); + } + + /// + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (!firstRender) + { + return; + } + + try + { + _module = await JSRuntime.InvokeAsync("import", ModulePath); + _isSupported = await _module.InvokeAsync("isAudioCaptureSupported"); + if (!_isSupported) + { + Context.SetErrorMessage("Audio recording is not supported by this browser."); + } + await InvokeAsync(StateHasChanged); + } + catch (JSException) + { + _isSupported = false; + Context.SetErrorMessage("Audio recording could not be initialized."); + await InvokeAsync(StateHasChanged); + } + } + + private Task ToggleRecordingAsync() + { + if (_isTranscribing) + { + CancelTranscription(); + return Task.CompletedTask; + } + + return _isRecording ? StopRecordingAsync() : StartRecordingAsync(); + } + + private async Task StartRecordingAsync() + { + _operationCts?.Cancel(); + var operationCts = new CancellationTokenSource(); + _operationCts = operationCts; + Context.SetErrorMessage(null); + + try + { + _module ??= await JSRuntime.InvokeAsync("import", ModulePath); + _recorder ??= await _module.InvokeAsync( + "createAudioRecorder", + MaximumBytes); + await _recorder.InvokeVoidAsync("start"); + if (!ReferenceEquals(_operationCts, operationCts) || + operationCts.IsCancellationRequested) + { + operationCts.Dispose(); + return; + } + + _isRecording = true; + Context.SetComposing(true); + Context.SetStatusMessage("Recording audio."); + await StartInterimTranscriptionAsync(); + } + catch (JSException) + { + if (ReferenceEquals(_operationCts, operationCts)) + { + _operationCts = null; + _isRecording = false; + Context.SetComposing(false); + Context.SetErrorMessage( + "Microphone access was not available. Check browser permissions."); + } + + operationCts.Dispose(); + } + } + + private async Task StopRecordingAsync() + { + var operationCts = _operationCts + ?? throw new InvalidOperationException( + "Audio recording does not have an active operation."); + var cancellationToken = operationCts.Token; + _isRecording = false; + _isTranscribing = true; + await InvokeAsync(StateHasChanged); + + try + { + var hadInterimTranscript = _isDictating; + await StopInterimTranscriptionAsync(); + var recording = await _recorder!.InvokeAsync("stop"); + cancellationToken.ThrowIfCancellationRequested(); + + if (recording.TooLarge || recording.Size > MaximumBytes) + { + if (recording.StreamReference is not null) + { + await recording.StreamReference.DisposeAsync(); + } + Context.SetErrorMessage( + $"Audio recordings must be {FormatBytes(MaximumBytes)} or smaller."); + return; + } + + if (recording.StreamReference is null || recording.Size == 0) + { + Context.SetErrorMessage( + "The browser did not capture audio. Record for at least one second and check the microphone input level."); + return; + } + + var mediaType = string.IsNullOrWhiteSpace(recording.MimeType) + ? "audio/webm" + : recording.MimeType; + await using var streamReference = recording.StreamReference; + await using var stream = await streamReference.OpenReadStreamAsync( + MaximumBytes, + cancellationToken); + var content = await DataContent.LoadFromAsync( + stream, + mediaType, + cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + content.Name = + $"recording-{DateTimeOffset.UtcNow:yyyyMMdd-HHmmss}.{GetExtension(mediaType)}"; + + if (AttachRecording && ReplaceExistingAudio) + { + foreach (var attachment in Context.Attachments + .Where(attachment => attachment.HasTopLevelMediaType("audio")) + .ToArray()) + { + await Context.RemoveAttachmentAsync(attachment); + } + } + + if (AttachRecording) + { + await Context.AddAttachmentAsync(content); + } + cancellationToken.ThrowIfCancellationRequested(); + await OnRecorded.InvokeAsync(content); + if (!ReferenceEquals(_operationCts, operationCts)) + { + return; + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Transcribe is not null) + { + Context.SetStatusMessage("Transcribing audio."); + var transcript = await Transcribe(content, cancellationToken); + if (!ReferenceEquals(_operationCts, operationCts)) + { + return; + } + + cancellationToken.ThrowIfCancellationRequested(); + if (!string.IsNullOrWhiteSpace(transcript)) + { + Context.Text = hadInterimTranscript + ? AppendText(_dictationPrefix, transcript.Trim()) + : AppendText(Context.Text, transcript.Trim()); + await OnTranscribed.InvokeAsync(transcript.Trim()); + Context.SetStatusMessage("Voice transcription ready."); + } + else + { + Context.SetErrorMessage("No speech was recognized in the recording."); + } + } + else if (AttachRecording) + { + Context.SetStatusMessage("Audio recording attached."); + } + else + { + Context.SetErrorMessage( + "A transcription callback is required when recordings are not attached."); + } + await Context.FocusAsync(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (JSException) + { + if (ReferenceEquals(_operationCts, operationCts)) + { + Context.SetErrorMessage("The audio recording could not be completed."); + } + } + catch (IOException) + { + if (ReferenceEquals(_operationCts, operationCts)) + { + Context.SetErrorMessage("The captured audio could not be read."); + } + } + catch (InvalidOperationException exception) + { + if (ReferenceEquals(_operationCts, operationCts)) + { + Context.SetErrorMessage(exception.Message); + } + } + finally + { + if (ReferenceEquals(_operationCts, operationCts)) + { + _operationCts = null; + _isRecording = false; + _isTranscribing = false; + if (!_isDisposed) + { + Context.SetComposing(false); + } + } + + operationCts.Dispose(); + } + } + + private void CancelTranscription() + { + _operationCts?.Cancel(); + _isTranscribing = false; + Context.SetComposing(false); + Context.SetStatusMessage("Audio transcription canceled."); + } + + private async Task StartInterimTranscriptionAsync() + { + if (!ShowInterimTranscript || _module is null) + { + return; + } + + try + { + if (!await _module.InvokeAsync( + "isLiveSpeechRecognitionSupported")) + { + return; + } + + _speechCallbackReference ??= DotNetObjectReference.Create(_speechCallbacks); + _speechRecognizer ??= await _module.InvokeAsync( + "createLiveSpeechRecognizer", + _speechCallbackReference, + SpeechRecognitionLanguage); + _dictationPrefix = Context.Text.Trim(); + _committedTranscript = string.Empty; + _isDictating = true; + await _speechRecognizer.InvokeVoidAsync("start"); + Context.SetStatusMessage("Recording and transcribing."); + } + catch (JSException) + { + _isDictating = false; + Context.SetStatusMessage("Recording audio. Live transcription is unavailable."); + } + } + + private async Task StopInterimTranscriptionAsync() + { + if (!_isDictating || _speechRecognizer is null) + { + return; + } + + _isDictating = false; + try + { + await _speechRecognizer.InvokeVoidAsync("stop"); + } + catch (JSException) + { + Context.SetStatusMessage("Transcribing the completed recording."); + } + } + + private Task HandleSpeechResultAsync( + string finalTranscript, + string interimTranscript) + { + return InvokeAsync(() => + { + if (!_isRecording || !_isDictating) + { + return; + } + + if (!string.IsNullOrWhiteSpace(finalTranscript)) + { + _committedTranscript = + AppendText(_committedTranscript, finalTranscript); + } + + Context.Text = AppendText( + AppendText(_dictationPrefix, _committedTranscript), + interimTranscript); + Context.SetStatusMessage("Recording and transcribing."); + }); + } + + private Task HandleSpeechErrorAsync() + { + return InvokeAsync(() => + { + _isDictating = false; + if (_isRecording) + { + Context.SetStatusMessage( + "Recording audio. Live transcription is unavailable."); + } + }); + } + + private string CssClass() + { + var css = _isRecording + ? "sc-ai-input__audio sc-ai-input__audio--recording" + : "sc-ai-input__audio"; + if (AdditionalAttributes?.TryGetValue("class", out var value) == true && + value is string additionalClass) + { + css = $"{css} {additionalClass}"; + } + + return css; + } + + private static string GetExtension(string mediaType) + { + if (mediaType.Contains("ogg", StringComparison.OrdinalIgnoreCase)) + { + return "ogg"; + } + + if (mediaType.Contains("mp4", StringComparison.OrdinalIgnoreCase)) + { + return "m4a"; + } + + if (mediaType.Contains("wav", StringComparison.OrdinalIgnoreCase)) + { + return "wav"; + } + + return "webm"; + } + + private static string FormatBytes(long bytes) + { + const long megabyte = 1024 * 1024; + return bytes >= megabyte && bytes % megabyte == 0 + ? $"{bytes / megabyte} MB" + : $"{bytes} bytes"; + } + + private static string AppendText(string existingText, string transcript) + { + return string.IsNullOrWhiteSpace(existingText) + ? transcript + : $"{existingText.TrimEnd()} {transcript}"; + } + + /// + /// Stops recording and releases browser resources. + /// + public async ValueTask DisposeAsync() + { + if (_isDisposed) + { + return; + } + + _isDisposed = true; + _changeSubscription?.Dispose(); + _operationCts?.Cancel(); + if (!_isTranscribing) + { + _operationCts?.Dispose(); + _operationCts = null; + } + Context.SetComposing(false); + + if (_recorder is not null) + { + try + { + await _recorder.InvokeVoidAsync("dispose"); + await _recorder.DisposeAsync(); + } + catch (JSDisconnectedException) + { + } + } + + if (_speechRecognizer is not null) + { + try + { + await _speechRecognizer.InvokeVoidAsync("dispose"); + await _speechRecognizer.DisposeAsync(); + } + catch (JSDisconnectedException) + { + } + } + + if (_module is not null) + { + try + { + await _module.DisposeAsync(); + } + catch (JSDisconnectedException) + { + } + } + + _speechCallbackReference?.Dispose(); + GC.SuppressFinalize(this); + } + + private sealed class SpeechCallbacks(AudioCaptureButton owner) + { + [JSInvokable] + public Task OnResultAsync(string finalTranscript, string interimTranscript) + { + return owner.HandleSpeechResultAsync(finalTranscript, interimTranscript); + } + + [JSInvokable] + public Task OnErrorAsync(string _, bool __) + { + return owner.HandleSpeechErrorAsync(); + } + } + + private sealed class AudioCaptureResult + { + public IJSStreamReference? StreamReference { get; set; } + + public string MimeType { get; set; } = string.Empty; + + public long Size { get; set; } + + public bool TooLarge { get; set; } + } +} diff --git a/src/Components/AI/src/Components/ConversationTurnRenderer.cs b/src/Components/AI/src/Components/ConversationTurnRenderer.cs index a3666f9aef00..0f532d4cb4c3 100644 --- a/src/Components/AI/src/Components/ConversationTurnRenderer.cs +++ b/src/Components/AI/src/Components/ConversationTurnRenderer.cs @@ -23,6 +23,7 @@ internal ConversationTurnRenderer( _turn = turn; _listContext = listContext; _requestRender = requestRender; + _turn.ResponseBlocksTruncated += OnResponseBlocksTruncated; _blockAddedSub = agentContext.RegisterOnBlockAdded((t, block) => { @@ -48,12 +49,28 @@ private void OnBlockAdded(ContentBlock block) _requestRender(); } + private void OnResponseBlocksTruncated(int count) + { + var retainedContainerCount = _turn.RequestBlocks.Count + count; + for (var index = _blockContainers.Count - 1; + index >= retainedContainerCount; + index--) + { + _blockContainers[index].Dispose(); + _blockContainers.RemoveAt(index); + } + + _requestRender(); + } + internal void RenderTo(RenderTreeBuilder builder, int baseSeq) { // Determine role from request blocks (user turn) or response blocks var role = _turn.RequestBlocks.Count > 0 ? "user" : "assistant"; builder.OpenElement(baseSeq, "div"); builder.AddAttribute(baseSeq + 1, "class", $"sc-ai-turn sc-ai-turn--{role}"); + builder.AddAttribute(baseSeq + 2, "role", "group"); + builder.AddAttribute(baseSeq + 3, "aria-label", "Conversation turn"); var seq = baseSeq + 10; foreach (var container in _blockContainers) { @@ -65,6 +82,7 @@ internal void RenderTo(RenderTreeBuilder builder, int baseSeq) public void Dispose() { + _turn.ResponseBlocksTruncated -= OnResponseBlocksTruncated; _blockAddedSub?.Dispose(); foreach (var container in _blockContainers) { diff --git a/src/Components/AI/src/Components/LiveSpeechButton.cs b/src/Components/AI/src/Components/LiveSpeechButton.cs new file mode 100644 index 000000000000..887402bc5a96 --- /dev/null +++ b/src/Components/AI/src/Components/LiveSpeechButton.cs @@ -0,0 +1,488 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.AspNetCore.Components.Rendering; +using Microsoft.JSInterop; + +namespace Microsoft.AspNetCore.Components.AI; + +/// +/// Captures continuous browser speech recognition, shows interim text in the nearest +/// , and optionally submits each completed utterance. +/// +public sealed class LiveSpeechButton : ComponentBase, IAsyncDisposable +{ + private const string ModulePath = + "./_content/Microsoft.AspNetCore.Components.AI/ai-chat.js"; + + private readonly SpeechCallbacks _callbacks; + private DotNetObjectReference? _callbackReference; + private IJSObjectReference? _module; + private IJSObjectReference? _recognizer; + private MessageInputContext? _subscribedContext; + private IDisposable? _changeSubscription; + private string _prefix = string.Empty; + private string _committedTranscript = string.Empty; + private bool _isEnabled; + private bool _isListening; + private bool _isStarting; + private bool _isFinalizing; + private bool _isSupported = true; + private bool _isDisposed; + + /// + /// Initializes a new instance of . + /// + public LiveSpeechButton() + { + _callbacks = new SpeechCallbacks(this); + } + + /// + /// Gets or sets the nearest message input. + /// + [CascadingParameter] + public MessageInputContext Context { get; set; } = default!; + + /// + /// Gets or sets the JavaScript runtime used to access browser speech recognition. + /// + [Inject] + internal IJSRuntime JSRuntime { get; set; } = default!; + + /// + /// Gets or sets whether each finalized utterance is submitted automatically. + /// + [Parameter] + public bool AutoSubmit { get; set; } = true; + + /// + /// Gets or sets the speech-recognition language. The browser default is used when omitted. + /// + [Parameter] + public string? Language { get; set; } + + /// + /// Gets or sets whether interim speech is displayed in the message composer. + /// + [Parameter] + public bool ShowInterimInComposer { get; set; } = true; + + /// + /// Gets or sets the accessible label shown before live speech starts. + /// + [Parameter] + public string StartLabel { get; set; } = "Start live voice"; + + /// + /// Gets or sets the accessible label shown while live speech is enabled. + /// + [Parameter] + public string StopLabel { get; set; } = "Stop live voice"; + + /// + /// Gets or sets custom button content based on whether live speech is enabled. + /// + [Parameter] + public RenderFragment? ChildContent { get; set; } + + /// + /// Gets or sets a callback invoked for each finalized transcript. + /// + [Parameter] + public EventCallback OnTranscript { get; set; } + + /// + /// Gets or sets a callback invoked when the visible interim transcript changes. + /// + [Parameter] + public EventCallback OnInterimTranscript { get; set; } + + /// + /// Gets or sets additional attributes applied to the live-speech button. + /// + [Parameter(CaptureUnmatchedValues = true)] + public Dictionary? AdditionalAttributes { get; set; } + + /// + protected override void OnParametersSet() + { + if (ReferenceEquals(_subscribedContext, Context)) + { + return; + } + + _changeSubscription?.Dispose(); + _subscribedContext = Context; + _changeSubscription = Context.RegisterOnChanged(OnContextChanged); + } + + /// + protected override void BuildRenderTree(RenderTreeBuilder builder) + { + var disabled = !_isSupported || + (!_isEnabled && (Context.IsConversationBusy || Context.IsComposing)); + var label = _isEnabled ? StopLabel : StartLabel; + + builder.OpenElement(0, "button"); + builder.AddMultipleAttributes(1, AdditionalAttributes); + builder.AddAttribute(2, "type", "button"); + builder.AddAttribute(3, "class", CssClass()); + builder.AddAttribute(4, "disabled", disabled); + builder.AddAttribute(5, "aria-label", label); + builder.AddAttribute(6, "aria-pressed", _isEnabled ? "true" : "false"); + builder.AddAttribute( + 7, + "onclick", + EventCallback.Factory.Create(this, ToggleAsync)); + + if (ChildContent is not null) + { + builder.AddContent(8, ChildContent(_isEnabled)); + } + else + { + builder.AddContent(9, label); + } + + builder.CloseElement(); + } + + /// + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (!firstRender) + { + return; + } + + try + { + _module = await JSRuntime.InvokeAsync("import", ModulePath); + _isSupported = await _module.InvokeAsync( + "isLiveSpeechRecognitionSupported"); + if (!_isSupported) + { + Context.SetStatusMessage( + "Live voice is not supported by this browser. Recorded voice notes are still available."); + } + await InvokeAsync(StateHasChanged); + } + catch (JSException) + { + _isSupported = false; + Context.SetErrorMessage("Live voice could not be initialized."); + await InvokeAsync(StateHasChanged); + } + } + + private Task ToggleAsync() + { + return _isEnabled ? StopAsync() : StartAsync(); + } + + private async Task StartAsync() + { + Context.SetErrorMessage(null); + + try + { + _module ??= await JSRuntime.InvokeAsync("import", ModulePath); + _callbackReference ??= DotNetObjectReference.Create(_callbacks); + _recognizer ??= await _module.InvokeAsync( + "createLiveSpeechRecognizer", + _callbackReference, + Language); + _prefix = Context.Text.Trim(); + _committedTranscript = string.Empty; + _isEnabled = true; + await StartListeningAsync(); + } + catch (JSException) + { + _isEnabled = false; + _isListening = false; + Context.SetComposing(false); + Context.SetErrorMessage( + "Microphone speech recognition was not available. Check browser permissions."); + } + } + + private async Task StartListeningAsync() + { + if (!_isEnabled || _isListening || _isStarting) + { + return; + } + + _isStarting = true; + try + { + await _recognizer!.InvokeVoidAsync("start"); + _isListening = true; + Context.SetComposing(true); + Context.SetStatusMessage("Listening for your next instruction."); + } + catch (JSException) + { + _isEnabled = false; + _isListening = false; + Context.SetComposing(false); + Context.SetErrorMessage( + "Microphone speech recognition was not available. Check browser permissions."); + } + finally + { + _isStarting = false; + await InvokeAsync(StateHasChanged); + } + } + + private async Task StopAsync() + { + _isEnabled = false; + _isListening = false; + Context.SetComposing(false); + if (_recognizer is not null) + { + await _recognizer.InvokeVoidAsync("stop"); + } + await OnInterimTranscript.InvokeAsync(string.Empty); + Context.SetStatusMessage("Live voice stopped."); + await InvokeAsync(StateHasChanged); + } + + private Task HandleResultAsync(string finalTranscript, string interimTranscript) + { + return InvokeAsync(async () => + { + if (!_isEnabled || _isFinalizing) + { + return; + } + + if (!string.IsNullOrWhiteSpace(finalTranscript)) + { + _committedTranscript = AppendText(_committedTranscript, finalTranscript); + } + + var recognizedText = + AppendText(_committedTranscript, interimTranscript); + if (ShowInterimInComposer) + { + Context.Text = ComposeText(interimTranscript); + } + await OnInterimTranscript.InvokeAsync(recognizedText); + if (string.IsNullOrWhiteSpace(finalTranscript)) + { + Context.SetStatusMessage("Listening..."); + return; + } + + await OnTranscript.InvokeAsync(finalTranscript.Trim()); + if (!AutoSubmit) + { + Context.SetStatusMessage("Listening for more."); + return; + } + + _isFinalizing = true; + _isListening = false; + try + { + await _recognizer!.InvokeVoidAsync("stop"); + Context.SetComposing(false); + Context.SetStatusMessage("Sending voice instruction."); + if (!ShowInterimInComposer) + { + Context.Text = ComposeText(string.Empty); + } + if (Context.CanSubmit) + { + var submitTask = Context.SubmitAsync(); + await OnInterimTranscript.InvokeAsync(string.Empty); + await submitTask; + } + + _prefix = string.Empty; + _committedTranscript = string.Empty; + } + finally + { + _isFinalizing = false; + } + + if (_isEnabled && CanResumeListening) + { + await StartListeningAsync(); + } + else if (_isEnabled) + { + Context.SetStatusMessage( + "Live voice is on and will resume after the current action."); + await InvokeAsync(StateHasChanged); + } + }); + } + + private void OnContextChanged() + { + _ = InvokeAsync(async () => + { + StateHasChanged(); + if (_isEnabled && + !_isListening && + !_isFinalizing && + CanResumeListening) + { + await StartListeningAsync(); + } + }); + } + + private bool CanResumeListening => + Context.Status is ConversationStatus.Idle or ConversationStatus.Error; + + private Task HandleStartedAsync() + { + return InvokeAsync(() => + { + if (!_isEnabled) + { + return; + } + + _isListening = true; + Context.SetComposing(true); + Context.SetErrorMessage(null); + Context.SetStatusMessage("Listening for your next instruction."); + StateHasChanged(); + }); + } + + private Task HandleErrorAsync(string error, bool isFatal) + { + return InvokeAsync(async () => + { + _isListening = false; + if (!isFatal) + { + Context.SetStatusMessage( + "Live voice was interrupted. Reconnecting automatically."); + StateHasChanged(); + return; + } + + _isEnabled = false; + Context.SetComposing(false); + await OnInterimTranscript.InvokeAsync(string.Empty); + Context.SetErrorMessage(error switch + { + "not-allowed" or "service-not-allowed" => + "Microphone access was denied. Allow microphone access to use live voice.", + "language-not-supported" => + "The selected live voice language is not supported by this browser.", + _ => "Live voice could not continue because speech recognition is not configured correctly.", + }); + StateHasChanged(); + }); + } + + private string ComposeText(string interimTranscript) + { + var text = AppendText(_prefix, _committedTranscript); + return AppendText(text, interimTranscript); + } + + private string CssClass() + { + var css = _isEnabled + ? "sc-ai-input__live-speech sc-ai-input__live-speech--active" + : "sc-ai-input__live-speech"; + if (_isEnabled && !_isListening) + { + css += " sc-ai-input__live-speech--waiting"; + } + if (AdditionalAttributes?.TryGetValue("class", out var value) == true && + value is string additionalClass) + { + css = $"{css} {additionalClass}"; + } + + return css; + } + + private static string AppendText(string existingText, string newText) + { + if (string.IsNullOrWhiteSpace(newText)) + { + return existingText.Trim(); + } + + return string.IsNullOrWhiteSpace(existingText) + ? newText.Trim() + : $"{existingText.TrimEnd()} {newText.Trim()}"; + } + + /// + /// Stops recognition and releases browser resources. + /// + public async ValueTask DisposeAsync() + { + if (_isDisposed) + { + return; + } + + _isDisposed = true; + _changeSubscription?.Dispose(); + Context.SetComposing(false); + + if (_recognizer is not null) + { + try + { + await _recognizer.InvokeVoidAsync("dispose"); + await _recognizer.DisposeAsync(); + } + catch (JSDisconnectedException) + { + } + } + + if (_module is not null) + { + try + { + await _module.DisposeAsync(); + } + catch (JSDisconnectedException) + { + } + } + + _callbackReference?.Dispose(); + GC.SuppressFinalize(this); + } + + private sealed class SpeechCallbacks(LiveSpeechButton owner) + { + [JSInvokable] + public Task OnStartedAsync() + { + return owner.HandleStartedAsync(); + } + + [JSInvokable] + public Task OnResultAsync(string finalTranscript, string interimTranscript) + { + return owner.HandleResultAsync(finalTranscript, interimTranscript); + } + + [JSInvokable] + public Task OnErrorAsync(string error, bool isFatal) + { + return owner.HandleErrorAsync(error, isFatal); + } + } +} diff --git a/src/Components/AI/src/Components/MediaContent.cs b/src/Components/AI/src/Components/MediaContent.cs new file mode 100644 index 000000000000..64b22e523685 --- /dev/null +++ b/src/Components/AI/src/Components/MediaContent.cs @@ -0,0 +1,124 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.AspNetCore.Components.Rendering; +using Microsoft.AspNetCore.Components.Media; +using Microsoft.Extensions.AI; +using System.Runtime.CompilerServices; + +namespace Microsoft.AspNetCore.Components.AI; + +/// +/// Renders binary AI content using the matching component from +/// . +/// +public sealed class MediaContent : ComponentBase +{ + private static readonly ConditionalWeakTable CacheKeys = new(); + private DataContent? _currentContent; + private MediaSource? _source; + + /// + /// Gets or sets the content to render. + /// + [Parameter, EditorRequired] + public DataContent Content { get; set; } = default!; + + /// + /// Gets or sets the accessible alternative text for image content. + /// + [Parameter] + public string? AlternativeText { get; set; } + + /// + /// Gets or sets additional attributes applied to the rendered media element. + /// + [Parameter(CaptureUnmatchedValues = true)] + public Dictionary? AdditionalAttributes { get; set; } + + /// + protected override void OnParametersSet() + { + ArgumentNullException.ThrowIfNull(Content); + + if (!ReferenceEquals(_currentContent, Content)) + { + _currentContent = Content; + _source = new MediaSource( + Content.Data.ToArray(), + Content.MediaType, + CacheKeys.GetValue( + Content, + static _ => new MediaCacheKey($"ai-media-{Guid.NewGuid():N}")).Value); + } + } + + /// + protected override void BuildRenderTree(RenderTreeBuilder builder) + { + var source = _source + ?? throw new InvalidOperationException($"{nameof(MediaContent)}.{nameof(Content)} is required."); + var attributes = CreateMediaAttributes(); + + if (Content.HasTopLevelMediaType("image")) + { + attributes["alt"] = AlternativeText ?? GetDisplayName("Attached image"); + builder.OpenComponent(0); + builder.AddComponentParameter(1, nameof(Media.Image.Source), source); + builder.AddComponentParameter(2, nameof(Media.Image.AdditionalAttributes), attributes); + builder.CloseComponent(); + } + else if (Content.HasTopLevelMediaType("audio")) + { + attributes.TryAdd("controls", true); + attributes.TryAdd("preload", "metadata"); + attributes.TryAdd("aria-label", GetDisplayName("Attached audio")); + builder.OpenComponent