diff --git a/src/Components/AI/samples/ClaimApp/ClaimApp.csproj b/src/Components/AI/samples/ClaimApp/ClaimApp.csproj new file mode 100644 index 000000000000..cfb7598907e6 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/ClaimApp.csproj @@ -0,0 +1,56 @@ + + + + $(DefaultNetCoreTargetFramework) + ComponentsAIClaimApp + false + enable + enable + true + aspnet-ComponentsAI-ClaimApp + true + true + $(NoWarn);NU1511 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Components/AI/samples/ClaimApp/Components/App.razor b/src/Components/AI/samples/ClaimApp/Components/App.razor new file mode 100644 index 000000000000..79c26fa1da47 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/App.razor @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Components/AI/samples/ClaimApp/Components/Layout/MainLayout.razor b/src/Components/AI/samples/ClaimApp/Components/Layout/MainLayout.razor new file mode 100644 index 000000000000..608e3e1f139a --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Layout/MainLayout.razor @@ -0,0 +1,11 @@ +@inherits LayoutComponentBase + +
+ @Body +
+ +
+ An unhandled error has occurred. + Reload + +
diff --git a/src/Components/AI/samples/ClaimApp/Components/Layout/MainLayout.razor.css b/src/Components/AI/samples/ClaimApp/Components/Layout/MainLayout.razor.css new file mode 100644 index 000000000000..1fdd134aa5ef --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Layout/MainLayout.razor.css @@ -0,0 +1,29 @@ +main { + min-height: 100dvh; +} + +#blazor-error-ui { + position: fixed; + right: 1rem; + bottom: 1rem; + left: 1rem; + z-index: 1000; + display: none; + box-sizing: border-box; + padding: 0.8rem 3rem 0.8rem 1rem; + border-radius: 0.75rem; + color: #601410; + background: #fce8e6; + box-shadow: 0 0.5rem 1.5rem rgba(44, 68, 104, 0.18); +} + +#blazor-error-ui .dismiss { + position: absolute; + top: 0.45rem; + right: 0.65rem; + border: 0; + color: inherit; + background: transparent; + cursor: pointer; + font-size: 1.25rem; +} diff --git a/src/Components/AI/samples/ClaimApp/Components/Pages/AIChat.razor b/src/Components/AI/samples/ClaimApp/Components/Pages/AIChat.razor new file mode 100644 index 000000000000..763af0f88316 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Pages/AIChat.razor @@ -0,0 +1,537 @@ +@page "/" +@page "/ai-chat" +@using AGUI.Abstractions +@using System.Text.Json +@using ComponentsAIClaimApp.Data +@using Microsoft.AspNetCore.Components.AI +@using Microsoft.Extensions.AI +@implements IDisposable +@inject IClaimAssistantBackend ClaimAssistant +@inject IChatClient ChatClient +@inject ILogger Logger +@inject ILoggerFactory LoggerFactory + +AutoSure claim assistant + +
+ Skip to claim assistant +
+
+ +
+ AutoSure claims +

Vehicle damage assessment

+
+
+
+ +
+
+ +
+ + +
+
+
+
+ +
+

Damage assessment agent

+ + + @ClaimState.Status + +
+
+
+ +
+ + + + + + + + + + + + + +
+ Claim assistant +

Start with what happened.

+

+ Describe the damage, add vehicle photos, or record a voice + note. You can also use one of the starters below. +

+
+ + + @if (!string.IsNullOrWhiteSpace(_liveVoiceTranscript)) + { +
+ Listening +

@_liveVoiceTranscript

+
+ } +
+
+
+ +
+
+ +
+
+
+
+ +@code { + private const string IdentifyVehicleAreasTool = "identify_vehicle_areas"; + private const string DisplayVehicleDamageTool = "display_vehicle_damage"; + + private readonly IReadOnlyList _suggestions = + [ + new( + "Front impact", + "My 2022 Toyota Camry SE was hit in Seattle, Washington. The front bumper is cracked, the hood looks bent, and the left headlight is broken."), + new( + "Side impact", + "A vehicle hit the driver's side at low speed. The front door and left fender are dented."), + new( + "Windshield damage", + "Road debris struck the windshield and also chipped the hood."), + ]; + + private ClaimConversation _conversation = default!; + private readonly List _claimPhotos = []; + private string? _liveVoiceTranscript; + private string _theme = "light"; + private ClaimState ClaimState => _conversation.Agent.State.Value; + + protected override void OnInitialized() + { + _conversation = CreateConversation("CLM-1042"); + } + + private ClaimConversation CreateConversation(string claimNumber) + { + var state = new ClaimState + { + ClaimNumber = claimNumber, + }; + var conversation = new ClaimConversation(); + UIAgent? agent = null; + + var displayVehicleDamage = AIFunctionFactory.Create( + (string[] areas) => DisplayVehicleDamage(agent!, areas), + DisplayVehicleDamageTool, + "Highlights affected areas on the vehicle diagram."); + + var chatOptions = new ChatOptions + { + RawRepresentationFactory = _ => new RunAgentInput + { + State = JsonSerializer.SerializeToElement( + agent!.State.Value, + ClaimStateJson.Options), + }, + }; + + agent = new UIAgent( + ChatClient, + options => + { + options.ChatOptions = chatOptions; + options.RegisterUIAction(displayVehicleDamage); + options.AddBlockHandler(new ClaimErrorContentHandler(Logger)); + options.StateMapper = context => MapClaimState(agent!, context); + }, + LoggerFactory, + state); + conversation.Agent = agent; + + conversation.StateChangedSubscription = agent.State.OnChanged(() => + { + _ = InvokeAsync(StateHasChanged); + }); + + return conversation; + } + + private static bool MapClaimState( + UIAgent agent, + StateMapperContext context) + { + var currentState = agent.State.Value; + ClaimState? mappedState = context.Update.RawRepresentation switch + { + StateSnapshotEvent snapshot => + snapshot.Snapshot.Deserialize(ClaimStateJson.Options), + StateDeltaEvent delta => ClaimStateJson.ApplyDelta(currentState, delta.Delta), + _ => null, + }; + + if (mappedState is null) + { + return false; + } + + context.SetState(mappedState); + return true; + } + + private static string DisplayVehicleDamage( + UIAgent agent, + string[] areas) + { + var updatedState = ClaimStateJson.Clone(agent.State.Value); + updatedState.AffectedAreas = areas + .Where(area => !string.IsNullOrWhiteSpace(area)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + updatedState.Status = "Vehicle diagram updated"; + agent.State.Value = updatedState; + + return JsonSerializer.Serialize(new + { + displayedAreas = updatedState.AffectedAreas, + }, ClaimStateJson.Options); + } + + private Task RecordApproval() + { + var updatedState = ClaimStateJson.Clone(ClaimState); + updatedState.RejectionReason = null; + _conversation.Agent.State.Value = updatedState; + + return Task.CompletedTask; + } + + private Task RecordRejection(string reason) + { + var updatedState = ClaimStateJson.Clone(ClaimState); + updatedState.RejectionReason = reason; + _conversation.Agent.State.Value = updatedState; + + return Task.CompletedTask; + } + + private Task RecordAdditionalEvidence() + { + var updatedState = ClaimStateJson.Clone(ClaimState); + updatedState.Status = "Add more evidence"; + updatedState.Decision = "Pending"; + updatedState.RejectionReason = null; + _conversation.Agent.State.Value = updatedState; + + return Task.CompletedTask; + } + + private Task RecordCancellation() + { + var updatedState = ClaimStateJson.Clone(ClaimState); + updatedState.Status = "Ready"; + updatedState.AssessmentSummary = updatedState.Confidence == 0 + ? "Assessment canceled. Add or update evidence when you are ready." + : updatedState.AssessmentSummary; + _conversation.Agent.State.Value = updatedState; + + return Task.CompletedTask; + } + + private Task SetLiveVoiceTranscript(string transcript) + { + _liveVoiceTranscript = string.IsNullOrWhiteSpace(transcript) + ? null + : transcript; + return Task.CompletedTask; + } + + private void AddClaimPhoto(ClaimPhotoPreview photo) + { + _claimPhotos.RemoveAll(existing => + string.Equals(existing.Name, photo.Name, StringComparison.Ordinal)); + _claimPhotos.Add(photo); + } + + private int FindingCountFor(string photoName) + => ClaimState.DamageFindings.Count(finding => + finding.PhotoNames.Any(name => + string.Equals(name, photoName, StringComparison.OrdinalIgnoreCase))); + + private Task SetTheme(string theme) + { + _theme = theme; + return Task.CompletedTask; + } + + private static string ProgressClass(bool complete, bool active) + => complete ? "claim-progress__step--complete" : + active ? "claim-progress__step--active" : + string.Empty; + + private static string FormatMoney(decimal value, string currency) + => $"{currency} {value:N0}"; + + public void Dispose() + { + _conversation.StateChangedSubscription?.Dispose(); + _conversation.Agent.Dispose(); + } + + private sealed class ClaimConversation + { + public UIAgent Agent { get; set; } = default!; + + public IDisposable? StateChangedSubscription { get; set; } + } +} diff --git a/src/Components/AI/samples/ClaimApp/Components/Pages/AIChat.razor.css b/src/Components/AI/samples/ClaimApp/Components/Pages/AIChat.razor.css new file mode 100644 index 000000000000..962a035ca409 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Pages/AIChat.razor.css @@ -0,0 +1,1022 @@ +.claim-app { + --claim-bg: oklch(0.965 0.008 255); + --claim-surface: oklch(1 0 0); + --claim-surface-alpha: oklch(1 0 0 / 0.96); + --claim-surface-muted: oklch(0.97 0.008 255); + --claim-surface-subtle: oklch(0.985 0.004 255); + --claim-text: oklch(0.24 0.035 258); + --claim-muted: oklch(0.49 0.028 258); + --claim-border: oklch(0.9 0.015 255); + --claim-border-strong: oklch(0.82 0.025 255); + --claim-accent: oklch(0.55 0.19 255); + --claim-accent-hover: oklch(0.49 0.2 255); + --claim-accent-soft: oklch(0.95 0.035 255); + --claim-accent-contrast: oklch(1 0 0); + --claim-success: oklch(0.5 0.14 155); + --claim-success-bg: oklch(0.95 0.04 155); + --claim-danger: oklch(0.51 0.18 30); + --claim-danger-bg: oklch(0.95 0.04 30); + --claim-warning-bg: oklch(0.95 0.05 55); + --claim-shadow: oklch(0.25 0.035 258 / 0.08); + box-sizing: border-box; + display: flex; + flex-direction: column; + gap: 0.75rem; + min-height: 100dvh; + padding: 0.75rem; + overflow: hidden; + color: var(--claim-text); + background: var(--claim-bg); + color-scheme: light; +} + +.claim-app[data-theme="dark"] { + --claim-bg: oklch(0.16 0.02 258); + --claim-surface: oklch(0.21 0.022 258); + --claim-surface-alpha: oklch(0.21 0.022 258 / 0.97); + --claim-surface-muted: oklch(0.26 0.022 258); + --claim-surface-subtle: oklch(0.23 0.021 258); + --claim-text: oklch(0.94 0.012 255); + --claim-muted: oklch(0.73 0.025 255); + --claim-border: oklch(0.32 0.028 258); + --claim-border-strong: oklch(0.43 0.035 258); + --claim-accent: oklch(0.7 0.16 250); + --claim-accent-hover: oklch(0.76 0.14 250); + --claim-accent-soft: oklch(0.28 0.055 255); + --claim-accent-contrast: oklch(0.15 0.02 258); + --claim-success: oklch(0.73 0.13 155); + --claim-success-bg: oklch(0.28 0.05 155); + --claim-danger: oklch(0.74 0.15 30); + --claim-danger-bg: oklch(0.29 0.055 30); + --claim-warning-bg: oklch(0.3 0.05 55); + --claim-shadow: oklch(0 0 0 / 0.28); + color-scheme: dark; +} + +.claim-app[data-theme="contrast"] { + --claim-bg: #000; + --claim-surface: #000; + --claim-surface-alpha: #000; + --claim-surface-muted: #111; + --claim-surface-subtle: #080808; + --claim-text: #fff; + --claim-muted: #fff; + --claim-border: #fff; + --claim-border-strong: #fff; + --claim-accent: #ff0; + --claim-accent-hover: #ff0; + --claim-accent-soft: #222200; + --claim-accent-contrast: #000; + --claim-success: #0f0; + --claim-success-bg: #001a00; + --claim-danger: #ff8a8a; + --claim-danger-bg: #260000; + --claim-warning-bg: #292900; + --claim-shadow: transparent; + color-scheme: dark; +} + +.claim-skip-link { + position: fixed; + top: 0.5rem; + left: 0.5rem; + z-index: 100; + padding: 0.65rem 0.85rem; + border-radius: 0.55rem; + color: var(--claim-accent-contrast); + background: var(--claim-accent); + transform: translateY(-200%); +} + +.claim-skip-link:focus { + transform: translateY(0); +} + +.claim-topbar { + display: flex; + flex: 0 0 auto; + align-items: center; + justify-content: space-between; + gap: 1rem; + width: 100%; + max-width: 88rem; + min-height: 3.5rem; + margin: 0 auto; + padding: 0 0.35rem; +} + +.claim-brand { + display: flex; + align-items: center; + gap: 0.75rem; + min-width: 0; +} + +.claim-brand__mark, +.claim-avatar { + display: grid; + flex: 0 0 auto; + place-items: center; + color: var(--claim-accent-contrast); + background: var(--claim-accent); +} + +.claim-brand__mark { + width: 2.25rem; + height: 2.25rem; + border-radius: 0.7rem; +} + +.claim-brand__mark svg { + width: 1.35rem; + fill: none; + stroke: currentColor; + stroke-linecap: round; + stroke-linejoin: round; + stroke-width: 1.8; +} + +.claim-eyebrow { + display: block; + color: var(--claim-accent); + font-size: 0.66rem; + font-weight: 800; + letter-spacing: 0.11em; + text-transform: uppercase; +} + +.claim-topbar h1 { + margin: 0.1rem 0 0; + overflow: hidden; + font-size: 1rem; + font-weight: 750; + letter-spacing: -0.015em; + text-overflow: ellipsis; + white-space: nowrap; +} + +.claim-topbar__actions { + display: flex; + align-items: center; + gap: 0.55rem; +} + +.claim-chat-header__status-dot { + width: 0.45rem; + height: 0.45rem; + border-radius: 50%; + background: var(--claim-success); + box-shadow: 0 0 0 0.18rem color-mix(in oklab, var(--claim-success) 16%, transparent); +} + +.claim-workspace { + display: grid; + flex: 0 1 auto; + grid-template-columns: 19.5rem minmax(0, 1fr); + align-items: start; + gap: 0.75rem; + width: 100%; + max-width: 88rem; + height: min(52rem, calc(100dvh - 5rem)); + min-height: 32rem; + margin: 0 auto; +} + +.claim-summary, +.claim-chat { + min-height: 0; + border: 1px solid var(--claim-border); + border-radius: 1rem; + background: var(--claim-surface-alpha); + box-shadow: 0 0.75rem 2.5rem var(--claim-shadow); +} + +.claim-summary { + display: flex; + flex-direction: column; + align-self: start; + max-height: 100%; + padding: 1rem; + overflow-y: auto; + scrollbar-color: var(--claim-border-strong) transparent; + scrollbar-width: thin; +} + +.claim-summary__heading { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 0.75rem; + margin-bottom: 0.75rem; +} + +.claim-summary h2 { + margin: 0.1rem 0 0; + font-size: 1.2rem; + letter-spacing: -0.02em; +} + +.claim-status { + padding: 0.32rem 0.55rem; + border-radius: 999px; + color: var(--claim-text); + background: var(--claim-surface-muted); + font-size: 0.68rem; + font-weight: 750; + text-align: center; +} + +.claim-status--approved { + color: var(--claim-success); + background: var(--claim-success-bg); +} + +.claim-status--rejected { + color: var(--claim-danger); + background: var(--claim-danger-bg); +} + +.claim-summary ::deep .vehicle-diagram { + padding: 0.65rem; + border-radius: 0.8rem; +} + +.claim-photo-gallery { + margin-bottom: 0.65rem; +} + +.claim-photo-gallery__heading { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.5rem; + margin-bottom: 0.45rem; +} + +.claim-photo-gallery__heading > span:last-child { + color: var(--claim-muted); + font-size: 0.62rem; +} + +.claim-photo-gallery__grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 0.4rem; +} + +.claim-photo-gallery figure { + position: relative; + min-width: 0; + margin: 0; + overflow: hidden; + border: 1px solid var(--claim-border); + border-radius: 0.7rem; + background: var(--claim-surface-muted); +} + +.claim-photo-gallery figure:first-child:nth-last-child(odd) { + grid-column: 1 / -1; +} + +.claim-photo-gallery ::deep img { + display: block; + width: 100%; + height: 5.5rem; + object-fit: cover; +} + +.claim-photo-gallery figure:first-child:nth-last-child(odd) ::deep img { + height: 8rem; +} + +.claim-photo-gallery figcaption { + padding: 0.35rem 0.45rem; + overflow: hidden; + color: var(--claim-muted); + font-size: 0.58rem; + text-overflow: ellipsis; + white-space: nowrap; +} + +.claim-photo--flagged { + border-color: var(--claim-danger) !important; + box-shadow: 0 0 0 0.12rem color-mix(in oklab, var(--claim-danger) 16%, transparent); +} + +.claim-photo__badge { + position: absolute; + top: 0.4rem; + right: 0.4rem; + padding: 0.22rem 0.4rem; + border-radius: 999px; + color: white; + background: var(--claim-danger); + font-size: 0.56rem; + font-weight: 800; +} + +.claim-photo-empty { + display: grid; + justify-items: center; + gap: 0.22rem; + margin-bottom: 0.65rem; + padding: 1.1rem 0.75rem; + border: 1px dashed var(--claim-border-strong); + border-radius: 0.8rem; + color: var(--claim-muted); + background: var(--claim-surface-subtle); + text-align: center; +} + +.claim-photo-empty strong { + color: var(--claim-text); + font-size: 0.76rem; +} + +.claim-photo-empty > span:last-child { + font-size: 0.64rem; +} + +.claim-photo-empty__icon { + display: grid; + width: 2rem; + height: 2rem; + margin-bottom: 0.2rem; + place-items: center; + border-radius: 0.55rem; + color: var(--claim-accent); + background: var(--claim-accent-soft); +} + +.claim-photo-empty__icon svg { + width: 1.15rem; + fill: none; + stroke: currentColor; + stroke-linecap: round; + stroke-linejoin: round; + stroke-width: 1.8; +} + +.claim-damage-map { + margin-bottom: 0.45rem; + border-bottom: 1px solid var(--claim-border); +} + +.claim-damage-map summary { + padding: 0.45rem 0; + color: var(--claim-muted); + cursor: pointer; + font-size: 0.65rem; + font-weight: 750; +} + +.claim-damage-map[open] summary { + color: var(--claim-text); +} + +.claim-summary ::deep .vehicle-diagram svg { + width: auto; + max-width: 100%; + height: clamp(7.5rem, 20dvh, 10.5rem); + margin: 0 auto; +} + +.claim-details { + display: grid; + gap: 0; + margin: 0.7rem 0; +} + +.claim-details div { + padding: 0.52rem 0; + border-bottom: 1px solid var(--claim-border); +} + +.claim-details dt { + margin-bottom: 0.12rem; + color: var(--claim-muted); + font-size: 0.62rem; + font-weight: 800; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.claim-details dd { + margin: 0; + color: var(--claim-text); + font-size: 0.78rem; + line-height: 1.35; +} + +.claim-progress { + margin: 0.1rem 0 0.75rem; +} + +.claim-findings { + margin: 0.2rem 0 0.75rem; +} + +.claim-findings__heading { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.5rem; +} + +.claim-findings__heading > span:last-child { + color: var(--claim-muted); + font-size: 0.62rem; +} + +.claim-findings ul { + display: grid; + gap: 0.4rem; + padding: 0; + margin: 0.45rem 0 0; + list-style: none; +} + +.claim-findings li { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 0.25rem 0.5rem; + padding: 0.55rem; + border: 1px solid var(--claim-border); + border-radius: 0.65rem; + background: var(--claim-surface-subtle); +} + +.claim-findings li > div { + display: grid; +} + +.claim-findings li strong { + font-size: 0.72rem; + text-transform: capitalize; +} + +.claim-findings li span, +.claim-findings li p { + color: var(--claim-muted); + font-size: 0.62rem; +} + +.claim-findings li p { + grid-column: 1 / -1; + margin: 0; + line-height: 1.35; +} + +.claim-finding-severity { + align-self: start; + padding: 0.18rem 0.35rem; + border-radius: 999px; + color: var(--claim-danger) !important; + background: var(--claim-danger-bg); + font-weight: 750; +} + +.claim-findings__next { + margin: 0.45rem 0 0; + padding: 0.55rem; + border-radius: 0.65rem; + color: var(--claim-muted); + background: var(--claim-accent-soft); + font-size: 0.64rem; + line-height: 1.35; +} + +.claim-findings__next strong { + display: block; + color: var(--claim-text); +} + +.claim-market { + display: grid; + gap: 0.65rem; + margin-top: 0.75rem; + padding-top: 0.75rem; + border-top: 1px solid var(--claim-border); +} + +.claim-market__heading { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.5rem; +} + +.claim-market__heading > span:last-child { + color: var(--claim-muted); + font-size: 0.58rem; +} + +.claim-estimate { + padding: 0.75rem; + border: 1px solid color-mix(in oklab, var(--claim-accent) 28%, var(--claim-border)); + border-radius: 0.75rem; + background: var(--claim-accent-soft); +} + +.claim-estimate > span { + display: block; + color: var(--claim-muted); + font-size: 0.62rem; + font-weight: 700; +} + +.claim-estimate strong { + display: block; + margin-top: 0.16rem; + color: var(--claim-text); + font-size: 1.15rem; + letter-spacing: -0.03em; +} + +.claim-estimate p, +.claim-parts p, +.claim-market__warning, +.claim-market__disclaimer { + margin: 0.3rem 0 0; + color: var(--claim-muted); + font-size: 0.62rem; + line-height: 1.45; +} + +.claim-parts, +.claim-sources ul { + display: grid; + gap: 0.45rem; + margin: 0; + padding: 0; + list-style: none; +} + +.claim-parts li { + padding: 0.65rem; + border: 1px solid var(--claim-border); + border-radius: 0.7rem; + background: var(--claim-surface); +} + +.claim-parts li > div { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 0.5rem; +} + +.claim-parts strong { + color: var(--claim-text); + font-size: 0.68rem; +} + +.claim-parts li > div span { + flex: none; + color: var(--claim-text); + font-size: 0.62rem; + font-weight: 750; +} + +.claim-parts a, +.claim-sources a { + display: inline-block; + margin-top: 0.35rem; + color: var(--claim-accent); + font-size: 0.6rem; + font-weight: 750; + overflow-wrap: anywhere; +} + +.claim-market__warning { + padding: 0.55rem 0.65rem; + border-left: 0.18rem solid var(--claim-warning); + background: color-mix(in oklab, var(--claim-warning) 8%, transparent); +} + +.claim-sources summary { + color: var(--claim-muted); + cursor: pointer; + font-size: 0.62rem; + font-weight: 750; +} + +.claim-sources ul { + margin-top: 0.35rem; +} + +.claim-market__disclaimer { + padding-top: 0.45rem; + border-top: 1px solid var(--claim-border); +} + +.claim-progress ol { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 0.35rem; + padding: 0; + margin: 0.45rem 0 0; + list-style: none; +} + +.claim-progress li { + display: grid; + gap: 0.28rem; + color: var(--claim-muted); + font-size: 0.62rem; + font-weight: 700; +} + +.claim-progress li > span { + display: grid; + width: 1.35rem; + height: 1.35rem; + place-items: center; + border: 1px solid var(--claim-border-strong); + border-radius: 50%; + font-size: 0.58rem; +} + +.claim-progress__step--active { + color: var(--claim-accent) !important; +} + +.claim-progress__step--active > span { + border-color: var(--claim-accent) !important; + background: var(--claim-accent-soft); +} + +.claim-progress__step--complete { + color: var(--claim-success) !important; +} + +.claim-progress__step--complete > span { + border-color: var(--claim-success) !important; + color: var(--claim-accent-contrast); + background: var(--claim-success); +} + +.claim-chat { + min-width: 0; + height: 100%; + overflow: hidden; +} + +.claim-live-transcript { + width: fit-content; + max-width: min(75%, 36rem); + margin: 0.5rem 1rem 0.75rem auto; + padding: 0.65rem 0.85rem; + border: 1px solid var(--claim-border); + border-radius: 1rem 1rem 0.25rem 1rem; + color: var(--claim-text); + background: var(--claim-accent-soft); +} + +.claim-live-transcript span { + display: block; + margin-bottom: 0.15rem; + color: var(--claim-accent); + font-size: 0.65rem; + font-weight: 750; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.claim-live-transcript p { + margin: 0; + line-height: 1.4; +} + +.claim-chat ::deep .sc-ai-chat-page { + height: 100%; + border: 0; + color-scheme: inherit; + background: transparent; + --sc-ai-color-bg: var(--claim-surface); + --sc-ai-color-surface: var(--claim-surface); + --sc-ai-color-surface-hover: var(--claim-surface-muted); + --sc-ai-color-text: var(--claim-text); + --sc-ai-color-text-secondary: var(--claim-muted); + --sc-ai-color-border: var(--claim-border); + --sc-ai-color-error-bg: var(--claim-danger-bg); + --sc-ai-color-bubble-user: var(--claim-accent-soft); + --sc-ai-color-bubble-user-text: var(--claim-text); + --sc-ai-color-bubble-assistant-text: var(--claim-text); + --sc-ai-max-width: 56rem; +} + +.claim-chat ::deep .sc-ai-chat-page__header { + padding: 0.8rem 1rem; + border-bottom: 1px solid var(--claim-border); + background: var(--claim-surface-subtle); +} + +.claim-chat-header { + display: flex; + align-items: center; + gap: 0.7rem; +} + +.claim-avatar { + width: 2rem; + height: 2rem; + border-radius: 0.62rem; +} + +.claim-avatar svg { + width: 1.15rem; + fill: none; + stroke: currentColor; + stroke-linecap: round; + stroke-linejoin: round; + stroke-width: 1.8; +} + +.claim-chat-header > div:last-child { + display: grid; + gap: 0.05rem; +} + +.claim-chat-header h2 { + margin: 0; + font-size: 0.88rem; + letter-spacing: -0.01em; +} + +.claim-chat-header span { + display: flex; + align-items: center; + gap: 0.38rem; + color: var(--claim-muted); + font-size: 0.68rem; +} + +.claim-chat ::deep .sc-ai-chat-page__body { + background: var(--claim-surface); + scrollbar-color: var(--claim-border-strong) transparent; + scrollbar-width: thin; +} + +.claim-chat ::deep .sc-ai-message-list { + padding: 1.25rem; +} + +.claim-chat ::deep .sc-ai-message-list:has(.claim-welcome) { + height: 100%; +} + +.claim-welcome { + display: grid; + align-content: end; + justify-items: start; + width: 100%; + height: 100%; + min-height: 15rem; + padding: 2rem 1rem clamp(1.5rem, 4dvh, 2.5rem); + color: var(--claim-muted); + text-align: left; +} + +.claim-welcome h3 { + margin: 0.5rem 0 0.35rem; + color: var(--claim-text); + font-size: clamp(1.5rem, 2.5vw, 2rem); + letter-spacing: -0.035em; +} + +.claim-welcome p { + max-width: 34rem; + margin: 0; + font-size: 0.92rem; + line-height: 1.55; +} + +.claim-chat ::deep .sc-ai-chat-page__footer { + padding: 0 1rem 1rem; + border-top: 0; + background: var(--claim-surface); +} + +.claim-chat ::deep .sc-ai-chat-page__input-container { + max-width: 56rem; +} + +.claim-chat ::deep .claim-tool-block, +.claim-chat ::deep .claim-action-block, +.claim-chat ::deep .claim-approval { + max-width: 34rem; + margin: 0.65rem 0; + padding: 0.85rem; + border: 1px solid var(--claim-border); + border-radius: 0.8rem; + background: var(--claim-surface); + box-shadow: 0 0.35rem 1rem var(--claim-shadow); +} + +.claim-chat ::deep .claim-tool-block__label { + display: block; + margin-bottom: 0.25rem; + color: var(--claim-accent); + font-size: 0.62rem; + font-weight: 800; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.claim-chat ::deep .claim-area-list { + display: flex; + flex-wrap: wrap; + gap: 0.35rem; + margin-top: 0.55rem; +} + +.claim-chat ::deep .claim-area-chip { + padding: 0.3rem 0.5rem; + border-radius: 999px; + color: var(--claim-danger); + background: var(--claim-danger-bg); + font-size: 0.7rem; + font-weight: 700; + text-transform: capitalize; +} + +.claim-chat ::deep .claim-action-block { + display: flex; + align-items: center; + gap: 0.55rem; + color: var(--claim-success); + background: var(--claim-success-bg); +} + +.claim-chat ::deep .claim-action-block__check { + display: grid; + width: 1.45rem; + height: 1.45rem; + place-items: center; + border-radius: 50%; + color: var(--claim-accent-contrast); + background: var(--claim-success); +} + +.claim-chat ::deep .claim-approval h3 { + margin: 0 0 0.25rem; + font-size: 0.9rem; +} + +.claim-chat ::deep .claim-approval p { + margin: 0 0 0.7rem; + color: var(--claim-muted); + font-size: 0.76rem; +} + +.claim-chat ::deep .claim-approval__reason { + display: grid; + gap: 0.3rem; + margin: 0.7rem 0; +} + +.claim-chat ::deep .claim-approval__reason label { + font-size: 0.72rem; + font-weight: 700; +} + +.claim-chat ::deep .claim-approval__reason textarea { + min-height: 4rem; + padding: 0.6rem; + border: 1px solid var(--claim-border-strong); + border-radius: 0.6rem; + color: var(--claim-text); + background: var(--claim-surface); + resize: vertical; +} + +.claim-chat ::deep .claim-approval__error { + color: var(--claim-danger); + font-size: 0.7rem; +} + +.claim-chat ::deep .claim-approval__actions { + display: flex; + gap: 0.45rem; +} + +.claim-chat ::deep .claim-approval__actions button, +.claim-chat ::deep .claim-cancel { + padding: 0.48rem 0.7rem; + border: 1px solid var(--claim-border-strong); + border-radius: 0.55rem; + color: var(--claim-text); + background: var(--claim-surface); + font-size: 0.72rem; + font-weight: 700; +} + +.claim-chat ::deep .claim-approval__actions button:first-child { + border-color: var(--claim-accent); + color: var(--claim-accent-contrast); + background: var(--claim-accent); +} + +.claim-chat ::deep .claim-cancel { + border-color: var(--claim-danger); + color: var(--claim-danger); +} + +@media (max-width: 980px) { + .claim-app { + min-height: 100dvh; + overflow: visible; + } + + .claim-workspace { + grid-template-columns: 1fr; + align-items: stretch; + height: auto; + min-height: 0; + } + + .claim-summary { + max-height: none; + overflow: visible; + } + + .claim-summary ::deep .vehicle-diagram svg { + height: 9rem; + } + + .claim-chat { + height: min(46rem, calc(100dvh - 1.5rem)); + min-height: 38rem; + } +} + +@media (max-width: 640.98px) { + .claim-app { + gap: 0.6rem; + padding: 0.6rem; + } + + .claim-topbar { + display: grid; + align-items: start; + gap: 0.4rem; + padding: 0.15rem; + } + + .claim-brand__mark { + display: none; + } + + .claim-topbar__actions { + justify-content: flex-end; + width: 100%; + } + + .claim-topbar h1 { + white-space: normal; + } + + .claim-workspace { + gap: 0.6rem; + } + + .claim-chat { + grid-row: 1; + } + + .claim-summary { + grid-row: 2; + } + + .claim-summary { + padding: 0.85rem; + } + + .claim-details { + grid-template-columns: 1fr 1fr; + column-gap: 0.8rem; + } + + .claim-details div:nth-child(-n + 3) { + grid-column: 1 / -1; + } + + .claim-chat { + height: calc(100dvh - 7rem); + min-height: 36rem; + border-radius: 0.85rem; + } + + .claim-chat ::deep .sc-ai-chat-page__header { + padding: 0.7rem; + } + + .claim-chat ::deep .sc-ai-message-list { + padding: 0.85rem; + } + + .claim-chat ::deep .sc-ai-chat-page__footer { + padding: 0.7rem; + } +} diff --git a/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimApprovalBlock.razor b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimApprovalBlock.razor new file mode 100644 index 000000000000..3968ec67b16b --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimApprovalBlock.razor @@ -0,0 +1,89 @@ +@using Microsoft.AspNetCore.Components.AI + +
+

Confirm the preliminary findings

+

Use this decision only after the photos and damage findings look complete.

+ + @if (Block.Status == ApprovalStatus.Pending) + { +
+ + + @if (!string.IsNullOrEmpty(_validationMessage)) + { + + @_validationMessage + + } +
+
+ + + +
+ } + else + { + + @(_additionalEvidenceRequested + ? "More evidence requested" + : Block.Status switch + { + ApprovalStatus.Approved => "Assessment approved", + _ => "Assessment rejected", + }) + + } +
+ +@code { + private readonly string _reasonId = $"claim-reason-{Guid.NewGuid():N}"; + private readonly string _errorId = $"claim-reason-error-{Guid.NewGuid():N}"; + private string _reason = string.Empty; + private string? _validationMessage; + private bool _additionalEvidenceRequested; + + [Parameter, EditorRequired] + public FunctionApprovalBlock Block { get; set; } = default!; + + [Parameter] + public EventCallback OnApproved { get; set; } + + [Parameter] + public EventCallback OnAdditionalEvidence { get; set; } + + [Parameter] + public EventCallback OnRejected { get; set; } + + private async Task ApproveAsync() + { + await OnApproved.InvokeAsync(); + Block.Approve(); + } + + private async Task RejectAsync() + { + if (string.IsNullOrWhiteSpace(_reason)) + { + _validationMessage = "Enter a reason before rejecting the assessment."; + return; + } + + _validationMessage = null; + var reason = _reason.Trim(); + await OnRejected.InvokeAsync(reason); + Block.Reject(reason); + } + + private async Task RequestMoreEvidence() + { + _additionalEvidenceRequested = true; + await OnAdditionalEvidence.InvokeAsync(); + Block.Reject("Additional evidence requested"); + } +} diff --git a/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimComposer.razor b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimComposer.razor new file mode 100644 index 000000000000..034fa7a6db57 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimComposer.razor @@ -0,0 +1,201 @@ +@using ComponentsAIClaimApp.Data +@using Microsoft.AspNetCore.Components.AI +@using Microsoft.AspNetCore.Components.Forms +@using Microsoft.Extensions.AI +@inject IClaimAssistantBackend ClaimAssistant + +
+ + + @if (Suggestions.Count > 0) + { +
+ @foreach (var suggestion in Suggestions) + { + + } +
+ } + + + + + + + + + + Add photo + + + + + @if (isRecording) + { + + Stop recording + } + else + { + + Record voice + } + + + + + + @if (isActive) + { + + Stop live voice + } + else + { + + Live voice + } + + + + + Cancel + + + + +
+ Drop photos here · recorded and live voice supported + + Model · @ModelName + @if (input.Attachments.Count > 0) + { + @($" · {input.Attachments.Count} attachment{(input.Attachments.Count == 1 ? string.Empty : "s")} ready") + } + +
+ + +
+ +@code { + private const long MaxAudioBytes = 8 * 1024 * 1024; + + [Parameter] + public IReadOnlyList Suggestions { get; set; } = []; + + [Parameter] + public string ModelName { get; set; } = "Foundry not configured"; + + [Parameter] + public EventCallback OnCanceled { get; set; } + + [Parameter] + public EventCallback OnLiveTranscriptChanged { get; set; } + + [Parameter] + public int ExistingPhotoCount { get; set; } + + [Parameter] + public long ExistingPhotoBytes { get; set; } + + [Parameter] + public EventCallback OnPhotoAdded { get; set; } + + private int RemainingPhotoCount => + Math.Max(0, ClaimLimits.MaximumPhotoCount - ExistingPhotoCount); + + private long RemainingPhotoBytes => + Math.Max(0, ClaimLimits.MaximumEvidenceBytes - ExistingPhotoBytes); + + private static void UseSuggestion( + MessageInputContext input, + ClaimPromptSuggestion suggestion) + { + input.Text = suggestion.Prompt; + input.SetStatusMessage($"{suggestion.Label} description selected."); + } + + private static async ValueTask CreateImageAttachmentAsync( + IBrowserFile file, + CancellationToken cancellationToken) + { + if (file.ContentType is not ( + "image/jpeg" or + "image/png" or + "image/webp" or + "image/gif")) + { + throw new InvalidOperationException( + "Choose a JPEG, PNG, WebP, or GIF image."); + } + + await using var stream = file.OpenReadStream( + ClaimLimits.MaximumPhotoBytes, + cancellationToken); + var content = await DataContent.LoadFromAsync( + stream, + file.ContentType, + cancellationToken); + content.Name = Path.GetFileName(file.Name); + return content; + } + + private async ValueTask TranscribeRecordingAsync( + DataContent recording, + CancellationToken cancellationToken) + { + return await ClaimAssistant.TranscribeAsync(recording, cancellationToken); + } + + private async Task SubmittedAsync(ChatMessage message) + { + foreach (var content in message.Contents + .OfType() + .Where(content => content.HasTopLevelMediaType("image"))) + { + await OnPhotoAdded.InvokeAsync(new ClaimPhotoPreview(content)); + } + } +} diff --git a/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimComposer.razor.css b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimComposer.razor.css new file mode 100644 index 000000000000..b593931c02fe --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimComposer.razor.css @@ -0,0 +1,335 @@ +.claim-composer { + position: relative; + display: grid; + gap: 0.55rem; + width: 100%; +} + +.claim-composer__suggestions { + display: flex; + gap: 0.4rem; + padding-bottom: 0.05rem; + overflow-x: auto; + scrollbar-width: none; +} + +.claim-composer__suggestions::-webkit-scrollbar { + display: none; +} + +.claim-composer__suggestions button { + flex: 0 0 auto; + min-height: 2rem; + padding: 0.35rem 0.65rem; + border: 1px solid var(--claim-border); + border-radius: 999px; + color: var(--claim-text); + background: var(--claim-surface); + cursor: pointer; + font-size: 0.7rem; + font-weight: 650; + transition: border-color 120ms ease, background 120ms ease, color 120ms ease; +} + +.claim-composer__suggestions button:hover:not(:disabled) { + border-color: var(--claim-accent); + color: var(--claim-accent); + background: var(--claim-accent-soft); +} + +.claim-composer ::deep .sc-ai-input { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + align-items: end; + overflow: hidden; + border: 1px solid var(--claim-border-strong); + border-radius: 1.35rem; + background: var(--claim-surface); + box-shadow: 0 0.5rem 1.5rem var(--claim-shadow); + transition: border-color 120ms ease, box-shadow 120ms ease; +} + +.claim-composer ::deep .sc-ai-input__body { + grid-column: 1 / -1; + grid-row: 1; +} + +.claim-composer ::deep .sc-ai-input__leading-actions { + grid-column: 1; + grid-row: 2; +} + +.claim-composer ::deep .sc-ai-input__send, +.claim-composer ::deep .sc-ai-input__stop { + grid-column: 2; + grid-row: 2; +} + +.claim-composer.sc-ai-drop-zone--active ::deep .sc-ai-input { + border-color: var(--claim-accent); + background: var(--claim-accent-soft); + box-shadow: + 0 0 0 0.2rem color-mix(in oklab, var(--claim-accent) 18%, transparent), + 0 0.5rem 1.5rem var(--claim-shadow); +} + +.claim-composer ::deep .sc-ai-input:focus-within { + border-color: var(--claim-accent); + box-shadow: + 0 0 0 0.18rem color-mix(in oklab, var(--claim-accent) 15%, transparent), + 0 0.5rem 1.5rem var(--claim-shadow); +} + +.claim-composer__label { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + +.claim-composer ::deep .claim-composer__textarea { + box-sizing: border-box; + display: block; + width: 100%; + min-height: 4rem; + max-height: 10rem; + padding: 0.9rem 0.95rem 0.45rem; + border: 0; + outline: 0; + color: var(--claim-text); + background: transparent; + line-height: 1.4; + text-align: left; + resize: none; +} + +.claim-composer ::deep .claim-composer__textarea::placeholder { + color: var(--claim-muted); +} + +.claim-composer ::deep .sc-ai-input__leading-actions { + display: flex; + align-items: center; +} + +.claim-composer ::deep .sc-ai-input__leading-actions { + gap: 0.5rem; + min-width: 0; +} + +.claim-composer ::deep .claim-composer__media-button { + position: relative; + display: inline-flex; + align-items: center; + gap: 0.35rem; + min-height: 2rem; + padding: 0.35rem 0.5rem; + overflow: hidden; + border: 0; + border-radius: 0.55rem; + color: var(--claim-muted); + background: transparent; + cursor: pointer; + font-size: 0.7rem; + font-weight: 650; + transition: color 120ms ease, background 120ms ease; +} + +.claim-composer ::deep .claim-composer__media-button:hover:not(:disabled) { + color: var(--claim-text); + background: var(--claim-surface-muted); +} + +.claim-composer ::deep .claim-composer__media-button svg { + width: 1.05rem; + height: 1.05rem; + fill: none; + stroke: currentColor; + stroke-linecap: round; + stroke-linejoin: round; + stroke-width: 1.8; +} + +.claim-composer ::deep .sc-ai-input__audio--recording { + color: var(--claim-danger); + background: var(--claim-danger-bg); +} + +.claim-composer ::deep .sc-ai-input__live-speech--active { + color: var(--claim-accent); + background: var(--claim-accent-soft); +} + +.claim-composer__recording-dot, +.claim-composer__live-dot { + width: 0.55rem; + height: 0.55rem; + border-radius: 50%; + background: currentColor; + animation: claim-recording-pulse 1.2s ease-in-out infinite; +} + +.claim-composer ::deep .claim-composer__file-input { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + opacity: 0; + cursor: pointer; +} + +.claim-composer ::deep .sc-ai-input__send { + display: grid; + flex: 0 0 auto; + width: 2.2rem; + height: 2.2rem; + padding: 0; + place-items: center; + border: 0; + border-radius: 50%; + color: var(--claim-accent-contrast); + background: var(--claim-accent); + cursor: pointer; + transition: background 120ms ease, transform 120ms ease, opacity 120ms ease; +} + +.claim-composer ::deep .sc-ai-input__send:hover:not(:disabled) { + background: var(--claim-accent-hover); + transform: translateY(-1px); +} + +.claim-composer ::deep .sc-ai-input__send svg { + width: 1.15rem; + fill: none; + stroke: currentColor; + stroke-linecap: round; + stroke-linejoin: round; + stroke-width: 1.8; +} + +.claim-composer ::deep button:disabled, +.claim-composer ::deep .claim-composer__file-input:disabled { + cursor: not-allowed; + opacity: 0.45; +} + +.claim-composer ::deep .claim-composer__attachments { + display: flex; + gap: 0.45rem; + padding: 0; + margin: 0; + overflow-x: auto; + list-style: none; +} + +.claim-composer ::deep .claim-composer__attachments li { + display: grid; + flex: 0 0 min(18rem, 80%); + grid-template-columns: 3rem minmax(0, 1fr) auto; + align-items: center; + gap: 0.55rem; + padding: 0.4rem; + border: 1px solid var(--claim-border); + border-radius: 0.75rem; + background: var(--claim-surface); +} + +.claim-composer ::deep .claim-composer__attachments img, +.claim-composer ::deep .claim-composer__attachments audio { + width: 3rem; + height: 2.5rem; + border-radius: 0.5rem; + object-fit: cover; +} + +.claim-composer ::deep .claim-composer__attachments span { + overflow: hidden; + font-size: 0.72rem; + font-weight: 650; + text-overflow: ellipsis; + white-space: nowrap; +} + +.claim-composer ::deep .claim-composer__attachments button { + min-height: 1.8rem; + padding: 0.25rem 0.4rem; + border: 0; + border-radius: 0.45rem; + color: var(--claim-danger); + background: transparent; + cursor: pointer; + font-size: 0.68rem; + font-weight: 700; +} + +.claim-composer ::deep .claim-composer__attachments button:hover { + background: var(--claim-danger-bg); +} + +.claim-composer__meta { + display: flex; + justify-content: space-between; + gap: 0.75rem; + min-height: 1rem; + padding: 0 0.15rem; + color: var(--claim-muted); + font-size: 0.62rem; +} + +.claim-composer ::deep .sc-ai-input__status { + overflow: hidden; + text-align: right; + text-overflow: ellipsis; + white-space: nowrap; +} + +.claim-composer ::deep .sc-ai-input__error { + color: var(--claim-danger); + font-size: 0.7rem; +} + +@keyframes claim-recording-pulse { + 0%, + 100% { + opacity: 1; + } + + 50% { + opacity: 0.35; + } +} + +@media (max-width: 640.98px) { + .claim-composer ::deep .claim-composer__media-button span { + display: none; + } + + .claim-composer ::deep .claim-composer__media-button { + width: 2rem; + padding: 0; + justify-content: center; + } + + .claim-composer ::deep .sc-ai-input__audio--recording { + width: auto; + padding: 0.35rem 0.5rem; + } + + .claim-composer ::deep .sc-ai-input__audio--recording span { + display: block; + } + + .claim-composer__meta > span:first-child { + display: none; + } + + .claim-composer ::deep .sc-ai-input__status { + width: 100%; + text-align: left; + } +} diff --git a/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimThemePicker.razor b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimThemePicker.razor new file mode 100644 index 000000000000..57f8935afa58 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimThemePicker.razor @@ -0,0 +1,64 @@ +@inject IJSRuntime JS +@inject ILogger Logger + +
+ @foreach (var option in s_options) + { + + } +
+ +@code { + private static readonly (string Value, string Label)[] s_options = + [ + ("light", "Light"), + ("dark", "Dark"), + ("contrast", "High contrast"), + ]; + private bool _initialized; + + [Parameter] + public string Theme { get; set; } = "light"; + + [Parameter] + public EventCallback ThemeChanged { get; set; } + + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (!firstRender || _initialized) + { + return; + } + + _initialized = true; + try + { + var theme = await JS.InvokeAsync("claimApp.getTheme"); + await ThemeChanged.InvokeAsync(IsSupported(theme) ? theme : "light"); + } + catch (JSException exception) + { + Logger.LogWarning(exception, "Could not restore the saved claim theme."); + } + } + + private async Task SelectThemeAsync(string theme) + { + await ThemeChanged.InvokeAsync(theme); + try + { + await JS.InvokeVoidAsync("claimApp.setTheme", theme); + } + catch (JSException exception) + { + Logger.LogWarning(exception, "Could not persist claim theme {Theme}.", theme); + } + } + + private static bool IsSupported(string theme) + => s_options.Any(option => option.Value == theme); +} diff --git a/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimThemePicker.razor.css b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimThemePicker.razor.css new file mode 100644 index 000000000000..9a95e7b7c503 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimThemePicker.razor.css @@ -0,0 +1,38 @@ +.claim-theme-picker { + display: flex; + gap: 0.15rem; + padding: 0.18rem; + border: 1px solid var(--claim-border); + border-radius: 0.7rem; + background: var(--claim-surface-alpha); +} + +.claim-theme-picker button { + min-height: 1.85rem; + padding: 0.3rem 0.55rem; + border: 0; + border-radius: 0.5rem; + color: var(--claim-muted); + background: transparent; + cursor: pointer; + font-size: 0.66rem; + font-weight: 700; + white-space: nowrap; +} + +.claim-theme-picker button:hover { + color: var(--claim-text); + background: var(--claim-surface-muted); +} + +.claim-theme-picker button[aria-pressed="true"] { + color: var(--claim-text); + background: var(--claim-surface-muted); + box-shadow: 0 0.1rem 0.35rem var(--claim-shadow); +} + +@media (max-width: 640.98px) { + .claim-theme-picker button { + padding-inline: 0.42rem; + } +} diff --git a/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimVehicleDiagram.razor b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimVehicleDiagram.razor new file mode 100644 index 000000000000..9eb1bb6ae5db --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimVehicleDiagram.razor @@ -0,0 +1,69 @@ +@using ComponentsAIClaimApp.Data + +
+ + Vehicle diagram with likely affected areas highlighted + @AffectedAreasDescription + + + + + + + + + + + + + + + + + + FRONT + + +
+ + @if (State.AffectedAreas.Count == 0) + { + No affected areas identified + } + else + { + @State.AffectedAreas.Count likely affected area@(State.AffectedAreas.Count == 1 ? "" : "s") + } +
+
+ +@code { + private readonly string _titleId = $"vehicle-title-{Guid.NewGuid():N}"; + private readonly string _descriptionId = $"vehicle-description-{Guid.NewGuid():N}"; + + [Parameter, EditorRequired] + public ClaimState State { get; set; } = default!; + + private string AffectedAreasDescription + => State.AffectedAreas.Count == 0 + ? "Top-down vehicle view. No affected areas are currently identified." + : $"Top-down vehicle view. Likely affected areas: {string.Join(", ", State.AffectedAreas)}."; + + private string AreaClass(string area) + => State.AffectedAreas.Contains(area, StringComparer.OrdinalIgnoreCase) + ? "vehicle-area vehicle-area--affected" + : "vehicle-area"; +} diff --git a/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimVehicleDiagram.razor.css b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimVehicleDiagram.razor.css new file mode 100644 index 000000000000..7ad040b72c7f --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Pages/ClaimVehicleDiagram.razor.css @@ -0,0 +1,86 @@ +.vehicle-diagram { + padding: 0.9rem; + border: 1px solid var(--claim-border); + border-radius: 1rem; + background: linear-gradient(180deg, var(--claim-surface), var(--claim-surface-muted)); +} + +.vehicle-diagram svg { + display: block; + width: 100%; + height: auto; + overflow: visible; +} + +.vehicle-shadow { + fill: var(--claim-shadow); +} + +.vehicle-outline { + fill: var(--claim-surface-muted); + stroke: var(--claim-muted); + stroke-width: 2.5; +} + +.vehicle-area { + fill: var(--claim-border); + stroke: var(--claim-muted); + stroke-width: 1.5; + transition: fill 180ms ease, stroke 180ms ease; +} + +.vehicle-area--affected { + fill: color-mix(in oklab, var(--claim-danger) 60%, var(--claim-surface)); + stroke: var(--claim-danger); + stroke-width: 2.5; + filter: drop-shadow(0 0 5px rgba(210, 72, 38, 0.3)); +} + +.vehicle-wheel { + fill: var(--claim-text); + stroke: var(--claim-bg); + stroke-width: 2; +} + +.vehicle-roof { + fill: var(--claim-surface-muted); + stroke: var(--claim-muted); + stroke-width: 1.5; +} + +.vehicle-rear-glass { + fill: var(--claim-border-strong); + stroke: var(--claim-muted); + stroke-width: 1.5; +} + +.vehicle-center-line { + fill: none; + stroke: var(--claim-muted); + stroke-dasharray: 3 4; + stroke-width: 1; +} + +.vehicle-orientation { + fill: var(--claim-muted); + font-size: 9px; + font-weight: 800; + letter-spacing: 0.12em; +} + +.vehicle-legend { + display: flex; + align-items: center; + justify-content: center; + gap: 0.45rem; + color: var(--claim-muted); + font-size: 0.76rem; +} + +.vehicle-legend__swatch { + width: 0.7rem; + height: 0.7rem; + border-radius: 0.2rem; + background: color-mix(in oklab, var(--claim-danger) 60%, var(--claim-surface)); + box-shadow: inset 0 0 0 1px var(--claim-danger); +} diff --git a/src/Components/AI/samples/ClaimApp/Components/Pages/VehicleDamageBlock.razor b/src/Components/AI/samples/ClaimApp/Components/Pages/VehicleDamageBlock.razor new file mode 100644 index 000000000000..42eb39d46b90 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Pages/VehicleDamageBlock.razor @@ -0,0 +1,52 @@ +@using System.Text.Json +@using ComponentsAIClaimApp.Data +@using Microsoft.AspNetCore.Components.AI + +
+ Backend tool + Likely affected vehicle areas + @if (Block.Result is null) + { +
Analyzing the accident description...
+ } + else + { +
+ @foreach (var area in Assessment.Areas) + { + @area.Replace('-', ' ') + } +
+ } +
+ +@code { + [Parameter, EditorRequired] + public FunctionInvocationContentBlock Block { get; set; } = default!; + + private VehicleAreaAssessment Assessment + { + get + { + var result = Block.Result?.Result; + if (result is VehicleAreaAssessment assessment) + { + return assessment; + } + + if (result is string json) + { + try + { + return JsonSerializer.Deserialize(json, ClaimStateJson.Options) + ?? new VehicleAreaAssessment(); + } + catch (JsonException) + { + } + } + + return new VehicleAreaAssessment(); + } + } +} diff --git a/src/Components/AI/samples/ClaimApp/Components/Pages/VehicleDiagramAction.razor b/src/Components/AI/samples/ClaimApp/Components/Pages/VehicleDiagramAction.razor new file mode 100644 index 000000000000..f8c100e7cda5 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Pages/VehicleDiagramAction.razor @@ -0,0 +1,25 @@ +@using Microsoft.AspNetCore.Components.AI + +
+ + @(Block.IsComplete ? "Vehicle diagram updated" : "Updating vehicle diagram") +
+ +@code { + private bool _invoking; + + [Parameter, EditorRequired] + public UIActionBlock Block { get; set; } = default!; + + protected override async Task OnParametersSetAsync() + { + if (Block.IsComplete || _invoking) + { + return; + } + + _invoking = true; + await Block.InvokeAsync(); + _invoking = false; + } +} diff --git a/src/Components/AI/samples/ClaimApp/Components/Routes.razor b/src/Components/AI/samples/ClaimApp/Components/Routes.razor new file mode 100644 index 000000000000..9ae5741cd21f --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/Routes.razor @@ -0,0 +1,12 @@ + + + + + + + Not found + +

The requested page was not found.

+ + + diff --git a/src/Components/AI/samples/ClaimApp/Components/_Imports.razor b/src/Components/AI/samples/ClaimApp/Components/_Imports.razor new file mode 100644 index 000000000000..0dae9eccc325 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Components/_Imports.razor @@ -0,0 +1,12 @@ +@using System.Net.Http +@using System.Net.Http.Json +@using ComponentsAIClaimApp +@using ComponentsAIClaimApp.Components +@using ComponentsAIClaimApp.Components.Layout +@using ComponentsAIClaimApp.Data +@using Microsoft.AspNetCore.Components.Endpoints +@using Microsoft.AspNetCore.Components.Forms +@using Microsoft.AspNetCore.Components.Routing +@using Microsoft.AspNetCore.Components.Web +@using Microsoft.JSInterop +@using static Microsoft.AspNetCore.Components.Web.RenderMode diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimAgentAddress.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimAgentAddress.cs new file mode 100644 index 000000000000..e88d883e538c --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimAgentAddress.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ComponentsAIClaimApp.Data; + +internal static class ClaimAgentAddress +{ + internal static Uri Resolve( + string? configuredBaseAddress, + string navigationBaseUri) + { + var value = string.IsNullOrWhiteSpace(configuredBaseAddress) + ? navigationBaseUri + : configuredBaseAddress; + if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || + uri.Scheme is not ("http" or "https") || + string.IsNullOrEmpty(uri.Host) || + !string.IsNullOrEmpty(uri.Query) || + !string.IsNullOrEmpty(uri.Fragment)) + { + throw new InvalidOperationException( + "ClaimAgent:BaseAddress must be an absolute HTTP or HTTPS URI without a query or fragment."); + } + + return uri.AbsolutePath.EndsWith('/', StringComparison.Ordinal) + ? uri + : new UriBuilder(uri) { Path = $"{uri.AbsolutePath}/" }.Uri; + } +} diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimAgentChatClient.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimAgentChatClient.cs new file mode 100644 index 000000000000..cfcfd44991ab --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimAgentChatClient.cs @@ -0,0 +1,138 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Text.Json; +using AGUI.Abstractions; +using Microsoft.Extensions.AI; + +namespace ComponentsAIClaimApp.Data; + +internal sealed class ClaimAgentChatClient : IChatClient +{ + private readonly ClaimAgentTransport _transport; + + public ClaimAgentChatClient( + IClaimAssistantBackend backend, + ILogger logger) + { + _transport = new ClaimAgentTransport(backend, logger); + } + + public async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var input = options?.RawRepresentationFactory?.Invoke(this) as RunAgentInput; + await foreach (var evt in _transport.SendAsync( + messages.ToList(), + input?.State, + cancellationToken)) + { + switch (evt) + { + case ClaimAgentErrorEvent error: + yield return new ChatResponseUpdate + { + RawRepresentation = new RunErrorEvent + { + Code = error.Code, + Message = error.Message, + }, + }; + break; + + case ClaimAgentTextEvent text: + yield return new ChatResponseUpdate + { + Role = ChatRole.Assistant, + MessageId = text.MessageId, + Contents = [new TextContent(text.Delta)], + }; + break; + + case ClaimAgentToolCallEvent toolCall: + yield return new ChatResponseUpdate + { + Role = ChatRole.Assistant, + MessageId = toolCall.MessageId, + Contents = + [ + new FunctionCallContent( + toolCall.ToolCallId, + toolCall.ToolName, + DeserializeArguments(toolCall.Arguments)), + ], + FinishReason = ChatFinishReason.ToolCalls, + }; + break; + + case ClaimAgentToolResultEvent toolResult: + yield return new ChatResponseUpdate + { + Contents = + [ + new FunctionResultContent( + toolResult.ToolCallId, + toolResult.Result), + ], + }; + break; + + case ClaimAgentApprovalRequestEvent approval: + var approvalCall = new FunctionCallContent( + approval.ToolCallId, + approval.ToolName, + DeserializeArguments(approval.Arguments)); + yield return new ChatResponseUpdate + { + Role = ChatRole.Assistant, + MessageId = approval.MessageId, + Contents = + [ + new ToolApprovalRequestContent( + approval.RequestId, + approvalCall), + ], + }; + break; + + case ClaimAgentStateSnapshotEvent snapshot: + yield return new ChatResponseUpdate + { + RawRepresentation = new StateSnapshotEvent + { + Snapshot = snapshot.Snapshot, + }, + }; + break; + + case ClaimAgentStateDeltaEvent delta: + yield return new ChatResponseUpdate + { + RawRepresentation = new StateDeltaEvent + { + Delta = delta.Delta, + }, + }; + break; + } + } + } + + public Task GetResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + CancellationToken cancellationToken = default) + => throw new NotSupportedException("This sample client uses streaming responses."); + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() + { + } + + private static IDictionary? DeserializeArguments(JsonElement arguments) + => arguments.Deserialize>(ClaimStateJson.Options); +} diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimAgentEventStreamResult.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimAgentEventStreamResult.cs new file mode 100644 index 000000000000..9c5afe343b8b --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimAgentEventStreamResult.cs @@ -0,0 +1,44 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using AGUI.Abstractions; +using AGUI.Formatting; +using Microsoft.AspNetCore.Http.Features; + +namespace ComponentsAIClaimApp.Data; + +internal sealed class ClaimAgentEventStreamResult : IResult +{ + private readonly IAsyncEnumerable _events; + private readonly IAGUIEventStreamFormatter _formatter; + private readonly CancellationToken _cancellationToken; + + public ClaimAgentEventStreamResult( + IAsyncEnumerable events, + IAGUIEventStreamFormatter formatter, + CancellationToken cancellationToken) + { + _events = events; + _formatter = formatter; + _cancellationToken = cancellationToken; + } + + public async Task ExecuteAsync(HttpContext httpContext) + { + ArgumentNullException.ThrowIfNull(httpContext); + + var response = httpContext.Response; + response.StatusCode = StatusCodes.Status200OK; + response.ContentType = _formatter.MediaType; + response.Headers.CacheControl = "no-cache,no-store"; + response.Headers.Pragma = "no-cache"; + + httpContext.Features.Get()?.DisableBuffering(); + + using var linked = CancellationTokenSource.CreateLinkedTokenSource( + httpContext.RequestAborted, + _cancellationToken); + await _formatter.WriteAsync(_events, response.Body, linked.Token); + await response.Body.FlushAsync(linked.Token); + } +} diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimAgentEvents.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimAgentEvents.cs new file mode 100644 index 000000000000..26386159cfec --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimAgentEvents.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; + +namespace ComponentsAIClaimApp.Data; + +internal abstract record ClaimAgentEvent; + +internal sealed record ClaimAgentErrorEvent(string Code, string Message) : ClaimAgentEvent; + +internal sealed record ClaimAgentTextEvent( + string MessageId, + string Delta) : ClaimAgentEvent; + +internal sealed record ClaimAgentToolCallEvent( + string MessageId, + string ToolCallId, + string ToolName, + JsonElement Arguments) : ClaimAgentEvent; + +internal sealed record ClaimAgentToolResultEvent( + string ToolCallId, + object? Result) : ClaimAgentEvent; + +internal sealed record ClaimAgentApprovalRequestEvent( + string MessageId, + string RequestId, + string ToolCallId, + string ToolName, + JsonElement Arguments) : ClaimAgentEvent; + +internal sealed record ClaimAgentStateSnapshotEvent(JsonElement Snapshot) : ClaimAgentEvent; + +internal sealed record ClaimAgentStateDeltaEvent(JsonElement Delta) : ClaimAgentEvent; diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimAgentOptions.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimAgentOptions.cs new file mode 100644 index 000000000000..6977ad3d5ec6 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimAgentOptions.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ComponentsAIClaimApp.Data; + +internal sealed class ClaimAgentOptions +{ + public string? BaseAddress { get; set; } +} diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimAgentTransport.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimAgentTransport.cs new file mode 100644 index 000000000000..82d0813fc5bc --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimAgentTransport.cs @@ -0,0 +1,447 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text.Json; +using Microsoft.Extensions.AI; + +namespace ComponentsAIClaimApp.Data; + +internal sealed class ClaimAgentTransport +{ + private const string IdentifyToolCallId = "identify-vehicle-areas"; + private const string DisplayToolCallId = "display-vehicle-damage"; + private const string ApprovalToolCallId = "submit-claim-assessment"; + private const string AdditionalEvidenceReason = "Additional evidence requested"; + private const string BackendErrorMessage = + "The claim assistant could not complete the request."; + private readonly IClaimAssistantBackend _backend; + private readonly ILogger _logger; + + public ClaimAgentTransport( + IClaimAssistantBackend backend, + ILogger logger) + { + _backend = backend; + _logger = logger; + } + + public async IAsyncEnumerable SendAsync( + IReadOnlyList messages, + JsonElement? stateSnapshot, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var lastMessage = messages.LastOrDefault(); + if (lastMessage?.Contents.OfType().LastOrDefault() is { } approval) + { + await foreach (var evt in CompleteApprovalAsync( + messages, + stateSnapshot, + approval, + cancellationToken)) + { + yield return evt; + } + + yield break; + } + + if (lastMessage?.Contents.OfType().LastOrDefault() is { } toolResult) + { + await foreach (var evt in ContinueAfterToolAsync( + messages, + stateSnapshot, + toolResult, + cancellationToken)) + { + yield return evt; + } + + yield break; + } + + var evidence = messages + .Where(message => message.Role == ChatRole.User) + .SelectMany(message => message.Contents) + .OfType() + .ToList(); + var media = evidence + .Where(content => content.HasTopLevelMediaType("image") || + content.HasTopLevelMediaType("audio")) + .ToList(); + var currentMedia = lastMessage?.Contents + .OfType() + .Where(content => content.HasTopLevelMediaType("image") || + content.HasTopLevelMediaType("audio")) + .ToList() ?? []; + var images = media + .Where(content => content.HasTopLevelMediaType("image")) + .ToList(); + var audio = media + .Where(content => content.HasTopLevelMediaType("audio")) + .ToList(); + if (images.Count > ClaimLimits.MaximumPhotoCount) + { + yield return new ClaimAgentErrorEvent( + "claim_evidence_limit", + "A claim can include up to six photos."); + yield break; + } + if (images.Any(image => + image.Data.Length > ClaimLimits.MaximumPhotoBytes)) + { + yield return new ClaimAgentErrorEvent( + "claim_evidence_limit", + "Each claim photo must be 8 MB or smaller."); + yield break; + } + if (evidence.Sum(content => (long)content.Data.Length) > + ClaimLimits.MaximumEvidenceBytes) + { + yield return new ClaimAgentErrorEvent( + "claim_evidence_limit", + "A claim can include up to 24 MB of total evidence."); + yield break; + } + + var state = ReadState(stateSnapshot); + var routing = await CallBackendAsync( + () => _backend.ShouldAnalyzeEvidenceAsync( + messages, + state, + currentMedia.Count, + cancellationToken)); + if (routing.Error is not null) + { + yield return new ClaimAgentErrorEvent( + "claim_conversation_failure", + routing.Error); + yield break; + } + + if (!routing.Value) + { + if (state.Confidence == 0) + { + state.Status = "Ready for claim details"; + } + yield return new ClaimAgentStateSnapshotEvent( + JsonSerializer.SerializeToElement(state, ClaimStateJson.Options)); + + var response = await CallBackendAsync( + () => _backend.GenerateResponseAsync( + messages, + state, + ClaimResponsePurpose.Conversation, + cancellationToken)); + if (response.Error is not null) + { + yield return new ClaimAgentErrorEvent( + "claim_conversation_failure", + response.Error); + yield break; + } + yield return TextMessage(response.Value); + yield break; + } + + var currentUserText = + lastMessage?.Text ?? "Review the attached claim evidence."; + var userText = string.Join( + Environment.NewLine, + messages + .Where(message => message.Role == ChatRole.User) + .Select(message => message.Text) + .Where(text => + !string.IsNullOrWhiteSpace(text) && + !string.Equals( + text, + "Review the attached claim evidence.", + StringComparison.OrdinalIgnoreCase))); + if (string.IsNullOrWhiteSpace(userText)) + { + userText = currentUserText; + } + + state.Status = "Analyzing evidence with Foundry"; + state.AccidentSummary = userText; + state.EvidenceSummary = DescribeMedia(media); + state.Decision = "Pending"; + state.RejectionReason = null; + + yield return new ClaimAgentStateSnapshotEvent( + JsonSerializer.SerializeToElement(state, ClaimStateJson.Options)); + + var analysis = await CallBackendAsync( + () => _backend.AnalyzeAsync( + userText, + images, + audio, + cancellationToken)); + if (analysis.Error is not null) + { + yield return new ClaimAgentErrorEvent( + "vision_analysis_failure", + analysis.Error); + yield break; + } + + var completedAnalysis = analysis.Value; + state.Status = "Evidence analyzed"; + state.AssessmentSummary = completedAnalysis.Summary; + state.AffectedAreas = []; + state.Confidence = completedAnalysis.Confidence; + state.Decision = "Pending"; + state.RejectionReason = null; + state.DamageFindings = completedAnalysis.Findings; + state.NextPhotoSuggestion = completedAnalysis.NextPhotoSuggestion; + state.NeedsHumanReview = completedAnalysis.NeedsHumanReview; + state.VoiceTranscript = completedAnalysis.VoiceTranscript; + state.RepairEstimate = completedAnalysis.RepairEstimate; + state.ReplacementParts = completedAnalysis.ReplacementParts; + state.ResearchSources = completedAnalysis.ResearchSources; + state.ResearchWarning = completedAnalysis.ResearchWarning; + + yield return new ClaimAgentStateSnapshotEvent( + JsonSerializer.SerializeToElement(state, ClaimStateJson.Options)); + + if (!HasReviewableAssessment(completedAnalysis)) + { + state.Status = "More evidence needed"; + state.Decision = "Pending"; + yield return new ClaimAgentStateSnapshotEvent( + JsonSerializer.SerializeToElement(state, ClaimStateJson.Options)); + + var response = await CallBackendAsync( + () => _backend.GenerateResponseAsync( + messages, + state, + ClaimResponsePurpose.MoreEvidence, + cancellationToken)); + if (response.Error is not null) + { + yield return new ClaimAgentErrorEvent( + "claim_conversation_failure", + response.Error); + yield break; + } + yield return TextMessage(response.Value); + yield break; + } + + var identifyToolCallId = CreateToolCallId(IdentifyToolCallId); + yield return ToolCall( + identifyToolCallId, + "identify_vehicle_areas", + new { description = userText }); + yield return new ClaimAgentToolResultEvent( + identifyToolCallId, + new VehicleAreaAssessment + { + Areas = completedAnalysis.AffectedAreas, + Severity = completedAnalysis.Findings + .Select(finding => finding.Severity) + .FirstOrDefault() ?? "Moderate", + }); + yield return ToolCall( + CreateToolCallId(DisplayToolCallId), + "display_vehicle_damage", + new { areas = completedAnalysis.AffectedAreas }); + } + + private async IAsyncEnumerable ContinueAfterToolAsync( + IReadOnlyList messages, + JsonElement? stateSnapshot, + FunctionResultContent toolResult, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + yield return new ClaimAgentToolResultEvent(toolResult.CallId, toolResult.Result); + + if (!IsToolCallId(toolResult.CallId, DisplayToolCallId)) + { + yield break; + } + + var state = ReadState(stateSnapshot); + state.Status = "Awaiting your decision"; + state.Decision = "Pending"; + + yield return new ClaimAgentStateDeltaEvent( + JsonSerializer.SerializeToElement(new object[] + { + new { op = "replace", path = "/status", value = state.Status }, + new { op = "replace", path = "/assessmentSummary", value = state.AssessmentSummary }, + new { op = "replace", path = "/confidence", value = state.Confidence }, + new { op = "replace", path = "/decision", value = state.Decision }, + new { op = "replace", path = "/damageFindings", value = state.DamageFindings }, + new { op = "replace", path = "/nextPhotoSuggestion", value = state.NextPhotoSuggestion }, + new { op = "replace", path = "/needsHumanReview", value = state.NeedsHumanReview }, + new { op = "replace", path = "/voiceTranscript", value = state.VoiceTranscript }, + new { op = "replace", path = "/repairEstimate", value = state.RepairEstimate }, + new { op = "replace", path = "/replacementParts", value = state.ReplacementParts }, + new { op = "replace", path = "/researchSources", value = state.ResearchSources }, + new { op = "replace", path = "/researchWarning", value = state.ResearchWarning }, + })); + + var response = await CallBackendAsync( + () => _backend.GenerateResponseAsync( + messages, + state, + ClaimResponsePurpose.AssessmentReady, + cancellationToken)); + if (response.Error is not null) + { + yield return new ClaimAgentErrorEvent( + "claim_conversation_failure", + response.Error); + yield break; + } + + yield return TextMessage(response.Value); + var approvalToolCallId = CreateToolCallId(ApprovalToolCallId); + yield return new ClaimAgentApprovalRequestEvent( + Guid.NewGuid().ToString("N"), + $"approve-{approvalToolCallId}", + approvalToolCallId, + "submit_claim_assessment", + JsonSerializer.SerializeToElement(new + { + state.AssessmentSummary, + state.Confidence, + }, ClaimStateJson.Options)); + } + + private async IAsyncEnumerable CompleteApprovalAsync( + IReadOnlyList messages, + JsonElement? stateSnapshot, + ToolApprovalResponseContent approval, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var state = ReadState(stateSnapshot); + var requestsAdditionalEvidence = + !approval.Approved && + (string.Equals( + approval.Reason, + AdditionalEvidenceReason, + StringComparison.Ordinal) || + string.Equals( + state.Status, + "Add more evidence", + StringComparison.Ordinal)); + state.Decision = requestsAdditionalEvidence + ? "Pending" + : approval.Approved ? "Approved" : "Rejected"; + state.Status = requestsAdditionalEvidence + ? "Add more evidence" + : approval.Approved ? "Assessment approved" : "Assessment rejected"; + state.RejectionReason = requestsAdditionalEvidence + ? null + : approval.Reason ?? state.RejectionReason; + + yield return new ClaimAgentToolResultEvent( + approval.ToolCall.CallId ?? ApprovalToolCallId, + requestsAdditionalEvidence + ? "The user requested another evidence collection turn." + : approval.Approved + ? "The user approved the assessment." + : $"The user rejected the assessment. Reason: {state.RejectionReason ?? "No reason supplied."}"); + + yield return new ClaimAgentStateDeltaEvent( + JsonSerializer.SerializeToElement(new object[] + { + new { op = "replace", path = "/status", value = state.Status }, + new { op = "replace", path = "/decision", value = state.Decision }, + new { op = "replace", path = "/rejectionReason", value = state.RejectionReason }, + })); + + var response = await CallBackendAsync( + () => _backend.GenerateResponseAsync( + messages, + state, + ClaimResponsePurpose.Decision, + cancellationToken)); + if (response.Error is not null) + { + yield return new ClaimAgentErrorEvent( + "claim_conversation_failure", + response.Error); + yield break; + } + + yield return TextMessage(response.Value); + } + + private async Task> CallBackendAsync( + Func> operation) + { + try + { + return new(await operation(), null); + } + catch (HttpRequestException exception) + { + return BackendFailure(exception); + } + catch (InvalidOperationException exception) + { + return BackendFailure(exception); + } + catch (JsonException exception) + { + return BackendFailure(exception); + } + } + + private BackendResult BackendFailure(Exception exception) + { + _logger.LogError(exception, "The claim assistant backend request failed."); + return new(default!, BackendErrorMessage); + } + + private static ClaimAgentTextEvent TextMessage(string text) + => new(Guid.NewGuid().ToString("N"), text); + + private static ClaimAgentToolCallEvent ToolCall( + string toolCallId, + string toolName, + object arguments) + => new( + Guid.NewGuid().ToString("N"), + toolCallId, + toolName, + JsonSerializer.SerializeToElement(arguments, ClaimStateJson.Options)); + + private static string CreateToolCallId(string prefix) + => $"{prefix}-{Guid.NewGuid():N}"; + + private static bool IsToolCallId(string? callId, string prefix) + => string.Equals(callId, prefix, StringComparison.Ordinal) || + callId?.StartsWith($"{prefix}-", StringComparison.Ordinal) == true; + + private static ClaimState ReadState(JsonElement? state) + => state?.Deserialize(ClaimStateJson.Options) ?? new ClaimState(); + + private static string DescribeMedia(IReadOnlyList media) + { + var imageCount = media.Count(content => content.HasTopLevelMediaType("image")); + var audioCount = media.Count(content => content.HasTopLevelMediaType("audio")); + var parts = new List(); + if (imageCount > 0) + { + parts.Add($"{imageCount} image{(imageCount == 1 ? string.Empty : "s")}"); + } + if (audioCount > 0) + { + parts.Add($"{audioCount} voice note{(audioCount == 1 ? string.Empty : "s")}"); + } + + return parts.Count == 0 ? "No evidence attached." : string.Join(" and ", parts); + } + + private static bool HasReviewableAssessment(ClaimDamageAnalysis analysis) + => analysis.Confidence >= 25 && + (analysis.Findings.Count > 0 || analysis.AffectedAreas.Count > 0); + + private readonly record struct BackendResult(T Value, string? Error); +} diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimDamageAnalyzer.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimDamageAnalyzer.cs new file mode 100644 index 000000000000..b5cdf2799f88 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimDamageAnalyzer.cs @@ -0,0 +1,744 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.RegularExpressions; +using Microsoft.Extensions.AI; + +namespace ComponentsAIClaimApp.Data; + +internal sealed class ClaimDamageAnalyzer : IClaimAssistantBackend +{ + private const string ChatApiVersion = "2025-01-01-preview"; + private const string TranscriptionApiVersion = "2025-03-01-preview"; + + private static readonly HttpClient s_httpClient = new() + { + Timeout = TimeSpan.FromMinutes(2), + }; + + private readonly ClaimFoundryOptions _options; + private readonly ConditionalWeakTable _transcripts = new(); + + public ClaimDamageAnalyzer(ClaimFoundryOptions options) + { + _options = options; + } + + public bool IsConfigured => + !string.IsNullOrWhiteSpace(_options.Endpoint) && + !string.IsNullOrWhiteSpace(_options.ApiKey); + + public string ModelName => IsConfigured + ? _options.ChatDeployment + : "Foundry not configured"; + + public async Task ShouldAnalyzeEvidenceAsync( + IReadOnlyList messages, + ClaimState state, + int currentMediaCount, + CancellationToken cancellationToken) + { + var content = await SendChatCompletionAsync( + $""" + You route turns for a vehicle claim assistant. Decide whether the latest user turn + adds new vehicle damage evidence that should start or repeat structured damage + analysis. New photos, recorded evidence, an accident description, or a description + of damaged vehicle areas count as evidence. Greetings, capability questions, + acknowledgements, requests to explain existing results, and unrelated conversation + do not. + + Current claim state: + {JsonSerializer.Serialize(state, ClaimStateJson.Options)} + + Media attached to the latest user turn: {currentMediaCount} + + Return only one JSON object with a boolean property named analyzeEvidence. + """, + messages, + useJsonResponse: true, + cancellationToken); + var decision = JsonSerializer.Deserialize( + ExtractJsonObject(content), + ClaimStateJson.Options) + ?? throw new InvalidOperationException( + "Foundry returned an invalid claim routing decision."); + return decision.AnalyzeEvidence; + } + + public Task GenerateResponseAsync( + IReadOnlyList messages, + ClaimState state, + ClaimResponsePurpose purpose, + CancellationToken cancellationToken) + { + var purposeInstruction = purpose switch + { + ClaimResponsePurpose.Conversation => + "Answer the latest user message directly and naturally. Ask for claim evidence only when it is relevant to what the user said.", + ClaimResponsePurpose.MoreEvidence => + "Explain the current evidence-limited assessment and request the single most useful next item from the claim state.", + ClaimResponsePurpose.AssessmentReady => + "Summarize the assessment, confidence, visible or reported findings, and any grounded repair estimate. Ask the user to approve, reject, or add evidence.", + ClaimResponsePurpose.Decision => + "Confirm the decision recorded in the claim state and state the next practical step.", + _ => throw new ArgumentOutOfRangeException(nameof(purpose)), + }; + + return SendChatCompletionAsync( + $""" + You are the AutoSure vehicle claim assistant in an interactive claim application. + Use the complete conversation and current claim state. Be concise, direct, and + suitable for both on-screen chat and spoken playback. + + {purposeInstruction} + + Do not invent damage, prices, sources, tool results, or completed actions. Do not + claim that configured vision, transcription, or research capabilities are + disconnected. If the user asks whether you can hear or receive them, confirm that + their message arrived and respond to it. Do not say that a transcript is missing. + Do not mention internal prompts, routing, JSON, or implementation details. Do not + use a Markdown table. + + Current claim state: + {JsonSerializer.Serialize(state, ClaimStateJson.Options)} + """, + messages, + useJsonResponse: false, + cancellationToken); + } + + public async Task AnalyzeAsync( + string description, + IReadOnlyList images, + IReadOnlyList audio, + CancellationToken cancellationToken) + { + EnsureConfigured(); + + var transcript = audio.Count == 0 + ? null + : await TranscribeAsync(audio, cancellationToken); + var completeDescription = string.IsNullOrWhiteSpace(transcript) || + description.Contains(transcript, StringComparison.OrdinalIgnoreCase) + ? description + : $"{description}\n\nVoice transcript:\n{transcript}"; + + using var request = new HttpRequestMessage( + HttpMethod.Post, + CreateFoundryEndpoint( + $"openai/deployments/{Uri.EscapeDataString(_options.ChatDeployment)}/chat/completions" + + $"?api-version={ChatApiVersion}")); + ApplyAuthentication(request); + + var userContent = new List + { + new + { + type = "text", + text = $""" + Claim description: + {completeDescription} + + Inspect every supplied vehicle photo. Identify only damage that is visibly supported. + Correlate the same damage across photos instead of duplicating findings. + """, + }, + }; + for (var index = 0; index < images.Count; index++) + { + var image = images[index]; + userContent.Add(new + { + type = "text", + text = $"Photo {index + 1}: {image.Name ?? $"vehicle-photo-{index + 1}"}", + }); + userContent.Add(new + { + type = "image_url", + image_url = new + { + url = image.Uri, + detail = "high", + }, + }); + } + + request.Content = JsonContent.Create(new + { + model = _options.ChatDeployment, + response_format = new + { + type = "json_object", + }, + messages = new object[] + { + new + { + role = "system", + content = """ + You are a vehicle damage intake assistant. Return one JSON object with: + summary: a concise assessment + confidence: integer 0-100 + affectedAreas: identifiers chosen from front-bumper, hood, windshield, + left-fender, left-door, right-fender, right-door, rear-bumper + findings: array of objects with area, damageType, severity, confidence, + evidence, and photoNames. photoNames must use the exact photo names + provided in the user content + nextPhotoSuggestion: the single most useful additional photo, or null + needsHumanReview: boolean + + Do not infer hidden structural or mechanical damage. Use "possible" and + require human review when image evidence is ambiguous. + """, + }, + new + { + role = "user", + content = userContent, + }, + }, + }); + + using var response = await s_httpClient.SendAsync(request, cancellationToken); + var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + if (!response.IsSuccessStatusCode) + { + throw new InvalidOperationException( + $"Vision analysis failed with HTTP {(int)response.StatusCode}."); + } + + using var envelope = JsonDocument.Parse(responseBody); + var content = envelope.RootElement + .GetProperty("choices")[0] + .GetProperty("message") + .GetProperty("content") + .GetString(); + if (string.IsNullOrWhiteSpace(content)) + { + throw new InvalidOperationException("Vision analysis returned an empty response."); + } + + var analysis = JsonSerializer.Deserialize( + content, + ClaimStateJson.Options) + ?? throw new InvalidOperationException( + "Vision analysis returned invalid JSON."); + ClaimResearchLink.Sanitize(analysis); + return await AddResearchAsync( + analysis, + completeDescription, + transcript, + cancellationToken); + } + + private async Task AddResearchAsync( + ClaimDamageAnalysis analysis, + string description, + string? transcript, + CancellationToken cancellationToken) + { + analysis.VoiceTranscript = transcript; + if (analysis.Findings.Count == 0) + { + analysis.ResearchWarning = + "Parts and pricing research starts after visible or reported damage is identified."; + return analysis; + } + if (!Regex.IsMatch(description, @"\b(?:19|20)\d{2}\b")) + { + analysis.ResearchWarning = + "Add the vehicle year, make, model, and repair location for grounded parts and pricing."; + return analysis; + } + + try + { + var research = await ResearchMarketAsync( + description, + analysis, + cancellationToken); + analysis.RepairEstimate = research.RepairEstimate; + analysis.ReplacementParts = research.ReplacementParts; + analysis.ResearchWarning = research.Warning; + } + catch (HttpRequestException exception) + { + analysis.ResearchWarning = $"Live market research failed: {exception.Message}"; + } + catch (InvalidOperationException exception) + { + analysis.ResearchWarning = $"Live market research failed: {exception.Message}"; + } + catch (JsonException exception) + { + analysis.ResearchWarning = $"Live market research failed: {exception.Message}"; + } + + return analysis; + } + + private async Task ResearchMarketAsync( + string description, + ClaimDamageAnalysis analysis, + CancellationToken cancellationToken) + { + using var request = new HttpRequestMessage( + HttpMethod.Post, + CreateFoundryEndpoint("openai/v1/responses")); + ApplyAuthentication(request); + request.Content = JsonContent.Create(new + { + model = _options.ChatDeployment, + tools = new[] { new { type = "web_search" } }, + input = $$""" + Research current public repair-cost and replacement-part sources for this + vehicle claim. Use web search. Do not invent prices, fitment, or URLs. + + Claim details: + {{description}} + + Damage analysis: + {{JsonSerializer.Serialize(analysis.Findings, ClaimStateJson.Options)}} + + Return only one JSON object with this shape: + { + "repairEstimate": { + "low": number, + "high": number, + "currency": "USD", + "basis": "short explanation of labor, paint, calibration, and parts assumptions" + }, + "replacementParts": [ + { + "name": "part name", + "priceLow": number, + "priceHigh": number, + "currency": "USD", + "fitment": "exact fitment caveat or verification needed", + "sourceTitle": "public source title", + "sourceUrl": "https://..." + } + ], + "warning": "missing vehicle, trim, VIN, location, labor, or teardown details" + } + + Use zero prices and explain the missing information when a grounded range + cannot be found. This is an intake estimate, not a repair authorization, + appraisal, or settlement value. + """, + }); + + using var response = await s_httpClient.SendAsync(request, cancellationToken); + var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + if (!response.IsSuccessStatusCode) + { + throw new InvalidOperationException( + $"Web research failed with HTTP {(int)response.StatusCode}."); + } + + using var envelope = JsonDocument.Parse(responseBody); + var outputText = GetResponseOutputText(envelope.RootElement); + var research = JsonSerializer.Deserialize( + ExtractJsonObject(outputText), + ClaimStateJson.Options) + ?? throw new InvalidOperationException("Web research returned invalid JSON."); + foreach (var part in research.ReplacementParts) + { + part.SourceUrl = ClaimResearchLink.Normalize(part.SourceUrl) ?? string.Empty; + } + analysis.ResearchSources = GetResearchSources(envelope.RootElement, research); + return research; + } + + internal async Task TranscribeAsync( + IReadOnlyList audio, + CancellationToken cancellationToken) + { + if (!IsConfigured) + { + throw new InvalidOperationException( + "Configure Microsoft Foundry before transcribing voice evidence."); + } + + var transcripts = new List(); + foreach (var recording in audio) + { + var text = await TranscribeAsync(recording, cancellationToken); + if (!string.IsNullOrWhiteSpace(text)) + { + transcripts.Add(text.Trim()); + } + } + + return string.Join(Environment.NewLine, transcripts); + } + + public async Task TranscribeAsync( + DataContent recording, + CancellationToken cancellationToken) + { + if (!IsConfigured) + { + throw new InvalidOperationException( + "Configure Microsoft Foundry before transcribing voice evidence."); + } + + var cached = _transcripts.GetOrCreateValue(recording); + await cached.Gate.WaitAsync(cancellationToken); + try + { + if (cached.Text is not null) + { + return cached.Text; + } + + if (!MediaTypeHeaderValue.TryParse( + recording.MediaType, + out var recordingMediaType)) + { + throw new InvalidOperationException( + "The captured audio has an invalid media type."); + } + + using var request = new HttpRequestMessage( + HttpMethod.Post, + CreateFoundryEndpoint( + $"openai/deployments/{Uri.EscapeDataString(_options.TranscriptionDeployment)}" + + $"/audio/transcriptions?api-version={TranscriptionApiVersion}")); + ApplyAuthentication(request); + using var form = new MultipartFormDataContent(); + using var audioContent = new ByteArrayContent(recording.Data.ToArray()); + audioContent.Headers.ContentType = recordingMediaType; + form.Add( + audioContent, + "file", + recording.Name ?? "claim-voice.webm"); + request.Content = form; + + HttpResponseMessage response; + try + { + response = await s_httpClient.SendAsync(request, cancellationToken); + } + catch (HttpRequestException exception) + { + throw new InvalidOperationException( + "Voice transcription could not reach Microsoft Foundry.", + exception); + } + catch (TaskCanceledException exception) + when (!cancellationToken.IsCancellationRequested) + { + throw new InvalidOperationException( + "Voice transcription timed out.", + exception); + } + + using (response) + { + var responseBody = + await response.Content.ReadAsStringAsync(cancellationToken); + if (!response.IsSuccessStatusCode) + { + throw new InvalidOperationException( + $"Voice transcription failed with HTTP {(int)response.StatusCode}."); + } + + JsonDocument envelope; + try + { + envelope = JsonDocument.Parse(responseBody); + } + catch (JsonException exception) + { + throw new InvalidOperationException( + "Voice transcription returned an invalid response.", + exception); + } + + using (envelope) + { + if (!envelope.RootElement.TryGetProperty("text", out var text)) + { + throw new InvalidOperationException( + "Voice transcription returned an invalid response."); + } + + cached.Text = text.GetString()?.Trim() ?? string.Empty; + return cached.Text; + } + } + } + finally + { + cached.Gate.Release(); + } + } + + private Uri CreateFoundryEndpoint(string relativePath) + { + if (!Uri.TryCreate(_options.Endpoint, UriKind.Absolute, out var endpoint) || + endpoint.AbsolutePath != "/" || + !string.IsNullOrEmpty(endpoint.Query) || + !string.IsNullOrEmpty(endpoint.Fragment)) + { + throw new InvalidOperationException( + "AZURE_OPENAI_ENDPOINT must be an absolute resource URI without a path, query, or fragment."); + } + + return new Uri( + $"{endpoint.GetLeftPart(UriPartial.Authority)}/{relativePath}", + UriKind.Absolute); + } + + private async Task SendChatCompletionAsync( + string systemPrompt, + IReadOnlyList messages, + bool useJsonResponse, + CancellationToken cancellationToken) + { + EnsureConfigured(); + + using var request = new HttpRequestMessage( + HttpMethod.Post, + CreateFoundryEndpoint( + $"openai/deployments/{Uri.EscapeDataString(_options.ChatDeployment)}/chat/completions" + + $"?api-version={ChatApiVersion}")); + ApplyAuthentication(request); + + var foundryMessages = new List + { + new + { + role = "system", + content = systemPrompt, + }, + }; + foreach (var message in messages) + { + var text = GetMessageText(message); + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + foundryMessages.Add(new + { + role = ToFoundryRole(message.Role), + content = text, + }); + } + + var payload = new Dictionary + { + ["model"] = _options.ChatDeployment, + ["max_completion_tokens"] = useJsonResponse ? 800 : 1_200, + ["reasoning_effort"] = "minimal", + ["messages"] = foundryMessages, + }; + if (useJsonResponse) + { + payload["response_format"] = new + { + type = "json_object", + }; + } + request.Content = JsonContent.Create(payload); + + HttpResponseMessage response; + try + { + response = await s_httpClient.SendAsync(request, cancellationToken); + } + catch (HttpRequestException exception) + { + throw new InvalidOperationException( + "The claim assistant could not reach Microsoft Foundry.", + exception); + } + catch (TaskCanceledException exception) + when (!cancellationToken.IsCancellationRequested) + { + throw new InvalidOperationException( + "The claim assistant timed out while waiting for Microsoft Foundry.", + exception); + } + + using (response) + { + var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + if (!response.IsSuccessStatusCode) + { + throw new InvalidOperationException( + $"The claim assistant failed with HTTP {(int)response.StatusCode}."); + } + + using var envelope = JsonDocument.Parse(responseBody); + var choice = envelope.RootElement.GetProperty("choices")[0]; + var content = choice + .GetProperty("message") + .GetProperty("content") + .GetString(); + if (string.IsNullOrWhiteSpace(content)) + { + var finishReason = choice.TryGetProperty( + "finish_reason", + out var finishReasonElement) + ? finishReasonElement.GetString() + : null; + var finishReasonSuffix = string.IsNullOrWhiteSpace(finishReason) + ? "." + : $" with finish reason '{finishReason}'."; + throw new InvalidOperationException( + $"Microsoft Foundry returned an empty claim response{finishReasonSuffix}"); + } + + return content.Trim(); + } + } + + private void EnsureConfigured() + { + if (!IsConfigured) + { + throw new InvalidOperationException( + "Configure AZURE_OPENAI_ENDPOINT and AZURE_OPENAI_API_KEY before using the claim assistant."); + } + } + + private static string GetMessageText(ChatMessage message) + { + var parts = new List(); + if (!string.IsNullOrWhiteSpace(message.Text)) + { + parts.Add(message.Text); + } + + parts.AddRange(message.Contents + .OfType() + .Select(content => + $"[Attached {content.MediaType}: {content.Name ?? "unnamed evidence"}]")); + return string.Join(Environment.NewLine, parts); + } + + private static string ToFoundryRole(ChatRole role) + => role == ChatRole.Assistant ? "assistant" : "user"; + + private void ApplyAuthentication(HttpRequestMessage request) + { + if (request.RequestUri?.Host.EndsWith(".azure.com", StringComparison.OrdinalIgnoreCase) is true) + { + request.Headers.Add("api-key", _options.ApiKey); + } + else + { + request.Headers.Authorization = + new AuthenticationHeaderValue("Bearer", _options.ApiKey); + } + } + + private sealed class CachedTranscript + { + public SemaphoreSlim Gate { get; } = new(1, 1); + + public string? Text { get; set; } + } + + private static string GetResponseOutputText(JsonElement root) + { + foreach (var item in root.GetProperty("output").EnumerateArray()) + { + if (!item.TryGetProperty("type", out var itemType) || + itemType.GetString() != "message" || + !item.TryGetProperty("content", out var content)) + { + continue; + } + + foreach (var part in content.EnumerateArray()) + { + if (part.TryGetProperty("type", out var partType) && + partType.GetString() == "output_text" && + part.TryGetProperty("text", out var text)) + { + return text.GetString() ?? string.Empty; + } + } + } + + throw new InvalidOperationException("Web research returned no text."); + } + + private static List GetResearchSources( + JsonElement root, + ClaimMarketResearch research) + { + var sources = new List(); + foreach (var item in root.GetProperty("output").EnumerateArray()) + { + if (!item.TryGetProperty("content", out var content)) + { + continue; + } + + foreach (var part in content.EnumerateArray()) + { + if (!part.TryGetProperty("annotations", out var annotations)) + { + continue; + } + + foreach (var annotation in annotations.EnumerateArray()) + { + if (annotation.TryGetProperty("type", out var type) && + type.GetString() == "url_citation" && + annotation.TryGetProperty("url", out var url) && + ClaimResearchLink.Normalize(url.GetString()) is { } safeUrl) + { + sources.Add(new() + { + Title = annotation.TryGetProperty("title", out var title) + ? title.GetString() ?? safeUrl + : safeUrl, + Url = safeUrl, + }); + } + } + } + } + + sources.AddRange(research.ReplacementParts + .Where(part => ClaimResearchLink.Normalize(part.SourceUrl) is not null) + .Select(part => new ClaimResearchSource + { + Title = part.SourceTitle, + Url = part.SourceUrl, + })); + + return sources + .DistinctBy(source => source.Url, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + private static string ExtractJsonObject(string value) + { + var firstBrace = value.IndexOf('{'); + var lastBrace = value.LastIndexOf('}'); + if (firstBrace < 0 || lastBrace <= firstBrace) + { + throw new InvalidOperationException("The model did not return a JSON object."); + } + + return value[firstBrace..(lastBrace + 1)]; + } + + private sealed class ClaimEvidenceDecision + { + public bool AnalyzeEvidence { get; set; } + } +} diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimErrorContentHandler.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimErrorContentHandler.cs new file mode 100644 index 000000000000..c473df51f38b --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimErrorContentHandler.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.AspNetCore.Components.AI; +using Microsoft.Extensions.AI; + +namespace ComponentsAIClaimApp.Data; + +internal sealed class ClaimErrorContentHandler(ILogger logger) + : ContentBlockHandler +{ + public const string DefaultMessage = "We couldn't complete the assessment. Please try again."; + + public override BlockMappingResult Handle( + BlockMappingContext context, + HandlerState state) + { + foreach (var content in context.UnhandledContents) + { + if (content is ErrorContent error) + { + context.MarkHandled(content); + logger.LogError( + "AG-UI claim assessment failed with code {ErrorCode}: {ErrorMessage}", + error.ErrorCode, + error.Message); + throw new InvalidOperationException(DefaultMessage); + } + } + + return BlockMappingResult.Pass(); + } + + internal sealed class HandlerState + { + } +} diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimFoundryOptions.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimFoundryOptions.cs new file mode 100644 index 000000000000..ebd30052cf8d --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimFoundryOptions.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ComponentsAIClaimApp.Data; + +internal sealed class ClaimFoundryOptions +{ + public string? Endpoint { get; set; } + + public string? ApiKey { get; set; } + + public string ChatDeployment { get; set; } = "gpt-5-mini"; + + public string TranscriptionDeployment { get; set; } = "gpt-4o-mini-transcribe"; +} diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimLimits.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimLimits.cs new file mode 100644 index 000000000000..e2968a0507d1 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimLimits.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ComponentsAIClaimApp.Data; + +internal static class ClaimLimits +{ + internal const int MaximumPhotoCount = 6; + internal const long MaximumPhotoBytes = 8 * 1024 * 1024; + internal const long MaximumEvidenceBytes = 24 * 1024 * 1024; + internal const long MaximumSerializedRequestBytes = 40 * 1024 * 1024; +} diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimPhotoPreview.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimPhotoPreview.cs new file mode 100644 index 000000000000..8ba42d4bf62f --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimPhotoPreview.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; + +namespace ComponentsAIClaimApp.Data; + +/// +/// Represents a vehicle photo selected for the current claim. +/// +public sealed record ClaimPhotoPreview(DataContent Content) +{ + /// + /// Gets the display name of the photo. + /// + public string Name => Content.Name ?? "Vehicle photo"; +} diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimPromptSuggestion.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimPromptSuggestion.cs new file mode 100644 index 000000000000..b5c47c7f544d --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimPromptSuggestion.cs @@ -0,0 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ComponentsAIClaimApp.Data; + +public sealed record ClaimPromptSuggestion(string Label, string Prompt); diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimResearchLink.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimResearchLink.cs new file mode 100644 index 000000000000..613e0d16c652 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimResearchLink.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ComponentsAIClaimApp.Data; + +internal static class ClaimResearchLink +{ + internal static string? Normalize(string? value) + { + if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || + uri.Scheme is not ("http" or "https") || + string.IsNullOrEmpty(uri.Host)) + { + return null; + } + + return uri.AbsoluteUri; + } + + internal static void Sanitize(ClaimDamageAnalysis analysis) + { + foreach (var part in analysis.ReplacementParts) + { + part.SourceUrl = Normalize(part.SourceUrl) ?? string.Empty; + } + + analysis.ResearchSources = analysis.ResearchSources + .Select(source => new ClaimResearchSource + { + Title = source.Title, + Url = Normalize(source.Url) ?? string.Empty, + }) + .Where(source => source.Url.Length > 0) + .ToList(); + } +} diff --git a/src/Components/AI/samples/ClaimApp/Data/ClaimState.cs b/src/Components/AI/samples/ClaimApp/Data/ClaimState.cs new file mode 100644 index 000000000000..5bae7c3f1af0 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/ClaimState.cs @@ -0,0 +1,321 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace ComponentsAIClaimApp.Data; + +/// +/// Contains the claim state synchronized between the AG-UI agent and the sample UI. +/// +public sealed class ClaimState +{ + /// + /// Gets or sets the claim number. + /// + public string ClaimNumber { get; set; } = "POC-1042"; + + /// + /// Gets or sets the current processing status. + /// + public string Status { get; set; } = "Ready"; + + /// + /// Gets or sets the accident description. + /// + public string AccidentSummary { get; set; } = "Describe the accident to start the assessment."; + + /// + /// Gets or sets the generated assessment summary. + /// + public string AssessmentSummary { get; set; } = "No assessment yet."; + + /// + /// Gets or sets the summary of image and voice evidence submitted with the claim. + /// + public string EvidenceSummary { get; set; } = "No evidence attached."; + + /// + /// Gets or sets the vehicle areas likely affected by the accident. + /// + public List AffectedAreas { get; set; } = []; + + /// + /// Gets or sets the assessment confidence percentage. + /// + public int Confidence { get; set; } + + /// + /// Gets or sets the user's assessment decision. + /// + public string Decision { get; set; } = "Pending"; + + /// + /// Gets or sets the reason supplied when rejecting the assessment. + /// + public string? RejectionReason { get; set; } + + /// + /// Gets or sets the visible damage findings correlated across the submitted photos. + /// + public List DamageFindings { get; set; } = []; + + /// + /// Gets or sets the most useful next photo requested by the damage analyzer. + /// + public string? NextPhotoSuggestion { get; set; } + + /// + /// Gets or sets a value indicating whether a human adjuster should review the findings. + /// + public bool NeedsHumanReview { get; set; } + + /// + /// Gets or sets the transcript produced from submitted voice evidence. + /// + public string? VoiceTranscript { get; set; } + + /// + /// Gets or sets the grounded repair-cost estimate. + /// + public ClaimRepairEstimate? RepairEstimate { get; set; } + + /// + /// Gets or sets likely replacement parts found through grounded market research. + /// + public List ReplacementParts { get; set; } = []; + + /// + /// Gets or sets the public sources used for parts and repair-cost research. + /// + public List ResearchSources { get; set; } = []; + + /// + /// Gets or sets a research limitation that should be shown to the user. + /// + public string? ResearchWarning { get; set; } +} + +/// +/// Describes one visible or reported vehicle damage finding. +/// +public sealed class ClaimDamageFinding +{ + /// + /// Gets or sets the vehicle area identifier. + /// + public string Area { get; set; } = string.Empty; + + /// + /// Gets or sets the observed damage type. + /// + public string DamageType { get; set; } = string.Empty; + + /// + /// Gets or sets the estimated severity. + /// + public string Severity { get; set; } = string.Empty; + + /// + /// Gets or sets the confidence percentage. + /// + public int Confidence { get; set; } + + /// + /// Gets or sets the evidence supporting the finding. + /// + public string Evidence { get; set; } = string.Empty; + + /// + /// Gets or sets the photo names supporting the finding. + /// + public List PhotoNames { get; set; } = []; +} + +/// +/// Describes a market-grounded repair estimate rather than a final settlement value. +/// +public sealed class ClaimRepairEstimate +{ + /// + /// Gets or sets the low end of the estimated repair range. + /// + public decimal Low { get; set; } + + /// + /// Gets or sets the high end of the estimated repair range. + /// + public decimal High { get; set; } + + /// + /// Gets or sets the ISO-style currency code. + /// + public string Currency { get; set; } = "USD"; + + /// + /// Gets or sets a concise explanation of the estimate basis. + /// + public string Basis { get; set; } = string.Empty; +} + +/// +/// Describes one likely replacement part and its current market range. +/// +public sealed class ClaimReplacementPart +{ + /// + /// Gets or sets the part name. + /// + public string Name { get; set; } = string.Empty; + + /// + /// Gets or sets the low observed part price. + /// + public decimal PriceLow { get; set; } + + /// + /// Gets or sets the high observed part price. + /// + public decimal PriceHigh { get; set; } + + /// + /// Gets or sets the price currency. + /// + public string Currency { get; set; } = "USD"; + + /// + /// Gets or sets fitment or verification guidance. + /// + public string Fitment { get; set; } = string.Empty; + + /// + /// Gets or sets the source title. + /// + public string SourceTitle { get; set; } = string.Empty; + + /// + /// Gets or sets the source URL. + /// + public string SourceUrl { get; set; } = string.Empty; +} + +/// +/// Identifies a public source used for grounded claim research. +/// +public sealed class ClaimResearchSource +{ + /// + /// Gets or sets the source title. + /// + public string Title { get; set; } = string.Empty; + + /// + /// Gets or sets the source URL. + /// + public string Url { get; set; } = string.Empty; +} + +public sealed class ClaimDamageAnalysis +{ + public string Summary { get; set; } = string.Empty; + + public int Confidence { get; set; } + + public List AffectedAreas { get; set; } = []; + + public List Findings { get; set; } = []; + + public string? NextPhotoSuggestion { get; set; } + + public bool NeedsHumanReview { get; set; } + + public string? VoiceTranscript { get; set; } + + public ClaimRepairEstimate? RepairEstimate { get; set; } + + public List ReplacementParts { get; set; } = []; + + public List ResearchSources { get; set; } = []; + + public string? ResearchWarning { get; set; } +} + +internal sealed class ClaimMarketResearch +{ + public ClaimRepairEstimate? RepairEstimate { get; set; } + + public List ReplacementParts { get; set; } = []; + + public string? Warning { get; set; } +} + +/// +/// Represents the result of identifying affected vehicle areas. +/// +public sealed class VehicleAreaAssessment +{ + /// + /// Gets or sets the affected vehicle area identifiers. + /// + public List Areas { get; set; } = []; + + /// + /// Gets or sets the estimated damage severity. + /// + public string Severity { get; set; } = "Moderate"; +} + +public enum ClaimResponsePurpose +{ + Conversation, + MoreEvidence, + AssessmentReady, + Decision, +} + +internal static class ClaimStateJson +{ + internal static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web); + + public static ClaimState Clone(ClaimState state) + => JsonSerializer.Deserialize(JsonSerializer.Serialize(state, Options), Options) + ?? new ClaimState(); + + public static ClaimState ApplyDelta(ClaimState current, JsonElement delta) + { + var root = JsonSerializer.SerializeToNode(current, Options)?.AsObject() + ?? new JsonObject(); + + foreach (var operation in delta.EnumerateArray()) + { + if (!operation.TryGetProperty("op", out var opElement) || + !operation.TryGetProperty("path", out var pathElement)) + { + continue; + } + + var path = pathElement.GetString(); + if (string.IsNullOrEmpty(path) || path[0] != '/' || path.IndexOf('/', 1) >= 0) + { + continue; + } + + var propertyName = path[1..].Replace("~1", "/", StringComparison.Ordinal) + .Replace("~0", "~", StringComparison.Ordinal); + var operationName = opElement.GetString(); + + if (operationName == "remove") + { + root.Remove(propertyName); + } + else if (operationName is "add" or "replace" && + operation.TryGetProperty("value", out var value)) + { + root[propertyName] = JsonNode.Parse(value.GetRawText()); + } + } + + return root.Deserialize(Options) ?? Clone(current); + } +} diff --git a/src/Components/AI/samples/ClaimApp/Data/IClaimAssistantBackend.cs b/src/Components/AI/samples/ClaimApp/Data/IClaimAssistantBackend.cs new file mode 100644 index 000000000000..33582566d2de --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Data/IClaimAssistantBackend.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; + +namespace ComponentsAIClaimApp.Data; + +public interface IClaimAssistantBackend +{ + string ModelName { get; } + + Task ShouldAnalyzeEvidenceAsync( + IReadOnlyList messages, + ClaimState state, + int currentMediaCount, + CancellationToken cancellationToken); + + Task GenerateResponseAsync( + IReadOnlyList messages, + ClaimState state, + ClaimResponsePurpose purpose, + CancellationToken cancellationToken); + + Task AnalyzeAsync( + string description, + IReadOnlyList images, + IReadOnlyList audio, + CancellationToken cancellationToken); + + Task TranscribeAsync( + DataContent recording, + CancellationToken cancellationToken); +} diff --git a/src/Components/AI/samples/ClaimApp/Program.cs b/src/Components/AI/samples/ClaimApp/Program.cs new file mode 100644 index 000000000000..c672c1b45791 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Program.cs @@ -0,0 +1,125 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Http; +using AGUI.Abstractions; +using AGUI.Client; +using AGUI.Formatting; +using AGUI.Server; +using ComponentsAIClaimApp.Components; +using ComponentsAIClaimApp.Data; +using Microsoft.AspNetCore.Components; +using Microsoft.AspNetCore.Http.Metadata; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; + +using JsonOptions = Microsoft.AspNetCore.Http.Json.JsonOptions; + +const string ClaimAgentEndpoint = "claim-agent"; +const string ClaimAgentHttpClient = "claim-agent"; + +var builder = WebApplication.CreateBuilder(args); + +var foundryOptions = new ClaimFoundryOptions(); +builder.Configuration.GetSection("AzureOpenAI").Bind(foundryOptions); +foundryOptions.Endpoint ??= builder.Configuration["AZURE_OPENAI_ENDPOINT"]; +foundryOptions.ApiKey ??= builder.Configuration["AZURE_OPENAI_API_KEY"]; +foundryOptions.ChatDeployment = + builder.Configuration["AZURE_OPENAI_CHAT_DEPLOYMENT"] ?? foundryOptions.ChatDeployment; +foundryOptions.TranscriptionDeployment = + builder.Configuration["AZURE_OPENAI_TRANSCRIPTION_DEPLOYMENT"] ?? foundryOptions.TranscriptionDeployment; + +builder.Services.AddRazorComponents() + .AddInteractiveServerComponents(); +builder.Services.TryAddEnumerable( + ServiceDescriptor.Singleton()); +builder.Services.Configure(options => +{ + options.SerializerOptions.TypeInfoResolverChain.Add( + AIJsonUtilities.DefaultOptions.TypeInfoResolver!); + options.SerializerOptions.TypeInfoResolverChain.Add( + AGUIJsonSerializerContext.Default); + AGUIJsonUtilities.RegisterInterruptContentTypes(options.SerializerOptions); +}); +builder.Services.AddSingleton(foundryOptions); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(services => + services.GetRequiredService()); +builder.Services.AddSingleton(); +builder.Services.Configure( + builder.Configuration.GetSection("ClaimAgent")); +builder.Services.AddHttpClient(ClaimAgentHttpClient, client => +{ + client.Timeout = Timeout.InfiniteTimeSpan; +}); +builder.Services.AddScoped(services => +{ + var httpClient = services.GetRequiredService() + .CreateClient(ClaimAgentHttpClient); + var options = services.GetRequiredService>().Value; + var navigationBaseUri = services.GetRequiredService().BaseUri; + httpClient.BaseAddress = ClaimAgentAddress.Resolve( + options.BaseAddress, + navigationBaseUri); + return new AGUIChatClient( + new AGUIChatClientOptions(httpClient, ClaimAgentEndpoint)); +}); + +var app = builder.Build(); + +if (!app.Environment.IsDevelopment()) +{ + app.UseExceptionHandler("/error", createScopeForErrors: true); + app.UseHsts(); + app.UseHttpsRedirection(); +} + +app.UseAntiforgery(); +app.MapStaticAssets(); +app.MapPost(ClaimAgentEndpoint, ( + RunAgentInput input, + IOptions jsonOptions, + ClaimAgentChatClient chatClient, + CancellationToken cancellationToken) => +{ + var serializerOptions = jsonOptions.Value.SerializerOptions; + var clientTools = input.Tools; + ChatRequestContext context; + try + { + // ClaimAgentChatClient emits client-tool calls. Hiding declarations here prevents + // AGUI.Server mixed invocation from treating later user turns as continuations and + // suppressing those function-call events. + input.Tools = null; + context = input.ToChatRequestContext( + serializerOptions, + new AGUIStreamOptions()); + } + finally + { + input.Tools = clientTools; + } + + context.ChatOptions.RawRepresentationFactory = _ => input; + var updates = chatClient.GetStreamingResponseAsync( + context.Messages, + context.ChatOptions, + cancellationToken); + var events = updates.AsAGUIEventStreamAsync(context, cancellationToken); + + return new ClaimAgentEventStreamResult( + events, + new SseEventStreamFormatter(), + cancellationToken); +}) + .WithMetadata(new ClaimAgentRequestSizeLimitMetadata()); +app.MapRazorComponents() + .AddInteractiveServerRenderMode(); + +app.Run(); + +file sealed class ClaimAgentRequestSizeLimitMetadata : IRequestSizeLimitMetadata +{ + public long? MaxRequestBodySize => ClaimLimits.MaximumSerializedRequestBytes; +} diff --git a/src/Components/AI/samples/ClaimApp/Properties/launchSettings.json b/src/Components/AI/samples/ClaimApp/Properties/launchSettings.json new file mode 100644 index 000000000000..35ed22f3278c --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/Properties/launchSettings.json @@ -0,0 +1,14 @@ +{ + "profiles": { + "ClaimApp": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://127.0.0.1:5099", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + }, + "$schema": "http://json.schemastore.org/launchsettings.json" +} diff --git a/src/Components/AI/samples/ClaimApp/README.md b/src/Components/AI/samples/ClaimApp/README.md new file mode 100644 index 000000000000..2cbd3591ada5 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/README.md @@ -0,0 +1,15 @@ +# Components.AI claim application + +This Blazor sample demonstrates multimodal claim intake over AG-UI with Microsoft Foundry chat, vision, transcription, and web search. + +Configure Azure OpenAI and run from the repository root: + +```bash +export AZURE_OPENAI_ENDPOINT="https://.openai.azure.com" +export AZURE_OPENAI_API_KEY="..." + +source activate.sh +dotnet run --project src/Components/AI/samples/ClaimApp/ClaimApp.csproj --no-restore +``` + +The sample uses `gpt-5-mini` for chat and vision and `gpt-4o-mini-transcribe` for recorded audio. Override them with `AZURE_OPENAI_CHAT_DEPLOYMENT` and `AZURE_OPENAI_TRANSCRIPTION_DEPLOYMENT`. diff --git a/src/Components/AI/samples/ClaimApp/wwwroot/app.css b/src/Components/AI/samples/ClaimApp/wwwroot/app.css new file mode 100644 index 000000000000..2510e1036420 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/wwwroot/app.css @@ -0,0 +1,40 @@ +html, +body { + min-height: 100%; + margin: 0; + font-family: "Aptos", "Segoe UI Variable", "Segoe UI", sans-serif; + background: oklch(0.97 0.012 255); +} + +button, +input, +textarea { + font: inherit; +} + +a { + color: #1767d1; +} + +h1:focus { + outline: none; +} + +button:focus-visible, +input:focus-visible, +textarea:focus-visible, +a:focus-visible { + outline: 0.2rem solid oklch(0.67 0.18 255); + outline-offset: 0.15rem; +} + +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + scroll-behavior: auto !important; + transition-duration: 0.01ms !important; + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + } +} diff --git a/src/Components/AI/samples/ClaimApp/wwwroot/claim-app.js b/src/Components/AI/samples/ClaimApp/wwwroot/claim-app.js new file mode 100644 index 000000000000..5e93faa21e08 --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/wwwroot/claim-app.js @@ -0,0 +1,20 @@ +(() => { + const themeKey = "components-ai-claim-theme"; + const supportedThemes = new Set(["light", "dark", "contrast"]); + + function getTheme() { + const theme = localStorage.getItem(themeKey); + return supportedThemes.has(theme) ? theme : "light"; + } + + function setTheme(theme) { + if (supportedThemes.has(theme)) { + localStorage.setItem(themeKey, theme); + } + } + + globalThis.claimApp = { + getTheme, + setTheme, + }; +})(); diff --git a/src/Components/AI/samples/ClaimApp/wwwroot/favicon.svg b/src/Components/AI/samples/ClaimApp/wwwroot/favicon.svg new file mode 100644 index 000000000000..8fc8fb60076b --- /dev/null +++ b/src/Components/AI/samples/ClaimApp/wwwroot/favicon.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/Components/AI/src/Blocks/DataContentBlock.cs b/src/Components/AI/src/Blocks/DataContentBlock.cs new file mode 100644 index 000000000000..86666bfe36da --- /dev/null +++ b/src/Components/AI/src/Blocks/DataContentBlock.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.AI; + +namespace Microsoft.AspNetCore.Components.AI; + +/// +/// Represents binary content, such as an image, audio clip, video, or file, in a conversation. +/// +public class DataContentBlock : ContentBlock +{ + /// + /// Gets or sets the binary content represented by this block. + /// + public DataContent Content { get; set; } = default!; +} diff --git a/src/Components/AI/src/Components/AudioCaptureButton.cs b/src/Components/AI/src/Components/AudioCaptureButton.cs new file mode 100644 index 000000000000..88a6e57c10e4 --- /dev/null +++ b/src/Components/AI/src/Components/AudioCaptureButton.cs @@ -0,0 +1,628 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.AspNetCore.Components.Rendering; +using Microsoft.Extensions.AI; +using Microsoft.JSInterop; +using System.Linq; + +namespace Microsoft.AspNetCore.Components.AI; + +/// +/// Records audio in the browser and transcribes or attaches it to the nearest +/// . +/// +public sealed class AudioCaptureButton : ComponentBase, IAsyncDisposable +{ + private const string ModulePath = + "./_content/Microsoft.AspNetCore.Components.AI/ai-chat.js"; + + private readonly SpeechCallbacks _speechCallbacks; + private DotNetObjectReference? _speechCallbackReference; + private IJSObjectReference? _module; + private IJSObjectReference? _recorder; + private IJSObjectReference? _speechRecognizer; + private MessageInputContext? _subscribedContext; + private IDisposable? _changeSubscription; + private CancellationTokenSource? _operationCts; + private string _dictationPrefix = string.Empty; + private string _committedTranscript = string.Empty; + private bool _isRecording; + private bool _isTranscribing; + private bool _isDictating; + private bool _isSupported = true; + private bool _isDisposed; + + /// + /// Initializes a new instance of . + /// + public AudioCaptureButton() + { + _speechCallbacks = new SpeechCallbacks(this); + } + + /// + /// Gets or sets the nearest message input. + /// + [CascadingParameter] + public MessageInputContext Context { get; set; } = default!; + + /// + /// Gets or sets the JavaScript runtime used to access browser recording APIs. + /// + [Inject] + internal IJSRuntime JSRuntime { get; set; } = default!; + + /// + /// Gets or sets the maximum recording size in bytes. + /// + [Parameter] + public long MaximumBytes { get; set; } = 10 * 1024 * 1024; + + /// + /// Gets or sets whether a new recording replaces existing audio attachments. + /// + [Parameter] + public bool ReplaceExistingAudio { get; set; } = true; + + /// + /// Gets or sets whether captured audio is added to the outgoing message. + /// + [Parameter] + public bool AttachRecording { get; set; } = true; + + /// + /// Gets or sets whether browser speech recognition updates the composer while recording. + /// + [Parameter] + public bool ShowInterimTranscript { get; set; } + + /// + /// Gets or sets the browser speech-recognition language. The browser default is used when omitted. + /// + [Parameter] + public string? SpeechRecognitionLanguage { get; set; } + + /// + /// Gets or sets the accessible label shown before recording starts. + /// + [Parameter] + public string StartLabel { get; set; } = "Record audio"; + + /// + /// Gets or sets the accessible label shown while recording. + /// + [Parameter] + public string StopLabel { get; set; } = "Stop recording"; + + /// + /// Gets or sets custom button content based on whether recording is active. + /// + [Parameter] + public RenderFragment? ChildContent { get; set; } + + /// + /// Gets or sets a callback invoked when audio has been captured. + /// + [Parameter] + public EventCallback OnRecorded { get; set; } + + /// + /// Gets or sets an optional callback that transcribes captured audio into composer text. + /// + [Parameter] + public Func>? Transcribe { get; set; } + + /// + /// Gets or sets a callback invoked after captured audio has been transcribed. + /// + [Parameter] + public EventCallback OnTranscribed { get; set; } + + /// + /// Gets or sets additional attributes applied to the recording button. + /// + [Parameter(CaptureUnmatchedValues = true)] + public Dictionary? AdditionalAttributes { get; set; } + + /// + protected override void OnParametersSet() + { + if (ReferenceEquals(_subscribedContext, Context)) + { + return; + } + + _changeSubscription?.Dispose(); + _subscribedContext = Context; + _changeSubscription = Context.RegisterOnChanged( + () => _ = InvokeAsync(StateHasChanged)); + } + + /// + protected override void BuildRenderTree(RenderTreeBuilder builder) + { + var isActive = _isRecording || _isTranscribing; + var disabled = !_isSupported || + (!isActive && (Context.IsConversationBusy || Context.IsComposing)); + var label = isActive ? StopLabel : StartLabel; + + builder.OpenElement(0, "button"); + builder.AddMultipleAttributes(1, AdditionalAttributes); + builder.AddAttribute(2, "type", "button"); + builder.AddAttribute(3, "class", CssClass()); + builder.AddAttribute(4, "disabled", disabled); + builder.AddAttribute(5, "aria-label", label); + builder.AddAttribute(6, "aria-pressed", isActive ? "true" : "false"); + builder.AddAttribute( + 7, + "onclick", + EventCallback.Factory.Create(this, ToggleRecordingAsync)); + + if (ChildContent is not null) + { + builder.AddContent(8, ChildContent(isActive)); + } + else + { + builder.AddContent(9, label); + } + + builder.CloseElement(); + } + + /// + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (!firstRender) + { + return; + } + + try + { + _module = await JSRuntime.InvokeAsync("import", ModulePath); + _isSupported = await _module.InvokeAsync("isAudioCaptureSupported"); + if (!_isSupported) + { + Context.SetErrorMessage("Audio recording is not supported by this browser."); + } + await InvokeAsync(StateHasChanged); + } + catch (JSException) + { + _isSupported = false; + Context.SetErrorMessage("Audio recording could not be initialized."); + await InvokeAsync(StateHasChanged); + } + } + + private Task ToggleRecordingAsync() + { + if (_isTranscribing) + { + CancelTranscription(); + return Task.CompletedTask; + } + + return _isRecording ? StopRecordingAsync() : StartRecordingAsync(); + } + + private async Task StartRecordingAsync() + { + _operationCts?.Cancel(); + var operationCts = new CancellationTokenSource(); + _operationCts = operationCts; + Context.SetErrorMessage(null); + + try + { + _module ??= await JSRuntime.InvokeAsync("import", ModulePath); + _recorder ??= await _module.InvokeAsync( + "createAudioRecorder", + MaximumBytes); + await _recorder.InvokeVoidAsync("start"); + if (!ReferenceEquals(_operationCts, operationCts) || + operationCts.IsCancellationRequested) + { + operationCts.Dispose(); + return; + } + + _isRecording = true; + Context.SetComposing(true); + Context.SetStatusMessage("Recording audio."); + await StartInterimTranscriptionAsync(); + } + catch (JSException) + { + if (ReferenceEquals(_operationCts, operationCts)) + { + _operationCts = null; + _isRecording = false; + Context.SetComposing(false); + Context.SetErrorMessage( + "Microphone access was not available. Check browser permissions."); + } + + operationCts.Dispose(); + } + } + + private async Task StopRecordingAsync() + { + var operationCts = _operationCts + ?? throw new InvalidOperationException( + "Audio recording does not have an active operation."); + var cancellationToken = operationCts.Token; + _isRecording = false; + _isTranscribing = true; + await InvokeAsync(StateHasChanged); + + try + { + var hadInterimTranscript = _isDictating; + await StopInterimTranscriptionAsync(); + var recording = await _recorder!.InvokeAsync("stop"); + cancellationToken.ThrowIfCancellationRequested(); + + if (recording.TooLarge || recording.Size > MaximumBytes) + { + if (recording.StreamReference is not null) + { + await recording.StreamReference.DisposeAsync(); + } + Context.SetErrorMessage( + $"Audio recordings must be {FormatBytes(MaximumBytes)} or smaller."); + return; + } + + if (recording.StreamReference is null || recording.Size == 0) + { + Context.SetErrorMessage( + "The browser did not capture audio. Record for at least one second and check the microphone input level."); + return; + } + + var mediaType = string.IsNullOrWhiteSpace(recording.MimeType) + ? "audio/webm" + : recording.MimeType; + await using var streamReference = recording.StreamReference; + await using var stream = await streamReference.OpenReadStreamAsync( + MaximumBytes, + cancellationToken); + var content = await DataContent.LoadFromAsync( + stream, + mediaType, + cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + content.Name = + $"recording-{DateTimeOffset.UtcNow:yyyyMMdd-HHmmss}.{GetExtension(mediaType)}"; + + if (AttachRecording && ReplaceExistingAudio) + { + foreach (var attachment in Context.Attachments + .Where(attachment => attachment.HasTopLevelMediaType("audio")) + .ToArray()) + { + await Context.RemoveAttachmentAsync(attachment); + } + } + + if (AttachRecording) + { + await Context.AddAttachmentAsync(content); + } + cancellationToken.ThrowIfCancellationRequested(); + await OnRecorded.InvokeAsync(content); + if (!ReferenceEquals(_operationCts, operationCts)) + { + return; + } + + cancellationToken.ThrowIfCancellationRequested(); + if (Transcribe is not null) + { + Context.SetStatusMessage("Transcribing audio."); + var transcript = await Transcribe(content, cancellationToken); + if (!ReferenceEquals(_operationCts, operationCts)) + { + return; + } + + cancellationToken.ThrowIfCancellationRequested(); + if (!string.IsNullOrWhiteSpace(transcript)) + { + Context.Text = hadInterimTranscript + ? AppendText(_dictationPrefix, transcript.Trim()) + : AppendText(Context.Text, transcript.Trim()); + await OnTranscribed.InvokeAsync(transcript.Trim()); + Context.SetStatusMessage("Voice transcription ready."); + } + else + { + Context.SetErrorMessage("No speech was recognized in the recording."); + } + } + else if (AttachRecording) + { + Context.SetStatusMessage("Audio recording attached."); + } + else + { + Context.SetErrorMessage( + "A transcription callback is required when recordings are not attached."); + } + await Context.FocusAsync(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (JSException) + { + if (ReferenceEquals(_operationCts, operationCts)) + { + Context.SetErrorMessage("The audio recording could not be completed."); + } + } + catch (IOException) + { + if (ReferenceEquals(_operationCts, operationCts)) + { + Context.SetErrorMessage("The captured audio could not be read."); + } + } + catch (InvalidOperationException exception) + { + if (ReferenceEquals(_operationCts, operationCts)) + { + Context.SetErrorMessage(exception.Message); + } + } + finally + { + if (ReferenceEquals(_operationCts, operationCts)) + { + _operationCts = null; + _isRecording = false; + _isTranscribing = false; + if (!_isDisposed) + { + Context.SetComposing(false); + } + } + + operationCts.Dispose(); + } + } + + private void CancelTranscription() + { + _operationCts?.Cancel(); + _isTranscribing = false; + Context.SetComposing(false); + Context.SetStatusMessage("Audio transcription canceled."); + } + + private async Task StartInterimTranscriptionAsync() + { + if (!ShowInterimTranscript || _module is null) + { + return; + } + + try + { + if (!await _module.InvokeAsync( + "isLiveSpeechRecognitionSupported")) + { + return; + } + + _speechCallbackReference ??= DotNetObjectReference.Create(_speechCallbacks); + _speechRecognizer ??= await _module.InvokeAsync( + "createLiveSpeechRecognizer", + _speechCallbackReference, + SpeechRecognitionLanguage); + _dictationPrefix = Context.Text.Trim(); + _committedTranscript = string.Empty; + _isDictating = true; + await _speechRecognizer.InvokeVoidAsync("start"); + Context.SetStatusMessage("Recording and transcribing."); + } + catch (JSException) + { + _isDictating = false; + Context.SetStatusMessage("Recording audio. Live transcription is unavailable."); + } + } + + private async Task StopInterimTranscriptionAsync() + { + if (!_isDictating || _speechRecognizer is null) + { + return; + } + + _isDictating = false; + try + { + await _speechRecognizer.InvokeVoidAsync("stop"); + } + catch (JSException) + { + Context.SetStatusMessage("Transcribing the completed recording."); + } + } + + private Task HandleSpeechResultAsync( + string finalTranscript, + string interimTranscript) + { + return InvokeAsync(() => + { + if (!_isRecording || !_isDictating) + { + return; + } + + if (!string.IsNullOrWhiteSpace(finalTranscript)) + { + _committedTranscript = + AppendText(_committedTranscript, finalTranscript); + } + + Context.Text = AppendText( + AppendText(_dictationPrefix, _committedTranscript), + interimTranscript); + Context.SetStatusMessage("Recording and transcribing."); + }); + } + + private Task HandleSpeechErrorAsync() + { + return InvokeAsync(() => + { + _isDictating = false; + if (_isRecording) + { + Context.SetStatusMessage( + "Recording audio. Live transcription is unavailable."); + } + }); + } + + private string CssClass() + { + var css = _isRecording + ? "sc-ai-input__audio sc-ai-input__audio--recording" + : "sc-ai-input__audio"; + if (AdditionalAttributes?.TryGetValue("class", out var value) == true && + value is string additionalClass) + { + css = $"{css} {additionalClass}"; + } + + return css; + } + + private static string GetExtension(string mediaType) + { + if (mediaType.Contains("ogg", StringComparison.OrdinalIgnoreCase)) + { + return "ogg"; + } + + if (mediaType.Contains("mp4", StringComparison.OrdinalIgnoreCase)) + { + return "m4a"; + } + + if (mediaType.Contains("wav", StringComparison.OrdinalIgnoreCase)) + { + return "wav"; + } + + return "webm"; + } + + private static string FormatBytes(long bytes) + { + const long megabyte = 1024 * 1024; + return bytes >= megabyte && bytes % megabyte == 0 + ? $"{bytes / megabyte} MB" + : $"{bytes} bytes"; + } + + private static string AppendText(string existingText, string transcript) + { + return string.IsNullOrWhiteSpace(existingText) + ? transcript + : $"{existingText.TrimEnd()} {transcript}"; + } + + /// + /// Stops recording and releases browser resources. + /// + public async ValueTask DisposeAsync() + { + if (_isDisposed) + { + return; + } + + _isDisposed = true; + _changeSubscription?.Dispose(); + _operationCts?.Cancel(); + if (!_isTranscribing) + { + _operationCts?.Dispose(); + _operationCts = null; + } + Context.SetComposing(false); + + if (_recorder is not null) + { + try + { + await _recorder.InvokeVoidAsync("dispose"); + await _recorder.DisposeAsync(); + } + catch (JSDisconnectedException) + { + } + } + + if (_speechRecognizer is not null) + { + try + { + await _speechRecognizer.InvokeVoidAsync("dispose"); + await _speechRecognizer.DisposeAsync(); + } + catch (JSDisconnectedException) + { + } + } + + if (_module is not null) + { + try + { + await _module.DisposeAsync(); + } + catch (JSDisconnectedException) + { + } + } + + _speechCallbackReference?.Dispose(); + GC.SuppressFinalize(this); + } + + private sealed class SpeechCallbacks(AudioCaptureButton owner) + { + [JSInvokable] + public Task OnResultAsync(string finalTranscript, string interimTranscript) + { + return owner.HandleSpeechResultAsync(finalTranscript, interimTranscript); + } + + [JSInvokable] + public Task OnErrorAsync(string _, bool __) + { + return owner.HandleSpeechErrorAsync(); + } + } + + private sealed class AudioCaptureResult + { + public IJSStreamReference? StreamReference { get; set; } + + public string MimeType { get; set; } = string.Empty; + + public long Size { get; set; } + + public bool TooLarge { get; set; } + } +} diff --git a/src/Components/AI/src/Components/ConversationTurnRenderer.cs b/src/Components/AI/src/Components/ConversationTurnRenderer.cs index a3666f9aef00..0f532d4cb4c3 100644 --- a/src/Components/AI/src/Components/ConversationTurnRenderer.cs +++ b/src/Components/AI/src/Components/ConversationTurnRenderer.cs @@ -23,6 +23,7 @@ internal ConversationTurnRenderer( _turn = turn; _listContext = listContext; _requestRender = requestRender; + _turn.ResponseBlocksTruncated += OnResponseBlocksTruncated; _blockAddedSub = agentContext.RegisterOnBlockAdded((t, block) => { @@ -48,12 +49,28 @@ private void OnBlockAdded(ContentBlock block) _requestRender(); } + private void OnResponseBlocksTruncated(int count) + { + var retainedContainerCount = _turn.RequestBlocks.Count + count; + for (var index = _blockContainers.Count - 1; + index >= retainedContainerCount; + index--) + { + _blockContainers[index].Dispose(); + _blockContainers.RemoveAt(index); + } + + _requestRender(); + } + internal void RenderTo(RenderTreeBuilder builder, int baseSeq) { // Determine role from request blocks (user turn) or response blocks var role = _turn.RequestBlocks.Count > 0 ? "user" : "assistant"; builder.OpenElement(baseSeq, "div"); builder.AddAttribute(baseSeq + 1, "class", $"sc-ai-turn sc-ai-turn--{role}"); + builder.AddAttribute(baseSeq + 2, "role", "group"); + builder.AddAttribute(baseSeq + 3, "aria-label", "Conversation turn"); var seq = baseSeq + 10; foreach (var container in _blockContainers) { @@ -65,6 +82,7 @@ internal void RenderTo(RenderTreeBuilder builder, int baseSeq) public void Dispose() { + _turn.ResponseBlocksTruncated -= OnResponseBlocksTruncated; _blockAddedSub?.Dispose(); foreach (var container in _blockContainers) { diff --git a/src/Components/AI/src/Components/LiveSpeechButton.cs b/src/Components/AI/src/Components/LiveSpeechButton.cs new file mode 100644 index 000000000000..887402bc5a96 --- /dev/null +++ b/src/Components/AI/src/Components/LiveSpeechButton.cs @@ -0,0 +1,488 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.AspNetCore.Components.Rendering; +using Microsoft.JSInterop; + +namespace Microsoft.AspNetCore.Components.AI; + +/// +/// Captures continuous browser speech recognition, shows interim text in the nearest +/// , and optionally submits each completed utterance. +/// +public sealed class LiveSpeechButton : ComponentBase, IAsyncDisposable +{ + private const string ModulePath = + "./_content/Microsoft.AspNetCore.Components.AI/ai-chat.js"; + + private readonly SpeechCallbacks _callbacks; + private DotNetObjectReference? _callbackReference; + private IJSObjectReference? _module; + private IJSObjectReference? _recognizer; + private MessageInputContext? _subscribedContext; + private IDisposable? _changeSubscription; + private string _prefix = string.Empty; + private string _committedTranscript = string.Empty; + private bool _isEnabled; + private bool _isListening; + private bool _isStarting; + private bool _isFinalizing; + private bool _isSupported = true; + private bool _isDisposed; + + /// + /// Initializes a new instance of . + /// + public LiveSpeechButton() + { + _callbacks = new SpeechCallbacks(this); + } + + /// + /// Gets or sets the nearest message input. + /// + [CascadingParameter] + public MessageInputContext Context { get; set; } = default!; + + /// + /// Gets or sets the JavaScript runtime used to access browser speech recognition. + /// + [Inject] + internal IJSRuntime JSRuntime { get; set; } = default!; + + /// + /// Gets or sets whether each finalized utterance is submitted automatically. + /// + [Parameter] + public bool AutoSubmit { get; set; } = true; + + /// + /// Gets or sets the speech-recognition language. The browser default is used when omitted. + /// + [Parameter] + public string? Language { get; set; } + + /// + /// Gets or sets whether interim speech is displayed in the message composer. + /// + [Parameter] + public bool ShowInterimInComposer { get; set; } = true; + + /// + /// Gets or sets the accessible label shown before live speech starts. + /// + [Parameter] + public string StartLabel { get; set; } = "Start live voice"; + + /// + /// Gets or sets the accessible label shown while live speech is enabled. + /// + [Parameter] + public string StopLabel { get; set; } = "Stop live voice"; + + /// + /// Gets or sets custom button content based on whether live speech is enabled. + /// + [Parameter] + public RenderFragment? ChildContent { get; set; } + + /// + /// Gets or sets a callback invoked for each finalized transcript. + /// + [Parameter] + public EventCallback OnTranscript { get; set; } + + /// + /// Gets or sets a callback invoked when the visible interim transcript changes. + /// + [Parameter] + public EventCallback OnInterimTranscript { get; set; } + + /// + /// Gets or sets additional attributes applied to the live-speech button. + /// + [Parameter(CaptureUnmatchedValues = true)] + public Dictionary? AdditionalAttributes { get; set; } + + /// + protected override void OnParametersSet() + { + if (ReferenceEquals(_subscribedContext, Context)) + { + return; + } + + _changeSubscription?.Dispose(); + _subscribedContext = Context; + _changeSubscription = Context.RegisterOnChanged(OnContextChanged); + } + + /// + protected override void BuildRenderTree(RenderTreeBuilder builder) + { + var disabled = !_isSupported || + (!_isEnabled && (Context.IsConversationBusy || Context.IsComposing)); + var label = _isEnabled ? StopLabel : StartLabel; + + builder.OpenElement(0, "button"); + builder.AddMultipleAttributes(1, AdditionalAttributes); + builder.AddAttribute(2, "type", "button"); + builder.AddAttribute(3, "class", CssClass()); + builder.AddAttribute(4, "disabled", disabled); + builder.AddAttribute(5, "aria-label", label); + builder.AddAttribute(6, "aria-pressed", _isEnabled ? "true" : "false"); + builder.AddAttribute( + 7, + "onclick", + EventCallback.Factory.Create(this, ToggleAsync)); + + if (ChildContent is not null) + { + builder.AddContent(8, ChildContent(_isEnabled)); + } + else + { + builder.AddContent(9, label); + } + + builder.CloseElement(); + } + + /// + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (!firstRender) + { + return; + } + + try + { + _module = await JSRuntime.InvokeAsync("import", ModulePath); + _isSupported = await _module.InvokeAsync( + "isLiveSpeechRecognitionSupported"); + if (!_isSupported) + { + Context.SetStatusMessage( + "Live voice is not supported by this browser. Recorded voice notes are still available."); + } + await InvokeAsync(StateHasChanged); + } + catch (JSException) + { + _isSupported = false; + Context.SetErrorMessage("Live voice could not be initialized."); + await InvokeAsync(StateHasChanged); + } + } + + private Task ToggleAsync() + { + return _isEnabled ? StopAsync() : StartAsync(); + } + + private async Task StartAsync() + { + Context.SetErrorMessage(null); + + try + { + _module ??= await JSRuntime.InvokeAsync("import", ModulePath); + _callbackReference ??= DotNetObjectReference.Create(_callbacks); + _recognizer ??= await _module.InvokeAsync( + "createLiveSpeechRecognizer", + _callbackReference, + Language); + _prefix = Context.Text.Trim(); + _committedTranscript = string.Empty; + _isEnabled = true; + await StartListeningAsync(); + } + catch (JSException) + { + _isEnabled = false; + _isListening = false; + Context.SetComposing(false); + Context.SetErrorMessage( + "Microphone speech recognition was not available. Check browser permissions."); + } + } + + private async Task StartListeningAsync() + { + if (!_isEnabled || _isListening || _isStarting) + { + return; + } + + _isStarting = true; + try + { + await _recognizer!.InvokeVoidAsync("start"); + _isListening = true; + Context.SetComposing(true); + Context.SetStatusMessage("Listening for your next instruction."); + } + catch (JSException) + { + _isEnabled = false; + _isListening = false; + Context.SetComposing(false); + Context.SetErrorMessage( + "Microphone speech recognition was not available. Check browser permissions."); + } + finally + { + _isStarting = false; + await InvokeAsync(StateHasChanged); + } + } + + private async Task StopAsync() + { + _isEnabled = false; + _isListening = false; + Context.SetComposing(false); + if (_recognizer is not null) + { + await _recognizer.InvokeVoidAsync("stop"); + } + await OnInterimTranscript.InvokeAsync(string.Empty); + Context.SetStatusMessage("Live voice stopped."); + await InvokeAsync(StateHasChanged); + } + + private Task HandleResultAsync(string finalTranscript, string interimTranscript) + { + return InvokeAsync(async () => + { + if (!_isEnabled || _isFinalizing) + { + return; + } + + if (!string.IsNullOrWhiteSpace(finalTranscript)) + { + _committedTranscript = AppendText(_committedTranscript, finalTranscript); + } + + var recognizedText = + AppendText(_committedTranscript, interimTranscript); + if (ShowInterimInComposer) + { + Context.Text = ComposeText(interimTranscript); + } + await OnInterimTranscript.InvokeAsync(recognizedText); + if (string.IsNullOrWhiteSpace(finalTranscript)) + { + Context.SetStatusMessage("Listening..."); + return; + } + + await OnTranscript.InvokeAsync(finalTranscript.Trim()); + if (!AutoSubmit) + { + Context.SetStatusMessage("Listening for more."); + return; + } + + _isFinalizing = true; + _isListening = false; + try + { + await _recognizer!.InvokeVoidAsync("stop"); + Context.SetComposing(false); + Context.SetStatusMessage("Sending voice instruction."); + if (!ShowInterimInComposer) + { + Context.Text = ComposeText(string.Empty); + } + if (Context.CanSubmit) + { + var submitTask = Context.SubmitAsync(); + await OnInterimTranscript.InvokeAsync(string.Empty); + await submitTask; + } + + _prefix = string.Empty; + _committedTranscript = string.Empty; + } + finally + { + _isFinalizing = false; + } + + if (_isEnabled && CanResumeListening) + { + await StartListeningAsync(); + } + else if (_isEnabled) + { + Context.SetStatusMessage( + "Live voice is on and will resume after the current action."); + await InvokeAsync(StateHasChanged); + } + }); + } + + private void OnContextChanged() + { + _ = InvokeAsync(async () => + { + StateHasChanged(); + if (_isEnabled && + !_isListening && + !_isFinalizing && + CanResumeListening) + { + await StartListeningAsync(); + } + }); + } + + private bool CanResumeListening => + Context.Status is ConversationStatus.Idle or ConversationStatus.Error; + + private Task HandleStartedAsync() + { + return InvokeAsync(() => + { + if (!_isEnabled) + { + return; + } + + _isListening = true; + Context.SetComposing(true); + Context.SetErrorMessage(null); + Context.SetStatusMessage("Listening for your next instruction."); + StateHasChanged(); + }); + } + + private Task HandleErrorAsync(string error, bool isFatal) + { + return InvokeAsync(async () => + { + _isListening = false; + if (!isFatal) + { + Context.SetStatusMessage( + "Live voice was interrupted. Reconnecting automatically."); + StateHasChanged(); + return; + } + + _isEnabled = false; + Context.SetComposing(false); + await OnInterimTranscript.InvokeAsync(string.Empty); + Context.SetErrorMessage(error switch + { + "not-allowed" or "service-not-allowed" => + "Microphone access was denied. Allow microphone access to use live voice.", + "language-not-supported" => + "The selected live voice language is not supported by this browser.", + _ => "Live voice could not continue because speech recognition is not configured correctly.", + }); + StateHasChanged(); + }); + } + + private string ComposeText(string interimTranscript) + { + var text = AppendText(_prefix, _committedTranscript); + return AppendText(text, interimTranscript); + } + + private string CssClass() + { + var css = _isEnabled + ? "sc-ai-input__live-speech sc-ai-input__live-speech--active" + : "sc-ai-input__live-speech"; + if (_isEnabled && !_isListening) + { + css += " sc-ai-input__live-speech--waiting"; + } + if (AdditionalAttributes?.TryGetValue("class", out var value) == true && + value is string additionalClass) + { + css = $"{css} {additionalClass}"; + } + + return css; + } + + private static string AppendText(string existingText, string newText) + { + if (string.IsNullOrWhiteSpace(newText)) + { + return existingText.Trim(); + } + + return string.IsNullOrWhiteSpace(existingText) + ? newText.Trim() + : $"{existingText.TrimEnd()} {newText.Trim()}"; + } + + /// + /// Stops recognition and releases browser resources. + /// + public async ValueTask DisposeAsync() + { + if (_isDisposed) + { + return; + } + + _isDisposed = true; + _changeSubscription?.Dispose(); + Context.SetComposing(false); + + if (_recognizer is not null) + { + try + { + await _recognizer.InvokeVoidAsync("dispose"); + await _recognizer.DisposeAsync(); + } + catch (JSDisconnectedException) + { + } + } + + if (_module is not null) + { + try + { + await _module.DisposeAsync(); + } + catch (JSDisconnectedException) + { + } + } + + _callbackReference?.Dispose(); + GC.SuppressFinalize(this); + } + + private sealed class SpeechCallbacks(LiveSpeechButton owner) + { + [JSInvokable] + public Task OnStartedAsync() + { + return owner.HandleStartedAsync(); + } + + [JSInvokable] + public Task OnResultAsync(string finalTranscript, string interimTranscript) + { + return owner.HandleResultAsync(finalTranscript, interimTranscript); + } + + [JSInvokable] + public Task OnErrorAsync(string error, bool isFatal) + { + return owner.HandleErrorAsync(error, isFatal); + } + } +} diff --git a/src/Components/AI/src/Components/MediaContent.cs b/src/Components/AI/src/Components/MediaContent.cs new file mode 100644 index 000000000000..64b22e523685 --- /dev/null +++ b/src/Components/AI/src/Components/MediaContent.cs @@ -0,0 +1,124 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.AspNetCore.Components.Rendering; +using Microsoft.AspNetCore.Components.Media; +using Microsoft.Extensions.AI; +using System.Runtime.CompilerServices; + +namespace Microsoft.AspNetCore.Components.AI; + +/// +/// Renders binary AI content using the matching component from +/// . +/// +public sealed class MediaContent : ComponentBase +{ + private static readonly ConditionalWeakTable CacheKeys = new(); + private DataContent? _currentContent; + private MediaSource? _source; + + /// + /// Gets or sets the content to render. + /// + [Parameter, EditorRequired] + public DataContent Content { get; set; } = default!; + + /// + /// Gets or sets the accessible alternative text for image content. + /// + [Parameter] + public string? AlternativeText { get; set; } + + /// + /// Gets or sets additional attributes applied to the rendered media element. + /// + [Parameter(CaptureUnmatchedValues = true)] + public Dictionary? AdditionalAttributes { get; set; } + + /// + protected override void OnParametersSet() + { + ArgumentNullException.ThrowIfNull(Content); + + if (!ReferenceEquals(_currentContent, Content)) + { + _currentContent = Content; + _source = new MediaSource( + Content.Data.ToArray(), + Content.MediaType, + CacheKeys.GetValue( + Content, + static _ => new MediaCacheKey($"ai-media-{Guid.NewGuid():N}")).Value); + } + } + + /// + protected override void BuildRenderTree(RenderTreeBuilder builder) + { + var source = _source + ?? throw new InvalidOperationException($"{nameof(MediaContent)}.{nameof(Content)} is required."); + var attributes = CreateMediaAttributes(); + + if (Content.HasTopLevelMediaType("image")) + { + attributes["alt"] = AlternativeText ?? GetDisplayName("Attached image"); + builder.OpenComponent(0); + builder.AddComponentParameter(1, nameof(Media.Image.Source), source); + builder.AddComponentParameter(2, nameof(Media.Image.AdditionalAttributes), attributes); + builder.CloseComponent(); + } + else if (Content.HasTopLevelMediaType("audio")) + { + attributes.TryAdd("controls", true); + attributes.TryAdd("preload", "metadata"); + attributes.TryAdd("aria-label", GetDisplayName("Attached audio")); + builder.OpenComponent