diff --git a/Core/Resgrid.Services/TtsAudioService.cs b/Core/Resgrid.Services/TtsAudioService.cs index a60ce57d..a5ab4c8d 100644 --- a/Core/Resgrid.Services/TtsAudioService.cs +++ b/Core/Resgrid.Services/TtsAudioService.cs @@ -74,6 +74,9 @@ public async Task RegenerateStaticPromptsAsync(IEnumerable prompts, Canc private static Exception CreateRequestFailure(string operation, RestResponse response) { + if (response.ErrorException is OperationCanceledException or TaskCanceledException) + return new OperationCanceledException($"The TTS service {operation} was canceled.", response.ErrorException); + var status = response.StatusCode == 0 ? "no-response" : response.StatusCode.ToString(); var detail = response.ErrorException?.Message; diff --git a/Tests/Resgrid.Tests/Workers/Console/Tasks/TtsStaticPromptRefreshTaskTests.cs b/Tests/Resgrid.Tests/Workers/Console/Tasks/TtsStaticPromptRefreshTaskTests.cs index 686e4e1d..d1911c88 100644 --- a/Tests/Resgrid.Tests/Workers/Console/Tasks/TtsStaticPromptRefreshTaskTests.cs +++ b/Tests/Resgrid.Tests/Workers/Console/Tasks/TtsStaticPromptRefreshTaskTests.cs @@ -27,6 +27,7 @@ public class TtsStaticPromptRefreshTaskTests private IContainer _testWorkerContainer; private string _originalServiceBaseUrl; private string _originalStaticPromptAdminKey; + private TimeSpan _originalRetryDelay; [SetUp] public void SetUp() @@ -34,9 +35,11 @@ public void SetUp() _originalWorkerContainer = WorkerBootstrapperContainerField.GetValue(null) as IContainer; _originalServiceBaseUrl = TtsConfig.ServiceBaseUrl; _originalStaticPromptAdminKey = TtsConfig.StaticPromptAdminKey; + _originalRetryDelay = TtsStaticPromptRefreshTask.RetryDelay; TtsConfig.ServiceBaseUrl = "https://tts.example.com"; TtsConfig.StaticPromptAdminKey = "prompt-admin-key"; + TtsStaticPromptRefreshTask.RetryDelay = TimeSpan.FromMilliseconds(1); } [TearDown] @@ -46,10 +49,11 @@ public void TearDown() _testWorkerContainer?.Dispose(); TtsConfig.ServiceBaseUrl = _originalServiceBaseUrl; TtsConfig.StaticPromptAdminKey = _originalStaticPromptAdminKey; + TtsStaticPromptRefreshTask.RetryDelay = _originalRetryDelay; } [Test] - public async Task process_async_should_rethrow_refresh_failures() + public async Task process_async_should_throw_after_all_retries_exhausted() { var failure = new InvalidOperationException("refresh failed"); var ttsAudioService = new Mock(MockBehavior.Strict); @@ -66,6 +70,10 @@ await FluentActions .Should() .ThrowAsync() .WithMessage("refresh failed"); + + ttsAudioService.Verify( + x => x.RegenerateStaticPromptsAsync(It.IsAny>(), It.IsAny()), + Times.Exactly(3)); } [Test] @@ -85,6 +93,40 @@ await FluentActions .Awaiting(() => task.ProcessAsync(new TtsStaticPromptRefreshCommand(1), progress.Object, cancellationTokenSource.Token)) .Should() .ThrowAsync(); + + ttsAudioService.Verify( + x => x.RegenerateStaticPromptsAsync(It.IsAny>(), It.IsAny()), + Times.Once); + } + + [Test] + public async Task process_async_should_succeed_on_retry_after_initial_failure() + { + var ttsAudioService = new Mock(MockBehavior.Strict); + var callCount = 0; + ttsAudioService + .Setup(x => x.RegenerateStaticPromptsAsync(It.IsAny>(), It.IsAny())) + .Returns(() => + { + callCount++; + if (callCount == 1) + throw new InvalidOperationException("transient failure"); + + return Task.CompletedTask; + }); + SetWorkerContainer(ttsAudioService.Object); + + var task = new TtsStaticPromptRefreshTask(Mock.Of()); + var progress = new Mock(MockBehavior.Loose); + + await FluentActions + .Awaiting(() => task.ProcessAsync(new TtsStaticPromptRefreshCommand(1), progress.Object, CancellationToken.None)) + .Should() + .NotThrowAsync(); + + ttsAudioService.Verify( + x => x.RegenerateStaticPromptsAsync(It.IsAny>(), It.IsAny()), + Times.Exactly(2)); } private void SetWorkerContainer(ITtsAudioService ttsAudioService) diff --git a/Workers/Resgrid.Workers.Console/Tasks/TtsStaticPromptRefreshTask.cs b/Workers/Resgrid.Workers.Console/Tasks/TtsStaticPromptRefreshTask.cs index 9ec71a7d..9121155e 100644 --- a/Workers/Resgrid.Workers.Console/Tasks/TtsStaticPromptRefreshTask.cs +++ b/Workers/Resgrid.Workers.Console/Tasks/TtsStaticPromptRefreshTask.cs @@ -15,6 +15,9 @@ namespace Resgrid.Workers.Console.Tasks { public class TtsStaticPromptRefreshTask : IQuidjiboHandler { + public const int MaxRetries = 3; + public static TimeSpan RetryDelay = TimeSpan.FromSeconds(30); + public string Name => "TTS Static Prompt Refresh"; public int Priority => 1; private readonly ILogger _logger; @@ -38,11 +41,41 @@ public async Task ProcessAsync(TtsStaticPromptRefreshCommand command, IQuidjiboP } var ttsAudioService = Bootstrapper.GetKernel().Resolve(); + var prompts = TwilioVoicePromptCatalog.GetStaticPrompts(); + Exception lastException = null; + + for (int attempt = 1; attempt <= MaxRetries; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); - _logger.LogInformation("TtsStaticPromptRefresh::Refreshing static prompts"); - await ttsAudioService.RegenerateStaticPromptsAsync(TwilioVoicePromptCatalog.GetStaticPrompts(), cancellationToken); + try + { + _logger.LogInformation("TtsStaticPromptRefresh::Refreshing static prompts (attempt {Attempt}/{MaxRetries})", attempt, MaxRetries); + await ttsAudioService.RegenerateStaticPromptsAsync(prompts, cancellationToken); + + _logger.LogInformation("TtsStaticPromptRefresh::Successfully refreshed static prompts on attempt {Attempt}", attempt); + progress.Report(100, $"Finishing the {Name} Task"); + return; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + lastException = ex; + + if (attempt < MaxRetries) + { + _logger.LogWarning(ex, "TtsStaticPromptRefresh::Attempt {Attempt}/{MaxRetries} failed, retrying in {DelaySeconds}s", attempt, MaxRetries, (int)RetryDelay.TotalSeconds); + await Task.Delay(RetryDelay, cancellationToken); + } + } + } - progress.Report(100, $"Finishing the {Name} Task"); + Resgrid.Framework.Logging.LogException(lastException); + _logger.LogError(lastException, "TtsStaticPromptRefresh::Failed to refresh static prompts after {MaxRetries} attempts", MaxRetries); + throw lastException!; } catch (OperationCanceledException) {