diff --git a/.gitignore b/.gitignore index ec73ceb3c0b..78750a9ed99 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,8 @@ msbuild.wrn # MSTest test Results [Tt]est[Rr]esult*/ +!src/Microsoft.DotNet.Helix/JobMonitor/TestResults/ +!src/Microsoft.DotNet.Helix/JobMonitor/TestResults/** [Bb]uild[Ll]og.* #NUNIT diff --git a/Arcade.slnx b/Arcade.slnx index 3a900cd4719..e1aea48ad87 100644 --- a/Arcade.slnx +++ b/Arcade.slnx @@ -5,7 +5,6 @@ - diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 81ecccdd17b..53bbf74927e 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -62,6 +62,12 @@ parameters: type: number default: 30 +# Maximum number of work items whose results may be downloaded, parsed, and +# uploaded concurrently. +- name: testResultUploadParallelism + type: number + default: 48 + # When 'true' (the default), Helix work items that exit 0 but have failed AzDO test results # are treated as failed: they count toward the monitor's exit code and are resubmitted by a # later invocation's retry pass. Set to 'false' to fall back to exit-code-only outcomes. @@ -215,6 +221,8 @@ jobs: --max-wait-minutes "$((${{ parameters.timeoutInMinutes }} - 5))" # Set the tool's timeout slightly lower than the Azure DevOps job timeout to allow it to exit gracefully. --stage-name '$(System.StageName)' --stage-attempt '$(System.StageAttempt)' + --job-attempt '$(System.JobAttempt)' + --test-result-upload-parallelism '${{ parameters.testResultUploadParallelism }}' ) organization='${{ parameters.organization }}' diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs deleted file mode 100644 index df9ce21216d..00000000000 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs +++ /dev/null @@ -1,680 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Net; -using System.Net.Http.Headers; -using System.Text; -using System.Text.Json; -using System.Net.Sockets; -using Microsoft.Arcade.Common; -using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; -using Microsoft.Extensions.Logging; - -namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; - -public sealed class AzureDevOpsResultPublisher : IDisposable -{ - private const int DefaultAttemptCount = 10; - private static readonly TimeSpan s_maximumRetryDelay = TimeSpan.FromSeconds(30); - private static readonly TimeSpan s_httpClientTimeout = TimeSpan.FromMinutes(5); - private static readonly JsonSerializerOptions s_serializerOptions = new(JsonSerializerDefaults.Web) - { - WriteIndented = false, - }; - - private readonly AsyncLocal _lastSendContent = new(); - private string s_lastSendContent - { - get => _lastSendContent.Value ?? string.Empty; - set => _lastSendContent.Value = value; - } - - private readonly AzureDevOpsReportingParameters _azdoParameters; - private readonly HttpClient _httpClient; - private readonly ILogger _logger; - - public AzureDevOpsResultPublisher( - AzureDevOpsReportingParameters azdoParameters, - ILogger logger) - { - _azdoParameters = azdoParameters; - _httpClient = CreateHttpClient(azdoParameters.AccessToken); - _logger = logger; - } - - public void Dispose() - { - _httpClient.Dispose(); - } - - public async Task UploadTestResultsWithSummaryAsync(List testResultFiles, object resultMetadata, CancellationToken cancellationToken = default) - { - var testResultReader = new LocalTestResultsReader(_logger, _azdoParameters.TestResultAttachmentMode); - - async Task> ParseAsync(string file) - { - DateTimeOffset startedAt = DateTimeOffset.UtcNow; - _logger.LogDebug("Parsing test result file '{FilePath}'.", file); - IReadOnlyList results = await testResultReader.ReadResultFileAsync(file, cancellationToken); - _logger.LogDebug( - "Parsed {ResultCount} test result(s) from '{FilePath}' in {Elapsed}.", - results.Count, - file, - DateTimeOffset.UtcNow - startedAt); - return results; - } - - Task>[] parseTasks = [.. testResultFiles.Select(ParseAsync)]; - IReadOnlyList[] parsedResults = await Task.WhenAll(parseTasks); - if (parsedResults.Length == 0) - { - _logger.LogWarning("No test result files were provided for upload"); - return new TestResultUploadSummary(true, 0); - } - - IReadOnlyList aggregatedResults = new ResultAggregator().Aggregate(parsedResults, _azdoParameters.UseFullyQualifiedTestName); - if (aggregatedResults.Count == 0) - { - _logger.LogDebug("Test results were discovered but none could be aggregated"); - return new TestResultUploadSummary(true, 0); - } - - long uploadedCount = await UploadTestResultsWithCountAsync(aggregatedResults, resultMetadata, cancellationToken); - return new TestResultUploadSummary( - AllPassed: ComputeAllPassed(aggregatedResults), - UploadedCount: uploadedCount); - } - - /// - /// A work item's uploaded results are only considered a failure when a test actually failed - /// or could not be parsed into a known outcome ("None"). "Inconclusive" is a legitimate, - /// non-failing outcome produced by the aggregator for data-driven tests that mix passing and - /// skipped data rows (see ), so it must not fail the work item. - /// - internal static bool ComputeAllPassed(IReadOnlyList results) - => results.All(result => result.Result != "Failed" && result.Result != "None"); - - public async Task UploadTestResultsWithCountAsync(IEnumerable results, object resultMetadata, CancellationToken cancellationToken = default) - { - try - { - long publishedTestCount = 0; - IReadOnlyList resultList = results as IReadOnlyList ?? results.ToList(); - var converted = ConvertResults(resultList, resultMetadata).ToList(); - foreach (List batch in Batch(converted, 1000, static t => Size(t.Converted))) - { - IReadOnlyList publishedTests = await PublishResultsAsync(batch, cancellationToken); - publishedTestCount += publishedTests.Count; - } - - _logger.LogDebug("Uploaded {Count} results", publishedTestCount); - - return publishedTestCount; - } - catch (TerminalError ex) - { - _logger.LogError(ex, "Failed to upload test results to Azure DevOps."); - throw; - } - } - - private async Task> PublishResultsAsync( - IReadOnlyList converted, - CancellationToken cancellationToken) - { - var testCaseResults = converted.Select(static c => c.Converted).ToList(); - var originalList = converted.Select(static c => c.Aggregated).ToList(); - - using HttpResponseMessage response = await SendWithRetryAsync( - HttpMethod.Post, - $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results?api-version=7.1-preview.6", - testCaseResults, - DefaultAttemptCount, - cancellationToken); - - IReadOnlyList publishedResults = await ReadPublishedResultsAsync(response, cancellationToken); - if (publishedResults.Count == 0) - { - _logger.LogWarning("The test run appears to have been closed, aborting test result uploads."); - return []; - } - - List publishedTestCases = []; - - foreach ((PublishedTestCaseResultReference published, AggregatedResult original, PublishedTestCase testCase) in publishedResults.Zip(originalList, testCaseResults)) - { - if (published.Id == -1) - { - _logger.LogWarning("Azure DevOps test ID returned -1, unable to attach files."); - continue; - } - - async Task IterateSubResultsAsync( - IReadOnlyList? publishedSubResults, - IReadOnlyList originalSubResults, - long testId) - { - if (publishedSubResults is null || publishedSubResults.Count == 0) - { - if (originalSubResults.Count > 0) - { - _logger.LogError("Published results do not include sub-results, attachments lost."); - } - - return; - } - - if (publishedSubResults.Count != originalSubResults.Count) - { - _logger.LogError("Published sub-result counts do not match uploaded attachments. Attachments lost."); - return; - } - - foreach ((PublishedSubResultReference publishedSubResult, AggregatedResult originalSubResult) subTriplet in publishedSubResults.Zip(originalSubResults, (publishedSubResult, originalSubResult) => (publishedSubResult, originalSubResult))) - { - foreach (TestResultAttachment attachment in subTriplet.originalSubResult.Attachments) - { - await SendAttachmentAsync(attachment, testId, subTriplet.publishedSubResult.Id, cancellationToken); - } - - await IterateSubResultsAsync(subTriplet.publishedSubResult.SubResults, subTriplet.originalSubResult.SubResults, testId); - } - } - - foreach (TestResultAttachment attachment in original.Attachments) - { - await SendAttachmentAsync(attachment, published.Id, null, cancellationToken); - } - - await IterateSubResultsAsync(published.SubResults, original.SubResults, published.Id); - - publishedTestCases.Add(testCase); - } - - return publishedTestCases; - } - - private async Task SendAttachmentAsync( - TestResultAttachment attachment, - long testId, - long? subResultId, - CancellationToken cancellationToken) - { - var request = new TestRunAttachmentRequest( - attachment.Name, - Convert.ToBase64String(Encoding.UTF8.GetBytes(attachment.Text))); - - string path = subResultId is long subId - ? $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results/{testId}/attachments?testSubResultId={subId}&api-version=7.1-preview.1" - : $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results/{testId}/attachments?api-version=7.1-preview.1"; - - using HttpResponseMessage response = await SendWithRetryAsync( - HttpMethod.Post, - path, - request, - DefaultAttemptCount, - cancellationToken); - _ = response; - } - - private IEnumerable ConvertResults(IEnumerable results, object resultMetadata) - { - static string GetResultGroupType(AggregationType aggregationType) - { - return aggregationType switch - { - AggregationType.Single => "None", - AggregationType.DataDriven => "dataDriven", - AggregationType.Rerun => "rerun", - _ => "None", - }; - } - - string comment = JsonSerializer.Serialize(resultMetadata) ?? string.Empty; - bool useFullyQualifiedName = _azdoParameters.UseFullyQualifiedTestName; - - string DisplayNameFor(AggregatedResult result) - => useFullyQualifiedName - ? TestNameFormatter.FormatDisplayName(result.FullyQualifiedName, result.Name) - : result.Name; - - PublishedSubResult ConvertToSubTest(AggregatedResult result) - { - var customFields = new List(); - if (result.IsFlaky) - { - customFields.Add(new CustomField("IsTestResultFlaky", true)); - } - - if ((result.AttemptId ?? 0) > 1) - { - customFields.Add(new CustomField("AttemptId", result.AttemptId!.Value - 1)); - } - - return new PublishedSubResult - { - Comment = comment, - CustomFields = customFields, - DisplayName = DisplayNameFor(result), - Outcome = result.Result, - DurationInMs = result.DurationSeconds * 1000.0, - StackTrace = result.StackTrace, - ErrorMessage = result.FailureMessage, - SubResults = result.SubResults.Count == 0 ? null : [.. result.SubResults.Select(ConvertToSubTest)], - ResultGroupType = GetResultGroupType(result.AggregationType), - }; - } - - ConvertedResult ConvertResult(AggregatedResult result) - { - var customFields = new List(); - if (result.IsFlaky) - { - customFields.Add(new CustomField("IsTestResultFlaky", true)); - } - - if (result.AggregationType == AggregationType.Rerun && result.SubResults.Count > 1) - { - customFields.Add(new CustomField("AttemptId", result.SubResults.Count - 1)); - } - - string displayName = DisplayNameFor(result); - - return new ConvertedResult( - new PublishedTestCase - { - TestCaseTitle = displayName, - AutomatedTestName = useFullyQualifiedName ? result.FullyQualifiedName : result.Name, - AutomatedTestType = "helix", - AutomatedTestStorage = comment, // TODO: This was workitem ID - Priority = 1, - DurationInMs = result.DurationSeconds * 1000.0, - Outcome = result.Result, - State = "Completed", - Comment = comment, - StackTrace = result.StackTrace, - ErrorMessage = result.FailureMessage, - SubResults = result.SubResults.Count == 0 ? null : [.. result.SubResults.Select(ConvertToSubTest)], - ResultGroupType = GetResultGroupType(result.AggregationType), - CustomFields = customFields, - }, - result); - } - - var converted = results.Select(ConvertResult).ToList(); - foreach (ConvertedResult? result in converted) - { - foreach (ConvertedResult chunk in Chunk(result, 950)) - { - yield return chunk; - } - } - } - - private static IEnumerable Chunk(ConvertedResult test, int limit) - { - if (Size(test.Converted) <= limit) - { - yield return test; - yield break; - } - - IEnumerable zippedSubTests = (test.Converted.SubResults ?? []) - .Zip(test.Aggregated.SubResults, (converted, aggregated) => new ChunkPair(converted, aggregated)); - - foreach (List zippedBatch in Batch(zippedSubTests, limit, static pair => Size(pair.Converted))) - { - yield return new ConvertedResult( - test.Converted with { SubResults = [.. zippedBatch.Select(static x => x.Converted)], Id = null }, - new AggregatedResult( - test.Aggregated.AggregationType, - test.Aggregated.Name, - test.Aggregated.DurationSeconds, - test.Aggregated.Result, - [.. zippedBatch.Select(static x => x.Aggregated)], - test.Aggregated.Attachments, - test.Aggregated.FailureMessage, - test.Aggregated.StackTrace, - isFlaky: test.Aggregated.IsFlaky, - attemptId: test.Aggregated.AttemptId, - fullyQualifiedName: test.Aggregated.FullyQualifiedName)); - } - } - - private static int Size(PublishedTestCase test) - { - return 1 + (test.SubResults?.Sum(Size) ?? 0); - } - - private static int Size(PublishedSubResult test) - { - return 1 + (test.SubResults?.Sum(Size) ?? 0); - } - - private static IEnumerable> Batch(IEnumerable items, int limit, Func getSize) - { - var currentBatch = new List(); - int currentSize = 0; - - foreach (T? item in items) - { - int size = getSize(item); - if (size > limit) - { - throw new InvalidOperationException("Cannot split a result larger than the batching limit."); - } - - if (currentSize + size > limit && currentBatch.Count > 0) - { - yield return currentBatch; - currentBatch = []; - currentSize = 0; - } - - currentBatch.Add(item); - currentSize += size; - } - - if (currentBatch.Count > 0) - { - yield return currentBatch; - } - } - - private static HttpClient CreateHttpClient(string? accessToken) - { - var client = new HttpClient - { - Timeout = s_httpClientTimeout - }; - client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - - if (!string.IsNullOrWhiteSpace(accessToken)) - { - string basicToken = Convert.ToBase64String(Encoding.ASCII.GetBytes($":{accessToken}")); - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", basicToken); - } - - return client; - } - - private async Task SendWithRetryAsync( - HttpMethod method, - string relativePath, - object? payload, - int attemptCount, - CancellationToken cancellationToken) - { - string? body = payload is null ? null : JsonSerializer.Serialize(payload, s_serializerOptions); - if (!string.IsNullOrEmpty(body)) - { - s_lastSendContent = body; - } - - HttpResponseMessage? successfulResponse = null; - Exception? lastException = null; - var retryHandler = new ExponentialRetry - { - MaxAttempts = attemptCount, - DelayBase = 3, - DelayConstant = 0, - MinRandomFactor = 1, - MaxRandomFactor = 1, - MaximumDelay = s_maximumRetryDelay, - RetryDelayCallback = (failedAttempt, delay) => - _logger.LogDebug( - "Azure DevOps {Method} request to '{RequestPath}' failed on attempt {Attempt} of {AttemptCount}. " - + "Waiting {RetryDelay} before the next attempt.", - method, - relativePath, - failedAttempt, - attemptCount, - delay), - }; - - bool succeeded = await retryHandler.RunAsync( - async attempt => - { - Uri baseUri = _azdoParameters.CollectionUri.AbsoluteUri.EndsWith('/') - ? _azdoParameters.CollectionUri - : new Uri(_azdoParameters.CollectionUri.AbsoluteUri + '/', UriKind.Absolute); - - using var request = new HttpRequestMessage(method, new Uri(baseUri, relativePath)); - if (body is not null) - { - request.Content = new StringContent(body, Encoding.UTF8, "application/json"); - } - - try - { - DateTimeOffset requestStartedAt = DateTimeOffset.UtcNow; - _logger.LogDebug( - "Sending Azure DevOps {Method} request to '{RequestPath}', attempt {Attempt} of {AttemptCount}.", - method, - relativePath, - attempt + 1, - attemptCount); - HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); - if (response.IsSuccessStatusCode) - { - _logger.LogDebug( - "Azure DevOps {Method} request to '{RequestPath}' completed with HTTP {StatusCode} " - + "on attempt {Attempt} of {AttemptCount} after {Elapsed}.", - method, - relativePath, - (int)response.StatusCode, - attempt + 1, - attemptCount, - DateTimeOffset.UtcNow - requestStartedAt); - successfulResponse = response; - return RetryResult.Success; - } - - using (response) - { - string responseBody = await response.Content.ReadAsStringAsync(cancellationToken); - bool isTransientStatus = (int)response.StatusCode >= 500 - || response.StatusCode == HttpStatusCode.TooManyRequests; - if (isTransientStatus) - { - TimeSpan? retryAfter = GetRetryDelay(response); - if (response.StatusCode == HttpStatusCode.TooManyRequests && retryAfter is null) - { - retryAfter = TimeSpan.FromSeconds(30); - } - - _logger.LogDebug( - "Azure DevOps {Method} request to '{RequestPath}' returned HTTP {StatusCode} " - + "on attempt {Attempt} of {AttemptCount} after {Elapsed}. Retrying.", - method, - relativePath, - (int)response.StatusCode, - attempt + 1, - attemptCount, - DateTimeOffset.UtcNow - requestStartedAt); - lastException = new AzureDevOpsReportingError( - $"Azure DevOps request failed with status code {(int)response.StatusCode}: {responseBody}"); - return RetryResult.Retry(retryAfter); - } - - if (responseBody.Contains("It may have been deleted", StringComparison.OrdinalIgnoreCase) - || responseBody.Contains("not authorized to access this resource", StringComparison.OrdinalIgnoreCase) - || responseBody.Contains("cannot be added or updated for a test run which is in Completed state", StringComparison.OrdinalIgnoreCase) - || response.StatusCode == HttpStatusCode.Forbidden - || response.StatusCode == HttpStatusCode.Unauthorized) - { - throw new TerminalError(responseBody); - } - - throw new AzureDevOpsReportingError( - $"Azure DevOps request failed with status code {(int)response.StatusCode}: {responseBody}"); - } - } - catch (Exception ex) when (IsTransientException(ex, cancellationToken)) - { - lastException = ex; - _logger.LogDebug( - ex, - "Transient Azure DevOps {Method} request failure for '{RequestPath}' on attempt " - + "{Attempt} of {AttemptCount}. Retrying.", - method, - relativePath, - attempt + 1, - attemptCount); - return RetryResult.Retry(); - } - }, - cancellationToken); - - return succeeded && successfulResponse is not null - ? successfulResponse - : throw lastException ?? new InvalidOperationException("Azure DevOps retry loop exited unexpectedly."); - } - - internal static bool IsTransientException(Exception exception, CancellationToken cancellationToken) - => !cancellationToken.IsCancellationRequested - && exception is OperationCanceledException { InnerException: TimeoutException } - or HttpRequestException - or TimeoutException - or SocketException - or IOException; - - private static TimeSpan? GetRetryDelay(HttpResponseMessage response) - { - RetryConditionHeaderValue? retryAfter = response.Headers.RetryAfter; - if (retryAfter?.Delta is { } delta && delta > TimeSpan.Zero) - { - return delta; - } - - if (retryAfter?.Date is { } date) - { - TimeSpan delay = date - DateTimeOffset.UtcNow; - if (delay > TimeSpan.Zero) - { - return delay; - } - } - - return null; - } - - private static async Task> ReadPublishedResultsAsync( - HttpResponseMessage response, - CancellationToken cancellationToken) - { - string content = await response.Content.ReadAsStringAsync(cancellationToken); - if (string.IsNullOrWhiteSpace(content)) - { - return []; - } - - using var document = JsonDocument.Parse(content); - JsonElement root = document.RootElement; - if (root.ValueKind == JsonValueKind.Array) - { - return [.. root.EnumerateArray().Select(ParsePublishedResult)]; - } - - if (root.TryGetProperty("value", out JsonElement value) && value.ValueKind == JsonValueKind.Array) - { - return [.. value.EnumerateArray().Select(ParsePublishedResult)]; - } - - return []; - } - - private static PublishedTestCaseResultReference ParsePublishedResult(JsonElement element) - { - var subResults = new List(); - if (element.TryGetProperty("subResults", out JsonElement subResultElement) && subResultElement.ValueKind == JsonValueKind.Array) - { - subResults.AddRange(subResultElement.EnumerateArray().Select(ParsePublishedSubResult)); - } - - return new PublishedTestCaseResultReference( - element.TryGetProperty("id", out JsonElement idElement) ? idElement.GetInt64() : -1, - subResults); - } - - private static PublishedSubResultReference ParsePublishedSubResult(JsonElement element) - { - var subResults = new List(); - if (element.TryGetProperty("subResults", out JsonElement subResultElement) && subResultElement.ValueKind == JsonValueKind.Array) - { - subResults.AddRange(subResultElement.EnumerateArray().Select(ParsePublishedSubResult)); - } - - return new PublishedSubResultReference( - element.TryGetProperty("id", out JsonElement idElement) ? idElement.GetInt64() : -1, - subResults); - } - - private sealed record ConvertedResult(PublishedTestCase Converted, AggregatedResult Aggregated); - - private sealed record ChunkPair(PublishedSubResult Converted, AggregatedResult Aggregated); - - private sealed record TestRunAttachmentRequest(string FileName, string Stream); - - private sealed record CustomField(string FieldName, object Value); - - private sealed record PublishedTestCase - { - public long? Id { get; init; } - - public string TestCaseTitle { get; init; } = string.Empty; - - public string AutomatedTestName { get; init; } = string.Empty; - - public string AutomatedTestType { get; init; } = string.Empty; - - public string AutomatedTestStorage { get; init; } = string.Empty; - - public int Priority { get; init; } - - public double DurationInMs { get; init; } - - public string Outcome { get; init; } = string.Empty; - - public string State { get; init; } = string.Empty; - - public string Comment { get; init; } = string.Empty; - - public string? StackTrace { get; init; } - - public string? ErrorMessage { get; init; } - - public List? SubResults { get; init; } - - public string ResultGroupType { get; init; } = string.Empty; - - public List? CustomFields { get; init; } - } - - private sealed record PublishedSubResult - { - public long? Id { get; init; } - - public string Comment { get; init; } = string.Empty; - - public List? CustomFields { get; init; } - - public string DisplayName { get; init; } = string.Empty; - - public string Outcome { get; init; } = string.Empty; - - public double DurationInMs { get; init; } - - public string? StackTrace { get; init; } - - public string? ErrorMessage { get; init; } - - public List? SubResults { get; init; } - - public string ResultGroupType { get; init; } = string.Empty; - } - - private sealed record PublishedTestCaseResultReference(long Id, IReadOnlyList SubResults); - - private sealed record PublishedSubResultReference(long Id, IReadOnlyList SubResults); -} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs deleted file mode 100644 index 95c63a90157..00000000000 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs +++ /dev/null @@ -1,219 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Globalization; -using System.Xml.Linq; -using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; -using Microsoft.Extensions.Logging; - -namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; - -public sealed class LocalTestResultsReader( - ILogger logger, - TestResultAttachmentMode attachmentMode = TestResultAttachmentMode.Failed) -{ - private readonly ILogger _logger = logger; - private readonly TestResultAttachmentMode _attachmentMode = attachmentMode; - - public static bool LooksLikeTestResultFile(string path) - { - string fileName = Path.GetFileName(path); - return fileName.EndsWith(".trx", StringComparison.OrdinalIgnoreCase) - || fileName.EndsWith("testResults.xml", StringComparison.OrdinalIgnoreCase) - || fileName.EndsWith("test-results.xml", StringComparison.OrdinalIgnoreCase) - || fileName.EndsWith("test_results.xml", StringComparison.OrdinalIgnoreCase) - || fileName.EndsWith("junit-results.xml", StringComparison.OrdinalIgnoreCase) - || fileName.EndsWith("junitresults.xml", StringComparison.OrdinalIgnoreCase); - } - - public async Task> ReadResultFileAsync(string filePath, CancellationToken cancellationToken = default) - { - try - { - using FileStream stream = File.OpenRead(filePath); - XDocument document = await XDocument.LoadAsync(stream, LoadOptions.PreserveWhitespace, cancellationToken); - string rootName = document.Root?.Name.LocalName ?? string.Empty; - string workItemName = new DirectoryInfo(Path.GetDirectoryName(filePath) ?? string.Empty).Name; - - return rootName switch - { - "assemblies" or "assembly" => ReadXunitResults(document), - "TestRun" => ReadTrxResults(document, workItemName), - "testsuites" or "testsuite" => ReadJUnitResults(document, workItemName), - _ => [], - }; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to parse test results file '{Path}'.", filePath); - return []; - } - } - - private IReadOnlyList ReadXunitResults(XDocument document) - { - return [.. - document.Descendants().Where(static e => e.Name.LocalName == "test").Select(test => - { - XElement? failure = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "failure"); - string? message = failure?.Elements().FirstOrDefault(static x => x.Name.LocalName == "message")?.Value?.Trim(); - string? stackTrace = failure?.Elements().FirstOrDefault(static x => x.Name.LocalName == "stack-trace")?.Value?.Trim(); - string? output = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "output")?.Value?.Trim(); - string? skipReason = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "reason")?.Value?.Trim(); - - string typeName = GetAttribute(test, "type") ?? string.Empty; - string method = GetAttribute(test, "method") ?? string.Empty; - string name = GetAttribute(test, "name") - ?? (!string.IsNullOrEmpty(typeName) && !string.IsNullOrEmpty(method) ? $"{typeName}.{method}" : method); - string normalizedOutcome = NormalizeOutcome(GetAttribute(test, "result")); - - List attachments = []; - AddAttachmentIfEnabled(attachments, "output.txt", output, normalizedOutcome); - - return new TestResult( - name, - "xunit", - typeName, - method, - ParseDouble(GetAttribute(test, "time")), - normalizedOutcome, - GetAttribute(failure, "exception-type"), - message, - stackTrace, - skipReason, - attachments); - })]; - } - - private IReadOnlyList ReadJUnitResults(XDocument document, string workItemName) - { - return [.. - document.Descendants().Where(static e => e.Name.LocalName == "testcase").Select(test => - { - XElement? failure = test.Elements().FirstOrDefault(static x => x.Name.LocalName is "failure" or "error"); - XElement? skipped = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "skipped"); - string? stdout = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "system-out")?.Value?.Trim(); - string? stderr = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "system-err")?.Value?.Trim(); - - string className = GetAttribute(test, "classname") ?? workItemName; - string method = GetAttribute(test, "name") ?? string.Empty; - string name = !string.IsNullOrEmpty(className) ? $"{className}.{method}" : method; - string result = skipped is not null ? "Skip" : failure is not null ? "Fail" : "Pass"; - - List attachments = []; - AddAttachmentIfEnabled(attachments, "stdout.txt", stdout, result); - AddAttachmentIfEnabled(attachments, "stderr.txt", stderr, result); - - return new TestResult( - name, - "junit", - className, - method, - ParseDouble(GetAttribute(test, "time")), - result, - null, - failure?.Value?.Trim(), - null, - skipped?.Value?.Trim(), - attachments); - })]; - } - - private IReadOnlyList ReadTrxResults(XDocument document, string workItemName) - { - Dictionary unitTestsById = document - .Descendants() - .Where(static e => e.Name.LocalName == "UnitTest") - .Select(static unitTest => (Id: GetAttribute(unitTest, "id"), Element: unitTest)) - .Where(static x => !string.IsNullOrEmpty(x.Id)) - .ToDictionary(static x => x.Id!, static x => x.Element, StringComparer.OrdinalIgnoreCase); - - return [.. - document.Descendants().Where(static e => e.Name.LocalName == "UnitTestResult").Select(result => - { - string testId = GetAttribute(result, "testId") ?? string.Empty; - unitTestsById.TryGetValue(testId, out XElement? unitTest); - XElement? testMethod = unitTest?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "TestMethod"); - - string className = GetAttribute(testMethod, "className") ?? workItemName; - string method = GetAttribute(testMethod, "name") ?? GetAttribute(result, "testName") ?? string.Empty; - string displayName = GetAttribute(result, "testName") - ?? (!string.IsNullOrEmpty(className) ? $"{className}.{method}" : method); - - XElement? output = result.Descendants().FirstOrDefault(static x => x.Name.LocalName == "Output"); - string? failureMessage = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "Message")?.Value?.Trim(); - string? stackTrace = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StackTrace")?.Value?.Trim(); - string? stdout = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StdOut")?.Value?.Trim(); - string? stderr = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StdErr")?.Value?.Trim(); - - string rawOutcome = GetAttribute(result, "outcome") ?? string.Empty; - string normalizedOutcome = NormalizeOutcome(rawOutcome); - string? skipReason = string.Equals(normalizedOutcome, "Skip", StringComparison.Ordinal) ? failureMessage : null; - - List attachments = []; - AddAttachmentIfEnabled(attachments, "stdout.txt", stdout, normalizedOutcome); - AddAttachmentIfEnabled(attachments, "stderr.txt", stderr, normalizedOutcome); - - return new TestResult( - displayName, - "trx", - className, - method, - ParseDuration(GetAttribute(result, "duration")), - normalizedOutcome, - null, - failureMessage, - stackTrace, - skipReason, - attachments); - })]; - } - - private static string? GetAttribute(XElement? element, string name) - => element?.Attribute(name)?.Value; - - private static double ParseDouble(string? value) - { - return double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double result) - ? result - : 0; - } - - private static double ParseDuration(string? value) - { - return TimeSpan.TryParse(value, CultureInfo.InvariantCulture, out TimeSpan result) - ? result.TotalSeconds - : ParseDouble(value); - } - - private static string NormalizeOutcome(string? value) - { - return value?.Trim().ToLowerInvariant() switch - { - "pass" or "passed" or "success" or "succeeded" => "Pass", - "skip" or "skipped" or "notexecuted" or "notrun" => "Skip", - "fail" or "failed" or "error" or "timeout" or "aborted" => "Fail", - _ => "None", - }; - } - - private void AddAttachmentIfEnabled( - List attachments, - string name, - string? text, - string normalizedOutcome) - { - bool includeAttachment = _attachmentMode switch - { - TestResultAttachmentMode.All => true, - TestResultAttachmentMode.Failed => string.Equals(normalizedOutcome, "Fail", StringComparison.Ordinal), - TestResultAttachmentMode.None => false, - _ => false, - }; - - if (includeAttachment && !string.IsNullOrWhiteSpace(text)) - { - attachments.Add(new TestResultAttachment(name, text)); - } - } -} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj deleted file mode 100644 index 9b6e4c57bdb..00000000000 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - $(NetMinimum) - enable - enable - - - - - - - - - - - - - - - diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md new file mode 100644 index 00000000000..e60e27122fb --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md @@ -0,0 +1,86 @@ +# Architecture + +## Ownership + +`JobMonitorRunner` is the single control-plane owner. It performs the entry +retry pass, obtains one immutable Azure DevOps/Helix snapshot per poll, updates +`MonitorState`, queues newly completed jobs, reports status from the snapshot, +and decides when normal completion is possible. + +Result publication is owned by `TestResultUploadPipeline`. The poller hands the +pipeline a completed job plus the work-item snapshot that established +completion. The pipeline never calls back into the poll loop and the reporter +never calls into the pipeline's external services. + +## Shared parallelism + +`Parallelism/ActionQueue` is the shared long-lived producer/consumer +primitive. It supports bounded and unbounded `Channel` instances with a +fixed worker set, cancellation, fault propagation, completion, drain, and +atomic queue counters. Only lightweight completed-job descriptors use the +unbounded form; work-item and finalization queues remain bounded. + +`Parallelism/ParallelAsync` is used for bounded snapshot reads whose complete +result is needed by the current poll. + +Parallelism budgets are independent: + +- job expansion has low parallelism because it only creates session state and + feeds work items; +- work-item processing uses `TestResultUploadParallelism`; +- test-run finalization has low parallelism because it performs small, + non-replayable writes. + +This prevents the old multiplication of "jobs × work items × result files" +tasks and keeps expensive state proportional to configured concurrency plus +bounded queue capacity. + +## Data flow + +```text +Azure DevOps + Helix snapshots + | + v + JobMonitorRunner + | + v + lightweight completed-job queue + | + v + bounded work-item queue + download -> parse -> aggregate -> publish + | + v + bounded finalization queue + attachment -> complete/tag +``` + +Backpressure is intentional. Job workers may wait while feeding a full +work-item queue, and work-item workers may wait on Azure DevOps throttling. +Completed-job acceptance remains non-blocking, so neither condition blocks the +poller or status reporter. + +`JobMonitorMetrics` is shared by the runner, services, publisher, rate-limit +gate, and upload pipeline. It records atomic request counts and operation +timings without emitting per-request information logs. The final aggregate +report is described in [Performance metrics](Components/PerformanceMetrics.md). + +## Logical stream identity + +Retry and outcome reconciliation operate on logical Helix job streams rather +than AzDO jobs. An AzDO job can submit multiple Helix jobs to the same queue, +so AzDO job identity plus queue does not uniquely identify a stream. + +`MonitorState` combines the stage identity, AzDO phase identity, queue, and +submitter-assigned logical Helix `jobName` (falling back to `TestRunName`). +Attempts are incarnation metadata rather than key components. Retry compares +the preserved submitter `System.JobAttempt` with the current timeline record; +resubmissions stamp the current stage attempt, preserve the submitter attempt, +and add `PreviousHelixJobName`. + +## Durability boundary + +The only durable "processed" marker is the Helix-job tag on a completed Azure +DevOps test run. A session is finalized only if every work item completed +cleanly. Any download, parse, publication, or finalization failure leaves the +run untagged so a later invocation replays the complete Helix job. diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/PerformanceMetrics.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/PerformanceMetrics.md new file mode 100644 index 00000000000..27b105ca129 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/PerformanceMetrics.md @@ -0,0 +1,44 @@ +# Performance metrics + +The monitor emits one aggregate performance block at normal completion or +timeout. Metrics are recorded with atomic counters and stopwatch timestamps; +request-level logging is not required and normal log volume is independent of +the number of requests. + +## Remote operations + +Azure DevOps metrics count every HTTP attempt, separated into control-plane, +test-result batch, and attachment requests. The report includes retries, +failed attempts, serialized request payload bytes, aggregate request time, and +the slowest request. A retried request contributes one attempt and its payload +bytes for each transmission. + +Helix metrics count attempts made through the shared retry wrapper. Result blob +downloads are reported separately because they use Azure Storage rather than +the Helix API. + +Rate-limit metrics distinguish server-directed deferrals from actual shared +gate waits. Deferrals report the delay guidance received from Azure DevOps; +gate waits report aggregate worker wait time and the longest individual wait. +Aggregate wait can exceed wall-clock time when several workers are delayed +concurrently. + +## Pipeline timings + +The report includes aggregate worker time and maximum single-operation time for: + +- work-item result discovery and download; +- XML parsing and aggregation; +- Azure DevOps publication; +- test-run creation; +- test-run completion and durable tagging. + +Upload throughput uses the interval from the first pipeline operation to the +last completed operation, not the full monitor lifetime spent waiting for +Helix work. Parsing is measured separately from Azure DevOps publication. + +Aggregate worker time can exceed monitor elapsed time because work executes in +parallel. Comparing aggregate time, maximum latency, request counts, and +throughput identifies whether a run is limited by result download, local +processing, Azure DevOps request latency, attachments, throttling, or final +test-run operations. diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md new file mode 100644 index 00000000000..649c4aa5edc --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md @@ -0,0 +1,27 @@ +# Polling and reconciliation + +Each poll concurrently refreshes work-item summaries for in-scope jobs that +have not yet been reconciled, with a fixed degree of parallelism. Terminal +snapshots are retained for later status reports instead of being downloaded +again. The resulting dictionary is the poll's immutable work-item snapshot and +is reused for: + +- completion fallback when a Helix job summary has not yet transitioned; +- outcome reconciliation; +- upload scheduling; +- failed-work-item links; +- aggregate status counts. + +No second service call is made for status. + +The one-shot entry retry pass and stage/job-attempt semantics are specified in +[the semantic document](../SemanticBehavior.md). It uses the same timeline +snapshot as the first poll to compare each Helix stream's preserved submitter +job attempt with the current timeline job attempt. Outcome updates are applied +oldest-to-newest so newer stage/job attempts and resubmission lineage supersede +older failures. + +The logical stream key includes stage, AzDO phase/job identity, queue, and the +submitter-assigned Helix `jobName` (or `TestRunName` when `jobName` is absent). +That discriminator is preserved by resubmission, allowing retries to collapse +only actual incarnations of the same logical Helix job without crossing stages. diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Shutdown.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Shutdown.md new file mode 100644 index 00000000000..7ef25ea12eb --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Shutdown.md @@ -0,0 +1,31 @@ +# Shutdown and drain + +## Normal completion + +The runner stops producing completed jobs, completes the job queue, waits for +job expansion, completes and drains the work-item queue, then completes and +drains finalization. This ordering guarantees that no downstream producer is +still active when its channel is closed. + +Work starts as soon as each Helix job completes, so normal drain contains only +the remaining tail. The drain log records elapsed time and aggregate pipeline +counts for runtime performance validation. + +At drain start, the monitor also records how many work items were first +observed terminal in the final poll, how many uploads became eligible at the +whole-job boundary in that poll, and how much remaining work came from the +final poll versus earlier polls. This distinguishes the unavoidable minimum +drain from pipeline lag accumulated before Helix completion. + +## Cancellation + +Cancellation does not drain uploads. The pipeline worker tokens are canceled +immediately, the timeout report is emitted, and latest in-flight Helix jobs are +canceled with an independent bounded token. Incomplete test runs remain +untagged and are replayed by a later invocation. + +## Crash recovery + +In-memory queue/session state is never required after restart. Completed tags, +failed-work-item attachments, Helix job properties, and resubmission lineage +are sufficient to reconstruct all required work. diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/StateAndStatus.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/StateAndStatus.md new file mode 100644 index 00000000000..1f3bde13cfd --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/StateAndStatus.md @@ -0,0 +1,20 @@ +# State and status + +`MonitorState` owns invocation state behind one lock. Collections are exposed +only through immutable snapshots. Upload workers update only narrow methods: +upload lifecycle, uploaded test outcomes, and durable completion. + +The runner publishes an immutable latest-poll snapshot. A dedicated periodic +reporting loop consumes that snapshot and the upload pipeline's atomic counters +every five minutes. Reporting therefore continues at a fixed cadence while the +poll loop is active and during final drain, without waiting for Helix file +access or Azure DevOps result uploads. + +An initial status is emitted after the first poll. Subsequent status messages +are timer-driven rather than triggered by job-count changes, so completion +bursts do not increase log volume. Normal logging reports semantic transitions +and aggregate counts. Verbose +logging adds queue depth, active worker counts, finalizer depth, and uploaded +result totals. It deliberately does not print every job, work item, file, or +request; verbose mode must remain usable on runs containing thousands of work +items and millions of results. diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md new file mode 100644 index 00000000000..fca4e209bf9 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md @@ -0,0 +1,46 @@ +# Test-result processing + +Result processing lives under `JobMonitor/TestResults`; the former standalone +`AzureDevOpsTestPublisher` project was removed because the job monitor was its +only product consumer. + +## Parsing + +`LocalTestResultsReader` uses `XmlReader` over a `FileStream`. xUnit and JUnit +records are materialized one result element at a time. TRX uses a small first +pass for test definitions followed by a result pass, because TRX result records +refer to definitions by ID and may precede them in the file. + +Malformed files are warned and omitted. Caller cancellation is propagated. +DTD processing is prohibited. + +Helix may append `.txt` to uploaded result artifacts, for example +`testResults.xml.txt` or `results.trx.txt`. File recognition strips that single +transport suffix before matching the supported xUnit, JUnit, and TRX names. +Recognized files whose XML root is unsupported are warned rather than silently +producing zero results. + +## Aggregation + +Existing single, data-driven, and rerun semantics are retained, including +flaky-result fields, fully qualified identity, attachment selection, and the +rule that `Inconclusive` does not fail a work item. + +Aggregation state is scoped to one work item. This bounds normal memory by the +largest concurrently processed work items rather than the whole Helix job or +build. A single pathological work item with millions of distinct tests still +sets the memory floor; handling that case would require spill-to-disk grouping. + +## Azure DevOps batching + +- A request contains at most 1,000 top-level results. +- Nested sub-results do not consume that request limit. +- A single oversized hierarchy is defensively split below 950 recursive nodes. +- Converted results are enumerated lazily into one request batch at a time. +- The serialized UTF-8 request body is retained only for the lifetime of that + request and its retries. +- Azure DevOps rate-limit guidance is applied through a service-wide gate so + concurrent workers slow down together instead of stampeding a throttled + endpoint independently. A response advances the gate for future requests; + the request that already received the response returns immediately rather + than redundantly adding the advertised delay to its own completion time. diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md new file mode 100644 index 00000000000..e1b8d91a0a9 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md @@ -0,0 +1,61 @@ +# Upload pipeline + +`TestResultUploadPipeline` contains three worker stages. The lightweight job +stage is unbounded so completed jobs cannot be dropped and upload backpressure +cannot block polling. The expensive work-item and finalization stages are +bounded. + +## Job expansion + +A completed Helix job is accepted once. A `JobUploadSession` captures the +completion-time work-item snapshot and owns: + +- a single-flight Azure DevOps test-run creation task; +- expected and finished work-item counts; +- failed-test work-item names; +- uploaded-result count; +- a sticky failure flag. + +The job stage expands work items into the global work-item queue. A job with no +work items proceeds directly to finalization. + +## Work-item processing + +Terminal work items are admitted as soon as polling observes their immutable +exit code, even if the containing Helix job is still running. This lets large +jobs overlap result publication with their remaining execution instead of +making every completed work item part of the final drain. + +Each worker: + +1. downloads recognized result files for one work item, retrying only + transient read failures; +2. obtains the session's single test-run ID; +3. parses, aggregates, batches, and publishes results; +4. records the upload summary and test-only failure outcome; +5. signals session completion. + +Work-item concurrency is global. A build with many jobs therefore cannot create +an unbounded task graph or multiply the configured Azure DevOps pressure. +Consumers can tune this global budget through the +`testResultUploadParallelism` pipeline-template parameter, which forwards to +the monitor's `--test-result-upload-parallelism` option. The default is 48, +selected from runtime validation with approximately 6,800 work items and +3.1 million results: it kept final drain below 2% while reducing service +throttling guidance compared with 64 workers. + +## Finalization + +Once the Helix job is complete, the last outstanding work item queues its +session for finalization. If any work item failed, the session remains +untagged. Otherwise finalization uploads the +failed-work-item attachment, marks the run completed, applies the Helix-job +tag, and only then marks the job durably processed. + +Test-run creation and attachment publication are not replayed after ambiguous +failures because they are non-idempotent POST operations. The final completion +PATCH is idempotent and uses bounded transient retries; repeating it applies +the same completed state and Helix-job tag to the same run without creating +duplicate results. Result publication also uses bounded transient retries +because losing an entire job's results is worse than the accepted duplicate +risk. diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md new file mode 100644 index 00000000000..18461550cce --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md @@ -0,0 +1,25 @@ +# Helix Job Monitor design + +The job monitor is a restart-safe coordinator between Azure DevOps and Helix. It +polls control-plane state, reconciles retries, publishes large test-result sets, +and computes the final stage outcome without tying progress reporting to result +upload latency. + +The design is split by concern: + +- [Semantic behavior](SemanticBehavior.md) defines externally + observable behavior and restart invariants. +- [Architecture](Architecture.md) describes process structure, ownership, + backpressure, and shared parallelism utilities. +- [Polling and reconciliation](Components/Polling.md) describes discovery, + stage-attempt scoping, retry, and outcome ordering. +- [Upload pipeline](Components/UploadPipeline.md) describes the bounded + job/work-item/finalization stages and durable completion boundary. +- [Test-result processing](Components/TestResults.md) describes XML parsing, + aggregation, batching, attachments, and Azure DevOps limits. +- [State and status](Components/StateAndStatus.md) describes thread-safe state, + snapshots, progress, and bounded verbose logging. +- [Performance metrics](Components/PerformanceMetrics.md) describes request + counts, retries, throughput, rate-limit waits, and pipeline stage timings. +- [Shutdown](Components/Shutdown.md) describes normal drain, cancellation, and + crash recovery. diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md similarity index 57% rename from src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md rename to src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md index 9e33357a7db..2d08a17ca52 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md @@ -1,9 +1,10 @@ -# JobMonitorRunner — Technical Specification +# Semantic behavior -This document is a behavioral specification of the Helix job monitor runner. -It describes *what* the runner must do, not *how* it currently does it. +This document specifies the externally observable behavior and restart +invariants of the Helix Job Monitor. It describes *what* the monitor must do, +not *how* it currently does it. -The current source lives at [JobMonitorRunner.cs](JobMonitorRunner.cs); use +The current source lives at [JobMonitorRunner.cs](../JobMonitorRunner.cs); use it only as the reference implementation, not as the specification. --- @@ -18,7 +19,8 @@ Azure DevOps pipeline stage. Its job is to: - Resubmit failed Helix work items once per invocation. - Upload Helix work-item test results to Azure DevOps. - Return an exit code that reflects whether the monitored pipeline jobs and - the latest completed Helix work items all succeeded. + the latest completed Helix work items and their processed test results all + succeeded. ## 2. Operating model @@ -32,12 +34,11 @@ principle has two halves that must both hold: never reach a terminal state (for example, work items stranded in `Waiting` after their queue was purged), and a superseded attempt's monitor is already gone, so nothing else will ever drive it. Waiting on it means waiting forever. -2. **All pipeline-submitted work must still complete.** The monitor cannot simply - discard a previous attempt's work either: that work represents tests the - pipeline asked to run. Any previous-attempt work that is not already - terminally passed (i.e. it failed or is still unfinished) and that the current - attempt has *not* already re-submitted must be **resubmitted into the current - attempt** so it is actually carried to completion (§2.3). +2. **Only work whose submitter did not rerun may be replayed.** Previous-stage + work still represents the current execution only when the matching AzDO + submitter job remains at the same `System.JobAttempt`. If the timeline shows + a newer submitter attempt, the old Helix work is superseded and must not be + resubmitted, even before the replacement Helix job is visible. Concretely, all decisions (retry, completion gating, upload, pass/fail) consider: @@ -52,30 +53,31 @@ Concretely, all decisions (retry, completion gating, upload, pass/fail) consider Jobs and work items from other stages must not be retried, uploaded, or used to fail this invocation. -**Why per-attempt, and why not just ignore previous attempts.** Azure DevOps -offers two distinct re-run gestures, and the monitor cannot tell them apart from -the timeline alone: +**Why both attempt axes are required.** Both Azure DevOps retry gestures advance +`System.StageAttempt`; the difference is which timeline jobs advance their +individual `System.JobAttempt`: - **Rerun the entire stage** — every job re-runs, including the Helix submitter jobs, so the current attempt already contains a fresh incarnation of every logical work stream. Previous-attempt incarnations are superseded and need no resubmission. -- **Retry failed jobs in the stage** — only failed jobs re-run. If the Helix +- **Retry failed jobs in the stage** — only selected failed jobs advance their + job attempt. If the Helix submitter jobs passed and only the monitor failed (e.g. it timed out), the - submitters do **not** re-run, so the current attempt contains **no** Helix work - at all. Naively scoping to the current attempt would make the monitor exit + monitor advances but the submitters remain at their original job attempts, so + the current stage attempt contains **no** fresh Helix work. Naively scoping to + the current stage attempt would make the monitor exit immediately as a success, silently discarding every result and failure from the previous attempt. Because of the second gesture, "current-attempt scope" is not the same as "ignore previous attempts." The monitor scopes *gating* to the current attempt but reconciles previous-attempt work into it by resubmission (§2.3), deciding -per logical work stream (not per attempt) whether a resubmission is needed. +per logical work stream by comparing the Helix job's submitter `System.JobAttempt` +with the matching current timeline record. -The monitor's own stage attempt is provided as an input (see §3) and defaults to -the `SYSTEM_STAGEATTEMPT` pipeline variable. When it is unknown the monitor -cannot distinguish attempts and falls back to build + stage scope, gating on -every attempt's work (historical behavior). +The monitor's stage and job attempts are provided as inputs (see §3), defaulting +to `SYSTEM_STAGEATTEMPT` and `SYSTEM_JOBATTEMPT`. ### 2.2 Durable state @@ -104,39 +106,48 @@ be the source of truth for cross-invocation correctness. Retry is the mechanism that reconciles previous-attempt work into the current attempt (§2.1). It operates on *logical work streams*, not on attempts: a work -stream is identified by the root Helix job in its `PreviousHelixJobName` lineage -(§5.7), which is stable across monitor resubmissions. - -1. Retry runs exactly once per invocation, on entry, before polling begins. +stream is identified by the submitter chain key (§5.7). The key combines the +stage identity, stable AzDO phase identity, Helix queue, and logical Helix job identity; the +AzDO job name and queue alone are not unique because one AzDO job may submit +multiple independent Helix jobs to the same queue. The combined identity is +stable across both stage attempts and monitor resubmissions. + +1. Retry runs exactly once on entry to a **retried monitor job** + (`System.JobAttempt > 1`). The initial monitor invocation reports failures + but never creates additional Helix work. 2. The set of work to resubmit is decided from a single Helix snapshot taken on entry. Work that fails after the monitor has started is not resubmitted during the current invocation; a later invocation may pick it up. 3. Retry decisions are made per work stream from its **latest incarnation across - all attempts** (the leaf of its lineage chain, breaking ties toward the higher - stage attempt). Let *L* be that incarnation: - - *L* is **still in flight** (running/waiting) and belongs to the **current - attempt** — leave it; the current attempt is actively driving it and - completion gating waits on it. This is the rerun-entire-stage case and also - prevents duplicate submissions when a previous-attempt incarnation of the - same stream is still running. - - *L* is **still in flight** and belongs to a **previous attempt** — the - previous attempt has abandoned it (its monitor is gone and nothing else will - drive it); resubmit the not-yet-passed items into the current attempt. + all attempts**, ordered by stage attempt, submitter job attempt, explicit + lineage depth, and finally Helix job ID. Let *L* be that incarnation: + - *L* belongs to the **current stage attempt** — leave it, whether running or + already failed. It belongs to the execution currently being monitored. + - The current timeline shows the matching submitter at a **higher job + attempt** than *L* — leave *L*. The submitter reran and superseded it; wait + for the newer submitter execution rather than duplicating it. Until the + replacement Helix job becomes visible, *L* is also excluded from current + status and pass/fail reconciliation so stale failures do not leak through + the visibility gap. + - The current timeline submitter attempt **equals** *L*'s + `System.JobAttempt` — the submitter did not rerun. Failed or unfinished + work in *L* may be resubmitted into the current stage attempt. - *L* is **completed and fully passed** — nothing to resubmit; its results are uploaded (if not already, §2.4) and its outcome counted. It is terminal, so it does not block completion. - - *L* is **completed with failures** — resubmit the failed items, regardless of - attempt. (For a current-attempt incarnation this is the pre-existing - per-invocation retry; for a previous-attempt one it carries the failure into - the current attempt.) + - Missing or incompatible submitter identity / `System.JobAttempt` metadata + makes retry classification ambiguous. Do not guess; record an actionable + failure rather than risk duplicating a rerun. - A needed resubmission is **not possible** (e.g. the queue was removed, so the work can never run again) — for previous-attempt in-flight work, whose failure is not otherwise recorded, surface it as an actionable hard failure so the invocation fails fast rather than waiting forever. (Completed-with- failures work that cannot be resubmitted already fails the build via outcome reconciliation, §2.5.) -4. Every resubmission is stamped with the **monitor's current stage attempt** - (not the original job's attempt) and linked back via `PreviousHelixJobName`. +4. Every resubmission is stamped with the **monitor's current stage attempt**, + preserves the original submitter's `System.JobAttempt`, records the + resubmitting monitor attempt as `JobMonitor.JobAttempt`, and links back via + `PreviousHelixJobName`. This is what brings the resubmitted work into current-attempt scope so the monitor gates on it; copying the original attempt would leave the monitor unable to see its own resubmission. @@ -155,8 +166,9 @@ These are the scenarios that a naive "scope strictly to the current attempt and ignore all previous-attempt jobs" design gets wrong, and how the model above addresses each: -1. **Retry-failed-jobs where only the monitor re-ran.** The submitters passed and - were not re-run, so the current attempt contains no Helix work. Naive scoping +1. **Retry-failed-jobs where only the monitor re-ran.** The stage and monitor + attempts advance, but the submitters retain their original job attempts, so + the current stage attempt contains no fresh Helix work. Naive scoping exits `0` immediately, discarding every previous-attempt result and failure. → The retry pass reconciles previous-attempt streams: passed work is uploaded and counted, failed/unfinished work is resubmitted into the current attempt @@ -171,14 +183,28 @@ addresses each: current incarnation) and resubmit them. → Decisions are re-derived from the Helix snapshot each invocation (latest incarnation + attempt + status per stream), not from in-memory state, so partial progress is self-correcting. -4. **Independent jobs with identical submission metadata.** One AzDO job can - submit multiple original Helix jobs to the same queue, so submitter and queue - cannot safely identify a stream. → Only an explicit `PreviousHelixJobName` - lineage collapses jobs; unlinked jobs remain independent (§5.7). -5. **Un-resubmittable work (e.g. purged queue).** Previous-attempt work that can +4. **Visibility race during a full-stage rerun.** The timeline already shows the + submitter at a newer job attempt, but its new Helix job is not visible yet. + Blindly replaying the old job duplicates the rerun. → Timeline job attempt, + not Helix visibility, suppresses replay. +5. **Rerun duplicates that are not lineage-linked.** A stage rerun's fresh Helix + job has no `PreviousHelixJobName` link to its previous-attempt counterpart; + they collapse only by chain key. → Outcome ordering breaks ties toward the + higher stage/job incarnation so the current attempt wins (§5.7). +6. **Un-resubmittable work (e.g. purged queue).** Previous-attempt work that can never run again would loop forever under any "just wait" or "just resubmit and wait" scheme. → Resubmission-not-possible is treated as an actionable hard failure so the invocation fails fast instead of hanging (§2.3.3). +7. **One AzDO job submits multiple Helix jobs to the same queue.** Grouping only + by AzDO job name and queue incorrectly merges independent streams. A failure + in one job can be overwritten by a same-named passing work item in another, + and retry can select only one of the jobs. → The stream key also includes the + submitter-assigned logical Helix `jobName` (falling back to `TestRunName`). + Resubmissions preserve that property, so incarnations of one logical job + still chain while sibling Helix jobs remain independent. +8. **Identical phase/queue/logical-job names in different stages.** → The stage + identity is part of the stream key, so outcomes and retry decisions cannot + cross stage boundaries. ### 2.4 Upload invariants @@ -255,6 +281,7 @@ inputs are: | AzDO collection URI + project | Construct the test-results URL used in failure reports. | | Stage name | Stage scope (see §2.1). | | Stage attempt | Per-attempt scope (see §2.1). Defaults to `SYSTEM_STAGEATTEMPT`; when unknown the monitor tracks jobs from every attempt of the stage. | +| Job attempt | Attempt of the monitor's own AzDO job. Retry reconciliation runs only after attempt 1. Defaults to `SYSTEM_JOBATTEMPT`. | | Polling interval | Delay between poll iterations; a minimum floor applies. | | Maximum wait | Reported in the timeout message; the timeout itself is enforced by the caller through cancellation. | | Job monitor name | Identifier of the monitor's own AzDO timeline record; used to exclude it from pass/fail. | @@ -271,14 +298,19 @@ behaviorally; method names are illustrative. - **List jobs for a build** — given the source filter and build ID, return all Helix jobs that the submitter recorded for the build. The source filter must be derivable from build metadata in lockstep with the - submitter (see §5.1). The returned set spans every attempt of the build; the + submitter (see §5.1). Discovery sends both the source and + `Properties[BuildId]` filters to Helix. Filtering `BuildId` only after + retrieving the source's jobs is prohibited: non-PR sources such as + `refs/heads/main` are long-lived and can contain thousands of historical + jobs. The returned set spans every attempt of the build; the runner keeps the whole stage's jobs (all attempts) so the retry pass can reconcile previous-attempt work (§2.3), and classifies each job as - current- or previous-attempt via `System.StageName` / `System.StageAttempt` - for gating (§2.1). + current- or previous-stage-attempt via `System.StageName` / + `System.StageAttempt`, and compares `System.JobAttempt` with the matching + timeline submitter for retry eligibility (§2.1). - **List work items for a job** — return all work-item summaries. -- **Download test results** — given a job and a set of work-item names, - download recognized result files into a working directory. Individual +- **Download test results** — given one job/work-item pair, download recognized + result files into a working directory. Individual per-file failures must not prevent the remaining files from being attempted. After the batch, transient failures cause the read-only download phase to be retried; permanent failures are logged and omitted. @@ -286,9 +318,10 @@ behaviorally; method names are illustrative. - **Resubmit failed work items** — given the original job and a set of failed (or unfinished) work items, submit a new Helix job that contains only those items. The new job must inherit the original's submitter identity (stage, - job name, display name, test-run name, queue) but be stamped with the - **resubmitting monitor's current stage attempt** (§2.3.4), and link back via - `PreviousHelixJobName`. May return "not possible" (e.g. queue gone), which the + job name, display name, test-run name, queue, and submitter job attempt), be + stamped with the **resubmitting monitor's current stage attempt** and + `JobMonitor.JobAttempt` (§2.3.4), and link back via `PreviousHelixJobName`. + May return "not possible" (e.g. queue gone), which the runner treats as an actionable hard failure for that work rather than silently skipping it (§2.3.3). @@ -326,16 +359,22 @@ or the runner will silently fail to see its own jobs. ### 5.3 Retry pass -1. Take a Helix snapshot of the whole stage (all attempts). +1. Take one current AzDO timeline snapshot and one Helix snapshot of the whole + stage (all attempts). 2. Reduce it to the latest incarnation of each logical work stream (§2.3.3): - the leaf of each lineage chain, keyed by its root Helix job. + the leaf of each lineage chain, keyed by logical stream key, ordered by stage + attempt, submitter job attempt, lineage depth, then Helix job ID. 3. For each latest incarnation, apply §2.3.3: - - Current-attempt incarnation — leave it; it is already being driven. + - Initial monitor job attempt — do not retry anything. + - Current-stage-attempt incarnation — leave it; it belongs to this execution. + - Timeline submitter attempt is newer than the Helix submitter attempt — + leave it; the submitter reran and superseded the old Helix work. - Previous-attempt, completed and fully passed — leave it (terminal); it will still be uploaded / reconciled by the poll loop. - Previous-attempt, completed with failures, or unfinished — ask the Helix service to resubmit the failed / not-yet-passed items, stamped with the - current stage attempt (§2.3.4). If resubmission is not possible, record it + current stage attempt while preserving the submitter job attempt (§2.3.4). + If resubmission is not possible, record it as a hard failure (§2.3.3). 4. Remember the AzDO submitter-job identifiers of successfully retried work; these are the jobs to exclude from the AzDO failure check while this @@ -357,8 +396,8 @@ Each iteration: (so completion/failure transitions are not missed). 4. Compute the set of completed Helix jobs (§5.5). 5. **First pass — upload**: for each completed Helix job not already - uploaded (per §2.2), upload its test results and remember it as - processed. This pass is the only one that triggers uploads. + uploaded (per §2.2), queue its completion-time work-item snapshot into the + bounded upload pipeline. This pass is the only one that triggers uploads. 6. **Second pass — outcome reconciliation**: for every completed Helix job in scope, ensure its per-work-item outcomes are reflected in the running outcome map (§5.7), processing lineage from oldest to newest so @@ -408,21 +447,129 @@ incarnations of the same item collapse onto a single entry. The chain key must be deterministic and uniqueness-preserving: -- Every original Helix job must produce a distinct key, including jobs submitted - by the same AzDO job to the same Helix queue. +- A single AzDO matrix leg that fans out to multiple Helix queues must + produce distinct keys (one per queue) so per-queue failures are + preserved. +- A single AzDO job that submits multiple independent Helix jobs to the same + queue must produce distinct keys (one per logical Helix job). The AzDO + phase/job name and queue are therefore necessary but not sufficient. - An original Helix job and its resubmission(s) on the same queue must produce the same key so the latest incarnation overwrites the older one. -- Jobs not linked by `PreviousHelixJobName` remain distinct. In particular, - submitter name and queue are insufficient to correlate jobs across stage - attempts because one AzDO job may submit multiple jobs to the same queue. -- If lineage cannot be resolved (the predecessor link points outside the - jobs the runner has observed), the referenced predecessor name is used as - the root so later incarnations still resolve consistently. +- The preferred key components are: + 1. `System.StageName`; + 2. `System.PhaseName`, falling back to `System.JobName`; + 3. the Helix queue; + 4. the submitter-assigned Helix `jobName`, falling back to `TestRunName`. + If no stable logical-job discriminator is available, the key is bound to the + root Helix job in the `PreviousHelixJobName` lineage rather than risk merging + unrelated jobs. +- Because these key components are stable across stage attempts, a + rerun-stage incarnation of the same logical job on the same queue collapses + onto the same key as its previous-attempt counterpart, even though the two + Helix jobs are **not** linked by + `PreviousHelixJobName` (only monitor resubmissions set that link). The map + must therefore let the later incarnation win when two incarnations share a + key: outcomes are applied in order of lineage depth, stage attempt, submitter + job attempt, then Helix job ID, rather than by Helix job-name sort alone, or a stale previous-attempt outcome + could nondeterministically overwrite the current one. `System.JobName` is + used only when `System.PhaseName` is unavailable because some pipelines stamp + every matrix job with `System.JobName=__default`. +- If lineage cannot be resolved (the predecessor link points outside the jobs + the runner has observed), the root predecessor name provides the + Helix-job-bound fallback. +- The submitter must assign different `jobName` values (or, when absent, + different `TestRunName` values) to independent Helix jobs submitted by the + same AzDO phase to the same queue. Without a stable distinguishing property, + no monitor can reliably correlate an unlinked stage-rerun job with its prior + incarnation while also distinguishing it from a sibling submission. The same key drives a parallel map of "failed work item console info" used to build the final failure report. When a later incarnation of a work item passes, its entry in that map is cleared. +#### 5.7.1 Stream and attempt examples + +The following examples use one logical stream: + +```text +StreamKey K1 = +( + StageName: Build, + PhaseName: build_windows_x64_Checked_NativeAOT, + QueueId: windows.10.amd64.open.rt, + LogicalJobName: runtime-tests +) + +Original Helix job H1: + System.StageAttempt = 1 + System.JobAttempt = 1 + +Monitor replay R2: + System.StageAttempt = 2 + System.JobAttempt = 1 + JobMonitor.JobAttempt = 2 + PreviousHelixJobName = H1 + +Fresh submitter rerun H2: + System.StageAttempt = 2 + System.JobAttempt = 2 + PreviousHelixJobName is absent +``` + +`System.JobAttempt` on a Helix job always identifies the AzDO job that +originally submitted that logical work. It is compared with the current +timeline attempt of that same submitter. It is **not** compared with the +monitor's own job attempt. `JobMonitor.JobAttempt` records which monitor +invocation created a replay and is diagnostic metadata only. + +`System.PhaseName` maps to the `refName` of the AzDO **Phase** timeline +record. When phase identity is unavailable, `System.JobName` maps to the +`refName` of the nested **Job** record. In matrix pipelines that nested job +name is frequently `__default`, which is why phase identity is preferred. + +#### 5.7.2 Retry and rerun permutation matrix + +| Scenario and observation time | Monitor | Current submitter timeline | Visible Helix work | Required behavior | +| --- | --- | --- | --- | --- | +| Initial execution; H1 fails | S1/M1 | J1 | H1 S1/J1 failed | Report the failure; attempt 1 never creates replay work. | +| Selective retry of only the monitor | S2/M2 | J1 completed | H1 S1/J1 failed | `J1 == J1`; replay H1 as R2. | +| Selective retry also selected the submitter, before H2 is visible | S2/M2 | J2 pending/running | Only H1 S1/J1 | `J2 > J1`; suppress H1 replay, status, and outcome using timeline state, without waiting for H2 visibility. | +| Selective retry after H2 is visible | S2/M2 | J2 | H1 S1/J1 and H2 S2/J2 | H2 supersedes H1; observe H2. | +| Full-stage rerun immediately after timeline creation | S2/M2 | J2 pending | Only H1 S1/J1 | Suppress H1 replay, status, and outcome because the submitter is part of the rerun. | +| Full-stage rerun after H2 submission | S2/M2 | J2 running/completed | H1 S1/J1 and H2 S2/J2 | H2 is authoritative; do not create a monitor replay. | +| Submitter reruns but intentionally submits no new Helix work | S2/M2 | J2 completed | Only H1 S1/J1 | Suppress H1; the newer submitter execution is authoritative. | +| Submitter did not rerun and H1 passed | S2/M2 | J1 | H1 S1/J1 passed | Upload/count H1; no replay. | +| Submitter did not rerun and H1 is failed or unfinished | S2/M2 | J1 | H1 S1/J1 failed/waiting | Replay the failed or unfinished items as R2. | +| R2 fails and the monitor is retried again | S3/M3 | J1 | H1 S1/J1 and R2 S2/J1 failed | R2 is the latest lineage leaf and still matches J1; replay its remaining failures as R3. | +| R2 remains unfinished when the monitor is retried | S3/M3 | J1 | R2 S2/J1 running/waiting | Treat the previous-stage leaf as abandoned and replay its unfinished items as R3. | +| R2 passed before the next monitor invocation | S3/M3 | J1 | R2 S2/J1 passed | Latest incarnation passed; no further replay. | +| Current-stage H2 is running when the monitor starts | S2/M2 | J2 | H2 S2/J2 running | Observe and gate on H2; never replay current-stage work on entry. | +| H2 fails after the monitor has started | S2/M2 | J2 | H2 changes running to failed | Report failure. Retry is entry-only; do not create replay work mid-invocation. | +| Monitor is retried after H2 failed | S3/M3 | J2 | H2 S2/J2 failed | `J2 == J2`; replay H2 into S3. | +| Submitter identity or attempt is missing | S2/M2 | Unknown | H1 S1/J1 failed | Fail safely; do not speculate and risk duplicate work. | +| Timeline attempt is lower than Helix metadata | S2/M2 | J1 | H1 claims S1/J2 | Treat metadata as inconsistent; do not replay. | +| Fresh H2 is visible but is not lineage-linked to H1 | S2/M2 | J2 | H1 S1/J1 and H2 S2/J2 | Collapse by `StreamKey`; the higher stage/job incarnation wins. | +| Monitor replay R2 is linked to H1 | S2/M2 | J1 | H1 and R2 with `Previous=H1` | Collapse explicit lineage to the leaf R2. | + +The central retry decision is therefore: + +```text +current submitter attempt > Helix System.JobAttempt + => the submitter reran; suppress monitor replay + +current submitter attempt == Helix System.JobAttempt + => the submitter did not rerun; failed/unfinished work may be replayed +``` + +#### 5.7.3 Identity-isolation matrix + +| Helix submissions | Example stream keys | Required behavior | +| --- | --- | --- | +| One AzDO job submits two logical jobs to the same queue | `(Build, phaseA, queue1, runtime-tests)` and `(Build, phaseA, queue1, nativeaot-smoke)` | Separate streams because logical job names differ. | +| One AzDO job submits to two queues | `(Build, phaseA, queue1, tests)` and `(Build, phaseA, queue2, tests)` | Separate streams because queues differ. | +| Two AzDO phases use the same queue and logical name | `(Build, phaseA, queue1, tests)` and `(Build, phaseB, queue1, tests)` | Separate streams because phase identities differ. | +| Two stages use identical phase, queue, and logical names | `(Build, phaseA, queue1, tests)` and `(Test, phaseA, queue1, tests)` | Separate streams because stage identities differ. | + ### 5.8 Failure reporting Failed Helix work items must produce clickable console-link warnings in the @@ -439,44 +586,31 @@ Warnings use AzDO `task.logissue type=warning` formatting; the final aggregated error uses `task.logissue type=error`. Informational status lines are plain logger output. -### 5.9 Test-result upload pipeline - -Uploads are asynchronous tasks tracked for normal completion. Their in-memory -lifecycle distinguishes queued, in-progress, durably completed, and failed -uploads; only a completed, tagged test run is considered durable. - -- Each upload is queued asynchronously and tracked. Multiple uploads may - proceed concurrently. -- Test results are downloaded before the AzDO test run is created. Transient - download failures are safe to retry and use a bounded retry budget. -- Test-run creation and completion/tagging are each attempted once. These - lifecycle writes determine the durable upload boundary, so replaying an - ambiguous response could create an extra run or incorrectly mark an - incompletely uploaded run as processed. -- Publishing test results and their attachments uses bounded retries for - throttling, server errors, and transient transport failures. These Azure - DevOps POST APIs do not expose an idempotency key or document deduplication. - A timeout or connection failure can occur after the service commits the - request but before the response reaches the client, so retrying may create - duplicate results or attachments. The design accepts that risk to avoid - losing an entire job's test results after a transient failure. -- Permanent failures and exhausted retries are logged as warnings and stop the - upload task without affecting pass/fail. -- The normal-termination path waits for queued uploads to drain before exiting. -- The cancellation path does not wait for pending or in-flight uploads. If an - upload has not completed and applied its Helix-job tag, it remains untagged; - durable-state discovery causes a later invocation to upload it again. - -The upload sequence per job is: create (or reuse) a test run with the plain -`{TestRunName}`, download results, upload them, complete the test run and tag -it with the Helix job name (`helixjob`). +### 5.9 Test-result durability + +- Terminal work items may publish results before their containing Helix job + completes. +- A Helix job is durably processed only after all admitted results are + published and its Azure DevOps test run is completed and tagged with the + Helix job name. +- A monitor that stops before that durable boundary leaves the job eligible + for replay by a later invocation. +- Failed processed test results contribute to the monitor exit code even when + the Helix work-item exit code is successful. +- Result publication may retry transient failures and therefore accepts the + documented duplicate-result risk rather than losing an entire result set. + +Channel structure, concurrency, retry budgets, and the concrete upload +sequence are implementation details documented in +[Upload pipeline](Components/UploadPipeline.md) and +[Test-result processing](Components/TestResults.md). ### 5.10 Status logging When a status log is due, the runner emits a one-line summary of work -counts (processed / completed / running / waiting jobs and work items). In -verbose mode it additionally emits a tree-style breakdown per job and work -item. The verbose tree is informational only. +counts (processed / completed / running / waiting jobs and work items). +Verbose mode adds bounded pipeline diagnostics (queued/active work and uploaded +result totals) but never emits a per-work-item tree. A Helix job is classified for status purposes as `Processed` (already uploaded), `Completed` (terminal but not yet uploaded), `Running` (has at diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IAzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IAzureDevOpsService.cs index 78838515fd7..94f11c713c7 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IAzureDevOpsService.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IAzureDevOpsService.cs @@ -59,12 +59,11 @@ Task CompleteTestRunAsync( CancellationToken cancellationToken); /// - /// Uploads test results for the specified work items into an existing test run. - /// Returns a dictionary mapping each work item and job name to its upload summary. + /// Uploads one work item's test results into an existing test run. /// - Task> UploadTestResultsAsync( + Task UploadTestResultsAsync( int testRunId, - IReadOnlyList results, + WorkItemTestResults results, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs index 2cc44abacdf..bd74546bbe7 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs @@ -16,10 +16,10 @@ public interface IHelixService { /// /// Returns Helix jobs associated with the current build/stage. - /// Implementations should query Helix using the given - /// filter (which scopes the query to the repo/branch/PR the build is for, mirroring - /// what the Helix job submitter records on each submission) and then narrow the - /// result to jobs stamped with . + /// Implementations should query Helix using both the given + /// and the job property BuildId=. The build property + /// must be filtered by the service rather than by retrieving every job for a long-lived + /// branch source and narrowing the result locally. /// Task> GetJobsForBuildAsync( string source, @@ -32,9 +32,9 @@ Task> GetJobsForBuildAsync( /// Work items without recognizable test result files may be omitted from the result. /// Individual file download failures should not prevent other result files from being downloaded. /// - Task> DownloadTestResultsAsync( + Task DownloadTestResultsAsync( string jobName, - IReadOnlyCollection workItemNames, + string workItemName, string workingDirectory, CancellationToken cancellationToken); /// @@ -58,14 +58,15 @@ Task CancelJobAsync( /// possible (e.g. the original queue no longer exists). /// The new job must preserve BuildId and StageName properties so it is discoverable by /// GetJobsForBuildAsync, and must be stamped with - /// (the resubmitting monitor's own stage attempt) rather than the original job's attempt, - /// so the monitor gates on its own resubmission. When - /// is null/empty the original job's attempt is preserved (build + stage back-compat). + /// (the resubmitting monitor's own stage attempt) so the monitor gates on its own + /// resubmission. It preserves the original submitter's System.JobAttempt and records + /// separately for diagnostics. /// Task ResubmitWorkItemsAsync( HelixJobInfo originalJob, IReadOnlyCollection failedWorkItems, string targetStageAttempt, + string monitorJobAttempt, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorMetrics.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorMetrics.cs new file mode 100644 index 00000000000..5a116e90c44 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorMetrics.cs @@ -0,0 +1,310 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; + +namespace Microsoft.DotNet.Helix.JobMonitor; + +internal enum AzureDevOpsRequestKind +{ + Control, + ResultBatch, + Attachment, +} + +internal enum PipelineOperation +{ + WorkItemDownload, + WorkItemPublish, + TestRunCreate, + TestRunComplete, + ResultParseAndAggregate, +} + +internal sealed class JobMonitorMetrics +{ + private readonly long _startedAt = Stopwatch.GetTimestamp(); + private long _pipelineStartedAt; + private long _pipelineFinishedAt; + private long _azdoControlRequests; + private long _azdoResultRequests; + private long _azdoAttachmentRequests; + private long _azdoRetries; + private long _azdoFailedAttempts; + private long _azdoPayloadBytes; + private long _azdoRequestTicks; + private long _azdoMaximumRequestTicks; + private long _helixRequests; + private long _helixRetries; + private long _helixFailedAttempts; + private long _resultBlobDownloads; + private long _resultBlobDownloadFailures; + private long _rateLimitWaits; + private long _rateLimitWaitTicks; + private long _maximumRateLimitWaitTicks; + private long _rateLimitDeferrals; + private long _rateLimitDeferredTicks; + private long _maximumRateLimitDeferralTicks; + private long _workItemDownloads; + private long _workItemDownloadTicks; + private long _maximumWorkItemDownloadTicks; + private long _workItemPublishes; + private long _workItemPublishTicks; + private long _maximumWorkItemPublishTicks; + private long _testRunCreates; + private long _testRunCreateTicks; + private long _maximumTestRunCreateTicks; + private long _testRunCompletes; + private long _testRunCompleteTicks; + private long _maximumTestRunCompleteTicks; + private long _parseOperations; + private long _parseTicks; + private long _maximumParseTicks; + + public static long StartOperation() => Stopwatch.GetTimestamp(); + + public void RecordAzureDevOpsRequest( + AzureDevOpsRequestKind kind, + int payloadBytes, + bool isRetry, + bool failed, + long startedAt) + { + switch (kind) + { + case AzureDevOpsRequestKind.Control: + Interlocked.Increment(ref _azdoControlRequests); + break; + case AzureDevOpsRequestKind.ResultBatch: + Interlocked.Increment(ref _azdoResultRequests); + break; + case AzureDevOpsRequestKind.Attachment: + Interlocked.Increment(ref _azdoAttachmentRequests); + break; + } + + if (isRetry) + { + Interlocked.Increment(ref _azdoRetries); + } + if (failed) + { + Interlocked.Increment(ref _azdoFailedAttempts); + } + + Interlocked.Add(ref _azdoPayloadBytes, payloadBytes); + RecordElapsed(ref _azdoRequestTicks, ref _azdoMaximumRequestTicks, startedAt); + } + + public void RecordHelixRequest(bool isRetry, bool failed) + { + Interlocked.Increment(ref _helixRequests); + if (isRetry) + { + Interlocked.Increment(ref _helixRetries); + } + if (failed) + { + Interlocked.Increment(ref _helixFailedAttempts); + } + } + + public void RecordResultBlobDownload(bool failed) + { + Interlocked.Increment(ref _resultBlobDownloads); + if (failed) + { + Interlocked.Increment(ref _resultBlobDownloadFailures); + } + } + + public void RecordRateLimitWait(long startedAt) + { + Interlocked.Increment(ref _rateLimitWaits); + RecordElapsed(ref _rateLimitWaitTicks, ref _maximumRateLimitWaitTicks, startedAt); + } + + public void RecordRateLimitDeferral(TimeSpan delay) + { + Interlocked.Increment(ref _rateLimitDeferrals); + long delayTicks = (long)(delay.TotalSeconds * Stopwatch.Frequency); + Interlocked.Add(ref _rateLimitDeferredTicks, delayTicks); + + long observed; + while (delayTicks > (observed = Interlocked.Read(ref _maximumRateLimitDeferralTicks))) + { + if (Interlocked.CompareExchange( + ref _maximumRateLimitDeferralTicks, + delayTicks, + observed) == observed) + { + break; + } + } + } + + public void RecordPipelineOperation(PipelineOperation operation, long startedAt) + { + RecordPipelineActivity(startedAt); + switch (operation) + { + case PipelineOperation.WorkItemDownload: + Interlocked.Increment(ref _workItemDownloads); + RecordElapsed(ref _workItemDownloadTicks, ref _maximumWorkItemDownloadTicks, startedAt); + break; + case PipelineOperation.WorkItemPublish: + Interlocked.Increment(ref _workItemPublishes); + RecordElapsed(ref _workItemPublishTicks, ref _maximumWorkItemPublishTicks, startedAt); + break; + case PipelineOperation.TestRunCreate: + Interlocked.Increment(ref _testRunCreates); + RecordElapsed(ref _testRunCreateTicks, ref _maximumTestRunCreateTicks, startedAt); + break; + case PipelineOperation.TestRunComplete: + Interlocked.Increment(ref _testRunCompletes); + RecordElapsed(ref _testRunCompleteTicks, ref _maximumTestRunCompleteTicks, startedAt); + break; + case PipelineOperation.ResultParseAndAggregate: + Interlocked.Increment(ref _parseOperations); + RecordElapsed(ref _parseTicks, ref _maximumParseTicks, startedAt); + break; + } + } + + public JobMonitorMetricsSnapshot Snapshot() + { + long controlRequests = Interlocked.Read(ref _azdoControlRequests); + long resultRequests = Interlocked.Read(ref _azdoResultRequests); + long attachmentRequests = Interlocked.Read(ref _azdoAttachmentRequests); + return new JobMonitorMetricsSnapshot( + Elapsed: Stopwatch.GetElapsedTime(_startedAt), + PipelineElapsed: GetPipelineElapsed(), + AzureDevOpsRequests: controlRequests + resultRequests + attachmentRequests, + AzureDevOpsControlRequests: controlRequests, + AzureDevOpsResultRequests: resultRequests, + AzureDevOpsAttachmentRequests: attachmentRequests, + AzureDevOpsRetries: Interlocked.Read(ref _azdoRetries), + AzureDevOpsFailedAttempts: Interlocked.Read(ref _azdoFailedAttempts), + AzureDevOpsPayloadBytes: Interlocked.Read(ref _azdoPayloadBytes), + AzureDevOpsRequestTime: GetElapsed(_azdoRequestTicks), + MaximumAzureDevOpsRequestTime: GetElapsed(_azdoMaximumRequestTicks), + HelixRequests: Interlocked.Read(ref _helixRequests), + HelixRetries: Interlocked.Read(ref _helixRetries), + HelixFailedAttempts: Interlocked.Read(ref _helixFailedAttempts), + ResultBlobDownloads: Interlocked.Read(ref _resultBlobDownloads), + ResultBlobDownloadFailures: Interlocked.Read(ref _resultBlobDownloadFailures), + RateLimitWaits: Interlocked.Read(ref _rateLimitWaits), + RateLimitWaitTime: GetElapsed(_rateLimitWaitTicks), + MaximumRateLimitWaitTime: GetElapsed(_maximumRateLimitWaitTicks), + RateLimitDeferrals: Interlocked.Read(ref _rateLimitDeferrals), + RateLimitDeferredTime: GetElapsed(_rateLimitDeferredTicks), + MaximumRateLimitDeferral: GetElapsed(_maximumRateLimitDeferralTicks), + WorkItemDownloads: Interlocked.Read(ref _workItemDownloads), + WorkItemDownloadTime: GetElapsed(_workItemDownloadTicks), + MaximumWorkItemDownloadTime: GetElapsed(_maximumWorkItemDownloadTicks), + WorkItemPublishes: Interlocked.Read(ref _workItemPublishes), + WorkItemPublishTime: GetElapsed(_workItemPublishTicks), + MaximumWorkItemPublishTime: GetElapsed(_maximumWorkItemPublishTicks), + TestRunCreates: Interlocked.Read(ref _testRunCreates), + TestRunCreateTime: GetElapsed(_testRunCreateTicks), + MaximumTestRunCreateTime: GetElapsed(_maximumTestRunCreateTicks), + TestRunCompletes: Interlocked.Read(ref _testRunCompletes), + TestRunCompleteTime: GetElapsed(_testRunCompleteTicks), + MaximumTestRunCompleteTime: GetElapsed(_maximumTestRunCompleteTicks), + ParseOperations: Interlocked.Read(ref _parseOperations), + ParseTime: GetElapsed(_parseTicks), + MaximumParseTime: GetElapsed(_maximumParseTicks)); + } + + private static void RecordElapsed(ref long totalTicks, ref long maximumTicks, long startedAt) + { + long elapsedTicks = Stopwatch.GetTimestamp() - startedAt; + Interlocked.Add(ref totalTicks, elapsedTicks); + + long observed; + while (elapsedTicks > (observed = Interlocked.Read(ref maximumTicks))) + { + if (Interlocked.CompareExchange(ref maximumTicks, elapsedTicks, observed) == observed) + { + break; + } + } + } + + private static TimeSpan GetElapsed(long stopwatchTicks) + => TimeSpan.FromSeconds((double)stopwatchTicks / Stopwatch.Frequency); + + private void RecordPipelineActivity(long startedAt) + { + long observedStart; + while ((observedStart = Interlocked.Read(ref _pipelineStartedAt)) == 0 + || startedAt < observedStart) + { + if (Interlocked.CompareExchange(ref _pipelineStartedAt, startedAt, observedStart) == observedStart) + { + break; + } + } + + long finishedAt = Stopwatch.GetTimestamp(); + long observedFinish; + while (finishedAt > (observedFinish = Interlocked.Read(ref _pipelineFinishedAt))) + { + if (Interlocked.CompareExchange(ref _pipelineFinishedAt, finishedAt, observedFinish) == observedFinish) + { + break; + } + } + } + + private TimeSpan GetPipelineElapsed() + { + long startedAt = Interlocked.Read(ref _pipelineStartedAt); + if (startedAt == 0) + { + return TimeSpan.Zero; + } + + long finishedAt = Math.Max(startedAt, Interlocked.Read(ref _pipelineFinishedAt)); + return TimeSpan.FromSeconds((double)(finishedAt - startedAt) / Stopwatch.Frequency); + } +} + +internal readonly record struct JobMonitorMetricsSnapshot( + TimeSpan Elapsed, + TimeSpan PipelineElapsed, + long AzureDevOpsRequests, + long AzureDevOpsControlRequests, + long AzureDevOpsResultRequests, + long AzureDevOpsAttachmentRequests, + long AzureDevOpsRetries, + long AzureDevOpsFailedAttempts, + long AzureDevOpsPayloadBytes, + TimeSpan AzureDevOpsRequestTime, + TimeSpan MaximumAzureDevOpsRequestTime, + long HelixRequests, + long HelixRetries, + long HelixFailedAttempts, + long ResultBlobDownloads, + long ResultBlobDownloadFailures, + long RateLimitWaits, + TimeSpan RateLimitWaitTime, + TimeSpan MaximumRateLimitWaitTime, + long RateLimitDeferrals, + TimeSpan RateLimitDeferredTime, + TimeSpan MaximumRateLimitDeferral, + long WorkItemDownloads, + TimeSpan WorkItemDownloadTime, + TimeSpan MaximumWorkItemDownloadTime, + long WorkItemPublishes, + TimeSpan WorkItemPublishTime, + TimeSpan MaximumWorkItemPublishTime, + long TestRunCreates, + TimeSpan TestRunCreateTime, + TimeSpan MaximumTestRunCreateTime, + long TestRunCompletes, + TimeSpan TestRunCompleteTime, + TimeSpan MaximumTestRunCompleteTime, + long ParseOperations, + TimeSpan ParseTime, + TimeSpan MaximumParseTime); diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs index 01ab7c203c8..b84dc9d37f8 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs @@ -55,7 +55,13 @@ public sealed class JobMonitorOptions /// public string StageAttempt { get; set; } - public int TestResultUploadParallelism { get; set; } = 4; + /// + /// Attempt number of the Azure DevOps monitor job. Retry reconciliation only runs when + /// this is greater than one. Defaults to the SYSTEM_JOBATTEMPT environment variable. + /// + public string JobAttempt { get; set; } + + public int TestResultUploadParallelism { get; set; } = 48; public TestResultAttachmentMode TestResultAttachmentMode { get; set; } = TestResultAttachmentMode.Failed; @@ -163,10 +169,15 @@ public static JobMonitorOptions Parse(string[] args) Description = "Attempt number of the Azure DevOps pipeline stage the monitor is running in. Used to scope monitoring to Helix jobs submitted by the current stage attempt so retries do not re-discover a previous attempt's work. Defaults to the SYSTEM_STAGEATTEMPT environment variable." }; + Option jobAttemptOption = new("--job-attempt") + { + Description = "Attempt number of the Azure DevOps monitor job. Retry reconciliation only runs after the initial job attempt. Defaults to the SYSTEM_JOBATTEMPT environment variable." + }; + Option testResultUploadParallelismOption = new("--test-result-upload-parallelism") { Description = "Maximum number of work items whose test results can be uploaded to Azure DevOps in parallel.", - DefaultValueFactory = _ => 4 + DefaultValueFactory = _ => 48 }; Option testResultAttachmentModeOption = new("--test-result-attachment-mode") @@ -216,6 +227,7 @@ public static JobMonitorOptions Parse(string[] args) rootCommand.Options.Add(workingDirectoryOption); rootCommand.Options.Add(stageNameOption); rootCommand.Options.Add(stageAttemptOption); + rootCommand.Options.Add(jobAttemptOption); rootCommand.Options.Add(testResultUploadParallelismOption); rootCommand.Options.Add(testResultAttachmentModeOption); rootCommand.Options.Add(failWorkItemsWithFailedTestsOption); @@ -241,6 +253,7 @@ public static JobMonitorOptions Parse(string[] args) WorkingDirectory = parseResult.GetValue(workingDirectoryOption), StageName = parseResult.GetValue(stageNameOption), StageAttempt = parseResult.GetValue(stageAttemptOption), + JobAttempt = parseResult.GetValue(jobAttemptOption), TestResultUploadParallelism = parseResult.GetValue(testResultUploadParallelismOption), TestResultAttachmentMode = parseResult.GetValue(testResultAttachmentModeOption), FailWorkItemsWithFailedTests = parseResult.GetValue(failWorkItemsWithFailedTestsOption), @@ -283,6 +296,7 @@ private void ApplyEnvironmentDefaults() SourceBranch ??= Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH"); StageName ??= Environment.GetEnvironmentVariable("SYSTEM_STAGENAME"); StageAttempt ??= Environment.GetEnvironmentVariable("SYSTEM_STAGEATTEMPT"); + JobAttempt ??= Environment.GetEnvironmentVariable("SYSTEM_JOBATTEMPT"); } private void Validate() diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index ed0d3e674d0..95ca6bf8375 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -10,12 +10,13 @@ using Microsoft.DotNet.Helix.Client; using Microsoft.DotNet.Helix.Client.Models; using Microsoft.DotNet.Helix.JobMonitor.Models; +using Microsoft.DotNet.Helix.JobMonitor.Parallelism; using Microsoft.Extensions.Logging; namespace Microsoft.DotNet.Helix.JobMonitor { /// - /// Orchestrates the per-invocation lifecycle described in JobMonitorRunner.Design.md: + /// Orchestrates the per-invocation lifecycle described in Design/SemanticBehavior.md: /// one-shot retry pass, poll loop (with upload + outcome reconciliation per iteration), /// final summary on completion, and timeout/cancel handling. All heavy lifting /// (status logging, uploads, state) lives in dedicated helpers. @@ -29,24 +30,35 @@ internal sealed class JobMonitorRunner : IJobMonitorRunner, IDisposable private readonly IAzureDevOpsService _azdo; private readonly IHelixService _helix; private readonly Func _delayFunc; + private readonly Func _statusDelayFunc; private readonly string _helixSource; private readonly MonitorState _state = new(); + private readonly JobMonitorMetrics _metrics; private readonly StatusReporter _reporter; - private readonly TestResultUploadQueue _uploads; + private readonly TestResultUploadPipeline _uploads; + private PollStatusSnapshot _latestStatus; /// /// Constructor for production use with real services. /// public JobMonitorRunner(JobMonitorOptions options, ILogger logger) - : this(options, - logger, - new AzureDevOpsService(options, logger), - new HelixService(string.IsNullOrEmpty(options.HelixAccessToken) - ? ApiFactory.GetAnonymous(options.HelixBaseUri) - : ApiFactory.GetAuthenticated(options.HelixBaseUri, options.HelixAccessToken), - logger), - delayFunc: null) + : this(options, logger, CreateProductionDependencies(options, logger)) + { + } + + private JobMonitorRunner( + JobMonitorOptions options, + ILogger logger, + ProductionDependencies dependencies) + : this( + options, + logger, + dependencies.AzureDevOps, + dependencies.Helix, + delayFunc: null, + statusDelayFunc: null, + metrics: dependencies.Metrics) { } @@ -58,13 +70,17 @@ internal JobMonitorRunner( ILogger logger, IAzureDevOpsService azdo, IHelixService helix, - Func delayFunc) + Func delayFunc, + Func statusDelayFunc = null, + JobMonitorMetrics metrics = null) { _options = options ?? throw new ArgumentNullException(nameof(options)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _azdo = azdo ?? throw new ArgumentNullException(nameof(azdo)); _helix = helix ?? throw new ArgumentNullException(nameof(helix)); _delayFunc = delayFunc ?? Task.Delay; + _statusDelayFunc = statusDelayFunc ?? Task.Delay; + _metrics = metrics ?? new JobMonitorMetrics(); Directory.CreateDirectory(_options.WorkingDirectory); _helixSource = HelixJobSource.Compute( @@ -74,8 +90,14 @@ internal JobMonitorRunner( _options.RepositoryName, _options.SourceBranch); - _reporter = new StatusReporter(_logger, _options, _helix, _state); - _uploads = new TestResultUploadQueue(_logger, _options, _azdo, _helix, _state); + _reporter = new StatusReporter(_logger, _options, _state); + _uploads = new TestResultUploadPipeline( + _logger, + _options, + _azdo, + _helix, + _state, + _metrics); } public async Task RunAsync(CancellationToken cancellationToken) @@ -84,13 +106,21 @@ public async Task RunAsync(CancellationToken cancellationToken) _state.AddProcessedHelixJobs(await _azdo.GetProcessedHelixJobNamesAsync(cancellationToken)); + using var statusCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + Task statusTask = ReportStatusPeriodicallyAsync(statusCts.Token); try { - IReadOnlyList jobsForFirstPoll = await ExecuteRetryPassAsync(cancellationToken); - return await RunPollLoopAsync(jobsForFirstPoll, cancellationToken); + IReadOnlyList timelineForFirstPoll = + HelixJobMonitorUtilities.FilterRecordsToStage( + await _azdo.GetTimelineRecordsAsync(cancellationToken), + _options.StageName); + IReadOnlyList jobsForFirstPoll = + await ExecuteRetryPassAsync(timelineForFirstPoll, cancellationToken); + return await RunPollLoopAsync(jobsForFirstPoll, timelineForFirstPoll, cancellationToken); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { + _uploads.Cancel(); // On cancellation (AzDO job timeout or build cancellation) the agent grants only a // few seconds before force-killing the process, so cancelling the in-flight Helix // jobs is the priority: do it immediately rather than waiting for the test-result @@ -100,7 +130,7 @@ public async Task RunAsync(CancellationToken cancellationToken) // treated as "processed" once their Azure DevOps test run reaches the Completed // state (the final upload step), so a job whose upload did not finish here is // re-uploaded in full by a subsequent monitor invocation. See - // JobMonitorRunner.Design.md ("Crash and timeout resilience"). + // Design/SemanticBehavior.md ("Crash and timeout resilience"). _reporter.ReportTimeout(); // Proactively cancel any Helix jobs we know about that haven't finished yet so @@ -109,21 +139,38 @@ public async Task RunAsync(CancellationToken cancellationToken) // cancelled. using var cancelCts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await CancelInFlightHelixJobsAsync(cancelCts.Token); + _reporter.LogPerformanceMetrics( + _metrics.Snapshot(), + _uploads.Snapshot, + isPartial: true); return 1; } + finally + { + statusCts.Cancel(); + try + { + await statusTask; + } + catch (OperationCanceledException) when (statusCts.IsCancellationRequested) + { + } + } } /// /// One-shot retry pass executed on entry. Reconciles the whole stage's Helix work /// (all attempts) into the current stage attempt: for each logical work stream it takes /// the latest incarnation and, when that incarnation is a previous attempt's failed or - /// unfinished work (or a current attempt's completed-with-failures work), resubmits the - /// not-yet-passed items into the current attempt. Returns the (stage snapshot ∪ + /// unfinished work whose submitter did not rerun, resubmits the not-yet-passed items into + /// the current attempt. Returns the (stage snapshot ∪ /// resubmitted jobs) so the first poll iteration sees the resubmissions immediately. - /// See JobMonitorRunner.Design.md §2.1 and §2.3. + /// See Design/SemanticBehavior.md §2.1 and §2.3. /// - private async Task> ExecuteRetryPassAsync(CancellationToken cancellationToken) + private async Task> ExecuteRetryPassAsync( + IReadOnlyList timelineRecords, + CancellationToken cancellationToken) { _reporter.LogRetryPassStart(); @@ -139,6 +186,16 @@ private async Task> ExecuteRetryPassAsync(Cancellati // Seed the cross-poll cache so PreviousHelixJobName walks resolve to the root Helix // job while grouping streams below. _state.ObserveJobs(stageJobs); + _state.SetTimelineRecords(timelineRecords); + + // The initial monitor invocation observes and reports failures; it does not create + // additional Helix work. Retry reconciliation is only meaningful after AzDO has + // retried the monitor job. + if (MonitorState.ParseJobAttempt(_options.JobAttempt) <= 1) + { + _reporter.LogRetryPassFoundNothing(); + return stageJobs; + } // Surfacing work items that passed by exit code but whose AzDO test results contain // failures: a prior monitor invocation may have uploaded failed tests for a job @@ -158,13 +215,33 @@ private async Task> ExecuteRetryPassAsync(Cancellati { bool previousAttempt = IsPreviousAttempt(latest); - // A current-attempt incarnation that is still in flight is gated on, not - // resubmitted. - if (!previousAttempt && !latest.IsCompleted) + // Work created in this stage attempt belongs to the current execution, whether + // it is running or has already failed. It is observed, not retried again on entry. + if (!previousAttempt) { continue; } + bool hasSubmitterAttempt = TryGetCurrentSubmitterAttempt( + latest, + timelineRecords, + out int currentSubmitterAttempt, + out string submitterIdentity); + int helixSubmitterAttempt = MonitorState.ParseJobAttempt(latest.JobAttempt); + + // The submitter itself reran. Its old Helix work is superseded even when the new + // Helix job has not become visible yet; resubmitting here would duplicate a full + // stage rerun or a selected failed-job retry. + if (IsSupersededBySubmitterRerun( + latest, + hasSubmitterAttempt, + currentSubmitterAttempt, + helixSubmitterAttempt)) + { + _state.MarkSupersededBySubmitterRerun(latest.JobName); + continue; + } + IReadOnlyCollection jobWorkItems = await _helix.ListWorkItemsAsync(latest.JobName, cancellationToken); @@ -188,6 +265,25 @@ private async Task> ExecuteRetryPassAsync(Cancellati continue; } + if (!hasSubmitterAttempt + || string.IsNullOrEmpty(latest.JobAttempt) + || currentSubmitterAttempt < helixSubmitterAttempt) + { + IReadOnlyCollection ambiguousWork = + [ + ..exitCodeFailures + .Concat(testOnlyFailures) + .DistinctBy(wi => wi.Name, StringComparer.OrdinalIgnoreCase) + ]; + _state.RecordAbandonedWork(latest, ambiguousWork); + LogWarning( + $"Cannot safely reconcile {ambiguousWork.Count} failed/unfinished work item(s) from " + + $"{latest.DisplayName}: submitter '{submitterIdentity ?? ""}' could not be " + + $"matched to compatible System.JobAttempt metadata in the current timeline. " + + "The work was not resubmitted to avoid duplicating a rerun."); + continue; + } + _reporter.LogRetryPassResubmission(latest, exitCodeFailures, testOnlyFailures); // exitCodeFailures and testOnlyFailures are disjoint by construction (one @@ -201,7 +297,7 @@ private async Task> ExecuteRetryPassAsync(Cancellati ]; HelixJobInfo resubmitted = await _helix.ResubmitWorkItemsAsync( - latest, failedWorkItems, _options.StageAttempt, cancellationToken); + latest, failedWorkItems, _options.StageAttempt, _options.JobAttempt, cancellationToken); if (resubmitted is null) { // Previous-attempt work that can never run again (e.g. its queue was removed) @@ -220,7 +316,9 @@ private async Task> ExecuteRetryPassAsync(Cancellati } resubmittedJobs.Add(resubmitted); - _state.RecordResubmission(latest.SubmitterJobName, failedWorkItems.Count); + _state.RecordResubmission( + latest.SubmitterPhaseName ?? latest.SubmitterJobName, + failedWorkItems.Count); } if (resubmittedJobs.Count == 0) @@ -231,7 +329,10 @@ private async Task> ExecuteRetryPassAsync(Cancellati return [.. stageJobs, .. resubmittedJobs]; } - private async Task RunPollLoopAsync(IReadOnlyList jobsForFirstPoll, CancellationToken cancellationToken) + private async Task RunPollLoopAsync( + IReadOnlyList jobsForFirstPoll, + IReadOnlyList timelineForFirstPoll, + CancellationToken cancellationToken) { var loopState = new PollLoopState(); @@ -239,8 +340,13 @@ private async Task RunPollLoopAsync(IReadOnlyList jobsForFirs { cancellationToken.ThrowIfCancellationRequested(); - int? exitCode = await PollOnceAsync(jobsForFirstPoll, loopState, cancellationToken); + int? exitCode = await PollOnceAsync( + jobsForFirstPoll, + timelineForFirstPoll, + loopState, + cancellationToken); jobsForFirstPoll = null; // first-poll seed is consumed + timelineForFirstPoll = null; if (exitCode.HasValue) { @@ -257,12 +363,16 @@ private async Task RunPollLoopAsync(IReadOnlyList jobsForFirs /// private async Task PollOnceAsync( IReadOnlyList jobsForFirstPoll, + IReadOnlyList timelineForFirstPoll, PollLoopState loopState, CancellationToken cancellationToken) { + int pollNumber = ++loopState.PollNumber; + // Fetch fresh snapshots, scoped to the monitor's stage. IReadOnlyList timelineRecords = - HelixJobMonitorUtilities.FilterRecordsToStage( + timelineForFirstPoll + ?? HelixJobMonitorUtilities.FilterRecordsToStage( await _azdo.GetTimelineRecordsAsync(cancellationToken), _options.StageName); @@ -287,42 +397,100 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken), _state.SetTimelineRecords(timelineRecords); _state.ObserveJobs(stageJobs); + // A rerun submitter supersedes its previous Helix stream before the replacement + // Helix job is necessarily visible. Keep those stale incarnations available for + // durable upload, but exclude them from current status and pass/fail reconciliation. + IReadOnlyList authoritativeJobs = + [ + .._state.GetLatestIncarnationPerStream(stageJobs) + .Where(job => !_state.IsSupersededBySubmitterRerun(job.JobName)) + ]; + var authoritativeJobNames = new HashSet( + authoritativeJobs.Select(static job => job.JobName), + StringComparer.OrdinalIgnoreCase); + // Helix job summaries can omit Finished for failed jobs even after all work // items have terminal exit codes, so fall back to per-work-item status. - IReadOnlyCollection completedJobs = await GetCompletedJobsAsync(stageJobs, cancellationToken); + IReadOnlyList jobsToRefresh = + [ + ..stageJobs.Where(job => + !_state.IsWorkItemOutcomesRecorded(job.JobName) + || !loopState.WorkItemsByJob.ContainsKey(job.JobName)) + ]; + IReadOnlyDictionary> refreshedWorkItems = + await GetWorkItemsAsync(jobsToRefresh, cancellationToken); + int newlyTerminalWorkItems = 0; + foreach ((string jobName, IReadOnlyCollection workItems) in refreshedWorkItems) + { + int previousTerminalCount = + loopState.WorkItemsByJob.TryGetValue(jobName, out IReadOnlyCollection previousWorkItems) + ? previousWorkItems.Count(static item => item.ExitCode.HasValue) + : 0; + int currentTerminalCount = workItems.Count(static item => item.ExitCode.HasValue); + // A Helix work-item exit code is immutable once assigned, so terminal counts + // increase monotonically without retaining or rebuilding per-item identity sets. + newlyTerminalWorkItems += Math.Max(0, currentTerminalCount - previousTerminalCount); + loopState.WorkItemsByJob[jobName] = workItems; + } + + IReadOnlyDictionary> workItemsByJob = + stageJobs.ToDictionary( + static job => job.JobName, + job => loopState.WorkItemsByJob[job.JobName], + StringComparer.OrdinalIgnoreCase); + IReadOnlyCollection completedJobs = GetCompletedJobs(stageJobs, workItemsByJob); var completedJobNames = new HashSet( completedJobs.Select(j => j.JobName), StringComparer.OrdinalIgnoreCase); - // First pass: upload + reconcile for any newly-completed jobs. + // Upload terminal work items as soon as they are visible, even while their Helix job + // is still running. Large jobs can finish hundreds of work items long before their + // final job transition; deferring those uploads creates an avoidable final drain. + foreach (HelixJobInfo job in stageJobs.Where(job => !_state.IsHelixJobProcessed(job.JobName))) + { + _uploads.TryEnqueue( + job, + workItemsByJob[job.JobName], + completedJobNames.Contains(job.JobName), + pollNumber); + } + + // First pass: reconcile outcomes for any newly-completed jobs. foreach (HelixJobInfo job in completedJobs.Where(j => !_state.IsHelixJobProcessed(j.JobName))) { - await ReconcileCompletedJobAsync(job, queueUpload: true, cancellationToken); + ReconcileCompletedJob( + job, + workItemsByJob[job.JobName], + queueUpload: false, + recordOutcomes: authoritativeJobNames.Contains(job.JobName), + discoveryPoll: pollNumber); } // Second pass: ensure outcomes for every completed job (any attempt) are reflected in // the running outcome map (oldest incarnation first, so linked resubmissions // supersede their predecessors). Idempotent — already-reconciled jobs early-return. foreach (HelixJobInfo job in MonitorState.OrderHelixJobsOldToNew( - MonitorState.GetLatestHelixJobAttempts(stageJobs) + MonitorState.GetLatestHelixJobAttempts(authoritativeJobs) .Where(j => completedJobNames.Contains(j.JobName)))) { - await ReconcileCompletedJobAsync(job, queueUpload: false, cancellationToken); + ReconcileCompletedJob( + job, + workItemsByJob[job.JobName], + queueUpload: false, + recordOutcomes: true, + discoveryPoll: pollNumber); } - _uploads.Prune(); - - bool shouldLogStatus = _options.Verbose - || loopState.LastObservedJobCount != stageJobs.Count - || loopState.LastObservedCompletedCount != completedJobs.Count - || (DateTime.UtcNow - loopState.LastStatusLogAt) >= TimeSpan.FromMinutes(5); - - if (shouldLogStatus) + var authoritativeCompletedJobNames = new HashSet( + authoritativeJobNames.Where(completedJobNames.Contains), + StringComparer.OrdinalIgnoreCase); + Volatile.Write( + ref _latestStatus, + new PollStatusSnapshot(authoritativeJobs, workItemsByJob, authoritativeCompletedJobNames)); + if (!loopState.HasLoggedInitialStatus) { - await _reporter.LogPollStatusAsync(stageJobs, completedJobNames, cancellationToken); - loopState.LastObservedJobCount = stageJobs.Count; - loopState.LastObservedCompletedCount = completedJobNames.Count; - loopState.LastStatusLogAt = DateTime.UtcNow; + LogLatestStatus(); + loopState.HasLoggedInitialStatus = true; } bool anyNonMonitorFailure = HelixJobMonitorUtilities.HasFailedNonMonitorJobs( @@ -337,7 +505,8 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken), return null; } - await _uploads.DrainAsync(cancellationToken); + await _uploads.DrainAsync(pollNumber, newlyTerminalWorkItems, cancellationToken); + _reporter.LogPerformanceMetrics(_metrics.Snapshot(), _uploads.Snapshot); _reporter.LogFinalFailedWorkItems(); _reporter.LogFinalSummary(_state.AssociatedJobsCount); @@ -361,10 +530,12 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken), /// queues a test-result upload. Idempotent: a second call without /// early-returns if the outcomes were already recorded. /// - private async Task ReconcileCompletedJobAsync( + private void ReconcileCompletedJob( HelixJobInfo helixJob, + IReadOnlyCollection workItems, bool queueUpload, - CancellationToken cancellationToken) + bool recordOutcomes, + int discoveryPoll) { // Already reconciled earlier in this invocation — nothing more to do (idempotent). if (_state.IsWorkItemOutcomesRecorded(helixJob.JobName)) @@ -372,9 +543,6 @@ private async Task ReconcileCompletedJobAsync( return; } - IReadOnlyCollection workItems = - await _helix.ListWorkItemsAsync(helixJob.JobName, cancellationToken); - // A previous monitor attempt for the same build already uploaded this job's results // (tracked via IsHelixJobProcessed, seeded on entry from the AzDO test-run tags). Its // work-item outcomes must still be reconciled so the final exit code accounts for @@ -383,36 +551,44 @@ private async Task ReconcileCompletedJobAsync( // completion / console-link logs are suppressed for such jobs. bool alreadyUploadedByPriorAttempt = _state.IsHelixJobProcessed(helixJob.JobName); - if (!alreadyUploadedByPriorAttempt) + if (!alreadyUploadedByPriorAttempt && recordOutcomes) { _reporter.LogJobProcessingStart(helixJob); _reporter.LogFailedWorkItemConsoleLinks(helixJob, workItems.Where(wi => wi.IsFailed)); } - _state.TryRecordWorkItemOutcomes(helixJob, workItems); + if (recordOutcomes) + { + _state.TryRecordWorkItemOutcomes(helixJob, workItems); + } + else + { + _state.MarkWorkItemOutcomesIgnored(helixJob.JobName); + } if (queueUpload && !alreadyUploadedByPriorAttempt) { - if (_state.TryQueueHelixJobUpload(helixJob.JobName)) - { - _uploads.Enqueue(helixJob, workItems, cancellationToken); - } + _uploads.TryEnqueue( + helixJob, + workItems, + isJobComplete: true, + discoveryPoll: discoveryPoll); } - if (!alreadyUploadedByPriorAttempt) + if (!alreadyUploadedByPriorAttempt && recordOutcomes) { _reporter.LogJobCompleted(helixJob, workItems); } } - private async Task> GetCompletedJobsAsync( + private IReadOnlyCollection GetCompletedJobs( IReadOnlyList jobs, - CancellationToken cancellationToken) + IReadOnlyDictionary> workItemsByJob) { var completed = new List(); foreach (HelixJobInfo job in jobs) { - if (job.IsCompleted || await AreAllWorkItemsTerminalAsync(job, cancellationToken)) + if (job.IsCompleted || AreAllWorkItemsTerminal(job, workItemsByJob[job.JobName])) { completed.Add(job); } @@ -421,18 +597,30 @@ private async Task> GetCompletedJobsAsync( return MonitorState.OrderHelixJobsOldToNew(completed); } - private async Task AreAllWorkItemsTerminalAsync(HelixJobInfo job, CancellationToken cancellationToken) + private static bool AreAllWorkItemsTerminal( + HelixJobInfo job, + IReadOnlyCollection workItems) { if (job.InitialWorkItemCount is not > 0) { return false; } - IReadOnlyCollection workItems = await _helix.ListWorkItemsAsync(job.JobName, cancellationToken); return workItems.Count >= job.InitialWorkItemCount.Value && workItems.All(wi => wi.ExitCode.HasValue); } + private Task>> GetWorkItemsAsync( + IReadOnlyList jobs, + CancellationToken cancellationToken) + => ParallelAsync.ToDictionaryAsync( + jobs, + parallelism: Math.Max(8, _options.TestResultUploadParallelism), + static job => job.JobName, + async (job, token) => await _helix.ListWorkItemsAsync(job.JobName, token), + StringComparer.OrdinalIgnoreCase, + cancellationToken); + private async Task CancelInFlightHelixJobsAsync(CancellationToken cancellationToken) { List inFlightJobs = @@ -490,8 +678,83 @@ private bool IsPreviousAttempt(HelixJobInfo job) && !string.IsNullOrEmpty(job.StageAttempt) && MonitorState.ParseStageAttempt(job.StageAttempt) < MonitorState.ParseStageAttempt(_options.StageAttempt); + private static bool TryGetCurrentSubmitterAttempt( + HelixJobInfo job, + IReadOnlyList timelineRecords, + out int attempt, + out string submitterIdentity) + { + // SendHelixJob stamps System.PhaseName from the AzDO phase refName. The nested + // job record frequently has refName "__default", so only use it when phase + // identity was not available on the Helix job. + string timelineRecordType; + if (!string.IsNullOrEmpty(job.SubmitterPhaseName)) + { + submitterIdentity = job.SubmitterPhaseName; + timelineRecordType = "Phase"; + } + else + { + submitterIdentity = job.SubmitterJobName; + timelineRecordType = "Job"; + } + + attempt = 0; + if (string.IsNullOrEmpty(submitterIdentity)) + { + return false; + } + + string identity = submitterIdentity; + AzureDevOpsTimelineRecord[] matchingRecords = + [ + ..timelineRecords + .Where(record => + string.Equals(record.Type, timelineRecordType, StringComparison.OrdinalIgnoreCase) + && string.Equals( + record.ReferenceName, + identity, + StringComparison.OrdinalIgnoreCase)) + ]; + + // A phase refName is normally unique. Job fallback can be "__default" for many + // matrix legs; treating any one of those as the submitter could duplicate work. + if (matchingRecords.Length != 1) + { + return false; + } + + attempt = matchingRecords[0].Attempt; + return true; + } + + private static bool IsSupersededBySubmitterRerun( + HelixJobInfo job, + bool hasSubmitterAttempt, + int currentSubmitterAttempt, + int helixSubmitterAttempt) + => hasSubmitterAttempt + && !string.IsNullOrEmpty(job.JobAttempt) + && currentSubmitterAttempt > helixSubmitterAttempt; + + private static ProductionDependencies CreateProductionDependencies( + JobMonitorOptions options, + ILogger logger) + { + var metrics = new JobMonitorMetrics(); + var azureDevOps = new AzureDevOpsService(options, logger, metrics); + var helix = new HelixService( + string.IsNullOrEmpty(options.HelixAccessToken) + ? ApiFactory.GetAnonymous(options.HelixBaseUri) + : ApiFactory.GetAuthenticated(options.HelixBaseUri, options.HelixAccessToken), + logger, + metrics); + return new ProductionDependencies(azureDevOps, helix, metrics); + } + public void Dispose() { + _uploads.Cancel(); (_azdo as IDisposable)?.Dispose(); (_helix as IDisposable)?.Dispose(); } @@ -499,6 +762,30 @@ public void Dispose() private Task Delay(CancellationToken cancellationToken) => _delayFunc(TimeSpan.FromSeconds(Math.Max(5, _options.PollingIntervalSeconds)), cancellationToken); + private async Task ReportStatusPeriodicallyAsync(CancellationToken cancellationToken) + { + while (true) + { + await _statusDelayFunc(TimeSpan.FromMinutes(5), cancellationToken); + LogLatestStatus(); + } + } + + private void LogLatestStatus() + { + PollStatusSnapshot snapshot = Volatile.Read(ref _latestStatus); + if (snapshot is null) + { + return; + } + + _reporter.LogPollStatus( + snapshot.Jobs, + snapshot.WorkItemsByJob, + snapshot.CompletedJobNames, + _uploads.Snapshot); + } + private void LogWarning(string message) => _logger.LogWarning("{Prefix}{Message}", AzdoWarningPrefix, message); @@ -511,9 +798,20 @@ private void LogWarning(Exception exception, string message) /// private sealed class PollLoopState { - public int LastObservedJobCount { get; set; } = -1; - public int LastObservedCompletedCount { get; set; } = -1; - public DateTime LastStatusLogAt { get; set; } = DateTime.UtcNow; + public int PollNumber { get; set; } + public bool HasLoggedInitialStatus { get; set; } + public Dictionary> WorkItemsByJob { get; } = + new(StringComparer.OrdinalIgnoreCase); } + + private sealed record PollStatusSnapshot( + IReadOnlyList Jobs, + IReadOnlyDictionary> WorkItemsByJob, + IReadOnlySet CompletedJobNames); + + private sealed record ProductionDependencies( + IAzureDevOpsService AzureDevOps, + IHelixService Helix, + JobMonitorMetrics Metrics); } } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj index a58bc2876d7..222d59d525b 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj @@ -2,6 +2,8 @@ $(NetMinimum) + enable + annotations Exe true true @@ -24,7 +26,6 @@ - diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs index 59bd17b463c..aa0f09a5c2b 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs @@ -16,14 +16,17 @@ namespace Microsoft.DotNet.Helix.JobMonitor.Models public sealed class HelixJobInfo { public const string PreviousHelixJobNamePropertyName = "PreviousHelixJobName"; + public const string LogicalJobNamePropertyName = "jobName"; /// /// Helix job property that records the Azure DevOps stage attempt during which the job /// was submitted. This is the exact AzDO predefined-variable name that the Helix submitter /// (SendHelixJob) copies onto every job, so the monitor reads and re-stamps the - /// same property when resubmitting (see JobMonitorRunner.Design.md §2.3). + /// same property when resubmitting (see Design/SemanticBehavior.md §2.3). /// public const string StageAttemptPropertyName = "System.StageAttempt"; + public const string JobAttemptPropertyName = "System.JobAttempt"; + public const string ResubmittedByJobAttemptPropertyName = "JobMonitor.JobAttempt"; public HelixJobInfo(JobSummary helixJob) { @@ -32,6 +35,7 @@ public HelixJobInfo(JobSummary helixJob) TestRunName = GetTestRunNameFromJob(helixJob); StageName = GetStringPropertyFromJob(helixJob, "System.StageName"); StageAttempt = GetStringPropertyFromJob(helixJob, StageAttemptPropertyName); + JobAttempt = GetStringPropertyFromJob(helixJob, JobAttemptPropertyName); QueueId = helixJob.QueueId; InitialWorkItemCount = helixJob.InitialWorkItemCount; Properties = helixJob.Properties; @@ -47,16 +51,29 @@ public HelixJobInfo( string queueId = null, string previousHelixJobName = null, int? initialWorkItemCount = null, - string stageAttempt = null) + string stageAttempt = null, + string jobAttempt = null, + string logicalJobName = null, + string submitterPhaseName = null) { JobName = jobName ?? throw new ArgumentNullException(nameof(jobName)); Status = status ?? throw new ArgumentNullException(nameof(status)); TestRunName = testRunName; StageName = stageName; StageAttempt = stageAttempt; + JobAttempt = jobAttempt; QueueId = queueId; InitialWorkItemCount = initialWorkItemCount; - Properties = CreateProperties(testRunName, stageName, submitterJobName, submitterJobDisplayName, previousHelixJobName, stageAttempt); + Properties = CreateProperties( + testRunName, + stageName, + submitterJobName, + submitterJobDisplayName, + previousHelixJobName, + stageAttempt, + jobAttempt, + logicalJobName, + submitterPhaseName); } public string JobName { get; } @@ -85,6 +102,13 @@ public HelixJobInfo( /// public string StageAttempt { get; } + /// + /// Attempt of the Azure DevOps submitter job that created this logical Helix work. + /// Monitor-created resubmissions preserve this value so it can be compared with the + /// current timeline record for the submitter. + /// + public string JobAttempt { get; } + /// /// Helix target queue this job ran on (e.g. "Ubuntu.2204.Amd64.Open"). Comes from the /// Helix JobSummary.QueueId. May be null on synthetic jobs. @@ -93,6 +117,20 @@ public HelixJobInfo( public string SubmitterJobName => GetStringProperty(Properties, "System.JobName"); + /// + /// Stable Azure DevOps phase/job reference name copied by the Helix SDK submitter. + /// Runtime pipelines can report System.JobName as __default for many + /// independent matrix jobs, while System.PhaseName remains unique and stable. + /// + public string SubmitterPhaseName => GetStringProperty(Properties, "System.PhaseName"); + + /// + /// Stable logical name assigned by the Helix SDK submitter. A single Azure DevOps job + /// can submit multiple Helix jobs to the same queue, so this value distinguishes those + /// independent streams while remaining stable across stage reruns and resubmissions. + /// + public string LogicalJobName => GetStringProperty(Properties, LogicalJobNamePropertyName); + /// /// Matrix-expanded Azure DevOps job display name (e.g. "Windows_NT Build_Release"), /// stamped onto the job from the System.JobDisplayName predefined variable. @@ -200,7 +238,10 @@ private static JObject CreateProperties( string submitterJobName, string submitterJobDisplayName, string previousHelixJobName, - string stageAttempt) + string stageAttempt, + string jobAttempt, + string logicalJobName, + string submitterPhaseName) { var properties = new JObject(); @@ -218,6 +259,10 @@ private static JObject CreateProperties( { properties[StageAttemptPropertyName] = stageAttempt; } + if (!string.IsNullOrEmpty(jobAttempt)) + { + properties[JobAttemptPropertyName] = jobAttempt; + } if (!string.IsNullOrEmpty(submitterJobName)) { @@ -234,6 +279,16 @@ private static JObject CreateProperties( properties[PreviousHelixJobNamePropertyName] = previousHelixJobName; } + if (!string.IsNullOrEmpty(logicalJobName)) + { + properties[LogicalJobNamePropertyName] = logicalJobName; + } + + if (!string.IsNullOrEmpty(submitterPhaseName)) + { + properties["System.PhaseName"] = submitterPhaseName; + } + return properties; } } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs index 54ef59424a8..e5e34fc993e 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs @@ -46,12 +46,18 @@ internal sealed class MonitorState // (HelixJobChainKey, WorkItemName). See GetHelixJobChainKey for the keying rationale. private readonly Dictionary<(string ChainKey, string WorkItemName), bool> _workItemOutcomes = new(WorkItemOutcomeKeyComparer.Instance); + private readonly HashSet<(string ChainKey, string WorkItemName)> _failedTestWorkItems + = new(WorkItemOutcomeKeyComparer.Instance); // Helix job names whose per-work-item outcomes have already been reconciled into // _workItemOutcomes. Prevents the second reconciliation pass from re-processing // jobs that were observed in an earlier poll. private readonly HashSet _workItemOutcomeJobs = new(StringComparer.OrdinalIgnoreCase); + // Previous-attempt jobs whose submitter has a newer timeline attempt. These remain + // uploadable history but must not contribute to current status or pass/fail. + private readonly HashSet _supersededJobNames = new(StringComparer.OrdinalIgnoreCase); + // Latest known console-link information for every failed work item, keyed the same // way as _workItemOutcomes. Cleared per key when a later incarnation passes. private readonly Dictionary<(string ChainKey, string WorkItemName), FailedWorkItemConsoleInfo> _failedWorkItemConsoleInfo @@ -239,6 +245,34 @@ public bool IsWorkItemOutcomesRecorded(string jobName) } } + /// + /// Marks a superseded job's outcomes as intentionally ignored. This prevents repeated + /// work-item refreshes without inserting stale outcomes into the current result map. + /// + public void MarkWorkItemOutcomesIgnored(string jobName) + { + lock (_sync) + { + _workItemOutcomeJobs.Add(jobName); + } + } + + public void MarkSupersededBySubmitterRerun(string jobName) + { + lock (_sync) + { + _supersededJobNames.Add(jobName); + } + } + + public bool IsSupersededBySubmitterRerun(string jobName) + { + lock (_sync) + { + return _supersededJobNames.Contains(jobName); + } + } + /// /// Atomically records all per-work-item outcomes for one completed Helix job: /// updates , the failure map, and the failed-work-item @@ -260,8 +294,21 @@ public bool TryRecordWorkItemOutcomes(HelixJobInfo helixJob, IReadOnlyCollection // Within the same Helix job lineage, the latest result overwrites the prior // one for the same work item name. Independent original Helix jobs have // different roots, even when they share an AzDO submitter and queue. - _workItemOutcomes[(chainKey, wi.Name)] = !wi.IsFailed; - TrackFailedWorkItemConsoleInfoLocked(helixJob, chainKey, wi); + var key = (chainKey, wi.Name); + bool passed = !wi.IsFailed && !_failedTestWorkItems.Contains(key); + // This marker only bridges the race where incremental test-result upload + // finishes before the Helix outcome is reconciled. A later incarnation in + // the same logical stream must be able to replace the failure with a pass. + _failedTestWorkItems.Remove(key); + _workItemOutcomes[key] = passed; + if (wi.IsFailed) + { + TrackFailedWorkItemConsoleInfoLocked(helixJob, chainKey, wi); + } + else if (passed) + { + _failedWorkItemConsoleInfo.Remove(key); + } } return true; @@ -306,6 +353,7 @@ public void ObserveTestResults( string chainKey = GetHelixJobChainKeyLocked(job); var key = (chainKey, entry.Key.WorkItemName); + _failedTestWorkItems.Add(key); _workItemOutcomes[key] = false; // Ensure the final failure report includes test-only failures too. @@ -318,9 +366,20 @@ public void ObserveTestResults( "see Azure DevOps test run results"); } } + } } + public void ObserveTestResult( + string jobName, + string workItemName, + TestResultUploadSummary summary) + => ObserveTestResults( + new Dictionary<(string JobName, string WorkItemName), TestResultUploadSummary> + { + [(jobName, workItemName)] = summary, + }); + /// /// Returns true if this is the first time a console-link warning is being emitted for /// the given (jobName, workItemName) key. Used to deduplicate console-link logging. @@ -385,10 +444,13 @@ public IReadOnlyList SnapshotFailedWorkItemConsoleInf } /// - /// Produces a key that rolls up work-item outcomes within a Helix resubmission lineage. - /// The chain is followed back through PreviousHelixJobName links and the root - /// Helix job name is used. This lets resubmissions overwrite prior outcomes without - /// folding independent original jobs that share the same AzDO submitter and Helix queue. + /// Produces a key that rolls up work-item outcomes within a logical Helix work stream. + /// The stage, AzDO phase name, Helix queue, and submitter-assigned logical job name jointly + /// identify a stream across stage reruns and monitor resubmissions. The phase name is + /// preferred because some pipelines stamp many independent jobs with + /// System.JobName=__default. When stable submitter metadata is unavailable, + /// lineage is followed back through PreviousHelixJobName and the root Helix job + /// name is used instead. /// public string GetHelixJobChainKey(HelixJobInfo job) { @@ -399,6 +461,29 @@ public string GetHelixJobChainKey(HelixJobInfo job) } private string GetHelixJobChainKeyLocked(HelixJobInfo job) + { + HelixJobInfo root = GetLineageRootLocked(job); + string submitterName = job.SubmitterPhaseName + ?? root.SubmitterPhaseName + ?? job.SubmitterJobName + ?? root.SubmitterJobName; + string queueId = job.QueueId ?? root.QueueId; + string logicalJobName = job.LogicalJobName + ?? root.LogicalJobName + ?? job.TestRunName + ?? root.TestRunName; + string stageName = job.StageName ?? root.StageName; + + if (!string.IsNullOrEmpty(submitterName) + && !string.IsNullOrEmpty(logicalJobName)) + { + return FormatSubmitterChainKey(stageName, submitterName, queueId, logicalJobName); + } + + return $"helix:{root.JobName}"; + } + + private HelixJobInfo GetLineageRootLocked(HelixJobInfo job) { HelixJobInfo current = job; var visited = new HashSet(StringComparer.OrdinalIgnoreCase); @@ -408,22 +493,30 @@ private string GetHelixJobChainKeyLocked(HelixJobInfo job) { if (!_associatedJobs.TryGetValue(current.PreviousHelixJobName, out HelixJobInfo previous)) { - return $"helix:{current.PreviousHelixJobName}"; + return new HelixJobInfo(current.PreviousHelixJobName, "finished"); } current = previous; } - return $"helix:{(current?.JobName ?? job.JobName)}"; + return current ?? job; } + private static string FormatSubmitterChainKey( + string stageName, + string submitterName, + string queueId, + string logicalJobName) + => $"stage:{stageName ?? string.Empty}|submitter:{submitterName}|queue:{queueId ?? string.Empty}|job:{logicalJobName}"; + /// /// From an arbitrary set of Helix jobs (possibly spanning multiple stage attempts), - /// return the single latest incarnation of each logical work stream — one job per root - /// Helix job (§5.7). Within a stream, resubmission lineage is collapsed to the leaf. - /// Unlinked jobs remain independent even when they share an AzDO submitter and queue. - /// Used by the retry pass to decide, per stream, whether previous-attempt work must be - /// reconciled into the current attempt. + /// return the single latest incarnation of each logical work stream — one job per + /// submitter chain key (§5.7). Within a stream, resubmission lineage is collapsed to the + /// leaf, and unlinked rerun duplicates (same submitter + queue on different stage + /// attempts, not connected by PreviousHelixJobName) are broken toward the highest + /// stage and submitter job attempts, then explicit lineage depth. Used by the retry pass + /// to decide whether previous-attempt work remains authoritative. /// public IReadOnlyList GetLatestIncarnationPerStream(IEnumerable jobs) { @@ -435,6 +528,8 @@ public IReadOnlyList GetLatestIncarnationPerStream(IEnumerable g .OrderBy(j => ParseStageAttempt(j.StageAttempt)) + .ThenBy(j => ParseJobAttempt(j.JobAttempt)) + .ThenBy(j => GetLineageDepth(j, _associatedJobs)) .ThenBy(j => j.JobName, StringComparer.OrdinalIgnoreCase) .Last()) ]; @@ -466,10 +561,30 @@ public void RecordAbandonedWork(HelixJobInfo job, IEnumerable w /// /// Parses a stage-attempt string (e.g. the System.StageAttempt property) into a - /// comparable integer. Unknown / unparseable values sort as attempt 1 (the first attempt). + /// comparable integer. Missing values are legacy metadata and sort as attempt 1. A + /// present malformed value violates the Helix submitter contract and is rejected. /// public static int ParseStageAttempt(string stageAttempt) - => int.TryParse(stageAttempt, out int attempt) ? attempt : 1; + => ParseAttempt(stageAttempt, nameof(stageAttempt)); + + public static int ParseJobAttempt(string jobAttempt) + => ParseAttempt(jobAttempt, nameof(jobAttempt)); + + private static int ParseAttempt(string value, string parameterName) + { + if (string.IsNullOrEmpty(value)) + { + return 1; + } + + if (int.TryParse(value, out int attempt) && attempt > 0) + { + return attempt; + } + + throw new InvalidOperationException( + $"Attempt metadata '{parameterName}' must be a positive integer, but was '{value}'."); + } /// /// From an arbitrary set of Helix jobs return only the leaves of each lineage chain — @@ -489,7 +604,9 @@ public static IReadOnlyList GetLatestHelixJobAttempts(IEnumerable< /// Orders Helix jobs from oldest incarnation to newest by following the /// PreviousHelixJobName link backwards, breaking ties toward the lower stage /// attempt. Used to ensure upload and outcome reconciliation process lineage in the - /// right order (older first, so newer incarnations supersede older ones). + /// right order (older first, so newer incarnations supersede older ones) — including + /// unlinked rerun duplicates on different attempts, where the higher stage/job + /// incarnation must reconcile last so it wins the outcome map (§5.7). /// public static IReadOnlyList OrderHelixJobsOldToNew(IEnumerable jobs) { @@ -499,6 +616,7 @@ public static IReadOnlyList OrderHelixJobsOldToNew(IEnumerable GetLineageDepth(j, jobByName)) .ThenBy(j => ParseStageAttempt(j.StageAttempt)) + .ThenBy(j => ParseJobAttempt(j.JobAttempt)) .ThenBy(j => j.JobName, StringComparer.OrdinalIgnoreCase) ]; } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ActionQueue.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ActionQueue.cs new file mode 100644 index 00000000000..3f14fa6ea3e --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ActionQueue.cs @@ -0,0 +1,144 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading.Channels; + +namespace Microsoft.DotNet.Helix.JobMonitor.Parallelism; + +internal sealed class ActionQueue : IAsyncDisposable +{ + private readonly Channel _channel; + private readonly Func _action; + private readonly CancellationTokenSource _shutdown = new(); + private readonly Task[] _workers; + private long _accepted; + private long _started; + private long _completed; + private int _active; + + public ActionQueue( + int capacity, + int parallelism, + Func action) + : this( + Channel.CreateBounded(new BoundedChannelOptions(capacity) + { + AllowSynchronousContinuations = false, + FullMode = BoundedChannelFullMode.Wait, + SingleReader = parallelism == 1, + SingleWriter = false, + }), + parallelism, + action) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity); + } + + public ActionQueue( + int parallelism, + Func action) + : this( + Channel.CreateUnbounded(new UnboundedChannelOptions + { + AllowSynchronousContinuations = false, + SingleReader = parallelism == 1, + SingleWriter = false, + }), + parallelism, + action) + { + } + + private ActionQueue( + Channel channel, + int parallelism, + Func action) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parallelism); + ArgumentNullException.ThrowIfNull(action); + + _action = action; + _channel = channel; + _workers = Enumerable.Range(0, parallelism) + .Select(_ => WorkerAsync()) + .ToArray(); + } + + public QueueSnapshot Snapshot => new( + Accepted: Interlocked.Read(ref _accepted), + Started: Interlocked.Read(ref _started), + Completed: Interlocked.Read(ref _completed), + Active: Volatile.Read(ref _active)); + + public bool TryEnqueue(T item) + { + if (!_channel.Writer.TryWrite(item)) + { + return false; + } + + Interlocked.Increment(ref _accepted); + return true; + } + + public async ValueTask EnqueueAsync(T item, CancellationToken cancellationToken) + { + await _channel.Writer.WriteAsync(item, cancellationToken); + Interlocked.Increment(ref _accepted); + } + + public void Complete() => _channel.Writer.TryComplete(); + + public void Cancel() => _shutdown.Cancel(); + + public Task DrainAsync() => Task.WhenAll(_workers); + + public async ValueTask DisposeAsync() + { + _channel.Writer.TryComplete(); + _shutdown.Cancel(); + + try + { + await Task.WhenAll(_workers); + } + catch (Exception) when (_shutdown.IsCancellationRequested) + { + } + + _shutdown.Dispose(); + } + + private async Task WorkerAsync() + { + await foreach (T item in _channel.Reader.ReadAllAsync(_shutdown.Token)) + { + Interlocked.Increment(ref _started); + Interlocked.Increment(ref _active); + try + { + await _action(item, _shutdown.Token); + } + catch (Exception ex) when (!_shutdown.IsCancellationRequested) + { + _channel.Writer.TryComplete(ex); + _shutdown.Cancel(); + throw; + } + finally + { + Interlocked.Decrement(ref _active); + Interlocked.Increment(ref _completed); + } + } + } +} + +internal readonly record struct QueueSnapshot( + long Accepted, + long Started, + long Completed, + int Active) +{ + public long Queued => Accepted - Started; +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ParallelAsync.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ParallelAsync.cs new file mode 100644 index 00000000000..2721775e4bf --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ParallelAsync.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.Helix.JobMonitor.Parallelism; + +internal static class ParallelAsync +{ + public static async Task> ToDictionaryAsync( + IEnumerable source, + int parallelism, + Func getKey, + Func> getValue, + IEqualityComparer comparer, + CancellationToken cancellationToken) + where TKey : notnull + { + TSource[] items = [.. source]; + // Each worker owns one unique array slot. The dictionary is constructed only after + // Parallel.ForEachAsync completes, so neither collection is mutated concurrently. + var values = new TValue[items.Length]; + + await Parallel.ForEachAsync( + Enumerable.Range(0, items.Length), + new ParallelOptions + { + CancellationToken = cancellationToken, + MaxDegreeOfParallelism = parallelism, + }, + async (index, token) => values[index] = await getValue(items[index], token)); + + var result = new Dictionary(items.Length, comparer); + for (int i = 0; i < items.Length; i++) + { + result.Add(getKey(items[i]), values[i]); + } + + return result; + } +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs new file mode 100644 index 00000000000..8648b6026ef --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs @@ -0,0 +1,68 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.Helix.JobMonitor; + +internal sealed class AzureDevOpsRateLimitGate +{ + private readonly JobMonitorMetrics _metrics; + private long _notBeforeUtcTicks; + + public AzureDevOpsRateLimitGate(JobMonitorMetrics metrics = null) + { + _metrics = metrics; + } + + /// + /// Extends the shared request deadline. A shared deadline coordinates all concurrent upload + /// workers; independent delays would allow other workers to continue issuing throttled calls. + /// + public void ExtendDeadline(TimeSpan delay) + { + if (delay <= TimeSpan.Zero) + { + return; + } + + _metrics?.RecordRateLimitDeferral(delay); + long candidate = DateTimeOffset.UtcNow.Add(delay).UtcTicks; + long observed; + while (candidate > (observed = Interlocked.Read(ref _notBeforeUtcTicks))) + { + if (Interlocked.CompareExchange(ref _notBeforeUtcTicks, candidate, observed) == observed) + { + break; + } + } + } + + /// Waits until the current shared request deadline has passed. + public async Task WaitAsync(CancellationToken cancellationToken) + { + long waitStartedAt = 0; + try + { + // Re-read after every delay because another worker may extend the deadline while this + // worker is waiting. + while (true) + { + long notBeforeTicks = Interlocked.Read(ref _notBeforeUtcTicks); + TimeSpan delay = TimeSpan.FromTicks(notBeforeTicks - DateTimeOffset.UtcNow.UtcTicks); + if (delay <= TimeSpan.Zero) + { + return; + } + + waitStartedAt = waitStartedAt == 0 ? JobMonitorMetrics.StartOperation() : waitStartedAt; + await Task.Delay(delay, cancellationToken); + } + } + finally + { + if (waitStartedAt != 0) + { + _metrics?.RecordRateLimitWait(waitStartedAt); + } + } + } +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs index fb0febf0ff8..344e6ddae87 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs @@ -5,8 +5,10 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; +using System.Net; using System.Net.Http; using System.Net.Http.Headers; +using System.Net.Sockets; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -21,6 +23,12 @@ namespace Microsoft.DotNet.Helix.JobMonitor { internal sealed class AzureDevOpsService : IAzureDevOpsService, IDisposable { + private const int ControlRequestAttemptCount = 5; + private const int ResultRequestAttemptCount = 10; + private static readonly TimeSpan s_maximumRetryDelay = TimeSpan.FromSeconds(30); + private static readonly System.Text.Json.JsonSerializerOptions s_serializerOptions = + new(System.Text.Json.JsonSerializerDefaults.Web); + // A test run tag is applied to every completed test run so we can recover the Helix job // name on a subsequent monitor attempt. The Helix job name (a GUID) is encoded as // "{HelixJobTagPrefix}{guidWithoutDashes}" because Azure DevOps only accepts alphanumeric @@ -52,23 +60,33 @@ internal sealed class AzureDevOpsService : IAzureDevOpsService, IDisposable private readonly JobMonitorOptions _options; private readonly ILogger _logger; private readonly HttpClient _azdoClient; - private readonly SemaphoreSlim _uploadSemaphore; + private readonly AzureDevOpsRateLimitGate _rateLimitGate; + private readonly JobMonitorMetrics _metrics; - public AzureDevOpsService(JobMonitorOptions options, ILogger logger) + public AzureDevOpsService( + JobMonitorOptions options, + ILogger logger, + JobMonitorMetrics metrics = null) { _options = options; _logger = logger; + _metrics = metrics ?? new JobMonitorMetrics(); + _rateLimitGate = new AzureDevOpsRateLimitGate(_metrics); _azdoClient = new HttpClient(); - _uploadSemaphore = new SemaphoreSlim(options.TestResultUploadParallelism); InitializeClient(); } - internal AzureDevOpsService(JobMonitorOptions options, ILogger logger, HttpClient azdoClient) + internal AzureDevOpsService( + JobMonitorOptions options, + ILogger logger, + HttpClient azdoClient, + JobMonitorMetrics metrics = null) { _options = options; _logger = logger; + _metrics = metrics ?? new JobMonitorMetrics(); + _rateLimitGate = new AzureDevOpsRateLimitGate(_metrics); _azdoClient = azdoClient ?? throw new ArgumentNullException(nameof(azdoClient)); - _uploadSemaphore = new SemaphoreSlim(options.TestResultUploadParallelism); InitializeClient(); } @@ -76,7 +94,9 @@ private void InitializeClient() { string encodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + _options.SystemAccessToken)); _azdoClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", encodedToken); + _azdoClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); _azdoClient.DefaultRequestHeaders.UserAgent.ParseAdd("dotnet-helix-job-monitor"); + _azdoClient.Timeout = TimeSpan.FromMinutes(5); } public async Task> GetTimelineRecordsAsync(CancellationToken cancellationToken) @@ -362,7 +382,7 @@ public async Task CompleteTestRunAsync( await SendAsync(new HttpMethod("PATCH"), $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=7.1", body, - retryTransientFailures: false, + retryTransientFailures: true, cancellationToken: cancellationToken); } @@ -398,74 +418,36 @@ await SendAsync( cancellationToken: cancellationToken); } - public async Task> UploadTestResultsAsync( + public async Task UploadTestResultsAsync( int testRunId, - IReadOnlyList results, + WorkItemTestResults results, CancellationToken cancellationToken) { - var reportingParameters = new AzureDevOpsReportingParameters( - new Uri(_options.CollectionUri, UriKind.Absolute), - _options.TeamProject, - testRunId.ToString(CultureInfo.InvariantCulture), - _options.SystemAccessToken, - _options.UseFullyQualifiedTestName, - _options.TestResultAttachmentMode); - using var publisher = new AzureDevOpsResultPublisher( - reportingParameters, - _logger); - - async Task UploadWorkItemAsync(WorkItemTestResults workItem) + if (results.TestResultFiles.Count == 0) { - if (workItem.TestResultFiles.Count == 0) - { - _logger.LogInformation("No test results to upload for work item {WorkItemId} in job {JobName}", workItem.WorkItemName, workItem.JobName); - return new TestResultUploadSummary(true, 0); - } + return new TestResultUploadSummary(true, 0); + } - DateTimeOffset waitStartedAt = DateTimeOffset.UtcNow; - _logger.LogDebug( - "Work item '{WorkItemName}' in job '{JobName}' is waiting for a test-result upload slot. " - + "{AvailableSlots} slot(s) are currently available.", - workItem.WorkItemName, - workItem.JobName, - _uploadSemaphore.CurrentCount); - await _uploadSemaphore.WaitAsync(cancellationToken); + var publisher = new AzureDevOpsResultPublisher( + _options.TestResultAttachmentMode, + _options.UseFullyQualifiedTestName, + _logger, + CreateResultTransport(testRunId), + _metrics); - try + return await publisher.UploadTestResultsWithSummaryAsync( + results.TestResultFiles, + new { - DateTimeOffset uploadStartedAt = DateTimeOffset.UtcNow; - _logger.LogDebug( - "Work item '{WorkItemName}' in job '{JobName}' acquired a test-result upload slot after {WaitElapsed}. " - + "Parsing and publishing {FileCount} file(s).", - workItem.WorkItemName, - workItem.JobName, - uploadStartedAt - waitStartedAt, - workItem.TestResultFiles.Count); - TestResultUploadSummary summary = await publisher.UploadTestResultsWithSummaryAsync( - workItem.TestResultFiles, - new - { - HelixJobId = workItem.JobName, - HelixWorkItemName = workItem.WorkItemName - }, - cancellationToken); - _logger.LogDebug( - "Work item '{WorkItemName}' in job '{JobName}' finished parsing and publishing after {UploadElapsed}.", - workItem.WorkItemName, - workItem.JobName, - DateTimeOffset.UtcNow - uploadStartedAt); - return summary; - } - finally - { - _uploadSemaphore.Release(); - } - } - - (WorkItemTestResults WorkItem, TestResultUploadSummary Summary)[] testSummaries = await Task.WhenAll(results.Select(async result => (result, await UploadWorkItemAsync(result)))); - return testSummaries.ToDictionary(t => (t.WorkItem.JobName, t.WorkItem.WorkItemName), t => t.Summary); + HelixJobId = results.JobName, + HelixWorkItemName = results.WorkItemName + }, + cancellationToken); } + internal IAzureDevOpsResultTransport CreateResultTransport(int testRunId) + => new AzureDevOpsResultTransport(this, testRunId); + private async Task SendAsync( HttpMethod method, string requestUri, @@ -477,53 +459,96 @@ private async Task SendAsync( return string.IsNullOrWhiteSpace(content) ? [] : JObject.Parse(content); } - // Sends a request and returns the raw response body as a string. Used for endpoints - // (such as test-run attachment downloads) that do not return JSON, where SendAsync's - // JObject parsing would fail or discard the payload. - private async Task SendForStringAsync( + private Task SendForStringAsync( HttpMethod method, string requestUri, JToken body = null, bool retryTransientFailures = true, CancellationToken cancellationToken = default) + => SendForStringAsync( + method, + requestUri, + body?.ToString(Formatting.None), + AzureDevOpsRequestKind.Control, + retryTransientFailures, + ControlRequestAttemptCount, + cancellationToken); + + // All Azure DevOps calls flow through this method so request validation, throttling, + // retries, and metrics remain consistent across control, result, and attachment traffic. + private async Task SendForStringAsync( + HttpMethod method, + string requestUri, + string serializedBody, + AzureDevOpsRequestKind requestKind, + bool retryTransientFailures, + int attemptCount, + CancellationToken cancellationToken) { - async Task SendOnceAsync() + int payloadBytes = serializedBody is null ? 0 : Encoding.UTF8.GetByteCount(serializedBody); + + async Task SendOnceAsync(int attempt) { + await _rateLimitGate.WaitAsync(cancellationToken); + long requestStartedAt = JobMonitorMetrics.StartOperation(); + bool failed = true; using var request = new HttpRequestMessage(method, requestUri); - if (body != null) + if (serializedBody != null) { - request.Content = new StringContent(body.ToString(Formatting.None), Encoding.UTF8, "application/json"); + request.Content = new StringContent(serializedBody, Encoding.UTF8, "application/json"); } - using HttpResponseMessage response = await _azdoClient.SendAsync(request, cancellationToken); - string content = response.Content != null ? await response.Content.ReadAsStringAsync(cancellationToken) : null; - if (!response.IsSuccessStatusCode) + try { - await HonorRateLimitAsync(response, requestUri, cancellationToken); - throw new HttpRequestException( - $"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}", - null, - response.StatusCode); - } + using HttpResponseMessage response = await _azdoClient.SendAsync(request, cancellationToken); + string content = response.Content != null ? await response.Content.ReadAsStringAsync(cancellationToken) : null; + TimeSpan? rateLimitDelay = GetRateLimitDelay(response); + if (response.StatusCode == HttpStatusCode.TooManyRequests && rateLimitDelay is null) + { + rateLimitDelay = TimeSpan.FromSeconds(30); + } + + if (rateLimitDelay is { } delay) + { + // The current request has completed. Extend only the shared deadline for + // requests that have not yet started. + _rateLimitGate.ExtendDeadline(delay); + } - await HonorRateLimitAsync(response, requestUri, cancellationToken); - return content; + if (!response.IsSuccessStatusCode) + { + ThrowForFailure(response, content, requestUri, requestKind, rateLimitDelay); + } + + failed = false; + return content; + } + finally + { + _metrics.RecordAzureDevOpsRequest( + requestKind, + payloadBytes, + isRetry: attempt > 0, + failed: failed, + startedAt: requestStartedAt); + } } if (!retryTransientFailures) { - return await SendOnceAsync(); + return await SendOnceAsync(0); } string result = null; Exception lastException = null; var retryHandler = new ExponentialRetry { - MaxAttempts = 5, - DelayBase = 2, + MaxAttempts = attemptCount, + DelayBase = requestKind == AzureDevOpsRequestKind.Control ? 2 : 3, DelayConstant = 0, MinRandomFactor = 1, MaxRandomFactor = 1, + MaximumDelay = s_maximumRetryDelay, RetryDelayCallback = (failedAttempt, delay) => _logger.LogDebug( "Azure DevOps {Method} request to '{RequestUri}' failed on attempt {Attempt} of {AttemptCount}. " @@ -531,22 +556,22 @@ async Task SendOnceAsync() method, requestUri, failedAttempt, - 5, + attemptCount, delay), }; bool succeeded = await retryHandler.RunAsync( - async _ => + async attempt => { try { - result = await SendOnceAsync(); + result = await SendOnceAsync(attempt); return RetryResult.Success; } - catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + catch (Exception ex) when (IsTransientException(ex, cancellationToken)) { lastException = ex; - return RetryResult.Retry(); + return RetryResult.Retry((ex as TransientAzureDevOpsRequestException)?.RetryAfter); } }, cancellationToken); @@ -556,64 +581,133 @@ async Task SendOnceAsync() : throw lastException ?? new InvalidOperationException("Retry failed without completing the Azure DevOps request."); } - // Honors Azure DevOps rate limiting guidance: - // https://learn.microsoft.com/azure/devops/integrate/concepts/rate-limits#api-client-experience - // If the response carries a Retry-After header (RFC 6585) we wait the specified amount of - // time before allowing the next request to be issued. We also log when the service reports - // a non-zero X-RateLimit-Delay so callers have visibility into throttling behavior. - private async Task HonorRateLimitAsync(HttpResponseMessage response, string requestUri, CancellationToken cancellationToken) + internal static bool IsTransientException(Exception exception, CancellationToken cancellationToken) + => !cancellationToken.IsCancellationRequested + && exception is OperationCanceledException { InnerException: TimeoutException } + or HttpRequestException + or TimeoutException + or SocketException + or IOException; + + internal static TimeSpan? GetRateLimitDelay(HttpResponseMessage response) { - TimeSpan? retryAfter = null; + TimeSpan delay = TimeSpan.Zero; RetryConditionHeaderValue retryAfterHeader = response.Headers.RetryAfter; - if (retryAfterHeader != null) + if (retryAfterHeader?.Delta is { } delta && delta > delay) { - if (retryAfterHeader.Delta.HasValue) - { - retryAfter = retryAfterHeader.Delta.Value; - } - else if (retryAfterHeader.Date.HasValue) + delay = delta; + } + + if (retryAfterHeader?.Date is { } date) + { + TimeSpan datedDelay = TimeSpan.FromTicks(date.UtcTicks - DateTimeOffset.UtcNow.UtcTicks); + if (datedDelay > delay) { - TimeSpan delta = retryAfterHeader.Date.Value - DateTimeOffset.UtcNow; - if (delta > TimeSpan.Zero) - { - retryAfter = delta; - } + delay = datedDelay; } } - TimeSpan delayToApply = TimeSpan.Zero; - if (response.Headers.TryGetValues("X-RateLimit-Delay", out IEnumerable delayValues) && double.TryParse(delayValues.FirstOrDefault(), NumberStyles.Float, CultureInfo.InvariantCulture, out double delaySeconds) && delaySeconds > 0) { TimeSpan rateLimitDelay = TimeSpan.FromSeconds(delaySeconds); - delayToApply = rateLimitDelay; - _logger.LogDebug( - "Azure DevOps reported X-RateLimit-Delay of {DelaySeconds:0.###}s on request to {RequestUri}.", - delaySeconds, - requestUri); + if (rateLimitDelay > delay) + { + delay = rateLimitDelay; + } } - if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero) + return delay > TimeSpan.Zero ? delay : null; + } + + private static void ThrowForFailure( + HttpResponseMessage response, + string responseBody, + string requestUri, + AzureDevOpsRequestKind requestKind, + TimeSpan? rateLimitDelay) + { + responseBody ??= string.Empty; + if (responseBody.Contains("It may have been deleted", StringComparison.OrdinalIgnoreCase) + || responseBody.Contains("not authorized to access this resource", StringComparison.OrdinalIgnoreCase) + || responseBody.Contains("cannot be added or updated for a test run which is in Completed state", StringComparison.OrdinalIgnoreCase) + || response.StatusCode == HttpStatusCode.Forbidden + || response.StatusCode == HttpStatusCode.Unauthorized) { - delayToApply = delayToApply > retryAfter.Value ? delayToApply : retryAfter.Value; + throw new TerminalError(responseBody); } - if (delayToApply > TimeSpan.Zero) + string message = $"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {responseBody}"; + if ((int)response.StatusCode >= 500 || response.StatusCode == HttpStatusCode.TooManyRequests) { - _logger.LogDebug( - "Azure DevOps rate limit back-off. Delaying next request by {DelaySeconds:0.###}s (request: {RequestUri}).", - delayToApply.TotalSeconds, - requestUri); - await Task.Delay(delayToApply, cancellationToken); + throw new TransientAzureDevOpsRequestException( + message, + response.StatusCode, + rateLimitDelay ?? (response.StatusCode == HttpStatusCode.TooManyRequests ? TimeSpan.FromSeconds(30) : null)); } + + if (requestKind == AzureDevOpsRequestKind.Control) + { + throw new HttpRequestException(message, null, response.StatusCode); + } + + throw new AzureDevOpsReportingError(message); + } + + private sealed class AzureDevOpsResultTransport( + AzureDevOpsService service, + int testRunId) : IAzureDevOpsResultTransport + { + public Task PublishResultsAsync(object results, CancellationToken cancellationToken) + => service.SendForStringAsync( + HttpMethod.Post, + $"{service._options.CollectionUri}{service._options.TeamProject}/_apis/test/runs/{testRunId}/results?api-version=7.1-preview.6", + System.Text.Json.JsonSerializer.Serialize(results, s_serializerOptions), + AzureDevOpsRequestKind.ResultBatch, + retryTransientFailures: true, + ResultRequestAttemptCount, + cancellationToken); + + public Task UploadAttachmentAsync( + long testResultId, + long? testSubResultId, + string fileName, + string stream, + CancellationToken cancellationToken) + { + string query = testSubResultId is long subResultId + ? $"?testSubResultId={subResultId}&api-version=7.1-preview.1" + : "?api-version=7.1-preview.1"; + var body = new JObject + { + ["fileName"] = fileName, + ["stream"] = stream, + }; + + return service.SendForStringAsync( + HttpMethod.Post, + $"{service._options.CollectionUri}{service._options.TeamProject}/_apis/test/runs/{testRunId}/results/{testResultId}/attachments{query}", + body.ToString(Formatting.None), + AzureDevOpsRequestKind.Attachment, + retryTransientFailures: true, + ResultRequestAttemptCount, + cancellationToken); + } + } + + private sealed class TransientAzureDevOpsRequestException( + string message, + HttpStatusCode statusCode, + TimeSpan? retryAfter) + : HttpRequestException(message, null, statusCode) + { + public TimeSpan? RetryAfter { get; } = retryAfter; } public void Dispose() { _azdoClient.Dispose(); - _uploadSemaphore.Dispose(); } } } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs index 5398ad43c0f..361afb5c041 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs @@ -21,13 +21,19 @@ namespace Microsoft.DotNet.Helix.JobMonitor { internal sealed class HelixService : IHelixService { + private const int MaximumJobsPerBuildQuery = 1_000; + private readonly ILogger _logger; private readonly IHelixApi _helixApi; private readonly IBlobClientFactory _blobClientFactory; private readonly IFileSystem _fileSystem; + private readonly JobMonitorMetrics _metrics; - public HelixService(IHelixApi helixApi, ILogger logger) - : this(helixApi, logger, new AzureBlobClientFactory(), new FileSystem()) + public HelixService( + IHelixApi helixApi, + ILogger logger, + JobMonitorMetrics metrics = null) + : this(helixApi, logger, new AzureBlobClientFactory(), new FileSystem(), metrics) { } @@ -35,12 +41,14 @@ internal HelixService( IHelixApi helixApi, ILogger logger, IBlobClientFactory blobClientFactory, - IFileSystem fileSystem) + IFileSystem fileSystem, + JobMonitorMetrics metrics = null) { _helixApi = helixApi ?? throw new ArgumentNullException(nameof(helixApi)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _blobClientFactory = blobClientFactory ?? throw new ArgumentNullException(nameof(blobClientFactory)); _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _metrics = metrics ?? new JobMonitorMetrics(); } public async Task> GetJobsForBuildAsync( @@ -53,15 +61,31 @@ public async Task> GetJobsForBuildAsync( throw new ArgumentException("A non-empty Helix source filter must be provided.", nameof(source)); } - var filterProperties = new Dictionary() + if (string.IsNullOrWhiteSpace(buildId)) + { + throw new ArgumentException("A non-empty build ID filter must be provided.", nameof(buildId)); + } + IImmutableDictionary filterProperties = new Dictionary() { ["BuildId"] = buildId, }.ToImmutableDictionary(); IImmutableList jobs = await RetryAsync( - async () => await _helixApi.Job.ListAsync(source: source, properties: filterProperties, count: 100_000), + async () => await _helixApi.Job.ListAsync( + source: source, + properties: filterProperties, + count: MaximumJobsPerBuildQuery), cancellationToken); + if (jobs.Count >= MaximumJobsPerBuildQuery) + { + throw new InvalidOperationException( + $"Helix returned {jobs.Count} jobs for build '{buildId}', reaching the query limit of " + + $"{MaximumJobsPerBuildQuery}. Refusing to monitor a potentially truncated result set."); + } + + // Keep the local check as a defensive contract boundary in case Helix returns a + // malformed or unexpectedly broad response. return [ ..jobs @@ -69,87 +93,71 @@ public async Task> GetJobsForBuildAsync( && properties.TryGetValue("BuildId", out JToken id) && buildId == id.Value()) .Select(j => new HelixJobInfo(j)) - ]; + ]; } - public async Task> DownloadTestResultsAsync( + public async Task DownloadTestResultsAsync( string jobName, - IReadOnlyCollection workItemNames, + string workItemName, string workingDirectory, CancellationToken cancellationToken) { - List downloadedFiles = []; List transientFailures = []; string outputDirectory = _fileSystem.PathCombine(workingDirectory, SanitizeDirName(jobName)); _fileSystem.CreateDirectory(outputDirectory); JobResultsUri resultsUri = await RetryAsync(() => _helixApi.Job.ResultsAsync(jobName), cancellationToken); + IImmutableList availableFiles = await RetryAsync( + () => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false), + cancellationToken); - foreach (string workItemName in workItemNames) + availableFiles = [.. availableFiles.Where(f => LooksLikeTestResultFile(f.Name))]; + if (availableFiles.Count == 0) { - IImmutableList availableFiles = await RetryAsync( - () => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false), - cancellationToken); + return new WorkItemTestResults(jobName, workItemName, []); + } + + string workItemDirectory = _fileSystem.PathCombine(outputDirectory, SanitizeDirName(workItemName)); + _fileSystem.CreateDirectory(workItemDirectory); - availableFiles = [.. availableFiles.Where(f => LooksLikeTestResultFile(f.Name))]; - if (availableFiles.Count == 0) + List workItemFiles = []; + foreach (UploadedFile file in availableFiles) + { + string relativePath = NormalizeUploadedFilePath(file.Name); + string destinationFile = _fileSystem.PathCombine(workItemDirectory, relativePath); + string directory = _fileSystem.GetDirectoryName(destinationFile); + if (!string.IsNullOrEmpty(directory)) { - continue; + _fileSystem.CreateDirectory(directory); } - string workItemDirectory = _fileSystem.PathCombine(outputDirectory, SanitizeDirName(workItemName)); - _fileSystem.CreateDirectory(workItemDirectory); - - List workItemFiles = []; - foreach (UploadedFile file in availableFiles) + try { - string relativePath = NormalizeUploadedFilePath(file.Name); - string destinationFile = _fileSystem.PathCombine(workItemDirectory, relativePath); - string directory = _fileSystem.GetDirectoryName(destinationFile); - if (!string.IsNullOrEmpty(directory)) - { - _fileSystem.CreateDirectory(directory); - } - - try - { - DateTimeOffset downloadStartedAt = DateTimeOffset.UtcNow; - _logger.LogDebug( - "Downloading test result file '{FileName}' for '{JobName}/{WorkItemName}'.", - file.Name, - jobName, - workItemName); - IBlobClient blobClient = _blobClientFactory.CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS); - await blobClient.DownloadToAsync(destinationFile, cancellationToken); - _logger.LogDebug( - "Downloaded test result file '{FileName}' for '{JobName}/{WorkItemName}' in {Elapsed}.", - file.Name, - jobName, - workItemName, - DateTimeOffset.UtcNow - downloadStartedAt); - workItemFiles.Add(destinationFile); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception ex) when (TransientFailureDetector.IsTransient(ex)) - { - transientFailures.Add(ex); - _logger.LogWarning(ex, - "Transient failure downloading '{FileName}' for '{JobName}/{WorkItemName}'. " - + "The remaining files will still be attempted before the download is retried.", - file.Name, - jobName, - workItemName); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItemName); - } + IBlobClient blobClient = _blobClientFactory.CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS); + await blobClient.DownloadToAsync(destinationFile, cancellationToken); + workItemFiles.Add(destinationFile); + _metrics.RecordResultBlobDownload(failed: false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) when (TransientFailureDetector.IsTransient(ex)) + { + _metrics.RecordResultBlobDownload(failed: true); + transientFailures.Add(ex); + _logger.LogWarning(ex, + "Transient failure downloading '{FileName}' for '{JobName}/{WorkItemName}'. " + + "The remaining files will still be attempted before the work item is retried.", + file.Name, + jobName, + workItemName); + } + catch (Exception ex) + { + _metrics.RecordResultBlobDownload(failed: true); + _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItemName); } - - downloadedFiles.Add(new WorkItemTestResults(jobName, workItemName, workItemFiles)); } if (transientFailures.Count > 0) @@ -159,7 +167,7 @@ public async Task> DownloadTestResultsAsync( new AggregateException(transientFailures)); } - return downloadedFiles; + return new WorkItemTestResults(jobName, workItemName, workItemFiles); } private static bool LooksLikeTestResultFile(string path) @@ -208,6 +216,7 @@ public async Task ResubmitWorkItemsAsync( HelixJobInfo originalJob, IReadOnlyCollection failedWorkItems, string targetStageAttempt, + string monitorJobAttempt, CancellationToken cancellationToken) { string originalJobName = originalJob.JobName; @@ -328,6 +337,12 @@ await RetryAsync( { resubmittedProperties = resubmittedProperties.SetItem(HelixJobInfo.StageAttemptPropertyName, resubmittedStageAttempt); } + if (!string.IsNullOrEmpty(monitorJobAttempt)) + { + resubmittedProperties = resubmittedProperties.SetItem( + HelixJobInfo.ResubmittedByJobAttemptPropertyName, + monitorJobAttempt); + } // 5. Build the new job creation request, copying over Source / Properties / Creator // so the resubmitted job remains discoverable (BuildId, System.StageName, TestRunName, etc.). @@ -352,6 +367,9 @@ await RetryAsync( string stageName = GetStringPropertyFromProperties(details.Properties, "System.StageName"); string submitterJobName = GetStringPropertyFromProperties(details.Properties, "System.JobName"); string submitterJobDisplayName = GetStringPropertyFromProperties(details.Properties, "System.JobDisplayName"); + string logicalJobName = GetStringPropertyFromProperties(details.Properties, HelixJobInfo.LogicalJobNamePropertyName); + string submitterPhaseName = GetStringPropertyFromProperties(details.Properties, "System.PhaseName"); + string submitterJobAttempt = GetStringPropertyFromProperties(details.Properties, HelixJobInfo.JobAttemptPropertyName); var newJobInfo = new HelixJobInfo( newJob.Name, @@ -362,7 +380,10 @@ await RetryAsync( submitterJobDisplayName, details.QueueId, originalJobName, - stageAttempt: resubmittedStageAttempt); + stageAttempt: resubmittedStageAttempt, + jobAttempt: submitterJobAttempt, + logicalJobName: logicalJobName, + submitterPhaseName: submitterPhaseName); _logger.LogInformation("Resubmitted {Count} failed work item(s) from '{OriginalJobName}' as new job '{NewJobName}'{nl}{JobUri}", filteredEntries.Count, @@ -401,6 +422,7 @@ private async Task RetryAsync(Func> action, CancellationToken canc { Exception last = null; T result = default; + int attempt = 0; var retryHandler = new ExponentialRetry { MaxAttempts = 5, @@ -420,13 +442,16 @@ private async Task RetryAsync(Func> action, CancellationToken canc bool succeeded = await retryHandler.RunAsync( async _ => { + int currentAttempt = attempt++; try { result = await action(); + _metrics.RecordHelixRequest(isRetry: currentAttempt > 0, failed: false); return RetryResult.Success; } catch (Exception ex) when (!cancellationToken.IsCancellationRequested) { + _metrics.RecordHelixRequest(isRetry: currentAttempt > 0, failed: true); last = ex; return RetryResult.Retry(); } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/StatusReporter.cs b/src/Microsoft.DotNet.Helix/JobMonitor/StatusReporter.cs index e736240a71e..aafa9b0fb44 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/StatusReporter.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/StatusReporter.cs @@ -33,14 +33,12 @@ internal sealed class StatusReporter private readonly ILogger _logger; private readonly JobMonitorOptions _options; - private readonly IHelixService _helix; private readonly MonitorState _state; - public StatusReporter(ILogger logger, JobMonitorOptions options, IHelixService helix, MonitorState state) + public StatusReporter(ILogger logger, JobMonitorOptions options, MonitorState state) { _logger = logger; _options = options; - _helix = helix; _state = state; } @@ -142,22 +140,21 @@ public void LogFailedWorkItemConsoleLinks(HelixJobInfo helixJob, IEnumerable - public async Task LogPollStatusAsync( + public void LogPollStatus( IReadOnlyList jobs, + IReadOnlyDictionary> workItemsByJob, IReadOnlySet completedJobNames, - CancellationToken cancellationToken) + UploadPipelineSnapshot uploads) { List orderedJobs = [ ..jobs.OrderBy(j => j.JobName, StringComparer.OrdinalIgnoreCase) ]; - var workItemsByJob = new Dictionary>(StringComparer.OrdinalIgnoreCase); foreach (HelixJobInfo job in orderedJobs) { - IReadOnlyCollection workItems = await _helix.ListWorkItemsAsync(job.JobName, cancellationToken); + IReadOnlyCollection workItems = workItemsByJob[job.JobName]; LogFailedWorkItemConsoleLinks(job, workItems.Where(wi => wi.IsFailedAndTerminal)); - workItemsByJob[job.JobName] = workItems; } JobWorkItemStatusCounts counts = ComputeCounts(orderedJobs, workItemsByJob, completedJobNames); @@ -177,7 +174,17 @@ public async Task LogPollStatusAsync( if (_options.Verbose) { - LogVerboseTree(orderedJobs, workItemsByJob, completedJobNames); + _logger.LogDebug( + "Upload pipeline: jobs {JobQueued} queued/{JobActive} active, work items " + + "{WorkItemQueued} queued/{WorkItemActive} active, finalizers " + + "{FinalizerQueued} queued/{FinalizerActive} active, {UploadedResults} results uploaded.", + uploads.Jobs.Queued, + uploads.Jobs.Active, + uploads.WorkItems.Queued, + uploads.WorkItems.Active, + uploads.Finalizers.Queued, + uploads.Finalizers.Active, + uploads.UploadedResults); } } @@ -201,6 +208,87 @@ public void LogFinalSummary(int totalAssociatedJobCount) DocumentationUri); } + public void LogPerformanceMetrics( + JobMonitorMetricsSnapshot metrics, + UploadPipelineSnapshot uploads, + bool isPartial = false) + { + double pipelineSeconds = Math.Max(metrics.PipelineElapsed.TotalSeconds, 0.001); + double resultThroughput = uploads.UploadedResults / pipelineSeconds; + double workItemThroughput = uploads.WorkItems.Completed / pipelineSeconds; + double payloadMiB = metrics.AzureDevOpsPayloadBytes / (1024d * 1024d); + string heading = isPartial + ? "⏱️ Partial performance metrics captured during cancellation" + : "⏱️ Performance metrics"; + string throughputLabel = isPartial + ? "Completed before cancellation" + : "Upload throughput"; + string throughputSummary = isPartial + ? $"{uploads.UploadedResults} test result(s); " + + $"{uploads.WorkItems.Completed} work item(s); rates unavailable for a partial snapshot" + : $"{uploads.UploadedResults} test result(s), {resultThroughput:0.##} result(s)/s; " + + $"{uploads.WorkItems.Completed} work item(s), {workItemThroughput:0.##} work item(s)/s"; + + _logger.LogInformation( + "{Heading} ({Elapsed} monitor elapsed, {PipelineElapsed} observed upload pipeline activity):{nl}" + + " {ThroughputLabel}: {ThroughputSummary}{nl}" + + " Azure DevOps HTTP: {AzdoRequests} request attempt(s) " + + "({ControlRequests} control, {ResultRequests} result batch, {AttachmentRequests} attachment), " + + "{AzdoRetries} retry attempt(s), {AzdoFailedAttempts} failed attempt(s), " + + "{PayloadMiB:0.##} MiB request payload; aggregate request time {AzdoRequestTime}, " + + "max {MaximumAzdoRequestTime}{nl}" + + " Helix/storage: {HelixRequests} retry-wrapped remote request attempt(s), " + + "{HelixRetries} retry attempt(s), {HelixFailedAttempts} failed attempt(s); " + + "{BlobDownloads} result blob download attempt(s), {BlobFailures} failed{nl}" + + " Rate limiting: {RateLimitDeferrals} server-directed deferral(s), " + + "{RateLimitDeferredTime} cumulative guidance, max {MaximumRateLimitDeferral}; " + + "{RateLimitWaits} shared-gate worker wait(s), aggregate {RateLimitWaitTime}, " + + "max {MaximumRateLimitWaitTime}{nl}" + + " Pipeline aggregate worker time (max single operation): download {DownloadTime} ({MaxDownloadTime}), " + + "parse/aggregate {ParseTime} ({MaxParseTime}), publish {PublishTime} ({MaxPublishTime}), " + + "create run {CreateTime} ({MaxCreateTime}), complete run {CompleteTime} ({MaxCompleteTime}).", + heading, + metrics.Elapsed, + metrics.PipelineElapsed, + Environment.NewLine, + throughputLabel, + throughputSummary, + Environment.NewLine, + metrics.AzureDevOpsRequests, + metrics.AzureDevOpsControlRequests, + metrics.AzureDevOpsResultRequests, + metrics.AzureDevOpsAttachmentRequests, + metrics.AzureDevOpsRetries, + metrics.AzureDevOpsFailedAttempts, + payloadMiB, + metrics.AzureDevOpsRequestTime, + metrics.MaximumAzureDevOpsRequestTime, + Environment.NewLine, + metrics.HelixRequests, + metrics.HelixRetries, + metrics.HelixFailedAttempts, + metrics.ResultBlobDownloads, + metrics.ResultBlobDownloadFailures, + Environment.NewLine, + metrics.RateLimitDeferrals, + metrics.RateLimitDeferredTime, + metrics.MaximumRateLimitDeferral, + metrics.RateLimitWaits, + metrics.RateLimitWaitTime, + metrics.MaximumRateLimitWaitTime, + Environment.NewLine, + metrics.WorkItemDownloadTime, + metrics.MaximumWorkItemDownloadTime, + metrics.ParseTime, + metrics.MaximumParseTime, + metrics.WorkItemPublishTime, + metrics.MaximumWorkItemPublishTime, + metrics.TestRunCreateTime, + metrics.MaximumTestRunCreateTime, + metrics.TestRunCompleteTime, + metrics.MaximumTestRunCompleteTime); + } + public void LogNonMonitorPipelineFailure() { // DO NOT CHANGE THIS LINE - it's matched by Build Analysis to ignore a failure @@ -295,86 +383,6 @@ private void LogWarning(string message) private void LogError(string message) => _logger.LogError("{Prefix}{Message}", AzdoErrorPrefix, message); - private void LogVerboseTree( - IReadOnlyList jobs, - IReadOnlyDictionary> workItemsByJob, - IReadOnlySet completedJobNames) - { - if (jobs.Count == 0) - { - _logger.LogInformation("⏳ Helix job details:{nl}└─ no Helix jobs discovered yet", Environment.NewLine); - return; - } - - var lines = new List(); - for (int jobIndex = 0; jobIndex < jobs.Count; jobIndex++) - { - HelixJobInfo job = jobs[jobIndex]; - IReadOnlyCollection workItems = workItemsByJob[job.JobName]; - AddVerboseJobLines( - lines, - job, - workItems, - GetJobStatus(job, workItems, completedJobNames), - isLastJob: jobIndex == jobs.Count - 1); - } - - _logger.LogInformation("⏳ Helix job details:{nl}{JobDetails}", - Environment.NewLine, - string.Join(Environment.NewLine, lines)); - } - - private static void AddVerboseJobLines( - List lines, - HelixJobInfo job, - IReadOnlyCollection workItems, - string jobStatus, - bool isLastJob) - { - string jobConnector = isLastJob ? "└─" : "├─"; - string childPrefix = isLastJob ? " " : "│ "; - lines.Add($"{jobConnector} 🧪 Helix job {job.DisplayName} [{jobStatus}]"); - - List orderedWorkItems = - [ - ..workItems.OrderBy(wi => wi.Name, StringComparer.OrdinalIgnoreCase) - ]; - - if (orderedWorkItems.Count == 0) - { - lines.Add($"{childPrefix}└─ no work items reported yet"); - return; - } - - for (int i = 0; i < orderedWorkItems.Count; i++) - { - WorkItemSummary workItem = orderedWorkItems[i]; - string connector = i == orderedWorkItems.Count - 1 ? "└─" : "├─"; - string console = workItem.IsFailedAndTerminal - ? $" | Console: {MonitorState.GetConsoleOutputText(workItem.ConsoleOutputUri)}" - : string.Empty; - lines.Add($"{childPrefix}{connector} {workItem.Name} ({workItem.FormattedState}){console}"); - } - } - - private string GetJobStatus( - HelixJobInfo job, - IReadOnlyCollection workItems, - IReadOnlySet completedJobNames) - { - if (_state.IsHelixJobProcessed(job.JobName)) - { - return "Processed"; - } - - if (completedJobNames.Contains(job.JobName)) - { - return "Completed"; - } - - return workItems.Count > 0 ? "Running" : "Waiting"; - } - private JobWorkItemStatusCounts ComputeCounts( IReadOnlyList jobs, IReadOnlyDictionary> workItemsByJob, diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs new file mode 100644 index 00000000000..63446d621e8 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs @@ -0,0 +1,576 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Concurrent; +using Microsoft.Arcade.Common; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; +using Microsoft.DotNet.Helix.Client.Models; +using Microsoft.DotNet.Helix.JobMonitor.Models; +using Microsoft.DotNet.Helix.JobMonitor.Parallelism; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DotNet.Helix.JobMonitor; + +internal sealed class TestResultUploadPipeline : IAsyncDisposable +{ + private const int MaximumTransientDownloadRetries = 2; + private const string AzdoWarningPrefix = "##vso[task.logissue type=warning]"; + + private readonly ILogger _logger; + private readonly JobMonitorOptions _options; + private readonly IAzureDevOpsService _azdo; + private readonly IHelixService _helix; + private readonly MonitorState _state; + private readonly JobMonitorMetrics _metrics; + private readonly ConcurrentDictionary _sessions = + new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _remainingWorkItemsByPoll = []; + private readonly ConcurrentDictionary _acceptedWorkItemsByPoll = []; + private readonly ActionQueue _jobs; + private readonly ActionQueue _workItems; + private readonly ActionQueue _finalizers; + private int _draining; + + public TestResultUploadPipeline( + ILogger logger, + JobMonitorOptions options, + IAzureDevOpsService azdo, + IHelixService helix, + MonitorState state, + JobMonitorMetrics metrics) + { + _logger = logger; + _options = options; + _azdo = azdo; + _helix = helix; + _state = state; + _metrics = metrics; + + int uploadParallelism = options.TestResultUploadParallelism; + _jobs = new ActionQueue( + parallelism: Math.Min(4, uploadParallelism), + ExpandJobAsync); + _workItems = new ActionQueue( + capacity: Math.Max(64, uploadParallelism * 8), + parallelism: uploadParallelism, + ProcessWorkItemAsync); + _finalizers = new ActionQueue( + capacity: Math.Max(64, uploadParallelism * 2), + parallelism: Math.Min(4, uploadParallelism), + FinalizeJobAsync); + } + + public UploadPipelineSnapshot Snapshot => new( + _jobs.Snapshot, + _workItems.Snapshot, + _finalizers.Snapshot, + _sessions.Values.Count(static session => session.HasFailed), + _sessions.Values.Sum(static session => session.UploadedResultCount)); + + public bool TryEnqueue( + HelixJobInfo job, + IReadOnlyCollection workItems, + bool isJobComplete, + int discoveryPoll) + { + if (Volatile.Read(ref _draining) != 0 || _state.IsHelixJobProcessed(job.JobName)) + { + return false; + } + + bool addedSession = false; + JobUploadSession session = _sessions.GetOrAdd( + job.JobName, + _ => + { + addedSession = true; + return new JobUploadSession(job); + }); + if (session.IsFinalized) + { + return false; + } + + IReadOnlyList newWorkItems = session.AddWorkItems( + workItems.Where(static workItem => workItem.ExitCode.HasValue), + isJobComplete); + if (newWorkItems.Count == 0 && !session.IsReadyToFinalize) + { + return false; + } + + if (!_jobs.TryEnqueue(new JobUploadRequest(session, newWorkItems, discoveryPoll))) + { + // Polling is the only producer, the queue is unbounded, and draining is rejected + // above, so this indicates a broken pipeline invariant rather than backpressure. + throw new InvalidOperationException("The test result upload pipeline stopped accepting jobs before drain began."); + } + + _remainingWorkItemsByPoll.AddOrUpdate( + discoveryPoll, + newWorkItems.Count, + (_, remaining) => remaining + newWorkItems.Count); + _acceptedWorkItemsByPoll.AddOrUpdate( + discoveryPoll, + newWorkItems.Count, + (_, accepted) => accepted + newWorkItems.Count); + + if (addedSession) + { + _state.TryQueueHelixJobUpload(job.JobName); + } + + return true; + } + + public async Task DrainAsync( + int finalPoll, + int newlyTerminalWorkItems, + CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _draining, 1) != 0) + { + return; + } + + DateTimeOffset startedAt = DateTimeOffset.UtcNow; + long finalPollRemainingWorkItems = GetRemainingWorkItems(finalPoll); + long priorPollBacklog = _remainingWorkItemsByPoll + .Where(pair => pair.Key != finalPoll) + .Sum(static pair => Math.Max(0, pair.Value)); + long remainingWorkItems = finalPollRemainingWorkItems + priorPollBacklog; + long finalPollEligibleWorkItems = _acceptedWorkItemsByPoll.TryGetValue(finalPoll, out long accepted) + ? accepted + : 0; + int remainingFinalizations = _sessions.Values.Count(static session => !session.IsFinalized); + + _logger.LogInformation( + "Starting final test result drain: {NewlyTerminalWorkItems} work item(s) were first observed " + + "terminal and {FinalPollEligibleWorkItems} work item upload(s) became eligible in the final poll; " + + "{RemainingWorkItems} work item upload(s) remain " + + "({FinalPollRemainingWorkItems} from the final poll, {PriorPollBacklog} from earlier polls), " + + "plus {RemainingFinalizations} job finalization(s).", + newlyTerminalWorkItems, + finalPollEligibleWorkItems, + remainingWorkItems, + finalPollRemainingWorkItems, + priorPollBacklog, + remainingFinalizations); + + _jobs.Complete(); + await _jobs.DrainAsync().WaitAsync(cancellationToken); + + _workItems.Complete(); + await _workItems.DrainAsync().WaitAsync(cancellationToken); + + _finalizers.Complete(); + await _finalizers.DrainAsync().WaitAsync(cancellationToken); + + UploadPipelineSnapshot snapshot = Snapshot; + _logger.LogInformation( + "Test result pipeline drained in {Elapsed}. {JobCount} job(s), {WorkItemCount} work item(s), " + + "and {ResultCount} result(s) were processed; {FailedJobCount} job upload(s) remain untagged.", + DateTimeOffset.UtcNow - startedAt, + _sessions.Count, + snapshot.WorkItems.Completed, + snapshot.UploadedResults, + snapshot.FailedJobs); + } + + public void Cancel() + { + _jobs.Cancel(); + _workItems.Cancel(); + _finalizers.Cancel(); + } + + public async ValueTask DisposeAsync() + { + Cancel(); + await _jobs.DisposeAsync(); + await _workItems.DisposeAsync(); + await _finalizers.DisposeAsync(); + } + + private async ValueTask ExpandJobAsync(JobUploadRequest request, CancellationToken cancellationToken) + { + JobUploadSession session = request.Session; + _state.MarkHelixJobUploadInProgress(session.Job.JobName); + + foreach (string workItemName in request.WorkItemNames) + { + await _workItems.EnqueueAsync( + new WorkItemUploadRequest(session, workItemName, request.DiscoveryPoll), + cancellationToken); + } + + if (session.TryQueueFinalizer()) + { + await _finalizers.EnqueueAsync(session, cancellationToken); + } + } + + private long GetRemainingWorkItems(int discoveryPoll) + => _remainingWorkItemsByPoll.TryGetValue(discoveryPoll, out long remaining) + ? Math.Max(0, remaining) + : 0; + + private async ValueTask ProcessWorkItemAsync( + WorkItemUploadRequest request, + CancellationToken cancellationToken) + { + JobUploadSession session = request.Session; + try + { + WorkItemTestResults downloaded; + long downloadStartedAt = JobMonitorMetrics.StartOperation(); + try + { + downloaded = await ExecuteDownloadWithRetryAsync( + session.Job, + request.WorkItemName, + cancellationToken); + } + finally + { + _metrics.RecordPipelineOperation( + PipelineOperation.WorkItemDownload, + downloadStartedAt); + } + + int testRunId = await session.GetOrCreateTestRunAsync( + () => CreateTestRunAsync(session.Job.TestRunName, cancellationToken)); + + TestResultUploadSummary summary = + await _azdo.UploadTestResultsAsync(testRunId, downloaded, cancellationToken); + + session.RecordSuccess( + request.WorkItemName, + downloaded.TestResultFiles.Count, + summary); + if (_options.FailWorkItemsWithFailedTests) + { + _state.ObserveTestResult(session.Job.JobName, request.WorkItemName, summary); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + session.RecordFailure(); + LogUploadFailure( + ex, + $"process test results for work item '{request.WorkItemName}' in job '{session.Job.DisplayName}'"); + } + finally + { + _remainingWorkItemsByPoll.AddOrUpdate( + request.DiscoveryPoll, + 0, + static (_, remaining) => remaining - 1); + if (session.MarkWorkItemFinished()) + { + await _finalizers.EnqueueAsync(session, cancellationToken); + } + } + } + + private async ValueTask FinalizeJobAsync( + JobUploadSession session, + CancellationToken cancellationToken) + { + try + { + if (session.HasFailed) + { + _state.MarkHelixJobUploadFailed(session.Job.JobName); + return; + } + + try + { + int testRunId = await session.GetOrCreateTestRunAsync( + () => CreateTestRunAsync(session.Job.TestRunName, cancellationToken)); + long completeStartedAt = JobMonitorMetrics.StartOperation(); + try + { + await _azdo.CompleteTestRunAsync( + testRunId, + session.Job.JobName, + session.FailedWorkItems, + cancellationToken); + } + finally + { + _metrics.RecordPipelineOperation( + PipelineOperation.TestRunComplete, + completeStartedAt); + } + + _state.TryMarkHelixJobProcessed(session.Job.JobName); + _logger.LogInformation( + "Test result processing completed for job '{JobName}': {WorkItemCount} work item(s), " + + "{ResultFileCount} recognized result file(s), and {UploadedCount} test result(s) uploaded.", + session.Job.DisplayName, + session.WorkItemCount, + session.ResultFileCount, + session.UploadedResultCount); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + session.RecordFailure(); + _state.MarkHelixJobUploadFailed(session.Job.JobName); + LogUploadFailure(ex, $"complete Azure DevOps test run for job '{session.Job.DisplayName}'"); + } + } + finally + { + session.MarkFinalized(); + } + } + + private async Task CreateTestRunAsync( + string testRunName, + CancellationToken cancellationToken) + { + long startedAt = JobMonitorMetrics.StartOperation(); + try + { + return await _azdo.CreateTestRunAsync(testRunName, cancellationToken); + } + finally + { + _metrics.RecordPipelineOperation(PipelineOperation.TestRunCreate, startedAt); + } + } + + private async Task ExecuteDownloadWithRetryAsync( + HelixJobInfo job, + string workItemName, + CancellationToken cancellationToken) + { + WorkItemTestResults result = null; + Exception lastException = null; + var retry = new ExponentialRetry + { + MaxAttempts = MaximumTransientDownloadRetries + 1, + RetryDelayCallback = (attempt, delay) => + _logger.LogDebug( + "Transient result download failure for '{JobName}/{WorkItemName}' on attempt {Attempt}. " + + "Retrying after {Delay}.", + job.DisplayName, + workItemName, + attempt, + delay), + }; + + bool succeeded = await retry.RunAsync( + async _ => + { + try + { + result = await _helix.DownloadTestResultsAsync( + job.JobName, + workItemName, + _options.WorkingDirectory, + cancellationToken); + return RetryResult.Success; + } + catch (Exception ex) when ( + !cancellationToken.IsCancellationRequested + && TransientFailureDetector.IsTransient(ex)) + { + lastException = ex; + return RetryResult.Retry(); + } + }, + cancellationToken); + + return succeeded + ? result + : throw lastException ?? new InvalidOperationException("Result download retry exited unexpectedly."); + } + + private void LogUploadFailure(Exception exception, string operation) + => _logger.LogWarning( + exception, + "{Prefix}Failed to {Operation}. The Helix job remains untagged so a later monitor invocation can replay it.", + AzdoWarningPrefix, + operation); + + private sealed record JobUploadRequest( + JobUploadSession Session, + IReadOnlyList WorkItemNames, + int DiscoveryPoll); + + private sealed record WorkItemUploadRequest( + JobUploadSession Session, + string WorkItemName, + int DiscoveryPoll); + + private sealed class JobUploadSession + { + private readonly object _sync = new(); + private readonly HashSet _failedWorkItems = new(StringComparer.OrdinalIgnoreCase); + private readonly HashSet _workItems = new(StringComparer.OrdinalIgnoreCase); + private Task _testRunTask; + private int _pendingWorkItems; + private int _jobComplete; + private int _finalizerQueued; + private int _finalized; + private int _failed; + private long _resultFileCount; + private long _uploadedResultCount; + + public JobUploadSession(HelixJobInfo job) + { + Job = job; + } + + public HelixJobInfo Job { get; } + + public int WorkItemCount + { + get { lock (_sync) { return _workItems.Count; } } + } + + public bool IsFinalized => Volatile.Read(ref _finalized) != 0; + + public bool IsReadyToFinalize + { + get + { + lock (_sync) + { + return IsReadyToFinalizeLocked(); + } + } + } + + public bool HasFailed => Volatile.Read(ref _failed) != 0; + + public long UploadedResultCount => Interlocked.Read(ref _uploadedResultCount); + + public long ResultFileCount => Interlocked.Read(ref _resultFileCount); + + public IReadOnlyCollection FailedWorkItems + { + get + { + lock (_sync) + { + return [.. _failedWorkItems]; + } + } + } + + public Task GetOrCreateTestRunAsync(Func> create) + { + lock (_sync) + { + return _testRunTask ??= InvokeCreate(); + } + + Task InvokeCreate() + { + try + { + return create(); + } + catch (Exception ex) + { + return Task.FromException(ex); + } + } + } + + public void RecordSuccess( + string workItemName, + int resultFileCount, + TestResultUploadSummary summary) + { + Interlocked.Add(ref _resultFileCount, resultFileCount); + Interlocked.Add(ref _uploadedResultCount, summary.UploadedCount); + if (!summary.AllPassed) + { + lock (_sync) + { + _failedWorkItems.Add(workItemName); + } + } + } + + public void RecordFailure() => Interlocked.Exchange(ref _failed, 1); + + public IReadOnlyList AddWorkItems( + IEnumerable workItems, + bool isJobComplete) + { + lock (_sync) + { + var added = new List(); + foreach (WorkItemSummary workItem in workItems) + { + if (_workItems.Add(workItem.Name)) + { + added.Add(workItem.Name); + _pendingWorkItems++; + } + } + + if (isJobComplete) + { + _jobComplete = 1; + } + + return added; + } + } + + public bool MarkWorkItemFinished() + { + lock (_sync) + { + _pendingWorkItems--; + return TryQueueFinalizerLocked(); + } + } + + public bool TryQueueFinalizer() + { + lock (_sync) + { + return TryQueueFinalizerLocked(); + } + } + + public void MarkFinalized() => Interlocked.Exchange(ref _finalized, 1); + + private bool TryQueueFinalizerLocked() + { + if (!IsReadyToFinalizeLocked() || _finalizerQueued != 0) + { + return false; + } + + _finalizerQueued = 1; + return true; + } + + private bool IsReadyToFinalizeLocked() + => _jobComplete != 0 && _pendingWorkItems == 0; + } +} + +internal readonly record struct UploadPipelineSnapshot( + QueueSnapshot Jobs, + QueueSnapshot WorkItems, + QueueSnapshot Finalizers, + int FailedJobs, + long UploadedResults); diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadQueue.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadQueue.cs deleted file mode 100644 index adfce8fb745..00000000000 --- a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadQueue.cs +++ /dev/null @@ -1,379 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Arcade.Common; -using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; -using Microsoft.DotNet.Helix.Client.Models; -using Microsoft.DotNet.Helix.JobMonitor.Models; -using Microsoft.Extensions.Logging; - -namespace Microsoft.DotNet.Helix.JobMonitor -{ - /// - /// Fire-and-forget queue for AzDO test-result uploads. Each queued upload runs as an - /// independent task. Transient reads and test-result/attachment publishing use bounded - /// retries; test-run creation and completion are attempted once. On normal completion the - /// queue is drained so results in flight when the runner exits are not lost. On cancellation - /// the queue is intentionally NOT drained: cancelling the in-flight Helix jobs takes priority, - /// and any unfinished upload is re-uploaded in full by a later monitor invocation (a Helix job - /// is only "processed" once its test run reaches the Completed state). - /// - internal sealed class TestResultUploadQueue - { - private const int MaximumTransientRetries = 2; - private const string AzdoWarningPrefix = "##vso[task.logissue type=warning]"; - - private readonly ILogger _logger; - private readonly JobMonitorOptions _options; - private readonly IAzureDevOpsService _azdo; - private readonly IHelixService _helix; - private readonly MonitorState _monitorState; - private readonly List _pending = []; - - public TestResultUploadQueue( - ILogger logger, - JobMonitorOptions options, - IAzureDevOpsService azdo, - IHelixService helix, - MonitorState monitorState) - { - _logger = logger; - _options = options; - _azdo = azdo; - _helix = helix; - _monitorState = monitorState; - } - - public void Enqueue(HelixJobInfo helixJob, IReadOnlyCollection workItems, CancellationToken cancellationToken) - { - IReadOnlyList workItemNames = [.. workItems.Select(w => w.Name)]; - var pendingUpload = new PendingUpload(helixJob); - // Scheduling uses CancellationToken.None so the upload task is always allowed to start. - // The upload body still observes the runner's token, so when the runner is cancelled the - // upload stops promptly and the job's results are re-uploaded by a later invocation. - pendingUpload.Task = Task.Run( - () => UploadAsync(pendingUpload, workItemNames, cancellationToken), - CancellationToken.None); - _pending.Add(pendingUpload); - } - - public void Prune() - { - _pending.RemoveAll(static upload => upload.Task.IsCompleted); - } - - public async Task DrainAsync(CancellationToken cancellationToken) - { - Prune(); - if (_pending.Count == 0) - { - return; - } - - _logger.LogInformation("Waiting for {Count} pending test result upload(s) to complete.", _pending.Count); - try - { - Task allUploads = Task.WhenAll(_pending.Select(upload => upload.Task)); - if (_options.Verbose) - { - LogPendingUploads(); - TimeSpan heartbeatInterval = TimeSpan.FromSeconds(Math.Max(1, _options.PollingIntervalSeconds)); - while (!allUploads.IsCompleted) - { - Task heartbeat = Task.Delay(heartbeatInterval, cancellationToken); - if (await Task.WhenAny(allUploads, heartbeat) == allUploads) - { - break; - } - - cancellationToken.ThrowIfCancellationRequested(); - Prune(); - LogPendingUploads(); - } - } - - await allUploads.WaitAsync(cancellationToken); - } - catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) - { - // One or more upload tasks were cancelled; treat as best-effort drained. - } - Prune(); - } - - private async Task UploadAsync( - PendingUpload pendingUpload, - IReadOnlyCollection workItemNames, - CancellationToken cancellationToken) - { - HelixJobInfo helixJob = pendingUpload.HelixJob; - _monitorState.MarkHelixJobUploadInProgress(helixJob.JobName); - - SetPhase(pendingUpload, $"downloading Helix test results for {workItemNames.Count} work item(s)"); - (bool downloadedSuccessfully, IReadOnlyList downloaded) = await TryExecuteWithRetryAsync( - () => _helix.DownloadTestResultsAsync( - helixJob.JobName, - workItemNames, - _options.WorkingDirectory, - cancellationToken), - "download the Helix test results", - helixJob, - testRunId: 0, - retryCount: MaximumTransientRetries, - cancellationToken); - if (!downloadedSuccessfully) - { - SetPhase(pendingUpload, "failed while downloading Helix test results"); - _monitorState.MarkHelixJobUploadFailed(helixJob.JobName); - return; - } - - SetPhase(pendingUpload, "creating the Azure DevOps test run"); - (bool created, int testRunId) = await TryExecuteAsync( - () => _azdo.CreateTestRunAsync(helixJob.TestRunName, cancellationToken), - "create the Azure DevOps test run", - helixJob, - testRunId: 0, - cancellationToken); - if (!created) - { - SetPhase(pendingUpload, "failed while creating the Azure DevOps test run"); - _monitorState.MarkHelixJobUploadFailed(helixJob.JobName); - return; - } - - SetPhase(pendingUpload, $"publishing {downloaded.Count} work item(s) to Azure DevOps test run {testRunId}"); - (bool uploadedSuccessfully, IReadOnlyDictionary<(string JobName, string WorkItemName), TestResultUploadSummary> testResults) - = await TryExecuteAsync( - () => _azdo.UploadTestResultsAsync(testRunId, downloaded, cancellationToken), - "upload the test results to Azure DevOps", - helixJob, - testRunId, - cancellationToken); - if (!uploadedSuccessfully) - { - SetPhase(pendingUpload, $"failed while publishing to Azure DevOps test run {testRunId}"); - _monitorState.MarkHelixJobUploadFailed(helixJob.JobName); - return; - } - - if (_options.FailWorkItemsWithFailedTests) - { - _monitorState.ObserveTestResults(testResults); - } - - IReadOnlyCollection failedWorkItems = - [ - .. testResults - .Where(kv => !kv.Value.AllPassed) - .Select(kv => kv.Key.WorkItemName) - ]; - - SetPhase(pendingUpload, $"completing and tagging Azure DevOps test run {testRunId}"); - (bool completed, _) = await TryExecuteAsync( - async () => - { - await _azdo.CompleteTestRunAsync(testRunId, helixJob.JobName, failedWorkItems, cancellationToken); - return true; - }, - "complete and tag the Azure DevOps test run", - helixJob, - testRunId, - cancellationToken); - if (!completed) - { - SetPhase(pendingUpload, $"failed while completing Azure DevOps test run {testRunId}"); - _monitorState.MarkHelixJobUploadFailed(helixJob.JobName); - return; - } - - SetPhase(pendingUpload, $"completed Azure DevOps test run {testRunId}"); - _monitorState.TryMarkHelixJobProcessed(helixJob.JobName); - - long uploadedCount = testResults.Values.Sum(r => r.UploadedCount); - _logger.LogInformation("{UploadedCount} test results for job '{JobName}' processed.", - uploadedCount, - helixJob.DisplayName); - } - - private Task<(bool Success, T Result)> TryExecuteAsync( - Func> operation, - string operationDescription, - HelixJobInfo helixJob, - int testRunId, - CancellationToken cancellationToken) - => TryExecuteWithRetryAsync( - operation, - operationDescription, - helixJob, - testRunId, - retryCount: 0, - cancellationToken); - - private async Task<(bool Success, T Result)> TryExecuteWithRetryAsync( - Func> operation, - string operationDescription, - HelixJobInfo helixJob, - int testRunId, - int retryCount, - CancellationToken cancellationToken) - { - try - { - T result = default; - Exception lastException = null; - var retry = new ExponentialRetry - { - MaxAttempts = retryCount + 1, - RetryDelayCallback = (failedAttempt, delay) => - _logger.LogDebug( - "Failed to {OperationDescription} for job {JobName}. Test run ID was {TestRunId}. " - + "Waiting {RetryDelay} before attempt {NextAttempt} of {AttemptCount}.", - operationDescription, - helixJob.DisplayName, - testRunId, - delay, - failedAttempt + 1, - retryCount + 1), - }; - - bool succeeded = await retry.RunAsync( - async attempt => - { - try - { - result = await operation(); - return true; - } - catch (Exception ex) when ( - !cancellationToken.IsCancellationRequested - && TransientFailureDetector.IsTransient(ex)) - { - lastException = ex; - _logger.LogDebug(ex, - "Failed to {OperationDescription} for job {JobName}. Test run ID was {TestRunId}. " - + "Transient attempt {Attempt} of {AttemptCount} failed.", - operationDescription, - helixJob.DisplayName, - testRunId, - attempt + 1, - retryCount + 1); - return false; - } - }, - cancellationToken); - - if (!succeeded) - { - cancellationToken.ThrowIfCancellationRequested(); - throw lastException ?? new InvalidOperationException("Upload retry loop exited unexpectedly."); - } - - return (true, result); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception ex) - { - string failureKind = TransientFailureDetector.IsTransient(ex) - ? retryCount > 0 - ? "Transient retry limit reached." - : "The operation may have partially completed and is not safe to replay in this invocation." - : "The failure is not retryable."; - _logger.LogWarning(ex, - "{Prefix}Failed to {OperationDescription} for job {JobName}. Test run ID was {TestRunId}. " - + "{FailureKind} The run remains untagged and a later monitor invocation may retry the upload.", - AzdoWarningPrefix, - operationDescription, - helixJob.DisplayName, - testRunId, - failureKind); - return (false, default); - } - } - - private void SetPhase(PendingUpload upload, string phase) - { - upload.SetPhase(phase); - if (_options.Verbose) - { - _logger.LogDebug( - "Test result upload for job '{JobName}' entered phase '{Phase}' after {Elapsed}.", - upload.HelixJob.DisplayName, - phase, - DateTimeOffset.UtcNow - upload.StartedAt); - } - } - - private void LogPendingUploads() - { - if (_pending.Count == 0) - { - return; - } - - DateTimeOffset now = DateTimeOffset.UtcNow; - string details = string.Join( - Environment.NewLine, - _pending - .OrderBy(upload => upload.StartedAt) - .Select(upload => - { - (string phase, DateTimeOffset phaseStartedAt) = upload.GetPhase(); - return $"- {upload.HelixJob.DisplayName}: phase='{phase}', " - + $"phase elapsed={now - phaseStartedAt:c}, total elapsed={now - upload.StartedAt:c}"; - })); - - _logger.LogDebug( - "{Count} test result upload(s) remain pending:{nl}{Details}", - _pending.Count, - Environment.NewLine, - details); - } - - private sealed class PendingUpload - { - private readonly object _sync = new(); - private string _phase = "queued"; - private DateTimeOffset _phaseStartedAt; - - public PendingUpload(HelixJobInfo helixJob) - { - HelixJob = helixJob; - StartedAt = DateTimeOffset.UtcNow; - _phaseStartedAt = StartedAt; - } - - public HelixJobInfo HelixJob { get; } - - public DateTimeOffset StartedAt { get; } - - public Task Task { get; set; } - - public void SetPhase(string phase) - { - lock (_sync) - { - _phase = phase; - _phaseStartedAt = DateTimeOffset.UtcNow; - } - } - - public (string Phase, DateTimeOffset PhaseStartedAt) GetPhase() - { - lock (_sync) - { - return (_phase, _phaseStartedAt); - } - } - } - - } -} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs new file mode 100644 index 00000000000..6882b48f494 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs @@ -0,0 +1,562 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; +using System.Text.Json; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; +using Microsoft.DotNet.Helix.JobMonitor; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; + +internal sealed class AzureDevOpsResultPublisher +{ + // Azure DevOps rejects requests containing more than 1,000 top-level TestCaseResult objects. + // Nested sub-results do not count toward this limit. + private const int MaximumResultsPerRequest = 1000; + + // Preserve the legacy bound on the recursive size of one result hierarchy independently + // from the top-level per-request limit. + private const int MaximumNodesPerResultHierarchy = 950; + + private readonly TestResultAttachmentMode _attachmentMode; + private readonly bool _useFullyQualifiedTestName; + private readonly ILogger _logger; + private readonly JobMonitorMetrics _metrics; + private readonly IAzureDevOpsResultTransport _transport; + + internal AzureDevOpsResultPublisher( + TestResultAttachmentMode attachmentMode, + bool useFullyQualifiedTestName, + ILogger logger, + IAzureDevOpsResultTransport transport, + JobMonitorMetrics? metrics = null) + { + _attachmentMode = attachmentMode; + _useFullyQualifiedTestName = useFullyQualifiedTestName; + _logger = logger; + _transport = transport; + _metrics = metrics ?? new JobMonitorMetrics(); + } + + public async Task UploadTestResultsWithSummaryAsync(List testResultFiles, object resultMetadata, CancellationToken cancellationToken = default) + { + long parseStartedAt = JobMonitorMetrics.StartOperation(); + bool parseRecorded = false; + try + { + var testResultReader = new LocalTestResultsReader(_logger, _attachmentMode); + + var parsedResults = new List>(testResultFiles.Count); + foreach (string file in testResultFiles) + { + parsedResults.Add(await testResultReader.ReadResultFileAsync(file, cancellationToken)); + } + + if (parsedResults.Count == 0) + { + _logger.LogWarning("No test result files were provided for upload"); + return new TestResultUploadSummary(true, 0); + } + + IReadOnlyList aggregatedResults = new ResultAggregator().Aggregate(parsedResults, _useFullyQualifiedTestName); + _metrics.RecordPipelineOperation(PipelineOperation.ResultParseAndAggregate, parseStartedAt); + parseRecorded = true; + if (aggregatedResults.Count == 0) + { + _logger.LogDebug("Test results were discovered but none could be aggregated"); + return new TestResultUploadSummary(true, 0); + } + + long publishStartedAt = JobMonitorMetrics.StartOperation(); + long uploadedCount; + try + { + uploadedCount = await UploadTestResultsWithCountAsync( + aggregatedResults, + resultMetadata, + cancellationToken); + } + finally + { + _metrics.RecordPipelineOperation(PipelineOperation.WorkItemPublish, publishStartedAt); + } + return new TestResultUploadSummary( + AllPassed: ComputeAllPassed(aggregatedResults), + UploadedCount: uploadedCount); + } + finally + { + if (!parseRecorded) + { + _metrics.RecordPipelineOperation(PipelineOperation.ResultParseAndAggregate, parseStartedAt); + } + } + } + + /// + /// A work item's uploaded results are only considered a failure when a test actually failed + /// or could not be parsed into a known outcome ("None"). "Inconclusive" is a legitimate, + /// non-failing outcome produced by the aggregator for data-driven tests that mix passing and + /// skipped data rows (see ), so it must not fail the work item. + /// + internal static bool ComputeAllPassed(IReadOnlyList results) + => results.All(result => result.Result != "Failed" && result.Result != "None"); + + public async Task UploadTestResultsWithCountAsync(IEnumerable results, object resultMetadata, CancellationToken cancellationToken = default) + { + try + { + long publishedTestCount = 0; + foreach (List requestBatch in CreateResultRequestBatches(ConvertResults(results, resultMetadata))) + { + IReadOnlyList publishedTests = await PublishResultsAsync(requestBatch, cancellationToken); + publishedTestCount += publishedTests.Count; + } + + _logger.LogDebug("Uploaded {Count} results", publishedTestCount); + + return publishedTestCount; + } + catch (TerminalError ex) + { + _logger.LogError(ex, "Failed to upload test results to Azure DevOps."); + throw; + } + } + + private async Task> PublishResultsAsync( + IReadOnlyList converted, + CancellationToken cancellationToken) + { + var testCaseResults = converted.Select(static c => c.Converted).ToList(); + var originalList = converted.Select(static c => c.Aggregated).ToList(); + + string response = await _transport.PublishResultsAsync(testCaseResults, cancellationToken); + IReadOnlyList publishedResults = ReadPublishedResults(response); + if (publishedResults.Count == 0) + { + _logger.LogWarning("The test run appears to have been closed, aborting test result uploads."); + return []; + } + + List publishedTestCases = []; + + foreach ((PublishedTestCaseResultReference published, AggregatedResult original, PublishedTestCase testCase) in publishedResults.Zip(originalList, testCaseResults)) + { + if (published.Id == -1) + { + _logger.LogWarning("Azure DevOps test ID returned -1, unable to attach files."); + continue; + } + + async Task IterateSubResultsAsync( + IReadOnlyList? publishedSubResults, + IReadOnlyList originalSubResults, + long testId) + { + if (publishedSubResults is null || publishedSubResults.Count == 0) + { + if (originalSubResults.Count > 0) + { + _logger.LogError("Published results do not include sub-results, attachments lost."); + } + + return; + } + + if (publishedSubResults.Count != originalSubResults.Count) + { + _logger.LogError("Published sub-result counts do not match uploaded attachments. Attachments lost."); + return; + } + + foreach ((PublishedSubResultReference publishedSubResult, AggregatedResult originalSubResult) subTriplet in publishedSubResults.Zip(originalSubResults, (publishedSubResult, originalSubResult) => (publishedSubResult, originalSubResult))) + { + foreach (TestResultAttachment attachment in subTriplet.originalSubResult.Attachments) + { + await SendAttachmentAsync(attachment, testId, subTriplet.publishedSubResult.Id, cancellationToken); + } + + await IterateSubResultsAsync(subTriplet.publishedSubResult.SubResults, subTriplet.originalSubResult.SubResults, testId); + } + } + + foreach (TestResultAttachment attachment in original.Attachments) + { + await SendAttachmentAsync(attachment, published.Id, null, cancellationToken); + } + + await IterateSubResultsAsync(published.SubResults, original.SubResults, published.Id); + + publishedTestCases.Add(testCase); + } + + return publishedTestCases; + } + + private async Task SendAttachmentAsync( + TestResultAttachment attachment, + long testId, + long? subResultId, + CancellationToken cancellationToken) + { + await _transport.UploadAttachmentAsync( + testId, + subResultId, + attachment.Name, + Convert.ToBase64String(Encoding.UTF8.GetBytes(attachment.Text)), + cancellationToken); + } + + private IEnumerable ConvertResults(IEnumerable results, object resultMetadata) + { + static string GetResultGroupType(AggregationType aggregationType) + { + return aggregationType switch + { + AggregationType.Single => "None", + AggregationType.DataDriven => "dataDriven", + AggregationType.Rerun => "rerun", + _ => "None", + }; + } + + string comment = JsonSerializer.Serialize(resultMetadata) ?? string.Empty; + bool useFullyQualifiedName = _useFullyQualifiedTestName; + + string DisplayNameFor(AggregatedResult result) + => useFullyQualifiedName + ? TestNameFormatter.FormatDisplayName(result.FullyQualifiedName, result.Name) + : result.Name; + + PublishedSubResult ConvertToSubTest(AggregatedResult result) + { + var customFields = new List(); + if (result.IsFlaky) + { + customFields.Add(new CustomField("IsTestResultFlaky", true)); + } + + if ((result.AttemptId ?? 0) > 1) + { + customFields.Add(new CustomField("AttemptId", result.AttemptId!.Value - 1)); + } + + return new PublishedSubResult + { + Comment = comment, + CustomFields = customFields, + DisplayName = DisplayNameFor(result), + Outcome = result.Result, + DurationInMs = result.DurationSeconds * 1000.0, + StackTrace = result.StackTrace, + ErrorMessage = result.FailureMessage, + SubResults = result.SubResults.Count == 0 ? null : [.. result.SubResults.Select(ConvertToSubTest)], + ResultGroupType = GetResultGroupType(result.AggregationType), + }; + } + + ConvertedResult ConvertResult(AggregatedResult result) + { + var customFields = new List(); + if (result.IsFlaky) + { + customFields.Add(new CustomField("IsTestResultFlaky", true)); + } + + if (result.AggregationType == AggregationType.Rerun && result.SubResults.Count > 1) + { + customFields.Add(new CustomField("AttemptId", result.SubResults.Count - 1)); + } + + string displayName = DisplayNameFor(result); + + return new ConvertedResult( + new PublishedTestCase + { + TestCaseTitle = displayName, + AutomatedTestName = useFullyQualifiedName ? result.FullyQualifiedName : result.Name, + AutomatedTestType = "helix", + AutomatedTestStorage = comment, // TODO: This was workitem ID + Priority = 1, + DurationInMs = result.DurationSeconds * 1000.0, + Outcome = result.Result, + State = "Completed", + Comment = comment, + StackTrace = result.StackTrace, + ErrorMessage = result.FailureMessage, + SubResults = result.SubResults.Count == 0 ? null : [.. result.SubResults.Select(ConvertToSubTest)], + ResultGroupType = GetResultGroupType(result.AggregationType), + CustomFields = customFields, + }, + result); + } + + foreach (AggregatedResult result in results) + { + foreach (ConvertedResult hierarchyPart in SplitOversizedResultHierarchy( + ConvertResult(result), + MaximumNodesPerResultHierarchy)) + { + yield return hierarchyPart; + } + } + } + + /// + /// Groups converted top-level results into Azure DevOps request bodies. Each item counts + /// once regardless of how many nested sub-results it contains. + /// + private static IEnumerable> CreateResultRequestBatches( + IEnumerable results) + => PartitionBySize(results, MaximumResultsPerRequest, static _ => 1); + + /// + /// Splits one logical data-driven or rerun result into multiple top-level payload entries + /// when its recursive hierarchy exceeds . + /// This is separate from grouping top-level entries into Azure DevOps request batches. + /// + private static IEnumerable SplitOversizedResultHierarchy( + ConvertedResult test, + int maximumNodesPerHierarchy) + { + if (CountResultTreeNodes(test.Converted) <= maximumNodesPerHierarchy) + { + yield return test; + yield break; + } + + if (maximumNodesPerHierarchy <= 1 || test.Converted.SubResults is not { Count: > 0 }) + { + throw new InvalidOperationException( + "A test-result hierarchy is deeper than the Azure DevOps hierarchy limit."); + } + + IEnumerable splitSubTests = (test.Converted.SubResults ?? []) + .Zip(test.Aggregated.SubResults, (converted, aggregated) => new ChunkPair(converted, aggregated)) + .SelectMany(pair => SplitOversizedSubResultHierarchy(pair, maximumNodesPerHierarchy - 1)); + + // Each emitted hierarchy includes the copied top-level result, leaving the remaining + // node budget for its sub-results. + foreach (List hierarchyPart in PartitionBySize( + splitSubTests, + maximumNodesPerHierarchy - 1, + static pair => CountResultTreeNodes(pair.Converted))) + { + yield return new ConvertedResult( + test.Converted with { SubResults = [.. hierarchyPart.Select(static x => x.Converted)], Id = null }, + CopyAggregatedResult( + test.Aggregated, + [.. hierarchyPart.Select(static x => x.Aggregated)])); + } + } + + private static IEnumerable SplitOversizedSubResultHierarchy( + ChunkPair test, + int maximumNodesPerHierarchy) + { + if (CountResultTreeNodes(test.Converted) <= maximumNodesPerHierarchy) + { + yield return test; + yield break; + } + + IEnumerable splitSubTests = (test.Converted.SubResults ?? []) + .Zip(test.Aggregated.SubResults, (converted, aggregated) => new ChunkPair(converted, aggregated)) + .SelectMany(pair => SplitOversizedSubResultHierarchy(pair, maximumNodesPerHierarchy - 1)); + + foreach (List hierarchyPart in PartitionBySize( + splitSubTests, + maximumNodesPerHierarchy - 1, + static pair => CountResultTreeNodes(pair.Converted))) + { + yield return new ChunkPair( + test.Converted with + { + SubResults = [.. hierarchyPart.Select(static x => x.Converted)], + Id = null, + }, + CopyAggregatedResult( + test.Aggregated, + [.. hierarchyPart.Select(static x => x.Aggregated)])); + } + } + + private static AggregatedResult CopyAggregatedResult( + AggregatedResult result, + IReadOnlyList subResults) + => new( + result.AggregationType, + result.Name, + result.DurationSeconds, + result.Result, + subResults, + result.Attachments, + result.FailureMessage, + result.StackTrace, + isFlaky: result.IsFlaky, + attemptId: result.AttemptId, + fullyQualifiedName: result.FullyQualifiedName); + + private static int CountResultTreeNodes(PublishedTestCase test) + { + return 1 + (test.SubResults?.Sum(CountResultTreeNodes) ?? 0); + } + + private static int CountResultTreeNodes(PublishedSubResult test) + { + return 1 + (test.SubResults?.Sum(CountResultTreeNodes) ?? 0); + } + + /// + /// Partitions items in order so the sum of values in each + /// partition does not exceed . + /// + private static IEnumerable> PartitionBySize( + IEnumerable items, + int maximumPartitionSize, + Func getSize) + { + var currentPartition = new List(); + int currentSize = 0; + + foreach (T? item in items) + { + int size = getSize(item); + if (size > maximumPartitionSize) + { + throw new InvalidOperationException("Cannot partition an item larger than the size limit."); + } + + if (currentSize + size > maximumPartitionSize && currentPartition.Count > 0) + { + yield return currentPartition; + currentPartition = []; + currentSize = 0; + } + + currentPartition.Add(item); + currentSize += size; + } + + if (currentPartition.Count > 0) + { + yield return currentPartition; + } + } + + private static IReadOnlyList ReadPublishedResults(string content) + { + if (string.IsNullOrWhiteSpace(content)) + { + return []; + } + + using var document = JsonDocument.Parse(content); + JsonElement root = document.RootElement; + if (root.ValueKind == JsonValueKind.Array) + { + return [.. root.EnumerateArray().Select(ParsePublishedResult)]; + } + + if (root.TryGetProperty("value", out JsonElement value) && value.ValueKind == JsonValueKind.Array) + { + return [.. value.EnumerateArray().Select(ParsePublishedResult)]; + } + + return []; + } + + private static PublishedTestCaseResultReference ParsePublishedResult(JsonElement element) + { + var subResults = new List(); + if (element.TryGetProperty("subResults", out JsonElement subResultElement) && subResultElement.ValueKind == JsonValueKind.Array) + { + subResults.AddRange(subResultElement.EnumerateArray().Select(ParsePublishedSubResult)); + } + + return new PublishedTestCaseResultReference( + element.TryGetProperty("id", out JsonElement idElement) ? idElement.GetInt64() : -1, + subResults); + } + + private static PublishedSubResultReference ParsePublishedSubResult(JsonElement element) + { + var subResults = new List(); + if (element.TryGetProperty("subResults", out JsonElement subResultElement) && subResultElement.ValueKind == JsonValueKind.Array) + { + subResults.AddRange(subResultElement.EnumerateArray().Select(ParsePublishedSubResult)); + } + + return new PublishedSubResultReference( + element.TryGetProperty("id", out JsonElement idElement) ? idElement.GetInt64() : -1, + subResults); + } + + private sealed record ConvertedResult(PublishedTestCase Converted, AggregatedResult Aggregated); + + private sealed record ChunkPair(PublishedSubResult Converted, AggregatedResult Aggregated); + + private sealed record CustomField(string FieldName, object Value); + + private sealed record PublishedTestCase + { + public long? Id { get; init; } + + public string TestCaseTitle { get; init; } = string.Empty; + + public string AutomatedTestName { get; init; } = string.Empty; + + public string AutomatedTestType { get; init; } = string.Empty; + + public string AutomatedTestStorage { get; init; } = string.Empty; + + public int Priority { get; init; } + + public double DurationInMs { get; init; } + + public string Outcome { get; init; } = string.Empty; + + public string State { get; init; } = string.Empty; + + public string Comment { get; init; } = string.Empty; + + public string? StackTrace { get; init; } + + public string? ErrorMessage { get; init; } + + public List? SubResults { get; init; } + + public string ResultGroupType { get; init; } = string.Empty; + + public List? CustomFields { get; init; } + } + + private sealed record PublishedSubResult + { + public long? Id { get; init; } + + public string Comment { get; init; } = string.Empty; + + public List? CustomFields { get; init; } + + public string DisplayName { get; init; } = string.Empty; + + public string Outcome { get; init; } = string.Empty; + + public double DurationInMs { get; init; } + + public string? StackTrace { get; init; } + + public string? ErrorMessage { get; init; } + + public List? SubResults { get; init; } + + public string ResultGroupType { get; init; } = string.Empty; + } + + private sealed record PublishedTestCaseResultReference(long Id, IReadOnlyList SubResults); + + private sealed record PublishedSubResultReference(long Id, IReadOnlyList SubResults); +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/IAzureDevOpsResultTransport.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/IAzureDevOpsResultTransport.cs new file mode 100644 index 00000000000..a74c333740b --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/IAzureDevOpsResultTransport.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; + +internal interface IAzureDevOpsResultTransport +{ + Task PublishResultsAsync(object results, CancellationToken cancellationToken); + + Task UploadAttachmentAsync( + long testResultId, + long? testSubResultId, + string fileName, + string stream, + CancellationToken cancellationToken); +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs new file mode 100644 index 00000000000..de208da4cc6 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs @@ -0,0 +1,334 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Xml; +using System.Xml.Linq; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; + +public sealed class LocalTestResultsReader( + ILogger logger, + TestResultAttachmentMode attachmentMode = TestResultAttachmentMode.Failed) +{ + private readonly ILogger _logger = logger; + private readonly TestResultAttachmentMode _attachmentMode = attachmentMode; + + public static bool LooksLikeTestResultFile(string path) + { + string fileName = Path.GetFileName(path); + if (fileName.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)) + { + fileName = fileName[..^4]; + } + + return fileName.EndsWith(".trx", StringComparison.OrdinalIgnoreCase) + || fileName.EndsWith("testResults.xml", StringComparison.OrdinalIgnoreCase) + || fileName.EndsWith("test-results.xml", StringComparison.OrdinalIgnoreCase) + || fileName.EndsWith("test_results.xml", StringComparison.OrdinalIgnoreCase) + || fileName.EndsWith("junit-results.xml", StringComparison.OrdinalIgnoreCase) + || fileName.EndsWith("junitresults.xml", StringComparison.OrdinalIgnoreCase); + } + + public async Task> ReadResultFileAsync( + string filePath, + CancellationToken cancellationToken = default) + { + try + { + var results = new List(); + await foreach (TestResult result in ReadResultsAsync(filePath, cancellationToken)) + { + results.Add(result); + } + + return results; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to parse test results file '{Path}'.", filePath); + return []; + } + } + + private async IAsyncEnumerable ReadResultsAsync( + string filePath, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + string rootName = await ReadRootNameAsync(filePath, cancellationToken); + string workItemName = new DirectoryInfo(Path.GetDirectoryName(filePath) ?? string.Empty).Name; + + switch (rootName) + { + case "assemblies": + case "assembly": + await foreach (TestResult result in ReadElementsAsync( + filePath, + "test", + ReadXunitResult, + cancellationToken)) + { + yield return result; + } + break; + + case "testsuites": + case "testsuite": + await foreach (TestResult result in ReadElementsAsync( + filePath, + "testcase", + element => ReadJUnitResult(element, workItemName), + cancellationToken)) + { + yield return result; + } + break; + + case "TestRun": + Dictionary definitions = + await ReadTrxDefinitionsAsync(filePath, cancellationToken); + await foreach (TestResult result in ReadElementsAsync( + filePath, + "UnitTestResult", + element => ReadTrxResult(element, workItemName, definitions), + cancellationToken)) + { + yield return result; + } + break; + + default: + _logger.LogWarning( + "Test result file '{Path}' has unsupported root element '{RootElement}' and will be skipped.", + filePath, + rootName); + break; + } + } + + private static async Task ReadRootNameAsync(string filePath, CancellationToken cancellationToken) + { + using XmlReader reader = CreateReader(filePath); + while (await reader.ReadAsync()) + { + cancellationToken.ThrowIfCancellationRequested(); + if (reader.NodeType == XmlNodeType.Element) + { + return reader.LocalName; + } + } + + return string.Empty; + } + + private static async IAsyncEnumerable ReadElementsAsync( + string filePath, + string elementName, + Func convert, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + using XmlReader reader = CreateReader(filePath); + while (await reader.ReadAsync()) + { + cancellationToken.ThrowIfCancellationRequested(); + if (reader.NodeType != XmlNodeType.Element || reader.LocalName != elementName) + { + continue; + } + + using XmlReader subtree = reader.ReadSubtree(); + await subtree.ReadAsync(); + yield return convert(XElement.Load(subtree, LoadOptions.PreserveWhitespace)); + } + } + + private static async Task> ReadTrxDefinitionsAsync( + string filePath, + CancellationToken cancellationToken) + { + var definitions = new Dictionary(StringComparer.OrdinalIgnoreCase); + using XmlReader reader = CreateReader(filePath); + while (await reader.ReadAsync()) + { + cancellationToken.ThrowIfCancellationRequested(); + if (reader.NodeType != XmlNodeType.Element || reader.LocalName != "UnitTest") + { + continue; + } + + using XmlReader subtree = reader.ReadSubtree(); + await subtree.ReadAsync(); + XElement unitTest = XElement.Load(subtree); + string? id = GetAttribute(unitTest, "id"); + XElement? method = unitTest.Descendants().FirstOrDefault(static x => x.Name.LocalName == "TestMethod"); + if (!string.IsNullOrEmpty(id)) + { + definitions[id] = new TestDefinition( + GetAttribute(method, "className"), + GetAttribute(method, "name")); + } + } + + return definitions; + } + + private TestResult ReadXunitResult(XElement test) + { + XElement? failure = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "failure"); + string? output = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "output")?.Value?.Trim(); + string typeName = GetAttribute(test, "type") ?? string.Empty; + string method = GetAttribute(test, "method") ?? string.Empty; + string name = GetAttribute(test, "name") + ?? (!string.IsNullOrEmpty(typeName) && !string.IsNullOrEmpty(method) ? $"{typeName}.{method}" : method); + string outcome = NormalizeOutcome(GetAttribute(test, "result")); + + List attachments = []; + AddAttachmentIfEnabled(attachments, "output.txt", output, outcome); + + return new TestResult( + name, + "xunit", + typeName, + method, + ParseDouble(GetAttribute(test, "time")), + outcome, + GetAttribute(failure, "exception-type"), + failure?.Elements().FirstOrDefault(static x => x.Name.LocalName == "message")?.Value?.Trim(), + failure?.Elements().FirstOrDefault(static x => x.Name.LocalName == "stack-trace")?.Value?.Trim(), + test.Elements().FirstOrDefault(static x => x.Name.LocalName == "reason")?.Value?.Trim(), + attachments); + } + + private TestResult ReadJUnitResult(XElement test, string workItemName) + { + XElement? failure = test.Elements().FirstOrDefault(static x => x.Name.LocalName is "failure" or "error"); + XElement? skipped = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "skipped"); + string className = GetAttribute(test, "classname") ?? workItemName; + string method = GetAttribute(test, "name") ?? string.Empty; + string outcome = skipped is not null ? "Skip" : failure is not null ? "Fail" : "Pass"; + + List attachments = []; + AddAttachmentIfEnabled( + attachments, + "stdout.txt", + test.Elements().FirstOrDefault(static x => x.Name.LocalName == "system-out")?.Value?.Trim(), + outcome); + AddAttachmentIfEnabled( + attachments, + "stderr.txt", + test.Elements().FirstOrDefault(static x => x.Name.LocalName == "system-err")?.Value?.Trim(), + outcome); + + return new TestResult( + !string.IsNullOrEmpty(className) ? $"{className}.{method}" : method, + "junit", + className, + method, + ParseDouble(GetAttribute(test, "time")), + outcome, + null, + failure?.Value?.Trim(), + null, + skipped?.Value?.Trim(), + attachments); + } + + private TestResult ReadTrxResult( + XElement result, + string workItemName, + IReadOnlyDictionary definitions) + { + string testId = GetAttribute(result, "testId") ?? string.Empty; + definitions.TryGetValue(testId, out TestDefinition? definition); + string className = definition?.ClassName ?? workItemName; + string method = definition?.Method ?? GetAttribute(result, "testName") ?? string.Empty; + string displayName = GetAttribute(result, "testName") + ?? (!string.IsNullOrEmpty(className) ? $"{className}.{method}" : method); + + XElement? output = result.Descendants().FirstOrDefault(static x => x.Name.LocalName == "Output"); + string? failureMessage = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "Message")?.Value?.Trim(); + string outcome = NormalizeOutcome(GetAttribute(result, "outcome")); + + List attachments = []; + AddAttachmentIfEnabled( + attachments, + "stdout.txt", + output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StdOut")?.Value?.Trim(), + outcome); + AddAttachmentIfEnabled( + attachments, + "stderr.txt", + output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StdErr")?.Value?.Trim(), + outcome); + + return new TestResult( + displayName, + "trx", + className, + method, + ParseDuration(GetAttribute(result, "duration")), + outcome, + null, + failureMessage, + output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StackTrace")?.Value?.Trim(), + string.Equals(outcome, "Skip", StringComparison.Ordinal) ? failureMessage : null, + attachments); + } + + private static XmlReader CreateReader(string filePath) + => XmlReader.Create(File.OpenRead(filePath), new XmlReaderSettings + { + Async = true, + CloseInput = true, + DtdProcessing = DtdProcessing.Prohibit, + IgnoreComments = true, + }); + + private static string? GetAttribute(XElement? element, string name) + => element?.Attribute(name)?.Value; + + private static double ParseDouble(string? value) + => double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double result) ? result : 0; + + private static double ParseDuration(string? value) + => TimeSpan.TryParse(value, CultureInfo.InvariantCulture, out TimeSpan result) + ? result.TotalSeconds + : ParseDouble(value); + + private static string NormalizeOutcome(string? value) + => value?.Trim().ToLowerInvariant() switch + { + "pass" or "passed" or "success" or "succeeded" => "Pass", + "skip" or "skipped" or "notexecuted" or "notrun" => "Skip", + "fail" or "failed" or "error" or "timeout" or "aborted" => "Fail", + _ => "None", + }; + + private void AddAttachmentIfEnabled( + List attachments, + string name, + string? text, + string normalizedOutcome) + { + bool includeAttachment = _attachmentMode switch + { + TestResultAttachmentMode.All => true, + TestResultAttachmentMode.Failed => normalizedOutcome == "Fail", + _ => false, + }; + + if (includeAttachment && !string.IsNullOrWhiteSpace(text)) + { + attachments.Add(new TestResultAttachment(name, text)); + } + } + + private sealed record TestDefinition(string? ClassName, string? Method); +} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingError.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/AzureDevOpsReportingError.cs similarity index 100% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingError.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/AzureDevOpsReportingError.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingParameters.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/AzureDevOpsReportingParameters.cs similarity index 100% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingParameters.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/AzureDevOpsReportingParameters.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/PackedTestReport.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/PackedTestReport.cs similarity index 100% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/PackedTestReport.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/PackedTestReport.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TerminalError.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TerminalError.cs similarity index 100% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TerminalError.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TerminalError.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResult.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TestResult.cs similarity index 100% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResult.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TestResult.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResultAttachment.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TestResultAttachment.cs similarity index 100% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResultAttachment.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TestResultAttachment.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResultAttachmentMode.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TestResultAttachmentMode.cs similarity index 100% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResultAttachmentMode.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TestResultAttachmentMode.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/ResultAggregator.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/ResultAggregator.cs similarity index 100% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/ResultAggregator.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/ResultAggregator.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestNameFormatter.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/TestNameFormatter.cs similarity index 100% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestNameFormatter.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/TestNameFormatter.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestResultUploadSummary.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/TestResultUploadSummary.cs similarity index 100% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestResultUploadSummary.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/TestResultUploadSummary.cs diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs index 59d61985642..649f2f480e2 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs @@ -2,9 +2,11 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Net.Http; -using System.Reflection; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; using System.Threading; +using System.Threading.Tasks; using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; using Microsoft.DotNet.Helix.JobMonitor; @@ -28,20 +30,9 @@ public void AttachmentModeDefaultsToFailed() } [Fact] - public void Constructor_ConfiguresHttpClientTimeoutForLongUploads() + public void JobMonitorUploadParallelismDefaultsToFortyEight() { - using var publisher = new AzureDevOpsResultPublisher( - new AzureDevOpsReportingParameters( - new Uri("https://dev.azure.com/dnceng-public/"), - "public", - "123", - "token"), - NullLogger.Instance); - - FieldInfo field = typeof(AzureDevOpsResultPublisher).GetField("_httpClient", BindingFlags.Instance | BindingFlags.NonPublic); - var client = Assert.IsType(field.GetValue(publisher)); - - Assert.Equal(TimeSpan.FromMinutes(5), client.Timeout); + Assert.Equal(48, new JobMonitorOptions().TestResultUploadParallelism); } [Theory] @@ -53,20 +44,17 @@ public void Constructor_ConfiguresHttpClientTimeoutForLongUploads() public void ComputeAllPassed_SingleResult_OnlyFailedAndNoneCountAsFailure(string result, bool expectedAllPassed) { var results = new[] { new AggregatedResult(AggregationType.Single, "Test1", 1, result) }; - Assert.Equal(expectedAllPassed, AzureDevOpsResultPublisher.ComputeAllPassed(results)); } [Fact] public void ComputeAllPassed_InconclusiveDataDrivenRollup_DoesNotFailTheWorkItem() { - // Mirrors the rollup the aggregator produces for a theory with some passing and some - // skipped data rows: no data row failed, but the mix isn't a clean pass or skip either. - var results = new[] - { - new AggregatedResult(AggregationType.Single, "Test1", 1, "Passed"), - new AggregatedResult(AggregationType.DataDriven, "Test2", 1, "Inconclusive"), - }; + AggregatedResult[] results = + [ + new(AggregationType.Single, "Test1", 1, "Passed"), + new(AggregationType.DataDriven, "Test2", 1, "Inconclusive"), + ]; Assert.True(AzureDevOpsResultPublisher.ComputeAllPassed(results)); } @@ -74,41 +62,215 @@ public void ComputeAllPassed_InconclusiveDataDrivenRollup_DoesNotFailTheWorkItem [Fact] public void ComputeAllPassed_AnyFailedResult_FailsTheWorkItem() { - var results = new[] - { - new AggregatedResult(AggregationType.Single, "Test1", 1, "Passed"), - new AggregatedResult(AggregationType.DataDriven, "Test2", 1, "Failed"), - }; + AggregatedResult[] results = + [ + new(AggregationType.Single, "Test1", 1, "Passed"), + new(AggregationType.DataDriven, "Test2", 1, "Failed"), + ]; Assert.False(AzureDevOpsResultPublisher.ComputeAllPassed(results)); } [Fact] - public void HttpClientTimeoutIsTransient() + public async Task UploadTestResultsWithCountAsync_BatchesByTopLevelResultCount() + { + var transport = new RecordingResultTransport(); + var publisher = CreatePublisher(transport); + AggregatedResult[] results = + [ + CreateDataDrivenResult("First", 600), + CreateDataDrivenResult("Second", 600), + ]; + + long uploadedCount = await publisher.UploadTestResultsWithCountAsync(results, new { }); + + Assert.Equal(2, uploadedCount); + Assert.Equal(new[] { 2 }, transport.RequestResultCounts); + } + + [Fact] + public async Task UploadTestResultsWithCountAsync_SplitsMoreThanOneThousandTopLevelResults() + { + var transport = new RecordingResultTransport(); + var publisher = CreatePublisher(transport); + AggregatedResult[] results = + [ + .. Enumerable.Range(0, 1001) + .Select(i => new AggregatedResult(AggregationType.Single, $"Test{i}", 1, "Passed")) + ]; + + long uploadedCount = await publisher.UploadTestResultsWithCountAsync(results, new { }); + + Assert.Equal(1001, uploadedCount); + Assert.Equal(new[] { 1000, 1 }, transport.RequestResultCounts); + } + + [Fact] + public async Task UploadTestResultsWithCountAsync_SplitHierarchiesIncludeRootInNodeLimit() + { + var transport = new RecordingResultTransport(); + var publisher = CreatePublisher(transport); + + long uploadedCount = await publisher.UploadTestResultsWithCountAsync( + [CreateDataDrivenResult("Theory", 950)], + new { }); + + Assert.Equal(2, uploadedCount); + Assert.Equal(new[] { 2 }, transport.RequestResultCounts); + Assert.Equal(new[] { 950, 2 }, transport.RequestHierarchyNodeCounts.Single()); + } + + [Fact] + public async Task UploadTestResultsWithCountAsync_RecursivelySplitsOversizedNestedHierarchies() { - Assert.True(AzureDevOpsResultPublisher.IsTransientException( - new OperationCanceledException("The request timed out.", new TimeoutException()), - CancellationToken.None)); + var transport = new RecordingResultTransport(); + var publisher = CreatePublisher(transport); + var nested = CreateDataDrivenResult("Nested", 950); + AggregatedResult[] results = + [ + new(AggregationType.DataDriven, "Outer", 1, "Passed", [nested]), + ]; + + long uploadedCount = await publisher.UploadTestResultsWithCountAsync(results, new { }); + + Assert.Equal(2, uploadedCount); + Assert.Equal(new[] { 950, 4 }, transport.RequestHierarchyNodeCounts.Single()); } [Fact] - public void CallerCancellationIsNotTransient() + public async Task UploadTestResultsWithCountAsync_DoesNotMaterializeAllConvertedResults() { - using var cancellation = new CancellationTokenSource(); - cancellation.Cancel(); + var transport = new BlockingResultTransport(); + var publisher = CreatePublisher(transport); + int enumerated = 0; + + IEnumerable Results() + { + for (int i = 0; i < 2_000; i++) + { + enumerated++; + yield return new AggregatedResult(AggregationType.Single, $"Test{i}", 1, "Passed"); + } + } + + Task upload = publisher.UploadTestResultsWithCountAsync(Results(), new { }); + await transport.FirstRequestStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.InRange(enumerated, 0, 1001); + transport.ReleaseFirstRequest.SetResult(); - Assert.False(AzureDevOpsResultPublisher.IsTransientException( - new OperationCanceledException("The request timed out.", new TimeoutException()), - cancellation.Token)); + Assert.Equal(2_000, await upload); + Assert.Equal(2, transport.RequestResultCounts.Count); } [Fact] - public void CancellationWithoutTimeoutIsNotTransient() + public async Task UploadTestResultsWithCountAsync_UsesSemanticAttachmentTransport() + { + var transport = new RecordingResultTransport(); + var publisher = CreatePublisher(transport); + var result = new AggregatedResult( + AggregationType.Single, + "Test", + 1, + "Failed", + attachments: [new TestResultAttachment("failure.txt", "details")]); + + Assert.Equal(1, await publisher.UploadTestResultsWithCountAsync([result], new { })); + + ResultAttachment attachment = Assert.Single(transport.Attachments); + Assert.Equal(1, attachment.TestResultId); + Assert.Null(attachment.TestSubResultId); + Assert.Equal("failure.txt", attachment.FileName); + Assert.Equal("details", System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(attachment.Stream))); + } + + private static AzureDevOpsResultPublisher CreatePublisher(IAzureDevOpsResultTransport transport) + => new( + TestResultAttachmentMode.Failed, + useFullyQualifiedTestName: false, + NullLogger.Instance, + transport); + + private static AggregatedResult CreateDataDrivenResult(string name, int subResultCount) + => new( + AggregationType.DataDriven, + name, + subResultCount, + "Passed", + [ + .. Enumerable.Range(0, subResultCount) + .Select(i => new AggregatedResult(AggregationType.Single, $"{name}_{i}", 1, "Passed")) + ]); + + private class RecordingResultTransport : IAzureDevOpsResultTransport + { + public List RequestResultCounts { get; } = []; + public List RequestHierarchyNodeCounts { get; } = []; + public List Attachments { get; } = []; + + public virtual Task PublishResultsAsync(object results, CancellationToken cancellationToken) + { + using JsonDocument requestBody = JsonDocument.Parse(JsonSerializer.Serialize(results)); + int resultCount = requestBody.RootElement.GetArrayLength(); + RequestResultCounts.Add(resultCount); + RequestHierarchyNodeCounts.Add( + [.. requestBody.RootElement.EnumerateArray().Select(CountHierarchyNodes)]); + + return Task.FromResult(JsonSerializer.Serialize(new + { + value = Enumerable.Range(1, resultCount).Select(id => new { id }) + })); + } + + public Task UploadAttachmentAsync( + long testResultId, + long? testSubResultId, + string fileName, + string stream, + CancellationToken cancellationToken) + { + Attachments.Add(new(testResultId, testSubResultId, fileName, stream)); + return Task.CompletedTask; + } + + private static int CountHierarchyNodes(JsonElement result) + { + if (!result.TryGetProperty("SubResults", out JsonElement subResults) && + !result.TryGetProperty("subResults", out subResults)) + { + return 1; + } + + return subResults.ValueKind == JsonValueKind.Array + ? 1 + subResults.EnumerateArray().Sum(CountHierarchyNodes) + : 1; + } + } + + private sealed class BlockingResultTransport : RecordingResultTransport { - Assert.False(AzureDevOpsResultPublisher.IsTransientException( - new OperationCanceledException(), - CancellationToken.None)); + public TaskCompletionSource FirstRequestStarted { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource ReleaseFirstRequest { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _requestCount; + + public override async Task PublishResultsAsync(object results, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _requestCount) == 1) + { + FirstRequestStarted.SetResult(); + await ReleaseFirstRequest.Task.WaitAsync(cancellationToken); + } + + return await base.PublishResultsAsync(results, cancellationToken); + } } + private sealed record ResultAttachment( + long TestResultId, + long? TestSubResultId, + string FileName, + string Stream); } } diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs index e2aeca84da7..571c94e71a1 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs @@ -3,12 +3,15 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; using AwesomeAssertions; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; using Microsoft.DotNet.Helix.JobMonitor; using Microsoft.DotNet.Helix.Sdk.Tests.Fakes; using Microsoft.Extensions.Logging.Abstractions; @@ -87,22 +90,149 @@ public async Task CompleteTestRunAsync_SendsCompletedStateAndHelixJobTag() } [Fact] - public async Task CompleteTestRunAsync_DoesNotRetryAmbiguousWrite() + public async Task CompleteTestRunAsync_RetriesTransientFailure() { + int attempt = 0; var handler = new RecordingHttpMessageHandler(_ => - new HttpResponseMessage(HttpStatusCode.ServiceUnavailable)); + Interlocked.Increment(ref attempt) == 1 + ? new HttpResponseMessage(HttpStatusCode.ServiceUnavailable) + : new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{}") + }); using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler)); - Func action = () => service.CompleteTestRunAsync( + await service.CompleteTestRunAsync( 123, HelixJobGuid, [], CancellationToken.None); - await action.Should().ThrowAsync(); + handler.Requests.Should().HaveCount(2); + handler.Bodies.Should().HaveCount(2); + foreach (string body in handler.Bodies) + { + JObject parsed = JObject.Parse(body); + parsed.Value("state").Should().Be("Completed"); + var tags = parsed["tags"].Should().BeOfType().Subject; + tags.Should().ContainSingle(); + tags[0].Value("name").Should().Be(HelixJobTag); + } + } + + [Fact] + public async Task RateLimitDelay_AppliesToNextRequest_NotCompletedRequest() + { + int requestCount = 0; + var handler = new RecordingHttpMessageHandler(_ => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + Interlocked.Increment(ref requestCount) == 1 + ? "{}" + : @"{""records"":[]}") + }; + + if (requestCount == 1) + { + response.Headers.TryAddWithoutValidation("X-RateLimit-Delay", "1"); + } + + return response; + }); + using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler)); + + var stopwatch = Stopwatch.StartNew(); + await service.CompleteTestRunAsync(123, HelixJobGuid, [], CancellationToken.None); + TimeSpan completionElapsed = stopwatch.Elapsed; + + await service.GetTimelineRecordsAsync(CancellationToken.None); + TimeSpan nextRequestElapsed = stopwatch.Elapsed - completionElapsed; + + completionElapsed.Should().BeLessThan(TimeSpan.FromMilliseconds(500)); + // If completion had awaited its own guidance, the shared deadline would have expired + // and the next request would not observe nearly the full delay. + nextRequestElapsed.Should().BeGreaterThan(TimeSpan.FromMilliseconds(750)); + handler.Requests.Should().HaveCount(2); + } + + [Fact] + public async Task ResultTransport_RetriesWritesAndRecordsExplicitRequestKinds() + { + int resultAttempts = 0; + int attachmentAttempts = 0; + var handler = new RecordingHttpMessageHandler(request => + { + bool isAttachment = request.RequestUri.AbsolutePath.EndsWith("/attachments", StringComparison.OrdinalIgnoreCase); + int attempt = isAttachment + ? Interlocked.Increment(ref attachmentAttempts) + : Interlocked.Increment(ref resultAttempts); + + if (attempt == 1) + { + var throttled = new HttpResponseMessage(HttpStatusCode.TooManyRequests); + throttled.Headers.RetryAfter = new System.Net.Http.Headers.RetryConditionHeaderValue( + TimeSpan.FromMilliseconds(25)); + return throttled; + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(isAttachment ? "{}" : @"{""value"":[{""id"":1}]}") + }; + }); + var metrics = new JobMonitorMetrics(); + using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler), metrics); + IAzureDevOpsResultTransport transport = service.CreateResultTransport(123); + + await transport.PublishResultsAsync(new[] { new { outcome = "Passed" } }, CancellationToken.None); + await transport.UploadAttachmentAsync(1, null, "log.txt", "YQ==", CancellationToken.None); + + JobMonitorMetricsSnapshot snapshot = metrics.Snapshot(); + snapshot.AzureDevOpsResultRequests.Should().Be(2); + snapshot.AzureDevOpsAttachmentRequests.Should().Be(2); + snapshot.AzureDevOpsControlRequests.Should().Be(0); + snapshot.AzureDevOpsRetries.Should().Be(2); + snapshot.AzureDevOpsFailedAttempts.Should().Be(2); + } + + [Fact] + public async Task ResultTransport_TerminalCompletedRunResponseDoesNotRetry() + { + var handler = new RecordingHttpMessageHandler(_ => + new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = new StringContent("cannot be added or updated for a test run which is in Completed state") + }); + using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler)); + + Func action = () => service.CreateResultTransport(123) + .PublishResultsAsync(new[] { new { outcome = "Passed" } }, CancellationToken.None); + + await action.Should().ThrowAsync(); handler.Requests.Should().ContainSingle(); } + [Fact] + public void AzureDevOpsTransportClassifiesTimeoutsAndRateLimitHeaders() + { + AzureDevOpsService.IsTransientException( + new OperationCanceledException("timeout", new TimeoutException()), + CancellationToken.None).Should().BeTrue(); + + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + AzureDevOpsService.IsTransientException( + new OperationCanceledException("timeout", new TimeoutException()), + cancellation.Token).Should().BeFalse(); + + using var response = new HttpResponseMessage(HttpStatusCode.OK); + response.Headers.RetryAfter = new System.Net.Http.Headers.RetryConditionHeaderValue(TimeSpan.FromSeconds(2)); + response.Headers.Add("X-RateLimit-Delay", "3.5"); + AzureDevOpsService.GetRateLimitDelay(response).Should().Be(TimeSpan.FromSeconds(3.5)); + } + [Fact] public async Task CompleteTestRunAsync_UploadsFailedWorkItemsAttachmentBeforePatch() { @@ -141,6 +271,24 @@ public async Task CompleteTestRunAsync_UploadsFailedWorkItemsAttachmentBeforePat patchRequest.RequestUri.ToString().Should().Be("https://dev.azure.com/dnceng-public/public/_apis/test/runs/123?api-version=7.1"); } + [Fact] + public async Task CompleteTestRunAsync_DoesNotRetryAmbiguousFailedWorkItemsAttachmentWrite() + { + var handler = new RecordingHttpMessageHandler(_ => + new HttpResponseMessage(HttpStatusCode.ServiceUnavailable)); + using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler)); + + Func action = () => service.CompleteTestRunAsync( + 123, + HelixJobGuid, + ["wi-1"], + CancellationToken.None); + + await action.Should().ThrowAsync(); + handler.Requests.Should().ContainSingle() + .Which.RequestUri.AbsolutePath.Should().EndWith("/attachments"); + } + [Fact] public async Task CompleteTestRunAsync_SkipsAttachment_WhenNoFailedWorkItems() { diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs index 5fce31eae8d..c76c8b647c9 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs @@ -25,12 +25,15 @@ internal sealed class FakeAzureDevOpsService : IAzureDevOpsService private readonly Queue _createFailures = []; private readonly Queue _uploadFailures = []; private readonly Queue _completeFailures = []; + private readonly Queue _timelineFailures = []; private readonly HashSet<(string JobName, string WorkItemName)> _recordedFailedTests = new(FailedTestWorkItemComparer.Instance); private readonly HashSet<(string JobName, string WorkItemName)> _uploadFailedTests = new(FailedTestWorkItemComparer.Instance); private int _timelineCallCount; private int _nextTestRunId; + private int _activeUploads; + private int _maximumConcurrentUploads; // Observable state for test assertions public List CreatedTestRuns { get; } = []; @@ -40,8 +43,10 @@ internal sealed class FakeAzureDevOpsService : IAzureDevOpsService public int CreateTestRunCallCount { get; private set; } public int UploadTestResultsCallCount { get; private set; } public int CompleteTestRunCallCount { get; private set; } + public int MaximumConcurrentUploads => Volatile.Read(ref _maximumConcurrentUploads); public TaskCompletionSource UploadStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource UploadCompleted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + public TaskCompletionSource TestRunCompleted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public Task UploadBlocker { get; set; } = Task.CompletedTask; /// @@ -79,6 +84,16 @@ public FakeAzureDevOpsService WithPreviouslyProcessedJob(string jobName) return this; } + public FakeAzureDevOpsService FailNextTimeline(Exception exception) + { + lock (_sync) + { + _timelineFailures.Enqueue(exception); + } + + return this; + } + public FakeAzureDevOpsService FailNextCreate(Exception exception = null) { lock (_sync) @@ -142,6 +157,14 @@ public FakeAzureDevOpsService WithFailedUpload(string helixJobName, string workI // IAzureDevOpsService implementation public Task> GetTimelineRecordsAsync(CancellationToken cancellationToken) { + lock (_sync) + { + if (_timelineFailures.TryDequeue(out Exception failure)) + { + return Task.FromException>(failure); + } + } + if (_timelineResponses.Count == 0) { _timelineCallCount++; @@ -213,58 +236,67 @@ public Task CompleteTestRunAsync(int testRunId, string helixJobName, IReadOnlyCo _recordedFailedTests.Add((helixJobName, workItemName)); } } + TestRunCompleted.TrySetResult(); return Task.CompletedTask; } } - public async Task> UploadTestResultsAsync( + public async Task UploadTestResultsAsync( int testRunId, - IReadOnlyList results, + WorkItemTestResults results, CancellationToken cancellationToken) { UploadStarted.TrySetResult(); - if (UploadBlockerIgnoresCancellation) + int active = Interlocked.Increment(ref _activeUploads); + int observedMaximum; + while (active > (observedMaximum = Volatile.Read(ref _maximumConcurrentUploads))) { - await UploadBlocker; - } - else - { - await UploadBlocker.WaitAsync(cancellationToken); + if (Interlocked.CompareExchange(ref _maximumConcurrentUploads, active, observedMaximum) == observedMaximum) + { + break; + } } - var summaries = new Dictionary<(string JobName, string WorkItemName), TestResultUploadSummary>(); - - lock (_sync) + try { - UploadTestResultsCallCount++; - if (_uploadFailures.Count > 0) + if (UploadBlockerIgnoresCancellation) { - throw _uploadFailures.Dequeue(); + await UploadBlocker; } - - if (!UploadedResultsByRunId.TryGetValue(testRunId, out List existing)) + else { - existing = []; - UploadedResultsByRunId[testRunId] = existing; + await UploadBlocker.WaitAsync(cancellationToken); } - existing.AddRange(results); - - foreach (string jobName in results.Select(r => r.JobName).Distinct(StringComparer.OrdinalIgnoreCase)) + lock (_sync) { - UploadedJobNames.Add(jobName); - } + UploadTestResultsCallCount++; + if (_uploadFailures.Count > 0) + { + throw _uploadFailures.Dequeue(); + } - foreach (WorkItemTestResults result in results) - { - bool allPassed = !_uploadFailedTests.Contains((result.JobName, result.WorkItemName)); - summaries[(result.JobName, result.WorkItemName)] = - new TestResultUploadSummary(allPassed, result.TestResultFiles.Count); + if (!UploadedResultsByRunId.TryGetValue(testRunId, out List existing)) + { + existing = []; + UploadedResultsByRunId[testRunId] = existing; + } + + existing.Add(results); + if (!UploadedJobNames.Contains(results.JobName, StringComparer.OrdinalIgnoreCase)) + { + UploadedJobNames.Add(results.JobName); + } + + bool allPassed = !_uploadFailedTests.Contains((results.JobName, results.WorkItemName)); + return new TestResultUploadSummary(allPassed, results.TestResultFiles.Count); } } - - UploadCompleted.TrySetResult(); - return summaries; + finally + { + Interlocked.Decrement(ref _activeUploads); + UploadCompleted.TrySetResult(); + } } private static HttpRequestException CreateTransientFailure(string message) diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs index 1b266aade5e..1ccd5645621 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs @@ -21,6 +21,8 @@ internal sealed class FakeHelixService : IHelixService private readonly HashSet _downloadFailureJobs = new(StringComparer.OrdinalIgnoreCase); private readonly Dictionary> _downloadFailures = new(StringComparer.OrdinalIgnoreCase); private readonly Dictionary> _customWorkItems = new(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _listWorkItemsCallCounts = + new(StringComparer.OrdinalIgnoreCase); private int _getJobsCallCount; /// @@ -68,6 +70,9 @@ public FakeHelixService WithWorkItems(string jobName, IReadOnlyCollectionNumber of times has been called. public int GetJobsCallCount => _getJobsCallCount; + public int GetListWorkItemsCallCount(string jobName) + => _listWorkItemsCallCounts.TryGetValue(jobName, out int count) ? count : 0; + public ConcurrentBag CanceledJobs { get; } = []; private HelixSnapshot CurrentSnapshot @@ -92,8 +97,8 @@ public Task> GetJobsForBuildAsync(string source, str return Task.FromResult>(_responses[index].Jobs); } - public Task> DownloadTestResultsAsync( - string jobName, IReadOnlyCollection workItemNames, string workingDirectory, CancellationToken cancellationToken) + public Task DownloadTestResultsAsync( + string jobName, string workItemName, string workingDirectory, CancellationToken cancellationToken) { if (_downloadFailureJobs.Contains(jobName)) { @@ -108,25 +113,21 @@ public Task> DownloadTestResultsAsync( if (CurrentSnapshot.TestResultsByJob.TryGetValue(jobName, out List explicitResults)) { - return Task.FromResult>(explicitResults); + WorkItemTestResults result = explicitResults.FirstOrDefault( + result => string.Equals(result.WorkItemName, workItemName, StringComparison.OrdinalIgnoreCase)) + ?? new WorkItemTestResults(jobName, workItemName, []); + return Task.FromResult(result); } - workItemNames = workItemNames - .Distinct(StringComparer.OrdinalIgnoreCase) - .DefaultIfEmpty($"{jobName}-synthetic") - .ToList(); - - IReadOnlyList generated = workItemNames - .Select(wi => new WorkItemTestResults(jobName, wi, [])) - .ToList(); - - return Task.FromResult(generated); + return Task.FromResult(new WorkItemTestResults(jobName, workItemName, [])); } public Task> ListWorkItemsAsync( string jobName, CancellationToken _) { + _listWorkItemsCallCounts.AddOrUpdate(jobName, 1, static (_, count) => count + 1); + if (_customWorkItems.TryGetValue(jobName, out IReadOnlyCollection customWorkItems)) { return Task.FromResult(customWorkItems); @@ -162,9 +163,10 @@ public Task CancelJobAsync(string jobName, CancellationToken cancellationToken) /// /// Tracks resubmission calls for test assertions. - /// Each entry is (originalJobName, failedWorkItemNames, newJobName, targetStageAttempt). + /// Each entry is (originalJobName, failedWorkItemNames, newJobName, targetStageAttempt, + /// monitorJobAttempt). /// - public List<(string OriginalJob, IReadOnlyCollection FailedItems, string NewJob, string TargetStageAttempt)> Resubmissions { get; } = []; + public List<(string OriginalJob, IReadOnlyCollection FailedItems, string NewJob, string TargetStageAttempt, string MonitorJobAttempt)> Resubmissions { get; } = []; /// The objects returned from successful resubmissions. public List ResubmittedJobInfos { get; } = []; @@ -194,6 +196,7 @@ public Task ResubmitWorkItemsAsync( HelixJobInfo originalJob, IReadOnlyCollection failedWorkItems, string targetStageAttempt, + string monitorJobAttempt, CancellationToken cancellationToken) { string originalJobName = originalJob.JobName; @@ -209,7 +212,7 @@ public Task ResubmitWorkItemsAsync( IReadOnlyCollection failedItemNames = [..failedWorkItems.Select(wi => wi.Name)]; if (_nullResubmissions.Contains(originalJobName)) { - Resubmissions.Add((originalJobName, failedItemNames, null, targetStageAttempt)); + Resubmissions.Add((originalJobName, failedItemNames, null, targetStageAttempt, monitorJobAttempt)); return Task.FromResult(null); } @@ -219,7 +222,7 @@ public Task ResubmitWorkItemsAsync( ? targetStageAttempt : (originalSnapshotJob?.StageAttempt ?? originalJob.StageAttempt); - Resubmissions.Add((originalJobName, failedItemNames, newJobName, targetStageAttempt)); + Resubmissions.Add((originalJobName, failedItemNames, newJobName, targetStageAttempt, monitorJobAttempt)); var newJobInfo = new HelixJobInfo( newJobName, "running", @@ -229,7 +232,10 @@ public Task ResubmitWorkItemsAsync( originalSnapshotJob?.SubmitterJobDisplayName ?? originalJob.SubmitterJobDisplayName, originalSnapshotJob?.QueueId ?? originalJob.QueueId, originalJobName, - stageAttempt: resubmittedStageAttempt); + stageAttempt: resubmittedStageAttempt, + jobAttempt: originalSnapshotJob?.JobAttempt ?? originalJob.JobAttempt, + logicalJobName: originalSnapshotJob?.LogicalJobName ?? originalJob.LogicalJobName, + submitterPhaseName: originalSnapshotJob?.SubmitterPhaseName ?? originalJob.SubmitterPhaseName); ResubmittedJobInfos.Add(newJobInfo); return Task.FromResult(newJobInfo); } diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs index 4f55e1bfa16..1811cb2aee3 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs @@ -26,16 +26,18 @@ namespace Microsoft.DotNet.Helix.Sdk.Tests public class HelixServiceTests { [Fact] - public async Task GetJobsForBuildAsync_PassesSourceThroughAndFiltersByBuildId() + public async Task GetJobsForBuildAsync_PassesSourceAndBuildIdFilters() { var api = CreateApi(); string capturedSource = null; + IImmutableDictionary capturedProperties = null; int? capturedCount = null; api.Job .Setup(j => j.ListAsync(null, It.IsAny(), null, null, It.IsAny>(), It.IsAny(), null, It.IsAny())) - .Callback, string, string, CancellationToken>((_, count, _, _, _, source, _, _) => + .Callback, string, string, CancellationToken>((_, count, _, _, properties, source, _, _) => { capturedCount = count; + capturedProperties = properties; capturedSource = source; }) .ReturnsAsync(ImmutableList.Create( @@ -64,7 +66,14 @@ public async Task GetJobsForBuildAsync_PassesSourceThroughAndFiltersByBuildId() CancellationToken.None); Assert.Equal("pr/public/dotnet/runtime/refs/pull/42/merge", capturedSource); - Assert.Equal(100_000, capturedCount); + Assert.Collection( + capturedProperties, + property => + { + Assert.Equal("BuildId", property.Key); + Assert.Equal("123", property.Value); + }); + Assert.Equal(1_000, capturedCount); Assert.Equal(2, jobs.Count); Assert.Equal("running-job", jobs[0].JobName); Assert.Equal("running", jobs[0].Status); @@ -85,6 +94,39 @@ await Assert.ThrowsAsync(() => service.GetJobsForBuildAsync(source: "", buildId: "123", CancellationToken.None)); } + [Fact] + public async Task GetJobsForBuildAsync_RequiresNonEmptyBuildId() + { + HelixService service = CreateService(CreateApi().Api.Object); + + await Assert.ThrowsAsync(() => + service.GetJobsForBuildAsync(source: "ci/public/dotnet/runtime/refs/heads/main", buildId: "", CancellationToken.None)); + } + + [Fact] + public async Task GetJobsForBuildAsync_RejectsPotentiallyTruncatedResponse() + { + var api = CreateApi(); + api.Job + .Setup(j => j.ListAsync(null, 1_000, null, null, It.IsAny>(), It.IsAny(), null, It.IsAny())) + .ReturnsAsync(ImmutableList.CreateRange( + Enumerable.Range(0, 1_000) + .Select(index => Job( + $"job-{index}", + finished: null, + new JObject { ["BuildId"] = "123" })))); + + HelixService service = CreateService(api.Api.Object); + + InvalidOperationException exception = await Assert.ThrowsAsync(() => + service.GetJobsForBuildAsync( + source: "ci/public/dotnet/runtime/refs/heads/main", + buildId: "123", + CancellationToken.None)); + + Assert.Contains("potentially truncated result set", exception.Message); + } + [Fact] public async Task DownloadTestResultsAsync_FiltersFilesUsesFileSystemAndContinuesAfterDownloadFailure() { @@ -96,7 +138,7 @@ public async Task DownloadTestResultsAsync_FiltersFilesUsesFileSystemAndContinue .Setup(w => w.ListFilesAsync("work:item", "job:name", false, It.IsAny())) .ReturnsAsync(ImmutableList.Create( new UploadedFile("logs/console.txt", "https://storage/logs/console.txt"), - new UploadedFile("nested/testResults.xml", "https://storage/nested/testResults.xml"), + new UploadedFile("nested/testResults.xml.txt", "https://storage/nested/testResults.xml.txt"), new UploadedFile("failed.trx", "https://storage/failed.trx"))); api.WorkItem .Setup(w => w.ListFilesAsync("no-results", "job:name", false, It.IsAny())) @@ -106,21 +148,19 @@ public async Task DownloadTestResultsAsync_FiltersFilesUsesFileSystemAndContinue blobClientFactory.FailDownloadsFor.Add("https://storage/failed.trx"); var fileSystem = new MockFileSystem(directorySeparator: Path.DirectorySeparatorChar.ToString()); - IReadOnlyList results = await CreateService(api.Api.Object, blobClientFactory, fileSystem) - .DownloadTestResultsAsync("job:name", ["work:item", "no-results"], "work", CancellationToken.None); - - WorkItemTestResults result = Assert.Single(results); + WorkItemTestResults result = await CreateService(api.Api.Object, blobClientFactory, fileSystem) + .DownloadTestResultsAsync("job:name", "work:item", "work", CancellationToken.None); Assert.Equal("job:name", result.JobName); Assert.Equal("work:item", result.WorkItemName); string jobDirectory = fileSystem.PathCombine("work", SanitizeForCurrentPlatform("job:name")); string workItemDirectory = fileSystem.PathCombine(jobDirectory, SanitizeForCurrentPlatform("work:item")); - string expectedResultFile = fileSystem.PathCombine(workItemDirectory, NormalizeForCurrentPlatform("nested/testResults.xml")); + string expectedResultFile = fileSystem.PathCombine(workItemDirectory, NormalizeForCurrentPlatform("nested/testResults.xml.txt")); Assert.Equal([expectedResultFile], result.TestResultFiles); Assert.Contains(jobDirectory, fileSystem.Directories); Assert.Contains(workItemDirectory, fileSystem.Directories); Assert.Contains(fileSystem.GetDirectoryName(expectedResultFile), fileSystem.Directories); Assert.Equal( - [new DownloadCall("https://storage/nested/testResults.xml", "?resultSas", expectedResultFile), + [new DownloadCall("https://storage/nested/testResults.xml.txt", "?resultSas", expectedResultFile), new DownloadCall("https://storage/failed.trx", "?resultSas", fileSystem.PathCombine(workItemDirectory, "failed.trx"))], blobClientFactory.Downloads); } @@ -143,7 +183,7 @@ public async Task DownloadTestResultsAsync_ReportsTransientFileFailureAfterAttem new HttpRequestException("Injected transient failure.", null, HttpStatusCode.ServiceUnavailable); Func action = () => CreateService(api.Api.Object, blobClientFactory, new MockFileSystem()) - .DownloadTestResultsAsync("job", ["work-item"], "work", CancellationToken.None); + .DownloadTestResultsAsync("job", "work-item", "work", CancellationToken.None); await Assert.ThrowsAsync(action); Assert.Equal(2, blobClientFactory.Downloads.Count); @@ -176,7 +216,7 @@ public async Task ResubmitWorkItemsAsync_ReturnsNullWhenRequiredJobDetailsAreMis }); HelixJobInfo result = await CreateService(api.Api.Object) - .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("missing")], targetStageAttempt: null, CancellationToken.None); + .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("missing")], targetStageAttempt: null, monitorJobAttempt: null, CancellationToken.None); Assert.Null(result); api.Storage.Verify(s => s.NewAsync(It.IsAny(), It.IsAny()), Times.Never); @@ -195,7 +235,7 @@ public async Task ResubmitWorkItemsAsync_ReturnsNullWhenJobListIsInvalidJson() }; HelixJobInfo result = await CreateService(api.Api.Object, blobClientFactory) - .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("work-a")], targetStageAttempt: null, CancellationToken.None); + .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("work-a")], targetStageAttempt: null, monitorJobAttempt: null, CancellationToken.None); Assert.Null(result); api.Storage.Verify(s => s.NewAsync(It.IsAny(), It.IsAny()), Times.Never); @@ -247,13 +287,20 @@ public async Task ResubmitWorkItemsAsync_UploadsFilteredJobListAndCreatesJobWith }; HelixJobInfo result = await CreateService(api.Api.Object, blobClientFactory) - .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("WORK-A"), WorkItem("work-b")], targetStageAttempt: null, CancellationToken.None); + .ResubmitWorkItemsAsync( + new HelixJobInfo("original-job", "finished"), + [WorkItem("WORK-A"), WorkItem("work-b")], + targetStageAttempt: "2", + monitorJobAttempt: "2", + CancellationToken.None); Assert.Equal("new-job", result.JobName); Assert.Equal("running", result.Status); Assert.Equal("custom run", result.TestRunName); Assert.Equal("test stage", result.StageName); Assert.Equal("original-job", result.PreviousHelixJobName); + Assert.Equal("2", result.StageAttempt); + Assert.Equal("1", result.JobAttempt); Assert.Equal("https://storage/job-list.json", blobClientFactory.DownloadedTextUri); UploadCall upload = Assert.Single(blobClientFactory.Uploads); @@ -276,6 +323,9 @@ public async Task ResubmitWorkItemsAsync_UploadsFilteredJobListAndCreatesJobWith Assert.Equal("123", capturedRequest.Properties["BuildId"]); Assert.Equal("custom run", capturedRequest.Properties["TestRunName"]); Assert.Equal("test stage", capturedRequest.Properties["System.StageName"]); + Assert.Equal("2", capturedRequest.Properties[HelixJobInfo.StageAttemptPropertyName]); + Assert.Equal("1", capturedRequest.Properties[HelixJobInfo.JobAttemptPropertyName]); + Assert.Equal("2", capturedRequest.Properties[HelixJobInfo.ResubmittedByJobAttemptPropertyName]); Assert.Equal("original-job", capturedRequest.Properties[HelixJobInfo.PreviousHelixJobNamePropertyName]); Assert.Equal("""{"nested":true}""", capturedRequest.Properties["ObjectProperty"]); @@ -322,6 +372,8 @@ private static async Task ResubmitAndCaptureRequestAsync( ["BuildId"] = "123", ["TestRunName"] = "custom run", ["System.StageName"] = "test stage", + [HelixJobInfo.StageAttemptPropertyName] = "1", + [HelixJobInfo.JobAttemptPropertyName] = "1", }; api.Job @@ -366,7 +418,7 @@ private static async Task ResubmitAndCaptureRequestAsync( }; await CreateService(api.Api.Object, blobClientFactory) - .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("work-a")], targetStageAttempt: null, CancellationToken.None); + .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("work-a")], targetStageAttempt: null, monitorJobAttempt: null, CancellationToken.None); Assert.NotNull(capturedRequest); return capturedRequest; @@ -408,6 +460,8 @@ private static JobDetails JobDetails() ["BuildId"] = "123", ["TestRunName"] = "custom run", ["System.StageName"] = "test stage", + [HelixJobInfo.StageAttemptPropertyName] = "1", + [HelixJobInfo.JobAttemptPropertyName] = "1", ["ObjectProperty"] = new JObject { ["nested"] = true, diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs index b5d59855570..c979d4048f3 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs @@ -2,11 +2,13 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using AwesomeAssertions; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; using Microsoft.DotNet.Helix.Client.Models; using Microsoft.DotNet.Helix.JobMonitor; using Microsoft.DotNet.Helix.JobMonitor.Models; @@ -21,6 +23,28 @@ namespace Microsoft.DotNet.Helix.Sdk.Tests [Collection("NonParallel")] public class JobMonitorRunnerTests { + [Theory] + [InlineData(null)] + [InlineData("")] + public void MissingAttemptMetadata_DefaultsToFirstAttempt(string value) + { + MonitorState.ParseStageAttempt(value).Should().Be(1); + MonitorState.ParseJobAttempt(value).Should().Be(1); + } + + [Theory] + [InlineData("invalid")] + [InlineData("0")] + [InlineData("-1")] + public void MalformedAttemptMetadata_IsRejected(string value) + { + Action parseStageAttempt = () => MonitorState.ParseStageAttempt(value); + Action parseJobAttempt = () => MonitorState.ParseJobAttempt(value); + + parseStageAttempt.Should().Throw(); + parseJobAttempt.Should().Throw(); + } + /// /// Single pipeline job goes from queued → in progress → completed (succeeded). /// No Helix jobs are ever submitted. @@ -435,15 +459,15 @@ public async Task BuildJobSubmitsHelixWork_WorkItemsPassed_ResultsUploaded_ExitZ // Poll 3: build job completed (it has submitted Helix work and exited) azdo.AddTimelineResponse( - MonitorJob(), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), PipelineJob("Test Linux", "completed", "succeeded")); // Polls 4-6: build job still completed (monitor waiting for Helix) azdo.AddTimelineResponse( - MonitorJob(), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), PipelineJob("Test Linux", "completed", "succeeded")); azdo.AddTimelineResponse( - MonitorJob(), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), PipelineJob("Test Linux", "completed", "succeeded")); azdo.AddTimelineResponse( MonitorJob(), @@ -503,7 +527,8 @@ public async Task CompletedHelixJob_QueuesTestResultUploadWithoutBlockingNextPol azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "inProgress")); azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished")], + jobs: [HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(passed: ["workitem-1"]), @@ -530,14 +555,20 @@ public async Task CompletedHelixJob_QueuesTestResultUploadWithoutBlockingNextPol exitCode.Should().Be(0); delayedBeforeUploadCompleted.Should().BeTrue(); azdo.TimelineCallCount.Should().Be(2); + // Initial monitor attempts do not run retry reconciliation; the completed job is + // listed once by the poll loop. + helix.GetListWorkItemsCallCount("helix-linux").Should().Be(1); azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-linux"]); azdo.CompletedTestRunIds.Should().ContainSingle(); logger.Messages.Should().Contain(message => - message.Contains("2 test results for job 'helix-linux' processed.", StringComparison.Ordinal)); + message.Contains( + "Test result processing completed for job 'Test Linux (helix-linux)': " + + "1 work item(s), 2 recognized result file(s), and 2 test result(s) uploaded.", + StringComparison.Ordinal)); } [Fact] - public async Task VerboseDrainReportsPendingUploadPhaseAndElapsedTime() + public async Task DrainReportsAggregatePipelineProgress() { var azdo = new FakeAzureDevOpsService(); var helix = new FakeHelixService(); @@ -546,10 +577,11 @@ public async Task VerboseDrainReportsPendingUploadPhaseAndElapsedTime() azdo.UploadBlocker = uploadRelease.Task; azdo.AddTimelineResponse( - MonitorJob(), - PipelineJob("Test Linux", "completed", "succeeded")); + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished")], + jobs: [HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(passed: ["workitem-1"]), @@ -576,11 +608,305 @@ public async Task VerboseDrainReportsPendingUploadPhaseAndElapsedTime() exitCode.Should().Be(0); logger.Messages.Should().Contain(message => - message.Contains("test result upload(s) remain pending", StringComparison.Ordinal) - && message.Contains("helix-linux", StringComparison.Ordinal) - && message.Contains("phase='publishing 1 work item(s) to Azure DevOps test run", StringComparison.Ordinal) - && message.Contains("phase elapsed=", StringComparison.Ordinal) - && message.Contains("total elapsed=", StringComparison.Ordinal)); + message.Contains( + "Starting final test result drain: 1 work item(s) were first observed terminal " + + "and 1 work item upload(s) became eligible in the final poll; " + + "1 work item upload(s) remain (1 from the final poll, 0 from earlier polls)", + StringComparison.Ordinal)); + logger.Messages.Should().Contain(message => + message.Contains("Test result pipeline drained in", StringComparison.Ordinal) + && message.Contains("1 job(s), 1 work item(s), and 1 result(s)", StringComparison.Ordinal)); + logger.Messages.Should().Contain(message => + message.Contains("Performance metrics", StringComparison.Ordinal) + && message.Contains("Azure DevOps HTTP:", StringComparison.Ordinal) + && message.Contains("Pipeline aggregate worker time", StringComparison.Ordinal)); + } + + [Fact] + public async Task DrainSeparatesFinalPollArrivalsFromEarlierBacklog() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var logger = new RecordingLogger(); + var uploadRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + azdo.UploadBlocker = uploadRelease.Task; + azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "inProgress")); + azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); + helix.AddResponse( + jobs: + [ + HelixJob("helix-earlier", "finished"), + HelixJob("helix-final", "running"), + ], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-earlier"] = PassFail(passed: ["earlier-workitem"]), + }, + testResultsByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-earlier"] = + [ + new WorkItemTestResults("helix-earlier", "earlier-workitem", ["earlier.trx"]) + ], + }); + helix.AddResponse( + jobs: + [ + HelixJob("helix-earlier", "finished"), + HelixJob("helix-final", "finished"), + ], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-earlier"] = PassFail(passed: ["earlier-workitem"]), + ["helix-final"] = PassFail(passed: ["final-workitem"]), + }, + testResultsByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-earlier"] = + [ + new WorkItemTestResults("helix-earlier", "earlier-workitem", ["earlier.trx"]) + ], + ["helix-final"] = + [ + new WorkItemTestResults("helix-final", "final-workitem", ["final.trx"]) + ], + }); + + var runner = new JobMonitorRunner( + DefaultOptions(), + logger, + azdo, + helix, + async (_, _) => await azdo.UploadStarted.Task.WaitAsync(TimeSpan.FromSeconds(5))); + + Task run = runner.RunAsync(CancellationToken.None); + await WaitForAsync(() => logger.Messages.Any(message => + message.Contains("Starting final test result drain:", StringComparison.Ordinal))); + + logger.Messages.Should().Contain(message => + message.Contains( + "1 work item(s) were first observed terminal and 1 work item upload(s) became eligible " + + "in the final poll; 2 work item upload(s) remain " + + "(1 from the final poll, 1 from earlier polls)", + StringComparison.Ordinal)); + + uploadRelease.SetResult(); + (await run.WaitAsync(TimeSpan.FromSeconds(5))).Should().Be(0); + } + + [Fact] + public async Task StatusReportsEveryFiveMinutesWhileDrainIsBlocked() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var logger = new RecordingLogger(); + var uploadRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var statusTick = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + int statusDelayCount = 0; + + azdo.UploadBlocker = uploadRelease.Task; + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("Test Linux", "completed", "succeeded")); + helix.AddResponse( + jobs: [HelixJob("helix-linux", "finished")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-linux"] = PassFail(passed: ["workitem-1"]), + }); + + var runner = new JobMonitorRunner( + DefaultOptions(), + logger, + azdo, + helix, + NoDelay, + async (delay, cancellationToken) => + { + delay.Should().Be(TimeSpan.FromMinutes(5)); + if (Interlocked.Increment(ref statusDelayCount) == 1) + { + await statusTick.Task.WaitAsync(cancellationToken); + } + else + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + }); + + Task run = runner.RunAsync(CancellationToken.None); + await WaitForAsync(() => logger.Messages.Any(message => + message.Contains("Starting final test result drain:", StringComparison.Ordinal))); + logger.Messages.Count(IsStatusMessage).Should().Be(1); + + statusTick.SetResult(); + await WaitForAsync(() => logger.Messages.Count(IsStatusMessage) == 2); + logger.Messages.Last(IsStatusMessage).Should().Contain("1 completed"); + + uploadRelease.SetResult(); + (await run.WaitAsync(TimeSpan.FromSeconds(5))).Should().Be(0); + + static bool IsStatusMessage(string message) + => message.Contains("ℹ️ Status:", StringComparison.Ordinal); + } + + [Fact] + public async Task UploadPipeline_BoundsWorkItemParallelismAndCreatesOneRunPerJob() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var uploadRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + string[] workItems = [.. Enumerable.Range(1, 20).Select(index => $"workitem-{index}")]; + + azdo.UploadBlocker = uploadRelease.Task; + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("Test Linux", "completed", "succeeded")); + helix.AddResponse( + jobs: [HelixJob("helix-linux", "finished")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-linux"] = PassFail(passed: workItems), + }, + testResultsByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-linux"] = + [ + ..workItems.Select(name => + new WorkItemTestResults("helix-linux", name, [$"{name}.trx"])) + ], + }); + + JobMonitorOptions options = DefaultOptions(); + options.TestResultUploadParallelism = 4; + var runner = new JobMonitorRunner(options, new RecordingLogger(), azdo, helix, NoDelay); + + Task run = runner.RunAsync(CancellationToken.None); + await WaitForAsync(() => azdo.MaximumConcurrentUploads == 4); + uploadRelease.SetResult(); + + (await run.WaitAsync(TimeSpan.FromSeconds(5))).Should().Be(0); + azdo.MaximumConcurrentUploads.Should().Be(4); + azdo.CreateTestRunCallCount.Should().Be(1); + azdo.CompleteTestRunCallCount.Should().Be(1); + azdo.UploadTestResultsCallCount.Should().Be(20); + } + + [Fact] + public async Task UploadPipeline_UploadsTerminalWorkItemsBeforeJobCompletesAndPreservesTestFailures() + { + var azdo = new FakeAzureDevOpsService() + .WithFailedUpload("helix-linux", "workitem-1"); + var helix = new FakeHelixService(); + + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("Test Linux", "inProgress")); + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("Test Linux", "completed", "succeeded")); + + helix.AddResponse( + jobs: [HelixJob("helix-linux", "running", initialWorkItemCount: 2)], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-linux"] = PassFail(passed: ["workitem-1"]), + }); + helix.AddResponse( + jobs: [HelixJob("helix-linux", "finished", initialWorkItemCount: 2)], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-linux"] = PassFail(passed: ["workitem-1", "workitem-2"]), + }); + + int delayCount = 0; + var runner = new JobMonitorRunner( + DefaultOptions(), + NullLogger.Instance, + azdo, + helix, + async (_, _) => + { + delayCount++; + await azdo.UploadCompleted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + azdo.UploadTestResultsCallCount.Should().Be(1); + azdo.CompleteTestRunCallCount.Should().Be(0); + }); + + int exitCode = await runner.RunAsync(CancellationToken.None); + + exitCode.Should().Be(1); + delayCount.Should().Be(1); + azdo.CreateTestRunCallCount.Should().Be(1); + azdo.UploadTestResultsCallCount.Should().Be(2); + azdo.CompleteTestRunCallCount.Should().Be(1); + } + + [Fact] + public void IncrementalTestFailure_DoesNotStickAcrossPassingIncarnation() + { + var state = new MonitorState(); + HelixJobInfo original = HelixJob( + "helix-original", + "finished", + submitterPhaseName: "Test_Linux", + queueId: "q", + logicalJobName: "tests"); + HelixJobInfo retry = HelixJob( + "helix-retry", + "finished", + submitterPhaseName: "Test_Linux", + queueId: "q", + previousHelixJobName: "helix-original", + logicalJobName: "tests"); + state.ObserveJobs([original, retry]); + + state.ObserveTestResult( + original.JobName, + "workitem", + new TestResultUploadSummary(AllPassed: false, UploadedCount: 1)); + state.TryRecordWorkItemOutcomes( + original, + [new WorkItemSummary("original/workitem", original.JobName, "workitem", "Finished") { ExitCode = 1 }]); + state.TryRecordWorkItemOutcomes( + retry, + [new WorkItemSummary("retry/workitem", retry.JobName, "workitem", "Finished") { ExitCode = 0 }]); + + state.HasFailedWorkItem.Should().BeFalse(); + state.SnapshotFailedWorkItemConsoleInfo().Should().BeEmpty(); + } + + [Fact] + public async Task UploadPipeline_DoesNotDropCompletedJobsWhenBacklogged() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + HelixJobInfo[] jobs = + [ + ..Enumerable.Range(1, 300) + .Select(index => HelixJob($"helix-{index}", "finished")) + ]; + var passFailByJob = jobs.ToDictionary( + static job => job.JobName, + static _ => PassFail(), + StringComparer.OrdinalIgnoreCase); + + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("Test Linux", "completed", "succeeded")); + helix.AddResponse(jobs, passFailByJob); + + var runner = new JobMonitorRunner( + DefaultOptions(), + new RecordingLogger(), + azdo, + helix, + NoDelay); + + (await runner.RunAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(10))).Should().Be(0); + azdo.CreateTestRunCallCount.Should().Be(300); + azdo.CompleteTestRunCallCount.Should().Be(300); } [Fact] @@ -626,7 +952,39 @@ public async Task PassedHelixWork_TransientUploadFailure_DoesNotReplayAmbiguousW azdo.CompletedTestRunIds.Should().BeEmpty(); azdo.UploadedJobNames.Should().BeEmpty(); logger.Messages.Should().Contain(message => - message.Contains("not safe to replay in this invocation", StringComparison.Ordinal)); + message.Contains("remains untagged", StringComparison.Ordinal) + && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal)); + } + + [Fact] + public async Task PassedHelixWork_CreateTestRunFailure_IsSingleFlightAndLeavesJobUntagged() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var logger = new RecordingLogger(); + + azdo.FailNextCreate(); + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("Test Linux", "completed", "succeeded")); + helix.AddResponse( + jobs: [HelixJob("helix-linux", "finished")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-linux"] = PassFail(passed: ["workitem-1", "workitem-2"]), + }); + + var runner = CreateRunner(azdo, helix, logger: logger); + int exitCode = await runner.RunAsync(CancellationToken.None); + + exitCode.Should().Be(0); + azdo.CreateTestRunCallCount.Should().Be(1); + azdo.UploadTestResultsCallCount.Should().Be(0); + azdo.CompleteTestRunCallCount.Should().Be(0); + azdo.CompletedTestRunIds.Should().BeEmpty(); + logger.Messages.Should().Contain(message => + message.Contains("remains untagged", StringComparison.Ordinal) + && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal)); } [Fact] @@ -665,7 +1023,71 @@ public async Task PassedHelixWork_TransientCompletionFailure_DoesNotReplayComple delayCount.Should().Be(0); azdo.CompletedTestRunIds.Should().BeEmpty(); logger.Messages.Should().Contain(message => - message.Contains("not safe to replay in this invocation", StringComparison.Ordinal)); + message.Contains("remains untagged", StringComparison.Ordinal) + && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal)); + } + + [Fact] + public async Task PassedHelixWork_PermanentDownloadFailure_DoesNotCreateTestRun() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var logger = new RecordingLogger(); + + helix.FailDownloadForJob("helix-linux"); + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("Test Linux", "completed", "succeeded")); + helix.AddResponse( + jobs: [HelixJob("helix-linux", "finished")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-linux"] = PassFail(passed: ["workitem-1"]), + }); + + var runner = CreateRunner(azdo, helix, logger: logger); + int exitCode = await runner.RunAsync(CancellationToken.None); + + exitCode.Should().Be(0); + azdo.CreateTestRunCallCount.Should().Be(0); + azdo.UploadTestResultsCallCount.Should().Be(0); + azdo.CompleteTestRunCallCount.Should().Be(0); + logger.Messages.Should().Contain(message => + message.Contains("remains untagged", StringComparison.Ordinal) + && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal)); + } + + [Fact] + public async Task MultipleWorkItems_OneUploadFails_JobRemainsUntagged() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var logger = new RecordingLogger(); + + azdo.FailNextUpload(new InvalidOperationException("Injected permanent upload failure.")); + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("Test Linux", "completed", "succeeded")); + helix.AddResponse( + jobs: [HelixJob("helix-linux", "finished")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-linux"] = PassFail(passed: ["workitem-1", "workitem-2"]), + }); + + JobMonitorOptions options = DefaultOptions(); + options.TestResultUploadParallelism = 1; + var runner = new JobMonitorRunner(options, logger, azdo, helix, NoDelay); + int exitCode = await runner.RunAsync(CancellationToken.None); + + exitCode.Should().Be(0); + azdo.CreateTestRunCallCount.Should().Be(1); + azdo.UploadTestResultsCallCount.Should().Be(2); + azdo.CompleteTestRunCallCount.Should().Be(0); + azdo.CompletedTestRunIds.Should().BeEmpty(); + logger.Messages.Should().Contain(message => + message.Contains("remains untagged", StringComparison.Ordinal) + && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal)); } [Fact] @@ -696,8 +1118,8 @@ public async Task PassedHelixWork_PermanentUploadFailure_DoesNotHangOrFailBuild( azdo.CompleteTestRunCallCount.Should().Be(0); azdo.CompletedTestRunIds.Should().BeEmpty(); logger.Messages.Should().Contain(message => - message.Contains("The failure is not retryable", StringComparison.Ordinal) - && message.Contains("later monitor invocation may retry the upload", StringComparison.Ordinal)); + message.Contains("remains untagged", StringComparison.Ordinal) + && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal)); } [Fact] @@ -924,6 +1346,108 @@ public async Task OneSubmitter_FansOutToMultipleQueues_SameWorkItemName_FailureN exitCode.Should().Be(1); } + /// + /// Regression: a single AzDO job can submit multiple independent Helix jobs to the same + /// queue. Their work-item names can overlap, so the submitter and queue alone are not a + /// unique stream identifier. The submitter-assigned logical Helix job name must preserve + /// one job's failure when a later sibling job reports the same work-item name as passed. + /// + [Fact] + public async Task OneSubmitter_SameQueue_DifferentLogicalJobs_FailureNotOverwrittenByPass() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var logger = new RecordingLogger(); + + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("__default", "completed", "succeeded")); + + helix.AddResponse( + jobs: + [ + HelixJob( + "helix-wasm-tests", + "finished", + submitterJobName: "__default", + queueId: "azurelinux.3.amd64.open.rt", + logicalJobName: "WasmTestOnChrome-MONO-ST"), + HelixJob( + "helix-wasm-build-tests", + "finished", + submitterJobName: "__default", + queueId: "azurelinux.3.amd64.open.rt", + logicalJobName: "WasmBuildTests"), + ], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-wasm-tests"] = PassFail(failed: ["System.Formats.Tar.Manual.Tests"]), + ["helix-wasm-build-tests"] = PassFail(passed: ["System.Formats.Tar.Manual.Tests"]), + }); + + var runner = CreateRunner(azdo, helix, logger: logger); + int exitCode = await runner.RunAsync(CancellationToken.None); + + exitCode.Should().Be(1); + logger.Messages.Should().Contain(message => + message.Contains("Work items: 2 submitted / 0 resubmitted / 1 failed", StringComparison.Ordinal)); + logger.Messages.Should().Contain(message => + message.Contains("Failed work item information:", StringComparison.Ordinal) + && message.Contains("System.Formats.Tar.Manual.Tests", StringComparison.Ordinal)); + } + + /// + /// Production regression: runtime stamps System.JobName=__default onto many + /// independent matrix jobs. Some of those jobs submit the same logical Helix scenario + /// to the same queue, so only System.PhaseName distinguishes their streams. + /// A pass from one phase must not erase a same-named failure from another phase. + /// + [Fact] + public async Task DefaultJobName_SameQueueAndLogicalJob_DifferentPhases_FailureNotOverwrittenByPass() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var logger = new RecordingLogger(); + + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("__default", "completed", "succeeded")); + + helix.AddResponse( + jobs: + [ + HelixJob( + "helix-wasm-eat", + "finished", + submitterJobName: "__default", + submitterPhaseName: "build_browser_wasm_linux_Release_LibraryTests_EAT", + queueId: "azurelinux.3.amd64.open.rt", + logicalJobName: "WasmTestOnChrome-MONO-ST"), + HelixJob( + "helix-wasm-smoke", + "finished", + submitterJobName: "__default", + submitterPhaseName: "build_browser_wasm_linux_Release_LibraryTests_Smoke", + queueId: "azurelinux.3.amd64.open.rt", + logicalJobName: "WasmTestOnChrome-MONO-ST"), + ], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-wasm-eat"] = PassFail(failed: ["System.Formats.Tar.Manual.Tests"]), + ["helix-wasm-smoke"] = PassFail(passed: ["System.Formats.Tar.Manual.Tests"]), + }); + + var runner = CreateRunner(azdo, helix, logger: logger); + int exitCode = await runner.RunAsync(CancellationToken.None); + + exitCode.Should().Be(1); + logger.Messages.Should().Contain(message => + message.Contains("Work items: 2 submitted / 0 resubmitted / 1 failed", StringComparison.Ordinal)); + logger.Messages.Should().Contain(message => + message.Contains("Failed work item information:", StringComparison.Ordinal) + && message.Contains("System.Formats.Tar.Manual.Tests", StringComparison.Ordinal)); + } + [Fact] public async Task OneSubmitter_SendsMultipleJobsToSameQueue_FailureNotOverwrittenByPass() { @@ -937,8 +1461,10 @@ public async Task OneSubmitter_SendsMultipleJobsToSameQueue_FailureNotOverwritte helix.AddResponse( jobs: [ - HelixJob("helix-failed", "finished", submitterJobName: "Linux_Build_Debug", queueId: "ubuntu.2204.amd64.open"), - HelixJob("helix-passed", "finished", submitterJobName: "Linux_Build_Debug", queueId: "ubuntu.2204.amd64.open"), + HelixJob("helix-failed", "finished", submitterJobName: "Linux_Build_Debug", + queueId: "ubuntu.2204.amd64.open", logicalJobName: "failed-job"), + HelixJob("helix-passed", "finished", submitterJobName: "Linux_Build_Debug", + queueId: "ubuntu.2204.amd64.open", logicalJobName: "passed-job"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -967,9 +1493,11 @@ public async Task OneSubmitter_SendsMultipleJobsToSameQueue_FailureNotOverwritte jobs: [ HelixJob("helix-failed", "finished", submitterJobName: "Linux_Build_Debug", - queueId: "ubuntu.2204.amd64.open", stageAttempt: "1"), + queueId: "ubuntu.2204.amd64.open", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "failed-job"), HelixJob("helix-passed", "finished", submitterJobName: "Linux_Build_Debug", - queueId: "ubuntu.2204.amd64.open", stageAttempt: "1"), + queueId: "ubuntu.2204.amd64.open", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "passed-job"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -981,11 +1509,14 @@ public async Task OneSubmitter_SendsMultipleJobsToSameQueue_FailureNotOverwritte jobs: [ HelixJob("helix-failed", "finished", submitterJobName: "Linux_Build_Debug", - queueId: "ubuntu.2204.amd64.open", stageAttempt: "1"), + queueId: "ubuntu.2204.amd64.open", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "failed-job"), HelixJob("helix-passed", "finished", submitterJobName: "Linux_Build_Debug", - queueId: "ubuntu.2204.amd64.open", stageAttempt: "1"), + queueId: "ubuntu.2204.amd64.open", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "passed-job"), HelixJob("helix-failed-resub", "finished", submitterJobName: "Linux_Build_Debug", - queueId: "ubuntu.2204.amd64.open", previousHelixJobName: "helix-failed", stageAttempt: "2"), + queueId: "ubuntu.2204.amd64.open", previousHelixJobName: "helix-failed", + stageAttempt: "2", jobAttempt: "1", logicalJobName: "failed-job"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -994,6 +1525,7 @@ public async Task OneSubmitter_SendsMultipleJobsToSameQueue_FailureNotOverwritte JobMonitorOptions attempt2Options = DefaultOptions(); attempt2Options.StageAttempt = "2"; + attempt2Options.JobAttempt = "2"; var attempt2Runner = new JobMonitorRunner( attempt2Options, NullLogger.Instance, @@ -1101,9 +1633,10 @@ public async Task StageScopedMonitor_IgnoresJobsOutsideStage_NoHelixJobs_ExitOne PipelineJob("Build Windows", "inProgress", parentId: "stage-build")); azdo.AddTimelineResponse( - StageRecord("Test", "stage-test", "inProgress"), - MonitorJob(parentId: "stage-test"), - PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test"), + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1, parentId: "stage-test"), StageRecord("Build", "stage-build", "inProgress"), PipelineJob("Build Windows", "inProgress", parentId: "stage-build")); @@ -1134,9 +1667,10 @@ public async Task StageScopedMonitor_IgnoresHelixJobsFromOtherStage_ExitOne() // Test stage job completes quickly with no Helix submissions. // Build stage job submits a Helix job, but that's not the monitor's concern. azdo.AddTimelineResponse( - StageRecord("Test", "stage-test", "inProgress"), - MonitorJob(parentId: "stage-test"), - PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test"), + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1, parentId: "stage-test"), StageRecord("Build", "stage-build", "inProgress"), PipelineJob("Build Windows", "inProgress", parentId: "stage-build")); @@ -1155,32 +1689,67 @@ public async Task StageScopedMonitor_IgnoresHelixJobsFromOtherStage_ExitOne() } /// - /// Corner case 1 (retry-failed-jobs where only the monitor re-ran). The stage's Helix - /// submitter jobs passed and were not re-run, so the current stage attempt contains no - /// Helix work of its own. The monitor must NOT exit 0 immediately (discarding the previous - /// attempt's results): it must upload the previous attempt's passed work and resubmit its - /// failed work into the current attempt, then gate on that resubmission. + /// The first monitor invocation observes and reports an already-failed Helix job but does + /// not create another Helix job. Work-item replay starts only when AzDO retries the monitor. /// [Fact] - public async Task AttemptScoped_RetryOnlyMonitor_ReconcilesPreviousAttemptWork() + public async Task AttemptScoped_InitialMonitorInvocation_DoesNotResubmit() { var azdo = new FakeAzureDevOpsService(); var helix = new FakeHelixService(); - // Submitters already succeeded in attempt 1; only the monitor is re-running as attempt 2. azdo.AddTimelineResponse( StageRecord("Test", "stage-test", "inProgress"), MonitorJob(parentId: "stage-test"), - PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test"), - PipelineJob("Test Windows", "completed", "succeeded", parentId: "stage-test")); + PipelineJob("Test_Linux", "completed", "succeeded", parentId: "stage-test")); + helix.AddResponse( + jobs: [HelixJob("helix-linux-a1", "finished", stageName: "Test", + submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "tests")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-linux-a1"] = PassFail(failed: ["wi"]), + }); + + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "1"; + options.JobAttempt = "1"; + + int exitCode = await new JobMonitorRunner( + options, NullLogger.Instance, azdo, helix, NoDelay).RunAsync(CancellationToken.None); + + exitCode.Should().Be(1); + helix.Resubmissions.Should().BeEmpty(); + azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-linux-a1"]); + } + + /// + /// Retry-failed-jobs advanced the stage and monitor to attempt 2, but the successful Helix + /// submitters remain at job attempt 1. Their previous-stage Helix work is therefore still + /// authoritative: passed work is retained and failed work is replayed. + /// + [Fact] + public async Task AttemptScoped_RetryOnlyMonitor_ReconcilesPreviousAttemptWork() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + + azdo.AddTimelineResponse( + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("Test_Linux", "completed", "succeeded", attempt: 1, parentId: "stage-test"), + PipelineJob("Test_Windows", "completed", "succeeded", attempt: 1, parentId: "stage-test")); HelixJobInfo linuxA1 = HelixJob("helix-linux-a1", "finished", stageName: "Test", - submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1"); + submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "tests"); HelixJobInfo winA1 = HelixJob("helix-win-a1", "finished", stageName: "Test", - submitterJobName: "Test_Windows", queueId: "q2", stageAttempt: "1"); + submitterJobName: "Test_Windows", queueId: "q2", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "tests"); HelixJobInfo winResub = HelixJob("helix-win-a1-resub", "finished", stageName: "Test", submitterJobName: "Test_Windows", queueId: "q2", previousHelixJobName: "helix-win-a1", - stageAttempt: "2"); + stageAttempt: "2", jobAttempt: "1", logicalJobName: "tests"); // Snapshot 0 (retry pass + poll 1): previous-attempt jobs only. helix.AddResponse( @@ -1203,6 +1772,7 @@ public async Task AttemptScoped_RetryOnlyMonitor_ReconcilesPreviousAttemptWork() JobMonitorOptions options = DefaultOptions(); options.StageAttempt = "2"; + options.JobAttempt = "2"; var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); @@ -1213,8 +1783,10 @@ public async Task AttemptScoped_RetryOnlyMonitor_ReconcilesPreviousAttemptWork() helix.Resubmissions[0].OriginalJob.Should().Be("helix-win-a1"); helix.Resubmissions[0].FailedItems.Should().BeEquivalentTo(["win-wi"]); helix.Resubmissions[0].TargetStageAttempt.Should().Be("2"); + helix.Resubmissions[0].MonitorJobAttempt.Should().Be("2"); helix.ResubmittedJobInfos.Should().ContainSingle() - .Which.StageAttempt.Should().Be("2"); + .Which.Should().Match(job => + job.StageAttempt == "2" && job.JobAttempt == "1"); // ...and the previous attempt's results (passed and failed) were still uploaded. azdo.UploadedJobNames.Should().Contain(["helix-linux-a1", "helix-win-a1", "helix-win-a1-resub"]); } @@ -1233,15 +1805,16 @@ public async Task AttemptScoped_StrandedWaitingPreviousWork_ResubmittedNotWaited var helix = new FakeHelixService(); azdo.AddTimelineResponse( - StageRecord("Test", "stage-test", "inProgress"), - MonitorJob(parentId: "stage-test"), - PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test")); + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("Test_Linux", "completed", "succeeded", attempt: 1, parentId: "stage-test")); HelixJobInfo stranded = HelixJob("helix-stranded-a1", "running", stageName: "Test", - submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1"); + submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1", jobAttempt: "1"); HelixJobInfo resub = HelixJob("helix-stranded-a1-r2", "finished", stageName: "Test", submitterJobName: "Test_Linux", queueId: "q1", previousHelixJobName: "helix-stranded-a1", - stageAttempt: "2"); + stageAttempt: "2", jobAttempt: "1"); // The stranded attempt-1 job's work items are stuck Waiting (never dispatched). helix.WithWorkItems("helix-stranded-a1", @@ -1264,6 +1837,7 @@ public async Task AttemptScoped_StrandedWaitingPreviousWork_ResubmittedNotWaited JobMonitorOptions options = DefaultOptions(); options.StageAttempt = "2"; + options.JobAttempt = "2"; var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); @@ -1276,125 +1850,411 @@ public async Task AttemptScoped_StrandedWaitingPreviousWork_ResubmittedNotWaited } /// - /// A completed monitor resubmission supersedes its still-running predecessor. The monitor - /// must gate on the linked current incarnation only and must not resubmit the predecessor. + /// A full-stage rerun has advanced the submitter to job attempt 2. The monitor starts + /// before the attempt-2 Helix job is visible. It must use the timeline attempt to suppress + /// replay of H1, wait for H2, and then gate on H2. /// [Fact] public async Task AttemptScoped_LinkedCurrentIncarnationExists_DoesNotResubmitPrevious() { var azdo = new FakeAzureDevOpsService(); var helix = new FakeHelixService(); + var logger = new RecordingLogger(); azdo.AddTimelineResponse( - StageRecord("Test", "stage-test", "inProgress"), - MonitorJob(parentId: "stage-test"), - PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test")); - - // The current incarnation is explicitly linked to its predecessor. - HelixJobInfo previousRunning = HelixJob("helix-x-a1", "running", stageName: "Test", - submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1"); + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelinePhase("Test_Linux", "Test Linux", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("__default", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)], parentId: "Test_Linux")); + azdo.AddTimelineResponse( + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelinePhase("Test_Linux", "Test Linux", "completed", "succeeded", attempt: 2, + previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("__default", "completed", "succeeded", attempt: 2, + previousAttempts: [PreviousAttempt(1)], parentId: "Test_Linux")); + + // Previous incarnation still running; fresh current incarnation of the SAME stream + // (same submitter + queue), not linked by PreviousHelixJobName. + HelixJobInfo previousRunning = HelixJob("helix-x-a1", "finished", stageName: "Test", + submitterJobName: "__default", submitterPhaseName: "Test_Linux", + queueId: "q1", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "tests"); HelixJobInfo currentDone = HelixJob("helix-x-a2", "finished", stageName: "Test", - submitterJobName: "Test_Linux", queueId: "q1", - previousHelixJobName: "helix-x-a1", stageAttempt: "2"); - helix.WithWorkItems("helix-x-a1", - [new WorkItemSummary("helix-x-a1/wi", "helix-x-a1", "wi", "Running")]); + submitterJobName: "__default", submitterPhaseName: "Test_Linux", + queueId: "q1", stageAttempt: "2", + jobAttempt: "2", logicalJobName: "tests"); + helix.AddResponse( + jobs: [previousRunning], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-x-a1"] = PassFail(failed: ["wi"]), + }); helix.AddResponse( jobs: [previousRunning, currentDone], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { + ["helix-x-a1"] = PassFail(failed: ["wi"]), ["helix-x-a2"] = PassFail(passed: ["wi"]), }); JobMonitorOptions options = DefaultOptions(); options.StageAttempt = "2"; + options.JobAttempt = "2"; + var runner = new JobMonitorRunner(options, logger, azdo, helix, NoDelay); + + int exitCode = await runner.RunAsync(CancellationToken.None); + + exitCode.Should().Be(0); + helix.Resubmissions.Should().BeEmpty(); + azdo.UploadedJobNames.Should().Contain("helix-x-a2"); + logger.Messages.Should().NotContain(message => + message.Contains("helix-x-a1", StringComparison.Ordinal) + && message.Contains("failed", StringComparison.OrdinalIgnoreCase)); + } + + /// + /// Corner case 5 (unlinked rerun duplicates). Two incarnations of the same work stream on + /// different attempts are NOT connected by PreviousHelixJobName (a stage rerun, not + /// a monitor resubmission). The higher stage/job incarnation must win the outcome map. + /// Job names are chosen so a naive job-name sort would let the failed attempt-1 job win. + /// + [Fact] + public async Task AttemptScoped_UnlinkedRerunDuplicates_HigherAttemptWinsOutcome() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + + azdo.AddTimelineResponse( + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("Test_Linux", "completed", "succeeded", attempt: 2, + previousAttempts: [PreviousAttempt(1)], parentId: "stage-test")); + + // Same stream (submitter + queue + logical job), not lineage-linked. "zzz-old" + // (attempt 1, failed) sorts after "aaa-new" (attempt 2, passed): a job-name-ordered + // reconciliation would let the stale failure overwrite the current pass. + HelixJobInfo oldFailed = HelixJob("zzz-old", "finished", stageName: "Test", + submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "tests"); + HelixJobInfo newPassed = HelixJob("aaa-new", "finished", stageName: "Test", + submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "2", + jobAttempt: "2", logicalJobName: "tests"); + + helix.AddResponse( + jobs: [oldFailed, newPassed], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["zzz-old"] = PassFail(failed: ["wi-1"]), + ["aaa-new"] = PassFail(passed: ["wi-1"]), + }); + + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); - // Gated on the current incarnation (completed) only; the still-running previous - // incarnation neither blocked termination nor was resubmitted. exitCode.Should().Be(0); helix.Resubmissions.Should().BeEmpty(); - azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-x-a2"]); } /// - /// Jobs on different attempts that are not connected by PreviousHelixJobName are - /// independent streams, even when their submitter and queue match. + /// Retry-failed-jobs can rerun only a subset of submitters. Retry eligibility is decided + /// independently for each submitter: A's newer timeline attempt supersedes HA1, while B's + /// unchanged attempt leaves HB1 eligible for monitor replay. + /// + [Fact] + public async Task AttemptScoped_MixedSubmitterRetries_ReconcilesPerSubmitterAttempt() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + + azdo.AddTimelineResponse( + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("A", "completed", "succeeded", attempt: 2, + previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("B", "completed", "succeeded", attempt: 1, parentId: "stage-test")); + + HelixJobInfo ha1 = HelixJob("ha1", "finished", stageName: "Test", + submitterJobName: "A", queueId: "q", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "tests"); + HelixJobInfo ha2 = HelixJob("ha2", "finished", stageName: "Test", + submitterJobName: "A", queueId: "q", stageAttempt: "2", + jobAttempt: "2", logicalJobName: "tests"); + HelixJobInfo hb1 = HelixJob("hb1", "finished", stageName: "Test", + submitterJobName: "B", queueId: "q", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "tests"); + HelixJobInfo rb2 = HelixJob("rb2", "finished", stageName: "Test", + submitterJobName: "B", queueId: "q", previousHelixJobName: "hb1", + stageAttempt: "2", jobAttempt: "1", logicalJobName: "tests"); + + helix.AddResponse( + jobs: [ha1, ha2, hb1], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["ha1"] = PassFail(failed: ["a"]), + ["ha2"] = PassFail(passed: ["a"]), + ["hb1"] = PassFail(failed: ["b"]), + }); + helix.ConfigureResubmission("hb1", "rb2"); + helix.AddResponse( + jobs: [ha1, ha2, hb1, rb2], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["ha1"] = PassFail(failed: ["a"]), + ["ha2"] = PassFail(passed: ["a"]), + ["hb1"] = PassFail(failed: ["b"]), + ["rb2"] = PassFail(passed: ["b"]), + }); + + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + + int exitCode = await new JobMonitorRunner( + options, NullLogger.Instance, azdo, helix, NoDelay).RunAsync(CancellationToken.None); + + exitCode.Should().Be(0); + helix.Resubmissions.Should().ContainSingle() + .Which.OriginalJob.Should().Be("hb1"); + } + + /// + /// A submitter that reran supersedes its previous Helix work even when the new execution + /// intentionally produces no replacement Helix job. + /// + [Fact] + public async Task AttemptScoped_RerunSubmitterProducesNoReplacement_DoesNotReplayOldWork() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + + azdo.AddTimelineResponse( + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("A", "completed", "succeeded", attempt: 2, + previousAttempts: [PreviousAttempt(1)], parentId: "stage-test")); + helix.AddResponse( + jobs: [HelixJob("ha1", "finished", stageName: "Test", + submitterJobName: "A", queueId: "q", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "tests")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["ha1"] = PassFail(failed: ["a"]), + }); + + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + + int exitCode = await new JobMonitorRunner( + options, NullLogger.Instance, azdo, helix, NoDelay).RunAsync(CancellationToken.None); + + exitCode.Should().Be(0); + helix.Resubmissions.Should().BeEmpty(); + } + + /// + /// Corner case 6 (un-resubmittable previous work). Previous-attempt work that cannot be + /// resubmitted (e.g. its Helix queue was removed) must fail the monitor fast with actionable + /// output rather than hanging forever or silently passing. /// [Fact] - public async Task AttemptScoped_UnlinkedJobsWithSameSubmitterAndQueue_RemainIndependent() + public async Task AttemptScoped_UnresubmittablePreviousWork_FailsFast() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var logger = new RecordingLogger(); + + azdo.AddTimelineResponse( + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("Test_Linux", "completed", "succeeded", attempt: 1, parentId: "stage-test")); + + HelixJobInfo purged = HelixJob("helix-purged-a1", "running", stageName: "Test", + submitterJobName: "Test_Linux", queueId: "gone", stageAttempt: "1", jobAttempt: "1"); + helix.WithWorkItems("helix-purged-a1", + [new WorkItemSummary("helix-purged-a1/wi-1", "helix-purged-a1", "wi-1", "Waiting")]); + + helix.AddResponse(jobs: [purged]); + helix.ConfigureNullResubmission("helix-purged-a1"); + + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + var runner = new JobMonitorRunner(options, logger, azdo, helix, NoDelay); + + int exitCode = await runner.RunAsync(CancellationToken.None); + + // Failed fast (did not hang), reported the abandoned work, and failed the monitor. + exitCode.Should().Be(1); + helix.Resubmissions.Should().ContainSingle() + .Which.NewJob.Should().BeNull(); + logger.Messages.Should().Contain(m => + m.Contains("Could not resubmit", StringComparison.Ordinal) + && m.Contains("previous attempt", StringComparison.Ordinal)); + } + + [Fact] + public async Task AttemptScoped_MissingSubmitterAttemptMetadata_DoesNotSpeculativelyResubmit() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var logger = new RecordingLogger(); + + azdo.AddTimelineResponse( + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("A", "completed", "succeeded", attempt: 1, parentId: "stage-test")); + helix.AddResponse( + jobs: [HelixJob("ha1", "finished", stageName: "Test", + submitterJobName: "A", queueId: "q", stageAttempt: "1", + logicalJobName: "tests")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["ha1"] = PassFail(failed: ["a"]), + }); + + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + + int exitCode = await new JobMonitorRunner( + options, logger, azdo, helix, NoDelay).RunAsync(CancellationToken.None); + + exitCode.Should().Be(1); + helix.Resubmissions.Should().BeEmpty(); + logger.Messages.Should().Contain(message => + message.Contains("could not be matched to compatible System.JobAttempt metadata", StringComparison.Ordinal)); + } + + [Fact] + public async Task AttemptScoped_TimelineAttemptOlderThanHelixMetadata_DoesNotSpeculativelyResubmit() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var logger = new RecordingLogger(); + + azdo.AddTimelineResponse( + StageRecord("Test", "stage-test", "inProgress", attempt: 3, + previousAttempts: [PreviousAttempt(1), PreviousAttempt(2)]), + MonitorJob(attempt: 3, previousAttempts: [PreviousAttempt(1), PreviousAttempt(2)], + parentId: "stage-test"), + PipelineJob("A", "completed", "succeeded", attempt: 1, parentId: "stage-test")); + helix.AddResponse( + jobs: [HelixJob("ha2", "finished", stageName: "Test", + submitterJobName: "A", queueId: "q", stageAttempt: "2", + jobAttempt: "2", logicalJobName: "tests")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["ha2"] = PassFail(failed: ["a"]), + }); + + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "3"; + options.JobAttempt = "3"; + + int exitCode = await new JobMonitorRunner( + options, logger, azdo, helix, NoDelay).RunAsync(CancellationToken.None); + + exitCode.Should().Be(1); + helix.Resubmissions.Should().BeEmpty(); + logger.Messages.Should().Contain(message => + message.Contains("could not be matched to compatible System.JobAttempt metadata", StringComparison.Ordinal)); + } + + [Fact] + public async Task AttemptScoped_AmbiguousDefaultJobIdentity_DoesNotSpeculativelyResubmit() { var azdo = new FakeAzureDevOpsService(); var helix = new FakeHelixService(); + var logger = new RecordingLogger(); azdo.AddTimelineResponse( - StageRecord("Test", "stage-test", "inProgress"), - MonitorJob(parentId: "stage-test"), - PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test")); - - // Matching submitter and queue are not enough to establish lineage. - HelixJobInfo oldFailed = HelixJob("zzz-old", "finished", stageName: "Test", - submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1"); - HelixJobInfo newPassed = HelixJob("aaa-new", "finished", stageName: "Test", - submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "2"); - + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("__default", "completed", "succeeded", attempt: 1, + parentId: "phase-a", id: "job-a"), + PipelineJob("__default", "completed", "succeeded", attempt: 1, + parentId: "phase-b", id: "job-b")); helix.AddResponse( - jobs: [oldFailed, newPassed], + jobs: [HelixJob("ha1", "finished", stageName: "Test", + submitterJobName: "__default", queueId: "q", stageAttempt: "1", + jobAttempt: "1", logicalJobName: "tests")], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { - ["zzz-old"] = PassFail(failed: ["wi-1"]), - ["aaa-new"] = PassFail(passed: ["wi-1"]), + ["ha1"] = PassFail(failed: ["a"]), }); - var runner = new JobMonitorRunner(DefaultOptions(), NullLogger.Instance, azdo, helix, NoDelay); + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; - int exitCode = await runner.RunAsync(CancellationToken.None); + int exitCode = await new JobMonitorRunner( + options, logger, azdo, helix, NoDelay).RunAsync(CancellationToken.None); exitCode.Should().Be(1); - helix.Resubmissions.Should().ContainSingle() - .Which.OriginalJob.Should().Be("zzz-old"); + helix.Resubmissions.Should().BeEmpty(); + logger.Messages.Should().Contain(message => + message.Contains("could not be matched to compatible System.JobAttempt metadata", StringComparison.Ordinal)); } - /// - /// Corner case 5 (un-resubmittable previous work). Previous-attempt work that cannot be - /// resubmitted (e.g. its Helix queue was removed) must fail the monitor fast with actionable - /// output rather than hanging forever or silently passing. - /// [Fact] - public async Task AttemptScoped_UnresubmittablePreviousWork_FailsFast() + public async Task AttemptScoped_CurrentStageFailure_IsObservedButNotReplayedOnEntry() { var azdo = new FakeAzureDevOpsService(); var helix = new FakeHelixService(); - var logger = new RecordingLogger(); azdo.AddTimelineResponse( - StageRecord("Test", "stage-test", "inProgress"), - MonitorJob(parentId: "stage-test"), - PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test")); - - HelixJobInfo purged = HelixJob("helix-purged-a1", "running", stageName: "Test", - submitterJobName: "Test_Linux", queueId: "gone", stageAttempt: "1"); - helix.WithWorkItems("helix-purged-a1", - [new WorkItemSummary("helix-purged-a1/wi-1", "helix-purged-a1", "wi-1", "Waiting")]); - - helix.AddResponse(jobs: [purged]); - helix.ConfigureNullResubmission("helix-purged-a1"); + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"), + PipelineJob("A", "completed", "succeeded", attempt: 2, + previousAttempts: [PreviousAttempt(1)], parentId: "stage-test")); + helix.AddResponse( + jobs: [HelixJob("ha2", "finished", stageName: "Test", + submitterJobName: "A", queueId: "q", stageAttempt: "2", + jobAttempt: "2", logicalJobName: "tests")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["ha2"] = PassFail(failed: ["a"]), + }); JobMonitorOptions options = DefaultOptions(); options.StageAttempt = "2"; - var runner = new JobMonitorRunner(options, logger, azdo, helix, NoDelay); + options.JobAttempt = "2"; - int exitCode = await runner.RunAsync(CancellationToken.None); + int exitCode = await new JobMonitorRunner( + options, NullLogger.Instance, azdo, helix, NoDelay).RunAsync(CancellationToken.None); - // Failed fast (did not hang), reported the abandoned work, and failed the monitor. exitCode.Should().Be(1); - helix.Resubmissions.Should().ContainSingle() - .Which.NewJob.Should().BeNull(); - logger.Messages.Should().Contain(m => - m.Contains("Could not resubmit", StringComparison.Ordinal) - && m.Contains("previous attempt", StringComparison.Ordinal)); + helix.Resubmissions.Should().BeEmpty(); + } + + [Fact] + public void SubmitterChainKey_IncludesStageIdentity() + { + var state = new MonitorState(); + HelixJobInfo build = HelixJob("build-job", "finished", stageName: "Build", + submitterPhaseName: "Tests", queueId: "q", logicalJobName: "tests"); + HelixJobInfo test = HelixJob("test-job", "finished", stageName: "Test", + submitterPhaseName: "Tests", queueId: "q", logicalJobName: "tests"); + + state.GetHelixJobChainKey(build).Should().NotBe(state.GetHelixJobChainKey(test)); } /// @@ -1449,9 +2309,9 @@ public async Task AttemptScopedMonitor_UnknownMonitorAttempt_TracksAllAttempts() [Fact] public async Task MultiAttempt_ResubmitsOnlyUnfinishedStreamsAcrossAttemptsAndMonitorCrash() { - HelixJobInfo a1 = HelixJob("a1", "finished", stageName: "Test", submitterJobName: "Test_A", queueId: "q1", stageAttempt: "1"); - HelixJobInfo b1 = HelixJob("b1", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", stageAttempt: "1"); - HelixJobInfo c1 = HelixJob("c1", "finished", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", stageAttempt: "1"); + HelixJobInfo a1 = HelixJob("a1", "finished", stageName: "Test", submitterJobName: "Test_A", queueId: "q1", stageAttempt: "1", jobAttempt: "1"); + HelixJobInfo b1 = HelixJob("b1", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", stageAttempt: "1", jobAttempt: "1"); + HelixJobInfo c1 = HelixJob("c1", "finished", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", stageAttempt: "1", jobAttempt: "1"); // Attempt 2: resubmit both failing streams (B, C) into attempt 2, then time out. FakeHelixService helix2 = await RunAttemptToTimeoutAsync( @@ -1471,8 +2331,8 @@ public async Task MultiAttempt_ResubmitsOnlyUnfinishedStreamsAcrossAttemptsAndMo // Attempt 3: B's attempt-2 resubmission failed; C's is still Waiting. Resubmit both // into attempt 3, then time out. - HelixJobInfo b2 = HelixJob("b2", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", previousHelixJobName: "b1", stageAttempt: "2"); - HelixJobInfo c2 = HelixJob("c2", "running", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c1", stageAttempt: "2"); + HelixJobInfo b2 = HelixJob("b2", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", previousHelixJobName: "b1", stageAttempt: "2", jobAttempt: "1"); + HelixJobInfo c2 = HelixJob("c2", "running", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c1", stageAttempt: "2", jobAttempt: "1"); FakeHelixService helix3 = await RunAttemptToTimeoutAsync( stageAttempt: "3", @@ -1494,15 +2354,17 @@ public async Task MultiAttempt_ResubmitsOnlyUnfinishedStreamsAcrossAttemptsAndMo // Attempt 5: B's attempt-3 resubmission PASSED; C's is still Waiting. The monitor must // resubmit ONLY the still-Waiting C stream, not the now-passing B stream, and complete // once C's resubmission finishes. - HelixJobInfo b3 = HelixJob("b3", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", previousHelixJobName: "b2", stageAttempt: "3"); - HelixJobInfo c3 = HelixJob("c3", "running", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c2", stageAttempt: "3"); - HelixJobInfo c5 = HelixJob("c5", "finished", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c3", stageAttempt: "5"); + HelixJobInfo b3 = HelixJob("b3", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", previousHelixJobName: "b2", stageAttempt: "3", jobAttempt: "1"); + HelixJobInfo c3 = HelixJob("c3", "running", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c2", stageAttempt: "3", jobAttempt: "1"); + HelixJobInfo c5 = HelixJob("c5", "finished", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c3", stageAttempt: "5", jobAttempt: "1"); var azdo5 = new FakeAzureDevOpsService(); azdo5.AddTimelineResponse( - StageRecord("Test", "stage-test", "inProgress"), - MonitorJob(parentId: "stage-test"), - PipelineJob("Test Suite", "completed", "succeeded", parentId: "stage-test")); + StageRecord("Test", "stage-test", "inProgress", attempt: 5), + MonitorJob(attempt: 5, parentId: "stage-test"), + PipelineJob("Test_C", "completed", "succeeded", attempt: 1, parentId: "stage-test"), + PipelineJob("Test_B", "completed", "succeeded", attempt: 1, parentId: "stage-test"), + PipelineJob("Test_A", "completed", "succeeded", attempt: 1, parentId: "stage-test")); // A and B were already uploaded by earlier attempts (durable AzDO tags). azdo5.WithPreviouslyProcessedJob("a1").WithPreviouslyProcessedJob("b3"); @@ -1529,6 +2391,7 @@ public async Task MultiAttempt_ResubmitsOnlyUnfinishedStreamsAcrossAttemptsAndMo var options5 = DefaultOptions(); options5.StageAttempt = "5"; + options5.JobAttempt = "5"; var runner5 = new JobMonitorRunner(options5, NullLogger.Instance, azdo5, helix5, NoDelay); int exit5 = await runner5.RunAsync(CancellationToken.None); @@ -1549,16 +2412,18 @@ public async Task MultiAttempt_ResubmitsOnlyUnfinishedStreamsAcrossAttemptsAndMo [Fact] public async Task MultiAttempt_PreviouslyWaitingStreamHasPassedOnRetry_NotResubmitted() { - HelixJobInfo a1 = HelixJob("a1", "finished", stageName: "Test", submitterJobName: "Test_A", queueId: "q1", stageAttempt: "1"); - HelixJobInfo b3 = HelixJob("b3", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", previousHelixJobName: "b2", stageAttempt: "3"); + HelixJobInfo a1 = HelixJob("a1", "finished", stageName: "Test", submitterJobName: "Test_A", queueId: "q1", stageAttempt: "1", jobAttempt: "1"); + HelixJobInfo b3 = HelixJob("b3", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", previousHelixJobName: "b2", stageAttempt: "3", jobAttempt: "1"); // Same C incarnation that was Waiting at the last timeout, now reported finished+passed. - HelixJobInfo c3 = HelixJob("c3", "finished", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c2", stageAttempt: "3"); + HelixJobInfo c3 = HelixJob("c3", "finished", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c2", stageAttempt: "3", jobAttempt: "1"); var azdo = new FakeAzureDevOpsService(); azdo.AddTimelineResponse( - StageRecord("Test", "stage-test", "inProgress"), - MonitorJob(parentId: "stage-test"), - PipelineJob("Test Suite", "completed", "succeeded", parentId: "stage-test")); + StageRecord("Test", "stage-test", "inProgress", attempt: 5), + MonitorJob(attempt: 5, parentId: "stage-test"), + PipelineJob("Test_A", "completed", "succeeded", attempt: 1, parentId: "stage-test"), + PipelineJob("Test_B", "completed", "succeeded", attempt: 1, parentId: "stage-test"), + PipelineJob("Test_C", "completed", "succeeded", attempt: 1, parentId: "stage-test")); azdo.WithPreviouslyProcessedJob("a1").WithPreviouslyProcessedJob("b3"); var helix = new FakeHelixService(); @@ -1573,6 +2438,7 @@ public async Task MultiAttempt_PreviouslyWaitingStreamHasPassedOnRetry_NotResubm var options = DefaultOptions(); options.StageAttempt = "5"; + options.JobAttempt = "5"; var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); @@ -1841,7 +2707,8 @@ public async Task StageScopedMonitor_OnStageRetry_OnlyRetriesFailedHelixWorkFrom helix2.AddResponse( jobs: [ - HelixJob("helix-test-linux", "finished", stageName: "Test", submitterJobName: "Test Linux"), + HelixJob("helix-test-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), HelixJob("helix-build-windows", "finished", stageName: "Build", submitterJobName: "Build Windows"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) @@ -1854,8 +2721,11 @@ public async Task StageScopedMonitor_OnStageRetry_OnlyRetriesFailedHelixWorkFrom helix2.AddResponse( jobs: [ - HelixJob("helix-test-linux", "finished", stageName: "Test", submitterJobName: "Test Linux"), - HelixJob("helix-test-linux-resub", "finished", stageName: "Test", submitterJobName: "Test Linux", previousHelixJobName: "helix-test-linux"), + HelixJob("helix-test-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-test-linux-resub", "finished", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-test-linux", + stageAttempt: "2", jobAttempt: "1"), HelixJob("helix-build-windows", "finished", stageName: "Build", submitterJobName: "Build Windows"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) @@ -1865,7 +2735,11 @@ public async Task StageScopedMonitor_OnStageRetry_OnlyRetriesFailedHelixWorkFrom ["helix-build-windows"] = PassFail(failed: ["build-fail"]), }); - var runner2 = CreateRunner(azdo2, helix2, stageName: "Test"); + JobMonitorOptions options2 = DefaultOptions(); + options2.StageName = "Test"; + options2.StageAttempt = "2"; + options2.JobAttempt = "2"; + var runner2 = new JobMonitorRunner(options2, NullLogger.Instance, azdo2, helix2, NoDelay); int exitCode2 = await runner2.RunAsync(CancellationToken.None); exitCode2.Should().Be(0); @@ -1880,9 +2754,10 @@ static void AddSingleMonitorStageTimeline(FakeAzureDevOpsService azdo, int attem PreviousAttemptReference[] previousAttempts = attempt == 1 ? null : [PreviousAttempt(1)]; azdo.AddTimelineResponse( - StageRecord("Test", "stage-test", "inProgress"), + StageRecord("Test", "stage-test", "inProgress", attempt: attempt, + previousAttempts: previousAttempts), MonitorJob(attempt: attempt, previousAttempts: previousAttempts, parentId: "stage-test"), - PipelineJob("Test Linux", "completed", "succeeded", attempt: attempt, previousAttempts: previousAttempts, parentId: "stage-test"), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1, parentId: "stage-test"), StageRecord("Build", "stage-build", "completed", "succeeded"), PipelineJob("Build Windows", "completed", "succeeded", parentId: "stage-build")); } @@ -1903,7 +2778,9 @@ public async Task StageScopedMonitor_OnRetry_IgnoresDefaultRefNameMonitorJobUnde AddRetriedStageTimeline(azdo); helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished", stageName: "Test", submitterJobName: "Build_Debug")], + jobs: [HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Build_Debug", submitterPhaseName: "Linux Build_Debug", + stageAttempt: "1", jobAttempt: "1")], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(failed: ["linux-fail"]), @@ -1912,8 +2789,12 @@ public async Task StageScopedMonitor_OnRetry_IgnoresDefaultRefNameMonitorJobUnde helix.AddResponse( jobs: [ - HelixJob("helix-linux", "finished", stageName: "Test", submitterJobName: "Build_Debug"), - HelixJob("helix-linux-resub", "finished", stageName: "Test", submitterJobName: "Build_Debug", previousHelixJobName: "helix-linux"), + HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Build_Debug", submitterPhaseName: "Linux Build_Debug", + stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "finished", stageName: "Test", + submitterJobName: "Build_Debug", submitterPhaseName: "Linux Build_Debug", + previousHelixJobName: "helix-linux", stageAttempt: "2", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -1924,6 +2805,8 @@ public async Task StageScopedMonitor_OnRetry_IgnoresDefaultRefNameMonitorJobUnde int delayCount = 0; var options = DefaultOptions(); options.StageName = "Test"; + options.StageAttempt = "2"; + options.JobAttempt = "2"; options.JobMonitorName = "HelixJobMonitor"; var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, (_, _) => @@ -1949,7 +2832,8 @@ public async Task StageScopedMonitor_OnRetry_IgnoresDefaultRefNameMonitorJobUnde static void AddRetriedStageTimeline(FakeAzureDevOpsService azdo) { azdo.AddTimelineResponse( - StageRecord("Test", "stage-test", "inProgress"), + StageRecord("Test", "stage-test", "inProgress", attempt: 2, + previousAttempts: [PreviousAttempt(1)]), new AzureDevOpsTimelineRecord { Id = "monitor-phase", @@ -1971,7 +2855,21 @@ static void AddRetriedStageTimeline(FakeAzureDevOpsService azdo) Attempt = 2, PreviousAttempts = [PreviousAttempt(1, recordId: "previous-monitor-job")], }, - PipelineJob("Linux Build_Debug", "completed", "succeeded", parentId: "stage-test")); + PipelinePhase( + "Linux Build_Debug", + "Linux Build_Debug", + "completed", + "succeeded", + attempt: 1, + parentId: "stage-test", + id: "linux-phase"), + PipelineJob( + "__default", + "completed", + "succeeded", + attempt: 1, + parentId: "linux-phase", + id: "linux-job")); } } @@ -2023,8 +2921,8 @@ public async Task MonitorTimesOut_Relaunched_UploadsRemainingResults() pollCount1++; if (pollCount1 >= 2) { - Task completed = await Task.WhenAny(azdo1.UploadCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5))); - Assert.Same(azdo1.UploadCompleted.Task, completed); + Task completed = await Task.WhenAny(azdo1.TestRunCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5))); + Assert.Same(azdo1.TestRunCompleted.Task, completed); cts.Cancel(); } }); @@ -2092,8 +2990,8 @@ public async Task MonitorTimesOut_CancelsLatestInFlightHelixJobs() var runner = new JobMonitorRunner(DefaultOptions(), logger, azdo, helix, async (_, _) => { - Task completed = await Task.WhenAny(azdo.UploadCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5))); - completed.Should().BeSameAs(azdo.UploadCompleted.Task); + Task completed = await Task.WhenAny(azdo.TestRunCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5))); + completed.Should().BeSameAs(azdo.TestRunCompleted.Task); cts.Cancel(); }); @@ -2224,8 +3122,8 @@ public async Task MonitorTimesOut_DoesNotReportOrCancelJobsThatFinishedAfterFirs // Wait until helix-good's results have actually been uploaded before // cancelling, so the monitor has had a chance to record its terminal // state. - Task completed = await Task.WhenAny(azdo.UploadCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5))); - completed.Should().BeSameAs(azdo.UploadCompleted.Task); + Task completed = await Task.WhenAny(azdo.TestRunCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5))); + completed.Should().BeSameAs(azdo.TestRunCompleted.Task); cts.Cancel(); } }); @@ -2293,8 +3191,8 @@ public async Task MonitorTimesOut_PartialProgress_Relaunched_CompletesSuccessful pollCount1++; if (pollCount1 >= 2) { - Task completed = await Task.WhenAny(azdo1.UploadCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5))); - Assert.Same(azdo1.UploadCompleted.Task, completed); + Task completed = await Task.WhenAny(azdo1.TestRunCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5))); + Assert.Same(azdo1.TestRunCompleted.Task, completed); cts.Cancel(); } }); @@ -2367,19 +3265,20 @@ public async Task RetryAttempt2_ResubmitsFailedWorkItems_ResubmissionPasses_Exit // Pipeline jobs already completed azdo.AddTimelineResponse( - MonitorJob(), - PipelineJob("Test Linux", "completed", "succeeded")); + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); // Extra polls while resubmitted job runs azdo.AddTimelineResponse( - MonitorJob(), - PipelineJob("Test Linux", "completed", "succeeded")); + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); azdo.AddTimelineResponse( - MonitorJob(), - PipelineJob("Test Linux", "completed", "succeeded")); + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); // Poll 1: original Helix job finished with 1 pass + 2 failures helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished")], + jobs: [HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(passed: ["wi-ok"], failed: ["wi-fail-1", "wi-fail-2"]), @@ -2389,7 +3288,14 @@ public async Task RetryAttempt2_ResubmitsFailedWorkItems_ResubmissionPasses_Exit // Poll 2: resubmitted job appears, running helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished"), HelixJob("helix-linux-resub", "running", previousHelixJobName: "helix-linux")], + jobs: + [ + HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "running", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", + stageAttempt: "2", jobAttempt: "1"), + ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(passed: ["wi-ok"], failed: ["wi-fail-1", "wi-fail-2"]), @@ -2397,14 +3303,24 @@ public async Task RetryAttempt2_ResubmitsFailedWorkItems_ResubmissionPasses_Exit // Poll 3: resubmitted job finished — both items now pass helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished"), HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux")], + jobs: + [ + HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "finished", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", + stageAttempt: "2", jobAttempt: "1"), + ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(passed: ["wi-ok"], failed: ["wi-fail-1", "wi-fail-2"]), ["helix-linux-resub"] = PassFail(passed: ["wi-fail-1", "wi-fail-2"]), }); - var runner = CreateRunner(azdo, helix); + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); // Resubmission healed the failures → exit 0 @@ -2433,15 +3349,16 @@ public async Task RetryAttempt2_ResubmitsFailedWorkItems_ResubmissionAlsoFails_E var helix = new FakeHelixService(); azdo.AddTimelineResponse( - MonitorJob(), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), PipelineJob("Test Linux", "completed", "succeeded")); azdo.AddTimelineResponse( - MonitorJob(), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), PipelineJob("Test Linux", "completed", "succeeded")); // Poll 1: original job has 1 failure helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished")], + jobs: [HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(passed: ["wi-ok"], failed: ["wi-fail"]), @@ -2450,14 +3367,24 @@ public async Task RetryAttempt2_ResubmitsFailedWorkItems_ResubmissionAlsoFails_E // Poll 2: resubmission finished but STILL fails helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished"), HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux")], + jobs: + [ + HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "finished", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", + stageAttempt: "2", jobAttempt: "1"), + ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(passed: ["wi-ok"], failed: ["wi-fail"]), ["helix-linux-resub"] = PassFail(failed: ["wi-fail"]), }); - var runner = CreateRunner(azdo, helix); + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); // Resubmission also failed → exit 1 @@ -2477,21 +3404,27 @@ public async Task RetryAttempt2_MultipleJobs_OnlyFailedItemsResubmitted_ExitZero var helix = new FakeHelixService(); azdo.AddTimelineResponse( - MonitorJob(), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), PipelineJob("Test Linux", "completed", "succeeded"), PipelineJob("Test Windows", "completed", "succeeded")); azdo.AddTimelineResponse( - MonitorJob(), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), PipelineJob("Test Linux", "completed", "succeeded"), PipelineJob("Test Windows", "completed", "succeeded")); azdo.AddTimelineResponse( - MonitorJob(), + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), PipelineJob("Test Linux", "completed", "succeeded"), PipelineJob("Test Windows", "completed", "succeeded")); // Poll 1: both original jobs finished with mixed results helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished"), HelixJob("helix-windows", "finished")], + jobs: + [ + HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-windows", "finished", stageName: "Test", + submitterJobName: "Test Windows", stageAttempt: "1", jobAttempt: "1"), + ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(passed: ["linux-ok-1", "linux-ok-2"], failed: ["linux-fail"]), @@ -2504,9 +3437,10 @@ public async Task RetryAttempt2_MultipleJobs_OnlyFailedItemsResubmitted_ExitZero helix.AddResponse( jobs: [ - HelixJob("helix-linux", "finished"), HelixJob("helix-windows", "finished"), - HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"), - HelixJob("helix-windows-resub", "running", previousHelixJobName: "helix-windows"), + HelixJob("helix-linux", "finished", stageName: "Test", submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-windows", "finished", stageName: "Test", submitterJobName: "Test Windows", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "finished", stageName: "Test", submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", stageAttempt: "2", jobAttempt: "1"), + HelixJob("helix-windows-resub", "running", stageName: "Test", submitterJobName: "Test Windows", previousHelixJobName: "helix-windows", stageAttempt: "2", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -2519,9 +3453,10 @@ public async Task RetryAttempt2_MultipleJobs_OnlyFailedItemsResubmitted_ExitZero helix.AddResponse( jobs: [ - HelixJob("helix-linux", "finished"), HelixJob("helix-windows", "finished"), - HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"), - HelixJob("helix-windows-resub", "finished", previousHelixJobName: "helix-windows"), + HelixJob("helix-linux", "finished", stageName: "Test", submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-windows", "finished", stageName: "Test", submitterJobName: "Test Windows", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "finished", stageName: "Test", submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", stageAttempt: "2", jobAttempt: "1"), + HelixJob("helix-windows-resub", "finished", stageName: "Test", submitterJobName: "Test Windows", previousHelixJobName: "helix-windows", stageAttempt: "2", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -2531,7 +3466,10 @@ public async Task RetryAttempt2_MultipleJobs_OnlyFailedItemsResubmitted_ExitZero ["helix-windows-resub"] = PassFail(passed: ["win-fail-1", "win-fail-2"]), }); - var runner = CreateRunner(azdo, helix); + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); exitCode.Should().Be(0); @@ -2590,20 +3528,31 @@ public async Task HelixJobFailsAfterMonitorEntry_IsNotResubmittedUntilNextEntry( azdo2.AddTimelineResponse(MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), PipelineJob("Test Linux", "completed", "succeeded")); helix2.AddResponse( - jobs: [HelixJob("helix-linux", "finished")], + jobs: [HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(failed: ["wi-fail"]), }); helix2.ConfigureResubmission("helix-linux", "helix-linux-resub"); helix2.AddResponse( - jobs: [HelixJob("helix-linux", "finished"), HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux")], + jobs: + [ + HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "finished", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", + stageAttempt: "2", jobAttempt: "1"), + ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux-resub"] = PassFail(passed: ["wi-fail"]), }); - var runner2 = CreateRunner(azdo2, helix2); + JobMonitorOptions options2 = DefaultOptions(); + options2.StageAttempt = "2"; + options2.JobAttempt = "2"; + var runner2 = new JobMonitorRunner(options2, NullLogger.Instance, azdo2, helix2, NoDelay); int exitCode2 = await runner2.RunAsync(CancellationToken.None); exitCode2.Should().Be(0); @@ -2701,14 +3650,21 @@ public async Task LatestCompletedIncarnationPartiallyHealed_ResubmitsOnlyRemaini var azdo = new FakeAzureDevOpsService(); var helix = new FakeHelixService(); - azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); - azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); + azdo.AddTimelineResponse( + MonitorJob(attempt: 3, previousAttempts: [PreviousAttempt(1), PreviousAttempt(2)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); + azdo.AddTimelineResponse( + MonitorJob(attempt: 3, previousAttempts: [PreviousAttempt(1), PreviousAttempt(2)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); helix.AddResponse( jobs: [ - HelixJob("helix-linux", "finished"), - HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"), + HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "finished", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", + stageAttempt: "2", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -2719,9 +3675,14 @@ public async Task LatestCompletedIncarnationPartiallyHealed_ResubmitsOnlyRemaini helix.AddResponse( jobs: [ - HelixJob("helix-linux", "finished"), - HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"), - HelixJob("helix-linux-resub-2", "finished", previousHelixJobName: "helix-linux-resub"), + HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "finished", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", + stageAttempt: "2", jobAttempt: "1"), + HelixJob("helix-linux-resub-2", "finished", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-linux-resub", + stageAttempt: "3", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -2730,7 +3691,10 @@ public async Task LatestCompletedIncarnationPartiallyHealed_ResubmitsOnlyRemaini ["helix-linux-resub-2"] = PassFail(passed: ["wi-2"]), }); - var runner = CreateRunner(azdo, helix); + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "3"; + options.JobAttempt = "3"; + var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); exitCode.Should().Be(0); @@ -2868,7 +3832,11 @@ public async Task RetryAfterMixedAzDOAndHelixFailures_RestartsFailedHelixWorkAnd PipelineJob("C", "completed", "succeeded", attempt: 2, previousAttempts: [PreviousAttempt(1)])); helix2.AddResponse( - jobs: [HelixJob("helix-a", "finished", submitterJobName: "A"), HelixJob("helix-b", "finished", submitterJobName: "B")], + jobs: + [ + HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"), + ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-a"] = PassFail(failed: ["a-fail"]), @@ -2879,10 +3847,10 @@ public async Task RetryAfterMixedAzDOAndHelixFailures_RestartsFailedHelixWorkAnd helix2.AddResponse( jobs: [ - HelixJob("helix-a", "finished", submitterJobName: "A"), - HelixJob("helix-b", "finished", submitterJobName: "B"), - HelixJob("helix-a-resub", "finished", submitterJobName: "A", previousHelixJobName: "helix-a"), - HelixJob("helix-b-resub", "finished", submitterJobName: "B", previousHelixJobName: "helix-b"), + HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-a-resub", "finished", stageName: "Test", submitterJobName: "A", previousHelixJobName: "helix-a", stageAttempt: "2", jobAttempt: "1"), + HelixJob("helix-b-resub", "finished", stageName: "Test", submitterJobName: "B", previousHelixJobName: "helix-b", stageAttempt: "2", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -2890,7 +3858,10 @@ public async Task RetryAfterMixedAzDOAndHelixFailures_RestartsFailedHelixWorkAnd ["helix-b-resub"] = PassFail(failed: ["b-fail"]), }); - var runner2 = CreateRunner(azdo2, helix2); + JobMonitorOptions options2 = DefaultOptions(); + options2.StageAttempt = "2"; + options2.JobAttempt = "2"; + var runner2 = new JobMonitorRunner(options2, NullLogger.Instance, azdo2, helix2, NoDelay); int exitCode2 = await runner2.RunAsync(CancellationToken.None); exitCode2.Should().Be(1); @@ -2922,10 +3893,10 @@ public async Task RetryAfterMixedAzDOAndHelixFailures_RestartsFailedHelixWorkAnd helix3.AddResponse( jobs: [ - HelixJob("helix-a", "finished", submitterJobName: "A"), - HelixJob("helix-b", "finished", submitterJobName: "B"), - HelixJob("helix-a-resub", "finished", submitterJobName: "A", previousHelixJobName: "helix-a"), - HelixJob("helix-b-resub", "finished", submitterJobName: "B", previousHelixJobName: "helix-b"), + HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-a-resub", "finished", stageName: "Test", submitterJobName: "A", previousHelixJobName: "helix-a", stageAttempt: "2", jobAttempt: "1"), + HelixJob("helix-b-resub", "finished", stageName: "Test", submitterJobName: "B", previousHelixJobName: "helix-b", stageAttempt: "2", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -2936,18 +3907,21 @@ public async Task RetryAfterMixedAzDOAndHelixFailures_RestartsFailedHelixWorkAnd helix3.AddResponse( jobs: [ - HelixJob("helix-a", "finished", submitterJobName: "A"), - HelixJob("helix-b", "finished", submitterJobName: "B"), - HelixJob("helix-a-resub", "finished", submitterJobName: "A", previousHelixJobName: "helix-a"), - HelixJob("helix-b-resub", "finished", submitterJobName: "B", previousHelixJobName: "helix-b"), - HelixJob("helix-b-resub-2", "finished", submitterJobName: "B", previousHelixJobName: "helix-b-resub"), + HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-a-resub", "finished", stageName: "Test", submitterJobName: "A", previousHelixJobName: "helix-a", stageAttempt: "2", jobAttempt: "1"), + HelixJob("helix-b-resub", "finished", stageName: "Test", submitterJobName: "B", previousHelixJobName: "helix-b", stageAttempt: "2", jobAttempt: "1"), + HelixJob("helix-b-resub-2", "finished", stageName: "Test", submitterJobName: "B", previousHelixJobName: "helix-b-resub", stageAttempt: "3", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-b-resub-2"] = PassFail(passed: ["b-fail"]), }); - var runner3 = CreateRunner(azdo3, helix3); + JobMonitorOptions options3 = DefaultOptions(); + options3.StageAttempt = "3"; + options3.JobAttempt = "3"; + var runner3 = new JobMonitorRunner(options3, NullLogger.Instance, azdo3, helix3, NoDelay); int exitCode3 = await runner3.RunAsync(CancellationToken.None); exitCode3.Should().Be(0); @@ -3002,17 +3976,15 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork() // Wait for helix-a's test-result upload to finish before cancelling so the // assertion below is deterministic. Otherwise the background upload task // (bound to the runner's cancellation token) races against cts1.Cancel(). - Task completed = await Task.WhenAny(azdo1.UploadCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5))); - completed.Should().BeSameAs(azdo1.UploadCompleted.Task); + Task completed = await Task.WhenAny(azdo1.TestRunCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5))); + completed.Should().BeSameAs(azdo1.TestRunCompleted.Task); cts1.Cancel(); }); int exitCode1 = await runner1.RunAsync(cts1.Token); exitCode1.Should().Be(1); - helix1.Resubmissions.Should().ContainSingle(); - helix1.Resubmissions[0].OriginalJob.Should().Be("helix-a"); - helix1.Resubmissions[0].FailedItems.Should().BeEquivalentTo(["a-fail"]); + helix1.Resubmissions.Should().BeEmpty(); azdo1.UploadedJobNames.Should().BeEquivalentTo(["helix-a"]); var azdo2 = new FakeAzureDevOpsService(); @@ -3071,9 +4043,9 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork() helix3.AddResponse( jobs: [ - HelixJob("helix-a", "finished", submitterJobName: "A"), - HelixJob("helix-a-resub", "running", submitterJobName: "A", previousHelixJobName: "helix-a"), - HelixJob("helix-b", "finished", submitterJobName: "B"), + HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-a-resub", "running", stageName: "Test", submitterJobName: "A", previousHelixJobName: "helix-a", stageAttempt: "2", jobAttempt: "1"), + HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -3083,10 +4055,10 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork() helix3.AddResponse( jobs: [ - HelixJob("helix-a", "finished", submitterJobName: "A"), - HelixJob("helix-a-resub", "running", submitterJobName: "A", previousHelixJobName: "helix-a"), - HelixJob("helix-b", "finished", submitterJobName: "B"), - HelixJob("helix-b-resub", "running", submitterJobName: "B", previousHelixJobName: "helix-b"), + HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-a-resub", "running", stageName: "Test", submitterJobName: "A", previousHelixJobName: "helix-a", stageAttempt: "2", jobAttempt: "1"), + HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-b-resub", "running", stageName: "Test", submitterJobName: "B", previousHelixJobName: "helix-b", stageAttempt: "3", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -3095,10 +4067,10 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork() helix3.AddResponse( jobs: [ - HelixJob("helix-a", "finished", submitterJobName: "A"), - HelixJob("helix-a-resub", "finished", submitterJobName: "A", previousHelixJobName: "helix-a"), - HelixJob("helix-b", "finished", submitterJobName: "B"), - HelixJob("helix-b-resub", "finished", submitterJobName: "B", previousHelixJobName: "helix-b"), + HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-a-resub", "finished", stageName: "Test", submitterJobName: "A", previousHelixJobName: "helix-a", stageAttempt: "2", jobAttempt: "1"), + HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-b-resub", "finished", stageName: "Test", submitterJobName: "B", previousHelixJobName: "helix-b", stageAttempt: "3", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -3107,7 +4079,10 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork() ["helix-b-resub"] = PassFail(passed: ["b-fail"]), }); - var runner3 = CreateRunner(azdo3, helix3); + JobMonitorOptions options3 = DefaultOptions(); + options3.StageAttempt = "3"; + options3.JobAttempt = "3"; + var runner3 = new JobMonitorRunner(options3, NullLogger.Instance, azdo3, helix3, NoDelay); int exitCode3 = await runner3.RunAsync(CancellationToken.None); exitCode3.Should().Be(0); @@ -3122,7 +4097,7 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork() /// the monitor's first attempt. The original failed results are still uploaded normally. /// [Fact] - public async Task Attempt1_ResubmitsFailedWorkItemsFoundOnEntry_ExitOne() + public async Task Attempt1_DoesNotResubmitFailedWorkItemsFoundOnEntry_ExitOne() { var azdo = new FakeAzureDevOpsService(); var helix = new FakeHelixService(); @@ -3142,9 +4117,7 @@ public async Task Attempt1_ResubmitsFailedWorkItemsFoundOnEntry_ExitOne() int exitCode = await runner.RunAsync(CancellationToken.None); exitCode.Should().Be(1); - helix.Resubmissions.Should().ContainSingle(); - helix.Resubmissions[0].OriginalJob.Should().Be("helix-linux"); - helix.Resubmissions[0].FailedItems.Should().BeEquivalentTo(["wi-fail"]); + helix.Resubmissions.Should().BeEmpty(); azdo.UploadedJobNames.Should().ContainSingle(); // only original } @@ -3175,9 +4148,7 @@ public async Task FailedAzdoSubmitter_WithoutSystemJobNameLink_IsNotIgnored() int exitCode = await runner.RunAsync(CancellationToken.None); exitCode.Should().Be(1); - helix.Resubmissions.Should().ContainSingle(); - helix.Resubmissions[0].OriginalJob.Should().Be("helix-a"); - helix.Resubmissions[0].FailedItems.Should().BeEquivalentTo(["a-fail"]); + helix.Resubmissions.Should().BeEmpty(); azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-a"]); } @@ -3192,18 +4163,22 @@ public async Task ResubmitReturnsNull_FailedSubmitterIsNotIgnored() var helix = new FakeHelixService(); azdo.AddTimelineResponse( - MonitorJob(), - PipelineJob("A", "completed", "failed")); + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("A", "completed", "failed", attempt: 1)); helix.AddResponse( - jobs: [HelixJob("helix-a", "finished", submitterJobName: "A")], + jobs: [HelixJob("helix-a", "finished", stageName: "Test", + submitterJobName: "A", stageAttempt: "1", jobAttempt: "1")], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-a"] = PassFail(failed: ["a-fail"]), }); helix.ConfigureNullResubmission("helix-a"); - var runner = CreateRunner(azdo, helix); + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); exitCode.Should().Be(1); @@ -3330,11 +4305,16 @@ public async Task PreviouslyProcessedFailedJob_StillContributesToRetryAndPassFai azdo.WithPreviouslyProcessedJob("helix-linux"); var helix = new FakeHelixService(); - azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); - azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); + azdo.AddTimelineResponse( + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); + azdo.AddTimelineResponse( + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished")], + jobs: [HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(failed: ["wi-fail"]), @@ -3343,15 +4323,21 @@ public async Task PreviouslyProcessedFailedJob_StillContributesToRetryAndPassFai helix.AddResponse( jobs: [ - HelixJob("helix-linux", "finished"), - HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"), + HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "finished", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", + stageAttempt: "2", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux-resub"] = PassFail(passed: ["wi-fail"]), }); - var runner = CreateRunner(azdo, helix); + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); exitCode.Should().Be(0); @@ -3373,11 +4359,16 @@ public async Task PreviouslyProcessedFailedJob_WithoutResubmission_StillFailsBui azdo.WithPreviouslyProcessedJob("helix-linux"); var helix = new FakeHelixService(); - azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); - azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); + azdo.AddTimelineResponse( + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); + azdo.AddTimelineResponse( + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished")], + jobs: [HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(failed: ["wi-fail"]), @@ -3385,7 +4376,10 @@ public async Task PreviouslyProcessedFailedJob_WithoutResubmission_StillFailsBui // The failed work item cannot be resubmitted (e.g. resubmission limit reached). helix.ConfigureNullResubmission("helix-linux"); - var runner = CreateRunner(azdo, helix); + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); exitCode.Should().Be(1); @@ -3523,7 +4517,8 @@ public async Task StageScopedMonitor_LineageCrossesStageBoundary_IgnoresOutOfSta helix.AddResponse( jobs: [ - HelixJob("helix-test", "finished", stageName: "Test"), + HelixJob("helix-test", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), HelixJob("helix-build-retry", "running", stageName: "Build", previousHelixJobName: "helix-test"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) @@ -3534,8 +4529,11 @@ public async Task StageScopedMonitor_LineageCrossesStageBoundary_IgnoresOutOfSta helix.AddResponse( jobs: [ - HelixJob("helix-test", "finished", stageName: "Test"), - HelixJob("helix-test-resub", "finished", stageName: "Test", previousHelixJobName: "helix-test"), + HelixJob("helix-test", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-test-resub", "finished", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-test", + stageAttempt: "2", jobAttempt: "1"), HelixJob("helix-build-retry", "running", stageName: "Build", previousHelixJobName: "helix-test"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) @@ -3544,7 +4542,11 @@ public async Task StageScopedMonitor_LineageCrossesStageBoundary_IgnoresOutOfSta ["helix-test-resub"] = PassFail(passed: ["test-fail"]), }); - var runner = CreateRunner(azdo, helix, stageName: "Test"); + JobMonitorOptions options = DefaultOptions(); + options.StageName = "Test"; + options.StageAttempt = "2"; + options.JobAttempt = "2"; + var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); exitCode.Should().Be(0); @@ -3589,10 +4591,15 @@ public async Task FinishedHelixJob_WithNonFinishedWorkItemState_IsFailure() var azdo = new FakeAzureDevOpsService(); var helix = new FakeHelixService(); - azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); - azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); + azdo.AddTimelineResponse( + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); + azdo.AddTimelineResponse( + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); - helix.AddResponse(jobs: [HelixJob("helix-linux", "finished")]); + helix.AddResponse(jobs: [HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")]); helix.WithWorkItems( "helix-linux", [new WorkItemSummary("helix-linux/wi-timeout", "helix-linux", "wi-timeout", "TimedOut") { ExitCode = 0 }]); @@ -3600,15 +4607,21 @@ public async Task FinishedHelixJob_WithNonFinishedWorkItemState_IsFailure() helix.AddResponse( jobs: [ - HelixJob("helix-linux", "finished"), - HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"), + HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "finished", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", + stageAttempt: "2", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux-resub"] = PassFail(passed: ["wi-timeout"]), }); - var runner = CreateRunner(azdo, helix); + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); exitCode.Should().Be(0); @@ -3663,6 +4676,19 @@ public async Task CompletedHelixJob_LogsFailedWorkItemConsoleLinks() message.Contains("Helix job: helix-linux", StringComparison.Ordinal)); } + [Fact] + public async Task NonCallerOperationCanceledException_Propagates() + { + var azdo = new FakeAzureDevOpsService() + .FailNextTimeline(new OperationCanceledException("Injected service timeout.")); + var runner = CreateRunner(azdo, new FakeHelixService()); + + Func action = () => runner.RunAsync(CancellationToken.None); + + await action.Should().ThrowAsync() + .WithMessage("Injected service timeout."); + } + [Fact] public async Task LoopStatus_LogsAggregateHelixJobWorkItemCounts() { @@ -3710,7 +4736,7 @@ public async Task LoopStatus_LogsAggregateHelixJobWorkItemCounts() } [Fact] - public async Task LoopStatus_VerboseLogsFullHelixJobWorkItemList() + public async Task LoopStatus_VerboseLogsBoundedPipelineDiagnostics() { var azdo = new FakeAzureDevOpsService(); var helix = new FakeHelixService(); @@ -3743,12 +4769,10 @@ public async Task LoopStatus_VerboseLogsFullHelixJobWorkItemList() exitCode.Should().Be(1); logger.Messages.Should().Contain(message => - message.Contains("Helix job details:", StringComparison.Ordinal) - && message.Contains("└─ 🧪 Helix job helix-linux [Running]", StringComparison.Ordinal) - && message.Contains(" ├─ wi-01 (Running)", StringComparison.Ordinal) - && message.Contains(" ├─ wi-10 (Running)", StringComparison.Ordinal) - && message.Contains(" ├─ wi-11 (Running)", StringComparison.Ordinal) - && message.Contains(" └─ wi-12 (Running)", StringComparison.Ordinal)); + message.Contains("Upload pipeline:", StringComparison.Ordinal)); + logger.Messages.Should().NotContain(message => + message.Contains("wi-01", StringComparison.Ordinal) + || message.Contains("wi-12", StringComparison.Ordinal)); } /// @@ -3790,7 +4814,7 @@ public async Task LoopStatus_WaitingWorkItems_CountedAsWaitingWhileJobIsRunning( logger.Messages.Should().Contain(message => message.Contains("0 processed / 0 completed / 0 running / 2 waiting work items", StringComparison.Ordinal)); logger.Messages.Should().Contain(message => - message.Contains("└─ 🧪 Helix job helix-linux [Running]", StringComparison.Ordinal)); + message.Contains("Upload pipeline:", StringComparison.Ordinal)); } /// @@ -3836,6 +4860,9 @@ public async Task HelixWorkItemPassesByExitCode_TestUploadReportsFailure_ExitOne helix.Resubmissions.Should().BeEmpty(); azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-linux"]); azdo.CompletedTestRunIds.Should().ContainSingle(); + IReadOnlyDictionary> failedTestWorkItems = + await azdo.GetFailedTestWorkItemsAsync(CancellationToken.None); + failedTestWorkItems["helix-linux"].Should().BeEquivalentTo(["workitem-1"]); } /// @@ -3859,14 +4886,19 @@ public async Task RetryPass_ResubmitsWorkItemThatPassedExitCodeButHasPriorFailed // It must not be resubmitted twice (once per failure reason). azdo.WithRecordedFailedTest("helix-linux", "workitem-2"); - azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); - azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); + azdo.AddTimelineResponse( + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); + azdo.AddTimelineResponse( + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); // Snapshot 1 (retry pass): only the original Helix job exists, finished, with // workitem-1 passing by exit code and workitem-2 failing by exit code. Without the // AzDO test-failure lookup the retry pass would only resubmit workitem-2. helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished")], + jobs: [HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(passed: ["workitem-1"], failed: ["workitem-2"]), @@ -3876,8 +4908,11 @@ public async Task RetryPass_ResubmitsWorkItemThatPassedExitCodeButHasPriorFailed helix.AddResponse( jobs: [ - HelixJob("helix-linux", "finished"), - HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"), + HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "finished", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", + stageAttempt: "2", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -3886,7 +4921,10 @@ public async Task RetryPass_ResubmitsWorkItemThatPassedExitCodeButHasPriorFailed }); helix.ConfigureResubmission("helix-linux", "helix-linux-resub"); - var runner = CreateRunner(azdo, helix, logger: logger); + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + var runner = new JobMonitorRunner(options, logger, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); helix.Resubmissions.Should().ContainSingle(); @@ -3922,11 +4960,16 @@ public async Task RetryPass_LogDistinguishesExitCodeAndTestFailureReasons() azdo.WithRecordedFailedTest("helix-linux", "wi-test-only"); - azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); - azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded")); + azdo.AddTimelineResponse( + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); + azdo.AddTimelineResponse( + MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), + PipelineJob("Test Linux", "completed", "succeeded", attempt: 1)); helix.AddResponse( - jobs: [HelixJob("helix-linux", "finished")], + jobs: [HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { ["helix-linux"] = PassFail(passed: ["wi-test-only"], failed: ["wi-exit-fail"]), @@ -3935,8 +4978,11 @@ public async Task RetryPass_LogDistinguishesExitCodeAndTestFailureReasons() helix.AddResponse( jobs: [ - HelixJob("helix-linux", "finished"), - HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"), + HelixJob("helix-linux", "finished", stageName: "Test", + submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"), + HelixJob("helix-linux-resub", "finished", stageName: "Test", + submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", + stageAttempt: "2", jobAttempt: "1"), ], passFailByJob: new(StringComparer.OrdinalIgnoreCase) { @@ -3945,7 +4991,10 @@ public async Task RetryPass_LogDistinguishesExitCodeAndTestFailureReasons() }); helix.ConfigureResubmission("helix-linux", "helix-linux-resub"); - var runner = CreateRunner(azdo, helix, logger: logger); + JobMonitorOptions options = DefaultOptions(); + options.StageAttempt = "2"; + options.JobAttempt = "2"; + var runner = new JobMonitorRunner(options, logger, azdo, helix, NoDelay); int exitCode = await runner.RunAsync(CancellationToken.None); exitCode.Should().Be(0); @@ -3999,6 +5048,9 @@ public async Task FailOnFailedTestsDisabled_TestFailuresIgnoredForOutcomeAndRetr exitCode.Should().Be(0); helix.Resubmissions.Should().BeEmpty(); azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-linux"]); + IReadOnlyDictionary> failedTestWorkItems = + await azdo.GetFailedTestWorkItemsAsync(CancellationToken.None); + failedTestWorkItems["helix-linux"].Should().BeEquivalentTo(["workitem-1"]); } // ----------------------------------------------------------------------- @@ -4017,6 +5069,8 @@ public async Task FailOnFailedTestsDisabled_TestFailuresIgnoredForOutcomeAndRetr BuildReason = "PullRequest", SourceBranch = "refs/pull/99999/merge", StageName = "Test", + StageAttempt = "1", + JobAttempt = "1", SystemAccessToken = "token", TeamProject = "public", WorkingDirectory = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "job-monitor-test"), @@ -4024,6 +5078,15 @@ public async Task FailOnFailedTestsDisabled_TestFailuresIgnoredForOutcomeAndRetr private static readonly Func NoDelay = (_, _) => Task.CompletedTask; + private static async Task WaitForAsync(Func condition) + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + while (!condition()) + { + await Task.Delay(10, timeout.Token); + } + } + private static WorkItemSummary WaitingItem(string jobName, string workItemName) => new($"{jobName}/{workItemName}", jobName, workItemName, "Waiting"); @@ -4043,9 +5106,11 @@ private static async Task RunAttemptToTimeoutAsync( { var azdo = new FakeAzureDevOpsService(); azdo.AddTimelineResponse( - StageRecord("Test", "stage-test", "inProgress"), - MonitorJob(parentId: "stage-test"), - PipelineJob("Test Suite", "completed", "succeeded", parentId: "stage-test")); + StageRecord("Test", "stage-test", "inProgress", attempt: int.Parse(stageAttempt)), + MonitorJob(attempt: int.Parse(stageAttempt), parentId: "stage-test"), + PipelineJob("Test_A", "completed", "succeeded", attempt: 1, parentId: "stage-test"), + PipelineJob("Test_B", "completed", "succeeded", attempt: 1, parentId: "stage-test"), + PipelineJob("Test_C", "completed", "succeeded", attempt: 1, parentId: "stage-test")); var helix = new FakeHelixService(); helix.AddResponse(jobs: entryLeaves, passFailByJob: passFailByJob); @@ -4060,6 +5125,8 @@ private static async Task RunAttemptToTimeoutAsync( var options = DefaultOptions(); options.StageAttempt = stageAttempt; + options.JobAttempt = stageAttempt; + options.JobAttempt = stageAttempt; using var cts = new CancellationTokenSource(); var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, @@ -4084,7 +5151,7 @@ private static JobMonitorRunner CreateRunner( private sealed class RecordingLogger : ILogger { - public List Messages { get; } = []; + public ConcurrentQueue Messages { get; } = []; public IDisposable BeginScope(TState state) => NullScope.Instance; @@ -4097,7 +5164,7 @@ public void Log( Exception exception, Func formatter) { - Messages.Add(formatter(state, exception)); + Messages.Enqueue(formatter(state, exception)); } private sealed class NullScope : IDisposable diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs index babcf1d287c..ed52e6a3124 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs @@ -15,6 +15,27 @@ namespace Microsoft.DotNet.Helix.Sdk.Tests { public class LocalTestResultsReaderTests { + [Theory] + [InlineData("results.trx")] + [InlineData("results.trx.txt")] + [InlineData("testResults.xml")] + [InlineData("testResults.xml.txt")] + [InlineData("junit-results.xml")] + [InlineData("junit-results.xml.txt")] + public void LooksLikeTestResultFile_RecognizesHelixTextSuffix(string fileName) + { + Assert.True(LocalTestResultsReader.LooksLikeTestResultFile(fileName)); + } + + [Theory] + [InlineData("results.xml")] + [InlineData("results.txt")] + [InlineData("testResults.xml.log")] + public void LooksLikeTestResultFile_RejectsUnknownNames(string fileName) + { + Assert.False(LocalTestResultsReader.LooksLikeTestResultFile(fileName)); + } + public static IEnumerable AttachmentModeCases() { foreach (string format in new[] { "xunit", "junit", "trx" }) @@ -96,7 +117,7 @@ public async Task LocalTestResultsReader_ReadsXunitFileFromDownloadedResults() try { File.WriteAllText( - Path.Combine(workItemDirectory, "testResults.xml"), + Path.Combine(workItemDirectory, "testResults.xml.txt"), """ @@ -108,7 +129,7 @@ public async Task LocalTestResultsReader_ReadsXunitFileFromDownloadedResults() """); var reader = new LocalTestResultsReader(NullLoggerFactory.Instance.CreateLogger()); - string filePath = Path.Combine(workItemDirectory, "testResults.xml"); + string filePath = Path.Combine(workItemDirectory, "testResults.xml.txt"); IReadOnlyList resultSets = await reader.ReadResultFileAsync(filePath); IReadOnlyList aggregate = new ResultAggregator().Aggregate([resultSets]); AggregatedResult result = Assert.Single(aggregate); diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj index 9a9eab6b519..c464b912e7d 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj @@ -21,7 +21,6 @@ - diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs index a79dae81f4a..8287f64c0f7 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs @@ -10,8 +10,23 @@ internal static class ScenarioHelpers { public const string DefaultMonitorName = "Helix Job Monitor"; - public static AzureDevOpsTimelineRecord StageRecord(string name, string id, string state, string result = null) - => new() { Type = "Stage", ReferenceName = name, Id = id, State = state, Result = result }; + public static AzureDevOpsTimelineRecord StageRecord( + string name, + string id, + string state, + string result = null, + int attempt = 1, + PreviousAttemptReference[] previousAttempts = null) + => new() + { + Type = "Stage", + ReferenceName = name, + Id = id, + State = state, + Result = result, + Attempt = attempt, + PreviousAttempts = previousAttempts, + }; public static AzureDevOpsTimelineRecord PipelineJob( string name, string state, string result = null, int attempt = 1, @@ -28,6 +43,22 @@ public static AzureDevOpsTimelineRecord PipelineJob( Id = id ?? name, }; + public static AzureDevOpsTimelineRecord PipelinePhase( + string referenceName, string name, string state, string result = null, int attempt = 1, + PreviousAttemptReference[] previousAttempts = null, string parentId = null, string id = null) + => new() + { + Type = "Phase", + ReferenceName = referenceName, + Name = name, + State = state, + Result = result, + Attempt = attempt, + PreviousAttempts = previousAttempts, + ParentId = parentId, + Id = id ?? referenceName, + }; + public static AzureDevOpsTimelineRecord MonitorJob( string name = DefaultMonitorName, int attempt = 1, PreviousAttemptReference[] previousAttempts = null, string parentId = null) @@ -54,7 +85,10 @@ public static HelixJobInfo HelixJob( string queueId = null, string previousHelixJobName = null, int? initialWorkItemCount = null, - string stageAttempt = null) + string stageAttempt = null, + string jobAttempt = null, + string logicalJobName = null, + string submitterPhaseName = null) => new( jobName, status, @@ -64,7 +98,10 @@ public static HelixJobInfo HelixJob( queueId: queueId, previousHelixJobName: previousHelixJobName, initialWorkItemCount: initialWorkItemCount, - stageAttempt: stageAttempt); + stageAttempt: stageAttempt, + jobAttempt: jobAttempt, + logicalJobName: logicalJobName, + submitterPhaseName: submitterPhaseName); public static HelixJobPassFail PassFail(string[] passed = null, string[] failed = null) => new(passed ?? [], failed ?? []); diff --git a/src/Microsoft.DotNet.Helix/Sdk/Readme.md b/src/Microsoft.DotNet.Helix/Sdk/Readme.md index d088cb62b8e..dd202f6a40b 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/Readme.md +++ b/src/Microsoft.DotNet.Helix/Sdk/Readme.md @@ -81,10 +81,16 @@ Useful parameters: - `continueOnError`: allow the pipeline to continue when the monitor job fails. Defaults to `false`. - `useFullyQualifiedTestName`: report fully qualified test names to Azure DevOps (see [Fully qualified test names](#fully-qualified-test-names)). Defaults to `false`. +Implementation and semantic design documents are indexed at +[JobMonitor/Design/README.md](../JobMonitor/Design/README.md). + Behavior notes: - The reporter uses its own `SYSTEM_ACCESSTOKEN`, so it does not depend on the shorter-lived token from the job that originally submitted the Helix work. - If parseable xUnit, JUnit, or TRX result files are available, those are uploaded. +- Result processing uses globally bounded work-item parallelism and streams XML + instead of loading complete result documents. Status polling remains + independent from result upload latency. - If no result files are found for a work item, no test results are uploaded for that work item; Helix work-item failures still affect the monitor job's final pass/fail status. - The reporter is safe to rerun because it checks for already-completed test runs and only processes new results.