From 00b70d058a06cc86211e01b411a4b781b6c24148 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Tue, 11 Aug 2026 16:39:41 -0700
Subject: [PATCH 01/21] Improve Helix result upload batching

Batch Azure DevOps result requests by the service's 1,000 top-level result limit and increase work-item upload parallelism to eight.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 77f2acbe-a455-4c48-9dee-58322d289862
---
 .../AzureDevOpsResultPublisher.cs             | 90 ++++++++++++++-----
 .../JobMonitor/JobMonitorOptions.cs           |  4 +-
 .../AzureDevOpsResultPublisherTests.cs        | 89 ++++++++++++++++++
 3 files changed, 157 insertions(+), 26 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs
index df9ce21216d..c10d8e7f50c 100644
--- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs
+++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs
@@ -15,6 +15,13 @@ namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher;
 public sealed class AzureDevOpsResultPublisher : IDisposable
 {
     private const int DefaultAttemptCount = 10;
+    // Azure DevOps rejects requests containing more than 1,000 top-level TestCaseResult objects.
+    // Nested sub-results do not count toward this limit.
+    private const int MaximumResultsPerRequest = 1000;
+
+    // Preserve the legacy bound on the recursive size of one result hierarchy independently
+    // from the top-level per-request limit.
+    private const int MaximumNodesPerResultHierarchy = 950;
     private static readonly TimeSpan s_maximumRetryDelay = TimeSpan.FromSeconds(30);
     private static readonly TimeSpan s_httpClientTimeout = TimeSpan.FromMinutes(5);
     private static readonly JsonSerializerOptions s_serializerOptions = new(JsonSerializerDefaults.Web)
@@ -36,9 +43,17 @@ private string s_lastSendContent
     public AzureDevOpsResultPublisher(
         AzureDevOpsReportingParameters azdoParameters,
         ILogger logger)
+        : this(azdoParameters, logger, CreateHttpClient(azdoParameters.AccessToken))
+    {
+    }
+
+    internal AzureDevOpsResultPublisher(
+        AzureDevOpsReportingParameters azdoParameters,
+        ILogger logger,
+        HttpClient httpClient)
     {
         _azdoParameters = azdoParameters;
-        _httpClient = CreateHttpClient(azdoParameters.AccessToken);
+        _httpClient = httpClient;
         _logger = logger;
     }
 
@@ -101,9 +116,9 @@ public async Task<long> UploadTestResultsWithCountAsync(IEnumerable<AggregatedRe
             long publishedTestCount = 0;
             IReadOnlyList<AggregatedResult> resultList = results as IReadOnlyList<AggregatedResult> ?? results.ToList();
             var converted = ConvertResults(resultList, resultMetadata).ToList();
-            foreach (List<ConvertedResult> batch in Batch(converted, 1000, static t => Size(t.Converted)))
+            foreach (List<ConvertedResult> requestBatch in CreateResultRequestBatches(converted))
             {
-                IReadOnlyList<PublishedTestCase> publishedTests = await PublishResultsAsync(batch, cancellationToken);
+                IReadOnlyList<PublishedTestCase> publishedTests = await PublishResultsAsync(requestBatch, cancellationToken);
                 publishedTestCount += publishedTests.Count;
             }
 
@@ -304,16 +319,33 @@ ConvertedResult ConvertResult(AggregatedResult result)
         var converted = results.Select(ConvertResult).ToList();
         foreach (ConvertedResult? result in converted)
         {
-            foreach (ConvertedResult chunk in Chunk(result, 950))
+            foreach (ConvertedResult hierarchyPart in SplitOversizedResultHierarchy(
+                result,
+                MaximumNodesPerResultHierarchy))
             {
-                yield return chunk;
+                yield return hierarchyPart;
             }
         }
     }
 
-    private static IEnumerable<ConvertedResult> Chunk(ConvertedResult test, int limit)
+    /// <summary>
+    /// Groups converted top-level results into Azure DevOps request bodies. Each item counts
+    /// once regardless of how many nested sub-results it contains.
+    /// </summary>
+    private static IEnumerable<List<ConvertedResult>> CreateResultRequestBatches(
+        IEnumerable<ConvertedResult> results)
+        => PartitionBySize(results, MaximumResultsPerRequest, static _ => 1);
+
+    /// <summary>
+    /// Splits one logical data-driven or rerun result into multiple top-level payload entries
+    /// when its recursive hierarchy exceeds <paramref name="maximumNodesPerHierarchy"/>.
+    /// This is separate from grouping top-level entries into Azure DevOps request batches.
+    /// </summary>
+    private static IEnumerable<ConvertedResult> SplitOversizedResultHierarchy(
+        ConvertedResult test,
+        int maximumNodesPerHierarchy)
     {
-        if (Size(test.Converted) <= limit)
+        if (CountResultTreeNodes(test.Converted) <= maximumNodesPerHierarchy)
         {
             yield return test;
             yield break;
@@ -322,16 +354,19 @@ private static IEnumerable<ConvertedResult> Chunk(ConvertedResult test, int limi
         IEnumerable<ChunkPair> zippedSubTests = (test.Converted.SubResults ?? [])
             .Zip(test.Aggregated.SubResults, (converted, aggregated) => new ChunkPair(converted, aggregated));
 
-        foreach (List<ChunkPair> zippedBatch in Batch(zippedSubTests, limit, static pair => Size(pair.Converted)))
+        foreach (List<ChunkPair> hierarchyPart in PartitionBySize(
+            zippedSubTests,
+            maximumNodesPerHierarchy,
+            static pair => CountResultTreeNodes(pair.Converted)))
         {
             yield return new ConvertedResult(
-                test.Converted with { SubResults = [.. zippedBatch.Select(static x => x.Converted)], Id = null },
+                test.Converted with { SubResults = [.. hierarchyPart.Select(static x => x.Converted)], Id = null },
                 new AggregatedResult(
                     test.Aggregated.AggregationType,
                     test.Aggregated.Name,
                     test.Aggregated.DurationSeconds,
                     test.Aggregated.Result,
-                    [.. zippedBatch.Select(static x => x.Aggregated)],
+                    [.. hierarchyPart.Select(static x => x.Aggregated)],
                     test.Aggregated.Attachments,
                     test.Aggregated.FailureMessage,
                     test.Aggregated.StackTrace,
@@ -341,43 +376,50 @@ [.. zippedBatch.Select(static x => x.Aggregated)],
         }
     }
 
-    private static int Size(PublishedTestCase test)
+    private static int CountResultTreeNodes(PublishedTestCase test)
     {
-        return 1 + (test.SubResults?.Sum(Size) ?? 0);
+        return 1 + (test.SubResults?.Sum(CountResultTreeNodes) ?? 0);
     }
 
-    private static int Size(PublishedSubResult test)
+    private static int CountResultTreeNodes(PublishedSubResult test)
     {
-        return 1 + (test.SubResults?.Sum(Size) ?? 0);
+        return 1 + (test.SubResults?.Sum(CountResultTreeNodes) ?? 0);
     }
 
-    private static IEnumerable<List<T>> Batch<T>(IEnumerable<T> items, int limit, Func<T, int> getSize)
+    /// <summary>
+    /// Partitions items in order so the sum of <paramref name="getSize"/> values in each
+    /// partition does not exceed <paramref name="maximumPartitionSize"/>.
+    /// </summary>
+    private static IEnumerable<List<T>> PartitionBySize<T>(
+        IEnumerable<T> items,
+        int maximumPartitionSize,
+        Func<T, int> getSize)
     {
-        var currentBatch = new List<T>();
+        var currentPartition = new List<T>();
         int currentSize = 0;
 
         foreach (T? item in items)
         {
             int size = getSize(item);
-            if (size > limit)
+            if (size > maximumPartitionSize)
             {
-                throw new InvalidOperationException("Cannot split a result larger than the batching limit.");
+                throw new InvalidOperationException("Cannot partition an item larger than the size limit.");
             }
 
-            if (currentSize + size > limit && currentBatch.Count > 0)
+            if (currentSize + size > maximumPartitionSize && currentPartition.Count > 0)
             {
-                yield return currentBatch;
-                currentBatch = [];
+                yield return currentPartition;
+                currentPartition = [];
                 currentSize = 0;
             }
 
-            currentBatch.Add(item);
+            currentPartition.Add(item);
             currentSize += size;
         }
 
-        if (currentBatch.Count > 0)
+        if (currentPartition.Count > 0)
         {
-            yield return currentBatch;
+            yield return currentPartition;
         }
     }
 
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs
index 01ab7c203c8..850b2c04d5b 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs
@@ -55,7 +55,7 @@ public sealed class JobMonitorOptions
         /// </summary>
         public string StageAttempt { get; set; }
 
-        public int TestResultUploadParallelism { get; set; } = 4;
+        public int TestResultUploadParallelism { get; set; } = 8;
 
         public TestResultAttachmentMode TestResultAttachmentMode { get; set; } = TestResultAttachmentMode.Failed;
 
@@ -166,7 +166,7 @@ public static JobMonitorOptions Parse(string[] args)
             Option<int> testResultUploadParallelismOption = new("--test-result-upload-parallelism")
             {
                 Description = "Maximum number of work items whose test results can be uploaded to Azure DevOps in parallel.",
-                DefaultValueFactory = _ => 4
+                DefaultValueFactory = _ => 8
             };
 
             Option<TestResultAttachmentMode> testResultAttachmentModeOption = new("--test-result-attachment-mode")
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
index 59d61985642..b040a34cc1b 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
@@ -2,9 +2,14 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
 using System.Net.Http;
 using System.Reflection;
+using System.Text.Json;
 using System.Threading;
+using System.Threading.Tasks;
 using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher;
 using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model;
 using Microsoft.DotNet.Helix.JobMonitor;
@@ -27,6 +32,12 @@ public void AttachmentModeDefaultsToFailed()
             Assert.Equal(TestResultAttachmentMode.Failed, new JobMonitorOptions().TestResultAttachmentMode);
         }
 
+        [Fact]
+        public void JobMonitorUploadParallelismDefaultsToEight()
+        {
+            Assert.Equal(8, new JobMonitorOptions().TestResultUploadParallelism);
+        }
+
         [Fact]
         public void Constructor_ConfiguresHttpClientTimeoutForLongUploads()
         {
@@ -110,5 +121,83 @@ public void CancellationWithoutTimeoutIsNotTransient()
                 CancellationToken.None));
         }
 
+        [Fact]
+        public async Task UploadTestResultsWithCountAsync_BatchesByTopLevelResultCount()
+        {
+            var handler = new RecordingResultHandler();
+            using var publisher = CreatePublisher(handler);
+            AggregatedResult[] results =
+            [
+                CreateDataDrivenResult("First", 600),
+                CreateDataDrivenResult("Second", 600),
+            ];
+
+            long uploadedCount = await publisher.UploadTestResultsWithCountAsync(results, new { });
+
+            Assert.Equal(2, uploadedCount);
+            Assert.Equal(new[] { 2 }, handler.RequestResultCounts);
+        }
+
+        [Fact]
+        public async Task UploadTestResultsWithCountAsync_SplitsMoreThanOneThousandTopLevelResults()
+        {
+            var handler = new RecordingResultHandler();
+            using var publisher = CreatePublisher(handler);
+            AggregatedResult[] results =
+            [
+                .. Enumerable.Range(0, 1001)
+                    .Select(i => new AggregatedResult(AggregationType.Single, $"Test{i}", 1, "Passed"))
+            ];
+
+            long uploadedCount = await publisher.UploadTestResultsWithCountAsync(results, new { });
+
+            Assert.Equal(1001, uploadedCount);
+            Assert.Equal(new[] { 1000, 1 }, handler.RequestResultCounts);
+        }
+
+        private static AzureDevOpsResultPublisher CreatePublisher(HttpMessageHandler handler)
+            => new(
+                new AzureDevOpsReportingParameters(
+                    new Uri("https://dev.azure.com/dnceng-public/"),
+                    "public",
+                    "123"),
+                NullLogger.Instance,
+                new HttpClient(handler));
+
+        private static AggregatedResult CreateDataDrivenResult(string name, int subResultCount)
+            => new(
+                AggregationType.DataDriven,
+                name,
+                subResultCount,
+                "Passed",
+                [
+                    .. Enumerable.Range(0, subResultCount)
+                        .Select(i => new AggregatedResult(AggregationType.Single, $"{name}_{i}", 1, "Passed"))
+                ]);
+
+        private sealed class RecordingResultHandler : HttpMessageHandler
+        {
+            public List<int> RequestResultCounts { get; } = [];
+
+            protected override async Task<HttpResponseMessage> SendAsync(
+                HttpRequestMessage request,
+                CancellationToken cancellationToken)
+            {
+                using JsonDocument requestBody = JsonDocument.Parse(
+                    await request.Content.ReadAsStringAsync(cancellationToken));
+                int resultCount = requestBody.RootElement.GetArrayLength();
+                RequestResultCounts.Add(resultCount);
+
+                string responseBody = JsonSerializer.Serialize(new
+                {
+                    value = Enumerable.Range(1, resultCount).Select(id => new { id })
+                });
+                return new HttpResponseMessage(HttpStatusCode.OK)
+                {
+                    Content = new StringContent(responseBody)
+                };
+            }
+        }
+
     }
 }

From 83526d0bb5d14511fbb047217a48dafa0ed7ffee Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Tue, 11 Aug 2026 16:50:58 -0700
Subject: [PATCH 02/21] Fix Helix result hierarchy split boundary

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 77f2acbe-a455-4c48-9dee-58322d289862
---
 .../AzureDevOpsResultPublisher.cs             |  4 ++-
 .../AzureDevOpsResultPublisherTests.cs        | 28 +++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs
index c10d8e7f50c..f259bff95fa 100644
--- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs
+++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs
@@ -354,9 +354,11 @@ private static IEnumerable<ConvertedResult> SplitOversizedResultHierarchy(
         IEnumerable<ChunkPair> zippedSubTests = (test.Converted.SubResults ?? [])
             .Zip(test.Aggregated.SubResults, (converted, aggregated) => new ChunkPair(converted, aggregated));
 
+        // Each emitted hierarchy includes the copied top-level result, leaving the remaining
+        // node budget for its sub-results.
         foreach (List<ChunkPair> hierarchyPart in PartitionBySize(
             zippedSubTests,
-            maximumNodesPerHierarchy,
+            maximumNodesPerHierarchy - 1,
             static pair => CountResultTreeNodes(pair.Converted)))
         {
             yield return new ConvertedResult(
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
index b040a34cc1b..cc6a0ae3ae9 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
@@ -155,6 +155,20 @@ .. Enumerable.Range(0, 1001)
             Assert.Equal(new[] { 1000, 1 }, handler.RequestResultCounts);
         }
 
+        [Fact]
+        public async Task UploadTestResultsWithCountAsync_SplitHierarchiesIncludeRootInNodeLimit()
+        {
+            var handler = new RecordingResultHandler();
+            using var publisher = CreatePublisher(handler);
+            AggregatedResult[] results = [CreateDataDrivenResult("Theory", 950)];
+
+            long uploadedCount = await publisher.UploadTestResultsWithCountAsync(results, new { });
+
+            Assert.Equal(2, uploadedCount);
+            Assert.Equal(new[] { 2 }, handler.RequestResultCounts);
+            Assert.Equal(new[] { 950, 2 }, handler.RequestHierarchyNodeCounts.Single());
+        }
+
         private static AzureDevOpsResultPublisher CreatePublisher(HttpMessageHandler handler)
             => new(
                 new AzureDevOpsReportingParameters(
@@ -178,6 +192,7 @@ .. Enumerable.Range(0, subResultCount)
         private sealed class RecordingResultHandler : HttpMessageHandler
         {
             public List<int> RequestResultCounts { get; } = [];
+            public List<int[]> RequestHierarchyNodeCounts { get; } = [];
 
             protected override async Task<HttpResponseMessage> SendAsync(
                 HttpRequestMessage request,
@@ -187,6 +202,8 @@ protected override async Task<HttpResponseMessage> SendAsync(
                     await request.Content.ReadAsStringAsync(cancellationToken));
                 int resultCount = requestBody.RootElement.GetArrayLength();
                 RequestResultCounts.Add(resultCount);
+                RequestHierarchyNodeCounts.Add(
+                    [.. requestBody.RootElement.EnumerateArray().Select(CountHierarchyNodes)]);
 
                 string responseBody = JsonSerializer.Serialize(new
                 {
@@ -197,6 +214,17 @@ protected override async Task<HttpResponseMessage> SendAsync(
                     Content = new StringContent(responseBody)
                 };
             }
+
+            private static int CountHierarchyNodes(JsonElement result)
+            {
+                if (!result.TryGetProperty("subResults", out JsonElement subResults) ||
+                    subResults.ValueKind != JsonValueKind.Array)
+                {
+                    return 1;
+                }
+
+                return 1 + subResults.EnumerateArray().Sum(CountHierarchyNodes);
+            }
         }
 
     }

From 585d5adb7afff313b759e09e909609d98192b842 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Tue, 11 Aug 2026 21:14:40 -0700
Subject: [PATCH 03/21] Rewrite Helix Job Monitor result pipeline

Replace the per-job upload task graph with a channel-based work-item pipeline, stream test result parsing, preserve Azure DevOps batching semantics, and document the monitor architecture and durability model.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .gitignore                                    |   2 +
 Arcade.slnx                                   |   1 -
 .../LocalTestResultsReader.cs                 | 219 ----------
 ...tNet.Helix.AzureDevOpsTestPublisher.csproj |  21 -
 .../JobMonitor/Design/Architecture.md         |  68 ++++
 .../JobMonitor/Design/Components/Polling.md   |  21 +
 .../JobMonitor/Design/Components/Shutdown.md  |  26 ++
 .../Design/Components/StateAndStatus.md       |  15 +
 .../Design/Components/TestResults.md          |  38 ++
 .../Design/Components/UploadPipeline.md       |  45 +++
 .../JobMonitor/Design/README.md               |  40 ++
 .../Interfaces/IAzureDevOpsService.cs         |   7 +-
 .../JobMonitor/Interfaces/IHelixService.cs    |   4 +-
 .../JobMonitor/JobMonitorRunner.Design.md     |  46 ++-
 .../JobMonitor/JobMonitorRunner.cs            |  75 ++--
 .../Microsoft.DotNet.Helix.JobMonitor.csproj  |   3 +-
 .../JobMonitor/MonitorState.cs                |  11 +
 .../JobMonitor/Parallelism/ActionQueue.cs     | 144 +++++++
 .../JobMonitor/Parallelism/ParallelAsync.cs   |  37 ++
 .../Services/AzureDevOpsRateLimitGate.cs      |  42 ++
 .../JobMonitor/Services/AzureDevOpsService.cs |  78 +---
 .../JobMonitor/Services/HelixService.cs       | 105 ++---
 .../JobMonitor/StatusReporter.cs              | 105 +----
 .../JobMonitor/TestResultUploadPipeline.cs    | 352 ++++++++++++++++
 .../JobMonitor/TestResultUploadQueue.cs       | 379 ------------------
 .../AzureDevOpsResultPublisher.cs             | 204 +++++++---
 .../TestResults/LocalTestResultsReader.cs     | 320 +++++++++++++++
 .../Model/AzureDevOpsReportingError.cs        |   0
 .../Model/AzureDevOpsReportingParameters.cs   |   0
 .../TestResults}/Model/PackedTestReport.cs    |   0
 .../TestResults}/Model/TerminalError.cs       |   0
 .../TestResults}/Model/TestResult.cs          |   0
 .../Model/TestResultAttachment.cs             |   0
 .../Model/TestResultAttachmentMode.cs         |   0
 .../TestResults}/ResultAggregator.cs          |   0
 .../TestResults}/TestNameFormatter.cs         |   0
 .../TestResults}/TestResultUploadSummary.cs   |   0
 .../AzureDevOpsResultPublisherTests.cs        |  84 +++-
 .../Fakes/FakeAzureDevOpsService.cs           |  75 ++--
 .../Fakes/FakeHelixService.cs                 |  27 +-
 .../HelixServiceTests.cs                      |   8 +-
 .../JobMonitorRunnerTests.cs                  | 139 +++++--
 .../Microsoft.DotNet.Helix.Sdk.Tests.csproj   |   1 -
 src/Microsoft.DotNet.Helix/Sdk/Readme.md      |   6 +
 44 files changed, 1726 insertions(+), 1022 deletions(-)
 delete mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs
 delete mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Shutdown.md
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/StateAndStatus.md
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ActionQueue.cs
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ParallelAsync.cs
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
 delete mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadQueue.cs
 rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestPublisher => JobMonitor/TestResults}/AzureDevOpsResultPublisher.cs (82%)
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs
 rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestPublisher => JobMonitor/TestResults}/Model/AzureDevOpsReportingError.cs (100%)
 rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestPublisher => JobMonitor/TestResults}/Model/AzureDevOpsReportingParameters.cs (100%)
 rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestPublisher => JobMonitor/TestResults}/Model/PackedTestReport.cs (100%)
 rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestPublisher => JobMonitor/TestResults}/Model/TerminalError.cs (100%)
 rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestPublisher => JobMonitor/TestResults}/Model/TestResult.cs (100%)
 rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestPublisher => JobMonitor/TestResults}/Model/TestResultAttachment.cs (100%)
 rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestPublisher => JobMonitor/TestResults}/Model/TestResultAttachmentMode.cs (100%)
 rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestPublisher => JobMonitor/TestResults}/ResultAggregator.cs (100%)
 rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestPublisher => JobMonitor/TestResults}/TestNameFormatter.cs (100%)
 rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestPublisher => JobMonitor/TestResults}/TestResultUploadSummary.cs (100%)

diff --git a/.gitignore b/.gitignore
index ec73ceb3c0b..78750a9ed99 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,6 +38,8 @@ msbuild.wrn
 
 # MSTest test Results
 [Tt]est[Rr]esult*/
+!src/Microsoft.DotNet.Helix/JobMonitor/TestResults/
+!src/Microsoft.DotNet.Helix/JobMonitor/TestResults/**
 [Bb]uild[Ll]og.*
 
 #NUNIT
diff --git a/Arcade.slnx b/Arcade.slnx
index 61ecac02ba9..fd8e5194bdb 100644
--- a/Arcade.slnx
+++ b/Arcade.slnx
@@ -5,7 +5,6 @@
     <Platform Name="x86" />
   </Configurations>
   <Folder Name="/Microsoft.DotNet.Helix/">
-    <Project Path="src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj" />
     <Project Path="src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj" />
     <Project Path="src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj" />
     <Project Path="src/Microsoft.DotNet.Helix/JobSender/Microsoft.DotNet.Helix.JobSender.csproj" />
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs
deleted file mode 100644
index 95c63a90157..00000000000
--- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs
+++ /dev/null
@@ -1,219 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-
-using System.Globalization;
-using System.Xml.Linq;
-using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model;
-using Microsoft.Extensions.Logging;
-
-namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher;
-
-public sealed class LocalTestResultsReader(
-    ILogger logger,
-    TestResultAttachmentMode attachmentMode = TestResultAttachmentMode.Failed)
-{
-    private readonly ILogger _logger = logger;
-    private readonly TestResultAttachmentMode _attachmentMode = attachmentMode;
-
-    public static bool LooksLikeTestResultFile(string path)
-    {
-        string fileName = Path.GetFileName(path);
-        return fileName.EndsWith(".trx", StringComparison.OrdinalIgnoreCase)
-               || fileName.EndsWith("testResults.xml", StringComparison.OrdinalIgnoreCase)
-               || fileName.EndsWith("test-results.xml", StringComparison.OrdinalIgnoreCase)
-               || fileName.EndsWith("test_results.xml", StringComparison.OrdinalIgnoreCase)
-               || fileName.EndsWith("junit-results.xml", StringComparison.OrdinalIgnoreCase)
-               || fileName.EndsWith("junitresults.xml", StringComparison.OrdinalIgnoreCase);
-    }
-
-    public async Task<IReadOnlyList<TestResult>> ReadResultFileAsync(string filePath, CancellationToken cancellationToken = default)
-    {
-        try
-        {
-            using FileStream stream = File.OpenRead(filePath);
-            XDocument document = await XDocument.LoadAsync(stream, LoadOptions.PreserveWhitespace, cancellationToken);
-            string rootName = document.Root?.Name.LocalName ?? string.Empty;
-            string workItemName = new DirectoryInfo(Path.GetDirectoryName(filePath) ?? string.Empty).Name;
-
-            return rootName switch
-            {
-                "assemblies" or "assembly" => ReadXunitResults(document),
-                "TestRun" => ReadTrxResults(document, workItemName),
-                "testsuites" or "testsuite" => ReadJUnitResults(document, workItemName),
-                _ => [],
-            };
-        }
-        catch (Exception ex)
-        {
-            _logger.LogWarning(ex, "Failed to parse test results file '{Path}'.", filePath);
-            return [];
-        }
-    }
-
-    private IReadOnlyList<TestResult> ReadXunitResults(XDocument document)
-    {
-        return [..
-            document.Descendants().Where(static e => e.Name.LocalName == "test").Select(test =>
-            {
-                XElement? failure = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "failure");
-                string? message = failure?.Elements().FirstOrDefault(static x => x.Name.LocalName == "message")?.Value?.Trim();
-                string? stackTrace = failure?.Elements().FirstOrDefault(static x => x.Name.LocalName == "stack-trace")?.Value?.Trim();
-                string? output = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "output")?.Value?.Trim();
-                string? skipReason = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "reason")?.Value?.Trim();
-
-                string typeName = GetAttribute(test, "type") ?? string.Empty;
-                string method = GetAttribute(test, "method") ?? string.Empty;
-                string name = GetAttribute(test, "name")
-                    ?? (!string.IsNullOrEmpty(typeName) && !string.IsNullOrEmpty(method) ? $"{typeName}.{method}" : method);
-                string normalizedOutcome = NormalizeOutcome(GetAttribute(test, "result"));
-
-                List<TestResultAttachment> attachments = [];
-                AddAttachmentIfEnabled(attachments, "output.txt", output, normalizedOutcome);
-
-                return new TestResult(
-                    name,
-                    "xunit",
-                    typeName,
-                    method,
-                    ParseDouble(GetAttribute(test, "time")),
-                    normalizedOutcome,
-                    GetAttribute(failure, "exception-type"),
-                    message,
-                    stackTrace,
-                    skipReason,
-                    attachments);
-            })];
-    }
-
-    private IReadOnlyList<TestResult> ReadJUnitResults(XDocument document, string workItemName)
-    {
-        return [..
-            document.Descendants().Where(static e => e.Name.LocalName == "testcase").Select(test =>
-            {
-                XElement? failure = test.Elements().FirstOrDefault(static x => x.Name.LocalName is "failure" or "error");
-                XElement? skipped = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "skipped");
-                string? stdout = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "system-out")?.Value?.Trim();
-                string? stderr = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "system-err")?.Value?.Trim();
-
-                string className = GetAttribute(test, "classname") ?? workItemName;
-                string method = GetAttribute(test, "name") ?? string.Empty;
-                string name = !string.IsNullOrEmpty(className) ? $"{className}.{method}" : method;
-                string result = skipped is not null ? "Skip" : failure is not null ? "Fail" : "Pass";
-
-                List<TestResultAttachment> attachments = [];
-                AddAttachmentIfEnabled(attachments, "stdout.txt", stdout, result);
-                AddAttachmentIfEnabled(attachments, "stderr.txt", stderr, result);
-
-                return new TestResult(
-                    name,
-                    "junit",
-                    className,
-                    method,
-                    ParseDouble(GetAttribute(test, "time")),
-                    result,
-                    null,
-                    failure?.Value?.Trim(),
-                    null,
-                    skipped?.Value?.Trim(),
-                    attachments);
-            })];
-    }
-
-    private IReadOnlyList<TestResult> ReadTrxResults(XDocument document, string workItemName)
-    {
-        Dictionary<string, XElement> unitTestsById = document
-            .Descendants()
-            .Where(static e => e.Name.LocalName == "UnitTest")
-            .Select(static unitTest => (Id: GetAttribute(unitTest, "id"), Element: unitTest))
-            .Where(static x => !string.IsNullOrEmpty(x.Id))
-            .ToDictionary(static x => x.Id!, static x => x.Element, StringComparer.OrdinalIgnoreCase);
-
-        return [..
-            document.Descendants().Where(static e => e.Name.LocalName == "UnitTestResult").Select(result =>
-            {
-                string testId = GetAttribute(result, "testId") ?? string.Empty;
-                unitTestsById.TryGetValue(testId, out XElement? unitTest);
-                XElement? testMethod = unitTest?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "TestMethod");
-
-                string className = GetAttribute(testMethod, "className") ?? workItemName;
-                string method = GetAttribute(testMethod, "name") ?? GetAttribute(result, "testName") ?? string.Empty;
-                string displayName = GetAttribute(result, "testName")
-                    ?? (!string.IsNullOrEmpty(className) ? $"{className}.{method}" : method);
-
-                XElement? output = result.Descendants().FirstOrDefault(static x => x.Name.LocalName == "Output");
-                string? failureMessage = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "Message")?.Value?.Trim();
-                string? stackTrace = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StackTrace")?.Value?.Trim();
-                string? stdout = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StdOut")?.Value?.Trim();
-                string? stderr = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StdErr")?.Value?.Trim();
-
-                string rawOutcome = GetAttribute(result, "outcome") ?? string.Empty;
-                string normalizedOutcome = NormalizeOutcome(rawOutcome);
-                string? skipReason = string.Equals(normalizedOutcome, "Skip", StringComparison.Ordinal) ? failureMessage : null;
-
-                List<TestResultAttachment> attachments = [];
-                AddAttachmentIfEnabled(attachments, "stdout.txt", stdout, normalizedOutcome);
-                AddAttachmentIfEnabled(attachments, "stderr.txt", stderr, normalizedOutcome);
-
-                return new TestResult(
-                    displayName,
-                    "trx",
-                    className,
-                    method,
-                    ParseDuration(GetAttribute(result, "duration")),
-                    normalizedOutcome,
-                    null,
-                    failureMessage,
-                    stackTrace,
-                    skipReason,
-                    attachments);
-            })];
-    }
-
-    private static string? GetAttribute(XElement? element, string name)
-        => element?.Attribute(name)?.Value;
-
-    private static double ParseDouble(string? value)
-    {
-        return double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double result)
-            ? result
-            : 0;
-    }
-
-    private static double ParseDuration(string? value)
-    {
-        return TimeSpan.TryParse(value, CultureInfo.InvariantCulture, out TimeSpan result)
-            ? result.TotalSeconds
-            : ParseDouble(value);
-    }
-
-    private static string NormalizeOutcome(string? value)
-    {
-        return value?.Trim().ToLowerInvariant() switch
-        {
-            "pass" or "passed" or "success" or "succeeded" => "Pass",
-            "skip" or "skipped" or "notexecuted" or "notrun" => "Skip",
-            "fail" or "failed" or "error" or "timeout" or "aborted" => "Fail",
-            _ => "None",
-        };
-    }
-
-    private void AddAttachmentIfEnabled(
-        List<TestResultAttachment> attachments,
-        string name,
-        string? text,
-        string normalizedOutcome)
-    {
-        bool includeAttachment = _attachmentMode switch
-        {
-            TestResultAttachmentMode.All => true,
-            TestResultAttachmentMode.Failed => string.Equals(normalizedOutcome, "Fail", StringComparison.Ordinal),
-            TestResultAttachmentMode.None => false,
-            _ => false,
-        };
-
-        if (includeAttachment && !string.IsNullOrWhiteSpace(text))
-        {
-            attachments.Add(new TestResultAttachment(name, text));
-        }
-    }
-}
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj
deleted file mode 100644
index 9b6e4c57bdb..00000000000
--- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj
+++ /dev/null
@@ -1,21 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>$(NetMinimum)</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\Common\Microsoft.Arcade.Common\Microsoft.Arcade.Common.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <InternalsVisibleTo Include="Microsoft.DotNet.Helix.Sdk.Tests" />
-  </ItemGroup>
-
-</Project>
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md
new file mode 100644
index 00000000000..c26add45b94
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md
@@ -0,0 +1,68 @@
+# Architecture
+
+## Ownership
+
+`JobMonitorRunner` is the single control-plane owner. It performs the entry
+retry pass, obtains one immutable Azure DevOps/Helix snapshot per poll, updates
+`MonitorState`, queues newly completed jobs, reports status from the snapshot,
+and decides when normal completion is possible.
+
+Result publication is owned by `TestResultUploadPipeline`. The poller hands the
+pipeline a completed job plus the work-item snapshot that established
+completion. The pipeline never calls back into the poll loop and the reporter
+never calls into the pipeline's external services.
+
+## Shared parallelism
+
+`Parallelism/ActionQueue<T>` is the shared long-lived producer/consumer
+primitive. It supports bounded and unbounded `Channel<T>` instances with a
+fixed worker set, cancellation, fault propagation, completion, drain, and
+atomic queue counters. Only lightweight completed-job descriptors use the
+unbounded form; work-item and finalization queues remain bounded.
+
+`Parallelism/ParallelAsync` is used for bounded snapshot reads whose complete
+result is needed by the current poll.
+
+Parallelism budgets are independent:
+
+- job expansion has low parallelism because it only creates session state and
+  feeds work items;
+- work-item processing uses `TestResultUploadParallelism`;
+- test-run finalization has low parallelism because it performs small,
+  non-replayable writes.
+
+This prevents the old multiplication of "jobs × work items × result files"
+tasks and keeps expensive state proportional to configured concurrency plus
+bounded queue capacity.
+
+## Data flow
+
+```text
+Azure DevOps + Helix snapshots
+             |
+             v
+      JobMonitorRunner
+             |
+             v
+ lightweight completed-job queue
+             |
+             v
+ bounded work-item queue
+   download -> parse -> aggregate -> publish
+             |
+             v
+ bounded finalization queue
+ attachment -> complete/tag
+```
+
+Backpressure is intentional. Job workers may wait while feeding a full
+work-item queue, and work-item workers may wait on Azure DevOps throttling.
+Completed-job acceptance remains non-blocking, so neither condition blocks the
+poller or status reporter.
+
+## Durability boundary
+
+The only durable "processed" marker is the Helix-job tag on a completed Azure
+DevOps test run. A session is finalized only if every work item completed
+cleanly. Any download, parse, publication, or finalization failure leaves the
+run untagged so a later invocation replays the complete Helix job.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
new file mode 100644
index 00000000000..4f256b0c3ed
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
@@ -0,0 +1,21 @@
+# Polling and reconciliation
+
+Each poll concurrently refreshes work-item summaries for in-scope jobs that
+have not yet been reconciled, with a fixed degree of parallelism. Terminal
+snapshots are retained for later status reports instead of being downloaded
+again. The resulting dictionary is the poll's immutable work-item snapshot and
+is reused for:
+
+- completion fallback when a Helix job summary has not yet transitioned;
+- outcome reconciliation;
+- upload scheduling;
+- failed-work-item links;
+- aggregate status counts.
+
+No second service call is made for status.
+
+The one-shot entry retry pass and stage-attempt semantics are specified in
+[the semantic document](../../JobMonitorRunner.Design.md). Outcome updates are
+applied oldest-to-newest so resubmissions and higher stage attempts supersede
+older failures without allowing identically named work from different
+submitter/queue streams to collide.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Shutdown.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Shutdown.md
new file mode 100644
index 00000000000..b886aed124e
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Shutdown.md
@@ -0,0 +1,26 @@
+# Shutdown and drain
+
+## Normal completion
+
+The runner stops producing completed jobs, completes the job queue, waits for
+job expansion, completes and drains the work-item queue, then completes and
+drains finalization. This ordering guarantees that no downstream producer is
+still active when its channel is closed.
+
+Work starts as soon as each Helix job completes, so normal drain contains only
+the remaining tail. The drain log records elapsed time and aggregate pipeline
+counts for runtime performance validation.
+
+## Cancellation
+
+Cancellation does not drain uploads. The pipeline worker tokens are canceled
+immediately, the timeout report is emitted, and latest in-flight Helix jobs are
+canceled with an independent bounded token. Incomplete test runs remain
+untagged and are replayed by a later invocation.
+
+## Crash recovery
+
+In-memory queue/session state is never required after restart. Completed tags,
+failed-work-item attachments, Helix job properties, and resubmission lineage
+are sufficient to reconstruct all required work.
+
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/StateAndStatus.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/StateAndStatus.md
new file mode 100644
index 00000000000..6c2e73a5739
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/StateAndStatus.md
@@ -0,0 +1,15 @@
+# State and status
+
+`MonitorState` owns invocation state behind one lock. Collections are exposed
+only through immutable snapshots. Upload workers update only narrow methods:
+upload lifecycle, uploaded test outcomes, and durable completion.
+
+The reporter consumes the current poll snapshot and atomic upload-pipeline
+counters. It never waits for Helix file access or Azure DevOps result uploads.
+
+Normal logging reports semantic transitions and aggregate counts. Verbose
+logging adds queue depth, active worker counts, finalizer depth, and uploaded
+result totals. It deliberately does not print every job, work item, file, or
+request; verbose mode must remain usable on runs containing thousands of work
+items and millions of results.
+
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md
new file mode 100644
index 00000000000..a7cf27e5e75
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md
@@ -0,0 +1,38 @@
+# Test-result processing
+
+Result processing lives under `JobMonitor/TestResults`; the former standalone
+`AzureDevOpsTestPublisher` project was removed because the job monitor was its
+only product consumer.
+
+## Parsing
+
+`LocalTestResultsReader` uses `XmlReader` over a `FileStream`. xUnit and JUnit
+records are materialized one result element at a time. TRX uses a small first
+pass for test definitions followed by a result pass, because TRX result records
+refer to definitions by ID and may precede them in the file.
+
+Malformed files are warned and omitted. Caller cancellation is propagated.
+DTD processing is prohibited.
+
+## Aggregation
+
+Existing single, data-driven, and rerun semantics are retained, including
+flaky-result fields, fully qualified identity, attachment selection, and the
+rule that `Inconclusive` does not fail a work item.
+
+Aggregation state is scoped to one work item. This bounds normal memory by the
+largest concurrently processed work items rather than the whole Helix job or
+build. A single pathological work item with millions of distinct tests still
+sets the memory floor; handling that case would require spill-to-disk grouping.
+
+## Azure DevOps batching
+
+- A request contains at most 1,000 top-level results.
+- Nested sub-results do not consume that request limit.
+- A single oversized hierarchy is defensively split below 950 recursive nodes.
+- Converted results are enumerated lazily into one request batch at a time.
+- The serialized UTF-8 request body is retained only for the lifetime of that
+  request and its retries.
+- Azure DevOps rate-limit guidance is applied through a service-wide gate so
+  concurrent workers slow down together instead of stampeding a throttled
+  endpoint independently.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
new file mode 100644
index 00000000000..1d7a0a48453
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
@@ -0,0 +1,45 @@
+# Upload pipeline
+
+`TestResultUploadPipeline` contains three worker stages. The lightweight job
+stage is unbounded so completed jobs cannot be dropped and upload backpressure
+cannot block polling. The expensive work-item and finalization stages are
+bounded.
+
+## Job expansion
+
+A completed Helix job is accepted once. A `JobUploadSession` captures the
+completion-time work-item snapshot and owns:
+
+- a single-flight Azure DevOps test-run creation task;
+- expected and finished work-item counts;
+- failed-test work-item names;
+- uploaded-result count;
+- a sticky failure flag.
+
+The job stage expands work items into the global work-item queue. A job with no
+work items proceeds directly to finalization.
+
+## Work-item processing
+
+Each worker:
+
+1. downloads recognized result files for one work item, retrying only
+   transient read failures;
+2. obtains the session's single test-run ID;
+3. parses, aggregates, batches, and publishes results;
+4. records the upload summary and test-only failure outcome;
+5. signals session completion.
+
+Work-item concurrency is global. A build with many jobs therefore cannot create
+an unbounded task graph or multiply the configured Azure DevOps pressure.
+
+## Finalization
+
+The last work item queues its session for finalization. If any work item
+failed, the session remains untagged. Otherwise finalization uploads the
+failed-work-item attachment, marks the run completed, applies the Helix-job
+tag, and only then marks the job durably processed.
+
+Create and complete are not replayed after ambiguous failures. Result and
+attachment publication use bounded transient retries because losing an entire
+job's results is worse than the accepted duplicate risk.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md
new file mode 100644
index 00000000000..6850eccd16b
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md
@@ -0,0 +1,40 @@
+# Helix Job Monitor design
+
+The job monitor is a restart-safe coordinator between Azure DevOps and Helix. It
+polls control-plane state, reconciles retries, publishes large test-result sets,
+and computes the final stage outcome without tying progress reporting to result
+upload latency.
+
+The design is split by concern:
+
+- [Semantic behavior](../JobMonitorRunner.Design.md) defines externally
+  observable behavior and restart invariants.
+- [Architecture](Architecture.md) describes process structure, ownership,
+  backpressure, and shared parallelism utilities.
+- [Polling and reconciliation](Components/Polling.md) describes discovery,
+  stage-attempt scoping, retry, and outcome ordering.
+- [Upload pipeline](Components/UploadPipeline.md) describes the bounded
+  job/work-item/finalization stages and durable completion boundary.
+- [Test-result processing](Components/TestResults.md) describes XML parsing,
+  aggregation, batching, attachments, and Azure DevOps limits.
+- [State and status](Components/StateAndStatus.md) describes thread-safe state,
+  snapshots, progress, and bounded verbose logging.
+- [Shutdown](Components/Shutdown.md) describes normal drain, cancellation, and
+  crash recovery.
+
+## Performance goals
+
+The monitor is designed for hundreds of Helix jobs, thousands of work items,
+and millions of test results.
+
+1. Polling and status reporting never wait for result downloads or Azure DevOps
+   result uploads.
+2. Every producer/consumer boundary is bounded.
+3. Parallelism is global per stage, not multiplied independently per Helix job.
+4. Result XML is read forward-only; complete XML documents are never retained.
+5. Azure DevOps requests contain up to 1,000 top-level results, independent of
+   nested sub-result count.
+6. Normal drain should contain only the upload tail that could not overlap
+   polling. Runtime validation is used to tune the default parallelism and
+   verify that the tail remains a small fraction of the monitor duration.
+
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IAzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IAzureDevOpsService.cs
index 78838515fd7..94f11c713c7 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IAzureDevOpsService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IAzureDevOpsService.cs
@@ -59,12 +59,11 @@ Task CompleteTestRunAsync(
             CancellationToken cancellationToken);
 
         /// <summary>
-        /// Uploads test results for the specified work items into an existing test run.
-        /// Returns a dictionary mapping each work item and job name to its upload summary.
+        /// Uploads one work item's test results into an existing test run.
         /// </summary>
-        Task<IReadOnlyDictionary<(string JobName, string WorkItemName), TestResultUploadSummary>> UploadTestResultsAsync(
+        Task<TestResultUploadSummary> UploadTestResultsAsync(
             int testRunId,
-            IReadOnlyList<WorkItemTestResults> results,
+            WorkItemTestResults results,
             CancellationToken cancellationToken);
     }
 }
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs
index 2cc44abacdf..1304bc66ce1 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs
@@ -32,9 +32,9 @@ Task<IReadOnlyList<HelixJobInfo>> GetJobsForBuildAsync(
         /// Work items without recognizable test result files may be omitted from the result.
         /// Individual file download failures should not prevent other result files from being downloaded.
         /// </summary>
-        Task<IReadOnlyList<WorkItemTestResults>> DownloadTestResultsAsync(
+        Task<WorkItemTestResults> DownloadTestResultsAsync(
             string jobName,
-            IReadOnlyCollection<string> workItemNames,
+            string workItemName,
             string workingDirectory, CancellationToken cancellationToken);
 
         /// <summary>
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md
index 248d5374b5a..1a4145d99fd 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md
@@ -283,8 +283,8 @@ behaviorally; method names are illustrative.
   current- or previous-attempt via `System.StageName` / `System.StageAttempt`
   for gating (§2.1).
 - **List work items for a job** — return all work-item summaries.
-- **Download test results** — given a job and a set of work-item names,
-  download recognized result files into a working directory. Individual
+- **Download test results** — given one job/work-item pair, download recognized
+  result files into a working directory. Individual
   per-file failures must not prevent the remaining files from being attempted.
   After the batch, transient failures cause the read-only download phase to be
   retried; permanent failures are logged and omitted.
@@ -364,8 +364,8 @@ Each iteration:
    (so completion/failure transitions are not missed).
 4. Compute the set of completed Helix jobs (§5.5).
 5. **First pass — upload**: for each completed Helix job not already
-   uploaded (per §2.2), upload its test results and remember it as
-   processed. This pass is the only one that triggers uploads.
+   uploaded (per §2.2), queue its completion-time work-item snapshot into the
+   bounded upload pipeline. This pass is the only one that triggers uploads.
 6. **Second pass — outcome reconciliation**: for every completed Helix
    job in scope, ensure its per-work-item outcomes are reflected in the
    running outcome map (§5.7), processing lineage from oldest to newest so
@@ -455,14 +455,19 @@ lines are plain logger output.
 
 ### 5.9 Test-result upload pipeline
 
-Uploads are asynchronous tasks tracked for normal completion. Their in-memory
-lifecycle distinguishes queued, in-progress, durably completed, and failed
-uploads; only a completed, tagged test run is considered durable.
-
-- Each upload is queued asynchronously and tracked. Multiple uploads may
-  proceed concurrently.
-- Test results are downloaded before the AzDO test run is created. Transient
-  download failures are safe to retry and use a bounded retry budget.
+Uploads use a non-dropping lightweight job-expansion channel plus bounded
+work-item and finalization channels. Their in-memory lifecycle distinguishes
+queued, in-progress, durably completed, and failed uploads; only a completed,
+tagged test run is considered durable.
+
+- Completed jobs are queued asynchronously and expanded into a globally
+  bounded work-item pipeline.
+- Test results are downloaded one work item at a time. Transient download
+  failures are safe to retry and use a bounded retry budget.
+- Work-item concurrency is global across all Helix jobs, so total parallelism
+  never multiplies by the number of completed jobs.
+- Test-run creation is single-flight per Helix job even when multiple work-item
+  workers arrive concurrently.
 - Test-run creation and completion/tagging are each attempted once. These
   lifecycle writes determine the durable upload boundary, so replaying an
   ambiguous response could create an extra run or incorrectly mark an
@@ -474,23 +479,24 @@ uploads; only a completed, tagged test run is considered durable.
   request but before the response reaches the client, so retrying may create
   duplicate results or attachments. The design accepts that risk to avoid
   losing an entire job's test results after a transient failure.
-- Permanent failures and exhausted retries are logged as warnings and stop the
-  upload task without affecting pass/fail.
+- Permanent failures and exhausted retries are logged as warnings and make the
+  job session ineligible for completion/tagging without affecting pass/fail.
 - The normal-termination path waits for queued uploads to drain before exiting.
 - The cancellation path does not wait for pending or in-flight uploads. If an
   upload has not completed and applied its Helix-job tag, it remains untagged;
   durable-state discovery causes a later invocation to upload it again.
 
-The upload sequence per job is: create (or reuse) a test run with the plain
-`{TestRunName}`, download results, upload them, complete the test run and tag
-it with the Helix job name (`helixjob<guid>`).
+The upload sequence per job is: download work-item results, lazily create one
+test run with the plain `{TestRunName}`, upload work items with bounded global
+parallelism, upload failure metadata, complete the run, and tag it with the
+Helix job name (`helixjob<guid>`).
 
 ### 5.10 Status logging
 
 When a status log is due, the runner emits a one-line summary of work
-counts (processed / completed / running / waiting jobs and work items). In
-verbose mode it additionally emits a tree-style breakdown per job and work
-item. The verbose tree is informational only.
+counts (processed / completed / running / waiting jobs and work items).
+Verbose mode adds bounded pipeline diagnostics (queued/active work and uploaded
+result totals) but never emits a per-work-item tree.
 
 A Helix job is classified for status purposes as `Processed` (already
 uploaded), `Completed` (terminal but not yet uploaded), `Running` (has at
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
index 15f52107ee5..1017ced7346 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
@@ -10,6 +10,7 @@
 using Microsoft.DotNet.Helix.Client;
 using Microsoft.DotNet.Helix.Client.Models;
 using Microsoft.DotNet.Helix.JobMonitor.Models;
+using Microsoft.DotNet.Helix.JobMonitor.Parallelism;
 using Microsoft.Extensions.Logging;
 
 namespace Microsoft.DotNet.Helix.JobMonitor
@@ -33,7 +34,7 @@ internal sealed class JobMonitorRunner : IJobMonitorRunner, IDisposable
 
         private readonly MonitorState _state = new();
         private readonly StatusReporter _reporter;
-        private readonly TestResultUploadQueue _uploads;
+        private readonly TestResultUploadPipeline _uploads;
 
         /// <summary>
         /// Constructor for production use with real services.
@@ -74,8 +75,8 @@ internal JobMonitorRunner(
                 _options.RepositoryName,
                 _options.SourceBranch);
 
-            _reporter = new StatusReporter(_logger, _options, _helix, _state);
-            _uploads = new TestResultUploadQueue(_logger, _options, _azdo, _helix, _state);
+            _reporter = new StatusReporter(_logger, _options, _state);
+            _uploads = new TestResultUploadPipeline(_logger, _options, _azdo, _helix, _state);
         }
 
         public async Task<int> RunAsync(CancellationToken cancellationToken)
@@ -91,6 +92,7 @@ public async Task<int> RunAsync(CancellationToken cancellationToken)
             }
             catch (OperationCanceledException)
             {
+                _uploads.Cancel();
                 // On cancellation (AzDO job timeout or build cancellation) the agent grants only a
                 // few seconds before force-killing the process, so cancelling the in-flight Helix
                 // jobs is the priority: do it immediately rather than waiting for the test-result
@@ -290,7 +292,25 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
 
             // Helix job summaries can omit Finished for failed jobs even after all work
             // items have terminal exit codes, so fall back to per-work-item status.
-            IReadOnlyCollection<HelixJobInfo> completedJobs = await GetCompletedJobsAsync(stageJobs, cancellationToken);
+            IReadOnlyList<HelixJobInfo> jobsToRefresh =
+            [
+                ..stageJobs.Where(job =>
+                    !_state.IsWorkItemOutcomesRecorded(job.JobName)
+                    || !loopState.WorkItemsByJob.ContainsKey(job.JobName))
+            ];
+            IReadOnlyDictionary<string, IReadOnlyCollection<WorkItemSummary>> refreshedWorkItems =
+                await GetWorkItemsAsync(jobsToRefresh, cancellationToken);
+            foreach ((string jobName, IReadOnlyCollection<WorkItemSummary> workItems) in refreshedWorkItems)
+            {
+                loopState.WorkItemsByJob[jobName] = workItems;
+            }
+
+            IReadOnlyDictionary<string, IReadOnlyCollection<WorkItemSummary>> workItemsByJob =
+                stageJobs.ToDictionary(
+                    static job => job.JobName,
+                    job => loopState.WorkItemsByJob[job.JobName],
+                    StringComparer.OrdinalIgnoreCase);
+            IReadOnlyCollection<HelixJobInfo> completedJobs = GetCompletedJobs(stageJobs, workItemsByJob);
             var completedJobNames = new HashSet<string>(
                 completedJobs.Select(j => j.JobName),
                 StringComparer.OrdinalIgnoreCase);
@@ -298,7 +318,7 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
             // First pass: upload + reconcile for any newly-completed jobs.
             foreach (HelixJobInfo job in completedJobs.Where(j => !_state.IsHelixJobProcessed(j.JobName)))
             {
-                await ReconcileCompletedJobAsync(job, queueUpload: true, cancellationToken);
+                ReconcileCompletedJob(job, workItemsByJob[job.JobName], queueUpload: true);
             }
 
             // Second pass: ensure outcomes for every completed job (any attempt) are reflected in
@@ -309,11 +329,9 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
                 MonitorState.GetLatestHelixJobAttempts(stageJobs)
                     .Where(j => completedJobNames.Contains(j.JobName))))
             {
-                await ReconcileCompletedJobAsync(job, queueUpload: false, cancellationToken);
+                ReconcileCompletedJob(job, workItemsByJob[job.JobName], queueUpload: false);
             }
 
-            _uploads.Prune();
-
             bool shouldLogStatus = _options.Verbose
                 || loopState.LastObservedJobCount != stageJobs.Count
                 || loopState.LastObservedCompletedCount != completedJobs.Count
@@ -321,7 +339,7 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
 
             if (shouldLogStatus)
             {
-                await _reporter.LogPollStatusAsync(stageJobs, completedJobNames, cancellationToken);
+                _reporter.LogPollStatus(stageJobs, workItemsByJob, completedJobNames, _uploads.Snapshot);
                 loopState.LastObservedJobCount = stageJobs.Count;
                 loopState.LastObservedCompletedCount = completedJobNames.Count;
                 loopState.LastStatusLogAt = DateTime.UtcNow;
@@ -363,10 +381,10 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
         /// queues a test-result upload. Idempotent: a second call without
         /// <paramref name="queueUpload"/> early-returns if the outcomes were already recorded.
         /// </summary>
-        private async Task ReconcileCompletedJobAsync(
+        private void ReconcileCompletedJob(
             HelixJobInfo helixJob,
-            bool queueUpload,
-            CancellationToken cancellationToken)
+            IReadOnlyCollection<WorkItemSummary> workItems,
+            bool queueUpload)
         {
             // Already reconciled earlier in this invocation — nothing more to do (idempotent).
             if (_state.IsWorkItemOutcomesRecorded(helixJob.JobName))
@@ -374,9 +392,6 @@ private async Task ReconcileCompletedJobAsync(
                 return;
             }
 
-            IReadOnlyCollection<WorkItemSummary> workItems =
-                await _helix.ListWorkItemsAsync(helixJob.JobName, cancellationToken);
-
             // A previous monitor attempt for the same build already uploaded this job's results
             // (tracked via IsHelixJobProcessed, seeded on entry from the AzDO test-run tags). Its
             // work-item outcomes must still be reconciled so the final exit code accounts for
@@ -395,10 +410,7 @@ private async Task ReconcileCompletedJobAsync(
 
             if (queueUpload && !alreadyUploadedByPriorAttempt)
             {
-                if (_state.TryQueueHelixJobUpload(helixJob.JobName))
-                {
-                    _uploads.Enqueue(helixJob, workItems, cancellationToken);
-                }
+                _uploads.TryEnqueue(helixJob, workItems);
             }
 
             if (!alreadyUploadedByPriorAttempt)
@@ -407,14 +419,14 @@ private async Task ReconcileCompletedJobAsync(
             }
         }
 
-        private async Task<IReadOnlyCollection<HelixJobInfo>> GetCompletedJobsAsync(
+        private IReadOnlyCollection<HelixJobInfo> GetCompletedJobs(
             IReadOnlyList<HelixJobInfo> jobs,
-            CancellationToken cancellationToken)
+            IReadOnlyDictionary<string, IReadOnlyCollection<WorkItemSummary>> workItemsByJob)
         {
             var completed = new List<HelixJobInfo>();
             foreach (HelixJobInfo job in jobs)
             {
-                if (job.IsCompleted || await AreAllWorkItemsTerminalAsync(job, cancellationToken))
+                if (job.IsCompleted || AreAllWorkItemsTerminal(job, workItemsByJob[job.JobName]))
                 {
                     completed.Add(job);
                 }
@@ -423,18 +435,30 @@ private async Task<IReadOnlyCollection<HelixJobInfo>> GetCompletedJobsAsync(
             return MonitorState.OrderHelixJobsOldToNew(completed);
         }
 
-        private async Task<bool> AreAllWorkItemsTerminalAsync(HelixJobInfo job, CancellationToken cancellationToken)
+        private static bool AreAllWorkItemsTerminal(
+            HelixJobInfo job,
+            IReadOnlyCollection<WorkItemSummary> workItems)
         {
             if (job.InitialWorkItemCount is not > 0)
             {
                 return false;
             }
 
-            IReadOnlyCollection<WorkItemSummary> workItems = await _helix.ListWorkItemsAsync(job.JobName, cancellationToken);
             return workItems.Count >= job.InitialWorkItemCount.Value
                 && workItems.All(wi => wi.ExitCode.HasValue);
         }
 
+        private Task<IReadOnlyDictionary<string, IReadOnlyCollection<WorkItemSummary>>> GetWorkItemsAsync(
+            IReadOnlyList<HelixJobInfo> jobs,
+            CancellationToken cancellationToken)
+            => ParallelAsync.ToDictionaryAsync(
+                jobs,
+                parallelism: Math.Max(8, _options.TestResultUploadParallelism),
+                static job => job.JobName,
+                async (job, token) => await _helix.ListWorkItemsAsync(job.JobName, token),
+                StringComparer.OrdinalIgnoreCase,
+                cancellationToken);
+
         private async Task CancelInFlightHelixJobsAsync(CancellationToken cancellationToken)
         {
             List<HelixJobInfo> inFlightJobs =
@@ -494,6 +518,7 @@ private bool IsPreviousAttempt(HelixJobInfo job)
 
         public void Dispose()
         {
+            _uploads.Cancel();
             (_azdo as IDisposable)?.Dispose();
             (_helix as IDisposable)?.Dispose();
         }
@@ -516,6 +541,8 @@ private sealed class PollLoopState
             public int LastObservedJobCount { get; set; } = -1;
             public int LastObservedCompletedCount { get; set; } = -1;
             public DateTime LastStatusLogAt { get; set; } = DateTime.UtcNow;
+            public Dictionary<string, IReadOnlyCollection<WorkItemSummary>> WorkItemsByJob { get; } =
+                new(StringComparer.OrdinalIgnoreCase);
         }
     }
 }
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj
index a58bc2876d7..222d59d525b 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj
@@ -2,6 +2,8 @@
 
   <PropertyGroup>
     <TargetFramework>$(NetMinimum)</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>annotations</Nullable>
     <OutputType>Exe</OutputType>
     <IsPackable>true</IsPackable>
     <PackAsTool>true</PackAsTool>
@@ -24,7 +26,6 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\Common\Microsoft.Arcade.Common\Microsoft.Arcade.Common.csproj" />
-    <ProjectReference Include="..\AzureDevOpsTestPublisher\Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj" />
     <ProjectReference Include="..\Client\CSharp\Microsoft.DotNet.Helix.Client.csproj" />
   </ItemGroup>
 
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
index 4ea36f76d9e..85533f2b380 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
@@ -320,9 +320,20 @@ public void ObserveTestResults(
                             "see Azure DevOps test run results");
                     }
                 }
+
             }
         }
 
+        public void ObserveTestResult(
+            string jobName,
+            string workItemName,
+            TestResultUploadSummary summary)
+            => ObserveTestResults(
+                new Dictionary<(string JobName, string WorkItemName), TestResultUploadSummary>
+                {
+                    [(jobName, workItemName)] = summary,
+                });
+
         /// <summary>
         /// Returns true if this is the first time a console-link warning is being emitted for
         /// the given (jobName, workItemName) key. Used to deduplicate console-link logging.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ActionQueue.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ActionQueue.cs
new file mode 100644
index 00000000000..3f14fa6ea3e
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ActionQueue.cs
@@ -0,0 +1,144 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Threading.Channels;
+
+namespace Microsoft.DotNet.Helix.JobMonitor.Parallelism;
+
+internal sealed class ActionQueue<T> : IAsyncDisposable
+{
+    private readonly Channel<T> _channel;
+    private readonly Func<T, CancellationToken, ValueTask> _action;
+    private readonly CancellationTokenSource _shutdown = new();
+    private readonly Task[] _workers;
+    private long _accepted;
+    private long _started;
+    private long _completed;
+    private int _active;
+
+    public ActionQueue(
+        int capacity,
+        int parallelism,
+        Func<T, CancellationToken, ValueTask> action)
+        : this(
+            Channel.CreateBounded<T>(new BoundedChannelOptions(capacity)
+            {
+                AllowSynchronousContinuations = false,
+                FullMode = BoundedChannelFullMode.Wait,
+                SingleReader = parallelism == 1,
+                SingleWriter = false,
+            }),
+            parallelism,
+            action)
+    {
+        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity);
+    }
+
+    public ActionQueue(
+        int parallelism,
+        Func<T, CancellationToken, ValueTask> action)
+        : this(
+            Channel.CreateUnbounded<T>(new UnboundedChannelOptions
+            {
+                AllowSynchronousContinuations = false,
+                SingleReader = parallelism == 1,
+                SingleWriter = false,
+            }),
+            parallelism,
+            action)
+    {
+    }
+
+    private ActionQueue(
+        Channel<T> channel,
+        int parallelism,
+        Func<T, CancellationToken, ValueTask> action)
+    {
+        ArgumentOutOfRangeException.ThrowIfNegativeOrZero(parallelism);
+        ArgumentNullException.ThrowIfNull(action);
+
+        _action = action;
+        _channel = channel;
+        _workers = Enumerable.Range(0, parallelism)
+            .Select(_ => WorkerAsync())
+            .ToArray();
+    }
+
+    public QueueSnapshot Snapshot => new(
+        Accepted: Interlocked.Read(ref _accepted),
+        Started: Interlocked.Read(ref _started),
+        Completed: Interlocked.Read(ref _completed),
+        Active: Volatile.Read(ref _active));
+
+    public bool TryEnqueue(T item)
+    {
+        if (!_channel.Writer.TryWrite(item))
+        {
+            return false;
+        }
+
+        Interlocked.Increment(ref _accepted);
+        return true;
+    }
+
+    public async ValueTask EnqueueAsync(T item, CancellationToken cancellationToken)
+    {
+        await _channel.Writer.WriteAsync(item, cancellationToken);
+        Interlocked.Increment(ref _accepted);
+    }
+
+    public void Complete() => _channel.Writer.TryComplete();
+
+    public void Cancel() => _shutdown.Cancel();
+
+    public Task DrainAsync() => Task.WhenAll(_workers);
+
+    public async ValueTask DisposeAsync()
+    {
+        _channel.Writer.TryComplete();
+        _shutdown.Cancel();
+
+        try
+        {
+            await Task.WhenAll(_workers);
+        }
+        catch (Exception) when (_shutdown.IsCancellationRequested)
+        {
+        }
+
+        _shutdown.Dispose();
+    }
+
+    private async Task WorkerAsync()
+    {
+        await foreach (T item in _channel.Reader.ReadAllAsync(_shutdown.Token))
+        {
+            Interlocked.Increment(ref _started);
+            Interlocked.Increment(ref _active);
+            try
+            {
+                await _action(item, _shutdown.Token);
+            }
+            catch (Exception ex) when (!_shutdown.IsCancellationRequested)
+            {
+                _channel.Writer.TryComplete(ex);
+                _shutdown.Cancel();
+                throw;
+            }
+            finally
+            {
+                Interlocked.Decrement(ref _active);
+                Interlocked.Increment(ref _completed);
+            }
+        }
+    }
+}
+
+internal readonly record struct QueueSnapshot(
+    long Accepted,
+    long Started,
+    long Completed,
+    int Active)
+{
+    public long Queued => Accepted - Started;
+}
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ParallelAsync.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ParallelAsync.cs
new file mode 100644
index 00000000000..f9b5f074fa7
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ParallelAsync.cs
@@ -0,0 +1,37 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.DotNet.Helix.JobMonitor.Parallelism;
+
+internal static class ParallelAsync
+{
+    public static async Task<IReadOnlyDictionary<TKey, TValue>> ToDictionaryAsync<TSource, TKey, TValue>(
+        IEnumerable<TSource> source,
+        int parallelism,
+        Func<TSource, TKey> getKey,
+        Func<TSource, CancellationToken, ValueTask<TValue>> getValue,
+        IEqualityComparer<TKey> comparer,
+        CancellationToken cancellationToken)
+        where TKey : notnull
+    {
+        TSource[] items = [.. source];
+        var values = new TValue[items.Length];
+
+        await Parallel.ForEachAsync(
+            Enumerable.Range(0, items.Length),
+            new ParallelOptions
+            {
+                CancellationToken = cancellationToken,
+                MaxDegreeOfParallelism = parallelism,
+            },
+            async (index, token) => values[index] = await getValue(items[index], token));
+
+        var result = new Dictionary<TKey, TValue>(items.Length, comparer);
+        for (int i = 0; i < items.Length; i++)
+        {
+            result.Add(getKey(items[i]), values[i]);
+        }
+
+        return result;
+    }
+}
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
new file mode 100644
index 00000000000..3676ff79805
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.DotNet.Helix.JobMonitor;
+
+internal sealed class AzureDevOpsRateLimitGate
+{
+    private long _notBeforeUtcTicks;
+
+    public void Defer(TimeSpan delay)
+    {
+        if (delay <= TimeSpan.Zero)
+        {
+            return;
+        }
+
+        long candidate = DateTimeOffset.UtcNow.Add(delay).UtcTicks;
+        long observed;
+        while (candidate > (observed = Interlocked.Read(ref _notBeforeUtcTicks)))
+        {
+            if (Interlocked.CompareExchange(ref _notBeforeUtcTicks, candidate, observed) == observed)
+            {
+                break;
+            }
+        }
+    }
+
+    public async Task WaitAsync(CancellationToken cancellationToken)
+    {
+        while (true)
+        {
+            long notBeforeTicks = Interlocked.Read(ref _notBeforeUtcTicks);
+            TimeSpan delay = new DateTimeOffset(notBeforeTicks, TimeSpan.Zero) - DateTimeOffset.UtcNow;
+            if (delay <= TimeSpan.Zero)
+            {
+                return;
+            }
+
+            await Task.Delay(delay, cancellationToken);
+        }
+    }
+}
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
index fb0febf0ff8..582d4c41e4f 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
@@ -52,14 +52,12 @@ internal sealed class AzureDevOpsService : IAzureDevOpsService, IDisposable
         private readonly JobMonitorOptions _options;
         private readonly ILogger _logger;
         private readonly HttpClient _azdoClient;
-        private readonly SemaphoreSlim _uploadSemaphore;
-
+        private readonly AzureDevOpsRateLimitGate _rateLimitGate = new();
         public AzureDevOpsService(JobMonitorOptions options, ILogger logger)
         {
             _options = options;
             _logger = logger;
             _azdoClient = new HttpClient();
-            _uploadSemaphore = new SemaphoreSlim(options.TestResultUploadParallelism);
             InitializeClient();
         }
 
@@ -68,7 +66,6 @@ internal AzureDevOpsService(JobMonitorOptions options, ILogger logger, HttpClien
             _options = options;
             _logger = logger;
             _azdoClient = azdoClient ?? throw new ArgumentNullException(nameof(azdoClient));
-            _uploadSemaphore = new SemaphoreSlim(options.TestResultUploadParallelism);
             InitializeClient();
         }
 
@@ -77,6 +74,7 @@ private void InitializeClient()
             string encodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + _options.SystemAccessToken));
             _azdoClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", encodedToken);
             _azdoClient.DefaultRequestHeaders.UserAgent.ParseAdd("dotnet-helix-job-monitor");
+            _azdoClient.Timeout = TimeSpan.FromMinutes(5);
         }
 
         public async Task<IReadOnlyList<AzureDevOpsTimelineRecord>> GetTimelineRecordsAsync(CancellationToken cancellationToken)
@@ -398,9 +396,9 @@ await SendAsync(
                 cancellationToken: cancellationToken);
         }
 
-        public async Task<IReadOnlyDictionary<(string JobName, string WorkItemName), TestResultUploadSummary>> UploadTestResultsAsync(
+        public async Task<TestResultUploadSummary> UploadTestResultsAsync(
             int testRunId,
-            IReadOnlyList<WorkItemTestResults> results,
+            WorkItemTestResults results,
             CancellationToken cancellationToken)
         {
             var reportingParameters = new AzureDevOpsReportingParameters(
@@ -410,60 +408,25 @@ await SendAsync(
                 _options.SystemAccessToken,
                 _options.UseFullyQualifiedTestName,
                 _options.TestResultAttachmentMode);
-            using var publisher = new AzureDevOpsResultPublisher(
+            var publisher = new AzureDevOpsResultPublisher(
                 reportingParameters,
-                _logger);
+                _logger,
+                _azdoClient,
+                _rateLimitGate);
 
-            async Task<TestResultUploadSummary> UploadWorkItemAsync(WorkItemTestResults workItem)
+            if (results.TestResultFiles.Count == 0)
             {
-                if (workItem.TestResultFiles.Count == 0)
-                {
-                    _logger.LogInformation("No test results to upload for work item {WorkItemId} in job {JobName}", workItem.WorkItemName, workItem.JobName);
-                    return new TestResultUploadSummary(true, 0);
-                }
-
-                DateTimeOffset waitStartedAt = DateTimeOffset.UtcNow;
-                _logger.LogDebug(
-                    "Work item '{WorkItemName}' in job '{JobName}' is waiting for a test-result upload slot. "
-                    + "{AvailableSlots} slot(s) are currently available.",
-                    workItem.WorkItemName,
-                    workItem.JobName,
-                    _uploadSemaphore.CurrentCount);
-                await _uploadSemaphore.WaitAsync(cancellationToken);
-
-                try
-                {
-                    DateTimeOffset uploadStartedAt = DateTimeOffset.UtcNow;
-                    _logger.LogDebug(
-                        "Work item '{WorkItemName}' in job '{JobName}' acquired a test-result upload slot after {WaitElapsed}. "
-                        + "Parsing and publishing {FileCount} file(s).",
-                        workItem.WorkItemName,
-                        workItem.JobName,
-                        uploadStartedAt - waitStartedAt,
-                        workItem.TestResultFiles.Count);
-                    TestResultUploadSummary summary = await publisher.UploadTestResultsWithSummaryAsync(
-                        workItem.TestResultFiles,
-                        new
-                        {
-                            HelixJobId = workItem.JobName,
-                            HelixWorkItemName = workItem.WorkItemName
-                        },
-                        cancellationToken);
-                    _logger.LogDebug(
-                        "Work item '{WorkItemName}' in job '{JobName}' finished parsing and publishing after {UploadElapsed}.",
-                        workItem.WorkItemName,
-                        workItem.JobName,
-                        DateTimeOffset.UtcNow - uploadStartedAt);
-                    return summary;
-                }
-                finally
-                {
-                    _uploadSemaphore.Release();
-                }
+                return new TestResultUploadSummary(true, 0);
             }
 
-            (WorkItemTestResults WorkItem, TestResultUploadSummary Summary)[] testSummaries = await Task.WhenAll(results.Select(async result => (result, await UploadWorkItemAsync(result))));
-            return testSummaries.ToDictionary(t => (t.WorkItem.JobName, t.WorkItem.WorkItemName), t => t.Summary);
+            return await publisher.UploadTestResultsWithSummaryAsync(
+                results.TestResultFiles,
+                new
+                {
+                    HelixJobId = results.JobName,
+                    HelixWorkItemName = results.WorkItemName
+                },
+                cancellationToken);
         }
 
         private async Task<JObject> SendAsync(
@@ -489,6 +452,7 @@ private async Task<string> SendForStringAsync(
         {
             async Task<string> SendOnceAsync()
             {
+                await _rateLimitGate.WaitAsync(cancellationToken);
                 using var request = new HttpRequestMessage(method, requestUri);
                 if (body != null)
                 {
@@ -602,18 +566,18 @@ private async Task HonorRateLimitAsync(HttpResponseMessage response, string requ
 
             if (delayToApply > TimeSpan.Zero)
             {
+                _rateLimitGate.Defer(delayToApply);
                 _logger.LogDebug(
                     "Azure DevOps rate limit back-off. Delaying next request by {DelaySeconds:0.###}s (request: {RequestUri}).",
                     delayToApply.TotalSeconds,
                     requestUri);
-                await Task.Delay(delayToApply, cancellationToken);
+                await _rateLimitGate.WaitAsync(cancellationToken);
             }
         }
 
         public void Dispose()
         {
             _azdoClient.Dispose();
-            _uploadSemaphore.Dispose();
         }
     }
 }
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
index 1ef050bfaa4..5896ab70a92 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
@@ -67,84 +67,65 @@ public async Task<IReadOnlyList<HelixJobInfo>> GetJobsForBuildAsync(
              ];
         }
 
-        public async Task<IReadOnlyList<WorkItemTestResults>> DownloadTestResultsAsync(
+        public async Task<WorkItemTestResults> DownloadTestResultsAsync(
             string jobName,
-            IReadOnlyCollection<string> workItemNames,
+            string workItemName,
             string workingDirectory,
             CancellationToken cancellationToken)
         {
-            List<WorkItemTestResults> downloadedFiles = [];
             List<Exception> transientFailures = [];
             string outputDirectory = _fileSystem.PathCombine(workingDirectory, SanitizeDirName(jobName));
             _fileSystem.CreateDirectory(outputDirectory);
 
             JobResultsUri resultsUri = await RetryAsync(() => _helixApi.Job.ResultsAsync(jobName), cancellationToken);
+            IImmutableList<UploadedFile> availableFiles = await RetryAsync(
+                () => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false),
+                cancellationToken);
 
-            foreach (string workItemName in workItemNames)
+            availableFiles = [.. availableFiles.Where(f => LooksLikeTestResultFile(f.Name))];
+            if (availableFiles.Count == 0)
             {
-                IImmutableList<UploadedFile> availableFiles = await RetryAsync(
-                    () => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false),
-                    cancellationToken);
+                return new WorkItemTestResults(jobName, workItemName, []);
+            }
+
+            string workItemDirectory = _fileSystem.PathCombine(outputDirectory, SanitizeDirName(workItemName));
+            _fileSystem.CreateDirectory(workItemDirectory);
 
-                availableFiles = [.. availableFiles.Where(f => LooksLikeTestResultFile(f.Name))];
-                if (availableFiles.Count == 0)
+            List<string> workItemFiles = [];
+            foreach (UploadedFile file in availableFiles)
+            {
+                string relativePath = NormalizeUploadedFilePath(file.Name);
+                string destinationFile = _fileSystem.PathCombine(workItemDirectory, relativePath);
+                string directory = _fileSystem.GetDirectoryName(destinationFile);
+                if (!string.IsNullOrEmpty(directory))
                 {
-                    continue;
+                    _fileSystem.CreateDirectory(directory);
                 }
 
-                string workItemDirectory = _fileSystem.PathCombine(outputDirectory, SanitizeDirName(workItemName));
-                _fileSystem.CreateDirectory(workItemDirectory);
-
-                List<string> workItemFiles = [];
-                foreach (UploadedFile file in availableFiles)
+                try
                 {
-                    string relativePath = NormalizeUploadedFilePath(file.Name);
-                    string destinationFile = _fileSystem.PathCombine(workItemDirectory, relativePath);
-                    string directory = _fileSystem.GetDirectoryName(destinationFile);
-                    if (!string.IsNullOrEmpty(directory))
-                    {
-                        _fileSystem.CreateDirectory(directory);
-                    }
-
-                    try
-                    {
-                        DateTimeOffset downloadStartedAt = DateTimeOffset.UtcNow;
-                        _logger.LogDebug(
-                            "Downloading test result file '{FileName}' for '{JobName}/{WorkItemName}'.",
-                            file.Name,
-                            jobName,
-                            workItemName);
-                        IBlobClient blobClient = _blobClientFactory.CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS);
-                        await blobClient.DownloadToAsync(destinationFile, cancellationToken);
-                        _logger.LogDebug(
-                            "Downloaded test result file '{FileName}' for '{JobName}/{WorkItemName}' in {Elapsed}.",
-                            file.Name,
-                            jobName,
-                            workItemName,
-                            DateTimeOffset.UtcNow - downloadStartedAt);
-                        workItemFiles.Add(destinationFile);
-                    }
-                    catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
-                    {
-                        throw;
-                    }
-                    catch (Exception ex) when (TransientFailureDetector.IsTransient(ex))
-                    {
-                        transientFailures.Add(ex);
-                        _logger.LogWarning(ex,
-                            "Transient failure downloading '{FileName}' for '{JobName}/{WorkItemName}'. "
-                            + "The remaining files will still be attempted before the download is retried.",
-                            file.Name,
-                            jobName,
-                            workItemName);
-                    }
-                    catch (Exception ex)
-                    {
-                        _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItemName);
-                    }
+                    IBlobClient blobClient = _blobClientFactory.CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS);
+                    await blobClient.DownloadToAsync(destinationFile, cancellationToken);
+                    workItemFiles.Add(destinationFile);
+                }
+                catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+                {
+                    throw;
+                }
+                catch (Exception ex) when (TransientFailureDetector.IsTransient(ex))
+                {
+                    transientFailures.Add(ex);
+                    _logger.LogWarning(ex,
+                        "Transient failure downloading '{FileName}' for '{JobName}/{WorkItemName}'. "
+                        + "The remaining files will still be attempted before the work item is retried.",
+                        file.Name,
+                        jobName,
+                        workItemName);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItemName);
                 }
-
-                downloadedFiles.Add(new WorkItemTestResults(jobName, workItemName, workItemFiles));
             }
 
             if (transientFailures.Count > 0)
@@ -154,7 +135,7 @@ public async Task<IReadOnlyList<WorkItemTestResults>> DownloadTestResultsAsync(
                     new AggregateException(transientFailures));
             }
 
-            return downloadedFiles;
+            return new WorkItemTestResults(jobName, workItemName, workItemFiles);
         }
 
         private static bool LooksLikeTestResultFile(string path)
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/StatusReporter.cs b/src/Microsoft.DotNet.Helix/JobMonitor/StatusReporter.cs
index e736240a71e..595333c6510 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/StatusReporter.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/StatusReporter.cs
@@ -33,14 +33,12 @@ internal sealed class StatusReporter
 
         private readonly ILogger _logger;
         private readonly JobMonitorOptions _options;
-        private readonly IHelixService _helix;
         private readonly MonitorState _state;
 
-        public StatusReporter(ILogger logger, JobMonitorOptions options, IHelixService helix, MonitorState state)
+        public StatusReporter(ILogger logger, JobMonitorOptions options, MonitorState state)
         {
             _logger = logger;
             _options = options;
-            _helix = helix;
             _state = state;
         }
 
@@ -142,22 +140,21 @@ public void LogFailedWorkItemConsoleLinks(HelixJobInfo helixJob, IEnumerable<Wor
         /// and work items). Also emits per-failure console-link warnings for any failed
         /// work items observed in this poll that haven't been reported yet.
         /// </summary>
-        public async Task LogPollStatusAsync(
+        public void LogPollStatus(
             IReadOnlyList<HelixJobInfo> jobs,
+            IReadOnlyDictionary<string, IReadOnlyCollection<WorkItemSummary>> workItemsByJob,
             IReadOnlySet<string> completedJobNames,
-            CancellationToken cancellationToken)
+            UploadPipelineSnapshot uploads)
         {
             List<HelixJobInfo> orderedJobs =
             [
                 ..jobs.OrderBy(j => j.JobName, StringComparer.OrdinalIgnoreCase)
             ];
 
-            var workItemsByJob = new Dictionary<string, IReadOnlyCollection<WorkItemSummary>>(StringComparer.OrdinalIgnoreCase);
             foreach (HelixJobInfo job in orderedJobs)
             {
-                IReadOnlyCollection<WorkItemSummary> workItems = await _helix.ListWorkItemsAsync(job.JobName, cancellationToken);
+                IReadOnlyCollection<WorkItemSummary> workItems = workItemsByJob[job.JobName];
                 LogFailedWorkItemConsoleLinks(job, workItems.Where(wi => wi.IsFailedAndTerminal));
-                workItemsByJob[job.JobName] = workItems;
             }
 
             JobWorkItemStatusCounts counts = ComputeCounts(orderedJobs, workItemsByJob, completedJobNames);
@@ -177,7 +174,17 @@ public async Task LogPollStatusAsync(
 
             if (_options.Verbose)
             {
-                LogVerboseTree(orderedJobs, workItemsByJob, completedJobNames);
+                _logger.LogDebug(
+                    "Upload pipeline: jobs {JobQueued} queued/{JobActive} active, work items "
+                    + "{WorkItemQueued} queued/{WorkItemActive} active, finalizers "
+                    + "{FinalizerQueued} queued/{FinalizerActive} active, {UploadedResults} results uploaded.",
+                    uploads.Jobs.Queued,
+                    uploads.Jobs.Active,
+                    uploads.WorkItems.Queued,
+                    uploads.WorkItems.Active,
+                    uploads.Finalizers.Queued,
+                    uploads.Finalizers.Active,
+                    uploads.UploadedResults);
             }
         }
 
@@ -295,86 +302,6 @@ private void LogWarning(string message)
         private void LogError(string message)
             => _logger.LogError("{Prefix}{Message}", AzdoErrorPrefix, message);
 
-        private void LogVerboseTree(
-            IReadOnlyList<HelixJobInfo> jobs,
-            IReadOnlyDictionary<string, IReadOnlyCollection<WorkItemSummary>> workItemsByJob,
-            IReadOnlySet<string> completedJobNames)
-        {
-            if (jobs.Count == 0)
-            {
-                _logger.LogInformation("⏳ Helix job details:{nl}└─ no Helix jobs discovered yet", Environment.NewLine);
-                return;
-            }
-
-            var lines = new List<string>();
-            for (int jobIndex = 0; jobIndex < jobs.Count; jobIndex++)
-            {
-                HelixJobInfo job = jobs[jobIndex];
-                IReadOnlyCollection<WorkItemSummary> workItems = workItemsByJob[job.JobName];
-                AddVerboseJobLines(
-                    lines,
-                    job,
-                    workItems,
-                    GetJobStatus(job, workItems, completedJobNames),
-                    isLastJob: jobIndex == jobs.Count - 1);
-            }
-
-            _logger.LogInformation("⏳ Helix job details:{nl}{JobDetails}",
-                Environment.NewLine,
-                string.Join(Environment.NewLine, lines));
-        }
-
-        private static void AddVerboseJobLines(
-            List<string> lines,
-            HelixJobInfo job,
-            IReadOnlyCollection<WorkItemSummary> workItems,
-            string jobStatus,
-            bool isLastJob)
-        {
-            string jobConnector = isLastJob ? "└─" : "├─";
-            string childPrefix = isLastJob ? "   " : "│  ";
-            lines.Add($"{jobConnector} 🧪 Helix job {job.DisplayName} [{jobStatus}]");
-
-            List<WorkItemSummary> orderedWorkItems =
-            [
-                ..workItems.OrderBy(wi => wi.Name, StringComparer.OrdinalIgnoreCase)
-            ];
-
-            if (orderedWorkItems.Count == 0)
-            {
-                lines.Add($"{childPrefix}└─ no work items reported yet");
-                return;
-            }
-
-            for (int i = 0; i < orderedWorkItems.Count; i++)
-            {
-                WorkItemSummary workItem = orderedWorkItems[i];
-                string connector = i == orderedWorkItems.Count - 1 ? "└─" : "├─";
-                string console = workItem.IsFailedAndTerminal
-                    ? $" | Console: {MonitorState.GetConsoleOutputText(workItem.ConsoleOutputUri)}"
-                    : string.Empty;
-                lines.Add($"{childPrefix}{connector} {workItem.Name} ({workItem.FormattedState}){console}");
-            }
-        }
-
-        private string GetJobStatus(
-            HelixJobInfo job,
-            IReadOnlyCollection<WorkItemSummary> workItems,
-            IReadOnlySet<string> completedJobNames)
-        {
-            if (_state.IsHelixJobProcessed(job.JobName))
-            {
-                return "Processed";
-            }
-
-            if (completedJobNames.Contains(job.JobName))
-            {
-                return "Completed";
-            }
-
-            return workItems.Count > 0 ? "Running" : "Waiting";
-        }
-
         private JobWorkItemStatusCounts ComputeCounts(
             IReadOnlyList<HelixJobInfo> jobs,
             IReadOnlyDictionary<string, IReadOnlyCollection<WorkItemSummary>> workItemsByJob,
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
new file mode 100644
index 00000000000..8fc8615a32f
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
@@ -0,0 +1,352 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Concurrent;
+using Microsoft.Arcade.Common;
+using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher;
+using Microsoft.DotNet.Helix.Client.Models;
+using Microsoft.DotNet.Helix.JobMonitor.Models;
+using Microsoft.DotNet.Helix.JobMonitor.Parallelism;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.DotNet.Helix.JobMonitor;
+
+internal sealed class TestResultUploadPipeline : IAsyncDisposable
+{
+    private const int MaximumTransientDownloadRetries = 2;
+    private const string AzdoWarningPrefix = "##vso[task.logissue type=warning]";
+
+    private readonly ILogger _logger;
+    private readonly JobMonitorOptions _options;
+    private readonly IAzureDevOpsService _azdo;
+    private readonly IHelixService _helix;
+    private readonly MonitorState _state;
+    private readonly ConcurrentDictionary<string, JobUploadSession> _sessions =
+        new(StringComparer.OrdinalIgnoreCase);
+    private readonly ActionQueue<JobUploadRequest> _jobs;
+    private readonly ActionQueue<WorkItemUploadRequest> _workItems;
+    private readonly ActionQueue<JobUploadSession> _finalizers;
+    private int _draining;
+
+    public TestResultUploadPipeline(
+        ILogger logger,
+        JobMonitorOptions options,
+        IAzureDevOpsService azdo,
+        IHelixService helix,
+        MonitorState state)
+    {
+        _logger = logger;
+        _options = options;
+        _azdo = azdo;
+        _helix = helix;
+        _state = state;
+
+        int uploadParallelism = options.TestResultUploadParallelism;
+        _jobs = new ActionQueue<JobUploadRequest>(
+            parallelism: Math.Min(4, uploadParallelism),
+            ExpandJobAsync);
+        _workItems = new ActionQueue<WorkItemUploadRequest>(
+            capacity: Math.Max(64, uploadParallelism * 8),
+            parallelism: uploadParallelism,
+            ProcessWorkItemAsync);
+        _finalizers = new ActionQueue<JobUploadSession>(
+            capacity: Math.Max(64, uploadParallelism * 2),
+            parallelism: Math.Min(4, uploadParallelism),
+            FinalizeJobAsync);
+    }
+
+    public UploadPipelineSnapshot Snapshot => new(
+        _jobs.Snapshot,
+        _workItems.Snapshot,
+        _finalizers.Snapshot,
+        _sessions.Values.Count(static session => session.HasFailed),
+        _sessions.Values.Sum(static session => session.UploadedResultCount));
+
+    public bool TryEnqueue(HelixJobInfo job, IReadOnlyCollection<WorkItemSummary> workItems)
+    {
+        if (Volatile.Read(ref _draining) != 0 || _state.IsHelixJobProcessed(job.JobName))
+        {
+            return false;
+        }
+
+        var session = new JobUploadSession(job, workItems);
+        if (!_sessions.TryAdd(job.JobName, session))
+        {
+            return false;
+        }
+
+        if (!_jobs.TryEnqueue(new JobUploadRequest(session)))
+        {
+            _sessions.TryRemove(job.JobName, out _);
+            return false;
+        }
+
+        _state.TryQueueHelixJobUpload(job.JobName);
+        return true;
+    }
+
+    public async Task DrainAsync(CancellationToken cancellationToken)
+    {
+        if (Interlocked.Exchange(ref _draining, 1) != 0)
+        {
+            return;
+        }
+
+        DateTimeOffset startedAt = DateTimeOffset.UtcNow;
+        _jobs.Complete();
+        await _jobs.DrainAsync().WaitAsync(cancellationToken);
+
+        _workItems.Complete();
+        await _workItems.DrainAsync().WaitAsync(cancellationToken);
+
+        _finalizers.Complete();
+        await _finalizers.DrainAsync().WaitAsync(cancellationToken);
+
+        UploadPipelineSnapshot snapshot = Snapshot;
+        _logger.LogInformation(
+            "Test result pipeline drained in {Elapsed}. {JobCount} job(s), {WorkItemCount} work item(s), "
+            + "and {ResultCount} result(s) were processed; {FailedJobCount} job upload(s) remain untagged.",
+            DateTimeOffset.UtcNow - startedAt,
+            snapshot.Jobs.Completed,
+            snapshot.WorkItems.Completed,
+            snapshot.UploadedResults,
+            snapshot.FailedJobs);
+    }
+
+    public void Cancel()
+    {
+        _jobs.Cancel();
+        _workItems.Cancel();
+        _finalizers.Cancel();
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        Cancel();
+        await _jobs.DisposeAsync();
+        await _workItems.DisposeAsync();
+        await _finalizers.DisposeAsync();
+    }
+
+    private async ValueTask ExpandJobAsync(JobUploadRequest request, CancellationToken cancellationToken)
+    {
+        JobUploadSession session = request.Session;
+        _state.MarkHelixJobUploadInProgress(session.Job.JobName);
+
+        if (session.WorkItems.Count == 0)
+        {
+            await _finalizers.EnqueueAsync(session, cancellationToken);
+            return;
+        }
+
+        foreach (WorkItemSummary workItem in session.WorkItems)
+        {
+            await _workItems.EnqueueAsync(new WorkItemUploadRequest(session, workItem.Name), cancellationToken);
+        }
+    }
+
+    private async ValueTask ProcessWorkItemAsync(
+        WorkItemUploadRequest request,
+        CancellationToken cancellationToken)
+    {
+        JobUploadSession session = request.Session;
+        try
+        {
+            WorkItemTestResults downloaded = await ExecuteDownloadWithRetryAsync(
+                session.Job,
+                request.WorkItemName,
+                cancellationToken);
+            int testRunId = await session.GetOrCreateTestRunAsync(
+                () => _azdo.CreateTestRunAsync(session.Job.TestRunName, cancellationToken));
+            TestResultUploadSummary summary =
+                await _azdo.UploadTestResultsAsync(testRunId, downloaded, cancellationToken);
+
+            session.RecordSuccess(request.WorkItemName, summary);
+            if (_options.FailWorkItemsWithFailedTests)
+            {
+                _state.ObserveTestResult(session.Job.JobName, request.WorkItemName, summary);
+            }
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            throw;
+        }
+        catch (Exception ex)
+        {
+            session.RecordFailure();
+            LogUploadFailure(
+                ex,
+                $"process test results for work item '{request.WorkItemName}' in job '{session.Job.DisplayName}'");
+        }
+        finally
+        {
+            if (session.MarkWorkItemFinished())
+            {
+                await _finalizers.EnqueueAsync(session, cancellationToken);
+            }
+        }
+    }
+
+    private async ValueTask FinalizeJobAsync(
+        JobUploadSession session,
+        CancellationToken cancellationToken)
+    {
+        if (session.HasFailed)
+        {
+            _state.MarkHelixJobUploadFailed(session.Job.JobName);
+            return;
+        }
+
+        try
+        {
+            int testRunId = await session.GetOrCreateTestRunAsync(
+                () => _azdo.CreateTestRunAsync(session.Job.TestRunName, cancellationToken));
+            await _azdo.CompleteTestRunAsync(
+                testRunId,
+                session.Job.JobName,
+                session.FailedWorkItems,
+                cancellationToken);
+
+            _state.TryMarkHelixJobProcessed(session.Job.JobName);
+            _logger.LogInformation(
+                "{UploadedCount} test results for job '{JobName}' processed.",
+                session.UploadedResultCount,
+                session.Job.DisplayName);
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            throw;
+        }
+        catch (Exception ex)
+        {
+            session.RecordFailure();
+            _state.MarkHelixJobUploadFailed(session.Job.JobName);
+            LogUploadFailure(ex, $"complete Azure DevOps test run for job '{session.Job.DisplayName}'");
+        }
+    }
+
+    private async Task<WorkItemTestResults> ExecuteDownloadWithRetryAsync(
+        HelixJobInfo job,
+        string workItemName,
+        CancellationToken cancellationToken)
+    {
+        WorkItemTestResults result = null;
+        Exception lastException = null;
+        var retry = new ExponentialRetry
+        {
+            MaxAttempts = MaximumTransientDownloadRetries + 1,
+            RetryDelayCallback = (attempt, delay) =>
+                _logger.LogDebug(
+                    "Transient result download failure for '{JobName}/{WorkItemName}' on attempt {Attempt}. "
+                    + "Retrying after {Delay}.",
+                    job.DisplayName,
+                    workItemName,
+                    attempt,
+                    delay),
+        };
+
+        bool succeeded = await retry.RunAsync(
+            async _ =>
+            {
+                try
+                {
+                    result = await _helix.DownloadTestResultsAsync(
+                        job.JobName,
+                        workItemName,
+                        _options.WorkingDirectory,
+                        cancellationToken);
+                    return RetryResult.Success;
+                }
+                catch (Exception ex) when (
+                    !cancellationToken.IsCancellationRequested
+                    && TransientFailureDetector.IsTransient(ex))
+                {
+                    lastException = ex;
+                    return RetryResult.Retry();
+                }
+            },
+            cancellationToken);
+
+        return succeeded
+            ? result
+            : throw lastException ?? new InvalidOperationException("Result download retry exited unexpectedly.");
+    }
+
+    private void LogUploadFailure(Exception exception, string operation)
+        => _logger.LogWarning(
+            exception,
+            "{Prefix}Failed to {Operation}. The Helix job remains untagged so a later monitor invocation can replay it.",
+            AzdoWarningPrefix,
+            operation);
+
+    private sealed record JobUploadRequest(JobUploadSession Session);
+
+    private sealed record WorkItemUploadRequest(JobUploadSession Session, string WorkItemName);
+
+    private sealed class JobUploadSession
+    {
+        private readonly object _sync = new();
+        private readonly HashSet<string> _failedWorkItems = new(StringComparer.OrdinalIgnoreCase);
+        private Task<int> _testRunTask;
+        private int _finishedWorkItems;
+        private int _failed;
+        private long _uploadedResultCount;
+
+        public JobUploadSession(HelixJobInfo job, IReadOnlyCollection<WorkItemSummary> workItems)
+        {
+            Job = job;
+            WorkItems = [.. workItems];
+        }
+
+        public HelixJobInfo Job { get; }
+
+        public IReadOnlyList<WorkItemSummary> WorkItems { get; }
+
+        public bool HasFailed => Volatile.Read(ref _failed) != 0;
+
+        public long UploadedResultCount => Interlocked.Read(ref _uploadedResultCount);
+
+        public IReadOnlyCollection<string> FailedWorkItems
+        {
+            get
+            {
+                lock (_sync)
+                {
+                    return [.. _failedWorkItems];
+                }
+            }
+        }
+
+        public Task<int> GetOrCreateTestRunAsync(Func<Task<int>> create)
+        {
+            lock (_sync)
+            {
+                return _testRunTask ??= create();
+            }
+        }
+
+        public void RecordSuccess(string workItemName, TestResultUploadSummary summary)
+        {
+            Interlocked.Add(ref _uploadedResultCount, summary.UploadedCount);
+            if (!summary.AllPassed)
+            {
+                lock (_sync)
+                {
+                    _failedWorkItems.Add(workItemName);
+                }
+            }
+        }
+
+        public void RecordFailure() => Interlocked.Exchange(ref _failed, 1);
+
+        public bool MarkWorkItemFinished()
+            => Interlocked.Increment(ref _finishedWorkItems) == WorkItems.Count;
+    }
+}
+
+internal readonly record struct UploadPipelineSnapshot(
+    QueueSnapshot Jobs,
+    QueueSnapshot WorkItems,
+    QueueSnapshot Finalizers,
+    int FailedJobs,
+    long UploadedResults);
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadQueue.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadQueue.cs
deleted file mode 100644
index adfce8fb745..00000000000
--- a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadQueue.cs
+++ /dev/null
@@ -1,379 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.Arcade.Common;
-using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher;
-using Microsoft.DotNet.Helix.Client.Models;
-using Microsoft.DotNet.Helix.JobMonitor.Models;
-using Microsoft.Extensions.Logging;
-
-namespace Microsoft.DotNet.Helix.JobMonitor
-{
-    /// <summary>
-    /// Fire-and-forget queue for AzDO test-result uploads. Each queued upload runs as an
-    /// independent task. Transient reads and test-result/attachment publishing use bounded
-    /// retries; test-run creation and completion are attempted once. On normal completion the
-    /// queue is drained so results in flight when the runner exits are not lost. On cancellation
-    /// the queue is intentionally NOT drained: cancelling the in-flight Helix jobs takes priority,
-    /// and any unfinished upload is re-uploaded in full by a later monitor invocation (a Helix job
-    /// is only "processed" once its test run reaches the Completed state).
-    /// </summary>
-    internal sealed class TestResultUploadQueue
-    {
-        private const int MaximumTransientRetries = 2;
-        private const string AzdoWarningPrefix = "##vso[task.logissue type=warning]";
-
-        private readonly ILogger _logger;
-        private readonly JobMonitorOptions _options;
-        private readonly IAzureDevOpsService _azdo;
-        private readonly IHelixService _helix;
-        private readonly MonitorState _monitorState;
-        private readonly List<PendingUpload> _pending = [];
-
-        public TestResultUploadQueue(
-            ILogger logger,
-            JobMonitorOptions options,
-            IAzureDevOpsService azdo,
-            IHelixService helix,
-            MonitorState monitorState)
-        {
-            _logger = logger;
-            _options = options;
-            _azdo = azdo;
-            _helix = helix;
-            _monitorState = monitorState;
-        }
-
-        public void Enqueue(HelixJobInfo helixJob, IReadOnlyCollection<WorkItemSummary> workItems, CancellationToken cancellationToken)
-        {
-            IReadOnlyList<string> workItemNames = [.. workItems.Select(w => w.Name)];
-            var pendingUpload = new PendingUpload(helixJob);
-            // Scheduling uses CancellationToken.None so the upload task is always allowed to start.
-            // The upload body still observes the runner's token, so when the runner is cancelled the
-            // upload stops promptly and the job's results are re-uploaded by a later invocation.
-            pendingUpload.Task = Task.Run(
-                () => UploadAsync(pendingUpload, workItemNames, cancellationToken),
-                CancellationToken.None);
-            _pending.Add(pendingUpload);
-        }
-
-        public void Prune()
-        {
-            _pending.RemoveAll(static upload => upload.Task.IsCompleted);
-        }
-
-        public async Task DrainAsync(CancellationToken cancellationToken)
-        {
-            Prune();
-            if (_pending.Count == 0)
-            {
-                return;
-            }
-
-            _logger.LogInformation("Waiting for {Count} pending test result upload(s) to complete.", _pending.Count);
-            try
-            {
-                Task allUploads = Task.WhenAll(_pending.Select(upload => upload.Task));
-                if (_options.Verbose)
-                {
-                    LogPendingUploads();
-                    TimeSpan heartbeatInterval = TimeSpan.FromSeconds(Math.Max(1, _options.PollingIntervalSeconds));
-                    while (!allUploads.IsCompleted)
-                    {
-                        Task heartbeat = Task.Delay(heartbeatInterval, cancellationToken);
-                        if (await Task.WhenAny(allUploads, heartbeat) == allUploads)
-                        {
-                            break;
-                        }
-
-                        cancellationToken.ThrowIfCancellationRequested();
-                        Prune();
-                        LogPendingUploads();
-                    }
-                }
-
-                await allUploads.WaitAsync(cancellationToken);
-            }
-            catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
-            {
-                // One or more upload tasks were cancelled; treat as best-effort drained.
-            }
-            Prune();
-        }
-
-        private async Task UploadAsync(
-            PendingUpload pendingUpload,
-            IReadOnlyCollection<string> workItemNames,
-            CancellationToken cancellationToken)
-        {
-            HelixJobInfo helixJob = pendingUpload.HelixJob;
-            _monitorState.MarkHelixJobUploadInProgress(helixJob.JobName);
-
-            SetPhase(pendingUpload, $"downloading Helix test results for {workItemNames.Count} work item(s)");
-            (bool downloadedSuccessfully, IReadOnlyList<WorkItemTestResults> downloaded) = await TryExecuteWithRetryAsync(
-                () => _helix.DownloadTestResultsAsync(
-                        helixJob.JobName,
-                        workItemNames,
-                        _options.WorkingDirectory,
-                        cancellationToken),
-                "download the Helix test results",
-                helixJob,
-                testRunId: 0,
-                retryCount: MaximumTransientRetries,
-                cancellationToken);
-            if (!downloadedSuccessfully)
-            {
-                SetPhase(pendingUpload, "failed while downloading Helix test results");
-                _monitorState.MarkHelixJobUploadFailed(helixJob.JobName);
-                return;
-            }
-
-            SetPhase(pendingUpload, "creating the Azure DevOps test run");
-            (bool created, int testRunId) = await TryExecuteAsync(
-                () => _azdo.CreateTestRunAsync(helixJob.TestRunName, cancellationToken),
-                "create the Azure DevOps test run",
-                helixJob,
-                testRunId: 0,
-                cancellationToken);
-            if (!created)
-            {
-                SetPhase(pendingUpload, "failed while creating the Azure DevOps test run");
-                _monitorState.MarkHelixJobUploadFailed(helixJob.JobName);
-                return;
-            }
-
-            SetPhase(pendingUpload, $"publishing {downloaded.Count} work item(s) to Azure DevOps test run {testRunId}");
-            (bool uploadedSuccessfully, IReadOnlyDictionary<(string JobName, string WorkItemName), TestResultUploadSummary> testResults)
-                = await TryExecuteAsync(
-                    () => _azdo.UploadTestResultsAsync(testRunId, downloaded, cancellationToken),
-                    "upload the test results to Azure DevOps",
-                    helixJob,
-                    testRunId,
-                    cancellationToken);
-            if (!uploadedSuccessfully)
-            {
-                SetPhase(pendingUpload, $"failed while publishing to Azure DevOps test run {testRunId}");
-                _monitorState.MarkHelixJobUploadFailed(helixJob.JobName);
-                return;
-            }
-
-            if (_options.FailWorkItemsWithFailedTests)
-            {
-                _monitorState.ObserveTestResults(testResults);
-            }
-
-            IReadOnlyCollection<string> failedWorkItems =
-            [
-                .. testResults
-                    .Where(kv => !kv.Value.AllPassed)
-                    .Select(kv => kv.Key.WorkItemName)
-            ];
-
-            SetPhase(pendingUpload, $"completing and tagging Azure DevOps test run {testRunId}");
-            (bool completed, _) = await TryExecuteAsync(
-                async () =>
-                {
-                    await _azdo.CompleteTestRunAsync(testRunId, helixJob.JobName, failedWorkItems, cancellationToken);
-                    return true;
-                },
-                "complete and tag the Azure DevOps test run",
-                helixJob,
-                testRunId,
-                cancellationToken);
-            if (!completed)
-            {
-                SetPhase(pendingUpload, $"failed while completing Azure DevOps test run {testRunId}");
-                _monitorState.MarkHelixJobUploadFailed(helixJob.JobName);
-                return;
-            }
-
-            SetPhase(pendingUpload, $"completed Azure DevOps test run {testRunId}");
-            _monitorState.TryMarkHelixJobProcessed(helixJob.JobName);
-
-            long uploadedCount = testResults.Values.Sum(r => r.UploadedCount);
-            _logger.LogInformation("{UploadedCount} test results for job '{JobName}' processed.",
-                uploadedCount,
-                helixJob.DisplayName);
-        }
-
-        private Task<(bool Success, T Result)> TryExecuteAsync<T>(
-            Func<Task<T>> operation,
-            string operationDescription,
-            HelixJobInfo helixJob,
-            int testRunId,
-            CancellationToken cancellationToken)
-            => TryExecuteWithRetryAsync(
-                operation,
-                operationDescription,
-                helixJob,
-                testRunId,
-                retryCount: 0,
-                cancellationToken);
-
-        private async Task<(bool Success, T Result)> TryExecuteWithRetryAsync<T>(
-            Func<Task<T>> operation,
-            string operationDescription,
-            HelixJobInfo helixJob,
-            int testRunId,
-            int retryCount,
-            CancellationToken cancellationToken)
-        {
-            try
-            {
-                T result = default;
-                Exception lastException = null;
-                var retry = new ExponentialRetry
-                {
-                    MaxAttempts = retryCount + 1,
-                    RetryDelayCallback = (failedAttempt, delay) =>
-                        _logger.LogDebug(
-                            "Failed to {OperationDescription} for job {JobName}. Test run ID was {TestRunId}. "
-                            + "Waiting {RetryDelay} before attempt {NextAttempt} of {AttemptCount}.",
-                            operationDescription,
-                            helixJob.DisplayName,
-                            testRunId,
-                            delay,
-                            failedAttempt + 1,
-                            retryCount + 1),
-                };
-
-                bool succeeded = await retry.RunAsync(
-                    async attempt =>
-                    {
-                        try
-                        {
-                            result = await operation();
-                            return true;
-                        }
-                        catch (Exception ex) when (
-                            !cancellationToken.IsCancellationRequested
-                            && TransientFailureDetector.IsTransient(ex))
-                        {
-                            lastException = ex;
-                            _logger.LogDebug(ex,
-                                "Failed to {OperationDescription} for job {JobName}. Test run ID was {TestRunId}. "
-                                + "Transient attempt {Attempt} of {AttemptCount} failed.",
-                                operationDescription,
-                                helixJob.DisplayName,
-                                testRunId,
-                                attempt + 1,
-                                retryCount + 1);
-                            return false;
-                        }
-                    },
-                    cancellationToken);
-
-                if (!succeeded)
-                {
-                    cancellationToken.ThrowIfCancellationRequested();
-                    throw lastException ?? new InvalidOperationException("Upload retry loop exited unexpectedly.");
-                }
-
-                return (true, result);
-            }
-            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
-            {
-                throw;
-            }
-            catch (Exception ex)
-            {
-                string failureKind = TransientFailureDetector.IsTransient(ex)
-                    ? retryCount > 0
-                        ? "Transient retry limit reached."
-                        : "The operation may have partially completed and is not safe to replay in this invocation."
-                    : "The failure is not retryable.";
-                _logger.LogWarning(ex,
-                    "{Prefix}Failed to {OperationDescription} for job {JobName}. Test run ID was {TestRunId}. "
-                    + "{FailureKind} The run remains untagged and a later monitor invocation may retry the upload.",
-                    AzdoWarningPrefix,
-                    operationDescription,
-                    helixJob.DisplayName,
-                    testRunId,
-                    failureKind);
-                return (false, default);
-            }
-        }
-
-        private void SetPhase(PendingUpload upload, string phase)
-        {
-            upload.SetPhase(phase);
-            if (_options.Verbose)
-            {
-                _logger.LogDebug(
-                    "Test result upload for job '{JobName}' entered phase '{Phase}' after {Elapsed}.",
-                    upload.HelixJob.DisplayName,
-                    phase,
-                    DateTimeOffset.UtcNow - upload.StartedAt);
-            }
-        }
-
-        private void LogPendingUploads()
-        {
-            if (_pending.Count == 0)
-            {
-                return;
-            }
-
-            DateTimeOffset now = DateTimeOffset.UtcNow;
-            string details = string.Join(
-                Environment.NewLine,
-                _pending
-                    .OrderBy(upload => upload.StartedAt)
-                    .Select(upload =>
-                    {
-                        (string phase, DateTimeOffset phaseStartedAt) = upload.GetPhase();
-                        return $"- {upload.HelixJob.DisplayName}: phase='{phase}', "
-                            + $"phase elapsed={now - phaseStartedAt:c}, total elapsed={now - upload.StartedAt:c}";
-                    }));
-
-            _logger.LogDebug(
-                "{Count} test result upload(s) remain pending:{nl}{Details}",
-                _pending.Count,
-                Environment.NewLine,
-                details);
-        }
-
-        private sealed class PendingUpload
-        {
-            private readonly object _sync = new();
-            private string _phase = "queued";
-            private DateTimeOffset _phaseStartedAt;
-
-            public PendingUpload(HelixJobInfo helixJob)
-            {
-                HelixJob = helixJob;
-                StartedAt = DateTimeOffset.UtcNow;
-                _phaseStartedAt = StartedAt;
-            }
-
-            public HelixJobInfo HelixJob { get; }
-
-            public DateTimeOffset StartedAt { get; }
-
-            public Task Task { get; set; }
-
-            public void SetPhase(string phase)
-            {
-                lock (_sync)
-                {
-                    _phase = phase;
-                    _phaseStartedAt = DateTimeOffset.UtcNow;
-                }
-            }
-
-            public (string Phase, DateTimeOffset PhaseStartedAt) GetPhase()
-            {
-                lock (_sync)
-                {
-                    return (_phase, _phaseStartedAt);
-                }
-            }
-        }
-
-    }
-}
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs
similarity index 82%
rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs
rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs
index f259bff95fa..e0fdfabceda 100644
--- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs
@@ -3,16 +3,18 @@
 
 using System.Net;
 using System.Net.Http.Headers;
+using System.Globalization;
 using System.Text;
 using System.Text.Json;
 using System.Net.Sockets;
 using Microsoft.Arcade.Common;
 using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model;
+using Microsoft.DotNet.Helix.JobMonitor;
 using Microsoft.Extensions.Logging;
 
 namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher;
 
-public sealed class AzureDevOpsResultPublisher : IDisposable
+internal sealed class AzureDevOpsResultPublisher : IDisposable
 {
     private const int DefaultAttemptCount = 10;
     // Azure DevOps rejects requests containing more than 1,000 top-level TestCaseResult objects.
@@ -29,59 +31,71 @@ public sealed class AzureDevOpsResultPublisher : IDisposable
         WriteIndented = false,
     };
 
-    private readonly AsyncLocal<string> _lastSendContent = new();
-    private string s_lastSendContent
-    {
-        get => _lastSendContent.Value ?? string.Empty;
-        set => _lastSendContent.Value = value;
-    }
-
     private readonly AzureDevOpsReportingParameters _azdoParameters;
     private readonly HttpClient _httpClient;
     private readonly ILogger _logger;
+    private readonly bool _ownsHttpClient;
+    private readonly AzureDevOpsRateLimitGate _rateLimitGate;
 
     public AzureDevOpsResultPublisher(
         AzureDevOpsReportingParameters azdoParameters,
         ILogger logger)
-        : this(azdoParameters, logger, CreateHttpClient(azdoParameters.AccessToken))
+        : this(
+            azdoParameters,
+            logger,
+            CreateHttpClient(azdoParameters.AccessToken),
+            new AzureDevOpsRateLimitGate(),
+            ownsHttpClient: true)
     {
     }
 
     internal AzureDevOpsResultPublisher(
         AzureDevOpsReportingParameters azdoParameters,
         ILogger logger,
-        HttpClient httpClient)
+        HttpClient httpClient,
+        AzureDevOpsRateLimitGate? rateLimitGate = null)
+        : this(
+            azdoParameters,
+            logger,
+            httpClient,
+            rateLimitGate ?? new AzureDevOpsRateLimitGate(),
+            ownsHttpClient: false)
+    {
+    }
+
+    private AzureDevOpsResultPublisher(
+        AzureDevOpsReportingParameters azdoParameters,
+        ILogger logger,
+        HttpClient httpClient,
+        AzureDevOpsRateLimitGate rateLimitGate,
+        bool ownsHttpClient)
     {
         _azdoParameters = azdoParameters;
         _httpClient = httpClient;
         _logger = logger;
+        _rateLimitGate = rateLimitGate;
+        _ownsHttpClient = ownsHttpClient;
     }
 
     public void Dispose()
     {
-        _httpClient.Dispose();
+        if (_ownsHttpClient)
+        {
+            _httpClient.Dispose();
+        }
     }
 
     public async Task<TestResultUploadSummary> UploadTestResultsWithSummaryAsync(List<string> testResultFiles, object resultMetadata, CancellationToken cancellationToken = default)
     {
         var testResultReader = new LocalTestResultsReader(_logger, _azdoParameters.TestResultAttachmentMode);
 
-        async Task<IReadOnlyList<TestResult>> ParseAsync(string file)
+        var parsedResults = new List<IReadOnlyList<TestResult>>(testResultFiles.Count);
+        foreach (string file in testResultFiles)
         {
-            DateTimeOffset startedAt = DateTimeOffset.UtcNow;
-            _logger.LogDebug("Parsing test result file '{FilePath}'.", file);
-            IReadOnlyList<TestResult> results = await testResultReader.ReadResultFileAsync(file, cancellationToken);
-            _logger.LogDebug(
-                "Parsed {ResultCount} test result(s) from '{FilePath}' in {Elapsed}.",
-                results.Count,
-                file,
-                DateTimeOffset.UtcNow - startedAt);
-            return results;
+            parsedResults.Add(await testResultReader.ReadResultFileAsync(file, cancellationToken));
         }
 
-        Task<IReadOnlyList<TestResult>>[] parseTasks = [.. testResultFiles.Select(ParseAsync)];
-        IReadOnlyList<TestResult>[] parsedResults = await Task.WhenAll(parseTasks);
-        if (parsedResults.Length == 0)
+        if (parsedResults.Count == 0)
         {
             _logger.LogWarning("No test result files were provided for upload");
             return new TestResultUploadSummary(true, 0);
@@ -114,9 +128,7 @@ public async Task<long> UploadTestResultsWithCountAsync(IEnumerable<AggregatedRe
         try
         {
             long publishedTestCount = 0;
-            IReadOnlyList<AggregatedResult> resultList = results as IReadOnlyList<AggregatedResult> ?? results.ToList();
-            var converted = ConvertResults(resultList, resultMetadata).ToList();
-            foreach (List<ConvertedResult> requestBatch in CreateResultRequestBatches(converted))
+            foreach (List<ConvertedResult> requestBatch in CreateResultRequestBatches(ConvertResults(results, resultMetadata)))
             {
                 IReadOnlyList<PublishedTestCase> publishedTests = await PublishResultsAsync(requestBatch, cancellationToken);
                 publishedTestCount += publishedTests.Count;
@@ -316,11 +328,10 @@ ConvertedResult ConvertResult(AggregatedResult result)
                 result);
         }
 
-        var converted = results.Select(ConvertResult).ToList();
-        foreach (ConvertedResult? result in converted)
+        foreach (AggregatedResult result in results)
         {
             foreach (ConvertedResult hierarchyPart in SplitOversizedResultHierarchy(
-                result,
+                ConvertResult(result),
                 MaximumNodesPerResultHierarchy))
             {
                 yield return hierarchyPart;
@@ -351,33 +362,78 @@ private static IEnumerable<ConvertedResult> SplitOversizedResultHierarchy(
             yield break;
         }
 
-        IEnumerable<ChunkPair> zippedSubTests = (test.Converted.SubResults ?? [])
-            .Zip(test.Aggregated.SubResults, (converted, aggregated) => new ChunkPair(converted, aggregated));
+        if (maximumNodesPerHierarchy <= 1 || test.Converted.SubResults is not { Count: > 0 })
+        {
+            throw new InvalidOperationException(
+                "A test-result hierarchy is deeper than the Azure DevOps hierarchy limit.");
+        }
+
+        IEnumerable<ChunkPair> splitSubTests = (test.Converted.SubResults ?? [])
+            .Zip(test.Aggregated.SubResults, (converted, aggregated) => new ChunkPair(converted, aggregated))
+            .SelectMany(pair => SplitOversizedSubResultHierarchy(pair, maximumNodesPerHierarchy - 1));
 
         // Each emitted hierarchy includes the copied top-level result, leaving the remaining
         // node budget for its sub-results.
         foreach (List<ChunkPair> hierarchyPart in PartitionBySize(
-            zippedSubTests,
+            splitSubTests,
             maximumNodesPerHierarchy - 1,
             static pair => CountResultTreeNodes(pair.Converted)))
         {
             yield return new ConvertedResult(
                 test.Converted with { SubResults = [.. hierarchyPart.Select(static x => x.Converted)], Id = null },
-                new AggregatedResult(
-                    test.Aggregated.AggregationType,
-                    test.Aggregated.Name,
-                    test.Aggregated.DurationSeconds,
-                    test.Aggregated.Result,
-                    [.. hierarchyPart.Select(static x => x.Aggregated)],
-                    test.Aggregated.Attachments,
-                    test.Aggregated.FailureMessage,
-                    test.Aggregated.StackTrace,
-                    isFlaky: test.Aggregated.IsFlaky,
-                    attemptId: test.Aggregated.AttemptId,
-                    fullyQualifiedName: test.Aggregated.FullyQualifiedName));
+                CopyAggregatedResult(
+                    test.Aggregated,
+                    [.. hierarchyPart.Select(static x => x.Aggregated)]));
+        }
+    }
+
+    private static IEnumerable<ChunkPair> SplitOversizedSubResultHierarchy(
+        ChunkPair test,
+        int maximumNodesPerHierarchy)
+    {
+        if (CountResultTreeNodes(test.Converted) <= maximumNodesPerHierarchy)
+        {
+            yield return test;
+            yield break;
+        }
+
+        IEnumerable<ChunkPair> splitSubTests = (test.Converted.SubResults ?? [])
+            .Zip(test.Aggregated.SubResults, (converted, aggregated) => new ChunkPair(converted, aggregated))
+            .SelectMany(pair => SplitOversizedSubResultHierarchy(pair, maximumNodesPerHierarchy - 1));
+
+        foreach (List<ChunkPair> hierarchyPart in PartitionBySize(
+            splitSubTests,
+            maximumNodesPerHierarchy - 1,
+            static pair => CountResultTreeNodes(pair.Converted)))
+        {
+            yield return new ChunkPair(
+                test.Converted with
+                {
+                    SubResults = [.. hierarchyPart.Select(static x => x.Converted)],
+                    Id = null,
+                },
+                CopyAggregatedResult(
+                    test.Aggregated,
+                    [.. hierarchyPart.Select(static x => x.Aggregated)]));
         }
     }
 
+    private static AggregatedResult CopyAggregatedResult(
+        AggregatedResult result,
+        IReadOnlyList<AggregatedResult> subResults)
+        => new(
+            result.AggregationType,
+            result.Name,
+            result.DurationSeconds,
+            result.Result,
+            subResults,
+            result.Attachments,
+            result.FailureMessage,
+            result.StackTrace,
+            isFlaky: result.IsFlaky,
+            attemptId: result.AttemptId,
+            fullyQualifiedName: result.FullyQualifiedName);
+
     private static int CountResultTreeNodes(PublishedTestCase test)
     {
         return 1 + (test.SubResults?.Sum(CountResultTreeNodes) ?? 0);
@@ -427,12 +483,8 @@ private static IEnumerable<List<T>> PartitionBySize<T>(
 
     private static HttpClient CreateHttpClient(string? accessToken)
     {
-        var client = new HttpClient
-        {
-            Timeout = s_httpClientTimeout
-        };
+        var client = new HttpClient { Timeout = s_httpClientTimeout };
         client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
-
         if (!string.IsNullOrWhiteSpace(accessToken))
         {
             string basicToken = Convert.ToBase64String(Encoding.ASCII.GetBytes($":{accessToken}"));
@@ -449,11 +501,7 @@ private async Task<HttpResponseMessage> SendWithRetryAsync(
         int attemptCount,
         CancellationToken cancellationToken)
     {
-        string? body = payload is null ? null : JsonSerializer.Serialize(payload, s_serializerOptions);
-        if (!string.IsNullOrEmpty(body))
-        {
-            s_lastSendContent = body;
-        }
+        byte[]? body = payload is null ? null : JsonSerializer.SerializeToUtf8Bytes(payload, s_serializerOptions);
 
         HttpResponseMessage? successfulResponse = null;
         Exception? lastException = null;
@@ -479,6 +527,8 @@ private async Task<HttpResponseMessage> SendWithRetryAsync(
         bool succeeded = await retryHandler.RunAsync(
             async attempt =>
             {
+                await _rateLimitGate.WaitAsync(cancellationToken);
+
                 Uri baseUri = _azdoParameters.CollectionUri.AbsoluteUri.EndsWith('/')
                     ? _azdoParameters.CollectionUri
                     : new Uri(_azdoParameters.CollectionUri.AbsoluteUri + '/', UriKind.Absolute);
@@ -486,7 +536,8 @@ private async Task<HttpResponseMessage> SendWithRetryAsync(
                 using var request = new HttpRequestMessage(method, new Uri(baseUri, relativePath));
                 if (body is not null)
                 {
-                    request.Content = new StringContent(body, Encoding.UTF8, "application/json");
+                    request.Content = new ByteArrayContent(body);
+                    request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
                 }
 
                 try
@@ -501,6 +552,11 @@ private async Task<HttpResponseMessage> SendWithRetryAsync(
                     HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken);
                     if (response.IsSuccessStatusCode)
                     {
+                        if (GetRateLimitDelay(response) is { } rateLimitDelay)
+                        {
+                            _rateLimitGate.Defer(rateLimitDelay);
+                        }
+
                         _logger.LogDebug(
                             "Azure DevOps {Method} request to '{RequestPath}' completed with HTTP {StatusCode} "
                             + "on attempt {Attempt} of {AttemptCount} after {Elapsed}.",
@@ -521,11 +577,15 @@ private async Task<HttpResponseMessage> SendWithRetryAsync(
                             || response.StatusCode == HttpStatusCode.TooManyRequests;
                         if (isTransientStatus)
                         {
-                            TimeSpan? retryAfter = GetRetryDelay(response);
+                            TimeSpan? retryAfter = GetRateLimitDelay(response);
                             if (response.StatusCode == HttpStatusCode.TooManyRequests && retryAfter is null)
                             {
                                 retryAfter = TimeSpan.FromSeconds(30);
                             }
+                            if (retryAfter is { } delay)
+                            {
+                                _rateLimitGate.Defer(delay);
+                            }
 
                             _logger.LogDebug(
                                 "Azure DevOps {Method} request to '{RequestPath}' returned HTTP {StatusCode} "
@@ -583,24 +643,40 @@ or TimeoutException
             or SocketException
             or IOException;
 
-    private static TimeSpan? GetRetryDelay(HttpResponseMessage response)
+    internal static TimeSpan? GetRateLimitDelay(HttpResponseMessage response)
     {
+        TimeSpan delay = TimeSpan.Zero;
         RetryConditionHeaderValue? retryAfter = response.Headers.RetryAfter;
-        if (retryAfter?.Delta is { } delta && delta > TimeSpan.Zero)
+        if (retryAfter?.Delta is { } delta && delta > delay)
         {
-            return delta;
+            delay = delta;
         }
 
         if (retryAfter?.Date is { } date)
         {
-            TimeSpan delay = date - DateTimeOffset.UtcNow;
-            if (delay > TimeSpan.Zero)
+            TimeSpan datedDelay = date - DateTimeOffset.UtcNow;
+            if (datedDelay > delay)
+            {
+                delay = datedDelay;
+            }
+        }
+
+        if (response.Headers.TryGetValues("X-RateLimit-Delay", out IEnumerable<string>? delayValues)
+            && double.TryParse(
+                delayValues.FirstOrDefault(),
+                NumberStyles.Float,
+                CultureInfo.InvariantCulture,
+                out double delaySeconds)
+            && delaySeconds > 0)
+        {
+            TimeSpan headerDelay = TimeSpan.FromSeconds(delaySeconds);
+            if (headerDelay > delay)
             {
-                return delay;
+                delay = headerDelay;
             }
         }
 
-        return null;
+        return delay > TimeSpan.Zero ? delay : null;
     }
 
     private static async Task<IReadOnlyList<PublishedTestCaseResultReference>> ReadPublishedResultsAsync(
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs
new file mode 100644
index 00000000000..e6a0f621b0b
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs
@@ -0,0 +1,320 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Globalization;
+using System.Runtime.CompilerServices;
+using System.Xml;
+using System.Xml.Linq;
+using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher;
+
+public sealed class LocalTestResultsReader(
+    ILogger logger,
+    TestResultAttachmentMode attachmentMode = TestResultAttachmentMode.Failed)
+{
+    private readonly ILogger _logger = logger;
+    private readonly TestResultAttachmentMode _attachmentMode = attachmentMode;
+
+    public static bool LooksLikeTestResultFile(string path)
+    {
+        string fileName = Path.GetFileName(path);
+        return fileName.EndsWith(".trx", StringComparison.OrdinalIgnoreCase)
+            || fileName.EndsWith("testResults.xml", StringComparison.OrdinalIgnoreCase)
+            || fileName.EndsWith("test-results.xml", StringComparison.OrdinalIgnoreCase)
+            || fileName.EndsWith("test_results.xml", StringComparison.OrdinalIgnoreCase)
+            || fileName.EndsWith("junit-results.xml", StringComparison.OrdinalIgnoreCase)
+            || fileName.EndsWith("junitresults.xml", StringComparison.OrdinalIgnoreCase);
+    }
+
+    public async Task<IReadOnlyList<TestResult>> ReadResultFileAsync(
+        string filePath,
+        CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            var results = new List<TestResult>();
+            await foreach (TestResult result in ReadResultsAsync(filePath, cancellationToken))
+            {
+                results.Add(result);
+            }
+
+            return results;
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            throw;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to parse test results file '{Path}'.", filePath);
+            return [];
+        }
+    }
+
+    private async IAsyncEnumerable<TestResult> ReadResultsAsync(
+        string filePath,
+        [EnumeratorCancellation] CancellationToken cancellationToken)
+    {
+        string rootName = await ReadRootNameAsync(filePath, cancellationToken);
+        string workItemName = new DirectoryInfo(Path.GetDirectoryName(filePath) ?? string.Empty).Name;
+
+        switch (rootName)
+        {
+            case "assemblies":
+            case "assembly":
+                await foreach (TestResult result in ReadElementsAsync(
+                    filePath,
+                    "test",
+                    ReadXunitResult,
+                    cancellationToken))
+                {
+                    yield return result;
+                }
+                break;
+
+            case "testsuites":
+            case "testsuite":
+                await foreach (TestResult result in ReadElementsAsync(
+                    filePath,
+                    "testcase",
+                    element => ReadJUnitResult(element, workItemName),
+                    cancellationToken))
+                {
+                    yield return result;
+                }
+                break;
+
+            case "TestRun":
+                Dictionary<string, TestDefinition> definitions =
+                    await ReadTrxDefinitionsAsync(filePath, cancellationToken);
+                await foreach (TestResult result in ReadElementsAsync(
+                    filePath,
+                    "UnitTestResult",
+                    element => ReadTrxResult(element, workItemName, definitions),
+                    cancellationToken))
+                {
+                    yield return result;
+                }
+                break;
+        }
+    }
+
+    private static async Task<string> ReadRootNameAsync(string filePath, CancellationToken cancellationToken)
+    {
+        using XmlReader reader = CreateReader(filePath);
+        while (await reader.ReadAsync())
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            if (reader.NodeType == XmlNodeType.Element)
+            {
+                return reader.LocalName;
+            }
+        }
+
+        return string.Empty;
+    }
+
+    private static async IAsyncEnumerable<TestResult> ReadElementsAsync(
+        string filePath,
+        string elementName,
+        Func<XElement, TestResult> convert,
+        [EnumeratorCancellation] CancellationToken cancellationToken)
+    {
+        using XmlReader reader = CreateReader(filePath);
+        while (await reader.ReadAsync())
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            if (reader.NodeType != XmlNodeType.Element || reader.LocalName != elementName)
+            {
+                continue;
+            }
+
+            using XmlReader subtree = reader.ReadSubtree();
+            yield return convert(XElement.Load(subtree, LoadOptions.PreserveWhitespace));
+        }
+    }
+
+    private static async Task<Dictionary<string, TestDefinition>> ReadTrxDefinitionsAsync(
+        string filePath,
+        CancellationToken cancellationToken)
+    {
+        var definitions = new Dictionary<string, TestDefinition>(StringComparer.OrdinalIgnoreCase);
+        using XmlReader reader = CreateReader(filePath);
+        while (await reader.ReadAsync())
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            if (reader.NodeType != XmlNodeType.Element || reader.LocalName != "UnitTest")
+            {
+                continue;
+            }
+
+            using XmlReader subtree = reader.ReadSubtree();
+            XElement unitTest = XElement.Load(subtree);
+            string? id = GetAttribute(unitTest, "id");
+            XElement? method = unitTest.Descendants().FirstOrDefault(static x => x.Name.LocalName == "TestMethod");
+            if (!string.IsNullOrEmpty(id))
+            {
+                definitions[id] = new TestDefinition(
+                    GetAttribute(method, "className"),
+                    GetAttribute(method, "name"));
+            }
+        }
+
+        return definitions;
+    }
+
+    private TestResult ReadXunitResult(XElement test)
+    {
+        XElement? failure = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "failure");
+        string? output = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "output")?.Value?.Trim();
+        string typeName = GetAttribute(test, "type") ?? string.Empty;
+        string method = GetAttribute(test, "method") ?? string.Empty;
+        string name = GetAttribute(test, "name")
+            ?? (!string.IsNullOrEmpty(typeName) && !string.IsNullOrEmpty(method) ? $"{typeName}.{method}" : method);
+        string outcome = NormalizeOutcome(GetAttribute(test, "result"));
+
+        List<TestResultAttachment> attachments = [];
+        AddAttachmentIfEnabled(attachments, "output.txt", output, outcome);
+
+        return new TestResult(
+            name,
+            "xunit",
+            typeName,
+            method,
+            ParseDouble(GetAttribute(test, "time")),
+            outcome,
+            GetAttribute(failure, "exception-type"),
+            failure?.Elements().FirstOrDefault(static x => x.Name.LocalName == "message")?.Value?.Trim(),
+            failure?.Elements().FirstOrDefault(static x => x.Name.LocalName == "stack-trace")?.Value?.Trim(),
+            test.Elements().FirstOrDefault(static x => x.Name.LocalName == "reason")?.Value?.Trim(),
+            attachments);
+    }
+
+    private TestResult ReadJUnitResult(XElement test, string workItemName)
+    {
+        XElement? failure = test.Elements().FirstOrDefault(static x => x.Name.LocalName is "failure" or "error");
+        XElement? skipped = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "skipped");
+        string className = GetAttribute(test, "classname") ?? workItemName;
+        string method = GetAttribute(test, "name") ?? string.Empty;
+        string outcome = skipped is not null ? "Skip" : failure is not null ? "Fail" : "Pass";
+
+        List<TestResultAttachment> attachments = [];
+        AddAttachmentIfEnabled(
+            attachments,
+            "stdout.txt",
+            test.Elements().FirstOrDefault(static x => x.Name.LocalName == "system-out")?.Value?.Trim(),
+            outcome);
+        AddAttachmentIfEnabled(
+            attachments,
+            "stderr.txt",
+            test.Elements().FirstOrDefault(static x => x.Name.LocalName == "system-err")?.Value?.Trim(),
+            outcome);
+
+        return new TestResult(
+            !string.IsNullOrEmpty(className) ? $"{className}.{method}" : method,
+            "junit",
+            className,
+            method,
+            ParseDouble(GetAttribute(test, "time")),
+            outcome,
+            null,
+            failure?.Value?.Trim(),
+            null,
+            skipped?.Value?.Trim(),
+            attachments);
+    }
+
+    private TestResult ReadTrxResult(
+        XElement result,
+        string workItemName,
+        IReadOnlyDictionary<string, TestDefinition> definitions)
+    {
+        string testId = GetAttribute(result, "testId") ?? string.Empty;
+        definitions.TryGetValue(testId, out TestDefinition? definition);
+        string className = definition?.ClassName ?? workItemName;
+        string method = definition?.Method ?? GetAttribute(result, "testName") ?? string.Empty;
+        string displayName = GetAttribute(result, "testName")
+            ?? (!string.IsNullOrEmpty(className) ? $"{className}.{method}" : method);
+
+        XElement? output = result.Descendants().FirstOrDefault(static x => x.Name.LocalName == "Output");
+        string? failureMessage = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "Message")?.Value?.Trim();
+        string outcome = NormalizeOutcome(GetAttribute(result, "outcome"));
+
+        List<TestResultAttachment> attachments = [];
+        AddAttachmentIfEnabled(
+            attachments,
+            "stdout.txt",
+            output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StdOut")?.Value?.Trim(),
+            outcome);
+        AddAttachmentIfEnabled(
+            attachments,
+            "stderr.txt",
+            output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StdErr")?.Value?.Trim(),
+            outcome);
+
+        return new TestResult(
+            displayName,
+            "trx",
+            className,
+            method,
+            ParseDuration(GetAttribute(result, "duration")),
+            outcome,
+            null,
+            failureMessage,
+            output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StackTrace")?.Value?.Trim(),
+            string.Equals(outcome, "Skip", StringComparison.Ordinal) ? failureMessage : null,
+            attachments);
+    }
+
+    private static XmlReader CreateReader(string filePath)
+        => XmlReader.Create(File.OpenRead(filePath), new XmlReaderSettings
+        {
+            Async = true,
+            CloseInput = true,
+            DtdProcessing = DtdProcessing.Prohibit,
+            IgnoreComments = true,
+        });
+
+    private static string? GetAttribute(XElement? element, string name)
+        => element?.Attribute(name)?.Value;
+
+    private static double ParseDouble(string? value)
+        => double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double result) ? result : 0;
+
+    private static double ParseDuration(string? value)
+        => TimeSpan.TryParse(value, CultureInfo.InvariantCulture, out TimeSpan result)
+            ? result.TotalSeconds
+            : ParseDouble(value);
+
+    private static string NormalizeOutcome(string? value)
+        => value?.Trim().ToLowerInvariant() switch
+        {
+            "pass" or "passed" or "success" or "succeeded" => "Pass",
+            "skip" or "skipped" or "notexecuted" or "notrun" => "Skip",
+            "fail" or "failed" or "error" or "timeout" or "aborted" => "Fail",
+            _ => "None",
+        };
+
+    private void AddAttachmentIfEnabled(
+        List<TestResultAttachment> attachments,
+        string name,
+        string? text,
+        string normalizedOutcome)
+    {
+        bool includeAttachment = _attachmentMode switch
+        {
+            TestResultAttachmentMode.All => true,
+            TestResultAttachmentMode.Failed => normalizedOutcome == "Fail",
+            _ => false,
+        };
+
+        if (includeAttachment && !string.IsNullOrWhiteSpace(text))
+        {
+            attachments.Add(new TestResultAttachment(name, text));
+        }
+    }
+
+    private sealed record TestDefinition(string? ClassName, string? Method);
+}
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingError.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/AzureDevOpsReportingError.cs
similarity index 100%
rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingError.cs
rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/AzureDevOpsReportingError.cs
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingParameters.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/AzureDevOpsReportingParameters.cs
similarity index 100%
rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingParameters.cs
rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/AzureDevOpsReportingParameters.cs
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/PackedTestReport.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/PackedTestReport.cs
similarity index 100%
rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/PackedTestReport.cs
rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/PackedTestReport.cs
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TerminalError.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TerminalError.cs
similarity index 100%
rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TerminalError.cs
rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TerminalError.cs
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResult.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TestResult.cs
similarity index 100%
rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResult.cs
rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TestResult.cs
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResultAttachment.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TestResultAttachment.cs
similarity index 100%
rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResultAttachment.cs
rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TestResultAttachment.cs
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResultAttachmentMode.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TestResultAttachmentMode.cs
similarity index 100%
rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResultAttachmentMode.cs
rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/Model/TestResultAttachmentMode.cs
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/ResultAggregator.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/ResultAggregator.cs
similarity index 100%
rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/ResultAggregator.cs
rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/ResultAggregator.cs
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestNameFormatter.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/TestNameFormatter.cs
similarity index 100%
rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestNameFormatter.cs
rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/TestNameFormatter.cs
diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestResultUploadSummary.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/TestResultUploadSummary.cs
similarity index 100%
rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestResultUploadSummary.cs
rename to src/Microsoft.DotNet.Helix/JobMonitor/TestResults/TestResultUploadSummary.cs
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
index cc6a0ae3ae9..f13409004bd 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
@@ -6,6 +6,7 @@
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Reflection;
 using System.Text.Json;
 using System.Threading;
@@ -121,6 +122,16 @@ public void CancellationWithoutTimeoutIsNotTransient()
                 CancellationToken.None));
         }
 
+        [Fact]
+        public void GetRateLimitDelay_UsesLargestAzureDevOpsDelayHeader()
+        {
+            using var response = new HttpResponseMessage(HttpStatusCode.OK);
+            response.Headers.RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromSeconds(2));
+            response.Headers.Add("X-RateLimit-Delay", "3.5");
+
+            Assert.Equal(TimeSpan.FromSeconds(3.5), AzureDevOpsResultPublisher.GetRateLimitDelay(response));
+        }
+
         [Fact]
         public async Task UploadTestResultsWithCountAsync_BatchesByTopLevelResultCount()
         {
@@ -169,6 +180,55 @@ public async Task UploadTestResultsWithCountAsync_SplitHierarchiesIncludeRootInN
             Assert.Equal(new[] { 950, 2 }, handler.RequestHierarchyNodeCounts.Single());
         }
 
+        [Fact]
+        public async Task UploadTestResultsWithCountAsync_RecursivelySplitsOversizedNestedHierarchies()
+        {
+            var handler = new RecordingResultHandler();
+            using var publisher = CreatePublisher(handler);
+            var nested = CreateDataDrivenResult("Nested", 950);
+            AggregatedResult[] results =
+            [
+                new AggregatedResult(
+                    AggregationType.DataDriven,
+                    "Outer",
+                    1,
+                    "Passed",
+                    [nested]),
+            ];
+
+            long uploadedCount = await publisher.UploadTestResultsWithCountAsync(results, new { });
+
+            Assert.Equal(2, uploadedCount);
+            Assert.Equal(new[] { 2 }, handler.RequestResultCounts);
+            Assert.Equal(new[] { 950, 4 }, handler.RequestHierarchyNodeCounts.Single());
+        }
+
+        [Fact]
+        public async Task UploadTestResultsWithCountAsync_DoesNotMaterializeAllConvertedResults()
+        {
+            var handler = new BlockingResultHandler();
+            using var publisher = CreatePublisher(handler);
+            int enumerated = 0;
+
+            IEnumerable<AggregatedResult> Results()
+            {
+                for (int i = 0; i < 2_000; i++)
+                {
+                    enumerated++;
+                    yield return new AggregatedResult(AggregationType.Single, $"Test{i}", 1, "Passed");
+                }
+            }
+
+            Task<long> upload = publisher.UploadTestResultsWithCountAsync(Results(), new { });
+            await handler.FirstRequestStarted.Task.WaitAsync(TimeSpan.FromSeconds(5));
+
+            Assert.InRange(enumerated, 0, 1001);
+            handler.ReleaseFirstRequest.SetResult();
+
+            Assert.Equal(2_000, await upload);
+            Assert.Equal(2, handler.RequestResultCounts.Count);
+        }
+
         private static AzureDevOpsResultPublisher CreatePublisher(HttpMessageHandler handler)
             => new(
                 new AzureDevOpsReportingParameters(
@@ -189,7 +249,7 @@ .. Enumerable.Range(0, subResultCount)
                         .Select(i => new AggregatedResult(AggregationType.Single, $"{name}_{i}", 1, "Passed"))
                 ]);
 
-        private sealed class RecordingResultHandler : HttpMessageHandler
+        private class RecordingResultHandler : HttpMessageHandler
         {
             public List<int> RequestResultCounts { get; } = [];
             public List<int[]> RequestHierarchyNodeCounts { get; } = [];
@@ -227,5 +287,27 @@ private static int CountHierarchyNodes(JsonElement result)
             }
         }
 
+        private sealed class BlockingResultHandler : RecordingResultHandler
+        {
+            public TaskCompletionSource FirstRequestStarted { get; } =
+                new(TaskCreationOptions.RunContinuationsAsynchronously);
+            public TaskCompletionSource ReleaseFirstRequest { get; } =
+                new(TaskCreationOptions.RunContinuationsAsynchronously);
+            private int _requestCount;
+
+            protected override async Task<HttpResponseMessage> SendAsync(
+                HttpRequestMessage request,
+                CancellationToken cancellationToken)
+            {
+                if (Interlocked.Increment(ref _requestCount) == 1)
+                {
+                    FirstRequestStarted.SetResult();
+                    await ReleaseFirstRequest.Task.WaitAsync(cancellationToken);
+                }
+
+                return await base.SendAsync(request, cancellationToken);
+            }
+        }
+
     }
 }
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs
index 5fce31eae8d..ea70f4585a9 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs
@@ -31,6 +31,8 @@ internal sealed class FakeAzureDevOpsService : IAzureDevOpsService
             = new(FailedTestWorkItemComparer.Instance);
         private int _timelineCallCount;
         private int _nextTestRunId;
+        private int _activeUploads;
+        private int _maximumConcurrentUploads;
 
         // Observable state for test assertions
         public List<string> CreatedTestRuns { get; } = [];
@@ -40,8 +42,10 @@ internal sealed class FakeAzureDevOpsService : IAzureDevOpsService
         public int CreateTestRunCallCount { get; private set; }
         public int UploadTestResultsCallCount { get; private set; }
         public int CompleteTestRunCallCount { get; private set; }
+        public int MaximumConcurrentUploads => Volatile.Read(ref _maximumConcurrentUploads);
         public TaskCompletionSource UploadStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
         public TaskCompletionSource UploadCompleted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+        public TaskCompletionSource TestRunCompleted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
         public Task UploadBlocker { get; set; } = Task.CompletedTask;
 
         /// <summary>
@@ -213,58 +217,67 @@ public Task CompleteTestRunAsync(int testRunId, string helixJobName, IReadOnlyCo
                         _recordedFailedTests.Add((helixJobName, workItemName));
                     }
                 }
+                TestRunCompleted.TrySetResult();
                 return Task.CompletedTask;
             }
         }
 
-        public async Task<IReadOnlyDictionary<(string JobName, string WorkItemName), TestResultUploadSummary>> UploadTestResultsAsync(
+        public async Task<TestResultUploadSummary> UploadTestResultsAsync(
             int testRunId,
-            IReadOnlyList<WorkItemTestResults> results,
+            WorkItemTestResults results,
             CancellationToken cancellationToken)
         {
             UploadStarted.TrySetResult();
-            if (UploadBlockerIgnoresCancellation)
+            int active = Interlocked.Increment(ref _activeUploads);
+            int observedMaximum;
+            while (active > (observedMaximum = Volatile.Read(ref _maximumConcurrentUploads)))
             {
-                await UploadBlocker;
-            }
-            else
-            {
-                await UploadBlocker.WaitAsync(cancellationToken);
+                if (Interlocked.CompareExchange(ref _maximumConcurrentUploads, active, observedMaximum) == observedMaximum)
+                {
+                    break;
+                }
             }
 
-            var summaries = new Dictionary<(string JobName, string WorkItemName), TestResultUploadSummary>();
-
-            lock (_sync)
+            try
             {
-                UploadTestResultsCallCount++;
-                if (_uploadFailures.Count > 0)
+                if (UploadBlockerIgnoresCancellation)
                 {
-                    throw _uploadFailures.Dequeue();
+                    await UploadBlocker;
                 }
-
-                if (!UploadedResultsByRunId.TryGetValue(testRunId, out List<WorkItemTestResults> existing))
+                else
                 {
-                    existing = [];
-                    UploadedResultsByRunId[testRunId] = existing;
+                    await UploadBlocker.WaitAsync(cancellationToken);
                 }
 
-                existing.AddRange(results);
-
-                foreach (string jobName in results.Select(r => r.JobName).Distinct(StringComparer.OrdinalIgnoreCase))
+                lock (_sync)
                 {
-                    UploadedJobNames.Add(jobName);
-                }
+                    UploadTestResultsCallCount++;
+                    if (_uploadFailures.Count > 0)
+                    {
+                        throw _uploadFailures.Dequeue();
+                    }
 
-                foreach (WorkItemTestResults result in results)
-                {
-                    bool allPassed = !_uploadFailedTests.Contains((result.JobName, result.WorkItemName));
-                    summaries[(result.JobName, result.WorkItemName)] =
-                        new TestResultUploadSummary(allPassed, result.TestResultFiles.Count);
+                    if (!UploadedResultsByRunId.TryGetValue(testRunId, out List<WorkItemTestResults> existing))
+                    {
+                        existing = [];
+                        UploadedResultsByRunId[testRunId] = existing;
+                    }
+
+                    existing.Add(results);
+                    if (!UploadedJobNames.Contains(results.JobName, StringComparer.OrdinalIgnoreCase))
+                    {
+                        UploadedJobNames.Add(results.JobName);
+                    }
+
+                    bool allPassed = !_uploadFailedTests.Contains((results.JobName, results.WorkItemName));
+                    return new TestResultUploadSummary(allPassed, results.TestResultFiles.Count);
                 }
             }
-
-            UploadCompleted.TrySetResult();
-            return summaries;
+            finally
+            {
+                Interlocked.Decrement(ref _activeUploads);
+                UploadCompleted.TrySetResult();
+            }
         }
 
         private static HttpRequestException CreateTransientFailure(string message)
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs
index 1b266aade5e..2a414ec8c4d 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs
@@ -21,6 +21,8 @@ internal sealed class FakeHelixService : IHelixService
         private readonly HashSet<string> _downloadFailureJobs = new(StringComparer.OrdinalIgnoreCase);
         private readonly Dictionary<string, Queue<Exception>> _downloadFailures = new(StringComparer.OrdinalIgnoreCase);
         private readonly Dictionary<string, IReadOnlyCollection<WorkItemSummary>> _customWorkItems = new(StringComparer.OrdinalIgnoreCase);
+        private readonly ConcurrentDictionary<string, int> _listWorkItemsCallCounts =
+            new(StringComparer.OrdinalIgnoreCase);
         private int _getJobsCallCount;
 
         /// <summary>
@@ -68,6 +70,9 @@ public FakeHelixService WithWorkItems(string jobName, IReadOnlyCollection<WorkIt
         /// <summary>Number of times <see cref="GetJobsForBuildAsync"/> has been called.</summary>
         public int GetJobsCallCount => _getJobsCallCount;
 
+        public int GetListWorkItemsCallCount(string jobName)
+            => _listWorkItemsCallCounts.TryGetValue(jobName, out int count) ? count : 0;
+
         public ConcurrentBag<string> CanceledJobs { get; } = [];
 
         private HelixSnapshot CurrentSnapshot
@@ -92,8 +97,8 @@ public Task<IReadOnlyList<HelixJobInfo>> GetJobsForBuildAsync(string source, str
             return Task.FromResult<IReadOnlyList<HelixJobInfo>>(_responses[index].Jobs);
         }
 
-        public Task<IReadOnlyList<WorkItemTestResults>> DownloadTestResultsAsync(
-            string jobName, IReadOnlyCollection<string> workItemNames, string workingDirectory, CancellationToken cancellationToken)
+        public Task<WorkItemTestResults> DownloadTestResultsAsync(
+            string jobName, string workItemName, string workingDirectory, CancellationToken cancellationToken)
         {
             if (_downloadFailureJobs.Contains(jobName))
             {
@@ -108,25 +113,21 @@ public Task<IReadOnlyList<WorkItemTestResults>> DownloadTestResultsAsync(
 
             if (CurrentSnapshot.TestResultsByJob.TryGetValue(jobName, out List<WorkItemTestResults> explicitResults))
             {
-                return Task.FromResult<IReadOnlyList<WorkItemTestResults>>(explicitResults);
+                WorkItemTestResults result = explicitResults.FirstOrDefault(
+                    result => string.Equals(result.WorkItemName, workItemName, StringComparison.OrdinalIgnoreCase))
+                    ?? new WorkItemTestResults(jobName, workItemName, []);
+                return Task.FromResult(result);
             }
 
-            workItemNames = workItemNames
-                .Distinct(StringComparer.OrdinalIgnoreCase)
-                .DefaultIfEmpty($"{jobName}-synthetic")
-                .ToList();
-
-            IReadOnlyList<WorkItemTestResults> generated = workItemNames
-                .Select(wi => new WorkItemTestResults(jobName, wi, []))
-                .ToList();
-
-            return Task.FromResult(generated);
+            return Task.FromResult(new WorkItemTestResults(jobName, workItemName, []));
         }
 
         public Task<IReadOnlyCollection<WorkItemSummary>> ListWorkItemsAsync(
             string jobName,
             CancellationToken _)
         {
+            _listWorkItemsCallCounts.AddOrUpdate(jobName, 1, static (_, count) => count + 1);
+
             if (_customWorkItems.TryGetValue(jobName, out IReadOnlyCollection<WorkItemSummary> customWorkItems))
             {
                 return Task.FromResult(customWorkItems);
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
index 6b2fca83d1b..12c5f47d850 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
@@ -106,10 +106,8 @@ public async Task DownloadTestResultsAsync_FiltersFilesUsesFileSystemAndContinue
             blobClientFactory.FailDownloadsFor.Add("https://storage/failed.trx");
             var fileSystem = new MockFileSystem(directorySeparator: Path.DirectorySeparatorChar.ToString());
 
-            IReadOnlyList<WorkItemTestResults> results = await CreateService(api.Api.Object, blobClientFactory, fileSystem)
-                .DownloadTestResultsAsync("job:name", ["work:item", "no-results"], "work", CancellationToken.None);
-
-            WorkItemTestResults result = Assert.Single(results);
+            WorkItemTestResults result = await CreateService(api.Api.Object, blobClientFactory, fileSystem)
+                .DownloadTestResultsAsync("job:name", "work:item", "work", CancellationToken.None);
             Assert.Equal("job:name", result.JobName);
             Assert.Equal("work:item", result.WorkItemName);
             string jobDirectory = fileSystem.PathCombine("work", SanitizeForCurrentPlatform("job:name"));
@@ -143,7 +141,7 @@ public async Task DownloadTestResultsAsync_ReportsTransientFileFailureAfterAttem
                 new HttpRequestException("Injected transient failure.", null, HttpStatusCode.ServiceUnavailable);
 
             Func<Task> action = () => CreateService(api.Api.Object, blobClientFactory, new MockFileSystem())
-                .DownloadTestResultsAsync("job", ["work-item"], "work", CancellationToken.None);
+                .DownloadTestResultsAsync("job", "work-item", "work", CancellationToken.None);
 
             await Assert.ThrowsAsync<IOException>(action);
             Assert.Equal(2, blobClientFactory.Downloads.Count);
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
index 6cea4d49d82..72ef0f85793 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
@@ -530,6 +530,9 @@ public async Task CompletedHelixJob_QueuesTestResultUploadWithoutBlockingNextPol
             exitCode.Should().Be(0);
             delayedBeforeUploadCompleted.Should().BeTrue();
             azdo.TimelineCallCount.Should().Be(2);
+            // One entry-retry scan and one first-poll snapshot; the second poll reuses the
+            // reconciled terminal snapshot instead of fetching it again.
+            helix.GetListWorkItemsCallCount("helix-linux").Should().Be(2);
             azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-linux"]);
             azdo.CompletedTestRunIds.Should().ContainSingle();
             logger.Messages.Should().Contain(message =>
@@ -537,7 +540,7 @@ public async Task CompletedHelixJob_QueuesTestResultUploadWithoutBlockingNextPol
         }
 
         [Fact]
-        public async Task VerboseDrainReportsPendingUploadPhaseAndElapsedTime()
+        public async Task DrainReportsAggregatePipelineProgress()
         {
             var azdo = new FakeAzureDevOpsService();
             var helix = new FakeHelixService();
@@ -576,11 +579,82 @@ public async Task VerboseDrainReportsPendingUploadPhaseAndElapsedTime()
 
             exitCode.Should().Be(0);
             logger.Messages.Should().Contain(message =>
-                message.Contains("test result upload(s) remain pending", StringComparison.Ordinal)
-                && message.Contains("helix-linux", StringComparison.Ordinal)
-                && message.Contains("phase='publishing 1 work item(s) to Azure DevOps test run", StringComparison.Ordinal)
-                && message.Contains("phase elapsed=", StringComparison.Ordinal)
-                && message.Contains("total elapsed=", StringComparison.Ordinal));
+                message.Contains("Test result pipeline drained in", StringComparison.Ordinal)
+                && message.Contains("1 job(s), 1 work item(s), and 1 result(s)", StringComparison.Ordinal));
+        }
+
+        [Fact]
+        public async Task UploadPipeline_BoundsWorkItemParallelismAndCreatesOneRunPerJob()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+            var uploadRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+            string[] workItems = [.. Enumerable.Range(1, 20).Select(index => $"workitem-{index}")];
+
+            azdo.UploadBlocker = uploadRelease.Task;
+            azdo.AddTimelineResponse(
+                MonitorJob(),
+                PipelineJob("Test Linux", "completed", "succeeded"));
+            helix.AddResponse(
+                jobs: [HelixJob("helix-linux", "finished")],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-linux"] = PassFail(passed: workItems),
+                },
+                testResultsByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-linux"] =
+                    [
+                        ..workItems.Select(name =>
+                            new WorkItemTestResults("helix-linux", name, [$"{name}.trx"]))
+                    ],
+                });
+
+            JobMonitorOptions options = DefaultOptions();
+            options.TestResultUploadParallelism = 4;
+            var runner = new JobMonitorRunner(options, new RecordingLogger(), azdo, helix, NoDelay);
+
+            Task<int> run = runner.RunAsync(CancellationToken.None);
+            await WaitForAsync(() => azdo.MaximumConcurrentUploads == 4);
+            uploadRelease.SetResult();
+
+            (await run.WaitAsync(TimeSpan.FromSeconds(5))).Should().Be(0);
+            azdo.MaximumConcurrentUploads.Should().Be(4);
+            azdo.CreateTestRunCallCount.Should().Be(1);
+            azdo.CompleteTestRunCallCount.Should().Be(1);
+            azdo.UploadTestResultsCallCount.Should().Be(20);
+        }
+
+        [Fact]
+        public async Task UploadPipeline_DoesNotDropCompletedJobsWhenBacklogged()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+            HelixJobInfo[] jobs =
+            [
+                ..Enumerable.Range(1, 300)
+                    .Select(index => HelixJob($"helix-{index}", "finished"))
+            ];
+            var passFailByJob = jobs.ToDictionary(
+                static job => job.JobName,
+                static _ => PassFail(),
+                StringComparer.OrdinalIgnoreCase);
+
+            azdo.AddTimelineResponse(
+                MonitorJob(),
+                PipelineJob("Test Linux", "completed", "succeeded"));
+            helix.AddResponse(jobs, passFailByJob);
+
+            var runner = new JobMonitorRunner(
+                DefaultOptions(),
+                new RecordingLogger(),
+                azdo,
+                helix,
+                NoDelay);
+
+            (await runner.RunAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(10))).Should().Be(0);
+            azdo.CreateTestRunCallCount.Should().Be(300);
+            azdo.CompleteTestRunCallCount.Should().Be(300);
         }
 
         [Fact]
@@ -626,7 +700,8 @@ public async Task PassedHelixWork_TransientUploadFailure_DoesNotReplayAmbiguousW
             azdo.CompletedTestRunIds.Should().BeEmpty();
             azdo.UploadedJobNames.Should().BeEmpty();
             logger.Messages.Should().Contain(message =>
-                message.Contains("not safe to replay in this invocation", StringComparison.Ordinal));
+                message.Contains("remains untagged", StringComparison.Ordinal)
+                && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal));
         }
 
         [Fact]
@@ -665,7 +740,8 @@ public async Task PassedHelixWork_TransientCompletionFailure_DoesNotReplayComple
             delayCount.Should().Be(0);
             azdo.CompletedTestRunIds.Should().BeEmpty();
             logger.Messages.Should().Contain(message =>
-                message.Contains("not safe to replay in this invocation", StringComparison.Ordinal));
+                message.Contains("remains untagged", StringComparison.Ordinal)
+                && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal));
         }
 
         [Fact]
@@ -696,8 +772,8 @@ public async Task PassedHelixWork_PermanentUploadFailure_DoesNotHangOrFailBuild(
             azdo.CompleteTestRunCallCount.Should().Be(0);
             azdo.CompletedTestRunIds.Should().BeEmpty();
             logger.Messages.Should().Contain(message =>
-                message.Contains("The failure is not retryable", StringComparison.Ordinal)
-                && message.Contains("later monitor invocation may retry the upload", StringComparison.Ordinal));
+                message.Contains("remains untagged", StringComparison.Ordinal)
+                && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal));
         }
 
         [Fact]
@@ -1947,8 +2023,8 @@ public async Task MonitorTimesOut_Relaunched_UploadsRemainingResults()
                     pollCount1++;
                     if (pollCount1 >= 2)
                     {
-                        Task completed = await Task.WhenAny(azdo1.UploadCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5)));
-                        Assert.Same(azdo1.UploadCompleted.Task, completed);
+                        Task completed = await Task.WhenAny(azdo1.TestRunCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5)));
+                        Assert.Same(azdo1.TestRunCompleted.Task, completed);
                         cts.Cancel();
                     }
                 });
@@ -2016,8 +2092,8 @@ public async Task MonitorTimesOut_CancelsLatestInFlightHelixJobs()
             var runner = new JobMonitorRunner(DefaultOptions(), logger, azdo, helix,
                 async (_, _) =>
                 {
-                    Task completed = await Task.WhenAny(azdo.UploadCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5)));
-                    completed.Should().BeSameAs(azdo.UploadCompleted.Task);
+                    Task completed = await Task.WhenAny(azdo.TestRunCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5)));
+                    completed.Should().BeSameAs(azdo.TestRunCompleted.Task);
                     cts.Cancel();
                 });
 
@@ -2148,8 +2224,8 @@ public async Task MonitorTimesOut_DoesNotReportOrCancelJobsThatFinishedAfterFirs
                         // Wait until helix-good's results have actually been uploaded before
                         // cancelling, so the monitor has had a chance to record its terminal
                         // state.
-                        Task completed = await Task.WhenAny(azdo.UploadCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5)));
-                        completed.Should().BeSameAs(azdo.UploadCompleted.Task);
+                        Task completed = await Task.WhenAny(azdo.TestRunCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5)));
+                        completed.Should().BeSameAs(azdo.TestRunCompleted.Task);
                         cts.Cancel();
                     }
                 });
@@ -2217,8 +2293,8 @@ public async Task MonitorTimesOut_PartialProgress_Relaunched_CompletesSuccessful
                     pollCount1++;
                     if (pollCount1 >= 2)
                     {
-                        Task completed = await Task.WhenAny(azdo1.UploadCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5)));
-                        Assert.Same(azdo1.UploadCompleted.Task, completed);
+                        Task completed = await Task.WhenAny(azdo1.TestRunCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5)));
+                        Assert.Same(azdo1.TestRunCompleted.Task, completed);
                         cts.Cancel();
                     }
                 });
@@ -2926,8 +3002,8 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork()
                     // Wait for helix-a's test-result upload to finish before cancelling so the
                     // assertion below is deterministic. Otherwise the background upload task
                     // (bound to the runner's cancellation token) races against cts1.Cancel().
-                    Task completed = await Task.WhenAny(azdo1.UploadCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5)));
-                    completed.Should().BeSameAs(azdo1.UploadCompleted.Task);
+                    Task completed = await Task.WhenAny(azdo1.TestRunCompleted.Task, Task.Delay(TimeSpan.FromSeconds(5)));
+                    completed.Should().BeSameAs(azdo1.TestRunCompleted.Task);
                     cts1.Cancel();
                 });
 
@@ -3634,7 +3710,7 @@ public async Task LoopStatus_LogsAggregateHelixJobWorkItemCounts()
         }
 
         [Fact]
-        public async Task LoopStatus_VerboseLogsFullHelixJobWorkItemList()
+        public async Task LoopStatus_VerboseLogsBoundedPipelineDiagnostics()
         {
             var azdo = new FakeAzureDevOpsService();
             var helix = new FakeHelixService();
@@ -3667,12 +3743,10 @@ public async Task LoopStatus_VerboseLogsFullHelixJobWorkItemList()
 
             exitCode.Should().Be(1);
             logger.Messages.Should().Contain(message =>
-                message.Contains("Helix job details:", StringComparison.Ordinal)
-                && message.Contains("└─ 🧪 Helix job helix-linux [Running]", StringComparison.Ordinal)
-                && message.Contains("   ├─ wi-01 (Running)", StringComparison.Ordinal)
-                && message.Contains("   ├─ wi-10 (Running)", StringComparison.Ordinal)
-                && message.Contains("   ├─ wi-11 (Running)", StringComparison.Ordinal)
-                && message.Contains("   └─ wi-12 (Running)", StringComparison.Ordinal));
+                message.Contains("Upload pipeline:", StringComparison.Ordinal));
+            logger.Messages.Should().NotContain(message =>
+                message.Contains("wi-01", StringComparison.Ordinal)
+                || message.Contains("wi-12", StringComparison.Ordinal));
         }
 
         /// <summary>
@@ -3714,7 +3788,7 @@ public async Task LoopStatus_WaitingWorkItems_CountedAsWaitingWhileJobIsRunning(
             logger.Messages.Should().Contain(message =>
                 message.Contains("0 processed / 0 completed / 0 running / 2 waiting work items", StringComparison.Ordinal));
             logger.Messages.Should().Contain(message =>
-                message.Contains("└─ 🧪 Helix job helix-linux [Running]", StringComparison.Ordinal));
+                message.Contains("Upload pipeline:", StringComparison.Ordinal));
         }
 
         /// <summary>
@@ -3948,6 +4022,15 @@ public async Task FailOnFailedTestsDisabled_TestFailuresIgnoredForOutcomeAndRetr
 
         private static readonly Func<TimeSpan, CancellationToken, Task> NoDelay = (_, _) => Task.CompletedTask;
 
+        private static async Task WaitForAsync(Func<bool> condition)
+        {
+            using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
+            while (!condition())
+            {
+                await Task.Delay(10, timeout.Token);
+            }
+        }
+
         private static WorkItemSummary WaitingItem(string jobName, string workItemName)
             => new($"{jobName}/{workItemName}", jobName, workItemName, "Waiting");
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj
index 9a9eab6b519..c464b912e7d 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj
@@ -21,7 +21,6 @@
     <ProjectReference Include="..\..\..\Common\Microsoft.Arcade.Common\Microsoft.Arcade.Common.csproj" />
     <ProjectReference Include="..\..\..\Common\Microsoft.Arcade.Test.Common\Microsoft.Arcade.Test.Common.csproj" />
     <ProjectReference Include="..\..\..\Microsoft.DotNet.Internal.DependencyInjection.Testing\Microsoft.DotNet.Internal.DependencyInjection.Testing.csproj" />
-    <ProjectReference Include="..\..\AzureDevOpsTestPublisher\Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj" />
     <ProjectReference Include="..\..\Sdk\Microsoft.DotNet.Helix.Sdk.csproj" />
     <ProjectReference Include="..\..\JobMonitor\Microsoft.DotNet.Helix.JobMonitor.csproj" />
   </ItemGroup>
diff --git a/src/Microsoft.DotNet.Helix/Sdk/Readme.md b/src/Microsoft.DotNet.Helix/Sdk/Readme.md
index d088cb62b8e..dd202f6a40b 100644
--- a/src/Microsoft.DotNet.Helix/Sdk/Readme.md
+++ b/src/Microsoft.DotNet.Helix/Sdk/Readme.md
@@ -81,10 +81,16 @@ Useful parameters:
 - `continueOnError`: allow the pipeline to continue when the monitor job fails. Defaults to `false`.
 - `useFullyQualifiedTestName`: report fully qualified test names to Azure DevOps (see [Fully qualified test names](#fully-qualified-test-names)). Defaults to `false`.
 
+Implementation and semantic design documents are indexed at
+[JobMonitor/Design/README.md](../JobMonitor/Design/README.md).
+
 Behavior notes:
 
 - The reporter uses its own `SYSTEM_ACCESSTOKEN`, so it does not depend on the shorter-lived token from the job that originally submitted the Helix work.
 - If parseable xUnit, JUnit, or TRX result files are available, those are uploaded.
+- Result processing uses globally bounded work-item parallelism and streams XML
+  instead of loading complete result documents. Status polling remains
+  independent from result upload latency.
 - If no result files are found for a work item, no test results are uploaded for that work item; Helix work-item failures still affect the monitor job's final pass/fail status.
 - The reporter is safe to rerun because it checks for already-completed test runs and only processes new results.
 

From 97923353ce032f1dc0af3d930867ae87312dd4fb Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Wed, 12 Aug 2026 07:36:44 -0700
Subject: [PATCH 04/21] Add final drain backlog metrics

Track work items first observed terminal in the final poll, uploads newly eligible at the whole-job boundary, and remaining final-poll versus earlier backlog.

Also expose upload parallelism through the shared Helix Job Monitor pipeline template.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../core-templates/job/helix-job-monitor.yml  |   7 +
 .../JobMonitor/Design/Components/Shutdown.md  |   7 +-
 .../Design/Components/UploadPipeline.md       |   3 +
 .../JobMonitor/JobMonitorRunner.cs            |  32 ++++-
 .../JobMonitor/TestResultUploadPipeline.cs    | 125 +++++++++++++-----
 .../JobMonitorRunnerTests.cs                  |  79 +++++++++++
 6 files changed, 216 insertions(+), 37 deletions(-)

diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml
index 81ecccdd17b..c89205b11b2 100644
--- a/eng/common/core-templates/job/helix-job-monitor.yml
+++ b/eng/common/core-templates/job/helix-job-monitor.yml
@@ -62,6 +62,12 @@ parameters:
   type: number
   default: 30
 
+# Maximum number of work items whose results may be downloaded, parsed, and
+# uploaded concurrently.
+- name: testResultUploadParallelism
+  type: number
+  default: 8
+
 # When 'true' (the default), Helix work items that exit 0 but have failed AzDO test results
 # are treated as failed: they count toward the monitor's exit code and are resubmitted by a
 # later invocation's retry pass. Set to 'false' to fall back to exit-code-only outcomes.
@@ -215,6 +221,7 @@ jobs:
         --max-wait-minutes            "$((${{ parameters.timeoutInMinutes }} - 5))" # Set the tool's timeout slightly lower than the Azure DevOps job timeout to allow it to exit gracefully.
         --stage-name                  '$(System.StageName)'
         --stage-attempt               '$(System.StageAttempt)'
+        --test-result-upload-parallelism '${{ parameters.testResultUploadParallelism }}'
       )
 
       organization='${{ parameters.organization }}'
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Shutdown.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Shutdown.md
index b886aed124e..7ef25ea12eb 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Shutdown.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Shutdown.md
@@ -11,6 +11,12 @@ Work starts as soon as each Helix job completes, so normal drain contains only
 the remaining tail. The drain log records elapsed time and aggregate pipeline
 counts for runtime performance validation.
 
+At drain start, the monitor also records how many work items were first
+observed terminal in the final poll, how many uploads became eligible at the
+whole-job boundary in that poll, and how much remaining work came from the
+final poll versus earlier polls. This distinguishes the unavoidable minimum
+drain from pipeline lag accumulated before Helix completion.
+
 ## Cancellation
 
 Cancellation does not drain uploads. The pipeline worker tokens are canceled
@@ -23,4 +29,3 @@ untagged and are replayed by a later invocation.
 In-memory queue/session state is never required after restart. Completed tags,
 failed-work-item attachments, Helix job properties, and resubmission lineage
 are sufficient to reconstruct all required work.
-
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
index 1d7a0a48453..c00a3b886bd 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
@@ -32,6 +32,9 @@ Each worker:
 
 Work-item concurrency is global. A build with many jobs therefore cannot create
 an unbounded task graph or multiply the configured Azure DevOps pressure.
+Consumers can tune this global budget through the
+`testResultUploadParallelism` pipeline-template parameter, which forwards to
+the monitor's `--test-result-upload-parallelism` option.
 
 ## Finalization
 
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
index 1017ced7346..5cebeebb782 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
@@ -263,6 +263,8 @@ private async Task<int> RunPollLoopAsync(IReadOnlyList<HelixJobInfo> jobsForFirs
             PollLoopState loopState,
             CancellationToken cancellationToken)
         {
+            int pollNumber = ++loopState.PollNumber;
+
             // Fetch fresh snapshots, scoped to the monitor's stage.
             IReadOnlyList<AzureDevOpsTimelineRecord> timelineRecords =
                 HelixJobMonitorUtilities.FilterRecordsToStage(
@@ -300,8 +302,17 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
             ];
             IReadOnlyDictionary<string, IReadOnlyCollection<WorkItemSummary>> refreshedWorkItems =
                 await GetWorkItemsAsync(jobsToRefresh, cancellationToken);
+            int newlyTerminalWorkItems = 0;
             foreach ((string jobName, IReadOnlyCollection<WorkItemSummary> workItems) in refreshedWorkItems)
             {
+                int previousTerminalCount =
+                    loopState.WorkItemsByJob.TryGetValue(jobName, out IReadOnlyCollection<WorkItemSummary> previousWorkItems)
+                        ? previousWorkItems.Count(static item => item.ExitCode.HasValue)
+                        : 0;
+                int currentTerminalCount = workItems.Count(static item => item.ExitCode.HasValue);
+                // A Helix work-item exit code is immutable once assigned, so terminal counts
+                // increase monotonically without retaining or rebuilding per-item identity sets.
+                newlyTerminalWorkItems += Math.Max(0, currentTerminalCount - previousTerminalCount);
                 loopState.WorkItemsByJob[jobName] = workItems;
             }
 
@@ -314,11 +325,14 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
             var completedJobNames = new HashSet<string>(
                 completedJobs.Select(j => j.JobName),
                 StringComparer.OrdinalIgnoreCase);
-
             // First pass: upload + reconcile for any newly-completed jobs.
             foreach (HelixJobInfo job in completedJobs.Where(j => !_state.IsHelixJobProcessed(j.JobName)))
             {
-                ReconcileCompletedJob(job, workItemsByJob[job.JobName], queueUpload: true);
+                ReconcileCompletedJob(
+                    job,
+                    workItemsByJob[job.JobName],
+                    queueUpload: true,
+                    discoveryPoll: pollNumber);
             }
 
             // Second pass: ensure outcomes for every completed job (any attempt) are reflected in
@@ -329,7 +343,11 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
                 MonitorState.GetLatestHelixJobAttempts(stageJobs)
                     .Where(j => completedJobNames.Contains(j.JobName))))
             {
-                ReconcileCompletedJob(job, workItemsByJob[job.JobName], queueUpload: false);
+                ReconcileCompletedJob(
+                    job,
+                    workItemsByJob[job.JobName],
+                    queueUpload: false,
+                    discoveryPoll: pollNumber);
             }
 
             bool shouldLogStatus = _options.Verbose
@@ -357,7 +375,7 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
                 return null;
             }
 
-            await _uploads.DrainAsync(cancellationToken);
+            await _uploads.DrainAsync(pollNumber, newlyTerminalWorkItems, cancellationToken);
             _reporter.LogFinalFailedWorkItems();
             _reporter.LogFinalSummary(_state.AssociatedJobsCount);
 
@@ -384,7 +402,8 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
         private void ReconcileCompletedJob(
             HelixJobInfo helixJob,
             IReadOnlyCollection<WorkItemSummary> workItems,
-            bool queueUpload)
+            bool queueUpload,
+            int discoveryPoll)
         {
             // Already reconciled earlier in this invocation — nothing more to do (idempotent).
             if (_state.IsWorkItemOutcomesRecorded(helixJob.JobName))
@@ -410,7 +429,7 @@ private void ReconcileCompletedJob(
 
             if (queueUpload && !alreadyUploadedByPriorAttempt)
             {
-                _uploads.TryEnqueue(helixJob, workItems);
+                _uploads.TryEnqueue(helixJob, workItems, discoveryPoll);
             }
 
             if (!alreadyUploadedByPriorAttempt)
@@ -538,6 +557,7 @@ private void LogWarning(Exception exception, string message)
         /// </summary>
         private sealed class PollLoopState
         {
+            public int PollNumber { get; set; }
             public int LastObservedJobCount { get; set; } = -1;
             public int LastObservedCompletedCount { get; set; } = -1;
             public DateTime LastStatusLogAt { get; set; } = DateTime.UtcNow;
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
index 8fc8615a32f..980e9556aaa 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
@@ -23,6 +23,7 @@ internal sealed class TestResultUploadPipeline : IAsyncDisposable
     private readonly MonitorState _state;
     private readonly ConcurrentDictionary<string, JobUploadSession> _sessions =
         new(StringComparer.OrdinalIgnoreCase);
+    private readonly ConcurrentDictionary<int, long> _remainingWorkItemsByPoll = [];
     private readonly ActionQueue<JobUploadRequest> _jobs;
     private readonly ActionQueue<WorkItemUploadRequest> _workItems;
     private readonly ActionQueue<JobUploadSession> _finalizers;
@@ -62,22 +63,33 @@ public TestResultUploadPipeline(
         _sessions.Values.Count(static session => session.HasFailed),
         _sessions.Values.Sum(static session => session.UploadedResultCount));
 
-    public bool TryEnqueue(HelixJobInfo job, IReadOnlyCollection<WorkItemSummary> workItems)
+    public bool TryEnqueue(
+        HelixJobInfo job,
+        IReadOnlyCollection<WorkItemSummary> workItems,
+        int discoveryPoll)
     {
         if (Volatile.Read(ref _draining) != 0 || _state.IsHelixJobProcessed(job.JobName))
         {
             return false;
         }
 
-        var session = new JobUploadSession(job, workItems);
+        var session = new JobUploadSession(job, workItems, discoveryPoll);
         if (!_sessions.TryAdd(job.JobName, session))
         {
             return false;
         }
 
+        _remainingWorkItemsByPoll.AddOrUpdate(
+            discoveryPoll,
+            workItems.Count,
+            (_, remaining) => remaining + workItems.Count);
         if (!_jobs.TryEnqueue(new JobUploadRequest(session)))
         {
             _sessions.TryRemove(job.JobName, out _);
+            _remainingWorkItemsByPoll.AddOrUpdate(
+                discoveryPoll,
+                0,
+                (_, remaining) => remaining - workItems.Count);
             return false;
         }
 
@@ -85,7 +97,10 @@ public bool TryEnqueue(HelixJobInfo job, IReadOnlyCollection<WorkItemSummary> wo
         return true;
     }
 
-    public async Task DrainAsync(CancellationToken cancellationToken)
+    public async Task DrainAsync(
+        int finalPoll,
+        int newlyTerminalWorkItems,
+        CancellationToken cancellationToken)
     {
         if (Interlocked.Exchange(ref _draining, 1) != 0)
         {
@@ -93,6 +108,29 @@ public async Task DrainAsync(CancellationToken cancellationToken)
         }
 
         DateTimeOffset startedAt = DateTimeOffset.UtcNow;
+        long finalPollRemainingWorkItems = GetRemainingWorkItems(finalPoll);
+        long priorPollBacklog = _remainingWorkItemsByPoll
+            .Where(pair => pair.Key != finalPoll)
+            .Sum(static pair => Math.Max(0, pair.Value));
+        long remainingWorkItems = finalPollRemainingWorkItems + priorPollBacklog;
+        int finalPollEligibleWorkItems = _sessions.Values
+            .Where(session => session.DiscoveryPoll == finalPoll)
+            .Sum(static session => session.WorkItems.Count);
+        int remainingFinalizations = _sessions.Values.Count(static session => !session.IsFinalized);
+
+        _logger.LogInformation(
+            "Starting final test result drain: {NewlyTerminalWorkItems} work item(s) were first observed "
+            + "terminal and {FinalPollEligibleWorkItems} work item upload(s) became eligible in the final poll; "
+            + "{RemainingWorkItems} work item upload(s) remain "
+            + "({FinalPollRemainingWorkItems} from the final poll, {PriorPollBacklog} from earlier polls), "
+            + "plus {RemainingFinalizations} job finalization(s).",
+            newlyTerminalWorkItems,
+            finalPollEligibleWorkItems,
+            remainingWorkItems,
+            finalPollRemainingWorkItems,
+            priorPollBacklog,
+            remainingFinalizations);
+
         _jobs.Complete();
         await _jobs.DrainAsync().WaitAsync(cancellationToken);
 
@@ -145,6 +183,11 @@ private async ValueTask ExpandJobAsync(JobUploadRequest request, CancellationTok
         }
     }
 
+    private long GetRemainingWorkItems(int discoveryPoll)
+        => _remainingWorkItemsByPoll.TryGetValue(discoveryPoll, out long remaining)
+            ? Math.Max(0, remaining)
+            : 0;
+
     private async ValueTask ProcessWorkItemAsync(
         WorkItemUploadRequest request,
         CancellationToken cancellationToken)
@@ -180,6 +223,10 @@ private async ValueTask ProcessWorkItemAsync(
         }
         finally
         {
+            _remainingWorkItemsByPoll.AddOrUpdate(
+                session.DiscoveryPoll,
+                0,
+                static (_, remaining) => remaining - 1);
             if (session.MarkWorkItemFinished())
             {
                 await _finalizers.EnqueueAsync(session, cancellationToken);
@@ -191,37 +238,44 @@ private async ValueTask FinalizeJobAsync(
         JobUploadSession session,
         CancellationToken cancellationToken)
     {
-        if (session.HasFailed)
-        {
-            _state.MarkHelixJobUploadFailed(session.Job.JobName);
-            return;
-        }
-
         try
         {
-            int testRunId = await session.GetOrCreateTestRunAsync(
-                () => _azdo.CreateTestRunAsync(session.Job.TestRunName, cancellationToken));
-            await _azdo.CompleteTestRunAsync(
-                testRunId,
-                session.Job.JobName,
-                session.FailedWorkItems,
-                cancellationToken);
+            if (session.HasFailed)
+            {
+                _state.MarkHelixJobUploadFailed(session.Job.JobName);
+                return;
+            }
 
-            _state.TryMarkHelixJobProcessed(session.Job.JobName);
-            _logger.LogInformation(
-                "{UploadedCount} test results for job '{JobName}' processed.",
-                session.UploadedResultCount,
-                session.Job.DisplayName);
-        }
-        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
-        {
-            throw;
+            try
+            {
+                int testRunId = await session.GetOrCreateTestRunAsync(
+                    () => _azdo.CreateTestRunAsync(session.Job.TestRunName, cancellationToken));
+                await _azdo.CompleteTestRunAsync(
+                    testRunId,
+                    session.Job.JobName,
+                    session.FailedWorkItems,
+                    cancellationToken);
+
+                _state.TryMarkHelixJobProcessed(session.Job.JobName);
+                _logger.LogInformation(
+                    "{UploadedCount} test results for job '{JobName}' processed.",
+                    session.UploadedResultCount,
+                    session.Job.DisplayName);
+            }
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+            {
+                throw;
+            }
+            catch (Exception ex)
+            {
+                session.RecordFailure();
+                _state.MarkHelixJobUploadFailed(session.Job.JobName);
+                LogUploadFailure(ex, $"complete Azure DevOps test run for job '{session.Job.DisplayName}'");
+            }
         }
-        catch (Exception ex)
+        finally
         {
-            session.RecordFailure();
-            _state.MarkHelixJobUploadFailed(session.Job.JobName);
-            LogUploadFailure(ex, $"complete Azure DevOps test run for job '{session.Job.DisplayName}'");
+            session.MarkFinalized();
         }
     }
 
@@ -289,19 +343,28 @@ private sealed class JobUploadSession
         private readonly HashSet<string> _failedWorkItems = new(StringComparer.OrdinalIgnoreCase);
         private Task<int> _testRunTask;
         private int _finishedWorkItems;
+        private int _finalized;
         private int _failed;
         private long _uploadedResultCount;
 
-        public JobUploadSession(HelixJobInfo job, IReadOnlyCollection<WorkItemSummary> workItems)
+        public JobUploadSession(
+            HelixJobInfo job,
+            IReadOnlyCollection<WorkItemSummary> workItems,
+            int discoveryPoll)
         {
             Job = job;
             WorkItems = [.. workItems];
+            DiscoveryPoll = discoveryPoll;
         }
 
         public HelixJobInfo Job { get; }
 
         public IReadOnlyList<WorkItemSummary> WorkItems { get; }
 
+        public int DiscoveryPoll { get; }
+
+        public bool IsFinalized => Volatile.Read(ref _finalized) != 0;
+
         public bool HasFailed => Volatile.Read(ref _failed) != 0;
 
         public long UploadedResultCount => Interlocked.Read(ref _uploadedResultCount);
@@ -341,6 +404,8 @@ public void RecordSuccess(string workItemName, TestResultUploadSummary summary)
 
         public bool MarkWorkItemFinished()
             => Interlocked.Increment(ref _finishedWorkItems) == WorkItems.Count;
+
+        public void MarkFinalized() => Interlocked.Exchange(ref _finalized, 1);
     }
 }
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
index 72ef0f85793..c40a84bba23 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
@@ -578,11 +578,90 @@ public async Task DrainReportsAggregatePipelineProgress()
             int exitCode = await run.WaitAsync(TimeSpan.FromSeconds(5));
 
             exitCode.Should().Be(0);
+            logger.Messages.Should().Contain(message =>
+                message.Contains(
+                    "Starting final test result drain: 1 work item(s) were first observed terminal "
+                    + "and 1 work item upload(s) became eligible in the final poll; "
+                    + "1 work item upload(s) remain (1 from the final poll, 0 from earlier polls)",
+                    StringComparison.Ordinal));
             logger.Messages.Should().Contain(message =>
                 message.Contains("Test result pipeline drained in", StringComparison.Ordinal)
                 && message.Contains("1 job(s), 1 work item(s), and 1 result(s)", StringComparison.Ordinal));
         }
 
+        [Fact]
+        public async Task DrainSeparatesFinalPollArrivalsFromEarlierBacklog()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+            var logger = new RecordingLogger();
+            var uploadRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+
+            azdo.UploadBlocker = uploadRelease.Task;
+            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "inProgress"));
+            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
+            helix.AddResponse(
+                jobs:
+                [
+                    HelixJob("helix-earlier", "finished"),
+                    HelixJob("helix-final", "running"),
+                ],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-earlier"] = PassFail(passed: ["earlier-workitem"]),
+                },
+                testResultsByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-earlier"] =
+                    [
+                        new WorkItemTestResults("helix-earlier", "earlier-workitem", ["earlier.trx"])
+                    ],
+                });
+            helix.AddResponse(
+                jobs:
+                [
+                    HelixJob("helix-earlier", "finished"),
+                    HelixJob("helix-final", "finished"),
+                ],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-earlier"] = PassFail(passed: ["earlier-workitem"]),
+                    ["helix-final"] = PassFail(passed: ["final-workitem"]),
+                },
+                testResultsByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-earlier"] =
+                    [
+                        new WorkItemTestResults("helix-earlier", "earlier-workitem", ["earlier.trx"])
+                    ],
+                    ["helix-final"] =
+                    [
+                        new WorkItemTestResults("helix-final", "final-workitem", ["final.trx"])
+                    ],
+                });
+
+            var runner = new JobMonitorRunner(
+                DefaultOptions(),
+                logger,
+                azdo,
+                helix,
+                async (_, _) => await azdo.UploadStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)));
+
+            Task<int> run = runner.RunAsync(CancellationToken.None);
+            await WaitForAsync(() => logger.Messages.Any(message =>
+                message.Contains("Starting final test result drain:", StringComparison.Ordinal)));
+
+            logger.Messages.Should().Contain(message =>
+                message.Contains(
+                    "1 work item(s) were first observed terminal and 1 work item upload(s) became eligible "
+                    + "in the final poll; 2 work item upload(s) remain "
+                    + "(1 from the final poll, 1 from earlier polls)",
+                    StringComparison.Ordinal));
+
+            uploadRelease.SetResult();
+            (await run.WaitAsync(TimeSpan.FromSeconds(5))).Should().Be(0);
+        }
+
         [Fact]
         public async Task UploadPipeline_BoundsWorkItemParallelismAndCreatesOneRunPerJob()
         {

From cc23c46fa517aaf6f2a5ac76917abac9a31c5060 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Wed, 12 Aug 2026 07:48:25 -0700
Subject: [PATCH 05/21] Preserve Job Monitor failure scenarios

Cache synchronous test-run creation failures in the per-job single-flight task and add regression coverage for creation, download, partial upload, and failed-test metadata behavior.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../JobMonitor/TestResultUploadPipeline.cs    |  14 ++-
 .../JobMonitorRunnerTests.cs                  | 100 ++++++++++++++++++
 2 files changed, 113 insertions(+), 1 deletion(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
index 980e9556aaa..13ddeb35156 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
@@ -384,7 +384,19 @@ public Task<int> GetOrCreateTestRunAsync(Func<Task<int>> create)
         {
             lock (_sync)
             {
-                return _testRunTask ??= create();
+                return _testRunTask ??= InvokeCreate();
+            }
+
+            Task<int> InvokeCreate()
+            {
+                try
+                {
+                    return create();
+                }
+                catch (Exception ex)
+                {
+                    return Task.FromException<int>(ex);
+                }
             }
         }
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
index c40a84bba23..0ef81b45eaa 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
@@ -783,6 +783,37 @@ public async Task PassedHelixWork_TransientUploadFailure_DoesNotReplayAmbiguousW
                 && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal));
         }
 
+        [Fact]
+        public async Task PassedHelixWork_CreateTestRunFailure_IsSingleFlightAndLeavesJobUntagged()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+            var logger = new RecordingLogger();
+
+            azdo.FailNextCreate();
+            azdo.AddTimelineResponse(
+                MonitorJob(),
+                PipelineJob("Test Linux", "completed", "succeeded"));
+            helix.AddResponse(
+                jobs: [HelixJob("helix-linux", "finished")],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-linux"] = PassFail(passed: ["workitem-1", "workitem-2"]),
+                });
+
+            var runner = CreateRunner(azdo, helix, logger: logger);
+            int exitCode = await runner.RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(0);
+            azdo.CreateTestRunCallCount.Should().Be(1);
+            azdo.UploadTestResultsCallCount.Should().Be(0);
+            azdo.CompleteTestRunCallCount.Should().Be(0);
+            azdo.CompletedTestRunIds.Should().BeEmpty();
+            logger.Messages.Should().Contain(message =>
+                message.Contains("remains untagged", StringComparison.Ordinal)
+                && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal));
+        }
+
         [Fact]
         public async Task PassedHelixWork_TransientCompletionFailure_DoesNotReplayCompletionSequence()
         {
@@ -823,6 +854,69 @@ public async Task PassedHelixWork_TransientCompletionFailure_DoesNotReplayComple
                 && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal));
         }
 
+        [Fact]
+        public async Task PassedHelixWork_PermanentDownloadFailure_DoesNotCreateTestRun()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+            var logger = new RecordingLogger();
+
+            helix.FailDownloadForJob("helix-linux");
+            azdo.AddTimelineResponse(
+                MonitorJob(),
+                PipelineJob("Test Linux", "completed", "succeeded"));
+            helix.AddResponse(
+                jobs: [HelixJob("helix-linux", "finished")],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-linux"] = PassFail(passed: ["workitem-1"]),
+                });
+
+            var runner = CreateRunner(azdo, helix, logger: logger);
+            int exitCode = await runner.RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(0);
+            azdo.CreateTestRunCallCount.Should().Be(0);
+            azdo.UploadTestResultsCallCount.Should().Be(0);
+            azdo.CompleteTestRunCallCount.Should().Be(0);
+            logger.Messages.Should().Contain(message =>
+                message.Contains("remains untagged", StringComparison.Ordinal)
+                && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal));
+        }
+
+        [Fact]
+        public async Task MultipleWorkItems_OneUploadFails_JobRemainsUntagged()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+            var logger = new RecordingLogger();
+
+            azdo.FailNextUpload(new InvalidOperationException("Injected permanent upload failure."));
+            azdo.AddTimelineResponse(
+                MonitorJob(),
+                PipelineJob("Test Linux", "completed", "succeeded"));
+            helix.AddResponse(
+                jobs: [HelixJob("helix-linux", "finished")],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-linux"] = PassFail(passed: ["workitem-1", "workitem-2"]),
+                });
+
+            JobMonitorOptions options = DefaultOptions();
+            options.TestResultUploadParallelism = 1;
+            var runner = new JobMonitorRunner(options, logger, azdo, helix, NoDelay);
+            int exitCode = await runner.RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(0);
+            azdo.CreateTestRunCallCount.Should().Be(1);
+            azdo.UploadTestResultsCallCount.Should().Be(2);
+            azdo.CompleteTestRunCallCount.Should().Be(0);
+            azdo.CompletedTestRunIds.Should().BeEmpty();
+            logger.Messages.Should().Contain(message =>
+                message.Contains("remains untagged", StringComparison.Ordinal)
+                && message.Contains("later monitor invocation can replay it", StringComparison.Ordinal));
+        }
+
         [Fact]
         public async Task PassedHelixWork_PermanentUploadFailure_DoesNotHangOrFailBuild()
         {
@@ -3913,6 +4007,9 @@ public async Task HelixWorkItemPassesByExitCode_TestUploadReportsFailure_ExitOne
             helix.Resubmissions.Should().BeEmpty();
             azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-linux"]);
             azdo.CompletedTestRunIds.Should().ContainSingle();
+            IReadOnlyDictionary<string, IReadOnlySet<string>> failedTestWorkItems =
+                await azdo.GetFailedTestWorkItemsAsync(CancellationToken.None);
+            failedTestWorkItems["helix-linux"].Should().BeEquivalentTo(["workitem-1"]);
         }
 
         /// <summary>
@@ -4076,6 +4173,9 @@ public async Task FailOnFailedTestsDisabled_TestFailuresIgnoredForOutcomeAndRetr
             exitCode.Should().Be(0);
             helix.Resubmissions.Should().BeEmpty();
             azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-linux"]);
+            IReadOnlyDictionary<string, IReadOnlySet<string>> failedTestWorkItems =
+                await azdo.GetFailedTestWorkItemsAsync(CancellationToken.None);
+            failedTestWorkItems["helix-linux"].Should().BeEquivalentTo(["workitem-1"]);
         }
 
         // -----------------------------------------------------------------------

From cd02bea71c8fcdaf478d378654fc69c456ced282 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Wed, 12 Aug 2026 08:02:09 -0700
Subject: [PATCH 06/21] Fix Job Monitor result discovery and status cadence

Recognize Helix result files with a trailing .txt transport suffix, report explicit per-job file and result counts, and emit aggregate status on an independent five-minute timer throughout final drain.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../Design/Components/StateAndStatus.md       | 13 ++--
 .../Design/Components/TestResults.md          |  6 ++
 .../JobMonitor/JobMonitorRunner.cs            | 68 +++++++++++++++----
 .../JobMonitor/TestResultUploadPipeline.cs    | 23 +++++--
 .../TestResults/LocalTestResultsReader.cs     | 12 ++++
 .../HelixServiceTests.cs                      |  6 +-
 .../JobMonitorRunnerTests.cs                  | 66 +++++++++++++++++-
 .../LocalTestResultsReaderTests.cs            | 25 ++++++-
 8 files changed, 188 insertions(+), 31 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/StateAndStatus.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/StateAndStatus.md
index 6c2e73a5739..1f3bde13cfd 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/StateAndStatus.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/StateAndStatus.md
@@ -4,12 +4,17 @@
 only through immutable snapshots. Upload workers update only narrow methods:
 upload lifecycle, uploaded test outcomes, and durable completion.
 
-The reporter consumes the current poll snapshot and atomic upload-pipeline
-counters. It never waits for Helix file access or Azure DevOps result uploads.
+The runner publishes an immutable latest-poll snapshot. A dedicated periodic
+reporting loop consumes that snapshot and the upload pipeline's atomic counters
+every five minutes. Reporting therefore continues at a fixed cadence while the
+poll loop is active and during final drain, without waiting for Helix file
+access or Azure DevOps result uploads.
 
-Normal logging reports semantic transitions and aggregate counts. Verbose
+An initial status is emitted after the first poll. Subsequent status messages
+are timer-driven rather than triggered by job-count changes, so completion
+bursts do not increase log volume. Normal logging reports semantic transitions
+and aggregate counts. Verbose
 logging adds queue depth, active worker counts, finalizer depth, and uploaded
 result totals. It deliberately does not print every job, work item, file, or
 request; verbose mode must remain usable on runs containing thousands of work
 items and millions of results.
-
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md
index a7cf27e5e75..1edfa3bcfae 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md
@@ -14,6 +14,12 @@ refer to definitions by ID and may precede them in the file.
 Malformed files are warned and omitted. Caller cancellation is propagated.
 DTD processing is prohibited.
 
+Helix may append `.txt` to uploaded result artifacts, for example
+`testResults.xml.txt` or `results.trx.txt`. File recognition strips that single
+transport suffix before matching the supported xUnit, JUnit, and TRX names.
+Recognized files whose XML root is unsupported are warned rather than silently
+producing zero results.
+
 ## Aggregation
 
 Existing single, data-driven, and rerun semantics are retained, including
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
index 5cebeebb782..8f8c8a10c61 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
@@ -30,11 +30,13 @@ internal sealed class JobMonitorRunner : IJobMonitorRunner, IDisposable
         private readonly IAzureDevOpsService _azdo;
         private readonly IHelixService _helix;
         private readonly Func<TimeSpan, CancellationToken, Task> _delayFunc;
+        private readonly Func<TimeSpan, CancellationToken, Task> _statusDelayFunc;
         private readonly string _helixSource;
 
         private readonly MonitorState _state = new();
         private readonly StatusReporter _reporter;
         private readonly TestResultUploadPipeline _uploads;
+        private PollStatusSnapshot _latestStatus;
 
         /// <summary>
         /// Constructor for production use with real services.
@@ -59,13 +61,15 @@ internal JobMonitorRunner(
             ILogger logger,
             IAzureDevOpsService azdo,
             IHelixService helix,
-            Func<TimeSpan, CancellationToken, Task> delayFunc)
+            Func<TimeSpan, CancellationToken, Task> delayFunc,
+            Func<TimeSpan, CancellationToken, Task> statusDelayFunc = null)
         {
             _options = options ?? throw new ArgumentNullException(nameof(options));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
             _azdo = azdo ?? throw new ArgumentNullException(nameof(azdo));
             _helix = helix ?? throw new ArgumentNullException(nameof(helix));
             _delayFunc = delayFunc ?? Task.Delay;
+            _statusDelayFunc = statusDelayFunc ?? Task.Delay;
             Directory.CreateDirectory(_options.WorkingDirectory);
 
             _helixSource = HelixJobSource.Compute(
@@ -85,6 +89,8 @@ public async Task<int> RunAsync(CancellationToken cancellationToken)
 
             _state.AddProcessedHelixJobs(await _azdo.GetProcessedHelixJobNamesAsync(cancellationToken));
 
+            using var statusCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+            Task statusTask = ReportStatusPeriodicallyAsync(statusCts.Token);
             try
             {
                 IReadOnlyList<HelixJobInfo> jobsForFirstPoll = await ExecuteRetryPassAsync(cancellationToken);
@@ -114,6 +120,17 @@ public async Task<int> RunAsync(CancellationToken cancellationToken)
 
                 return 1;
             }
+            finally
+            {
+                statusCts.Cancel();
+                try
+                {
+                    await statusTask;
+                }
+                catch (OperationCanceledException) when (statusCts.IsCancellationRequested)
+                {
+                }
+            }
         }
 
         /// <summary>
@@ -350,17 +367,13 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
                     discoveryPoll: pollNumber);
             }
 
-            bool shouldLogStatus = _options.Verbose
-                || loopState.LastObservedJobCount != stageJobs.Count
-                || loopState.LastObservedCompletedCount != completedJobs.Count
-                || (DateTime.UtcNow - loopState.LastStatusLogAt) >= TimeSpan.FromMinutes(5);
-
-            if (shouldLogStatus)
+            Volatile.Write(
+                ref _latestStatus,
+                new PollStatusSnapshot(stageJobs, workItemsByJob, completedJobNames));
+            if (!loopState.HasLoggedInitialStatus)
             {
-                _reporter.LogPollStatus(stageJobs, workItemsByJob, completedJobNames, _uploads.Snapshot);
-                loopState.LastObservedJobCount = stageJobs.Count;
-                loopState.LastObservedCompletedCount = completedJobNames.Count;
-                loopState.LastStatusLogAt = DateTime.UtcNow;
+                LogLatestStatus();
+                loopState.HasLoggedInitialStatus = true;
             }
 
             bool anyNonMonitorFailure = HelixJobMonitorUtilities.HasFailedNonMonitorJobs(
@@ -545,6 +558,30 @@ public void Dispose()
         private Task Delay(CancellationToken cancellationToken)
             => _delayFunc(TimeSpan.FromSeconds(Math.Max(5, _options.PollingIntervalSeconds)), cancellationToken);
 
+        private async Task ReportStatusPeriodicallyAsync(CancellationToken cancellationToken)
+        {
+            while (true)
+            {
+                await _statusDelayFunc(TimeSpan.FromMinutes(5), cancellationToken);
+                LogLatestStatus();
+            }
+        }
+
+        private void LogLatestStatus()
+        {
+            PollStatusSnapshot snapshot = Volatile.Read(ref _latestStatus);
+            if (snapshot is null)
+            {
+                return;
+            }
+
+            _reporter.LogPollStatus(
+                snapshot.Jobs,
+                snapshot.WorkItemsByJob,
+                snapshot.CompletedJobNames,
+                _uploads.Snapshot);
+        }
+
         private void LogWarning(string message)
             => _logger.LogWarning("{Prefix}{Message}", AzdoWarningPrefix, message);
 
@@ -558,11 +595,14 @@ private void LogWarning(Exception exception, string message)
         private sealed class PollLoopState
         {
             public int PollNumber { get; set; }
-            public int LastObservedJobCount { get; set; } = -1;
-            public int LastObservedCompletedCount { get; set; } = -1;
-            public DateTime LastStatusLogAt { get; set; } = DateTime.UtcNow;
+            public bool HasLoggedInitialStatus { get; set; }
             public Dictionary<string, IReadOnlyCollection<WorkItemSummary>> WorkItemsByJob { get; } =
                 new(StringComparer.OrdinalIgnoreCase);
         }
+
+        private sealed record PollStatusSnapshot(
+            IReadOnlyList<HelixJobInfo> Jobs,
+            IReadOnlyDictionary<string, IReadOnlyCollection<WorkItemSummary>> WorkItemsByJob,
+            IReadOnlySet<string> CompletedJobNames);
     }
 }
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
index 13ddeb35156..3a814496114 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
@@ -204,7 +204,10 @@ private async ValueTask ProcessWorkItemAsync(
             TestResultUploadSummary summary =
                 await _azdo.UploadTestResultsAsync(testRunId, downloaded, cancellationToken);
 
-            session.RecordSuccess(request.WorkItemName, summary);
+            session.RecordSuccess(
+                request.WorkItemName,
+                downloaded.TestResultFiles.Count,
+                summary);
             if (_options.FailWorkItemsWithFailedTests)
             {
                 _state.ObserveTestResult(session.Job.JobName, request.WorkItemName, summary);
@@ -258,9 +261,12 @@ await _azdo.CompleteTestRunAsync(
 
                 _state.TryMarkHelixJobProcessed(session.Job.JobName);
                 _logger.LogInformation(
-                    "{UploadedCount} test results for job '{JobName}' processed.",
-                    session.UploadedResultCount,
-                    session.Job.DisplayName);
+                    "Test result processing completed for job '{JobName}': {WorkItemCount} work item(s), "
+                    + "{ResultFileCount} recognized result file(s), and {UploadedCount} test result(s) uploaded.",
+                    session.Job.DisplayName,
+                    session.WorkItems.Count,
+                    session.ResultFileCount,
+                    session.UploadedResultCount);
             }
             catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
             {
@@ -345,6 +351,7 @@ private sealed class JobUploadSession
         private int _finishedWorkItems;
         private int _finalized;
         private int _failed;
+        private long _resultFileCount;
         private long _uploadedResultCount;
 
         public JobUploadSession(
@@ -369,6 +376,8 @@ public JobUploadSession(
 
         public long UploadedResultCount => Interlocked.Read(ref _uploadedResultCount);
 
+        public long ResultFileCount => Interlocked.Read(ref _resultFileCount);
+
         public IReadOnlyCollection<string> FailedWorkItems
         {
             get
@@ -400,8 +409,12 @@ Task<int> InvokeCreate()
             }
         }
 
-        public void RecordSuccess(string workItemName, TestResultUploadSummary summary)
+        public void RecordSuccess(
+            string workItemName,
+            int resultFileCount,
+            TestResultUploadSummary summary)
         {
+            Interlocked.Add(ref _resultFileCount, resultFileCount);
             Interlocked.Add(ref _uploadedResultCount, summary.UploadedCount);
             if (!summary.AllPassed)
             {
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs
index e6a0f621b0b..1d32642e9e8 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs
@@ -20,6 +20,11 @@ public sealed class LocalTestResultsReader(
     public static bool LooksLikeTestResultFile(string path)
     {
         string fileName = Path.GetFileName(path);
+        if (fileName.EndsWith(".txt", StringComparison.OrdinalIgnoreCase))
+        {
+            fileName = fileName[..^4];
+        }
+
         return fileName.EndsWith(".trx", StringComparison.OrdinalIgnoreCase)
             || fileName.EndsWith("testResults.xml", StringComparison.OrdinalIgnoreCase)
             || fileName.EndsWith("test-results.xml", StringComparison.OrdinalIgnoreCase)
@@ -98,6 +103,13 @@ private async IAsyncEnumerable<TestResult> ReadResultsAsync(
                     yield return result;
                 }
                 break;
+
+            default:
+                _logger.LogWarning(
+                    "Test result file '{Path}' has unsupported root element '{RootElement}' and will be skipped.",
+                    filePath,
+                    rootName);
+                break;
         }
     }
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
index 12c5f47d850..7a5917eac1c 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
@@ -96,7 +96,7 @@ public async Task DownloadTestResultsAsync_FiltersFilesUsesFileSystemAndContinue
                 .Setup(w => w.ListFilesAsync("work:item", "job:name", false, It.IsAny<CancellationToken>()))
                 .ReturnsAsync(ImmutableList.Create(
                     new UploadedFile("logs/console.txt", "https://storage/logs/console.txt"),
-                    new UploadedFile("nested/testResults.xml", "https://storage/nested/testResults.xml"),
+                    new UploadedFile("nested/testResults.xml.txt", "https://storage/nested/testResults.xml.txt"),
                     new UploadedFile("failed.trx", "https://storage/failed.trx")));
             api.WorkItem
                 .Setup(w => w.ListFilesAsync("no-results", "job:name", false, It.IsAny<CancellationToken>()))
@@ -112,13 +112,13 @@ public async Task DownloadTestResultsAsync_FiltersFilesUsesFileSystemAndContinue
             Assert.Equal("work:item", result.WorkItemName);
             string jobDirectory = fileSystem.PathCombine("work", SanitizeForCurrentPlatform("job:name"));
             string workItemDirectory = fileSystem.PathCombine(jobDirectory, SanitizeForCurrentPlatform("work:item"));
-            string expectedResultFile = fileSystem.PathCombine(workItemDirectory, NormalizeForCurrentPlatform("nested/testResults.xml"));
+            string expectedResultFile = fileSystem.PathCombine(workItemDirectory, NormalizeForCurrentPlatform("nested/testResults.xml.txt"));
             Assert.Equal([expectedResultFile], result.TestResultFiles);
             Assert.Contains(jobDirectory, fileSystem.Directories);
             Assert.Contains(workItemDirectory, fileSystem.Directories);
             Assert.Contains(fileSystem.GetDirectoryName(expectedResultFile), fileSystem.Directories);
             Assert.Equal(
-                [new DownloadCall("https://storage/nested/testResults.xml", "?resultSas", expectedResultFile),
+                [new DownloadCall("https://storage/nested/testResults.xml.txt", "?resultSas", expectedResultFile),
                  new DownloadCall("https://storage/failed.trx", "?resultSas", fileSystem.PathCombine(workItemDirectory, "failed.trx"))],
                 blobClientFactory.Downloads);
         }
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
index 0ef81b45eaa..63f0ffa1267 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
@@ -536,7 +537,10 @@ public async Task CompletedHelixJob_QueuesTestResultUploadWithoutBlockingNextPol
             azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-linux"]);
             azdo.CompletedTestRunIds.Should().ContainSingle();
             logger.Messages.Should().Contain(message =>
-                message.Contains("2 test results for job 'helix-linux' processed.", StringComparison.Ordinal));
+                message.Contains(
+                    "Test result processing completed for job 'helix-linux': "
+                    + "1 work item(s), 2 recognized result file(s), and 2 test result(s) uploaded.",
+                    StringComparison.Ordinal));
         }
 
         [Fact]
@@ -662,6 +666,62 @@ await WaitForAsync(() => logger.Messages.Any(message =>
             (await run.WaitAsync(TimeSpan.FromSeconds(5))).Should().Be(0);
         }
 
+        [Fact]
+        public async Task StatusReportsEveryFiveMinutesWhileDrainIsBlocked()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+            var logger = new RecordingLogger();
+            var uploadRelease = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+            var statusTick = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+            int statusDelayCount = 0;
+
+            azdo.UploadBlocker = uploadRelease.Task;
+            azdo.AddTimelineResponse(
+                MonitorJob(),
+                PipelineJob("Test Linux", "completed", "succeeded"));
+            helix.AddResponse(
+                jobs: [HelixJob("helix-linux", "finished")],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-linux"] = PassFail(passed: ["workitem-1"]),
+                });
+
+            var runner = new JobMonitorRunner(
+                DefaultOptions(),
+                logger,
+                azdo,
+                helix,
+                NoDelay,
+                async (delay, cancellationToken) =>
+                {
+                    delay.Should().Be(TimeSpan.FromMinutes(5));
+                    if (Interlocked.Increment(ref statusDelayCount) == 1)
+                    {
+                        await statusTick.Task.WaitAsync(cancellationToken);
+                    }
+                    else
+                    {
+                        await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
+                    }
+                });
+
+            Task<int> run = runner.RunAsync(CancellationToken.None);
+            await WaitForAsync(() => logger.Messages.Any(message =>
+                message.Contains("Starting final test result drain:", StringComparison.Ordinal)));
+            logger.Messages.Count(IsStatusMessage).Should().Be(1);
+
+            statusTick.SetResult();
+            await WaitForAsync(() => logger.Messages.Count(IsStatusMessage) == 2);
+            logger.Messages.Last(IsStatusMessage).Should().Contain("1 completed");
+
+            uploadRelease.SetResult();
+            (await run.WaitAsync(TimeSpan.FromSeconds(5))).Should().Be(0);
+
+            static bool IsStatusMessage(string message)
+                => message.Contains("ℹ️ Status:", StringComparison.Ordinal);
+        }
+
         [Fact]
         public async Task UploadPipeline_BoundsWorkItemParallelismAndCreatesOneRunPerJob()
         {
@@ -4270,7 +4330,7 @@ private static JobMonitorRunner CreateRunner(
 
         private sealed class RecordingLogger : ILogger
         {
-            public List<string> Messages { get; } = [];
+            public ConcurrentQueue<string> Messages { get; } = [];
 
             public IDisposable BeginScope<TState>(TState state) => NullScope.Instance;
 
@@ -4283,7 +4343,7 @@ public void Log<TState>(
                 Exception exception,
                 Func<TState, Exception, string> formatter)
             {
-                Messages.Add(formatter(state, exception));
+                Messages.Enqueue(formatter(state, exception));
             }
 
             private sealed class NullScope : IDisposable
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs
index babcf1d287c..ed52e6a3124 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs
@@ -15,6 +15,27 @@ namespace Microsoft.DotNet.Helix.Sdk.Tests
 {
     public class LocalTestResultsReaderTests
     {
+        [Theory]
+        [InlineData("results.trx")]
+        [InlineData("results.trx.txt")]
+        [InlineData("testResults.xml")]
+        [InlineData("testResults.xml.txt")]
+        [InlineData("junit-results.xml")]
+        [InlineData("junit-results.xml.txt")]
+        public void LooksLikeTestResultFile_RecognizesHelixTextSuffix(string fileName)
+        {
+            Assert.True(LocalTestResultsReader.LooksLikeTestResultFile(fileName));
+        }
+
+        [Theory]
+        [InlineData("results.xml")]
+        [InlineData("results.txt")]
+        [InlineData("testResults.xml.log")]
+        public void LooksLikeTestResultFile_RejectsUnknownNames(string fileName)
+        {
+            Assert.False(LocalTestResultsReader.LooksLikeTestResultFile(fileName));
+        }
+
         public static IEnumerable<object[]> AttachmentModeCases()
         {
             foreach (string format in new[] { "xunit", "junit", "trx" })
@@ -96,7 +117,7 @@ public async Task LocalTestResultsReader_ReadsXunitFileFromDownloadedResults()
             try
             {
                 File.WriteAllText(
-                    Path.Combine(workItemDirectory, "testResults.xml"),
+                    Path.Combine(workItemDirectory, "testResults.xml.txt"),
                     """
                     <assemblies>
                       <assembly name="Sample.Tests.dll" total="1" passed="1" failed="0" skipped="0">
@@ -108,7 +129,7 @@ public async Task LocalTestResultsReader_ReadsXunitFileFromDownloadedResults()
                     """);
 
                 var reader = new LocalTestResultsReader(NullLoggerFactory.Instance.CreateLogger<LocalTestResultsReader>());
-                string filePath = Path.Combine(workItemDirectory, "testResults.xml");
+                string filePath = Path.Combine(workItemDirectory, "testResults.xml.txt");
                 IReadOnlyList<TestResult> resultSets = await reader.ReadResultFileAsync(filePath);
                 IReadOnlyList<AggregatedResult> aggregate = new ResultAggregator().Aggregate([resultSets]);
                 AggregatedResult result = Assert.Single(aggregate);

From 714d09f5f67a7045f9667c5ff6c50fa9389811fc Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Wed, 12 Aug 2026 12:31:01 -0700
Subject: [PATCH 07/21] Add Job Monitor performance metrics

Report request counts, retries, payload volume, latency, rate-limit behavior, pipeline-active throughput, and aggregate/max stage timings without per-request information logging.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../JobMonitor/Design/Architecture.md         |   5 +
 .../Design/Components/PerformanceMetrics.md   |  44 +++
 .../JobMonitor/Design/README.md               |   3 +-
 .../JobMonitor/JobMonitorMetrics.cs           | 310 ++++++++++++++++++
 .../JobMonitor/JobMonitorRunner.cs            |  62 +++-
 .../Services/AzureDevOpsRateLimitGate.cs      |  33 +-
 .../JobMonitor/Services/AzureDevOpsService.cs |  77 ++++-
 .../JobMonitor/Services/HelixService.cs       |  19 +-
 .../JobMonitor/StatusReporter.cs              |  81 +++++
 .../JobMonitor/TestResultUploadPipeline.cs    |  65 +++-
 .../TestResults/AzureDevOpsResultPublisher.cs | 106 ++++--
 .../AzureDevOpsResultPublisherTests.cs        |  62 +++-
 .../JobMonitorRunnerTests.cs                  |   4 +
 13 files changed, 789 insertions(+), 82 deletions(-)
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/PerformanceMetrics.md
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorMetrics.cs

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md
index c26add45b94..119901c5547 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md
@@ -60,6 +60,11 @@ work-item queue, and work-item workers may wait on Azure DevOps throttling.
 Completed-job acceptance remains non-blocking, so neither condition blocks the
 poller or status reporter.
 
+`JobMonitorMetrics` is shared by the runner, services, publisher, rate-limit
+gate, and upload pipeline. It records atomic request counts and operation
+timings without emitting per-request information logs. The final aggregate
+report is described in [Performance metrics](Components/PerformanceMetrics.md).
+
 ## Durability boundary
 
 The only durable "processed" marker is the Helix-job tag on a completed Azure
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/PerformanceMetrics.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/PerformanceMetrics.md
new file mode 100644
index 00000000000..27b105ca129
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/PerformanceMetrics.md
@@ -0,0 +1,44 @@
+# Performance metrics
+
+The monitor emits one aggregate performance block at normal completion or
+timeout. Metrics are recorded with atomic counters and stopwatch timestamps;
+request-level logging is not required and normal log volume is independent of
+the number of requests.
+
+## Remote operations
+
+Azure DevOps metrics count every HTTP attempt, separated into control-plane,
+test-result batch, and attachment requests. The report includes retries,
+failed attempts, serialized request payload bytes, aggregate request time, and
+the slowest request. A retried request contributes one attempt and its payload
+bytes for each transmission.
+
+Helix metrics count attempts made through the shared retry wrapper. Result blob
+downloads are reported separately because they use Azure Storage rather than
+the Helix API.
+
+Rate-limit metrics distinguish server-directed deferrals from actual shared
+gate waits. Deferrals report the delay guidance received from Azure DevOps;
+gate waits report aggregate worker wait time and the longest individual wait.
+Aggregate wait can exceed wall-clock time when several workers are delayed
+concurrently.
+
+## Pipeline timings
+
+The report includes aggregate worker time and maximum single-operation time for:
+
+- work-item result discovery and download;
+- XML parsing and aggregation;
+- Azure DevOps publication;
+- test-run creation;
+- test-run completion and durable tagging.
+
+Upload throughput uses the interval from the first pipeline operation to the
+last completed operation, not the full monitor lifetime spent waiting for
+Helix work. Parsing is measured separately from Azure DevOps publication.
+
+Aggregate worker time can exceed monitor elapsed time because work executes in
+parallel. Comparing aggregate time, maximum latency, request counts, and
+throughput identifies whether a run is limited by result download, local
+processing, Azure DevOps request latency, attachments, throttling, or final
+test-run operations.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md
index 6850eccd16b..5f888c4e805 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md
@@ -19,6 +19,8 @@ The design is split by concern:
   aggregation, batching, attachments, and Azure DevOps limits.
 - [State and status](Components/StateAndStatus.md) describes thread-safe state,
   snapshots, progress, and bounded verbose logging.
+- [Performance metrics](Components/PerformanceMetrics.md) describes request
+  counts, retries, throughput, rate-limit waits, and pipeline stage timings.
 - [Shutdown](Components/Shutdown.md) describes normal drain, cancellation, and
   crash recovery.
 
@@ -37,4 +39,3 @@ and millions of test results.
 6. Normal drain should contain only the upload tail that could not overlap
    polling. Runtime validation is used to tune the default parallelism and
    verify that the tail remains a small fraction of the monitor duration.
-
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorMetrics.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorMetrics.cs
new file mode 100644
index 00000000000..5a116e90c44
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorMetrics.cs
@@ -0,0 +1,310 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+
+namespace Microsoft.DotNet.Helix.JobMonitor;
+
+internal enum AzureDevOpsRequestKind
+{
+    Control,
+    ResultBatch,
+    Attachment,
+}
+
+internal enum PipelineOperation
+{
+    WorkItemDownload,
+    WorkItemPublish,
+    TestRunCreate,
+    TestRunComplete,
+    ResultParseAndAggregate,
+}
+
+internal sealed class JobMonitorMetrics
+{
+    private readonly long _startedAt = Stopwatch.GetTimestamp();
+    private long _pipelineStartedAt;
+    private long _pipelineFinishedAt;
+    private long _azdoControlRequests;
+    private long _azdoResultRequests;
+    private long _azdoAttachmentRequests;
+    private long _azdoRetries;
+    private long _azdoFailedAttempts;
+    private long _azdoPayloadBytes;
+    private long _azdoRequestTicks;
+    private long _azdoMaximumRequestTicks;
+    private long _helixRequests;
+    private long _helixRetries;
+    private long _helixFailedAttempts;
+    private long _resultBlobDownloads;
+    private long _resultBlobDownloadFailures;
+    private long _rateLimitWaits;
+    private long _rateLimitWaitTicks;
+    private long _maximumRateLimitWaitTicks;
+    private long _rateLimitDeferrals;
+    private long _rateLimitDeferredTicks;
+    private long _maximumRateLimitDeferralTicks;
+    private long _workItemDownloads;
+    private long _workItemDownloadTicks;
+    private long _maximumWorkItemDownloadTicks;
+    private long _workItemPublishes;
+    private long _workItemPublishTicks;
+    private long _maximumWorkItemPublishTicks;
+    private long _testRunCreates;
+    private long _testRunCreateTicks;
+    private long _maximumTestRunCreateTicks;
+    private long _testRunCompletes;
+    private long _testRunCompleteTicks;
+    private long _maximumTestRunCompleteTicks;
+    private long _parseOperations;
+    private long _parseTicks;
+    private long _maximumParseTicks;
+
+    public static long StartOperation() => Stopwatch.GetTimestamp();
+
+    public void RecordAzureDevOpsRequest(
+        AzureDevOpsRequestKind kind,
+        int payloadBytes,
+        bool isRetry,
+        bool failed,
+        long startedAt)
+    {
+        switch (kind)
+        {
+            case AzureDevOpsRequestKind.Control:
+                Interlocked.Increment(ref _azdoControlRequests);
+                break;
+            case AzureDevOpsRequestKind.ResultBatch:
+                Interlocked.Increment(ref _azdoResultRequests);
+                break;
+            case AzureDevOpsRequestKind.Attachment:
+                Interlocked.Increment(ref _azdoAttachmentRequests);
+                break;
+        }
+
+        if (isRetry)
+        {
+            Interlocked.Increment(ref _azdoRetries);
+        }
+        if (failed)
+        {
+            Interlocked.Increment(ref _azdoFailedAttempts);
+        }
+
+        Interlocked.Add(ref _azdoPayloadBytes, payloadBytes);
+        RecordElapsed(ref _azdoRequestTicks, ref _azdoMaximumRequestTicks, startedAt);
+    }
+
+    public void RecordHelixRequest(bool isRetry, bool failed)
+    {
+        Interlocked.Increment(ref _helixRequests);
+        if (isRetry)
+        {
+            Interlocked.Increment(ref _helixRetries);
+        }
+        if (failed)
+        {
+            Interlocked.Increment(ref _helixFailedAttempts);
+        }
+    }
+
+    public void RecordResultBlobDownload(bool failed)
+    {
+        Interlocked.Increment(ref _resultBlobDownloads);
+        if (failed)
+        {
+            Interlocked.Increment(ref _resultBlobDownloadFailures);
+        }
+    }
+
+    public void RecordRateLimitWait(long startedAt)
+    {
+        Interlocked.Increment(ref _rateLimitWaits);
+        RecordElapsed(ref _rateLimitWaitTicks, ref _maximumRateLimitWaitTicks, startedAt);
+    }
+
+    public void RecordRateLimitDeferral(TimeSpan delay)
+    {
+        Interlocked.Increment(ref _rateLimitDeferrals);
+        long delayTicks = (long)(delay.TotalSeconds * Stopwatch.Frequency);
+        Interlocked.Add(ref _rateLimitDeferredTicks, delayTicks);
+
+        long observed;
+        while (delayTicks > (observed = Interlocked.Read(ref _maximumRateLimitDeferralTicks)))
+        {
+            if (Interlocked.CompareExchange(
+                ref _maximumRateLimitDeferralTicks,
+                delayTicks,
+                observed) == observed)
+            {
+                break;
+            }
+        }
+    }
+
+    public void RecordPipelineOperation(PipelineOperation operation, long startedAt)
+    {
+        RecordPipelineActivity(startedAt);
+        switch (operation)
+        {
+            case PipelineOperation.WorkItemDownload:
+                Interlocked.Increment(ref _workItemDownloads);
+                RecordElapsed(ref _workItemDownloadTicks, ref _maximumWorkItemDownloadTicks, startedAt);
+                break;
+            case PipelineOperation.WorkItemPublish:
+                Interlocked.Increment(ref _workItemPublishes);
+                RecordElapsed(ref _workItemPublishTicks, ref _maximumWorkItemPublishTicks, startedAt);
+                break;
+            case PipelineOperation.TestRunCreate:
+                Interlocked.Increment(ref _testRunCreates);
+                RecordElapsed(ref _testRunCreateTicks, ref _maximumTestRunCreateTicks, startedAt);
+                break;
+            case PipelineOperation.TestRunComplete:
+                Interlocked.Increment(ref _testRunCompletes);
+                RecordElapsed(ref _testRunCompleteTicks, ref _maximumTestRunCompleteTicks, startedAt);
+                break;
+            case PipelineOperation.ResultParseAndAggregate:
+                Interlocked.Increment(ref _parseOperations);
+                RecordElapsed(ref _parseTicks, ref _maximumParseTicks, startedAt);
+                break;
+        }
+    }
+
+    public JobMonitorMetricsSnapshot Snapshot()
+    {
+        long controlRequests = Interlocked.Read(ref _azdoControlRequests);
+        long resultRequests = Interlocked.Read(ref _azdoResultRequests);
+        long attachmentRequests = Interlocked.Read(ref _azdoAttachmentRequests);
+        return new JobMonitorMetricsSnapshot(
+            Elapsed: Stopwatch.GetElapsedTime(_startedAt),
+            PipelineElapsed: GetPipelineElapsed(),
+            AzureDevOpsRequests: controlRequests + resultRequests + attachmentRequests,
+            AzureDevOpsControlRequests: controlRequests,
+            AzureDevOpsResultRequests: resultRequests,
+            AzureDevOpsAttachmentRequests: attachmentRequests,
+            AzureDevOpsRetries: Interlocked.Read(ref _azdoRetries),
+            AzureDevOpsFailedAttempts: Interlocked.Read(ref _azdoFailedAttempts),
+            AzureDevOpsPayloadBytes: Interlocked.Read(ref _azdoPayloadBytes),
+            AzureDevOpsRequestTime: GetElapsed(_azdoRequestTicks),
+            MaximumAzureDevOpsRequestTime: GetElapsed(_azdoMaximumRequestTicks),
+            HelixRequests: Interlocked.Read(ref _helixRequests),
+            HelixRetries: Interlocked.Read(ref _helixRetries),
+            HelixFailedAttempts: Interlocked.Read(ref _helixFailedAttempts),
+            ResultBlobDownloads: Interlocked.Read(ref _resultBlobDownloads),
+            ResultBlobDownloadFailures: Interlocked.Read(ref _resultBlobDownloadFailures),
+            RateLimitWaits: Interlocked.Read(ref _rateLimitWaits),
+            RateLimitWaitTime: GetElapsed(_rateLimitWaitTicks),
+            MaximumRateLimitWaitTime: GetElapsed(_maximumRateLimitWaitTicks),
+            RateLimitDeferrals: Interlocked.Read(ref _rateLimitDeferrals),
+            RateLimitDeferredTime: GetElapsed(_rateLimitDeferredTicks),
+            MaximumRateLimitDeferral: GetElapsed(_maximumRateLimitDeferralTicks),
+            WorkItemDownloads: Interlocked.Read(ref _workItemDownloads),
+            WorkItemDownloadTime: GetElapsed(_workItemDownloadTicks),
+            MaximumWorkItemDownloadTime: GetElapsed(_maximumWorkItemDownloadTicks),
+            WorkItemPublishes: Interlocked.Read(ref _workItemPublishes),
+            WorkItemPublishTime: GetElapsed(_workItemPublishTicks),
+            MaximumWorkItemPublishTime: GetElapsed(_maximumWorkItemPublishTicks),
+            TestRunCreates: Interlocked.Read(ref _testRunCreates),
+            TestRunCreateTime: GetElapsed(_testRunCreateTicks),
+            MaximumTestRunCreateTime: GetElapsed(_maximumTestRunCreateTicks),
+            TestRunCompletes: Interlocked.Read(ref _testRunCompletes),
+            TestRunCompleteTime: GetElapsed(_testRunCompleteTicks),
+            MaximumTestRunCompleteTime: GetElapsed(_maximumTestRunCompleteTicks),
+            ParseOperations: Interlocked.Read(ref _parseOperations),
+            ParseTime: GetElapsed(_parseTicks),
+            MaximumParseTime: GetElapsed(_maximumParseTicks));
+    }
+
+    private static void RecordElapsed(ref long totalTicks, ref long maximumTicks, long startedAt)
+    {
+        long elapsedTicks = Stopwatch.GetTimestamp() - startedAt;
+        Interlocked.Add(ref totalTicks, elapsedTicks);
+
+        long observed;
+        while (elapsedTicks > (observed = Interlocked.Read(ref maximumTicks)))
+        {
+            if (Interlocked.CompareExchange(ref maximumTicks, elapsedTicks, observed) == observed)
+            {
+                break;
+            }
+        }
+    }
+
+    private static TimeSpan GetElapsed(long stopwatchTicks)
+        => TimeSpan.FromSeconds((double)stopwatchTicks / Stopwatch.Frequency);
+
+    private void RecordPipelineActivity(long startedAt)
+    {
+        long observedStart;
+        while ((observedStart = Interlocked.Read(ref _pipelineStartedAt)) == 0
+            || startedAt < observedStart)
+        {
+            if (Interlocked.CompareExchange(ref _pipelineStartedAt, startedAt, observedStart) == observedStart)
+            {
+                break;
+            }
+        }
+
+        long finishedAt = Stopwatch.GetTimestamp();
+        long observedFinish;
+        while (finishedAt > (observedFinish = Interlocked.Read(ref _pipelineFinishedAt)))
+        {
+            if (Interlocked.CompareExchange(ref _pipelineFinishedAt, finishedAt, observedFinish) == observedFinish)
+            {
+                break;
+            }
+        }
+    }
+
+    private TimeSpan GetPipelineElapsed()
+    {
+        long startedAt = Interlocked.Read(ref _pipelineStartedAt);
+        if (startedAt == 0)
+        {
+            return TimeSpan.Zero;
+        }
+
+        long finishedAt = Math.Max(startedAt, Interlocked.Read(ref _pipelineFinishedAt));
+        return TimeSpan.FromSeconds((double)(finishedAt - startedAt) / Stopwatch.Frequency);
+    }
+}
+
+internal readonly record struct JobMonitorMetricsSnapshot(
+    TimeSpan Elapsed,
+    TimeSpan PipelineElapsed,
+    long AzureDevOpsRequests,
+    long AzureDevOpsControlRequests,
+    long AzureDevOpsResultRequests,
+    long AzureDevOpsAttachmentRequests,
+    long AzureDevOpsRetries,
+    long AzureDevOpsFailedAttempts,
+    long AzureDevOpsPayloadBytes,
+    TimeSpan AzureDevOpsRequestTime,
+    TimeSpan MaximumAzureDevOpsRequestTime,
+    long HelixRequests,
+    long HelixRetries,
+    long HelixFailedAttempts,
+    long ResultBlobDownloads,
+    long ResultBlobDownloadFailures,
+    long RateLimitWaits,
+    TimeSpan RateLimitWaitTime,
+    TimeSpan MaximumRateLimitWaitTime,
+    long RateLimitDeferrals,
+    TimeSpan RateLimitDeferredTime,
+    TimeSpan MaximumRateLimitDeferral,
+    long WorkItemDownloads,
+    TimeSpan WorkItemDownloadTime,
+    TimeSpan MaximumWorkItemDownloadTime,
+    long WorkItemPublishes,
+    TimeSpan WorkItemPublishTime,
+    TimeSpan MaximumWorkItemPublishTime,
+    long TestRunCreates,
+    TimeSpan TestRunCreateTime,
+    TimeSpan MaximumTestRunCreateTime,
+    long TestRunCompletes,
+    TimeSpan TestRunCompleteTime,
+    TimeSpan MaximumTestRunCompleteTime,
+    long ParseOperations,
+    TimeSpan ParseTime,
+    TimeSpan MaximumParseTime);
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
index 8f8c8a10c61..32b5a9b8253 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
@@ -34,6 +34,7 @@ internal sealed class JobMonitorRunner : IJobMonitorRunner, IDisposable
         private readonly string _helixSource;
 
         private readonly MonitorState _state = new();
+        private readonly JobMonitorMetrics _metrics;
         private readonly StatusReporter _reporter;
         private readonly TestResultUploadPipeline _uploads;
         private PollStatusSnapshot _latestStatus;
@@ -42,14 +43,22 @@ internal sealed class JobMonitorRunner : IJobMonitorRunner, IDisposable
         /// Constructor for production use with real services.
         /// </summary>
         public JobMonitorRunner(JobMonitorOptions options, ILogger logger)
-            : this(options,
-                  logger,
-                  new AzureDevOpsService(options, logger),
-                  new HelixService(string.IsNullOrEmpty(options.HelixAccessToken)
-                      ? ApiFactory.GetAnonymous(options.HelixBaseUri)
-                      : ApiFactory.GetAuthenticated(options.HelixBaseUri, options.HelixAccessToken),
-                  logger),
-                  delayFunc: null)
+            : this(options, logger, CreateProductionDependencies(options, logger))
+        {
+        }
+
+        private JobMonitorRunner(
+            JobMonitorOptions options,
+            ILogger logger,
+            ProductionDependencies dependencies)
+            : this(
+                options,
+                logger,
+                dependencies.AzureDevOps,
+                dependencies.Helix,
+                delayFunc: null,
+                statusDelayFunc: null,
+                metrics: dependencies.Metrics)
         {
         }
 
@@ -62,7 +71,8 @@ internal JobMonitorRunner(
             IAzureDevOpsService azdo,
             IHelixService helix,
             Func<TimeSpan, CancellationToken, Task> delayFunc,
-            Func<TimeSpan, CancellationToken, Task> statusDelayFunc = null)
+            Func<TimeSpan, CancellationToken, Task> statusDelayFunc = null,
+            JobMonitorMetrics metrics = null)
         {
             _options = options ?? throw new ArgumentNullException(nameof(options));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -70,6 +80,7 @@ internal JobMonitorRunner(
             _helix = helix ?? throw new ArgumentNullException(nameof(helix));
             _delayFunc = delayFunc ?? Task.Delay;
             _statusDelayFunc = statusDelayFunc ?? Task.Delay;
+            _metrics = metrics ?? new JobMonitorMetrics();
             Directory.CreateDirectory(_options.WorkingDirectory);
 
             _helixSource = HelixJobSource.Compute(
@@ -80,7 +91,13 @@ internal JobMonitorRunner(
                 _options.SourceBranch);
 
             _reporter = new StatusReporter(_logger, _options, _state);
-            _uploads = new TestResultUploadPipeline(_logger, _options, _azdo, _helix, _state);
+            _uploads = new TestResultUploadPipeline(
+                _logger,
+                _options,
+                _azdo,
+                _helix,
+                _state,
+                _metrics);
         }
 
         public async Task<int> RunAsync(CancellationToken cancellationToken)
@@ -117,6 +134,10 @@ public async Task<int> RunAsync(CancellationToken cancellationToken)
                 // cancelled.
                 using var cancelCts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
                 await CancelInFlightHelixJobsAsync(cancelCts.Token);
+                _reporter.LogPerformanceMetrics(
+                    _metrics.Snapshot(),
+                    _uploads.Snapshot,
+                    isPartial: true);
 
                 return 1;
             }
@@ -389,6 +410,7 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
             }
 
             await _uploads.DrainAsync(pollNumber, newlyTerminalWorkItems, cancellationToken);
+            _reporter.LogPerformanceMetrics(_metrics.Snapshot(), _uploads.Snapshot);
             _reporter.LogFinalFailedWorkItems();
             _reporter.LogFinalSummary(_state.AssociatedJobsCount);
 
@@ -548,6 +570,21 @@ private bool IsPreviousAttempt(HelixJobInfo job)
                 && !string.IsNullOrEmpty(job.StageAttempt)
                 && MonitorState.ParseStageAttempt(job.StageAttempt) < MonitorState.ParseStageAttempt(_options.StageAttempt);
 
+        private static ProductionDependencies CreateProductionDependencies(
+            JobMonitorOptions options,
+            ILogger logger)
+        {
+            var metrics = new JobMonitorMetrics();
+            var azureDevOps = new AzureDevOpsService(options, logger, metrics);
+            var helix = new HelixService(
+                string.IsNullOrEmpty(options.HelixAccessToken)
+                    ? ApiFactory.GetAnonymous(options.HelixBaseUri)
+                    : ApiFactory.GetAuthenticated(options.HelixBaseUri, options.HelixAccessToken),
+                logger,
+                metrics);
+            return new ProductionDependencies(azureDevOps, helix, metrics);
+        }
+
         public void Dispose()
         {
             _uploads.Cancel();
@@ -604,5 +641,10 @@ private sealed record PollStatusSnapshot(
             IReadOnlyList<HelixJobInfo> Jobs,
             IReadOnlyDictionary<string, IReadOnlyCollection<WorkItemSummary>> WorkItemsByJob,
             IReadOnlySet<string> CompletedJobNames);
+
+        private sealed record ProductionDependencies(
+            IAzureDevOpsService AzureDevOps,
+            IHelixService Helix,
+            JobMonitorMetrics Metrics);
     }
 }
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
index 3676ff79805..d73d99ffeee 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
@@ -5,8 +5,14 @@ namespace Microsoft.DotNet.Helix.JobMonitor;
 
 internal sealed class AzureDevOpsRateLimitGate
 {
+    private readonly JobMonitorMetrics _metrics;
     private long _notBeforeUtcTicks;
 
+    public AzureDevOpsRateLimitGate(JobMonitorMetrics metrics = null)
+    {
+        _metrics = metrics;
+    }
+
     public void Defer(TimeSpan delay)
     {
         if (delay <= TimeSpan.Zero)
@@ -14,6 +20,7 @@ public void Defer(TimeSpan delay)
             return;
         }
 
+        _metrics?.RecordRateLimitDeferral(delay);
         long candidate = DateTimeOffset.UtcNow.Add(delay).UtcTicks;
         long observed;
         while (candidate > (observed = Interlocked.Read(ref _notBeforeUtcTicks)))
@@ -27,16 +34,28 @@ public void Defer(TimeSpan delay)
 
     public async Task WaitAsync(CancellationToken cancellationToken)
     {
-        while (true)
+        long waitStartedAt = 0;
+        try
         {
-            long notBeforeTicks = Interlocked.Read(ref _notBeforeUtcTicks);
-            TimeSpan delay = new DateTimeOffset(notBeforeTicks, TimeSpan.Zero) - DateTimeOffset.UtcNow;
-            if (delay <= TimeSpan.Zero)
+            while (true)
             {
-                return;
-            }
+                long notBeforeTicks = Interlocked.Read(ref _notBeforeUtcTicks);
+                TimeSpan delay = new DateTimeOffset(notBeforeTicks, TimeSpan.Zero) - DateTimeOffset.UtcNow;
+                if (delay <= TimeSpan.Zero)
+                {
+                    return;
+                }
 
-            await Task.Delay(delay, cancellationToken);
+                waitStartedAt = waitStartedAt == 0 ? JobMonitorMetrics.StartOperation() : waitStartedAt;
+                await Task.Delay(delay, cancellationToken);
+            }
+        }
+        finally
+        {
+            if (waitStartedAt != 0)
+            {
+                _metrics?.RecordRateLimitWait(waitStartedAt);
+            }
         }
     }
 }
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
index 582d4c41e4f..730bd826a76 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
@@ -52,19 +52,32 @@ internal sealed class AzureDevOpsService : IAzureDevOpsService, IDisposable
         private readonly JobMonitorOptions _options;
         private readonly ILogger _logger;
         private readonly HttpClient _azdoClient;
-        private readonly AzureDevOpsRateLimitGate _rateLimitGate = new();
-        public AzureDevOpsService(JobMonitorOptions options, ILogger logger)
+        private readonly AzureDevOpsRateLimitGate _rateLimitGate;
+        private readonly JobMonitorMetrics _metrics;
+
+        public AzureDevOpsService(
+            JobMonitorOptions options,
+            ILogger logger,
+            JobMonitorMetrics metrics = null)
         {
             _options = options;
             _logger = logger;
+            _metrics = metrics ?? new JobMonitorMetrics();
+            _rateLimitGate = new AzureDevOpsRateLimitGate(_metrics);
             _azdoClient = new HttpClient();
             InitializeClient();
         }
 
-        internal AzureDevOpsService(JobMonitorOptions options, ILogger logger, HttpClient azdoClient)
+        internal AzureDevOpsService(
+            JobMonitorOptions options,
+            ILogger logger,
+            HttpClient azdoClient,
+            JobMonitorMetrics metrics = null)
         {
             _options = options;
             _logger = logger;
+            _metrics = metrics ?? new JobMonitorMetrics();
+            _rateLimitGate = new AzureDevOpsRateLimitGate(_metrics);
             _azdoClient = azdoClient ?? throw new ArgumentNullException(nameof(azdoClient));
             InitializeClient();
         }
@@ -412,7 +425,8 @@ public async Task<TestResultUploadSummary> UploadTestResultsAsync(
                 reportingParameters,
                 _logger,
                 _azdoClient,
-                _rateLimitGate);
+                _rateLimitGate,
+                _metrics);
 
             if (results.TestResultFiles.Count == 0)
             {
@@ -450,28 +464,59 @@ private async Task<string> SendForStringAsync(
             bool retryTransientFailures = true,
             CancellationToken cancellationToken = default)
         {
+            string serializedBody = body?.ToString(Formatting.None);
+            int payloadBytes = serializedBody is null ? 0 : Encoding.UTF8.GetByteCount(serializedBody);
+            int attempt = 0;
+
             async Task<string> SendOnceAsync()
             {
                 await _rateLimitGate.WaitAsync(cancellationToken);
+                int currentAttempt = attempt++;
+                long requestStartedAt = JobMonitorMetrics.StartOperation();
+                bool failed = true;
+                bool metricsRecorded = false;
                 using var request = new HttpRequestMessage(method, requestUri);
-                if (body != null)
+                if (serializedBody != null)
                 {
-                    request.Content = new StringContent(body.ToString(Formatting.None), Encoding.UTF8, "application/json");
+                    request.Content = new StringContent(serializedBody, Encoding.UTF8, "application/json");
                 }
 
-                using HttpResponseMessage response = await _azdoClient.SendAsync(request, cancellationToken);
-                string content = response.Content != null ? await response.Content.ReadAsStringAsync(cancellationToken) : null;
-                if (!response.IsSuccessStatusCode)
+                try
                 {
+                    using HttpResponseMessage response = await _azdoClient.SendAsync(request, cancellationToken);
+                    string content = response.Content != null ? await response.Content.ReadAsStringAsync(cancellationToken) : null;
+                    failed = !response.IsSuccessStatusCode;
+                    _metrics.RecordAzureDevOpsRequest(
+                        AzureDevOpsRequestKind.Control,
+                        payloadBytes,
+                        isRetry: currentAttempt > 0,
+                        failed: failed,
+                        startedAt: requestStartedAt);
+                    metricsRecorded = true;
+                    if (!response.IsSuccessStatusCode)
+                    {
+                        await HonorRateLimitAsync(response, requestUri, cancellationToken);
+                        throw new HttpRequestException(
+                            $"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}",
+                            null,
+                            response.StatusCode);
+                    }
+
                     await HonorRateLimitAsync(response, requestUri, cancellationToken);
-                    throw new HttpRequestException(
-                        $"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}",
-                        null,
-                        response.StatusCode);
+                    return content;
+                }
+                finally
+                {
+                    if (!metricsRecorded)
+                    {
+                        _metrics.RecordAzureDevOpsRequest(
+                            AzureDevOpsRequestKind.Control,
+                            payloadBytes,
+                            isRetry: currentAttempt > 0,
+                            failed: failed,
+                            startedAt: requestStartedAt);
+                    }
                 }
-
-                await HonorRateLimitAsync(response, requestUri, cancellationToken);
-                return content;
             }
 
             if (!retryTransientFailures)
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
index 5896ab70a92..78f15778437 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
@@ -25,9 +25,13 @@ internal sealed class HelixService : IHelixService
         private readonly IHelixApi _helixApi;
         private readonly IBlobClientFactory _blobClientFactory;
         private readonly IFileSystem _fileSystem;
+        private readonly JobMonitorMetrics _metrics;
 
-        public HelixService(IHelixApi helixApi, ILogger logger)
-            : this(helixApi, logger, new AzureBlobClientFactory(), new FileSystem())
+        public HelixService(
+            IHelixApi helixApi,
+            ILogger logger,
+            JobMonitorMetrics metrics = null)
+            : this(helixApi, logger, new AzureBlobClientFactory(), new FileSystem(), metrics)
         {
         }
 
@@ -35,12 +39,14 @@ internal HelixService(
             IHelixApi helixApi,
             ILogger logger,
             IBlobClientFactory blobClientFactory,
-            IFileSystem fileSystem)
+            IFileSystem fileSystem,
+            JobMonitorMetrics metrics = null)
         {
             _helixApi = helixApi ?? throw new ArgumentNullException(nameof(helixApi));
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
             _blobClientFactory = blobClientFactory ?? throw new ArgumentNullException(nameof(blobClientFactory));
             _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
+            _metrics = metrics ?? new JobMonitorMetrics();
         }
 
         public async Task<IReadOnlyList<HelixJobInfo>> GetJobsForBuildAsync(
@@ -107,6 +113,7 @@ public async Task<WorkItemTestResults> DownloadTestResultsAsync(
                     IBlobClient blobClient = _blobClientFactory.CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS);
                     await blobClient.DownloadToAsync(destinationFile, cancellationToken);
                     workItemFiles.Add(destinationFile);
+                    _metrics.RecordResultBlobDownload(failed: false);
                 }
                 catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
                 {
@@ -114,6 +121,7 @@ public async Task<WorkItemTestResults> DownloadTestResultsAsync(
                 }
                 catch (Exception ex) when (TransientFailureDetector.IsTransient(ex))
                 {
+                    _metrics.RecordResultBlobDownload(failed: true);
                     transientFailures.Add(ex);
                     _logger.LogWarning(ex,
                         "Transient failure downloading '{FileName}' for '{JobName}/{WorkItemName}'. "
@@ -124,6 +132,7 @@ public async Task<WorkItemTestResults> DownloadTestResultsAsync(
                 }
                 catch (Exception ex)
                 {
+                    _metrics.RecordResultBlobDownload(failed: true);
                     _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItemName);
                 }
             }
@@ -377,6 +386,7 @@ private async Task<T> RetryAsync<T>(Func<Task<T>> action, CancellationToken canc
         {
             Exception last = null;
             T result = default;
+            int attempt = 0;
             var retryHandler = new ExponentialRetry
             {
                 MaxAttempts = 5,
@@ -396,13 +406,16 @@ private async Task<T> RetryAsync<T>(Func<Task<T>> action, CancellationToken canc
             bool succeeded = await retryHandler.RunAsync(
                 async _ =>
                 {
+                    int currentAttempt = attempt++;
                     try
                     {
                         result = await action();
+                        _metrics.RecordHelixRequest(isRetry: currentAttempt > 0, failed: false);
                         return RetryResult.Success;
                     }
                     catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
                     {
+                        _metrics.RecordHelixRequest(isRetry: currentAttempt > 0, failed: true);
                         last = ex;
                         return RetryResult.Retry();
                     }
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/StatusReporter.cs b/src/Microsoft.DotNet.Helix/JobMonitor/StatusReporter.cs
index 595333c6510..aafa9b0fb44 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/StatusReporter.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/StatusReporter.cs
@@ -208,6 +208,87 @@ public void LogFinalSummary(int totalAssociatedJobCount)
                 DocumentationUri);
         }
 
+        public void LogPerformanceMetrics(
+            JobMonitorMetricsSnapshot metrics,
+            UploadPipelineSnapshot uploads,
+            bool isPartial = false)
+        {
+            double pipelineSeconds = Math.Max(metrics.PipelineElapsed.TotalSeconds, 0.001);
+            double resultThroughput = uploads.UploadedResults / pipelineSeconds;
+            double workItemThroughput = uploads.WorkItems.Completed / pipelineSeconds;
+            double payloadMiB = metrics.AzureDevOpsPayloadBytes / (1024d * 1024d);
+            string heading = isPartial
+                ? "⏱️ Partial performance metrics captured during cancellation"
+                : "⏱️ Performance metrics";
+            string throughputLabel = isPartial
+                ? "Completed before cancellation"
+                : "Upload throughput";
+            string throughputSummary = isPartial
+                ? $"{uploads.UploadedResults} test result(s); "
+                    + $"{uploads.WorkItems.Completed} work item(s); rates unavailable for a partial snapshot"
+                : $"{uploads.UploadedResults} test result(s), {resultThroughput:0.##} result(s)/s; "
+                    + $"{uploads.WorkItems.Completed} work item(s), {workItemThroughput:0.##} work item(s)/s";
+
+            _logger.LogInformation(
+                "{Heading} ({Elapsed} monitor elapsed, {PipelineElapsed} observed upload pipeline activity):{nl}"
+              + "   {ThroughputLabel}: {ThroughputSummary}{nl}"
+              + "   Azure DevOps HTTP: {AzdoRequests} request attempt(s) "
+              + "({ControlRequests} control, {ResultRequests} result batch, {AttachmentRequests} attachment), "
+              + "{AzdoRetries} retry attempt(s), {AzdoFailedAttempts} failed attempt(s), "
+              + "{PayloadMiB:0.##} MiB request payload; aggregate request time {AzdoRequestTime}, "
+              + "max {MaximumAzdoRequestTime}{nl}"
+              + "   Helix/storage: {HelixRequests} retry-wrapped remote request attempt(s), "
+              + "{HelixRetries} retry attempt(s), {HelixFailedAttempts} failed attempt(s); "
+              + "{BlobDownloads} result blob download attempt(s), {BlobFailures} failed{nl}"
+              + "   Rate limiting: {RateLimitDeferrals} server-directed deferral(s), "
+              + "{RateLimitDeferredTime} cumulative guidance, max {MaximumRateLimitDeferral}; "
+              + "{RateLimitWaits} shared-gate worker wait(s), aggregate {RateLimitWaitTime}, "
+              + "max {MaximumRateLimitWaitTime}{nl}"
+              + "   Pipeline aggregate worker time (max single operation): download {DownloadTime} ({MaxDownloadTime}), "
+              + "parse/aggregate {ParseTime} ({MaxParseTime}), publish {PublishTime} ({MaxPublishTime}), "
+              + "create run {CreateTime} ({MaxCreateTime}), complete run {CompleteTime} ({MaxCompleteTime}).",
+                heading,
+                metrics.Elapsed,
+                metrics.PipelineElapsed,
+                Environment.NewLine,
+                throughputLabel,
+                throughputSummary,
+                Environment.NewLine,
+                metrics.AzureDevOpsRequests,
+                metrics.AzureDevOpsControlRequests,
+                metrics.AzureDevOpsResultRequests,
+                metrics.AzureDevOpsAttachmentRequests,
+                metrics.AzureDevOpsRetries,
+                metrics.AzureDevOpsFailedAttempts,
+                payloadMiB,
+                metrics.AzureDevOpsRequestTime,
+                metrics.MaximumAzureDevOpsRequestTime,
+                Environment.NewLine,
+                metrics.HelixRequests,
+                metrics.HelixRetries,
+                metrics.HelixFailedAttempts,
+                metrics.ResultBlobDownloads,
+                metrics.ResultBlobDownloadFailures,
+                Environment.NewLine,
+                metrics.RateLimitDeferrals,
+                metrics.RateLimitDeferredTime,
+                metrics.MaximumRateLimitDeferral,
+                metrics.RateLimitWaits,
+                metrics.RateLimitWaitTime,
+                metrics.MaximumRateLimitWaitTime,
+                Environment.NewLine,
+                metrics.WorkItemDownloadTime,
+                metrics.MaximumWorkItemDownloadTime,
+                metrics.ParseTime,
+                metrics.MaximumParseTime,
+                metrics.WorkItemPublishTime,
+                metrics.MaximumWorkItemPublishTime,
+                metrics.TestRunCreateTime,
+                metrics.MaximumTestRunCreateTime,
+                metrics.TestRunCompleteTime,
+                metrics.MaximumTestRunCompleteTime);
+        }
+
         public void LogNonMonitorPipelineFailure()
         {
             // DO NOT CHANGE THIS LINE - it's matched by Build Analysis to ignore a failure
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
index 3a814496114..6797f278185 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
@@ -21,6 +21,7 @@ internal sealed class TestResultUploadPipeline : IAsyncDisposable
     private readonly IAzureDevOpsService _azdo;
     private readonly IHelixService _helix;
     private readonly MonitorState _state;
+    private readonly JobMonitorMetrics _metrics;
     private readonly ConcurrentDictionary<string, JobUploadSession> _sessions =
         new(StringComparer.OrdinalIgnoreCase);
     private readonly ConcurrentDictionary<int, long> _remainingWorkItemsByPoll = [];
@@ -34,13 +35,15 @@ public TestResultUploadPipeline(
         JobMonitorOptions options,
         IAzureDevOpsService azdo,
         IHelixService helix,
-        MonitorState state)
+        MonitorState state,
+        JobMonitorMetrics metrics)
     {
         _logger = logger;
         _options = options;
         _azdo = azdo;
         _helix = helix;
         _state = state;
+        _metrics = metrics;
 
         int uploadParallelism = options.TestResultUploadParallelism;
         _jobs = new ActionQueue<JobUploadRequest>(
@@ -195,12 +198,25 @@ private async ValueTask ProcessWorkItemAsync(
         JobUploadSession session = request.Session;
         try
         {
-            WorkItemTestResults downloaded = await ExecuteDownloadWithRetryAsync(
-                session.Job,
-                request.WorkItemName,
-                cancellationToken);
+            WorkItemTestResults downloaded;
+            long downloadStartedAt = JobMonitorMetrics.StartOperation();
+            try
+            {
+                downloaded = await ExecuteDownloadWithRetryAsync(
+                    session.Job,
+                    request.WorkItemName,
+                    cancellationToken);
+            }
+            finally
+            {
+                _metrics.RecordPipelineOperation(
+                    PipelineOperation.WorkItemDownload,
+                    downloadStartedAt);
+            }
+
             int testRunId = await session.GetOrCreateTestRunAsync(
-                () => _azdo.CreateTestRunAsync(session.Job.TestRunName, cancellationToken));
+                () => CreateTestRunAsync(session.Job.TestRunName, cancellationToken));
+
             TestResultUploadSummary summary =
                 await _azdo.UploadTestResultsAsync(testRunId, downloaded, cancellationToken);
 
@@ -252,12 +268,22 @@ private async ValueTask FinalizeJobAsync(
             try
             {
                 int testRunId = await session.GetOrCreateTestRunAsync(
-                    () => _azdo.CreateTestRunAsync(session.Job.TestRunName, cancellationToken));
-                await _azdo.CompleteTestRunAsync(
-                    testRunId,
-                    session.Job.JobName,
-                    session.FailedWorkItems,
-                    cancellationToken);
+                    () => CreateTestRunAsync(session.Job.TestRunName, cancellationToken));
+                long completeStartedAt = JobMonitorMetrics.StartOperation();
+                try
+                {
+                    await _azdo.CompleteTestRunAsync(
+                        testRunId,
+                        session.Job.JobName,
+                        session.FailedWorkItems,
+                        cancellationToken);
+                }
+                finally
+                {
+                    _metrics.RecordPipelineOperation(
+                        PipelineOperation.TestRunComplete,
+                        completeStartedAt);
+                }
 
                 _state.TryMarkHelixJobProcessed(session.Job.JobName);
                 _logger.LogInformation(
@@ -285,6 +311,21 @@ await _azdo.CompleteTestRunAsync(
         }
     }
 
+    private async Task<int> CreateTestRunAsync(
+        string testRunName,
+        CancellationToken cancellationToken)
+    {
+        long startedAt = JobMonitorMetrics.StartOperation();
+        try
+        {
+            return await _azdo.CreateTestRunAsync(testRunName, cancellationToken);
+        }
+        finally
+        {
+            _metrics.RecordPipelineOperation(PipelineOperation.TestRunCreate, startedAt);
+        }
+    }
+
     private async Task<WorkItemTestResults> ExecuteDownloadWithRetryAsync(
         HelixJobInfo job,
         string workItemName,
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs
index e0fdfabceda..96bc0e596d9 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs
@@ -36,29 +36,32 @@ internal sealed class AzureDevOpsResultPublisher : IDisposable
     private readonly ILogger _logger;
     private readonly bool _ownsHttpClient;
     private readonly AzureDevOpsRateLimitGate _rateLimitGate;
+    private readonly JobMonitorMetrics _metrics;
 
     public AzureDevOpsResultPublisher(
         AzureDevOpsReportingParameters azdoParameters,
         ILogger logger)
-        : this(
-            azdoParameters,
-            logger,
-            CreateHttpClient(azdoParameters.AccessToken),
-            new AzureDevOpsRateLimitGate(),
-            ownsHttpClient: true)
     {
+        _azdoParameters = azdoParameters;
+        _httpClient = CreateHttpClient(azdoParameters.AccessToken);
+        _logger = logger;
+        _metrics = new JobMonitorMetrics();
+        _rateLimitGate = new AzureDevOpsRateLimitGate(_metrics);
+        _ownsHttpClient = true;
     }
 
     internal AzureDevOpsResultPublisher(
         AzureDevOpsReportingParameters azdoParameters,
         ILogger logger,
         HttpClient httpClient,
-        AzureDevOpsRateLimitGate? rateLimitGate = null)
+        AzureDevOpsRateLimitGate? rateLimitGate = null,
+        JobMonitorMetrics? metrics = null)
         : this(
             azdoParameters,
             logger,
             httpClient,
             rateLimitGate ?? new AzureDevOpsRateLimitGate(),
+            metrics ?? new JobMonitorMetrics(),
             ownsHttpClient: false)
     {
     }
@@ -68,12 +71,14 @@ private AzureDevOpsResultPublisher(
         ILogger logger,
         HttpClient httpClient,
         AzureDevOpsRateLimitGate rateLimitGate,
+        JobMonitorMetrics metrics,
         bool ownsHttpClient)
     {
         _azdoParameters = azdoParameters;
         _httpClient = httpClient;
         _logger = logger;
         _rateLimitGate = rateLimitGate;
+        _metrics = metrics;
         _ownsHttpClient = ownsHttpClient;
     }
 
@@ -87,31 +92,57 @@ public void Dispose()
 
     public async Task<TestResultUploadSummary> UploadTestResultsWithSummaryAsync(List<string> testResultFiles, object resultMetadata, CancellationToken cancellationToken = default)
     {
-        var testResultReader = new LocalTestResultsReader(_logger, _azdoParameters.TestResultAttachmentMode);
-
-        var parsedResults = new List<IReadOnlyList<TestResult>>(testResultFiles.Count);
-        foreach (string file in testResultFiles)
+        long parseStartedAt = JobMonitorMetrics.StartOperation();
+        bool parseRecorded = false;
+        try
         {
-            parsedResults.Add(await testResultReader.ReadResultFileAsync(file, cancellationToken));
-        }
+            var testResultReader = new LocalTestResultsReader(_logger, _azdoParameters.TestResultAttachmentMode);
 
-        if (parsedResults.Count == 0)
-        {
-            _logger.LogWarning("No test result files were provided for upload");
-            return new TestResultUploadSummary(true, 0);
-        }
+            var parsedResults = new List<IReadOnlyList<TestResult>>(testResultFiles.Count);
+            foreach (string file in testResultFiles)
+            {
+                parsedResults.Add(await testResultReader.ReadResultFileAsync(file, cancellationToken));
+            }
+
+            if (parsedResults.Count == 0)
+            {
+                _logger.LogWarning("No test result files were provided for upload");
+                return new TestResultUploadSummary(true, 0);
+            }
+
+            IReadOnlyList<AggregatedResult> aggregatedResults = new ResultAggregator().Aggregate(parsedResults, _azdoParameters.UseFullyQualifiedTestName);
+            _metrics.RecordPipelineOperation(PipelineOperation.ResultParseAndAggregate, parseStartedAt);
+            parseRecorded = true;
+            if (aggregatedResults.Count == 0)
+            {
+                _logger.LogDebug("Test results were discovered but none could be aggregated");
+                return new TestResultUploadSummary(true, 0);
+            }
 
-        IReadOnlyList<AggregatedResult> aggregatedResults = new ResultAggregator().Aggregate(parsedResults, _azdoParameters.UseFullyQualifiedTestName);
-        if (aggregatedResults.Count == 0)
+            long publishStartedAt = JobMonitorMetrics.StartOperation();
+            long uploadedCount;
+            try
+            {
+                uploadedCount = await UploadTestResultsWithCountAsync(
+                    aggregatedResults,
+                    resultMetadata,
+                    cancellationToken);
+            }
+            finally
+            {
+                _metrics.RecordPipelineOperation(PipelineOperation.WorkItemPublish, publishStartedAt);
+            }
+            return new TestResultUploadSummary(
+                AllPassed: ComputeAllPassed(aggregatedResults),
+                UploadedCount: uploadedCount);
+        }
+        finally
         {
-            _logger.LogDebug("Test results were discovered but none could be aggregated");
-            return new TestResultUploadSummary(true, 0);
+            if (!parseRecorded)
+            {
+                _metrics.RecordPipelineOperation(PipelineOperation.ResultParseAndAggregate, parseStartedAt);
+            }
         }
-
-        long uploadedCount = await UploadTestResultsWithCountAsync(aggregatedResults, resultMetadata, cancellationToken);
-        return new TestResultUploadSummary(
-            AllPassed: ComputeAllPassed(aggregatedResults),
-            UploadedCount: uploadedCount);
     }
 
     /// <summary>
@@ -157,6 +188,7 @@ private async Task<IReadOnlyList<PublishedTestCase>> PublishResultsAsync(
             $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results?api-version=7.1-preview.6",
             testCaseResults,
             DefaultAttemptCount,
+            AzureDevOpsRequestKind.ResultBatch,
             cancellationToken);
 
         IReadOnlyList<PublishedTestCaseResultReference> publishedResults = await ReadPublishedResultsAsync(response, cancellationToken);
@@ -240,6 +272,7 @@ private async Task SendAttachmentAsync(
             path,
             request,
             DefaultAttemptCount,
+            AzureDevOpsRequestKind.Attachment,
             cancellationToken);
         _ = response;
     }
@@ -499,6 +532,7 @@ private async Task<HttpResponseMessage> SendWithRetryAsync(
         string relativePath,
         object? payload,
         int attemptCount,
+        AzureDevOpsRequestKind requestKind,
         CancellationToken cancellationToken)
     {
         byte[]? body = payload is null ? null : JsonSerializer.SerializeToUtf8Bytes(payload, s_serializerOptions);
@@ -528,6 +562,8 @@ private async Task<HttpResponseMessage> SendWithRetryAsync(
             async attempt =>
             {
                 await _rateLimitGate.WaitAsync(cancellationToken);
+                long requestStartedAt = JobMonitorMetrics.StartOperation();
+                bool failed = true;
 
                 Uri baseUri = _azdoParameters.CollectionUri.AbsoluteUri.EndsWith('/')
                     ? _azdoParameters.CollectionUri
@@ -542,7 +578,7 @@ private async Task<HttpResponseMessage> SendWithRetryAsync(
 
                 try
                 {
-                    DateTimeOffset requestStartedAt = DateTimeOffset.UtcNow;
+                    DateTimeOffset logStartedAt = DateTimeOffset.UtcNow;
                     _logger.LogDebug(
                         "Sending Azure DevOps {Method} request to '{RequestPath}', attempt {Attempt} of {AttemptCount}.",
                         method,
@@ -565,7 +601,8 @@ private async Task<HttpResponseMessage> SendWithRetryAsync(
                             (int)response.StatusCode,
                             attempt + 1,
                             attemptCount,
-                            DateTimeOffset.UtcNow - requestStartedAt);
+                            DateTimeOffset.UtcNow - logStartedAt);
+                        failed = false;
                         successfulResponse = response;
                         return RetryResult.Success;
                     }
@@ -595,7 +632,7 @@ private async Task<HttpResponseMessage> SendWithRetryAsync(
                                 (int)response.StatusCode,
                                 attempt + 1,
                                 attemptCount,
-                                DateTimeOffset.UtcNow - requestStartedAt);
+                                DateTimeOffset.UtcNow - logStartedAt);
                             lastException = new AzureDevOpsReportingError(
                                 $"Azure DevOps request failed with status code {(int)response.StatusCode}: {responseBody}");
                             return RetryResult.Retry(retryAfter);
@@ -627,6 +664,15 @@ private async Task<HttpResponseMessage> SendWithRetryAsync(
                         attemptCount);
                     return RetryResult.Retry();
                 }
+                finally
+                {
+                    _metrics.RecordAzureDevOpsRequest(
+                        requestKind,
+                        body?.Length ?? 0,
+                        isRetry: attempt > 0,
+                        failed: failed,
+                        startedAt: requestStartedAt);
+                }
             },
             cancellationToken);
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
index f13409004bd..7d66d3e9cc0 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
@@ -153,7 +153,8 @@ public async Task UploadTestResultsWithCountAsync_BatchesByTopLevelResultCount()
         public async Task UploadTestResultsWithCountAsync_SplitsMoreThanOneThousandTopLevelResults()
         {
             var handler = new RecordingResultHandler();
-            using var publisher = CreatePublisher(handler);
+            var metrics = new JobMonitorMetrics();
+            using var publisher = CreatePublisher(handler, metrics);
             AggregatedResult[] results =
             [
                 .. Enumerable.Range(0, 1001)
@@ -164,6 +165,15 @@ .. Enumerable.Range(0, 1001)
 
             Assert.Equal(1001, uploadedCount);
             Assert.Equal(new[] { 1000, 1 }, handler.RequestResultCounts);
+            JobMonitorMetricsSnapshot snapshot = metrics.Snapshot();
+            Assert.Equal(2, snapshot.AzureDevOpsRequests);
+            Assert.Equal(2, snapshot.AzureDevOpsResultRequests);
+            Assert.Equal(0, snapshot.AzureDevOpsControlRequests);
+            Assert.Equal(0, snapshot.AzureDevOpsAttachmentRequests);
+            Assert.Equal(0, snapshot.AzureDevOpsRetries);
+            Assert.Equal(0, snapshot.AzureDevOpsFailedAttempts);
+            Assert.True(snapshot.AzureDevOpsPayloadBytes > 0);
+            Assert.True(snapshot.MaximumAzureDevOpsRequestTime > TimeSpan.Zero);
         }
 
         [Fact]
@@ -229,14 +239,40 @@ IEnumerable<AggregatedResult> Results()
             Assert.Equal(2, handler.RequestResultCounts.Count);
         }
 
-        private static AzureDevOpsResultPublisher CreatePublisher(HttpMessageHandler handler)
+        [Fact]
+        public async Task UploadTestResultsWithCountAsync_RecordsThrottledRetryWait()
+        {
+            var handler = new ThrottlingResultHandler();
+            var metrics = new JobMonitorMetrics();
+            using var publisher = CreatePublisher(handler, metrics);
+            AggregatedResult[] results =
+            [
+                new(AggregationType.Single, "Test", 1, "Passed")
+            ];
+
+            Assert.Equal(1, await publisher.UploadTestResultsWithCountAsync(results, new { }));
+
+            JobMonitorMetricsSnapshot snapshot = metrics.Snapshot();
+            Assert.Equal(2, snapshot.AzureDevOpsResultRequests);
+            Assert.Equal(1, snapshot.AzureDevOpsRetries);
+            Assert.Equal(1, snapshot.AzureDevOpsFailedAttempts);
+            Assert.Equal(1, snapshot.RateLimitDeferrals);
+            Assert.True(snapshot.RateLimitDeferredTime >= TimeSpan.FromMilliseconds(50));
+            Assert.True(snapshot.MaximumRateLimitDeferral >= TimeSpan.FromMilliseconds(50));
+        }
+
+        private static AzureDevOpsResultPublisher CreatePublisher(
+            HttpMessageHandler handler,
+            JobMonitorMetrics metrics = null)
             => new(
                 new AzureDevOpsReportingParameters(
                     new Uri("https://dev.azure.com/dnceng-public/"),
                     "public",
                     "123"),
                 NullLogger.Instance,
-                new HttpClient(handler));
+                new HttpClient(handler),
+                new AzureDevOpsRateLimitGate(metrics),
+                metrics);
 
         private static AggregatedResult CreateDataDrivenResult(string name, int subResultCount)
             => new(
@@ -309,5 +345,25 @@ protected override async Task<HttpResponseMessage> SendAsync(
             }
         }
 
+        private sealed class ThrottlingResultHandler : RecordingResultHandler
+        {
+            private int _requestCount;
+
+            protected override Task<HttpResponseMessage> SendAsync(
+                HttpRequestMessage request,
+                CancellationToken cancellationToken)
+            {
+                if (Interlocked.Increment(ref _requestCount) == 1)
+                {
+                    var response = new HttpResponseMessage(HttpStatusCode.TooManyRequests);
+                    response.Headers.RetryAfter = new RetryConditionHeaderValue(
+                        TimeSpan.FromMilliseconds(50));
+                    return Task.FromResult(response);
+                }
+
+                return base.SendAsync(request, cancellationToken);
+            }
+        }
+
     }
 }
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
index 63f0ffa1267..9dd2a8a3f9c 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
@@ -591,6 +591,10 @@ public async Task DrainReportsAggregatePipelineProgress()
             logger.Messages.Should().Contain(message =>
                 message.Contains("Test result pipeline drained in", StringComparison.Ordinal)
                 && message.Contains("1 job(s), 1 work item(s), and 1 result(s)", StringComparison.Ordinal));
+            logger.Messages.Should().Contain(message =>
+                message.Contains("Performance metrics", StringComparison.Ordinal)
+                && message.Contains("Azure DevOps HTTP:", StringComparison.Ordinal)
+                && message.Contains("Pipeline aggregate worker time", StringComparison.Ordinal));
         }
 
         [Fact]

From 92c4d052b9d56be3748680682e477417a6e5e466 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Thu, 13 Aug 2026 02:37:45 -0700
Subject: [PATCH 08/21] Preserve failures across same-queue Helix jobs

Use the submitter-assigned logical Helix job name in work-stream identity so independent jobs from one AzDO submitter and queue cannot overwrite same-named work-item outcomes. Preserve the identifier through monitor resubmissions and document the revised reconciliation semantics.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../JobMonitor/Design/Components/Polling.md   |  2 +-
 .../JobMonitor/JobMonitorRunner.Design.md     | 13 +++--
 .../JobMonitor/Models/HelixJobInfo.cs         | 28 ++++++++-
 .../JobMonitor/MonitorState.cs                | 52 +++++++++--------
 .../JobMonitor/Services/HelixService.cs       |  4 +-
 .../Fakes/FakeHelixService.cs                 |  3 +-
 .../JobMonitorRunnerTests.cs                  | 58 +++++++++++++++++--
 .../ScenarioHelpers/ScenarioHelpers.cs        |  6 +-
 8 files changed, 127 insertions(+), 39 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
index 4f256b0c3ed..99e96274e09 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
@@ -18,4 +18,4 @@ The one-shot entry retry pass and stage-attempt semantics are specified in
 [the semantic document](../../JobMonitorRunner.Design.md). Outcome updates are
 applied oldest-to-newest so resubmissions and higher stage attempts supersede
 older failures without allowing identically named work from different
-submitter/queue streams to collide.
+submitter/queue/logical-job streams to collide.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md
index 1a4145d99fd..e0aefe1b8db 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md
@@ -334,7 +334,7 @@ or the runner will silently fail to see its own jobs.
 
 1. Take a Helix snapshot of the whole stage (all attempts).
 2. Reduce it to the latest incarnation of each logical work stream (§2.3.3):
-   the leaf of each lineage chain, keyed by submitter chain key, preferring the
+   the leaf of each lineage chain, keyed by logical stream key, preferring the
    higher stage attempt on ties.
 3. For each latest incarnation, apply §2.3.3:
    - Current-attempt incarnation — leave it; it is already being driven.
@@ -420,9 +420,10 @@ The chain key must be deterministic and uniqueness-preserving:
   preserved.
 - An original Helix job and its resubmission(s) on the same queue must
   produce the same key so the latest incarnation overwrites the older one.
-- Because the chain key is built from the AzDO `System.JobName` + queue — both
-  stable across stage attempts — a rerun-stage incarnation of the same job on
-  the same queue collapses onto the same key as its previous-attempt
+- Because the chain key is built from the AzDO `System.JobName`, queue, and
+  submitter-assigned Helix `jobName` — all stable across stage attempts — a
+  rerun-stage incarnation of the same logical job on the same queue collapses
+  onto the same key as its previous-attempt
   counterpart, even though the two Helix jobs are **not** linked by
   `PreviousHelixJobName` (only monitor resubmissions set that link). The map
   must therefore let the **later stage attempt win** when two incarnations share
@@ -432,6 +433,10 @@ The chain key must be deterministic and uniqueness-preserving:
 - If lineage cannot be resolved (the predecessor link points outside the
   jobs the runner has observed), the key falls back to a Helix-job-bound
   identifier so independent jobs don't collide.
+- Multiple independent Helix jobs submitted by the same AzDO job to the same
+  queue remain distinct because their submitter-assigned `jobName` values
+  differ. This prevents a passing work item in one logical job from erasing a
+  same-named failure in another.
 
 The same key drives a parallel map of "failed work item console info" used
 to build the final failure report. When a later incarnation of a work item
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs
index 59bd17b463c..b8e541a1d69 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs
@@ -16,6 +16,7 @@ namespace Microsoft.DotNet.Helix.JobMonitor.Models
     public sealed class HelixJobInfo
     {
         public const string PreviousHelixJobNamePropertyName = "PreviousHelixJobName";
+        public const string LogicalJobNamePropertyName = "jobName";
 
         /// <summary>
         /// Helix job property that records the Azure DevOps stage attempt during which the job
@@ -47,7 +48,8 @@ public HelixJobInfo(
             string queueId = null,
             string previousHelixJobName = null,
             int? initialWorkItemCount = null,
-            string stageAttempt = null)
+            string stageAttempt = null,
+            string logicalJobName = null)
         {
             JobName = jobName ?? throw new ArgumentNullException(nameof(jobName));
             Status = status ?? throw new ArgumentNullException(nameof(status));
@@ -56,7 +58,14 @@ public HelixJobInfo(
             StageAttempt = stageAttempt;
             QueueId = queueId;
             InitialWorkItemCount = initialWorkItemCount;
-            Properties = CreateProperties(testRunName, stageName, submitterJobName, submitterJobDisplayName, previousHelixJobName, stageAttempt);
+            Properties = CreateProperties(
+                testRunName,
+                stageName,
+                submitterJobName,
+                submitterJobDisplayName,
+                previousHelixJobName,
+                stageAttempt,
+                logicalJobName);
         }
 
         public string JobName { get; }
@@ -93,6 +102,13 @@ public HelixJobInfo(
 
         public string SubmitterJobName => GetStringProperty(Properties, "System.JobName");
 
+        /// <summary>
+        /// Stable logical name assigned by the Helix SDK submitter. A single Azure DevOps job
+        /// can submit multiple Helix jobs to the same queue, so this value distinguishes those
+        /// independent streams while remaining stable across stage reruns and resubmissions.
+        /// </summary>
+        public string LogicalJobName => GetStringProperty(Properties, LogicalJobNamePropertyName);
+
         /// <summary>
         /// Matrix-expanded Azure DevOps job display name (e.g. "Windows_NT Build_Release"),
         /// stamped onto the job from the <c>System.JobDisplayName</c> predefined variable.
@@ -200,7 +216,8 @@ private static JObject CreateProperties(
             string submitterJobName,
             string submitterJobDisplayName,
             string previousHelixJobName,
-            string stageAttempt)
+            string stageAttempt,
+            string logicalJobName)
         {
             var properties = new JObject();
 
@@ -234,6 +251,11 @@ private static JObject CreateProperties(
                 properties[PreviousHelixJobNamePropertyName] = previousHelixJobName;
             }
 
+            if (!string.IsNullOrEmpty(logicalJobName))
+            {
+                properties[LogicalJobNamePropertyName] = logicalJobName;
+            }
+
             return properties;
         }
     }
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
index 85533f2b380..e95cf501166 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
@@ -398,16 +398,12 @@ public IReadOnlyList<FailedWorkItemConsoleInfo> SnapshotFailedWorkItemConsoleInf
         }
 
         /// <summary>
-        /// Produces a key that rolls up work-item outcomes within a logical AzDO submitter
-        /// chain. When the job carries an AzDO <c>System.JobName</c>, the chain key is based
-        /// on that name combined with the Helix <c>QueueId</c> (so resubmissions of the same
-        /// AzDO job to the same queue share the same key while a single AzDO matrix leg that
-        /// fans out to multiple Helix queues — each producing its own Helix job under the
-        /// same <c>System.JobName</c> — stays distinct and cannot overwrite a sibling queue's
-        /// failure with a pass). When there is no submitter name (test scenarios, manual
-        /// Helix submissions), the chain is followed back through <c>PreviousHelixJobName</c>
-        /// links to the root and the root Helix job name is used instead, so that retries
-        /// still overwrite prior failures correctly.
+        /// Produces a key that rolls up work-item outcomes within a logical Helix work stream.
+        /// The AzDO submitter name, Helix queue, and submitter-assigned logical job name jointly
+        /// identify a stream across stage reruns and monitor resubmissions. The logical job name
+        /// is essential because one AzDO job can submit multiple independent Helix jobs to the
+        /// same queue. When stable submitter metadata is unavailable, lineage is followed back
+        /// through <c>PreviousHelixJobName</c> and the root Helix job name is used instead.
         /// </summary>
         public string GetSubmitterChainKey(HelixJobInfo job)
         {
@@ -419,11 +415,25 @@ public string GetSubmitterChainKey(HelixJobInfo job)
 
         private string GetSubmitterChainKeyLocked(HelixJobInfo job)
         {
-            if (!string.IsNullOrEmpty(job.SubmitterJobName))
+            HelixJobInfo root = GetLineageRootLocked(job);
+            string submitterJobName = job.SubmitterJobName ?? root.SubmitterJobName;
+            string queueId = job.QueueId ?? root.QueueId;
+            string logicalJobName = job.LogicalJobName
+                ?? root.LogicalJobName
+                ?? job.TestRunName
+                ?? root.TestRunName;
+
+            if (!string.IsNullOrEmpty(submitterJobName)
+                && !string.IsNullOrEmpty(logicalJobName))
             {
-                return FormatSubmitterChainKey(job.SubmitterJobName, job.QueueId);
+                return FormatSubmitterChainKey(submitterJobName, queueId, logicalJobName);
             }
 
+            return $"helix:{root.JobName}";
+        }
+
+        private HelixJobInfo GetLineageRootLocked(HelixJobInfo job)
+        {
             HelixJobInfo current = job;
             var visited = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
             while (current is not null
@@ -432,24 +442,20 @@ private string GetSubmitterChainKeyLocked(HelixJobInfo job)
             {
                 if (!_associatedJobs.TryGetValue(current.PreviousHelixJobName, out HelixJobInfo previous))
                 {
-                    return $"helix:{current.PreviousHelixJobName}";
-                }
-
-                if (!string.IsNullOrEmpty(previous.SubmitterJobName))
-                {
-                    return FormatSubmitterChainKey(previous.SubmitterJobName, previous.QueueId);
+                    return new HelixJobInfo(current.PreviousHelixJobName, "finished");
                 }
 
                 current = previous;
             }
 
-            return $"helix:{(current?.JobName ?? job.JobName)}";
+            return current ?? job;
         }
 
-        private static string FormatSubmitterChainKey(string submitterJobName, string queueId)
-            => string.IsNullOrEmpty(queueId)
-                ? $"submitter:{submitterJobName}"
-                : $"submitter:{submitterJobName}|queue:{queueId}";
+        private static string FormatSubmitterChainKey(
+            string submitterJobName,
+            string queueId,
+            string logicalJobName)
+            => $"submitter:{submitterJobName}|queue:{queueId ?? string.Empty}|job:{logicalJobName}";
 
         /// <summary>
         /// From an arbitrary set of Helix jobs (possibly spanning multiple stage attempts),
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
index 78f15778437..c0431d74fe2 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
@@ -337,6 +337,7 @@ await RetryAsync(
             string stageName = GetStringPropertyFromProperties(details.Properties, "System.StageName");
             string submitterJobName = GetStringPropertyFromProperties(details.Properties, "System.JobName");
             string submitterJobDisplayName = GetStringPropertyFromProperties(details.Properties, "System.JobDisplayName");
+            string logicalJobName = GetStringPropertyFromProperties(details.Properties, HelixJobInfo.LogicalJobNamePropertyName);
 
             var newJobInfo = new HelixJobInfo(
                 newJob.Name,
@@ -347,7 +348,8 @@ await RetryAsync(
                 submitterJobDisplayName,
                 details.QueueId,
                 originalJobName,
-                stageAttempt: resubmittedStageAttempt);
+                stageAttempt: resubmittedStageAttempt,
+                logicalJobName: logicalJobName);
 
             _logger.LogInformation("Resubmitted {Count} failed work item(s) from '{OriginalJobName}' as new job '{NewJobName}'{nl}{JobUri}",
                 filteredEntries.Count,
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs
index 2a414ec8c4d..c37e6417a87 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs
@@ -230,7 +230,8 @@ public Task<HelixJobInfo> ResubmitWorkItemsAsync(
                 originalSnapshotJob?.SubmitterJobDisplayName ?? originalJob.SubmitterJobDisplayName,
                 originalSnapshotJob?.QueueId ?? originalJob.QueueId,
                 originalJobName,
-                stageAttempt: resubmittedStageAttempt);
+                stageAttempt: resubmittedStageAttempt,
+                logicalJobName: originalSnapshotJob?.LogicalJobName ?? originalJob.LogicalJobName);
             ResubmittedJobInfos.Add(newJobInfo);
             return Task.FromResult(newJobInfo);
         }
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
index 9dd2a8a3f9c..700fff8cf1a 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
@@ -1237,6 +1237,56 @@ public async Task OneSubmitter_FansOutToMultipleQueues_SameWorkItemName_FailureN
             exitCode.Should().Be(1);
         }
 
+        /// <summary>
+        /// Regression: a single AzDO job can submit multiple independent Helix jobs to the same
+        /// queue. Their work-item names can overlap, so the submitter and queue alone are not a
+        /// unique stream identifier. The submitter-assigned logical Helix job name must preserve
+        /// one job's failure when a later sibling job reports the same work-item name as passed.
+        /// </summary>
+        [Fact]
+        public async Task OneSubmitter_SameQueue_DifferentLogicalJobs_FailureNotOverwrittenByPass()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+            var logger = new RecordingLogger();
+
+            azdo.AddTimelineResponse(
+                MonitorJob(),
+                PipelineJob("__default", "completed", "succeeded"));
+
+            helix.AddResponse(
+                jobs:
+                [
+                    HelixJob(
+                        "helix-wasm-tests",
+                        "finished",
+                        submitterJobName: "__default",
+                        queueId: "azurelinux.3.amd64.open.rt",
+                        logicalJobName: "WasmTestOnChrome-MONO-ST"),
+                    HelixJob(
+                        "helix-wasm-build-tests",
+                        "finished",
+                        submitterJobName: "__default",
+                        queueId: "azurelinux.3.amd64.open.rt",
+                        logicalJobName: "WasmBuildTests"),
+                ],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-wasm-tests"] = PassFail(failed: ["System.Formats.Tar.Manual.Tests"]),
+                    ["helix-wasm-build-tests"] = PassFail(passed: ["System.Formats.Tar.Manual.Tests"]),
+                });
+
+            var runner = CreateRunner(azdo, helix, logger: logger);
+            int exitCode = await runner.RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(1);
+            logger.Messages.Should().Contain(message =>
+                message.Contains("Work items: 2 submitted / 1 resubmitted / 1 failed", StringComparison.Ordinal));
+            logger.Messages.Should().Contain(message =>
+                message.Contains("Failed work item information:", StringComparison.Ordinal)
+                && message.Contains("System.Formats.Tar.Manual.Tests", StringComparison.Ordinal));
+        }
+
         [Fact]
         public async Task StageRerun_UploadsNewHelixWorkItemsWithoutReuploadingPreviousWorkItems()
         {
@@ -1524,9 +1574,9 @@ public async Task AttemptScoped_FastRerun_CurrentIncarnationExists_DoesNotResubm
             // Previous incarnation still running; fresh current incarnation of the SAME stream
             // (same submitter + queue), not linked by PreviousHelixJobName.
             HelixJobInfo previousRunning = HelixJob("helix-x-a1", "running", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1");
+                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1", logicalJobName: "tests");
             HelixJobInfo currentDone = HelixJob("helix-x-a2", "finished", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "2");
+                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "2", logicalJobName: "tests");
             helix.WithWorkItems("helix-x-a1",
                 [new WorkItemSummary("helix-x-a1/wi", "helix-x-a1", "wi", "Running")]);
 
@@ -1571,9 +1621,9 @@ public async Task AttemptScoped_UnlinkedRerunDuplicates_HigherAttemptWinsOutcome
             // sorts AFTER "aaa-new" (attempt 2, passed): a job-name-ordered reconciliation would
             // let the failed attempt-1 outcome overwrite the passing attempt-2 one.
             HelixJobInfo oldFailed = HelixJob("zzz-old", "finished", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1");
+                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1", logicalJobName: "tests");
             HelixJobInfo newPassed = HelixJob("aaa-new", "finished", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "2");
+                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "2", logicalJobName: "tests");
 
             helix.AddResponse(
                 jobs: [oldFailed, newPassed],
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs
index a79dae81f4a..6ebeca2595b 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs
@@ -54,7 +54,8 @@ public static HelixJobInfo HelixJob(
             string queueId = null,
             string previousHelixJobName = null,
             int? initialWorkItemCount = null,
-            string stageAttempt = null)
+            string stageAttempt = null,
+            string logicalJobName = null)
             => new(
                 jobName,
                 status,
@@ -64,7 +65,8 @@ public static HelixJobInfo HelixJob(
                 queueId: queueId,
                 previousHelixJobName: previousHelixJobName,
                 initialWorkItemCount: initialWorkItemCount,
-                stageAttempt: stageAttempt);
+                stageAttempt: stageAttempt,
+                logicalJobName: logicalJobName);
 
         public static HelixJobPassFail PassFail(string[] passed = null, string[] failed = null)
             => new(passed ?? [], failed ?? []);

From f94829acd82fb3c9a1246bd14f05774e74338db2 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Thu, 13 Aug 2026 06:49:54 -0700
Subject: [PATCH 09/21] Use stable Helix submitter stream identity

Prefer System.PhaseName when reconciling work-item outcomes because runtime stamps System.JobName=__default across independent matrix jobs. Preserve phase identity through resubmission, move semantic documentation into the new Design tree, and set the runtime-validated upload parallelism default to 48.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../core-templates/job/helix-job-monitor.yml  |  2 +-
 .../JobMonitor/Design/Components/Polling.md   |  2 +-
 .../Design/Components/UploadPipeline.md       |  5 +-
 .../JobMonitor/Design/README.md               |  2 +-
 .../SemanticBehavior.md}                      | 27 +++++-----
 .../JobMonitor/JobMonitorOptions.cs           |  4 +-
 .../JobMonitor/JobMonitorRunner.cs            |  6 +--
 .../JobMonitor/Models/HelixJobInfo.cs         | 23 ++++++--
 .../JobMonitor/MonitorState.cs                | 24 +++++----
 .../JobMonitor/Services/HelixService.cs       |  4 +-
 .../AzureDevOpsResultPublisherTests.cs        |  4 +-
 .../Fakes/FakeHelixService.cs                 |  3 +-
 .../JobMonitorRunnerTests.cs                  | 52 +++++++++++++++++++
 .../ScenarioHelpers/ScenarioHelpers.cs        |  6 ++-
 14 files changed, 123 insertions(+), 41 deletions(-)
 rename src/Microsoft.DotNet.Helix/JobMonitor/{JobMonitorRunner.Design.md => Design/SemanticBehavior.md} (96%)

diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml
index c89205b11b2..3148e718caf 100644
--- a/eng/common/core-templates/job/helix-job-monitor.yml
+++ b/eng/common/core-templates/job/helix-job-monitor.yml
@@ -66,7 +66,7 @@ parameters:
 # uploaded concurrently.
 - name: testResultUploadParallelism
   type: number
-  default: 8
+  default: 48
 
 # When 'true' (the default), Helix work items that exit 0 but have failed AzDO test results
 # are treated as failed: they count toward the monitor's exit code and are resubmitted by a
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
index 99e96274e09..5555c25f647 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
@@ -15,7 +15,7 @@ is reused for:
 No second service call is made for status.
 
 The one-shot entry retry pass and stage-attempt semantics are specified in
-[the semantic document](../../JobMonitorRunner.Design.md). Outcome updates are
+[the semantic document](../SemanticBehavior.md). Outcome updates are
 applied oldest-to-newest so resubmissions and higher stage attempts supersede
 older failures without allowing identically named work from different
 submitter/queue/logical-job streams to collide.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
index c00a3b886bd..98347d76082 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
@@ -34,7 +34,10 @@ Work-item concurrency is global. A build with many jobs therefore cannot create
 an unbounded task graph or multiply the configured Azure DevOps pressure.
 Consumers can tune this global budget through the
 `testResultUploadParallelism` pipeline-template parameter, which forwards to
-the monitor's `--test-result-upload-parallelism` option.
+the monitor's `--test-result-upload-parallelism` option. The default is 48,
+selected from runtime validation with approximately 6,800 work items and
+3.1 million results: it kept final drain below 2% while reducing service
+throttling guidance compared with 64 workers.
 
 ## Finalization
 
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md
index 5f888c4e805..eb836961601 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md
@@ -7,7 +7,7 @@ upload latency.
 
 The design is split by concern:
 
-- [Semantic behavior](../JobMonitorRunner.Design.md) defines externally
+- [Semantic behavior](SemanticBehavior.md) defines externally
   observable behavior and restart invariants.
 - [Architecture](Architecture.md) describes process structure, ownership,
   backpressure, and shared parallelism utilities.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
similarity index 96%
rename from src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md
rename to src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
index e0aefe1b8db..a9a30d8a816 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
@@ -1,9 +1,10 @@
-# JobMonitorRunner — Technical Specification
+# Semantic behavior
 
-This document is a behavioral specification of the Helix job monitor runner.
-It describes *what* the runner must do, not *how* it currently does it.
+This document specifies the externally observable behavior and restart
+invariants of the Helix Job Monitor. It describes *what* the monitor must do,
+not *how* it currently does it.
 
-The current source lives at [JobMonitorRunner.cs](JobMonitorRunner.cs); use
+The current source lives at [JobMonitorRunner.cs](../JobMonitorRunner.cs); use
 it only as the reference implementation, not as the specification.
 
 ---
@@ -420,23 +421,25 @@ The chain key must be deterministic and uniqueness-preserving:
   preserved.
 - An original Helix job and its resubmission(s) on the same queue must
   produce the same key so the latest incarnation overwrites the older one.
-- Because the chain key is built from the AzDO `System.JobName`, queue, and
+- Because the chain key is built from the AzDO `System.PhaseName`, queue, and
   submitter-assigned Helix `jobName` — all stable across stage attempts — a
   rerun-stage incarnation of the same logical job on the same queue collapses
-  onto the same key as its previous-attempt
-  counterpart, even though the two Helix jobs are **not** linked by
+  onto the same key as its previous-attempt counterpart, even though the two
+  Helix jobs are **not** linked by
   `PreviousHelixJobName` (only monitor resubmissions set that link). The map
   must therefore let the **later stage attempt win** when two incarnations share
   a key: outcomes must be applied in order of (lineage depth, then stage
   attempt), not by Helix job-name sort, or a stale previous-attempt outcome
-  could nondeterministically overwrite the current one.
+  could nondeterministically overwrite the current one. `System.JobName` is
+  used only when `System.PhaseName` is unavailable because some pipelines stamp
+  every matrix job with `System.JobName=__default`.
 - If lineage cannot be resolved (the predecessor link points outside the
   jobs the runner has observed), the key falls back to a Helix-job-bound
   identifier so independent jobs don't collide.
-- Multiple independent Helix jobs submitted by the same AzDO job to the same
-  queue remain distinct because their submitter-assigned `jobName` values
-  differ. This prevents a passing work item in one logical job from erasing a
-  same-named failure in another.
+- Multiple independent AzDO phases and logical Helix jobs targeting the same
+  queue remain distinct because `System.PhaseName` and submitter-assigned
+  `jobName` are both part of the key. This prevents a passing work item in one
+  stream from erasing a same-named failure in another.
 
 The same key drives a parallel map of "failed work item console info" used
 to build the final failure report. When a later incarnation of a work item
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs
index 850b2c04d5b..53ada6a35c9 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs
@@ -55,7 +55,7 @@ public sealed class JobMonitorOptions
         /// </summary>
         public string StageAttempt { get; set; }
 
-        public int TestResultUploadParallelism { get; set; } = 8;
+        public int TestResultUploadParallelism { get; set; } = 48;
 
         public TestResultAttachmentMode TestResultAttachmentMode { get; set; } = TestResultAttachmentMode.Failed;
 
@@ -166,7 +166,7 @@ public static JobMonitorOptions Parse(string[] args)
             Option<int> testResultUploadParallelismOption = new("--test-result-upload-parallelism")
             {
                 Description = "Maximum number of work items whose test results can be uploaded to Azure DevOps in parallel.",
-                DefaultValueFactory = _ => 8
+                DefaultValueFactory = _ => 48
             };
 
             Option<TestResultAttachmentMode> testResultAttachmentModeOption = new("--test-result-attachment-mode")
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
index 32b5a9b8253..a43f64d702e 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
@@ -16,7 +16,7 @@
 namespace Microsoft.DotNet.Helix.JobMonitor
 {
     /// <summary>
-    /// Orchestrates the per-invocation lifecycle described in <c>JobMonitorRunner.Design.md</c>:
+    /// Orchestrates the per-invocation lifecycle described in <c>Design/SemanticBehavior.md</c>:
     /// one-shot retry pass, poll loop (with upload + outcome reconciliation per iteration),
     /// final summary on completion, and timeout/cancel handling. All heavy lifting
     /// (status logging, uploads, state) lives in dedicated helpers.
@@ -125,7 +125,7 @@ public async Task<int> RunAsync(CancellationToken cancellationToken)
                 // treated as "processed" once their Azure DevOps test run reaches the Completed
                 // state (the final upload step), so a job whose upload did not finish here is
                 // re-uploaded in full by a subsequent monitor invocation. See
-                // JobMonitorRunner.Design.md ("Crash and timeout resilience").
+                // Design/SemanticBehavior.md ("Crash and timeout resilience").
                 _reporter.ReportTimeout();
 
                 // Proactively cancel any Helix jobs we know about that haven't finished yet so
@@ -161,7 +161,7 @@ public async Task<int> RunAsync(CancellationToken cancellationToken)
         /// unfinished work (or a current attempt's completed-with-failures work), resubmits the
         /// not-yet-passed items into the current attempt. Returns the (stage snapshot ∪
         /// resubmitted jobs) so the first poll iteration sees the resubmissions immediately.
-        /// See JobMonitorRunner.Design.md §2.1 and §2.3.
+        /// See Design/SemanticBehavior.md §2.1 and §2.3.
         /// </summary>
         private async Task<IReadOnlyList<HelixJobInfo>> ExecuteRetryPassAsync(CancellationToken cancellationToken)
         {
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs
index b8e541a1d69..39cd0f5525f 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs
@@ -22,7 +22,7 @@ public sealed class HelixJobInfo
         /// Helix job property that records the Azure DevOps stage attempt during which the job
         /// was submitted. This is the exact AzDO predefined-variable name that the Helix submitter
         /// (<c>SendHelixJob</c>) copies onto every job, so the monitor reads and re-stamps the
-        /// same property when resubmitting (see JobMonitorRunner.Design.md §2.3).
+        /// same property when resubmitting (see Design/SemanticBehavior.md §2.3).
         /// </summary>
         public const string StageAttemptPropertyName = "System.StageAttempt";
 
@@ -49,7 +49,8 @@ public HelixJobInfo(
             string previousHelixJobName = null,
             int? initialWorkItemCount = null,
             string stageAttempt = null,
-            string logicalJobName = null)
+            string logicalJobName = null,
+            string submitterPhaseName = null)
         {
             JobName = jobName ?? throw new ArgumentNullException(nameof(jobName));
             Status = status ?? throw new ArgumentNullException(nameof(status));
@@ -65,7 +66,8 @@ public HelixJobInfo(
                 submitterJobDisplayName,
                 previousHelixJobName,
                 stageAttempt,
-                logicalJobName);
+                logicalJobName,
+                submitterPhaseName);
         }
 
         public string JobName { get; }
@@ -102,6 +104,13 @@ public HelixJobInfo(
 
         public string SubmitterJobName => GetStringProperty(Properties, "System.JobName");
 
+        /// <summary>
+        /// Stable Azure DevOps phase/job reference name copied by the Helix SDK submitter.
+        /// Runtime pipelines can report <c>System.JobName</c> as <c>__default</c> for many
+        /// independent matrix jobs, while <c>System.PhaseName</c> remains unique and stable.
+        /// </summary>
+        public string SubmitterPhaseName => GetStringProperty(Properties, "System.PhaseName");
+
         /// <summary>
         /// Stable logical name assigned by the Helix SDK submitter. A single Azure DevOps job
         /// can submit multiple Helix jobs to the same queue, so this value distinguishes those
@@ -217,7 +226,8 @@ private static JObject CreateProperties(
             string submitterJobDisplayName,
             string previousHelixJobName,
             string stageAttempt,
-            string logicalJobName)
+            string logicalJobName,
+            string submitterPhaseName)
         {
             var properties = new JObject();
 
@@ -256,6 +266,11 @@ private static JObject CreateProperties(
                 properties[LogicalJobNamePropertyName] = logicalJobName;
             }
 
+            if (!string.IsNullOrEmpty(submitterPhaseName))
+            {
+                properties["System.PhaseName"] = submitterPhaseName;
+            }
+
             return properties;
         }
     }
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
index e95cf501166..77fa512713e 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
@@ -399,11 +399,12 @@ public IReadOnlyList<FailedWorkItemConsoleInfo> SnapshotFailedWorkItemConsoleInf
 
         /// <summary>
         /// Produces a key that rolls up work-item outcomes within a logical Helix work stream.
-        /// The AzDO submitter name, Helix queue, and submitter-assigned logical job name jointly
-        /// identify a stream across stage reruns and monitor resubmissions. The logical job name
-        /// is essential because one AzDO job can submit multiple independent Helix jobs to the
-        /// same queue. When stable submitter metadata is unavailable, lineage is followed back
-        /// through <c>PreviousHelixJobName</c> and the root Helix job name is used instead.
+        /// The AzDO phase name, Helix queue, and submitter-assigned logical job name jointly
+        /// identify a stream across stage reruns and monitor resubmissions. The phase name is
+        /// preferred because some pipelines stamp many independent jobs with
+        /// <c>System.JobName=__default</c>. When stable submitter metadata is unavailable,
+        /// lineage is followed back through <c>PreviousHelixJobName</c> and the root Helix job
+        /// name is used instead.
         /// </summary>
         public string GetSubmitterChainKey(HelixJobInfo job)
         {
@@ -416,17 +417,20 @@ public string GetSubmitterChainKey(HelixJobInfo job)
         private string GetSubmitterChainKeyLocked(HelixJobInfo job)
         {
             HelixJobInfo root = GetLineageRootLocked(job);
-            string submitterJobName = job.SubmitterJobName ?? root.SubmitterJobName;
+            string submitterName = job.SubmitterPhaseName
+                ?? root.SubmitterPhaseName
+                ?? job.SubmitterJobName
+                ?? root.SubmitterJobName;
             string queueId = job.QueueId ?? root.QueueId;
             string logicalJobName = job.LogicalJobName
                 ?? root.LogicalJobName
                 ?? job.TestRunName
                 ?? root.TestRunName;
 
-            if (!string.IsNullOrEmpty(submitterJobName)
+            if (!string.IsNullOrEmpty(submitterName)
                 && !string.IsNullOrEmpty(logicalJobName))
             {
-                return FormatSubmitterChainKey(submitterJobName, queueId, logicalJobName);
+                return FormatSubmitterChainKey(submitterName, queueId, logicalJobName);
             }
 
             return $"helix:{root.JobName}";
@@ -452,10 +456,10 @@ private HelixJobInfo GetLineageRootLocked(HelixJobInfo job)
         }
 
         private static string FormatSubmitterChainKey(
-            string submitterJobName,
+            string submitterName,
             string queueId,
             string logicalJobName)
-            => $"submitter:{submitterJobName}|queue:{queueId ?? string.Empty}|job:{logicalJobName}";
+            => $"submitter:{submitterName}|queue:{queueId ?? string.Empty}|job:{logicalJobName}";
 
         /// <summary>
         /// From an arbitrary set of Helix jobs (possibly spanning multiple stage attempts),
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
index c0431d74fe2..333df0b2e21 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
@@ -338,6 +338,7 @@ await RetryAsync(
             string submitterJobName = GetStringPropertyFromProperties(details.Properties, "System.JobName");
             string submitterJobDisplayName = GetStringPropertyFromProperties(details.Properties, "System.JobDisplayName");
             string logicalJobName = GetStringPropertyFromProperties(details.Properties, HelixJobInfo.LogicalJobNamePropertyName);
+            string submitterPhaseName = GetStringPropertyFromProperties(details.Properties, "System.PhaseName");
 
             var newJobInfo = new HelixJobInfo(
                 newJob.Name,
@@ -349,7 +350,8 @@ await RetryAsync(
                 details.QueueId,
                 originalJobName,
                 stageAttempt: resubmittedStageAttempt,
-                logicalJobName: logicalJobName);
+                logicalJobName: logicalJobName,
+                submitterPhaseName: submitterPhaseName);
 
             _logger.LogInformation("Resubmitted {Count} failed work item(s) from '{OriginalJobName}' as new job '{NewJobName}'{nl}{JobUri}",
                 filteredEntries.Count,
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
index 7d66d3e9cc0..de483430b69 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
@@ -34,9 +34,9 @@ public void AttachmentModeDefaultsToFailed()
         }
 
         [Fact]
-        public void JobMonitorUploadParallelismDefaultsToEight()
+        public void JobMonitorUploadParallelismDefaultsToFortyEight()
         {
-            Assert.Equal(8, new JobMonitorOptions().TestResultUploadParallelism);
+            Assert.Equal(48, new JobMonitorOptions().TestResultUploadParallelism);
         }
 
         [Fact]
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs
index c37e6417a87..edbab2dc0a7 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs
@@ -231,7 +231,8 @@ public Task<HelixJobInfo> ResubmitWorkItemsAsync(
                 originalSnapshotJob?.QueueId ?? originalJob.QueueId,
                 originalJobName,
                 stageAttempt: resubmittedStageAttempt,
-                logicalJobName: originalSnapshotJob?.LogicalJobName ?? originalJob.LogicalJobName);
+                logicalJobName: originalSnapshotJob?.LogicalJobName ?? originalJob.LogicalJobName,
+                submitterPhaseName: originalSnapshotJob?.SubmitterPhaseName ?? originalJob.SubmitterPhaseName);
             ResubmittedJobInfos.Add(newJobInfo);
             return Task.FromResult(newJobInfo);
         }
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
index 700fff8cf1a..02a17296bed 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
@@ -1287,6 +1287,58 @@ public async Task OneSubmitter_SameQueue_DifferentLogicalJobs_FailureNotOverwrit
                 && message.Contains("System.Formats.Tar.Manual.Tests", StringComparison.Ordinal));
         }
 
+        /// <summary>
+        /// Production regression: runtime stamps <c>System.JobName=__default</c> onto many
+        /// independent matrix jobs. Some of those jobs submit the same logical Helix scenario
+        /// to the same queue, so only <c>System.PhaseName</c> distinguishes their streams.
+        /// A pass from one phase must not erase a same-named failure from another phase.
+        /// </summary>
+        [Fact]
+        public async Task DefaultJobName_SameQueueAndLogicalJob_DifferentPhases_FailureNotOverwrittenByPass()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+            var logger = new RecordingLogger();
+
+            azdo.AddTimelineResponse(
+                MonitorJob(),
+                PipelineJob("__default", "completed", "succeeded"));
+
+            helix.AddResponse(
+                jobs:
+                [
+                    HelixJob(
+                        "helix-wasm-eat",
+                        "finished",
+                        submitterJobName: "__default",
+                        submitterPhaseName: "build_browser_wasm_linux_Release_LibraryTests_EAT",
+                        queueId: "azurelinux.3.amd64.open.rt",
+                        logicalJobName: "WasmTestOnChrome-MONO-ST"),
+                    HelixJob(
+                        "helix-wasm-smoke",
+                        "finished",
+                        submitterJobName: "__default",
+                        submitterPhaseName: "build_browser_wasm_linux_Release_LibraryTests_Smoke",
+                        queueId: "azurelinux.3.amd64.open.rt",
+                        logicalJobName: "WasmTestOnChrome-MONO-ST"),
+                ],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-wasm-eat"] = PassFail(failed: ["System.Formats.Tar.Manual.Tests"]),
+                    ["helix-wasm-smoke"] = PassFail(passed: ["System.Formats.Tar.Manual.Tests"]),
+                });
+
+            var runner = CreateRunner(azdo, helix, logger: logger);
+            int exitCode = await runner.RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(1);
+            logger.Messages.Should().Contain(message =>
+                message.Contains("Work items: 2 submitted / 1 resubmitted / 1 failed", StringComparison.Ordinal));
+            logger.Messages.Should().Contain(message =>
+                message.Contains("Failed work item information:", StringComparison.Ordinal)
+                && message.Contains("System.Formats.Tar.Manual.Tests", StringComparison.Ordinal));
+        }
+
         [Fact]
         public async Task StageRerun_UploadsNewHelixWorkItemsWithoutReuploadingPreviousWorkItems()
         {
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs
index 6ebeca2595b..dbdcf9b277f 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs
@@ -55,7 +55,8 @@ public static HelixJobInfo HelixJob(
             string previousHelixJobName = null,
             int? initialWorkItemCount = null,
             string stageAttempt = null,
-            string logicalJobName = null)
+            string logicalJobName = null,
+            string submitterPhaseName = null)
             => new(
                 jobName,
                 status,
@@ -66,7 +67,8 @@ public static HelixJobInfo HelixJob(
                 previousHelixJobName: previousHelixJobName,
                 initialWorkItemCount: initialWorkItemCount,
                 stageAttempt: stageAttempt,
-                logicalJobName: logicalJobName);
+                logicalJobName: logicalJobName,
+                submitterPhaseName: submitterPhaseName);
 
         public static HelixJobPassFail PassFail(string[] passed = null, string[] failed = null)
             => new(passed ?? [], failed ?? []);

From f3b524b34d7df2b18cc47f6c9da54349bff337e8 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Thu, 13 Aug 2026 07:01:44 -0700
Subject: [PATCH 10/21] Document same-queue Helix stream identity

Specify that AzDO job identity and queue are insufficient when one job submits multiple Helix jobs to the same queue. Define the logical job discriminator, resubmission preservation, safe fallback, and submitter uniqueness requirement.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../JobMonitor/Design/Architecture.md         | 12 ++++++
 .../JobMonitor/Design/Components/Polling.md   |  6 +++
 .../JobMonitor/Design/SemanticBehavior.md     | 43 +++++++++++++------
 3 files changed, 49 insertions(+), 12 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md
index 119901c5547..8a5948008d1 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md
@@ -65,6 +65,18 @@ gate, and upload pipeline. It records atomic request counts and operation
 timings without emitting per-request information logs. The final aggregate
 report is described in [Performance metrics](Components/PerformanceMetrics.md).
 
+## Logical stream identity
+
+Retry and outcome reconciliation operate on logical Helix job streams rather
+than AzDO jobs. An AzDO job can submit multiple Helix jobs to the same queue,
+so AzDO job identity plus queue does not uniquely identify a stream.
+
+`MonitorState` combines the AzDO phase identity, queue, and submitter-assigned
+logical Helix `jobName` (falling back to `TestRunName`). Resubmissions preserve
+these properties and add `PreviousHelixJobName`, so a resubmission chains to
+its original job while sibling submissions from the same AzDO job and queue
+remain independent.
+
 ## Durability boundary
 
 The only durable "processed" marker is the Helix-job tag on a completed Azure
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
index 5555c25f647..6dcd3295838 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
@@ -19,3 +19,9 @@ The one-shot entry retry pass and stage-attempt semantics are specified in
 applied oldest-to-newest so resubmissions and higher stage attempts supersede
 older failures without allowing identically named work from different
 submitter/queue/logical-job streams to collide.
+
+The logical stream key is not merely the AzDO job name plus queue. One AzDO job
+may submit multiple Helix jobs to the same queue, so the key also includes the
+submitter-assigned Helix `jobName` (or `TestRunName` when `jobName` is absent).
+That discriminator is preserved by resubmission, allowing retries to collapse
+only actual incarnations of the same logical Helix job.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
index a9a30d8a816..64279dbbe59 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
@@ -105,9 +105,11 @@ be the source of truth for cross-invocation correctness.
 
 Retry is the mechanism that reconciles previous-attempt work into the current
 attempt (§2.1). It operates on *logical work streams*, not on attempts: a work
-stream is identified by the submitter chain key (§5.7) — the AzDO `System.JobName`
-plus the Helix queue — which is stable across both stage attempts and monitor
-resubmissions.
+stream is identified by the submitter chain key (§5.7). The key combines the
+stable AzDO phase identity, Helix queue, and logical Helix job identity; the
+AzDO job name and queue alone are not unique because one AzDO job may submit
+multiple independent Helix jobs to the same queue. The combined identity is
+stable across both stage attempts and monitor resubmissions.
 
 1. Retry runs exactly once per invocation, on entry, before polling begins.
 2. The set of work to resubmit is decided from a single Helix snapshot taken on
@@ -186,6 +188,13 @@ addresses each:
    never run again would loop forever under any "just wait" or "just resubmit and
    wait" scheme. → Resubmission-not-possible is treated as an actionable hard
    failure so the invocation fails fast instead of hanging (§2.3.3).
+7. **One AzDO job submits multiple Helix jobs to the same queue.** Grouping only
+   by AzDO job name and queue incorrectly merges independent streams. A failure
+   in one job can be overwritten by a same-named passing work item in another,
+   and retry can select only one of the jobs. → The stream key also includes the
+   submitter-assigned logical Helix `jobName` (falling back to `TestRunName`).
+   Resubmissions preserve that property, so incarnations of one logical job
+   still chain while sibling Helix jobs remain independent.
 
 ### 2.4 Upload invariants
 
@@ -419,10 +428,19 @@ The chain key must be deterministic and uniqueness-preserving:
 - A single AzDO matrix leg that fans out to multiple Helix queues must
   produce distinct keys (one per queue) so per-queue failures are
   preserved.
+- A single AzDO job that submits multiple independent Helix jobs to the same
+  queue must produce distinct keys (one per logical Helix job). The AzDO
+  phase/job name and queue are therefore necessary but not sufficient.
 - An original Helix job and its resubmission(s) on the same queue must
   produce the same key so the latest incarnation overwrites the older one.
-- Because the chain key is built from the AzDO `System.PhaseName`, queue, and
-  submitter-assigned Helix `jobName` — all stable across stage attempts — a
+- The preferred key components are:
+  1. `System.PhaseName`, falling back to `System.JobName`;
+  2. the Helix queue;
+  3. the submitter-assigned Helix `jobName`, falling back to `TestRunName`.
+  If no stable logical-job discriminator is available, the key is bound to the
+  root Helix job in the `PreviousHelixJobName` lineage rather than risk merging
+  unrelated jobs.
+- Because these key components are stable across stage attempts, a
   rerun-stage incarnation of the same logical job on the same queue collapses
   onto the same key as its previous-attempt counterpart, even though the two
   Helix jobs are **not** linked by
@@ -433,13 +451,14 @@ The chain key must be deterministic and uniqueness-preserving:
   could nondeterministically overwrite the current one. `System.JobName` is
   used only when `System.PhaseName` is unavailable because some pipelines stamp
   every matrix job with `System.JobName=__default`.
-- If lineage cannot be resolved (the predecessor link points outside the
-  jobs the runner has observed), the key falls back to a Helix-job-bound
-  identifier so independent jobs don't collide.
-- Multiple independent AzDO phases and logical Helix jobs targeting the same
-  queue remain distinct because `System.PhaseName` and submitter-assigned
-  `jobName` are both part of the key. This prevents a passing work item in one
-  stream from erasing a same-named failure in another.
+- If lineage cannot be resolved (the predecessor link points outside the jobs
+  the runner has observed), the root predecessor name provides the
+  Helix-job-bound fallback.
+- The submitter must assign different `jobName` values (or, when absent,
+  different `TestRunName` values) to independent Helix jobs submitted by the
+  same AzDO phase to the same queue. Without a stable distinguishing property,
+  no monitor can reliably correlate an unlinked stage-rerun job with its prior
+  incarnation while also distinguishing it from a sibling submission.
 
 The same key drives a parallel map of "failed work item console info" used
 to build the final failure report. When a later incarnation of a work item

From bedde536e2d02ce256d2690df574e70d277186f1 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Thu, 13 Aug 2026 09:03:52 -0700
Subject: [PATCH 11/21] Correct Job Monitor retry attempt semantics

Use Azure DevOps submitter job attempts to distinguish selective retries from full-stage reruns, preserve lineage metadata, and add scenario coverage for retry races and stage isolation.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../core-templates/job/helix-job-monitor.yml  |   1 +
 .../JobMonitor/Design/Architecture.md         |  11 +-
 .../JobMonitor/Design/Components/Polling.md   |  16 +-
 .../JobMonitor/Design/SemanticBehavior.md     | 132 +--
 .../JobMonitor/Interfaces/IHelixService.cs    |   7 +-
 .../JobMonitor/JobMonitorOptions.cs           |  14 +
 .../JobMonitor/JobMonitorRunner.cs            | 127 ++-
 .../JobMonitor/Models/HelixJobInfo.cs         |  18 +
 .../JobMonitor/MonitorState.cs                |  22 +-
 .../JobMonitor/Services/HelixService.cs       |   9 +
 .../Fakes/FakeHelixService.cs                 |  11 +-
 .../HelixServiceTests.cs                      |  22 +-
 .../JobMonitorRunnerTests.cs                  | 782 +++++++++++++-----
 .../ScenarioHelpers/ScenarioHelpers.cs        |  21 +-
 14 files changed, 884 insertions(+), 309 deletions(-)

diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml
index 3148e718caf..53bbf74927e 100644
--- a/eng/common/core-templates/job/helix-job-monitor.yml
+++ b/eng/common/core-templates/job/helix-job-monitor.yml
@@ -221,6 +221,7 @@ jobs:
         --max-wait-minutes            "$((${{ parameters.timeoutInMinutes }} - 5))" # Set the tool's timeout slightly lower than the Azure DevOps job timeout to allow it to exit gracefully.
         --stage-name                  '$(System.StageName)'
         --stage-attempt               '$(System.StageAttempt)'
+        --job-attempt                 '$(System.JobAttempt)'
         --test-result-upload-parallelism '${{ parameters.testResultUploadParallelism }}'
       )
 
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md
index 8a5948008d1..e60e27122fb 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Architecture.md
@@ -71,11 +71,12 @@ Retry and outcome reconciliation operate on logical Helix job streams rather
 than AzDO jobs. An AzDO job can submit multiple Helix jobs to the same queue,
 so AzDO job identity plus queue does not uniquely identify a stream.
 
-`MonitorState` combines the AzDO phase identity, queue, and submitter-assigned
-logical Helix `jobName` (falling back to `TestRunName`). Resubmissions preserve
-these properties and add `PreviousHelixJobName`, so a resubmission chains to
-its original job while sibling submissions from the same AzDO job and queue
-remain independent.
+`MonitorState` combines the stage identity, AzDO phase identity, queue, and
+submitter-assigned logical Helix `jobName` (falling back to `TestRunName`).
+Attempts are incarnation metadata rather than key components. Retry compares
+the preserved submitter `System.JobAttempt` with the current timeline record;
+resubmissions stamp the current stage attempt, preserve the submitter attempt,
+and add `PreviousHelixJobName`.
 
 ## Durability boundary
 
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
index 6dcd3295838..649c4aa5edc 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/Polling.md
@@ -14,14 +14,14 @@ is reused for:
 
 No second service call is made for status.
 
-The one-shot entry retry pass and stage-attempt semantics are specified in
-[the semantic document](../SemanticBehavior.md). Outcome updates are
-applied oldest-to-newest so resubmissions and higher stage attempts supersede
-older failures without allowing identically named work from different
-submitter/queue/logical-job streams to collide.
+The one-shot entry retry pass and stage/job-attempt semantics are specified in
+[the semantic document](../SemanticBehavior.md). It uses the same timeline
+snapshot as the first poll to compare each Helix stream's preserved submitter
+job attempt with the current timeline job attempt. Outcome updates are applied
+oldest-to-newest so newer stage/job attempts and resubmission lineage supersede
+older failures.
 
-The logical stream key is not merely the AzDO job name plus queue. One AzDO job
-may submit multiple Helix jobs to the same queue, so the key also includes the
+The logical stream key includes stage, AzDO phase/job identity, queue, and the
 submitter-assigned Helix `jobName` (or `TestRunName` when `jobName` is absent).
 That discriminator is preserved by resubmission, allowing retries to collapse
-only actual incarnations of the same logical Helix job.
+only actual incarnations of the same logical Helix job without crossing stages.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
index 64279dbbe59..57eaf56d3b9 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
@@ -33,12 +33,11 @@ principle has two halves that must both hold:
    never reach a terminal state (for example, work items stranded in `Waiting`
    after their queue was purged), and a superseded attempt's monitor is already
    gone, so nothing else will ever drive it. Waiting on it means waiting forever.
-2. **All pipeline-submitted work must still complete.** The monitor cannot simply
-   discard a previous attempt's work either: that work represents tests the
-   pipeline asked to run. Any previous-attempt work that is not already
-   terminally passed (i.e. it failed or is still unfinished) and that the current
-   attempt has *not* already re-submitted must be **resubmitted into the current
-   attempt** so it is actually carried to completion (§2.3).
+2. **Only work whose submitter did not rerun may be replayed.** Previous-stage
+   work still represents the current execution only when the matching AzDO
+   submitter job remains at the same `System.JobAttempt`. If the timeline shows
+   a newer submitter attempt, the old Helix work is superseded and must not be
+   resubmitted, even before the replacement Helix job is visible.
 
 Concretely, all decisions (retry, completion gating, upload, pass/fail) consider:
 
@@ -53,30 +52,31 @@ Concretely, all decisions (retry, completion gating, upload, pass/fail) consider
 Jobs and work items from other stages must not be retried, uploaded, or used to
 fail this invocation.
 
-**Why per-attempt, and why not just ignore previous attempts.** Azure DevOps
-offers two distinct re-run gestures, and the monitor cannot tell them apart from
-the timeline alone:
+**Why both attempt axes are required.** Both Azure DevOps retry gestures advance
+`System.StageAttempt`; the difference is which timeline jobs advance their
+individual `System.JobAttempt`:
 
 - **Rerun the entire stage** — every job re-runs, including the Helix submitter
   jobs, so the current attempt already contains a fresh incarnation of every
   logical work stream. Previous-attempt incarnations are superseded and need no
   resubmission.
-- **Retry failed jobs in the stage** — only failed jobs re-run. If the Helix
+- **Retry failed jobs in the stage** — only selected failed jobs advance their
+  job attempt. If the Helix
   submitter jobs passed and only the monitor failed (e.g. it timed out), the
-  submitters do **not** re-run, so the current attempt contains **no** Helix work
-  at all. Naively scoping to the current attempt would make the monitor exit
+  monitor advances but the submitters remain at their original job attempts, so
+  the current stage attempt contains **no** fresh Helix work. Naively scoping to
+  the current stage attempt would make the monitor exit
   immediately as a success, silently discarding every result and failure from the
   previous attempt.
 
 Because of the second gesture, "current-attempt scope" is not the same as
 "ignore previous attempts." The monitor scopes *gating* to the current attempt
 but reconciles previous-attempt work into it by resubmission (§2.3), deciding
-per logical work stream (not per attempt) whether a resubmission is needed.
+per logical work stream by comparing the Helix job's submitter `System.JobAttempt`
+with the matching current timeline record.
 
-The monitor's own stage attempt is provided as an input (see §3) and defaults to
-the `SYSTEM_STAGEATTEMPT` pipeline variable. When it is unknown the monitor
-cannot distinguish attempts and falls back to build + stage scope, gating on
-every attempt's work (historical behavior).
+The monitor's stage and job attempts are provided as inputs (see §3), defaulting
+to `SYSTEM_STAGEATTEMPT` and `SYSTEM_JOBATTEMPT`.
 
 ### 2.2 Durable state
 
@@ -106,41 +106,44 @@ be the source of truth for cross-invocation correctness.
 Retry is the mechanism that reconciles previous-attempt work into the current
 attempt (§2.1). It operates on *logical work streams*, not on attempts: a work
 stream is identified by the submitter chain key (§5.7). The key combines the
-stable AzDO phase identity, Helix queue, and logical Helix job identity; the
+stage identity, stable AzDO phase identity, Helix queue, and logical Helix job identity; the
 AzDO job name and queue alone are not unique because one AzDO job may submit
 multiple independent Helix jobs to the same queue. The combined identity is
 stable across both stage attempts and monitor resubmissions.
 
-1. Retry runs exactly once per invocation, on entry, before polling begins.
+1. Retry runs exactly once on entry to a **retried monitor job**
+   (`System.JobAttempt > 1`). The initial monitor invocation reports failures
+   but never creates additional Helix work.
 2. The set of work to resubmit is decided from a single Helix snapshot taken on
    entry. Work that fails after the monitor has started is not resubmitted during
    the current invocation; a later invocation may pick it up.
 3. Retry decisions are made per work stream from its **latest incarnation across
-   all attempts** (the leaf of its lineage chain, breaking ties toward the higher
-   stage attempt). Let *L* be that incarnation:
-   - *L* is **still in flight** (running/waiting) and belongs to the **current
-     attempt** — leave it; the current attempt is actively driving it and
-     completion gating waits on it. This is the rerun-entire-stage case and also
-     prevents duplicate submissions when a previous-attempt incarnation of the
-     same stream is still running.
-   - *L* is **still in flight** and belongs to a **previous attempt** — the
-     previous attempt has abandoned it (its monitor is gone and nothing else will
-     drive it); resubmit the not-yet-passed items into the current attempt.
+   all attempts**, ordered by stage attempt, submitter job attempt, explicit
+   lineage depth, and finally Helix job ID. Let *L* be that incarnation:
+   - *L* belongs to the **current stage attempt** — leave it, whether running or
+     already failed. It belongs to the execution currently being monitored.
+   - The current timeline shows the matching submitter at a **higher job
+     attempt** than *L* — leave *L*. The submitter reran and superseded it; wait
+     for the newer submitter execution rather than duplicating it.
+   - The current timeline submitter attempt **equals** *L*'s
+     `System.JobAttempt` — the submitter did not rerun. Failed or unfinished
+     work in *L* may be resubmitted into the current stage attempt.
    - *L* is **completed and fully passed** — nothing to resubmit; its results are
      uploaded (if not already, §2.4) and its outcome counted. It is terminal, so
      it does not block completion.
-   - *L* is **completed with failures** — resubmit the failed items, regardless of
-     attempt. (For a current-attempt incarnation this is the pre-existing
-     per-invocation retry; for a previous-attempt one it carries the failure into
-     the current attempt.)
+   - Missing or incompatible submitter identity / `System.JobAttempt` metadata
+     makes retry classification ambiguous. Do not guess; record an actionable
+     failure rather than risk duplicating a rerun.
    - A needed resubmission is **not possible** (e.g. the queue was removed, so the
      work can never run again) — for previous-attempt in-flight work, whose
      failure is not otherwise recorded, surface it as an actionable hard failure
      so the invocation fails fast rather than waiting forever. (Completed-with-
      failures work that cannot be resubmitted already fails the build via outcome
      reconciliation, §2.5.)
-4. Every resubmission is stamped with the **monitor's current stage attempt**
-   (not the original job's attempt) and linked back via `PreviousHelixJobName`.
+4. Every resubmission is stamped with the **monitor's current stage attempt**,
+   preserves the original submitter's `System.JobAttempt`, records the
+   resubmitting monitor attempt as `JobMonitor.JobAttempt`, and links back via
+   `PreviousHelixJobName`.
    This is what brings the resubmitted work into current-attempt scope so the
    monitor gates on it; copying the original attempt would leave the monitor
    unable to see its own resubmission.
@@ -159,8 +162,9 @@ These are the scenarios that a naive "scope strictly to the current attempt and
 ignore all previous-attempt jobs" design gets wrong, and how the model above
 addresses each:
 
-1. **Retry-failed-jobs where only the monitor re-ran.** The submitters passed and
-   were not re-run, so the current attempt contains no Helix work. Naive scoping
+1. **Retry-failed-jobs where only the monitor re-ran.** The stage and monitor
+   attempts advance, but the submitters retain their original job attempts, so
+   the current stage attempt contains no fresh Helix work. Naive scoping
    exits `0` immediately, discarding every previous-attempt result and failure.
    → The retry pass reconciles previous-attempt streams: passed work is uploaded
    and counted, failed/unfinished work is resubmitted into the current attempt
@@ -175,15 +179,14 @@ addresses each:
    current incarnation) and resubmit them. → Decisions are re-derived from the
    Helix snapshot each invocation (latest incarnation + attempt + status per
    stream), not from in-memory state, so partial progress is self-correcting.
-4. **Previous-attempt work that is still legitimately running during a fast stage
-   rerun.** A rerun submits a fresh current-attempt incarnation while the
-   previous one is still running; blindly resubmitting the previous unfinished
-   work would triple-submit. → When a current-attempt incarnation already exists
-   for a stream, the previous one is left alone (§2.3.3, first bullet).
+4. **Visibility race during a full-stage rerun.** The timeline already shows the
+   submitter at a newer job attempt, but its new Helix job is not visible yet.
+   Blindly replaying the old job duplicates the rerun. → Timeline job attempt,
+   not Helix visibility, suppresses replay.
 5. **Rerun duplicates that are not lineage-linked.** A stage rerun's fresh Helix
    job has no `PreviousHelixJobName` link to its previous-attempt counterpart;
    they collapse only by chain key. → Outcome ordering breaks ties toward the
-   higher stage attempt so the current attempt wins (§5.7).
+   higher stage/job incarnation so the current attempt wins (§5.7).
 6. **Un-resubmittable work (e.g. purged queue).** Previous-attempt work that can
    never run again would loop forever under any "just wait" or "just resubmit and
    wait" scheme. → Resubmission-not-possible is treated as an actionable hard
@@ -195,6 +198,9 @@ addresses each:
    submitter-assigned logical Helix `jobName` (falling back to `TestRunName`).
    Resubmissions preserve that property, so incarnations of one logical job
    still chain while sibling Helix jobs remain independent.
+8. **Identical phase/queue/logical-job names in different stages.** → The stage
+   identity is part of the stream key, so outcomes and retry decisions cannot
+   cross stage boundaries.
 
 ### 2.4 Upload invariants
 
@@ -290,8 +296,9 @@ behaviorally; method names are illustrative.
   submitter (see §5.1). The returned set spans every attempt of the build; the
   runner keeps the whole stage's jobs (all attempts) so the retry pass can
   reconcile previous-attempt work (§2.3), and classifies each job as
-  current- or previous-attempt via `System.StageName` / `System.StageAttempt`
-  for gating (§2.1).
+  current- or previous-stage-attempt via `System.StageName` /
+  `System.StageAttempt`, and compares `System.JobAttempt` with the matching
+  timeline submitter for retry eligibility (§2.1).
 - **List work items for a job** — return all work-item summaries.
 - **Download test results** — given one job/work-item pair, download recognized
   result files into a working directory. Individual
@@ -302,9 +309,10 @@ behaviorally; method names are illustrative.
 - **Resubmit failed work items** — given the original job and a set of
   failed (or unfinished) work items, submit a new Helix job that contains only
   those items. The new job must inherit the original's submitter identity (stage,
-  job name, display name, test-run name, queue) but be stamped with the
-  **resubmitting monitor's current stage attempt** (§2.3.4), and link back via
-  `PreviousHelixJobName`. May return "not possible" (e.g. queue gone), which the
+  job name, display name, test-run name, queue, and submitter job attempt), be
+  stamped with the **resubmitting monitor's current stage attempt** and
+  `JobMonitor.JobAttempt` (§2.3.4), and link back via `PreviousHelixJobName`.
+  May return "not possible" (e.g. queue gone), which the
   runner treats as an actionable hard failure for that work rather than silently
   skipping it (§2.3.3).
 
@@ -342,17 +350,22 @@ or the runner will silently fail to see its own jobs.
 
 ### 5.3 Retry pass
 
-1. Take a Helix snapshot of the whole stage (all attempts).
+1. Take one current AzDO timeline snapshot and one Helix snapshot of the whole
+   stage (all attempts).
 2. Reduce it to the latest incarnation of each logical work stream (§2.3.3):
-   the leaf of each lineage chain, keyed by logical stream key, preferring the
-   higher stage attempt on ties.
+   the leaf of each lineage chain, keyed by logical stream key, ordered by stage
+   attempt, submitter job attempt, lineage depth, then Helix job ID.
 3. For each latest incarnation, apply §2.3.3:
-   - Current-attempt incarnation — leave it; it is already being driven.
+   - Initial monitor job attempt — do not retry anything.
+   - Current-stage-attempt incarnation — leave it; it belongs to this execution.
+   - Timeline submitter attempt is newer than the Helix submitter attempt —
+     leave it; the submitter reran and superseded the old Helix work.
    - Previous-attempt, completed and fully passed — leave it (terminal); it will
      still be uploaded / reconciled by the poll loop.
    - Previous-attempt, completed with failures, or unfinished — ask the Helix
      service to resubmit the failed / not-yet-passed items, stamped with the
-     current stage attempt (§2.3.4). If resubmission is not possible, record it
+     current stage attempt while preserving the submitter job attempt (§2.3.4).
+     If resubmission is not possible, record it
      as a hard failure (§2.3.3).
 4. Remember the AzDO submitter-job identifiers of successfully retried work;
    these are the jobs to exclude from the AzDO failure check while this
@@ -434,9 +447,10 @@ The chain key must be deterministic and uniqueness-preserving:
 - An original Helix job and its resubmission(s) on the same queue must
   produce the same key so the latest incarnation overwrites the older one.
 - The preferred key components are:
-  1. `System.PhaseName`, falling back to `System.JobName`;
-  2. the Helix queue;
-  3. the submitter-assigned Helix `jobName`, falling back to `TestRunName`.
+  1. `System.StageName`;
+  2. `System.PhaseName`, falling back to `System.JobName`;
+  3. the Helix queue;
+  4. the submitter-assigned Helix `jobName`, falling back to `TestRunName`.
   If no stable logical-job discriminator is available, the key is bound to the
   root Helix job in the `PreviousHelixJobName` lineage rather than risk merging
   unrelated jobs.
@@ -445,9 +459,9 @@ The chain key must be deterministic and uniqueness-preserving:
   onto the same key as its previous-attempt counterpart, even though the two
   Helix jobs are **not** linked by
   `PreviousHelixJobName` (only monitor resubmissions set that link). The map
-  must therefore let the **later stage attempt win** when two incarnations share
-  a key: outcomes must be applied in order of (lineage depth, then stage
-  attempt), not by Helix job-name sort, or a stale previous-attempt outcome
+  must therefore let the later incarnation win when two incarnations share a
+  key: outcomes are applied in order of lineage depth, stage attempt, submitter
+  job attempt, then Helix job ID, rather than by Helix job-name sort alone, or a stale previous-attempt outcome
   could nondeterministically overwrite the current one. `System.JobName` is
   used only when `System.PhaseName` is unavailable because some pipelines stamp
   every matrix job with `System.JobName=__default`.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs
index 1304bc66ce1..6808ace5d85 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs
@@ -58,14 +58,15 @@ Task CancelJobAsync(
         /// possible (e.g. the original queue no longer exists).
         /// The new job must preserve BuildId and StageName properties so it is discoverable by
         /// GetJobsForBuildAsync, and must be stamped with <paramref name="targetStageAttempt"/>
-        /// (the resubmitting monitor's own stage attempt) rather than the original job's attempt,
-        /// so the monitor gates on its own resubmission. When <paramref name="targetStageAttempt"/>
-        /// is null/empty the original job's attempt is preserved (build + stage back-compat).
+        /// (the resubmitting monitor's own stage attempt) so the monitor gates on its own
+        /// resubmission. It preserves the original submitter's System.JobAttempt and records
+        /// <paramref name="monitorJobAttempt"/> separately for diagnostics.
         /// </summary>
         Task<HelixJobInfo> ResubmitWorkItemsAsync(
             HelixJobInfo originalJob,
             IReadOnlyCollection<WorkItemSummary> failedWorkItems,
             string targetStageAttempt,
+            string monitorJobAttempt,
             CancellationToken cancellationToken);
     }
 }
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs
index 53ada6a35c9..b84dc9d37f8 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs
@@ -55,6 +55,12 @@ public sealed class JobMonitorOptions
         /// </summary>
         public string StageAttempt { get; set; }
 
+        /// <summary>
+        /// Attempt number of the Azure DevOps monitor job. Retry reconciliation only runs when
+        /// this is greater than one. Defaults to the SYSTEM_JOBATTEMPT environment variable.
+        /// </summary>
+        public string JobAttempt { get; set; }
+
         public int TestResultUploadParallelism { get; set; } = 48;
 
         public TestResultAttachmentMode TestResultAttachmentMode { get; set; } = TestResultAttachmentMode.Failed;
@@ -163,6 +169,11 @@ public static JobMonitorOptions Parse(string[] args)
                 Description = "Attempt number of the Azure DevOps pipeline stage the monitor is running in. Used to scope monitoring to Helix jobs submitted by the current stage attempt so retries do not re-discover a previous attempt's work. Defaults to the SYSTEM_STAGEATTEMPT environment variable."
             };
 
+            Option<string> jobAttemptOption = new("--job-attempt")
+            {
+                Description = "Attempt number of the Azure DevOps monitor job. Retry reconciliation only runs after the initial job attempt. Defaults to the SYSTEM_JOBATTEMPT environment variable."
+            };
+
             Option<int> testResultUploadParallelismOption = new("--test-result-upload-parallelism")
             {
                 Description = "Maximum number of work items whose test results can be uploaded to Azure DevOps in parallel.",
@@ -216,6 +227,7 @@ public static JobMonitorOptions Parse(string[] args)
             rootCommand.Options.Add(workingDirectoryOption);
             rootCommand.Options.Add(stageNameOption);
             rootCommand.Options.Add(stageAttemptOption);
+            rootCommand.Options.Add(jobAttemptOption);
             rootCommand.Options.Add(testResultUploadParallelismOption);
             rootCommand.Options.Add(testResultAttachmentModeOption);
             rootCommand.Options.Add(failWorkItemsWithFailedTestsOption);
@@ -241,6 +253,7 @@ public static JobMonitorOptions Parse(string[] args)
                     WorkingDirectory = parseResult.GetValue(workingDirectoryOption),
                     StageName = parseResult.GetValue(stageNameOption),
                     StageAttempt = parseResult.GetValue(stageAttemptOption),
+                    JobAttempt = parseResult.GetValue(jobAttemptOption),
                     TestResultUploadParallelism = parseResult.GetValue(testResultUploadParallelismOption),
                     TestResultAttachmentMode = parseResult.GetValue(testResultAttachmentModeOption),
                     FailWorkItemsWithFailedTests = parseResult.GetValue(failWorkItemsWithFailedTestsOption),
@@ -283,6 +296,7 @@ private void ApplyEnvironmentDefaults()
             SourceBranch ??= Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH");
             StageName ??= Environment.GetEnvironmentVariable("SYSTEM_STAGENAME");
             StageAttempt ??= Environment.GetEnvironmentVariable("SYSTEM_STAGEATTEMPT");
+            JobAttempt ??= Environment.GetEnvironmentVariable("SYSTEM_JOBATTEMPT");
         }
 
         private void Validate()
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
index a43f64d702e..c54c07c3f30 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
@@ -110,8 +110,13 @@ public async Task<int> RunAsync(CancellationToken cancellationToken)
             Task statusTask = ReportStatusPeriodicallyAsync(statusCts.Token);
             try
             {
-                IReadOnlyList<HelixJobInfo> jobsForFirstPoll = await ExecuteRetryPassAsync(cancellationToken);
-                return await RunPollLoopAsync(jobsForFirstPoll, cancellationToken);
+                IReadOnlyList<AzureDevOpsTimelineRecord> timelineForFirstPoll =
+                    HelixJobMonitorUtilities.FilterRecordsToStage(
+                        await _azdo.GetTimelineRecordsAsync(cancellationToken),
+                        _options.StageName);
+                IReadOnlyList<HelixJobInfo> jobsForFirstPoll =
+                    await ExecuteRetryPassAsync(timelineForFirstPoll, cancellationToken);
+                return await RunPollLoopAsync(jobsForFirstPoll, timelineForFirstPoll, cancellationToken);
             }
             catch (OperationCanceledException)
             {
@@ -158,12 +163,14 @@ public async Task<int> RunAsync(CancellationToken cancellationToken)
         /// One-shot retry pass executed on entry. Reconciles the whole stage's Helix work
         /// (all attempts) into the current stage attempt: for each logical work stream it takes
         /// the latest incarnation and, when that incarnation is a previous attempt's failed or
-        /// unfinished work (or a current attempt's completed-with-failures work), resubmits the
-        /// not-yet-passed items into the current attempt. Returns the (stage snapshot ∪
+        /// unfinished work whose submitter did not rerun, resubmits the not-yet-passed items into
+        /// the current attempt. Returns the (stage snapshot ∪
         /// resubmitted jobs) so the first poll iteration sees the resubmissions immediately.
         /// See Design/SemanticBehavior.md §2.1 and §2.3.
         /// </summary>
-        private async Task<IReadOnlyList<HelixJobInfo>> ExecuteRetryPassAsync(CancellationToken cancellationToken)
+        private async Task<IReadOnlyList<HelixJobInfo>> ExecuteRetryPassAsync(
+            IReadOnlyList<AzureDevOpsTimelineRecord> timelineRecords,
+            CancellationToken cancellationToken)
         {
             _reporter.LogRetryPassStart();
 
@@ -179,6 +186,16 @@ private async Task<IReadOnlyList<HelixJobInfo>> ExecuteRetryPassAsync(Cancellati
             // Seed the cross-poll cache so submitter-chain-key lineage (PreviousHelixJobName
             // walks) resolves while grouping streams below.
             _state.ObserveJobs(stageJobs);
+            _state.SetTimelineRecords(timelineRecords);
+
+            // The initial monitor invocation observes and reports failures; it does not create
+            // additional Helix work. Retry reconciliation is only meaningful after AzDO has
+            // retried the monitor job.
+            if (MonitorState.ParseJobAttempt(_options.JobAttempt) <= 1)
+            {
+                _reporter.LogRetryPassFoundNothing();
+                return stageJobs;
+            }
 
             // Surfacing work items that passed by exit code but whose AzDO test results contain
             // failures: a prior monitor invocation may have uploaded failed tests for a job
@@ -198,10 +215,24 @@ private async Task<IReadOnlyList<HelixJobInfo>> ExecuteRetryPassAsync(Cancellati
             {
                 bool previousAttempt = IsPreviousAttempt(latest);
 
-                // A current-attempt incarnation that is still in flight is gated on, not
-                // resubmitted (this also covers the fast-rerun case where a fresh current-attempt
-                // incarnation exists while a previous one is still running — §2.3.1 case 4).
-                if (!previousAttempt && !latest.IsCompleted)
+                // Work created in this stage attempt belongs to the current execution, whether
+                // it is running or has already failed. It is observed, not retried again on entry.
+                if (!previousAttempt)
+                {
+                    continue;
+                }
+
+                bool hasSubmitterAttempt = TryGetCurrentSubmitterAttempt(
+                    latest,
+                    timelineRecords,
+                    out int currentSubmitterAttempt,
+                    out string submitterIdentity);
+                int helixSubmitterAttempt = MonitorState.ParseJobAttempt(latest.JobAttempt);
+
+                // The submitter itself reran. Its old Helix work is superseded even when the new
+                // Helix job has not become visible yet; resubmitting here would duplicate a full
+                // stage rerun or a selected failed-job retry.
+                if (hasSubmitterAttempt && currentSubmitterAttempt > helixSubmitterAttempt)
                 {
                     continue;
                 }
@@ -229,6 +260,25 @@ private async Task<IReadOnlyList<HelixJobInfo>> ExecuteRetryPassAsync(Cancellati
                     continue;
                 }
 
+                if (!hasSubmitterAttempt
+                    || string.IsNullOrEmpty(latest.JobAttempt)
+                    || currentSubmitterAttempt < helixSubmitterAttempt)
+                {
+                    IReadOnlyCollection<WorkItemSummary> ambiguousWork =
+                    [
+                        ..exitCodeFailures
+                            .Concat(testOnlyFailures)
+                            .DistinctBy(wi => wi.Name, StringComparer.OrdinalIgnoreCase)
+                    ];
+                    _state.RecordAbandonedWork(latest, ambiguousWork);
+                    LogWarning(
+                        $"Cannot safely reconcile {ambiguousWork.Count} failed/unfinished work item(s) from "
+                        + $"{latest.DisplayName}: submitter '{submitterIdentity ?? "<unknown>"}' could not be "
+                        + $"matched to compatible System.JobAttempt metadata in the current timeline. "
+                        + "The work was not resubmitted to avoid duplicating a rerun.");
+                    continue;
+                }
+
                 _reporter.LogRetryPassResubmission(latest, exitCodeFailures, testOnlyFailures);
 
                 // exitCodeFailures and testOnlyFailures are disjoint by construction (one
@@ -242,7 +292,7 @@ private async Task<IReadOnlyList<HelixJobInfo>> ExecuteRetryPassAsync(Cancellati
                 ];
 
                 HelixJobInfo resubmitted = await _helix.ResubmitWorkItemsAsync(
-                    latest, failedWorkItems, _options.StageAttempt, cancellationToken);
+                    latest, failedWorkItems, _options.StageAttempt, _options.JobAttempt, cancellationToken);
                 if (resubmitted is null)
                 {
                     // Previous-attempt work that can never run again (e.g. its queue was removed)
@@ -261,7 +311,9 @@ private async Task<IReadOnlyList<HelixJobInfo>> ExecuteRetryPassAsync(Cancellati
                 }
 
                 resubmittedJobs.Add(resubmitted);
-                _state.RecordResubmission(latest.SubmitterJobName, failedWorkItems.Count);
+                _state.RecordResubmission(
+                    latest.SubmitterPhaseName ?? latest.SubmitterJobName,
+                    failedWorkItems.Count);
             }
 
             if (resubmittedJobs.Count == 0)
@@ -272,7 +324,10 @@ private async Task<IReadOnlyList<HelixJobInfo>> ExecuteRetryPassAsync(Cancellati
             return [.. stageJobs, .. resubmittedJobs];
         }
 
-        private async Task<int> RunPollLoopAsync(IReadOnlyList<HelixJobInfo> jobsForFirstPoll, CancellationToken cancellationToken)
+        private async Task<int> RunPollLoopAsync(
+            IReadOnlyList<HelixJobInfo> jobsForFirstPoll,
+            IReadOnlyList<AzureDevOpsTimelineRecord> timelineForFirstPoll,
+            CancellationToken cancellationToken)
         {
             var loopState = new PollLoopState();
 
@@ -280,8 +335,13 @@ private async Task<int> RunPollLoopAsync(IReadOnlyList<HelixJobInfo> jobsForFirs
             {
                 cancellationToken.ThrowIfCancellationRequested();
 
-                int? exitCode = await PollOnceAsync(jobsForFirstPoll, loopState, cancellationToken);
+                int? exitCode = await PollOnceAsync(
+                    jobsForFirstPoll,
+                    timelineForFirstPoll,
+                    loopState,
+                    cancellationToken);
                 jobsForFirstPoll = null; // first-poll seed is consumed
+                timelineForFirstPoll = null;
 
                 if (exitCode.HasValue)
                 {
@@ -298,6 +358,7 @@ private async Task<int> RunPollLoopAsync(IReadOnlyList<HelixJobInfo> jobsForFirs
         /// </summary>
         private async Task<int?> PollOnceAsync(
             IReadOnlyList<HelixJobInfo> jobsForFirstPoll,
+            IReadOnlyList<AzureDevOpsTimelineRecord> timelineForFirstPoll,
             PollLoopState loopState,
             CancellationToken cancellationToken)
         {
@@ -305,7 +366,8 @@ private async Task<int> RunPollLoopAsync(IReadOnlyList<HelixJobInfo> jobsForFirs
 
             // Fetch fresh snapshots, scoped to the monitor's stage.
             IReadOnlyList<AzureDevOpsTimelineRecord> timelineRecords =
-                HelixJobMonitorUtilities.FilterRecordsToStage(
+                timelineForFirstPoll
+                ?? HelixJobMonitorUtilities.FilterRecordsToStage(
                     await _azdo.GetTimelineRecordsAsync(cancellationToken),
                     _options.StageName);
 
@@ -570,6 +632,43 @@ private bool IsPreviousAttempt(HelixJobInfo job)
                 && !string.IsNullOrEmpty(job.StageAttempt)
                 && MonitorState.ParseStageAttempt(job.StageAttempt) < MonitorState.ParseStageAttempt(_options.StageAttempt);
 
+        private static bool TryGetCurrentSubmitterAttempt(
+            HelixJobInfo job,
+            IReadOnlyList<AzureDevOpsTimelineRecord> timelineRecords,
+            out int attempt,
+            out string submitterIdentity)
+        {
+            submitterIdentity = job.SubmitterPhaseName ?? job.SubmitterJobName;
+            attempt = 0;
+            if (string.IsNullOrEmpty(submitterIdentity))
+            {
+                return false;
+            }
+
+            string identity = submitterIdentity;
+            int[] matchingAttempts =
+            [
+                ..timelineRecords
+                    .Where(record =>
+                        (string.Equals(record.Type, "Job", StringComparison.OrdinalIgnoreCase)
+                            || string.Equals(record.Type, "Phase", StringComparison.OrdinalIgnoreCase))
+                        && string.Equals(
+                            record.ReferenceName,
+                            identity,
+                            StringComparison.OrdinalIgnoreCase))
+                    .Select(record => record.Attempt)
+                    .Distinct()
+            ];
+
+            if (matchingAttempts.Length == 0)
+            {
+                return false;
+            }
+
+            attempt = matchingAttempts.Max();
+            return true;
+        }
+
         private static ProductionDependencies CreateProductionDependencies(
             JobMonitorOptions options,
             ILogger logger)
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs
index 39cd0f5525f..aa0f09a5c2b 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs
@@ -25,6 +25,8 @@ public sealed class HelixJobInfo
         /// same property when resubmitting (see Design/SemanticBehavior.md §2.3).
         /// </summary>
         public const string StageAttemptPropertyName = "System.StageAttempt";
+        public const string JobAttemptPropertyName = "System.JobAttempt";
+        public const string ResubmittedByJobAttemptPropertyName = "JobMonitor.JobAttempt";
 
         public HelixJobInfo(JobSummary helixJob)
         {
@@ -33,6 +35,7 @@ public HelixJobInfo(JobSummary helixJob)
             TestRunName = GetTestRunNameFromJob(helixJob);
             StageName = GetStringPropertyFromJob(helixJob, "System.StageName");
             StageAttempt = GetStringPropertyFromJob(helixJob, StageAttemptPropertyName);
+            JobAttempt = GetStringPropertyFromJob(helixJob, JobAttemptPropertyName);
             QueueId = helixJob.QueueId;
             InitialWorkItemCount = helixJob.InitialWorkItemCount;
             Properties = helixJob.Properties;
@@ -49,6 +52,7 @@ public HelixJobInfo(
             string previousHelixJobName = null,
             int? initialWorkItemCount = null,
             string stageAttempt = null,
+            string jobAttempt = null,
             string logicalJobName = null,
             string submitterPhaseName = null)
         {
@@ -57,6 +61,7 @@ public HelixJobInfo(
             TestRunName = testRunName;
             StageName = stageName;
             StageAttempt = stageAttempt;
+            JobAttempt = jobAttempt;
             QueueId = queueId;
             InitialWorkItemCount = initialWorkItemCount;
             Properties = CreateProperties(
@@ -66,6 +71,7 @@ public HelixJobInfo(
                 submitterJobDisplayName,
                 previousHelixJobName,
                 stageAttempt,
+                jobAttempt,
                 logicalJobName,
                 submitterPhaseName);
         }
@@ -96,6 +102,13 @@ public HelixJobInfo(
         /// </summary>
         public string StageAttempt { get; }
 
+        /// <summary>
+        /// Attempt of the Azure DevOps submitter job that created this logical Helix work.
+        /// Monitor-created resubmissions preserve this value so it can be compared with the
+        /// current timeline record for the submitter.
+        /// </summary>
+        public string JobAttempt { get; }
+
         /// <summary>
         /// Helix target queue this job ran on (e.g. "Ubuntu.2204.Amd64.Open"). Comes from the
         /// Helix <c>JobSummary.QueueId</c>. May be null on synthetic jobs.
@@ -226,6 +239,7 @@ private static JObject CreateProperties(
             string submitterJobDisplayName,
             string previousHelixJobName,
             string stageAttempt,
+            string jobAttempt,
             string logicalJobName,
             string submitterPhaseName)
         {
@@ -245,6 +259,10 @@ private static JObject CreateProperties(
             {
                 properties[StageAttemptPropertyName] = stageAttempt;
             }
+            if (!string.IsNullOrEmpty(jobAttempt))
+            {
+                properties[JobAttemptPropertyName] = jobAttempt;
+            }
 
             if (!string.IsNullOrEmpty(submitterJobName))
             {
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
index 77fa512713e..8ee18ea7991 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
@@ -399,7 +399,7 @@ public IReadOnlyList<FailedWorkItemConsoleInfo> SnapshotFailedWorkItemConsoleInf
 
         /// <summary>
         /// Produces a key that rolls up work-item outcomes within a logical Helix work stream.
-        /// The AzDO phase name, Helix queue, and submitter-assigned logical job name jointly
+        /// The stage, AzDO phase name, Helix queue, and submitter-assigned logical job name jointly
         /// identify a stream across stage reruns and monitor resubmissions. The phase name is
         /// preferred because some pipelines stamp many independent jobs with
         /// <c>System.JobName=__default</c>. When stable submitter metadata is unavailable,
@@ -426,11 +426,12 @@ private string GetSubmitterChainKeyLocked(HelixJobInfo job)
                 ?? root.LogicalJobName
                 ?? job.TestRunName
                 ?? root.TestRunName;
+            string stageName = job.StageName ?? root.StageName;
 
             if (!string.IsNullOrEmpty(submitterName)
                 && !string.IsNullOrEmpty(logicalJobName))
             {
-                return FormatSubmitterChainKey(submitterName, queueId, logicalJobName);
+                return FormatSubmitterChainKey(stageName, submitterName, queueId, logicalJobName);
             }
 
             return $"helix:{root.JobName}";
@@ -456,10 +457,11 @@ private HelixJobInfo GetLineageRootLocked(HelixJobInfo job)
         }
 
         private static string FormatSubmitterChainKey(
+            string stageName,
             string submitterName,
             string queueId,
             string logicalJobName)
-            => $"submitter:{submitterName}|queue:{queueId ?? string.Empty}|job:{logicalJobName}";
+            => $"stage:{stageName ?? string.Empty}|submitter:{submitterName}|queue:{queueId ?? string.Empty}|job:{logicalJobName}";
 
         /// <summary>
         /// From an arbitrary set of Helix jobs (possibly spanning multiple stage attempts),
@@ -467,8 +469,8 @@ private static string FormatSubmitterChainKey(
         /// submitter chain key (§5.7). Within a stream, resubmission lineage is collapsed to the
         /// leaf, and unlinked rerun duplicates (same submitter + queue on different stage
         /// attempts, not connected by <c>PreviousHelixJobName</c>) are broken toward the highest
-        /// stage attempt. Used by the retry pass to decide, per stream, whether previous-attempt
-        /// work must be reconciled into the current attempt.
+        /// stage and submitter job attempts, then explicit lineage depth. Used by the retry pass
+        /// to decide whether previous-attempt work remains authoritative.
         /// </summary>
         public IReadOnlyList<HelixJobInfo> GetLatestIncarnationPerStream(IEnumerable<HelixJobInfo> jobs)
         {
@@ -480,6 +482,8 @@ public IReadOnlyList<HelixJobInfo> GetLatestIncarnationPerStream(IEnumerable<Hel
                         .GroupBy(GetSubmitterChainKeyLocked, StringComparer.OrdinalIgnoreCase)
                         .Select(g => g
                             .OrderBy(j => ParseStageAttempt(j.StageAttempt))
+                            .ThenBy(j => ParseJobAttempt(j.JobAttempt))
+                            .ThenBy(j => GetLineageDepth(j, _associatedJobs))
                             .ThenBy(j => j.JobName, StringComparer.OrdinalIgnoreCase)
                             .Last())
                 ];
@@ -516,6 +520,9 @@ public void RecordAbandonedWork(HelixJobInfo job, IEnumerable<WorkItemSummary> w
         public static int ParseStageAttempt(string stageAttempt)
             => int.TryParse(stageAttempt, out int attempt) ? attempt : 1;
 
+        public static int ParseJobAttempt(string jobAttempt)
+            => int.TryParse(jobAttempt, out int attempt) ? attempt : 1;
+
         /// <summary>
         /// From an arbitrary set of Helix jobs return only the leaves of each lineage chain —
         /// jobs that are not pointed at by any other job's <c>PreviousHelixJobName</c>.
@@ -535,8 +542,8 @@ public static IReadOnlyList<HelixJobInfo> GetLatestHelixJobAttempts(IEnumerable<
         /// <c>PreviousHelixJobName</c> link backwards, breaking ties toward the lower stage
         /// attempt. Used to ensure upload and outcome reconciliation process lineage in the
         /// right order (older first, so newer incarnations supersede older ones) — including
-        /// unlinked rerun duplicates on different attempts, where the higher stage attempt must
-        /// reconcile last so it wins the outcome map (§5.7).
+        /// unlinked rerun duplicates on different attempts, where the higher stage/job
+        /// incarnation must reconcile last so it wins the outcome map (§5.7).
         /// </summary>
         public static IReadOnlyList<HelixJobInfo> OrderHelixJobsOldToNew(IEnumerable<HelixJobInfo> jobs)
         {
@@ -546,6 +553,7 @@ public static IReadOnlyList<HelixJobInfo> OrderHelixJobsOldToNew(IEnumerable<Hel
                 ..jobs
                     .OrderBy(j => GetLineageDepth(j, jobByName))
                     .ThenBy(j => ParseStageAttempt(j.StageAttempt))
+                    .ThenBy(j => ParseJobAttempt(j.JobAttempt))
                     .ThenBy(j => j.JobName, StringComparer.OrdinalIgnoreCase)
             ];
         }
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
index 333df0b2e21..6a7924f52cd 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
@@ -193,6 +193,7 @@ public async Task<HelixJobInfo> ResubmitWorkItemsAsync(
             HelixJobInfo originalJob,
             IReadOnlyCollection<WorkItemSummary> failedWorkItems,
             string targetStageAttempt,
+            string monitorJobAttempt,
             CancellationToken cancellationToken)
         {
             string originalJobName = originalJob.JobName;
@@ -313,6 +314,12 @@ await RetryAsync(
             {
                 resubmittedProperties = resubmittedProperties.SetItem(HelixJobInfo.StageAttemptPropertyName, resubmittedStageAttempt);
             }
+            if (!string.IsNullOrEmpty(monitorJobAttempt))
+            {
+                resubmittedProperties = resubmittedProperties.SetItem(
+                    HelixJobInfo.ResubmittedByJobAttemptPropertyName,
+                    monitorJobAttempt);
+            }
 
             // 5. Build the new job creation request, copying over Source / Properties / Creator
             //    so the resubmitted job remains discoverable (BuildId, System.StageName, TestRunName, etc.).
@@ -339,6 +346,7 @@ await RetryAsync(
             string submitterJobDisplayName = GetStringPropertyFromProperties(details.Properties, "System.JobDisplayName");
             string logicalJobName = GetStringPropertyFromProperties(details.Properties, HelixJobInfo.LogicalJobNamePropertyName);
             string submitterPhaseName = GetStringPropertyFromProperties(details.Properties, "System.PhaseName");
+            string submitterJobAttempt = GetStringPropertyFromProperties(details.Properties, HelixJobInfo.JobAttemptPropertyName);
 
             var newJobInfo = new HelixJobInfo(
                 newJob.Name,
@@ -350,6 +358,7 @@ await RetryAsync(
                 details.QueueId,
                 originalJobName,
                 stageAttempt: resubmittedStageAttempt,
+                jobAttempt: submitterJobAttempt,
                 logicalJobName: logicalJobName,
                 submitterPhaseName: submitterPhaseName);
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs
index edbab2dc0a7..1ccd5645621 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs
@@ -163,9 +163,10 @@ public Task CancelJobAsync(string jobName, CancellationToken cancellationToken)
 
         /// <summary>
         /// Tracks resubmission calls for test assertions.
-        /// Each entry is (originalJobName, failedWorkItemNames, newJobName, targetStageAttempt).
+        /// Each entry is (originalJobName, failedWorkItemNames, newJobName, targetStageAttempt,
+        /// monitorJobAttempt).
         /// </summary>
-        public List<(string OriginalJob, IReadOnlyCollection<string> FailedItems, string NewJob, string TargetStageAttempt)> Resubmissions { get; } = [];
+        public List<(string OriginalJob, IReadOnlyCollection<string> FailedItems, string NewJob, string TargetStageAttempt, string MonitorJobAttempt)> Resubmissions { get; } = [];
 
         /// <summary>The <see cref="HelixJobInfo"/> objects returned from successful resubmissions.</summary>
         public List<HelixJobInfo> ResubmittedJobInfos { get; } = [];
@@ -195,6 +196,7 @@ public Task<HelixJobInfo> ResubmitWorkItemsAsync(
             HelixJobInfo originalJob,
             IReadOnlyCollection<WorkItemSummary> failedWorkItems,
             string targetStageAttempt,
+            string monitorJobAttempt,
             CancellationToken cancellationToken)
         {
             string originalJobName = originalJob.JobName;
@@ -210,7 +212,7 @@ public Task<HelixJobInfo> ResubmitWorkItemsAsync(
             IReadOnlyCollection<string> failedItemNames = [..failedWorkItems.Select(wi => wi.Name)];
             if (_nullResubmissions.Contains(originalJobName))
             {
-                Resubmissions.Add((originalJobName, failedItemNames, null, targetStageAttempt));
+                Resubmissions.Add((originalJobName, failedItemNames, null, targetStageAttempt, monitorJobAttempt));
                 return Task.FromResult<HelixJobInfo>(null);
             }
 
@@ -220,7 +222,7 @@ public Task<HelixJobInfo> ResubmitWorkItemsAsync(
                 ? targetStageAttempt
                 : (originalSnapshotJob?.StageAttempt ?? originalJob.StageAttempt);
 
-            Resubmissions.Add((originalJobName, failedItemNames, newJobName, targetStageAttempt));
+            Resubmissions.Add((originalJobName, failedItemNames, newJobName, targetStageAttempt, monitorJobAttempt));
             var newJobInfo = new HelixJobInfo(
                 newJobName,
                 "running",
@@ -231,6 +233,7 @@ public Task<HelixJobInfo> ResubmitWorkItemsAsync(
                 originalSnapshotJob?.QueueId ?? originalJob.QueueId,
                 originalJobName,
                 stageAttempt: resubmittedStageAttempt,
+                jobAttempt: originalSnapshotJob?.JobAttempt ?? originalJob.JobAttempt,
                 logicalJobName: originalSnapshotJob?.LogicalJobName ?? originalJob.LogicalJobName,
                 submitterPhaseName: originalSnapshotJob?.SubmitterPhaseName ?? originalJob.SubmitterPhaseName);
             ResubmittedJobInfos.Add(newJobInfo);
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
index 7a5917eac1c..dcc94dab878 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
@@ -174,7 +174,7 @@ public async Task ResubmitWorkItemsAsync_ReturnsNullWhenRequiredJobDetailsAreMis
                 });
 
             HelixJobInfo result = await CreateService(api.Api.Object)
-                .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("missing")], targetStageAttempt: null, CancellationToken.None);
+                .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("missing")], targetStageAttempt: null, monitorJobAttempt: null, CancellationToken.None);
 
             Assert.Null(result);
             api.Storage.Verify(s => s.NewAsync(It.IsAny<ContainerCreationRequest>(), It.IsAny<CancellationToken>()), Times.Never);
@@ -193,7 +193,7 @@ public async Task ResubmitWorkItemsAsync_ReturnsNullWhenJobListIsInvalidJson()
             };
 
             HelixJobInfo result = await CreateService(api.Api.Object, blobClientFactory)
-                .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("work-a")], targetStageAttempt: null, CancellationToken.None);
+                .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("work-a")], targetStageAttempt: null, monitorJobAttempt: null, CancellationToken.None);
 
             Assert.Null(result);
             api.Storage.Verify(s => s.NewAsync(It.IsAny<ContainerCreationRequest>(), It.IsAny<CancellationToken>()), Times.Never);
@@ -245,13 +245,20 @@ public async Task ResubmitWorkItemsAsync_UploadsFilteredJobListAndCreatesJobWith
             };
 
             HelixJobInfo result = await CreateService(api.Api.Object, blobClientFactory)
-                .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("WORK-A"), WorkItem("work-b")], targetStageAttempt: null, CancellationToken.None);
+                .ResubmitWorkItemsAsync(
+                    new HelixJobInfo("original-job", "finished"),
+                    [WorkItem("WORK-A"), WorkItem("work-b")],
+                    targetStageAttempt: "2",
+                    monitorJobAttempt: "2",
+                    CancellationToken.None);
 
             Assert.Equal("new-job", result.JobName);
             Assert.Equal("running", result.Status);
             Assert.Equal("custom run", result.TestRunName);
             Assert.Equal("test stage", result.StageName);
             Assert.Equal("original-job", result.PreviousHelixJobName);
+            Assert.Equal("2", result.StageAttempt);
+            Assert.Equal("1", result.JobAttempt);
 
             Assert.Equal("https://storage/job-list.json", blobClientFactory.DownloadedTextUri);
             UploadCall upload = Assert.Single(blobClientFactory.Uploads);
@@ -274,6 +281,9 @@ public async Task ResubmitWorkItemsAsync_UploadsFilteredJobListAndCreatesJobWith
             Assert.Equal("123", capturedRequest.Properties["BuildId"]);
             Assert.Equal("custom run", capturedRequest.Properties["TestRunName"]);
             Assert.Equal("test stage", capturedRequest.Properties["System.StageName"]);
+            Assert.Equal("2", capturedRequest.Properties[HelixJobInfo.StageAttemptPropertyName]);
+            Assert.Equal("1", capturedRequest.Properties[HelixJobInfo.JobAttemptPropertyName]);
+            Assert.Equal("2", capturedRequest.Properties[HelixJobInfo.ResubmittedByJobAttemptPropertyName]);
             Assert.Equal("original-job", capturedRequest.Properties[HelixJobInfo.PreviousHelixJobNamePropertyName]);
             Assert.Equal("""{"nested":true}""", capturedRequest.Properties["ObjectProperty"]);
 
@@ -320,6 +330,8 @@ private static async Task<JobCreationRequest> ResubmitAndCaptureRequestAsync(
                 ["BuildId"] = "123",
                 ["TestRunName"] = "custom run",
                 ["System.StageName"] = "test stage",
+                [HelixJobInfo.StageAttemptPropertyName] = "1",
+                [HelixJobInfo.JobAttemptPropertyName] = "1",
             };
 
             api.Job
@@ -364,7 +376,7 @@ private static async Task<JobCreationRequest> ResubmitAndCaptureRequestAsync(
             };
 
             await CreateService(api.Api.Object, blobClientFactory)
-                .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("work-a")], targetStageAttempt: null, CancellationToken.None);
+                .ResubmitWorkItemsAsync(new HelixJobInfo("original-job", "finished"), [WorkItem("work-a")], targetStageAttempt: null, monitorJobAttempt: null, CancellationToken.None);
 
             Assert.NotNull(capturedRequest);
             return capturedRequest;
@@ -406,6 +418,8 @@ private static JobDetails JobDetails()
                     ["BuildId"] = "123",
                     ["TestRunName"] = "custom run",
                     ["System.StageName"] = "test stage",
+                    [HelixJobInfo.StageAttemptPropertyName] = "1",
+                    [HelixJobInfo.JobAttemptPropertyName] = "1",
                     ["ObjectProperty"] = new JObject
                     {
                         ["nested"] = true,
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
index 02a17296bed..99d6dc9e9c0 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
@@ -436,15 +436,15 @@ public async Task BuildJobSubmitsHelixWork_WorkItemsPassed_ResultsUploaded_ExitZ
 
             // Poll 3: build job completed (it has submitted Helix work and exited)
             azdo.AddTimelineResponse(
-                MonitorJob(),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
                 PipelineJob("Test Linux", "completed", "succeeded"));
 
             // Polls 4-6: build job still completed (monitor waiting for Helix)
             azdo.AddTimelineResponse(
-                MonitorJob(),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
                 PipelineJob("Test Linux", "completed", "succeeded"));
             azdo.AddTimelineResponse(
-                MonitorJob(),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
                 PipelineJob("Test Linux", "completed", "succeeded"));
             azdo.AddTimelineResponse(
                 MonitorJob(),
@@ -504,7 +504,8 @@ public async Task CompletedHelixJob_QueuesTestResultUploadWithoutBlockingNextPol
             azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "inProgress"));
             azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished")],
+                jobs: [HelixJob("helix-linux", "finished", stageName: "Test",
+                    submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(passed: ["workitem-1"]),
@@ -531,14 +532,14 @@ public async Task CompletedHelixJob_QueuesTestResultUploadWithoutBlockingNextPol
             exitCode.Should().Be(0);
             delayedBeforeUploadCompleted.Should().BeTrue();
             azdo.TimelineCallCount.Should().Be(2);
-            // One entry-retry scan and one first-poll snapshot; the second poll reuses the
-            // reconciled terminal snapshot instead of fetching it again.
-            helix.GetListWorkItemsCallCount("helix-linux").Should().Be(2);
+            // Initial monitor attempts do not run retry reconciliation; the completed job is
+            // listed once by the poll loop.
+            helix.GetListWorkItemsCallCount("helix-linux").Should().Be(1);
             azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-linux"]);
             azdo.CompletedTestRunIds.Should().ContainSingle();
             logger.Messages.Should().Contain(message =>
                 message.Contains(
-                    "Test result processing completed for job 'helix-linux': "
+                    "Test result processing completed for job 'Test Linux (helix-linux)': "
                     + "1 work item(s), 2 recognized result file(s), and 2 test result(s) uploaded.",
                     StringComparison.Ordinal));
         }
@@ -553,10 +554,11 @@ public async Task DrainReportsAggregatePipelineProgress()
 
             azdo.UploadBlocker = uploadRelease.Task;
             azdo.AddTimelineResponse(
-                MonitorJob(),
-                PipelineJob("Test Linux", "completed", "succeeded"));
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished")],
+                jobs: [HelixJob("helix-linux", "finished", stageName: "Test",
+                    submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(passed: ["workitem-1"]),
@@ -1281,7 +1283,7 @@ public async Task OneSubmitter_SameQueue_DifferentLogicalJobs_FailureNotOverwrit
 
             exitCode.Should().Be(1);
             logger.Messages.Should().Contain(message =>
-                message.Contains("Work items: 2 submitted / 1 resubmitted / 1 failed", StringComparison.Ordinal));
+                message.Contains("Work items: 2 submitted / 0 resubmitted / 1 failed", StringComparison.Ordinal));
             logger.Messages.Should().Contain(message =>
                 message.Contains("Failed work item information:", StringComparison.Ordinal)
                 && message.Contains("System.Formats.Tar.Manual.Tests", StringComparison.Ordinal));
@@ -1333,7 +1335,7 @@ public async Task DefaultJobName_SameQueueAndLogicalJob_DifferentPhases_FailureN
 
             exitCode.Should().Be(1);
             logger.Messages.Should().Contain(message =>
-                message.Contains("Work items: 2 submitted / 1 resubmitted / 1 failed", StringComparison.Ordinal));
+                message.Contains("Work items: 2 submitted / 0 resubmitted / 1 failed", StringComparison.Ordinal));
             logger.Messages.Should().Contain(message =>
                 message.Contains("Failed work item information:", StringComparison.Ordinal)
                 && message.Contains("System.Formats.Tar.Manual.Tests", StringComparison.Ordinal));
@@ -1432,9 +1434,10 @@ public async Task StageScopedMonitor_IgnoresJobsOutsideStage_NoHelixJobs_ExitOne
                 PipelineJob("Build Windows", "inProgress", parentId: "stage-build"));
 
             azdo.AddTimelineResponse(
-                StageRecord("Test", "stage-test", "inProgress"),
-                MonitorJob(parentId: "stage-test"),
-                PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test"),
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1, parentId: "stage-test"),
                 StageRecord("Build", "stage-build", "inProgress"),
                 PipelineJob("Build Windows", "inProgress", parentId: "stage-build"));
 
@@ -1465,9 +1468,10 @@ public async Task StageScopedMonitor_IgnoresHelixJobsFromOtherStage_ExitOne()
             // Test stage job completes quickly with no Helix submissions.
             // Build stage job submits a Helix job, but that's not the monitor's concern.
             azdo.AddTimelineResponse(
-                StageRecord("Test", "stage-test", "inProgress"),
-                MonitorJob(parentId: "stage-test"),
-                PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test"),
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1, parentId: "stage-test"),
                 StageRecord("Build", "stage-build", "inProgress"),
                 PipelineJob("Build Windows", "inProgress", parentId: "stage-build"));
 
@@ -1486,32 +1490,67 @@ public async Task StageScopedMonitor_IgnoresHelixJobsFromOtherStage_ExitOne()
         }
 
         /// <summary>
-        /// Corner case 1 (retry-failed-jobs where only the monitor re-ran). The stage's Helix
-        /// submitter jobs passed and were not re-run, so the current stage attempt contains no
-        /// Helix work of its own. The monitor must NOT exit 0 immediately (discarding the previous
-        /// attempt's results): it must upload the previous attempt's passed work and resubmit its
-        /// failed work into the current attempt, then gate on that resubmission.
+        /// The first monitor invocation observes and reports an already-failed Helix job but does
+        /// not create another Helix job. Work-item replay starts only when AzDO retries the monitor.
         /// </summary>
         [Fact]
-        public async Task AttemptScoped_RetryOnlyMonitor_ReconcilesPreviousAttemptWork()
+        public async Task AttemptScoped_InitialMonitorInvocation_DoesNotResubmit()
         {
             var azdo = new FakeAzureDevOpsService();
             var helix = new FakeHelixService();
 
-            // Submitters already succeeded in attempt 1; only the monitor is re-running as attempt 2.
             azdo.AddTimelineResponse(
                 StageRecord("Test", "stage-test", "inProgress"),
                 MonitorJob(parentId: "stage-test"),
-                PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test"),
-                PipelineJob("Test Windows", "completed", "succeeded", parentId: "stage-test"));
+                PipelineJob("Test_Linux", "completed", "succeeded", parentId: "stage-test"));
+            helix.AddResponse(
+                jobs: [HelixJob("helix-linux-a1", "finished", stageName: "Test",
+                    submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1",
+                    jobAttempt: "1", logicalJobName: "tests")],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-linux-a1"] = PassFail(failed: ["wi"]),
+                });
+
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "1";
+            options.JobAttempt = "1";
+
+            int exitCode = await new JobMonitorRunner(
+                options, NullLogger.Instance, azdo, helix, NoDelay).RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(1);
+            helix.Resubmissions.Should().BeEmpty();
+            azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-linux-a1"]);
+        }
+
+        /// <summary>
+        /// Retry-failed-jobs advanced the stage and monitor to attempt 2, but the successful Helix
+        /// submitters remain at job attempt 1. Their previous-stage Helix work is therefore still
+        /// authoritative: passed work is retained and failed work is replayed.
+        /// </summary>
+        [Fact]
+        public async Task AttemptScoped_RetryOnlyMonitor_ReconcilesPreviousAttemptWork()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+
+            azdo.AddTimelineResponse(
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("Test_Linux", "completed", "succeeded", attempt: 1, parentId: "stage-test"),
+                PipelineJob("Test_Windows", "completed", "succeeded", attempt: 1, parentId: "stage-test"));
 
             HelixJobInfo linuxA1 = HelixJob("helix-linux-a1", "finished", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1");
+                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1",
+                jobAttempt: "1", logicalJobName: "tests");
             HelixJobInfo winA1 = HelixJob("helix-win-a1", "finished", stageName: "Test",
-                submitterJobName: "Test_Windows", queueId: "q2", stageAttempt: "1");
+                submitterJobName: "Test_Windows", queueId: "q2", stageAttempt: "1",
+                jobAttempt: "1", logicalJobName: "tests");
             HelixJobInfo winResub = HelixJob("helix-win-a1-resub", "finished", stageName: "Test",
                 submitterJobName: "Test_Windows", queueId: "q2", previousHelixJobName: "helix-win-a1",
-                stageAttempt: "2");
+                stageAttempt: "2", jobAttempt: "1", logicalJobName: "tests");
 
             // Snapshot 0 (retry pass + poll 1): previous-attempt jobs only.
             helix.AddResponse(
@@ -1534,6 +1573,7 @@ public async Task AttemptScoped_RetryOnlyMonitor_ReconcilesPreviousAttemptWork()
 
             JobMonitorOptions options = DefaultOptions();
             options.StageAttempt = "2";
+            options.JobAttempt = "2";
             var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
 
             int exitCode = await runner.RunAsync(CancellationToken.None);
@@ -1544,8 +1584,10 @@ public async Task AttemptScoped_RetryOnlyMonitor_ReconcilesPreviousAttemptWork()
             helix.Resubmissions[0].OriginalJob.Should().Be("helix-win-a1");
             helix.Resubmissions[0].FailedItems.Should().BeEquivalentTo(["win-wi"]);
             helix.Resubmissions[0].TargetStageAttempt.Should().Be("2");
+            helix.Resubmissions[0].MonitorJobAttempt.Should().Be("2");
             helix.ResubmittedJobInfos.Should().ContainSingle()
-                .Which.StageAttempt.Should().Be("2");
+                .Which.Should().Match<HelixJobInfo>(job =>
+                    job.StageAttempt == "2" && job.JobAttempt == "1");
             // ...and the previous attempt's results (passed and failed) were still uploaded.
             azdo.UploadedJobNames.Should().Contain(["helix-linux-a1", "helix-win-a1", "helix-win-a1-resub"]);
         }
@@ -1564,15 +1606,16 @@ public async Task AttemptScoped_StrandedWaitingPreviousWork_ResubmittedNotWaited
             var helix = new FakeHelixService();
 
             azdo.AddTimelineResponse(
-                StageRecord("Test", "stage-test", "inProgress"),
-                MonitorJob(parentId: "stage-test"),
-                PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test"));
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("Test_Linux", "completed", "succeeded", attempt: 1, parentId: "stage-test"));
 
             HelixJobInfo stranded = HelixJob("helix-stranded-a1", "running", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1");
+                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1", jobAttempt: "1");
             HelixJobInfo resub = HelixJob("helix-stranded-a1-r2", "finished", stageName: "Test",
                 submitterJobName: "Test_Linux", queueId: "q1", previousHelixJobName: "helix-stranded-a1",
-                stageAttempt: "2");
+                stageAttempt: "2", jobAttempt: "1");
 
             // The stranded attempt-1 job's work items are stuck Waiting (never dispatched).
             helix.WithWorkItems("helix-stranded-a1",
@@ -1595,6 +1638,7 @@ public async Task AttemptScoped_StrandedWaitingPreviousWork_ResubmittedNotWaited
 
             JobMonitorOptions options = DefaultOptions();
             options.StageAttempt = "2";
+            options.JobAttempt = "2";
             var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
 
             int exitCode = await runner.RunAsync(CancellationToken.None);
@@ -1607,10 +1651,9 @@ public async Task AttemptScoped_StrandedWaitingPreviousWork_ResubmittedNotWaited
         }
 
         /// <summary>
-        /// Corner case 4 (fast rerun-entire-stage). A stage rerun submits a fresh current-attempt
-        /// incarnation of a work stream while the previous attempt's incarnation of the same stream
-        /// is still running. The monitor must gate on the current incarnation only, and must NOT
-        /// resubmit the still-running previous incarnation (no duplicate/triple submission).
+        /// A full-stage rerun has advanced the submitter to job attempt 2. The monitor starts
+        /// before the attempt-2 Helix job is visible. It must use the timeline attempt to suppress
+        /// replay of H1, wait for H2, and then gate on H2.
         /// </summary>
         [Fact]
         public async Task AttemptScoped_FastRerun_CurrentIncarnationExists_DoesNotResubmitPrevious()
@@ -1619,43 +1662,57 @@ public async Task AttemptScoped_FastRerun_CurrentIncarnationExists_DoesNotResubm
             var helix = new FakeHelixService();
 
             azdo.AddTimelineResponse(
-                StageRecord("Test", "stage-test", "inProgress"),
-                MonitorJob(parentId: "stage-test"),
-                PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test"));
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("Test_Linux", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"));
+            azdo.AddTimelineResponse(
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("Test_Linux", "completed", "succeeded", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"));
 
             // Previous incarnation still running; fresh current incarnation of the SAME stream
             // (same submitter + queue), not linked by PreviousHelixJobName.
-            HelixJobInfo previousRunning = HelixJob("helix-x-a1", "running", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1", logicalJobName: "tests");
+            HelixJobInfo previousRunning = HelixJob("helix-x-a1", "finished", stageName: "Test",
+                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1",
+                jobAttempt: "1", logicalJobName: "tests");
             HelixJobInfo currentDone = HelixJob("helix-x-a2", "finished", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "2", logicalJobName: "tests");
-            helix.WithWorkItems("helix-x-a1",
-                [new WorkItemSummary("helix-x-a1/wi", "helix-x-a1", "wi", "Running")]);
+                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "2",
+                jobAttempt: "2", logicalJobName: "tests");
 
+            helix.AddResponse(
+                jobs: [previousRunning],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-x-a1"] = PassFail(failed: ["wi"]),
+                });
             helix.AddResponse(
                 jobs: [previousRunning, currentDone],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
+                    ["helix-x-a1"] = PassFail(failed: ["wi"]),
                     ["helix-x-a2"] = PassFail(passed: ["wi"]),
                 });
 
             JobMonitorOptions options = DefaultOptions();
             options.StageAttempt = "2";
+            options.JobAttempt = "2";
             var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
 
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
-            // Gated on the current incarnation (completed) only; the still-running previous
-            // incarnation neither blocked termination nor was resubmitted.
             exitCode.Should().Be(0);
             helix.Resubmissions.Should().BeEmpty();
-            azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-x-a2"]);
+            azdo.UploadedJobNames.Should().Contain("helix-x-a2");
         }
 
         /// <summary>
         /// Corner case 5 (unlinked rerun duplicates). Two incarnations of the same work stream on
         /// different attempts are NOT connected by <c>PreviousHelixJobName</c> (a stage rerun, not
-        /// a monitor resubmission). The higher stage attempt's outcome must win the outcome map.
+        /// a monitor resubmission). The higher stage/job incarnation must win the outcome map.
         /// Job names are chosen so a naive job-name sort would let the failed attempt-1 job win.
         /// </summary>
         [Fact]
@@ -1665,17 +1722,21 @@ public async Task AttemptScoped_UnlinkedRerunDuplicates_HigherAttemptWinsOutcome
             var helix = new FakeHelixService();
 
             azdo.AddTimelineResponse(
-                StageRecord("Test", "stage-test", "inProgress"),
-                MonitorJob(parentId: "stage-test"),
-                PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test"));
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("Test_Linux", "completed", "succeeded", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"));
 
             // Same stream (submitter + queue), not lineage-linked. "zzz-old" (attempt 1, failed)
             // sorts AFTER "aaa-new" (attempt 2, passed): a job-name-ordered reconciliation would
             // let the failed attempt-1 outcome overwrite the passing attempt-2 one.
             HelixJobInfo oldFailed = HelixJob("zzz-old", "finished", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1", logicalJobName: "tests");
+                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1",
+                jobAttempt: "1", logicalJobName: "tests");
             HelixJobInfo newPassed = HelixJob("aaa-new", "finished", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "2", logicalJobName: "tests");
+                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "2",
+                jobAttempt: "2", logicalJobName: "tests");
 
             helix.AddResponse(
                 jobs: [oldFailed, newPassed],
@@ -1687,6 +1748,7 @@ public async Task AttemptScoped_UnlinkedRerunDuplicates_HigherAttemptWinsOutcome
 
             JobMonitorOptions options = DefaultOptions();
             options.StageAttempt = "2";
+            options.JobAttempt = "2";
             var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
 
             int exitCode = await runner.RunAsync(CancellationToken.None);
@@ -1696,6 +1758,105 @@ public async Task AttemptScoped_UnlinkedRerunDuplicates_HigherAttemptWinsOutcome
             helix.Resubmissions.Should().BeEmpty();
         }
 
+        /// <summary>
+        /// Retry-failed-jobs can rerun only a subset of submitters. Retry eligibility is decided
+        /// independently for each submitter: A's newer timeline attempt supersedes HA1, while B's
+        /// unchanged attempt leaves HB1 eligible for monitor replay.
+        /// </summary>
+        [Fact]
+        public async Task AttemptScoped_MixedSubmitterRetries_ReconcilesPerSubmitterAttempt()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+
+            azdo.AddTimelineResponse(
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("A", "completed", "succeeded", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("B", "completed", "succeeded", attempt: 1, parentId: "stage-test"));
+
+            HelixJobInfo ha1 = HelixJob("ha1", "finished", stageName: "Test",
+                submitterJobName: "A", queueId: "q", stageAttempt: "1",
+                jobAttempt: "1", logicalJobName: "tests");
+            HelixJobInfo ha2 = HelixJob("ha2", "finished", stageName: "Test",
+                submitterJobName: "A", queueId: "q", stageAttempt: "2",
+                jobAttempt: "2", logicalJobName: "tests");
+            HelixJobInfo hb1 = HelixJob("hb1", "finished", stageName: "Test",
+                submitterJobName: "B", queueId: "q", stageAttempt: "1",
+                jobAttempt: "1", logicalJobName: "tests");
+            HelixJobInfo rb2 = HelixJob("rb2", "finished", stageName: "Test",
+                submitterJobName: "B", queueId: "q", previousHelixJobName: "hb1",
+                stageAttempt: "2", jobAttempt: "1", logicalJobName: "tests");
+
+            helix.AddResponse(
+                jobs: [ha1, ha2, hb1],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["ha1"] = PassFail(failed: ["a"]),
+                    ["ha2"] = PassFail(passed: ["a"]),
+                    ["hb1"] = PassFail(failed: ["b"]),
+                });
+            helix.ConfigureResubmission("hb1", "rb2");
+            helix.AddResponse(
+                jobs: [ha1, ha2, hb1, rb2],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["ha1"] = PassFail(failed: ["a"]),
+                    ["ha2"] = PassFail(passed: ["a"]),
+                    ["hb1"] = PassFail(failed: ["b"]),
+                    ["rb2"] = PassFail(passed: ["b"]),
+                });
+
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+
+            int exitCode = await new JobMonitorRunner(
+                options, NullLogger.Instance, azdo, helix, NoDelay).RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(0);
+            helix.Resubmissions.Should().ContainSingle()
+                .Which.OriginalJob.Should().Be("hb1");
+        }
+
+        /// <summary>
+        /// A submitter that reran supersedes its previous Helix work even when the new execution
+        /// intentionally produces no replacement Helix job.
+        /// </summary>
+        [Fact]
+        public async Task AttemptScoped_RerunSubmitterProducesNoReplacement_DoesNotReplayOldWork()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+
+            azdo.AddTimelineResponse(
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("A", "completed", "succeeded", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"));
+            helix.AddResponse(
+                jobs: [HelixJob("ha1", "finished", stageName: "Test",
+                    submitterJobName: "A", queueId: "q", stageAttempt: "1",
+                    jobAttempt: "1", logicalJobName: "tests")],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["ha1"] = PassFail(failed: ["a"]),
+                });
+
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+
+            int exitCode = await new JobMonitorRunner(
+                options, NullLogger.Instance, azdo, helix, NoDelay).RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(1);
+            helix.Resubmissions.Should().BeEmpty();
+        }
+
         /// <summary>
         /// Corner case 6 (un-resubmittable previous work). Previous-attempt work that cannot be
         /// resubmitted (e.g. its Helix queue was removed) must fail the monitor fast with actionable
@@ -1709,12 +1870,13 @@ public async Task AttemptScoped_UnresubmittablePreviousWork_FailsFast()
             var logger = new RecordingLogger();
 
             azdo.AddTimelineResponse(
-                StageRecord("Test", "stage-test", "inProgress"),
-                MonitorJob(parentId: "stage-test"),
-                PipelineJob("Test Linux", "completed", "succeeded", parentId: "stage-test"));
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("Test_Linux", "completed", "succeeded", attempt: 1, parentId: "stage-test"));
 
             HelixJobInfo purged = HelixJob("helix-purged-a1", "running", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "gone", stageAttempt: "1");
+                submitterJobName: "Test_Linux", queueId: "gone", stageAttempt: "1", jobAttempt: "1");
             helix.WithWorkItems("helix-purged-a1",
                 [new WorkItemSummary("helix-purged-a1/wi-1", "helix-purged-a1", "wi-1", "Waiting")]);
 
@@ -1723,6 +1885,7 @@ public async Task AttemptScoped_UnresubmittablePreviousWork_FailsFast()
 
             JobMonitorOptions options = DefaultOptions();
             options.StageAttempt = "2";
+            options.JobAttempt = "2";
             var runner = new JobMonitorRunner(options, logger, azdo, helix, NoDelay);
 
             int exitCode = await runner.RunAsync(CancellationToken.None);
@@ -1736,6 +1899,52 @@ public async Task AttemptScoped_UnresubmittablePreviousWork_FailsFast()
                 && m.Contains("previous attempt", StringComparison.Ordinal));
         }
 
+        [Fact]
+        public async Task AttemptScoped_MissingSubmitterAttemptMetadata_DoesNotSpeculativelyResubmit()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+            var logger = new RecordingLogger();
+
+            azdo.AddTimelineResponse(
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("A", "completed", "succeeded", attempt: 1, parentId: "stage-test"));
+            helix.AddResponse(
+                jobs: [HelixJob("ha1", "finished", stageName: "Test",
+                    submitterJobName: "A", queueId: "q", stageAttempt: "1",
+                    logicalJobName: "tests")],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["ha1"] = PassFail(failed: ["a"]),
+                });
+
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+
+            int exitCode = await new JobMonitorRunner(
+                options, logger, azdo, helix, NoDelay).RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(1);
+            helix.Resubmissions.Should().BeEmpty();
+            logger.Messages.Should().Contain(message =>
+                message.Contains("could not be matched to compatible System.JobAttempt metadata", StringComparison.Ordinal));
+        }
+
+        [Fact]
+        public void SubmitterChainKey_IncludesStageIdentity()
+        {
+            var state = new MonitorState();
+            HelixJobInfo build = HelixJob("build-job", "finished", stageName: "Build",
+                submitterPhaseName: "Tests", queueId: "q", logicalJobName: "tests");
+            HelixJobInfo test = HelixJob("test-job", "finished", stageName: "Test",
+                submitterPhaseName: "Tests", queueId: "q", logicalJobName: "tests");
+
+            state.GetSubmitterChainKey(build).Should().NotBe(state.GetSubmitterChainKey(test));
+        }
+
         /// <summary>
         /// Backward compatibility: when the monitor's own stage attempt is unknown (no
         /// <c>SYSTEM_STAGEATTEMPT</c>), attempt scoping is disabled and the monitor tracks jobs
@@ -1788,9 +1997,9 @@ public async Task AttemptScopedMonitor_UnknownMonitorAttempt_TracksAllAttempts()
         [Fact]
         public async Task MultiAttempt_ResubmitsOnlyUnfinishedStreamsAcrossAttemptsAndMonitorCrash()
         {
-            HelixJobInfo a1 = HelixJob("a1", "finished", stageName: "Test", submitterJobName: "Test_A", queueId: "q1", stageAttempt: "1");
-            HelixJobInfo b1 = HelixJob("b1", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", stageAttempt: "1");
-            HelixJobInfo c1 = HelixJob("c1", "finished", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", stageAttempt: "1");
+            HelixJobInfo a1 = HelixJob("a1", "finished", stageName: "Test", submitterJobName: "Test_A", queueId: "q1", stageAttempt: "1", jobAttempt: "1");
+            HelixJobInfo b1 = HelixJob("b1", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", stageAttempt: "1", jobAttempt: "1");
+            HelixJobInfo c1 = HelixJob("c1", "finished", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", stageAttempt: "1", jobAttempt: "1");
 
             // Attempt 2: resubmit both failing streams (B, C) into attempt 2, then time out.
             FakeHelixService helix2 = await RunAttemptToTimeoutAsync(
@@ -1810,8 +2019,8 @@ public async Task MultiAttempt_ResubmitsOnlyUnfinishedStreamsAcrossAttemptsAndMo
 
             // Attempt 3: B's attempt-2 resubmission failed; C's is still Waiting. Resubmit both
             // into attempt 3, then time out.
-            HelixJobInfo b2 = HelixJob("b2", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", previousHelixJobName: "b1", stageAttempt: "2");
-            HelixJobInfo c2 = HelixJob("c2", "running", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c1", stageAttempt: "2");
+            HelixJobInfo b2 = HelixJob("b2", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", previousHelixJobName: "b1", stageAttempt: "2", jobAttempt: "1");
+            HelixJobInfo c2 = HelixJob("c2", "running", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c1", stageAttempt: "2", jobAttempt: "1");
 
             FakeHelixService helix3 = await RunAttemptToTimeoutAsync(
                 stageAttempt: "3",
@@ -1833,15 +2042,17 @@ public async Task MultiAttempt_ResubmitsOnlyUnfinishedStreamsAcrossAttemptsAndMo
             // Attempt 5: B's attempt-3 resubmission PASSED; C's is still Waiting. The monitor must
             // resubmit ONLY the still-Waiting C stream, not the now-passing B stream, and complete
             // once C's resubmission finishes.
-            HelixJobInfo b3 = HelixJob("b3", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", previousHelixJobName: "b2", stageAttempt: "3");
-            HelixJobInfo c3 = HelixJob("c3", "running", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c2", stageAttempt: "3");
-            HelixJobInfo c5 = HelixJob("c5", "finished", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c3", stageAttempt: "5");
+            HelixJobInfo b3 = HelixJob("b3", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", previousHelixJobName: "b2", stageAttempt: "3", jobAttempt: "1");
+            HelixJobInfo c3 = HelixJob("c3", "running", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c2", stageAttempt: "3", jobAttempt: "1");
+            HelixJobInfo c5 = HelixJob("c5", "finished", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c3", stageAttempt: "5", jobAttempt: "1");
 
             var azdo5 = new FakeAzureDevOpsService();
             azdo5.AddTimelineResponse(
-                StageRecord("Test", "stage-test", "inProgress"),
-                MonitorJob(parentId: "stage-test"),
-                PipelineJob("Test Suite", "completed", "succeeded", parentId: "stage-test"));
+                StageRecord("Test", "stage-test", "inProgress", attempt: 5),
+                MonitorJob(attempt: 5, parentId: "stage-test"),
+                PipelineJob("Test_C", "completed", "succeeded", attempt: 1, parentId: "stage-test"),
+                PipelineJob("Test_B", "completed", "succeeded", attempt: 1, parentId: "stage-test"),
+                PipelineJob("Test_A", "completed", "succeeded", attempt: 1, parentId: "stage-test"));
             // A and B were already uploaded by earlier attempts (durable AzDO tags).
             azdo5.WithPreviouslyProcessedJob("a1").WithPreviouslyProcessedJob("b3");
 
@@ -1868,6 +2079,7 @@ public async Task MultiAttempt_ResubmitsOnlyUnfinishedStreamsAcrossAttemptsAndMo
 
             var options5 = DefaultOptions();
             options5.StageAttempt = "5";
+            options5.JobAttempt = "5";
             var runner5 = new JobMonitorRunner(options5, NullLogger.Instance, azdo5, helix5, NoDelay);
 
             int exit5 = await runner5.RunAsync(CancellationToken.None);
@@ -1888,16 +2100,18 @@ public async Task MultiAttempt_ResubmitsOnlyUnfinishedStreamsAcrossAttemptsAndMo
         [Fact]
         public async Task MultiAttempt_PreviouslyWaitingStreamHasPassedOnRetry_NotResubmitted()
         {
-            HelixJobInfo a1 = HelixJob("a1", "finished", stageName: "Test", submitterJobName: "Test_A", queueId: "q1", stageAttempt: "1");
-            HelixJobInfo b3 = HelixJob("b3", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", previousHelixJobName: "b2", stageAttempt: "3");
+            HelixJobInfo a1 = HelixJob("a1", "finished", stageName: "Test", submitterJobName: "Test_A", queueId: "q1", stageAttempt: "1", jobAttempt: "1");
+            HelixJobInfo b3 = HelixJob("b3", "finished", stageName: "Test", submitterJobName: "Test_B", queueId: "q1", previousHelixJobName: "b2", stageAttempt: "3", jobAttempt: "1");
             // Same C incarnation that was Waiting at the last timeout, now reported finished+passed.
-            HelixJobInfo c3 = HelixJob("c3", "finished", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c2", stageAttempt: "3");
+            HelixJobInfo c3 = HelixJob("c3", "finished", stageName: "Test", submitterJobName: "Test_C", queueId: "q1", previousHelixJobName: "c2", stageAttempt: "3", jobAttempt: "1");
 
             var azdo = new FakeAzureDevOpsService();
             azdo.AddTimelineResponse(
-                StageRecord("Test", "stage-test", "inProgress"),
-                MonitorJob(parentId: "stage-test"),
-                PipelineJob("Test Suite", "completed", "succeeded", parentId: "stage-test"));
+                StageRecord("Test", "stage-test", "inProgress", attempt: 5),
+                MonitorJob(attempt: 5, parentId: "stage-test"),
+                PipelineJob("Test_A", "completed", "succeeded", attempt: 1, parentId: "stage-test"),
+                PipelineJob("Test_B", "completed", "succeeded", attempt: 1, parentId: "stage-test"),
+                PipelineJob("Test_C", "completed", "succeeded", attempt: 1, parentId: "stage-test"));
             azdo.WithPreviouslyProcessedJob("a1").WithPreviouslyProcessedJob("b3");
 
             var helix = new FakeHelixService();
@@ -1912,6 +2126,7 @@ public async Task MultiAttempt_PreviouslyWaitingStreamHasPassedOnRetry_NotResubm
 
             var options = DefaultOptions();
             options.StageAttempt = "5";
+            options.JobAttempt = "5";
             var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
 
             int exitCode = await runner.RunAsync(CancellationToken.None);
@@ -2180,7 +2395,8 @@ public async Task StageScopedMonitor_OnStageRetry_OnlyRetriesFailedHelixWorkFrom
             helix2.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-test-linux", "finished", stageName: "Test", submitterJobName: "Test Linux"),
+                    HelixJob("helix-test-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
                     HelixJob("helix-build-windows", "finished", stageName: "Build", submitterJobName: "Build Windows"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
@@ -2193,8 +2409,11 @@ public async Task StageScopedMonitor_OnStageRetry_OnlyRetriesFailedHelixWorkFrom
             helix2.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-test-linux", "finished", stageName: "Test", submitterJobName: "Test Linux"),
-                    HelixJob("helix-test-linux-resub", "finished", stageName: "Test", submitterJobName: "Test Linux", previousHelixJobName: "helix-test-linux"),
+                    HelixJob("helix-test-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-test-linux-resub", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-test-linux",
+                        stageAttempt: "2", jobAttempt: "1"),
                     HelixJob("helix-build-windows", "finished", stageName: "Build", submitterJobName: "Build Windows"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
@@ -2204,7 +2423,11 @@ public async Task StageScopedMonitor_OnStageRetry_OnlyRetriesFailedHelixWorkFrom
                     ["helix-build-windows"] = PassFail(failed: ["build-fail"]),
                 });
 
-            var runner2 = CreateRunner(azdo2, helix2, stageName: "Test");
+            JobMonitorOptions options2 = DefaultOptions();
+            options2.StageName = "Test";
+            options2.StageAttempt = "2";
+            options2.JobAttempt = "2";
+            var runner2 = new JobMonitorRunner(options2, NullLogger.Instance, azdo2, helix2, NoDelay);
             int exitCode2 = await runner2.RunAsync(CancellationToken.None);
 
             exitCode2.Should().Be(0);
@@ -2219,9 +2442,10 @@ static void AddSingleMonitorStageTimeline(FakeAzureDevOpsService azdo, int attem
                 PreviousAttemptReference[] previousAttempts = attempt == 1 ? null : [PreviousAttempt(1)];
 
                 azdo.AddTimelineResponse(
-                    StageRecord("Test", "stage-test", "inProgress"),
+                    StageRecord("Test", "stage-test", "inProgress", attempt: attempt,
+                        previousAttempts: previousAttempts),
                     MonitorJob(attempt: attempt, previousAttempts: previousAttempts, parentId: "stage-test"),
-                    PipelineJob("Test Linux", "completed", "succeeded", attempt: attempt, previousAttempts: previousAttempts, parentId: "stage-test"),
+                    PipelineJob("Test Linux", "completed", "succeeded", attempt: 1, parentId: "stage-test"),
                     StageRecord("Build", "stage-build", "completed", "succeeded"),
                     PipelineJob("Build Windows", "completed", "succeeded", parentId: "stage-build"));
             }
@@ -2242,7 +2466,9 @@ public async Task StageScopedMonitor_OnRetry_IgnoresDefaultRefNameMonitorJobUnde
             AddRetriedStageTimeline(azdo);
 
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished", stageName: "Test", submitterJobName: "Build_Debug")],
+                jobs: [HelixJob("helix-linux", "finished", stageName: "Test",
+                    submitterJobName: "Build_Debug", submitterPhaseName: "Linux Build_Debug",
+                    stageAttempt: "1", jobAttempt: "1")],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(failed: ["linux-fail"]),
@@ -2251,8 +2477,12 @@ public async Task StageScopedMonitor_OnRetry_IgnoresDefaultRefNameMonitorJobUnde
             helix.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-linux", "finished", stageName: "Test", submitterJobName: "Build_Debug"),
-                    HelixJob("helix-linux-resub", "finished", stageName: "Test", submitterJobName: "Build_Debug", previousHelixJobName: "helix-linux"),
+                    HelixJob("helix-linux", "finished", stageName: "Test",
+                        submitterJobName: "Build_Debug", submitterPhaseName: "Linux Build_Debug",
+                        stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "finished", stageName: "Test",
+                        submitterJobName: "Build_Debug", submitterPhaseName: "Linux Build_Debug",
+                        previousHelixJobName: "helix-linux", stageAttempt: "2", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
@@ -2263,6 +2493,8 @@ public async Task StageScopedMonitor_OnRetry_IgnoresDefaultRefNameMonitorJobUnde
             int delayCount = 0;
             var options = DefaultOptions();
             options.StageName = "Test";
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
             options.JobMonitorName = "HelixJobMonitor";
             var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix,
                 (_, _) =>
@@ -2288,7 +2520,8 @@ public async Task StageScopedMonitor_OnRetry_IgnoresDefaultRefNameMonitorJobUnde
             static void AddRetriedStageTimeline(FakeAzureDevOpsService azdo)
             {
                 azdo.AddTimelineResponse(
-                    StageRecord("Test", "stage-test", "inProgress"),
+                    StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                        previousAttempts: [PreviousAttempt(1)]),
                     new AzureDevOpsTimelineRecord
                     {
                         Id = "monitor-phase",
@@ -2706,19 +2939,20 @@ public async Task RetryAttempt2_ResubmitsFailedWorkItems_ResubmissionPasses_Exit
 
             // Pipeline jobs already completed
             azdo.AddTimelineResponse(
-                MonitorJob(),
-                PipelineJob("Test Linux", "completed", "succeeded"));
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
             // Extra polls while resubmitted job runs
             azdo.AddTimelineResponse(
-                MonitorJob(),
-                PipelineJob("Test Linux", "completed", "succeeded"));
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
             azdo.AddTimelineResponse(
-                MonitorJob(),
-                PipelineJob("Test Linux", "completed", "succeeded"));
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
 
             // Poll 1: original Helix job finished with 1 pass + 2 failures
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished")],
+                jobs: [HelixJob("helix-linux", "finished", stageName: "Test",
+                    submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(passed: ["wi-ok"], failed: ["wi-fail-1", "wi-fail-2"]),
@@ -2728,7 +2962,14 @@ public async Task RetryAttempt2_ResubmitsFailedWorkItems_ResubmissionPasses_Exit
 
             // Poll 2: resubmitted job appears, running
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished"), HelixJob("helix-linux-resub", "running", previousHelixJobName: "helix-linux")],
+                jobs:
+                [
+                    HelixJob("helix-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "running", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-linux",
+                        stageAttempt: "2", jobAttempt: "1"),
+                ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(passed: ["wi-ok"], failed: ["wi-fail-1", "wi-fail-2"]),
@@ -2736,14 +2977,24 @@ public async Task RetryAttempt2_ResubmitsFailedWorkItems_ResubmissionPasses_Exit
 
             // Poll 3: resubmitted job finished — both items now pass
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished"), HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux")],
+                jobs:
+                [
+                    HelixJob("helix-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-linux",
+                        stageAttempt: "2", jobAttempt: "1"),
+                ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(passed: ["wi-ok"], failed: ["wi-fail-1", "wi-fail-2"]),
                     ["helix-linux-resub"] = PassFail(passed: ["wi-fail-1", "wi-fail-2"]),
                 });
 
-            var runner = CreateRunner(azdo, helix);
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+            var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             // Resubmission healed the failures → exit 0
@@ -2772,15 +3023,16 @@ public async Task RetryAttempt2_ResubmitsFailedWorkItems_ResubmissionAlsoFails_E
             var helix = new FakeHelixService();
 
             azdo.AddTimelineResponse(
-                MonitorJob(),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
                 PipelineJob("Test Linux", "completed", "succeeded"));
             azdo.AddTimelineResponse(
-                MonitorJob(),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
                 PipelineJob("Test Linux", "completed", "succeeded"));
 
             // Poll 1: original job has 1 failure
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished")],
+                jobs: [HelixJob("helix-linux", "finished", stageName: "Test",
+                    submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(passed: ["wi-ok"], failed: ["wi-fail"]),
@@ -2789,14 +3041,24 @@ public async Task RetryAttempt2_ResubmitsFailedWorkItems_ResubmissionAlsoFails_E
 
             // Poll 2: resubmission finished but STILL fails
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished"), HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux")],
+                jobs:
+                [
+                    HelixJob("helix-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-linux",
+                        stageAttempt: "2", jobAttempt: "1"),
+                ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(passed: ["wi-ok"], failed: ["wi-fail"]),
                     ["helix-linux-resub"] = PassFail(failed: ["wi-fail"]),
                 });
 
-            var runner = CreateRunner(azdo, helix);
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+            var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             // Resubmission also failed → exit 1
@@ -2816,21 +3078,27 @@ public async Task RetryAttempt2_MultipleJobs_OnlyFailedItemsResubmitted_ExitZero
             var helix = new FakeHelixService();
 
             azdo.AddTimelineResponse(
-                MonitorJob(),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
                 PipelineJob("Test Linux", "completed", "succeeded"),
                 PipelineJob("Test Windows", "completed", "succeeded"));
             azdo.AddTimelineResponse(
-                MonitorJob(),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
                 PipelineJob("Test Linux", "completed", "succeeded"),
                 PipelineJob("Test Windows", "completed", "succeeded"));
             azdo.AddTimelineResponse(
-                MonitorJob(),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
                 PipelineJob("Test Linux", "completed", "succeeded"),
                 PipelineJob("Test Windows", "completed", "succeeded"));
 
             // Poll 1: both original jobs finished with mixed results
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished"), HelixJob("helix-windows", "finished")],
+                jobs:
+                [
+                    HelixJob("helix-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-windows", "finished", stageName: "Test",
+                        submitterJobName: "Test Windows", stageAttempt: "1", jobAttempt: "1"),
+                ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(passed: ["linux-ok-1", "linux-ok-2"], failed: ["linux-fail"]),
@@ -2843,9 +3111,10 @@ public async Task RetryAttempt2_MultipleJobs_OnlyFailedItemsResubmitted_ExitZero
             helix.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-linux", "finished"), HelixJob("helix-windows", "finished"),
-                    HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"),
-                    HelixJob("helix-windows-resub", "running", previousHelixJobName: "helix-windows"),
+                    HelixJob("helix-linux", "finished", stageName: "Test", submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-windows", "finished", stageName: "Test", submitterJobName: "Test Windows", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "finished", stageName: "Test", submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", stageAttempt: "2", jobAttempt: "1"),
+                    HelixJob("helix-windows-resub", "running", stageName: "Test", submitterJobName: "Test Windows", previousHelixJobName: "helix-windows", stageAttempt: "2", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
@@ -2858,9 +3127,10 @@ public async Task RetryAttempt2_MultipleJobs_OnlyFailedItemsResubmitted_ExitZero
             helix.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-linux", "finished"), HelixJob("helix-windows", "finished"),
-                    HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"),
-                    HelixJob("helix-windows-resub", "finished", previousHelixJobName: "helix-windows"),
+                    HelixJob("helix-linux", "finished", stageName: "Test", submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-windows", "finished", stageName: "Test", submitterJobName: "Test Windows", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "finished", stageName: "Test", submitterJobName: "Test Linux", previousHelixJobName: "helix-linux", stageAttempt: "2", jobAttempt: "1"),
+                    HelixJob("helix-windows-resub", "finished", stageName: "Test", submitterJobName: "Test Windows", previousHelixJobName: "helix-windows", stageAttempt: "2", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
@@ -2870,7 +3140,10 @@ public async Task RetryAttempt2_MultipleJobs_OnlyFailedItemsResubmitted_ExitZero
                     ["helix-windows-resub"] = PassFail(passed: ["win-fail-1", "win-fail-2"]),
                 });
 
-            var runner = CreateRunner(azdo, helix);
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+            var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             exitCode.Should().Be(0);
@@ -2929,20 +3202,31 @@ public async Task HelixJobFailsAfterMonitorEntry_IsNotResubmittedUntilNextEntry(
             azdo2.AddTimelineResponse(MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]), PipelineJob("Test Linux", "completed", "succeeded"));
 
             helix2.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished")],
+                jobs: [HelixJob("helix-linux", "finished", stageName: "Test",
+                    submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(failed: ["wi-fail"]),
                 });
             helix2.ConfigureResubmission("helix-linux", "helix-linux-resub");
             helix2.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished"), HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux")],
+                jobs:
+                [
+                    HelixJob("helix-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-linux",
+                        stageAttempt: "2", jobAttempt: "1"),
+                ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux-resub"] = PassFail(passed: ["wi-fail"]),
                 });
 
-            var runner2 = CreateRunner(azdo2, helix2);
+            JobMonitorOptions options2 = DefaultOptions();
+            options2.StageAttempt = "2";
+            options2.JobAttempt = "2";
+            var runner2 = new JobMonitorRunner(options2, NullLogger.Instance, azdo2, helix2, NoDelay);
             int exitCode2 = await runner2.RunAsync(CancellationToken.None);
 
             exitCode2.Should().Be(0);
@@ -3040,14 +3324,21 @@ public async Task LatestCompletedIncarnationPartiallyHealed_ResubmitsOnlyRemaini
             var azdo = new FakeAzureDevOpsService();
             var helix = new FakeHelixService();
 
-            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
-            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
+            azdo.AddTimelineResponse(
+                MonitorJob(attempt: 3, previousAttempts: [PreviousAttempt(1), PreviousAttempt(2)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
+            azdo.AddTimelineResponse(
+                MonitorJob(attempt: 3, previousAttempts: [PreviousAttempt(1), PreviousAttempt(2)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
 
             helix.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-linux", "finished"),
-                    HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"),
+                    HelixJob("helix-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-linux",
+                        stageAttempt: "2", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
@@ -3058,9 +3349,14 @@ public async Task LatestCompletedIncarnationPartiallyHealed_ResubmitsOnlyRemaini
             helix.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-linux", "finished"),
-                    HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"),
-                    HelixJob("helix-linux-resub-2", "finished", previousHelixJobName: "helix-linux-resub"),
+                    HelixJob("helix-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-linux",
+                        stageAttempt: "2", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub-2", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-linux-resub",
+                        stageAttempt: "3", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
@@ -3069,7 +3365,10 @@ public async Task LatestCompletedIncarnationPartiallyHealed_ResubmitsOnlyRemaini
                     ["helix-linux-resub-2"] = PassFail(passed: ["wi-2"]),
                 });
 
-            var runner = CreateRunner(azdo, helix);
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "3";
+            options.JobAttempt = "3";
+            var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             exitCode.Should().Be(0);
@@ -3207,7 +3506,11 @@ public async Task RetryAfterMixedAzDOAndHelixFailures_RestartsFailedHelixWorkAnd
                 PipelineJob("C", "completed", "succeeded", attempt: 2, previousAttempts: [PreviousAttempt(1)]));
 
             helix2.AddResponse(
-                jobs: [HelixJob("helix-a", "finished", submitterJobName: "A"), HelixJob("helix-b", "finished", submitterJobName: "B")],
+                jobs:
+                [
+                    HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"),
+                ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-a"] = PassFail(failed: ["a-fail"]),
@@ -3218,10 +3521,10 @@ public async Task RetryAfterMixedAzDOAndHelixFailures_RestartsFailedHelixWorkAnd
             helix2.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-a", "finished", submitterJobName: "A"),
-                    HelixJob("helix-b", "finished", submitterJobName: "B"),
-                    HelixJob("helix-a-resub", "finished", submitterJobName: "A", previousHelixJobName: "helix-a"),
-                    HelixJob("helix-b-resub", "finished", submitterJobName: "B", previousHelixJobName: "helix-b"),
+                    HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-a-resub", "finished", stageName: "Test", submitterJobName: "A", previousHelixJobName: "helix-a", stageAttempt: "2", jobAttempt: "1"),
+                    HelixJob("helix-b-resub", "finished", stageName: "Test", submitterJobName: "B", previousHelixJobName: "helix-b", stageAttempt: "2", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
@@ -3229,7 +3532,10 @@ public async Task RetryAfterMixedAzDOAndHelixFailures_RestartsFailedHelixWorkAnd
                     ["helix-b-resub"] = PassFail(failed: ["b-fail"]),
                 });
 
-            var runner2 = CreateRunner(azdo2, helix2);
+            JobMonitorOptions options2 = DefaultOptions();
+            options2.StageAttempt = "2";
+            options2.JobAttempt = "2";
+            var runner2 = new JobMonitorRunner(options2, NullLogger.Instance, azdo2, helix2, NoDelay);
             int exitCode2 = await runner2.RunAsync(CancellationToken.None);
 
             exitCode2.Should().Be(1);
@@ -3261,10 +3567,10 @@ public async Task RetryAfterMixedAzDOAndHelixFailures_RestartsFailedHelixWorkAnd
             helix3.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-a", "finished", submitterJobName: "A"),
-                    HelixJob("helix-b", "finished", submitterJobName: "B"),
-                    HelixJob("helix-a-resub", "finished", submitterJobName: "A", previousHelixJobName: "helix-a"),
-                    HelixJob("helix-b-resub", "finished", submitterJobName: "B", previousHelixJobName: "helix-b"),
+                    HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-a-resub", "finished", stageName: "Test", submitterJobName: "A", previousHelixJobName: "helix-a", stageAttempt: "2", jobAttempt: "1"),
+                    HelixJob("helix-b-resub", "finished", stageName: "Test", submitterJobName: "B", previousHelixJobName: "helix-b", stageAttempt: "2", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
@@ -3275,18 +3581,21 @@ public async Task RetryAfterMixedAzDOAndHelixFailures_RestartsFailedHelixWorkAnd
             helix3.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-a", "finished", submitterJobName: "A"),
-                    HelixJob("helix-b", "finished", submitterJobName: "B"),
-                    HelixJob("helix-a-resub", "finished", submitterJobName: "A", previousHelixJobName: "helix-a"),
-                    HelixJob("helix-b-resub", "finished", submitterJobName: "B", previousHelixJobName: "helix-b"),
-                    HelixJob("helix-b-resub-2", "finished", submitterJobName: "B", previousHelixJobName: "helix-b-resub"),
+                    HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-a-resub", "finished", stageName: "Test", submitterJobName: "A", previousHelixJobName: "helix-a", stageAttempt: "2", jobAttempt: "1"),
+                    HelixJob("helix-b-resub", "finished", stageName: "Test", submitterJobName: "B", previousHelixJobName: "helix-b", stageAttempt: "2", jobAttempt: "1"),
+                    HelixJob("helix-b-resub-2", "finished", stageName: "Test", submitterJobName: "B", previousHelixJobName: "helix-b-resub", stageAttempt: "3", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-b-resub-2"] = PassFail(passed: ["b-fail"]),
                 });
 
-            var runner3 = CreateRunner(azdo3, helix3);
+            JobMonitorOptions options3 = DefaultOptions();
+            options3.StageAttempt = "3";
+            options3.JobAttempt = "3";
+            var runner3 = new JobMonitorRunner(options3, NullLogger.Instance, azdo3, helix3, NoDelay);
             int exitCode3 = await runner3.RunAsync(CancellationToken.None);
 
             exitCode3.Should().Be(0);
@@ -3349,9 +3658,7 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork()
             int exitCode1 = await runner1.RunAsync(cts1.Token);
 
             exitCode1.Should().Be(1);
-            helix1.Resubmissions.Should().ContainSingle();
-            helix1.Resubmissions[0].OriginalJob.Should().Be("helix-a");
-            helix1.Resubmissions[0].FailedItems.Should().BeEquivalentTo(["a-fail"]);
+            helix1.Resubmissions.Should().BeEmpty();
             azdo1.UploadedJobNames.Should().BeEquivalentTo(["helix-a"]);
 
             var azdo2 = new FakeAzureDevOpsService();
@@ -3410,9 +3717,9 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork()
             helix3.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-a", "finished", submitterJobName: "A"),
-                    HelixJob("helix-a-resub", "running", submitterJobName: "A", previousHelixJobName: "helix-a"),
-                    HelixJob("helix-b", "finished", submitterJobName: "B"),
+                    HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-a-resub", "running", stageName: "Test", submitterJobName: "A", previousHelixJobName: "helix-a", stageAttempt: "2", jobAttempt: "1"),
+                    HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
@@ -3422,10 +3729,10 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork()
             helix3.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-a", "finished", submitterJobName: "A"),
-                    HelixJob("helix-a-resub", "running", submitterJobName: "A", previousHelixJobName: "helix-a"),
-                    HelixJob("helix-b", "finished", submitterJobName: "B"),
-                    HelixJob("helix-b-resub", "running", submitterJobName: "B", previousHelixJobName: "helix-b"),
+                    HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-a-resub", "running", stageName: "Test", submitterJobName: "A", previousHelixJobName: "helix-a", stageAttempt: "2", jobAttempt: "1"),
+                    HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-b-resub", "running", stageName: "Test", submitterJobName: "B", previousHelixJobName: "helix-b", stageAttempt: "3", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
@@ -3434,10 +3741,10 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork()
             helix3.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-a", "finished", submitterJobName: "A"),
-                    HelixJob("helix-a-resub", "finished", submitterJobName: "A", previousHelixJobName: "helix-a"),
-                    HelixJob("helix-b", "finished", submitterJobName: "B"),
-                    HelixJob("helix-b-resub", "finished", submitterJobName: "B", previousHelixJobName: "helix-b"),
+                    HelixJob("helix-a", "finished", stageName: "Test", submitterJobName: "A", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-a-resub", "finished", stageName: "Test", submitterJobName: "A", previousHelixJobName: "helix-a", stageAttempt: "2", jobAttempt: "1"),
+                    HelixJob("helix-b", "finished", stageName: "Test", submitterJobName: "B", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-b-resub", "finished", stageName: "Test", submitterJobName: "B", previousHelixJobName: "helix-b", stageAttempt: "3", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
@@ -3446,7 +3753,10 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork()
                     ["helix-b-resub"] = PassFail(passed: ["b-fail"]),
                 });
 
-            var runner3 = CreateRunner(azdo3, helix3);
+            JobMonitorOptions options3 = DefaultOptions();
+            options3.StageAttempt = "3";
+            options3.JobAttempt = "3";
+            var runner3 = new JobMonitorRunner(options3, NullLogger.Instance, azdo3, helix3, NoDelay);
             int exitCode3 = await runner3.RunAsync(CancellationToken.None);
 
             exitCode3.Should().Be(0);
@@ -3461,7 +3771,7 @@ public async Task RetryOnEntryWithCrashes_ResubmitsOnlyLatestFailedWork()
         /// the monitor's first attempt. The original failed results are still uploaded normally.
         /// </summary>
         [Fact]
-        public async Task Attempt1_ResubmitsFailedWorkItemsFoundOnEntry_ExitOne()
+        public async Task Attempt1_DoesNotResubmitFailedWorkItemsFoundOnEntry_ExitOne()
         {
             var azdo = new FakeAzureDevOpsService();
             var helix = new FakeHelixService();
@@ -3481,9 +3791,7 @@ public async Task Attempt1_ResubmitsFailedWorkItemsFoundOnEntry_ExitOne()
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             exitCode.Should().Be(1);
-            helix.Resubmissions.Should().ContainSingle();
-            helix.Resubmissions[0].OriginalJob.Should().Be("helix-linux");
-            helix.Resubmissions[0].FailedItems.Should().BeEquivalentTo(["wi-fail"]);
+            helix.Resubmissions.Should().BeEmpty();
             azdo.UploadedJobNames.Should().ContainSingle(); // only original
         }
 
@@ -3514,9 +3822,7 @@ public async Task FailedAzdoSubmitter_WithoutSystemJobNameLink_IsNotIgnored()
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             exitCode.Should().Be(1);
-            helix.Resubmissions.Should().ContainSingle();
-            helix.Resubmissions[0].OriginalJob.Should().Be("helix-a");
-            helix.Resubmissions[0].FailedItems.Should().BeEquivalentTo(["a-fail"]);
+            helix.Resubmissions.Should().BeEmpty();
             azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-a"]);
         }
 
@@ -3531,18 +3837,22 @@ public async Task ResubmitReturnsNull_FailedSubmitterIsNotIgnored()
             var helix = new FakeHelixService();
 
             azdo.AddTimelineResponse(
-                MonitorJob(),
-                PipelineJob("A", "completed", "failed"));
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("A", "completed", "failed", attempt: 1));
 
             helix.AddResponse(
-                jobs: [HelixJob("helix-a", "finished", submitterJobName: "A")],
+                jobs: [HelixJob("helix-a", "finished", stageName: "Test",
+                    submitterJobName: "A", stageAttempt: "1", jobAttempt: "1")],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-a"] = PassFail(failed: ["a-fail"]),
                 });
             helix.ConfigureNullResubmission("helix-a");
 
-            var runner = CreateRunner(azdo, helix);
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+            var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             exitCode.Should().Be(1);
@@ -3669,11 +3979,16 @@ public async Task PreviouslyProcessedFailedJob_StillContributesToRetryAndPassFai
             azdo.WithPreviouslyProcessedJob("helix-linux");
             var helix = new FakeHelixService();
 
-            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
-            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
+            azdo.AddTimelineResponse(
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
+            azdo.AddTimelineResponse(
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
 
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished")],
+                jobs: [HelixJob("helix-linux", "finished", stageName: "Test",
+                    submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(failed: ["wi-fail"]),
@@ -3682,15 +3997,21 @@ public async Task PreviouslyProcessedFailedJob_StillContributesToRetryAndPassFai
             helix.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-linux", "finished"),
-                    HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"),
+                    HelixJob("helix-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-linux",
+                        stageAttempt: "2", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux-resub"] = PassFail(passed: ["wi-fail"]),
                 });
 
-            var runner = CreateRunner(azdo, helix);
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+            var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             exitCode.Should().Be(0);
@@ -3712,11 +4033,16 @@ public async Task PreviouslyProcessedFailedJob_WithoutResubmission_StillFailsBui
             azdo.WithPreviouslyProcessedJob("helix-linux");
             var helix = new FakeHelixService();
 
-            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
-            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
+            azdo.AddTimelineResponse(
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
+            azdo.AddTimelineResponse(
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
 
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished")],
+                jobs: [HelixJob("helix-linux", "finished", stageName: "Test",
+                    submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(failed: ["wi-fail"]),
@@ -3724,7 +4050,10 @@ public async Task PreviouslyProcessedFailedJob_WithoutResubmission_StillFailsBui
             // The failed work item cannot be resubmitted (e.g. resubmission limit reached).
             helix.ConfigureNullResubmission("helix-linux");
 
-            var runner = CreateRunner(azdo, helix);
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+            var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             exitCode.Should().Be(1);
@@ -3862,7 +4191,8 @@ public async Task StageScopedMonitor_LineageCrossesStageBoundary_IgnoresOutOfSta
             helix.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-test", "finished", stageName: "Test"),
+                    HelixJob("helix-test", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
                     HelixJob("helix-build-retry", "running", stageName: "Build", previousHelixJobName: "helix-test"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
@@ -3873,8 +4203,11 @@ public async Task StageScopedMonitor_LineageCrossesStageBoundary_IgnoresOutOfSta
             helix.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-test", "finished", stageName: "Test"),
-                    HelixJob("helix-test-resub", "finished", stageName: "Test", previousHelixJobName: "helix-test"),
+                    HelixJob("helix-test", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-test-resub", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-test",
+                        stageAttempt: "2", jobAttempt: "1"),
                     HelixJob("helix-build-retry", "running", stageName: "Build", previousHelixJobName: "helix-test"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
@@ -3883,7 +4216,11 @@ public async Task StageScopedMonitor_LineageCrossesStageBoundary_IgnoresOutOfSta
                     ["helix-test-resub"] = PassFail(passed: ["test-fail"]),
                 });
 
-            var runner = CreateRunner(azdo, helix, stageName: "Test");
+            JobMonitorOptions options = DefaultOptions();
+            options.StageName = "Test";
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+            var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             exitCode.Should().Be(0);
@@ -3928,10 +4265,15 @@ public async Task FinishedHelixJob_WithNonFinishedWorkItemState_IsFailure()
             var azdo = new FakeAzureDevOpsService();
             var helix = new FakeHelixService();
 
-            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
-            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
+            azdo.AddTimelineResponse(
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
+            azdo.AddTimelineResponse(
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
 
-            helix.AddResponse(jobs: [HelixJob("helix-linux", "finished")]);
+            helix.AddResponse(jobs: [HelixJob("helix-linux", "finished", stageName: "Test",
+                submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")]);
             helix.WithWorkItems(
                 "helix-linux",
                 [new WorkItemSummary("helix-linux/wi-timeout", "helix-linux", "wi-timeout", "TimedOut") { ExitCode = 0 }]);
@@ -3939,15 +4281,21 @@ public async Task FinishedHelixJob_WithNonFinishedWorkItemState_IsFailure()
             helix.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-linux", "finished"),
-                    HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"),
+                    HelixJob("helix-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-linux",
+                        stageAttempt: "2", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux-resub"] = PassFail(passed: ["wi-timeout"]),
                 });
 
-            var runner = CreateRunner(azdo, helix);
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+            var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             exitCode.Should().Be(0);
@@ -4199,14 +4547,19 @@ public async Task RetryPass_ResubmitsWorkItemThatPassedExitCodeButHasPriorFailed
             // It must not be resubmitted twice (once per failure reason).
             azdo.WithRecordedFailedTest("helix-linux", "workitem-2");
 
-            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
-            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
+            azdo.AddTimelineResponse(
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
+            azdo.AddTimelineResponse(
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
 
             // Snapshot 1 (retry pass): only the original Helix job exists, finished, with
             // workitem-1 passing by exit code and workitem-2 failing by exit code. Without the
             // AzDO test-failure lookup the retry pass would only resubmit workitem-2.
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished")],
+                jobs: [HelixJob("helix-linux", "finished", stageName: "Test",
+                    submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(passed: ["workitem-1"], failed: ["workitem-2"]),
@@ -4216,8 +4569,11 @@ public async Task RetryPass_ResubmitsWorkItemThatPassedExitCodeButHasPriorFailed
             helix.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-linux", "finished"),
-                    HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"),
+                    HelixJob("helix-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-linux",
+                        stageAttempt: "2", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
@@ -4226,7 +4582,10 @@ public async Task RetryPass_ResubmitsWorkItemThatPassedExitCodeButHasPriorFailed
                 });
             helix.ConfigureResubmission("helix-linux", "helix-linux-resub");
 
-            var runner = CreateRunner(azdo, helix, logger: logger);
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+            var runner = new JobMonitorRunner(options, logger, azdo, helix, NoDelay);
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             helix.Resubmissions.Should().ContainSingle();
@@ -4262,11 +4621,16 @@ public async Task RetryPass_LogDistinguishesExitCodeAndTestFailureReasons()
 
             azdo.WithRecordedFailedTest("helix-linux", "wi-test-only");
 
-            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
-            azdo.AddTimelineResponse(MonitorJob(), PipelineJob("Test Linux", "completed", "succeeded"));
+            azdo.AddTimelineResponse(
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
+            azdo.AddTimelineResponse(
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)]),
+                PipelineJob("Test Linux", "completed", "succeeded", attempt: 1));
 
             helix.AddResponse(
-                jobs: [HelixJob("helix-linux", "finished")],
+                jobs: [HelixJob("helix-linux", "finished", stageName: "Test",
+                    submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1")],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
                     ["helix-linux"] = PassFail(passed: ["wi-test-only"], failed: ["wi-exit-fail"]),
@@ -4275,8 +4639,11 @@ public async Task RetryPass_LogDistinguishesExitCodeAndTestFailureReasons()
             helix.AddResponse(
                 jobs:
                 [
-                    HelixJob("helix-linux", "finished"),
-                    HelixJob("helix-linux-resub", "finished", previousHelixJobName: "helix-linux"),
+                    HelixJob("helix-linux", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", stageAttempt: "1", jobAttempt: "1"),
+                    HelixJob("helix-linux-resub", "finished", stageName: "Test",
+                        submitterJobName: "Test Linux", previousHelixJobName: "helix-linux",
+                        stageAttempt: "2", jobAttempt: "1"),
                 ],
                 passFailByJob: new(StringComparer.OrdinalIgnoreCase)
                 {
@@ -4285,7 +4652,10 @@ public async Task RetryPass_LogDistinguishesExitCodeAndTestFailureReasons()
                 });
             helix.ConfigureResubmission("helix-linux", "helix-linux-resub");
 
-            var runner = CreateRunner(azdo, helix, logger: logger);
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+            var runner = new JobMonitorRunner(options, logger, azdo, helix, NoDelay);
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             exitCode.Should().Be(0);
@@ -4360,6 +4730,8 @@ public async Task FailOnFailedTestsDisabled_TestFailuresIgnoredForOutcomeAndRetr
             BuildReason = "PullRequest",
             SourceBranch = "refs/pull/99999/merge",
             StageName = "Test",
+            StageAttempt = "1",
+            JobAttempt = "1",
             SystemAccessToken = "token",
             TeamProject = "public",
             WorkingDirectory = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "job-monitor-test"),
@@ -4395,9 +4767,11 @@ private static async Task<FakeHelixService> RunAttemptToTimeoutAsync(
         {
             var azdo = new FakeAzureDevOpsService();
             azdo.AddTimelineResponse(
-                StageRecord("Test", "stage-test", "inProgress"),
-                MonitorJob(parentId: "stage-test"),
-                PipelineJob("Test Suite", "completed", "succeeded", parentId: "stage-test"));
+                StageRecord("Test", "stage-test", "inProgress", attempt: int.Parse(stageAttempt)),
+                MonitorJob(attempt: int.Parse(stageAttempt), parentId: "stage-test"),
+                PipelineJob("Test_A", "completed", "succeeded", attempt: 1, parentId: "stage-test"),
+                PipelineJob("Test_B", "completed", "succeeded", attempt: 1, parentId: "stage-test"),
+                PipelineJob("Test_C", "completed", "succeeded", attempt: 1, parentId: "stage-test"));
 
             var helix = new FakeHelixService();
             helix.AddResponse(jobs: entryLeaves, passFailByJob: passFailByJob);
@@ -4412,6 +4786,8 @@ private static async Task<FakeHelixService> RunAttemptToTimeoutAsync(
 
             var options = DefaultOptions();
             options.StageAttempt = stageAttempt;
+            options.JobAttempt = stageAttempt;
+            options.JobAttempt = stageAttempt;
 
             using var cts = new CancellationTokenSource();
             var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix,
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs
index dbdcf9b277f..645dfc7b85a 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs
@@ -10,8 +10,23 @@ internal static class ScenarioHelpers
     {
         public const string DefaultMonitorName = "Helix Job Monitor";
 
-        public static AzureDevOpsTimelineRecord StageRecord(string name, string id, string state, string result = null)
-            => new() { Type = "Stage", ReferenceName = name, Id = id, State = state, Result = result };
+        public static AzureDevOpsTimelineRecord StageRecord(
+            string name,
+            string id,
+            string state,
+            string result = null,
+            int attempt = 1,
+            PreviousAttemptReference[] previousAttempts = null)
+            => new()
+            {
+                Type = "Stage",
+                ReferenceName = name,
+                Id = id,
+                State = state,
+                Result = result,
+                Attempt = attempt,
+                PreviousAttempts = previousAttempts,
+            };
 
         public static AzureDevOpsTimelineRecord PipelineJob(
             string name, string state, string result = null, int attempt = 1,
@@ -55,6 +70,7 @@ public static HelixJobInfo HelixJob(
             string previousHelixJobName = null,
             int? initialWorkItemCount = null,
             string stageAttempt = null,
+            string jobAttempt = null,
             string logicalJobName = null,
             string submitterPhaseName = null)
             => new(
@@ -67,6 +83,7 @@ public static HelixJobInfo HelixJob(
                 previousHelixJobName: previousHelixJobName,
                 initialWorkItemCount: initialWorkItemCount,
                 stageAttempt: stageAttempt,
+                jobAttempt: jobAttempt,
                 logicalJobName: logicalJobName,
                 submitterPhaseName: submitterPhaseName);
 

From e8a760c9c46d8c6412f650852cf9910a6fcb3988 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Fri, 14 Aug 2026 07:38:47 -0700
Subject: [PATCH 12/21] Document Job Monitor retry permutations

Specify stream identities and timing-sensitive retry behavior, make phase-to-timeline matching explicit, and reject ambiguous fallback job identities.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../JobMonitor/Design/SemanticBehavior.md     |  84 +++++++++++
 .../JobMonitor/JobMonitorRunner.cs            |  29 +++-
 .../JobMonitorRunnerTests.cs                  | 138 +++++++++++++++++-
 .../ScenarioHelpers/ScenarioHelpers.cs        |  16 ++
 4 files changed, 252 insertions(+), 15 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
index 57eaf56d3b9..a6629e43554 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
@@ -277,6 +277,7 @@ inputs are:
 | AzDO collection URI + project | Construct the test-results URL used in failure reports. |
 | Stage name | Stage scope (see §2.1). |
 | Stage attempt | Per-attempt scope (see §2.1). Defaults to `SYSTEM_STAGEATTEMPT`; when unknown the monitor tracks jobs from every attempt of the stage. |
+| Job attempt | Attempt of the monitor's own AzDO job. Retry reconciliation runs only after attempt 1. Defaults to `SYSTEM_JOBATTEMPT`. |
 | Polling interval | Delay between poll iterations; a minimum floor applies. |
 | Maximum wait | Reported in the timeout message; the timeout itself is enforced by the caller through cancellation. |
 | Job monitor name | Identifier of the monitor's own AzDO timeline record; used to exclude it from pass/fail. |
@@ -478,6 +479,89 @@ The same key drives a parallel map of "failed work item console info" used
 to build the final failure report. When a later incarnation of a work item
 passes, its entry in that map is cleared.
 
+#### 5.7.1 Stream and attempt examples
+
+The following examples use one logical stream:
+
+```text
+StreamKey K1 =
+(
+  StageName: Build,
+  PhaseName: build_windows_x64_Checked_NativeAOT,
+  QueueId: windows.10.amd64.open.rt,
+  LogicalJobName: runtime-tests
+)
+
+Original Helix job H1:
+  System.StageAttempt = 1
+  System.JobAttempt = 1
+
+Monitor replay R2:
+  System.StageAttempt = 2
+  System.JobAttempt = 1
+  JobMonitor.JobAttempt = 2
+  PreviousHelixJobName = H1
+
+Fresh submitter rerun H2:
+  System.StageAttempt = 2
+  System.JobAttempt = 2
+  PreviousHelixJobName is absent
+```
+
+`System.JobAttempt` on a Helix job always identifies the AzDO job that
+originally submitted that logical work. It is compared with the current
+timeline attempt of that same submitter. It is **not** compared with the
+monitor's own job attempt. `JobMonitor.JobAttempt` records which monitor
+invocation created a replay and is diagnostic metadata only.
+
+`System.PhaseName` maps to the `refName` of the AzDO **Phase** timeline
+record. When phase identity is unavailable, `System.JobName` maps to the
+`refName` of the nested **Job** record. In matrix pipelines that nested job
+name is frequently `__default`, which is why phase identity is preferred.
+
+#### 5.7.2 Retry and rerun permutation matrix
+
+| Scenario and observation time | Monitor | Current submitter timeline | Visible Helix work | Required behavior |
+| --- | --- | --- | --- | --- |
+| Initial execution; H1 fails | S1/M1 | J1 | H1 S1/J1 failed | Report the failure; attempt 1 never creates replay work. |
+| Selective retry of only the monitor | S2/M2 | J1 completed | H1 S1/J1 failed | `J1 == J1`; replay H1 as R2. |
+| Selective retry also selected the submitter, before H2 is visible | S2/M2 | J2 pending/running | Only H1 S1/J1 | `J2 > J1`; suppress H1 using timeline state, without waiting for H2 visibility. |
+| Selective retry after H2 is visible | S2/M2 | J2 | H1 S1/J1 and H2 S2/J2 | H2 supersedes H1; observe H2. |
+| Full-stage rerun immediately after timeline creation | S2/M2 | J2 pending | Only H1 S1/J1 | Suppress H1 because the submitter is part of the rerun. |
+| Full-stage rerun after H2 submission | S2/M2 | J2 running/completed | H1 S1/J1 and H2 S2/J2 | H2 is authoritative; do not create a monitor replay. |
+| Submitter reruns but intentionally submits no new Helix work | S2/M2 | J2 completed | Only H1 S1/J1 | Suppress H1; the newer submitter execution is authoritative. |
+| Submitter did not rerun and H1 passed | S2/M2 | J1 | H1 S1/J1 passed | Upload/count H1; no replay. |
+| Submitter did not rerun and H1 is failed or unfinished | S2/M2 | J1 | H1 S1/J1 failed/waiting | Replay the failed or unfinished items as R2. |
+| R2 fails and the monitor is retried again | S3/M3 | J1 | H1 S1/J1 and R2 S2/J1 failed | R2 is the latest lineage leaf and still matches J1; replay its remaining failures as R3. |
+| R2 remains unfinished when the monitor is retried | S3/M3 | J1 | R2 S2/J1 running/waiting | Treat the previous-stage leaf as abandoned and replay its unfinished items as R3. |
+| R2 passed before the next monitor invocation | S3/M3 | J1 | R2 S2/J1 passed | Latest incarnation passed; no further replay. |
+| Current-stage H2 is running when the monitor starts | S2/M2 | J2 | H2 S2/J2 running | Observe and gate on H2; never replay current-stage work on entry. |
+| H2 fails after the monitor has started | S2/M2 | J2 | H2 changes running to failed | Report failure. Retry is entry-only; do not create replay work mid-invocation. |
+| Monitor is retried after H2 failed | S3/M3 | J2 | H2 S2/J2 failed | `J2 == J2`; replay H2 into S3. |
+| Submitter identity or attempt is missing | S2/M2 | Unknown | H1 S1/J1 failed | Fail safely; do not speculate and risk duplicate work. |
+| Timeline attempt is lower than Helix metadata | S2/M2 | J1 | H1 claims S1/J2 | Treat metadata as inconsistent; do not replay. |
+| Fresh H2 is visible but is not lineage-linked to H1 | S2/M2 | J2 | H1 S1/J1 and H2 S2/J2 | Collapse by `StreamKey`; the higher stage/job incarnation wins. |
+| Monitor replay R2 is linked to H1 | S2/M2 | J1 | H1 and R2 with `Previous=H1` | Collapse explicit lineage to the leaf R2. |
+
+The central retry decision is therefore:
+
+```text
+current submitter attempt > Helix System.JobAttempt
+    => the submitter reran; suppress monitor replay
+
+current submitter attempt == Helix System.JobAttempt
+    => the submitter did not rerun; failed/unfinished work may be replayed
+```
+
+#### 5.7.3 Identity-isolation matrix
+
+| Helix submissions | Example stream keys | Required behavior |
+| --- | --- | --- |
+| One AzDO job submits two logical jobs to the same queue | `(Build, phaseA, queue1, runtime-tests)` and `(Build, phaseA, queue1, nativeaot-smoke)` | Separate streams because logical job names differ. |
+| One AzDO job submits to two queues | `(Build, phaseA, queue1, tests)` and `(Build, phaseA, queue2, tests)` | Separate streams because queues differ. |
+| Two AzDO phases use the same queue and logical name | `(Build, phaseA, queue1, tests)` and `(Build, phaseB, queue1, tests)` | Separate streams because phase identities differ. |
+| Two stages use identical phase, queue, and logical names | `(Build, phaseA, queue1, tests)` and `(Test, phaseA, queue1, tests)` | Separate streams because stage identities differ. |
+
 ### 5.8 Failure reporting
 
 Failed Helix work items must produce clickable console-link warnings in the
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
index c54c07c3f30..590d98718c4 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
@@ -638,7 +638,21 @@ private static bool TryGetCurrentSubmitterAttempt(
             out int attempt,
             out string submitterIdentity)
         {
-            submitterIdentity = job.SubmitterPhaseName ?? job.SubmitterJobName;
+            // SendHelixJob stamps System.PhaseName from the AzDO phase refName. The nested
+            // job record frequently has refName "__default", so only use it when phase
+            // identity was not available on the Helix job.
+            string timelineRecordType;
+            if (!string.IsNullOrEmpty(job.SubmitterPhaseName))
+            {
+                submitterIdentity = job.SubmitterPhaseName;
+                timelineRecordType = "Phase";
+            }
+            else
+            {
+                submitterIdentity = job.SubmitterJobName;
+                timelineRecordType = "Job";
+            }
+
             attempt = 0;
             if (string.IsNullOrEmpty(submitterIdentity))
             {
@@ -646,26 +660,25 @@ private static bool TryGetCurrentSubmitterAttempt(
             }
 
             string identity = submitterIdentity;
-            int[] matchingAttempts =
+            AzureDevOpsTimelineRecord[] matchingRecords =
             [
                 ..timelineRecords
                     .Where(record =>
-                        (string.Equals(record.Type, "Job", StringComparison.OrdinalIgnoreCase)
-                            || string.Equals(record.Type, "Phase", StringComparison.OrdinalIgnoreCase))
+                        string.Equals(record.Type, timelineRecordType, StringComparison.OrdinalIgnoreCase)
                         && string.Equals(
                             record.ReferenceName,
                             identity,
                             StringComparison.OrdinalIgnoreCase))
-                    .Select(record => record.Attempt)
-                    .Distinct()
             ];
 
-            if (matchingAttempts.Length == 0)
+            // A phase refName is normally unique. Job fallback can be "__default" for many
+            // matrix legs; treating any one of those as the submitter could duplicate work.
+            if (matchingRecords.Length != 1)
             {
                 return false;
             }
 
-            attempt = matchingAttempts.Max();
+            attempt = matchingRecords[0].Attempt;
             return true;
         }
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
index 99d6dc9e9c0..1c1916729fa 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
@@ -1665,22 +1665,28 @@ public async Task AttemptScoped_FastRerun_CurrentIncarnationExists_DoesNotResubm
                 StageRecord("Test", "stage-test", "inProgress", attempt: 2,
                     previousAttempts: [PreviousAttempt(1)]),
                 MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
-                PipelineJob("Test_Linux", "inProgress", attempt: 2,
-                    previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"));
+                PipelinePhase("Test_Linux", "Test Linux", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("__default", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)], parentId: "Test_Linux"));
             azdo.AddTimelineResponse(
                 StageRecord("Test", "stage-test", "inProgress", attempt: 2,
                     previousAttempts: [PreviousAttempt(1)]),
                 MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
-                PipelineJob("Test_Linux", "completed", "succeeded", attempt: 2,
-                    previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"));
+                PipelinePhase("Test_Linux", "Test Linux", "completed", "succeeded", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("__default", "completed", "succeeded", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)], parentId: "Test_Linux"));
 
             // Previous incarnation still running; fresh current incarnation of the SAME stream
             // (same submitter + queue), not linked by PreviousHelixJobName.
             HelixJobInfo previousRunning = HelixJob("helix-x-a1", "finished", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "1",
+                submitterJobName: "__default", submitterPhaseName: "Test_Linux",
+                queueId: "q1", stageAttempt: "1",
                 jobAttempt: "1", logicalJobName: "tests");
             HelixJobInfo currentDone = HelixJob("helix-x-a2", "finished", stageName: "Test",
-                submitterJobName: "Test_Linux", queueId: "q1", stageAttempt: "2",
+                submitterJobName: "__default", submitterPhaseName: "Test_Linux",
+                queueId: "q1", stageAttempt: "2",
                 jobAttempt: "2", logicalJobName: "tests");
 
             helix.AddResponse(
@@ -1933,6 +1939,110 @@ public async Task AttemptScoped_MissingSubmitterAttemptMetadata_DoesNotSpeculati
                 message.Contains("could not be matched to compatible System.JobAttempt metadata", StringComparison.Ordinal));
         }
 
+        [Fact]
+        public async Task AttemptScoped_TimelineAttemptOlderThanHelixMetadata_DoesNotSpeculativelyResubmit()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+            var logger = new RecordingLogger();
+
+            azdo.AddTimelineResponse(
+                StageRecord("Test", "stage-test", "inProgress", attempt: 3,
+                    previousAttempts: [PreviousAttempt(1), PreviousAttempt(2)]),
+                MonitorJob(attempt: 3, previousAttempts: [PreviousAttempt(1), PreviousAttempt(2)],
+                    parentId: "stage-test"),
+                PipelineJob("A", "completed", "succeeded", attempt: 1, parentId: "stage-test"));
+            helix.AddResponse(
+                jobs: [HelixJob("ha2", "finished", stageName: "Test",
+                    submitterJobName: "A", queueId: "q", stageAttempt: "2",
+                    jobAttempt: "2", logicalJobName: "tests")],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["ha2"] = PassFail(failed: ["a"]),
+                });
+
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "3";
+            options.JobAttempt = "3";
+
+            int exitCode = await new JobMonitorRunner(
+                options, logger, azdo, helix, NoDelay).RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(1);
+            helix.Resubmissions.Should().BeEmpty();
+            logger.Messages.Should().Contain(message =>
+                message.Contains("could not be matched to compatible System.JobAttempt metadata", StringComparison.Ordinal));
+        }
+
+        [Fact]
+        public async Task AttemptScoped_AmbiguousDefaultJobIdentity_DoesNotSpeculativelyResubmit()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+            var logger = new RecordingLogger();
+
+            azdo.AddTimelineResponse(
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("__default", "completed", "succeeded", attempt: 1,
+                    parentId: "phase-a", id: "job-a"),
+                PipelineJob("__default", "completed", "succeeded", attempt: 1,
+                    parentId: "phase-b", id: "job-b"));
+            helix.AddResponse(
+                jobs: [HelixJob("ha1", "finished", stageName: "Test",
+                    submitterJobName: "__default", queueId: "q", stageAttempt: "1",
+                    jobAttempt: "1", logicalJobName: "tests")],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["ha1"] = PassFail(failed: ["a"]),
+                });
+
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+
+            int exitCode = await new JobMonitorRunner(
+                options, logger, azdo, helix, NoDelay).RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(1);
+            helix.Resubmissions.Should().BeEmpty();
+            logger.Messages.Should().Contain(message =>
+                message.Contains("could not be matched to compatible System.JobAttempt metadata", StringComparison.Ordinal));
+        }
+
+        [Fact]
+        public async Task AttemptScoped_CurrentStageFailure_IsObservedButNotReplayedOnEntry()
+        {
+            var azdo = new FakeAzureDevOpsService();
+            var helix = new FakeHelixService();
+
+            azdo.AddTimelineResponse(
+                StageRecord("Test", "stage-test", "inProgress", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)]),
+                MonitorJob(attempt: 2, previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"),
+                PipelineJob("A", "completed", "succeeded", attempt: 2,
+                    previousAttempts: [PreviousAttempt(1)], parentId: "stage-test"));
+            helix.AddResponse(
+                jobs: [HelixJob("ha2", "finished", stageName: "Test",
+                    submitterJobName: "A", queueId: "q", stageAttempt: "2",
+                    jobAttempt: "2", logicalJobName: "tests")],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["ha2"] = PassFail(failed: ["a"]),
+                });
+
+            JobMonitorOptions options = DefaultOptions();
+            options.StageAttempt = "2";
+            options.JobAttempt = "2";
+
+            int exitCode = await new JobMonitorRunner(
+                options, NullLogger.Instance, azdo, helix, NoDelay).RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(1);
+            helix.Resubmissions.Should().BeEmpty();
+        }
+
         [Fact]
         public void SubmitterChainKey_IncludesStageIdentity()
         {
@@ -2543,7 +2653,21 @@ static void AddRetriedStageTimeline(FakeAzureDevOpsService azdo)
                         Attempt = 2,
                         PreviousAttempts = [PreviousAttempt(1, recordId: "previous-monitor-job")],
                     },
-                    PipelineJob("Linux Build_Debug", "completed", "succeeded", parentId: "stage-test"));
+                    PipelinePhase(
+                        "Linux Build_Debug",
+                        "Linux Build_Debug",
+                        "completed",
+                        "succeeded",
+                        attempt: 1,
+                        parentId: "stage-test",
+                        id: "linux-phase"),
+                    PipelineJob(
+                        "__default",
+                        "completed",
+                        "succeeded",
+                        attempt: 1,
+                        parentId: "linux-phase",
+                        id: "linux-job"));
             }
         }
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs
index 645dfc7b85a..8287f64c0f7 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs
@@ -43,6 +43,22 @@ public static AzureDevOpsTimelineRecord PipelineJob(
                 Id = id ?? name,
             };
 
+        public static AzureDevOpsTimelineRecord PipelinePhase(
+            string referenceName, string name, string state, string result = null, int attempt = 1,
+            PreviousAttemptReference[] previousAttempts = null, string parentId = null, string id = null)
+            => new()
+            {
+                Type = "Phase",
+                ReferenceName = referenceName,
+                Name = name,
+                State = state,
+                Result = result,
+                Attempt = attempt,
+                PreviousAttempts = previousAttempts,
+                ParentId = parentId,
+                Id = id ?? referenceName,
+            };
+
         public static AzureDevOpsTimelineRecord MonitorJob(
             string name = DefaultMonitorName, int attempt = 1,
             PreviousAttemptReference[] previousAttempts = null, string parentId = null)

From 5dfc13ebdf5c97a583df7b4d0d8cda1caf281005 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Fri, 14 Aug 2026 08:14:13 -0700
Subject: [PATCH 13/21] Suppress superseded retry outcomes

Keep previous-attempt results uploadable while excluding streams superseded by newer submitter attempts from current status and pass/fail reconciliation.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../JobMonitor/Design/SemanticBehavior.md     |  9 ++--
 .../JobMonitor/JobMonitorRunner.cs            | 51 ++++++++++++++++---
 .../JobMonitor/MonitorState.cs                | 32 ++++++++++++
 .../JobMonitorRunnerTests.cs                  |  8 ++-
 4 files changed, 89 insertions(+), 11 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
index a6629e43554..ee7aa74df0c 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
@@ -124,7 +124,10 @@ stable across both stage attempts and monitor resubmissions.
      already failed. It belongs to the execution currently being monitored.
    - The current timeline shows the matching submitter at a **higher job
      attempt** than *L* — leave *L*. The submitter reran and superseded it; wait
-     for the newer submitter execution rather than duplicating it.
+     for the newer submitter execution rather than duplicating it. Until the
+     replacement Helix job becomes visible, *L* is also excluded from current
+     status and pass/fail reconciliation so stale failures do not leak through
+     the visibility gap.
    - The current timeline submitter attempt **equals** *L*'s
      `System.JobAttempt` — the submitter did not rerun. Failed or unfinished
      work in *L* may be resubmitted into the current stage attempt.
@@ -525,9 +528,9 @@ name is frequently `__default`, which is why phase identity is preferred.
 | --- | --- | --- | --- | --- |
 | Initial execution; H1 fails | S1/M1 | J1 | H1 S1/J1 failed | Report the failure; attempt 1 never creates replay work. |
 | Selective retry of only the monitor | S2/M2 | J1 completed | H1 S1/J1 failed | `J1 == J1`; replay H1 as R2. |
-| Selective retry also selected the submitter, before H2 is visible | S2/M2 | J2 pending/running | Only H1 S1/J1 | `J2 > J1`; suppress H1 using timeline state, without waiting for H2 visibility. |
+| Selective retry also selected the submitter, before H2 is visible | S2/M2 | J2 pending/running | Only H1 S1/J1 | `J2 > J1`; suppress H1 replay, status, and outcome using timeline state, without waiting for H2 visibility. |
 | Selective retry after H2 is visible | S2/M2 | J2 | H1 S1/J1 and H2 S2/J2 | H2 supersedes H1; observe H2. |
-| Full-stage rerun immediately after timeline creation | S2/M2 | J2 pending | Only H1 S1/J1 | Suppress H1 because the submitter is part of the rerun. |
+| Full-stage rerun immediately after timeline creation | S2/M2 | J2 pending | Only H1 S1/J1 | Suppress H1 replay, status, and outcome because the submitter is part of the rerun. |
 | Full-stage rerun after H2 submission | S2/M2 | J2 running/completed | H1 S1/J1 and H2 S2/J2 | H2 is authoritative; do not create a monitor replay. |
 | Submitter reruns but intentionally submits no new Helix work | S2/M2 | J2 completed | Only H1 S1/J1 | Suppress H1; the newer submitter execution is authoritative. |
 | Submitter did not rerun and H1 passed | S2/M2 | J1 | H1 S1/J1 passed | Upload/count H1; no replay. |
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
index 590d98718c4..a1fa47e4567 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
@@ -232,8 +232,13 @@ private async Task<IReadOnlyList<HelixJobInfo>> ExecuteRetryPassAsync(
                 // The submitter itself reran. Its old Helix work is superseded even when the new
                 // Helix job has not become visible yet; resubmitting here would duplicate a full
                 // stage rerun or a selected failed-job retry.
-                if (hasSubmitterAttempt && currentSubmitterAttempt > helixSubmitterAttempt)
+                if (IsSupersededBySubmitterRerun(
+                    latest,
+                    hasSubmitterAttempt,
+                    currentSubmitterAttempt,
+                    helixSubmitterAttempt))
                 {
+                    _state.MarkSupersededBySubmitterRerun(latest.JobName);
                     continue;
                 }
 
@@ -392,6 +397,18 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
             _state.SetTimelineRecords(timelineRecords);
             _state.ObserveJobs(stageJobs);
 
+            // A rerun submitter supersedes its previous Helix stream before the replacement
+            // Helix job is necessarily visible. Keep those stale incarnations available for
+            // durable upload, but exclude them from current status and pass/fail reconciliation.
+            IReadOnlyList<HelixJobInfo> authoritativeJobs =
+            [
+                .._state.GetLatestIncarnationPerStream(stageJobs)
+                    .Where(job => !_state.IsSupersededBySubmitterRerun(job.JobName))
+            ];
+            var authoritativeJobNames = new HashSet<string>(
+                authoritativeJobs.Select(static job => job.JobName),
+                StringComparer.OrdinalIgnoreCase);
+
             // Helix job summaries can omit Finished for failed jobs even after all work
             // items have terminal exit codes, so fall back to per-work-item status.
             IReadOnlyList<HelixJobInfo> jobsToRefresh =
@@ -432,6 +449,7 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
                     job,
                     workItemsByJob[job.JobName],
                     queueUpload: true,
+                    recordOutcomes: authoritativeJobNames.Contains(job.JobName),
                     discoveryPoll: pollNumber);
             }
 
@@ -440,19 +458,23 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
             // so newer incarnations — including higher-attempt rerun duplicates — supersede older
             // ones). Idempotent — already-reconciled jobs early-return.
             foreach (HelixJobInfo job in MonitorState.OrderHelixJobsOldToNew(
-                MonitorState.GetLatestHelixJobAttempts(stageJobs)
+                MonitorState.GetLatestHelixJobAttempts(authoritativeJobs)
                     .Where(j => completedJobNames.Contains(j.JobName))))
             {
                 ReconcileCompletedJob(
                     job,
                     workItemsByJob[job.JobName],
                     queueUpload: false,
+                    recordOutcomes: true,
                     discoveryPoll: pollNumber);
             }
 
+            var authoritativeCompletedJobNames = new HashSet<string>(
+                authoritativeJobNames.Where(completedJobNames.Contains),
+                StringComparer.OrdinalIgnoreCase);
             Volatile.Write(
                 ref _latestStatus,
-                new PollStatusSnapshot(stageJobs, workItemsByJob, completedJobNames));
+                new PollStatusSnapshot(authoritativeJobs, workItemsByJob, authoritativeCompletedJobNames));
             if (!loopState.HasLoggedInitialStatus)
             {
                 LogLatestStatus();
@@ -500,6 +522,7 @@ private void ReconcileCompletedJob(
             HelixJobInfo helixJob,
             IReadOnlyCollection<WorkItemSummary> workItems,
             bool queueUpload,
+            bool recordOutcomes,
             int discoveryPoll)
         {
             // Already reconciled earlier in this invocation — nothing more to do (idempotent).
@@ -516,20 +539,27 @@ private void ReconcileCompletedJob(
             // completion / console-link logs are suppressed for such jobs.
             bool alreadyUploadedByPriorAttempt = _state.IsHelixJobProcessed(helixJob.JobName);
 
-            if (!alreadyUploadedByPriorAttempt)
+            if (!alreadyUploadedByPriorAttempt && recordOutcomes)
             {
                 _reporter.LogJobProcessingStart(helixJob);
                 _reporter.LogFailedWorkItemConsoleLinks(helixJob, workItems.Where(wi => wi.IsFailed));
             }
 
-            _state.TryRecordWorkItemOutcomes(helixJob, workItems);
+            if (recordOutcomes)
+            {
+                _state.TryRecordWorkItemOutcomes(helixJob, workItems);
+            }
+            else
+            {
+                _state.MarkWorkItemOutcomesIgnored(helixJob.JobName);
+            }
 
             if (queueUpload && !alreadyUploadedByPriorAttempt)
             {
                 _uploads.TryEnqueue(helixJob, workItems, discoveryPoll);
             }
 
-            if (!alreadyUploadedByPriorAttempt)
+            if (!alreadyUploadedByPriorAttempt && recordOutcomes)
             {
                 _reporter.LogJobCompleted(helixJob, workItems);
             }
@@ -682,6 +712,15 @@ private static bool TryGetCurrentSubmitterAttempt(
             return true;
         }
 
+        private static bool IsSupersededBySubmitterRerun(
+            HelixJobInfo job,
+            bool hasSubmitterAttempt,
+            int currentSubmitterAttempt,
+            int helixSubmitterAttempt)
+            => hasSubmitterAttempt
+                && !string.IsNullOrEmpty(job.JobAttempt)
+                && currentSubmitterAttempt > helixSubmitterAttempt;
+
         private static ProductionDependencies CreateProductionDependencies(
             JobMonitorOptions options,
             ILogger logger)
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
index 8ee18ea7991..d66bcf7b2e6 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
@@ -52,6 +52,10 @@ internal sealed class MonitorState
         // jobs that were observed in an earlier poll.
         private readonly HashSet<string> _workItemOutcomeJobs = new(StringComparer.OrdinalIgnoreCase);
 
+        // Previous-attempt jobs whose submitter has a newer timeline attempt. These remain
+        // uploadable history but must not contribute to current status or pass/fail.
+        private readonly HashSet<string> _supersededJobNames = new(StringComparer.OrdinalIgnoreCase);
+
         // Latest known console-link information for every failed work item, keyed the same
         // way as _workItemOutcomes. Cleared per key when a later incarnation passes.
         private readonly Dictionary<(string ChainKey, string WorkItemName), FailedWorkItemConsoleInfo> _failedWorkItemConsoleInfo
@@ -239,6 +243,34 @@ public bool IsWorkItemOutcomesRecorded(string jobName)
             }
         }
 
+        /// <summary>
+        /// Marks a superseded job's outcomes as intentionally ignored. This prevents repeated
+        /// work-item refreshes without inserting stale outcomes into the current result map.
+        /// </summary>
+        public void MarkWorkItemOutcomesIgnored(string jobName)
+        {
+            lock (_sync)
+            {
+                _workItemOutcomeJobs.Add(jobName);
+            }
+        }
+
+        public void MarkSupersededBySubmitterRerun(string jobName)
+        {
+            lock (_sync)
+            {
+                _supersededJobNames.Add(jobName);
+            }
+        }
+
+        public bool IsSupersededBySubmitterRerun(string jobName)
+        {
+            lock (_sync)
+            {
+                return _supersededJobNames.Contains(jobName);
+            }
+        }
+
         /// <summary>
         /// Atomically records all per-work-item outcomes for one completed Helix job:
         /// updates <see cref="WorkItemOutcomeCount"/>, the failure map, and the failed-work-item
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
index 1c1916729fa..1373eb4111c 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
@@ -1660,6 +1660,7 @@ public async Task AttemptScoped_FastRerun_CurrentIncarnationExists_DoesNotResubm
         {
             var azdo = new FakeAzureDevOpsService();
             var helix = new FakeHelixService();
+            var logger = new RecordingLogger();
 
             azdo.AddTimelineResponse(
                 StageRecord("Test", "stage-test", "inProgress", attempt: 2,
@@ -1706,13 +1707,16 @@ public async Task AttemptScoped_FastRerun_CurrentIncarnationExists_DoesNotResubm
             JobMonitorOptions options = DefaultOptions();
             options.StageAttempt = "2";
             options.JobAttempt = "2";
-            var runner = new JobMonitorRunner(options, NullLogger.Instance, azdo, helix, NoDelay);
+            var runner = new JobMonitorRunner(options, logger, azdo, helix, NoDelay);
 
             int exitCode = await runner.RunAsync(CancellationToken.None);
 
             exitCode.Should().Be(0);
             helix.Resubmissions.Should().BeEmpty();
             azdo.UploadedJobNames.Should().Contain("helix-x-a2");
+            logger.Messages.Should().NotContain(message =>
+                message.Contains("helix-x-a1", StringComparison.Ordinal)
+                && message.Contains("failed", StringComparison.OrdinalIgnoreCase));
         }
 
         /// <summary>
@@ -1859,7 +1863,7 @@ public async Task AttemptScoped_RerunSubmitterProducesNoReplacement_DoesNotRepla
             int exitCode = await new JobMonitorRunner(
                 options, NullLogger.Instance, azdo, helix, NoDelay).RunAsync(CancellationToken.None);
 
-            exitCode.Should().Be(1);
+            exitCode.Should().Be(0);
             helix.Resubmissions.Should().BeEmpty();
         }
 

From 60cacf179fe476be6934d9fca922ac7b7b45750f Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Wed, 19 Aug 2026 09:19:23 -0700
Subject: [PATCH 14/21] Retry idempotent test run completion

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../Design/Components/UploadPipeline.md       | 10 +++++---
 .../JobMonitor/Services/AzureDevOpsService.cs |  2 +-
 .../AzureDevOpsServiceTests.cs                | 24 +++++++++++++++----
 3 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
index 98347d76082..98d3ad3d990 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
@@ -46,6 +46,10 @@ failed, the session remains untagged. Otherwise finalization uploads the
 failed-work-item attachment, marks the run completed, applies the Helix-job
 tag, and only then marks the job durably processed.
 
-Create and complete are not replayed after ambiguous failures. Result and
-attachment publication use bounded transient retries because losing an entire
-job's results is worse than the accepted duplicate risk.
+Test-run creation and attachment publication are not replayed after ambiguous
+failures because they are non-idempotent POST operations. The final completion
+PATCH is idempotent and uses bounded transient retries; repeating it applies
+the same completed state and Helix-job tag to the same run without creating
+duplicate results. Result publication also uses bounded transient retries
+because losing an entire job's results is worse than the accepted duplicate
+risk.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
index 730bd826a76..0073975d02b 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
@@ -373,7 +373,7 @@ public async Task CompleteTestRunAsync(
             await SendAsync(new HttpMethod("PATCH"),
                 $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=7.1",
                 body,
-                retryTransientFailures: false,
+                retryTransientFailures: true,
                 cancellationToken: cancellationToken);
         }
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs
index e2aeca84da7..2dc32937781 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs
@@ -87,20 +87,34 @@ public async Task CompleteTestRunAsync_SendsCompletedStateAndHelixJobTag()
         }
 
         [Fact]
-        public async Task CompleteTestRunAsync_DoesNotRetryAmbiguousWrite()
+        public async Task CompleteTestRunAsync_RetriesTransientFailure()
         {
+            int attempt = 0;
             var handler = new RecordingHttpMessageHandler(_ =>
-                new HttpResponseMessage(HttpStatusCode.ServiceUnavailable));
+                Interlocked.Increment(ref attempt) == 1
+                    ? new HttpResponseMessage(HttpStatusCode.ServiceUnavailable)
+                    : new HttpResponseMessage(HttpStatusCode.OK)
+                    {
+                        Content = new StringContent("{}")
+                    });
             using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler));
 
-            Func<Task> action = () => service.CompleteTestRunAsync(
+            await service.CompleteTestRunAsync(
                 123,
                 HelixJobGuid,
                 [],
                 CancellationToken.None);
 
-            await action.Should().ThrowAsync<HttpRequestException>();
-            handler.Requests.Should().ContainSingle();
+            handler.Requests.Should().HaveCount(2);
+                handler.Bodies.Should().HaveCount(2);
+                foreach (string body in handler.Bodies)
+                {
+                    JObject parsed = JObject.Parse(body);
+                    parsed.Value<string>("state").Should().Be("Completed");
+                    var tags = parsed["tags"].Should().BeOfType<JArray>().Subject;
+                    tags.Should().ContainSingle();
+                    tags[0].Value<string>("name").Should().Be(HelixJobTag);
+                }
         }
 
         [Fact]

From 88ac08b80eef15e2816477146689fa20e28b019a Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Wed, 19 Aug 2026 13:05:29 -0700
Subject: [PATCH 15/21] Upload terminal Helix work items incrementally

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../Design/Components/UploadPipeline.md       |  10 +-
 .../JobMonitor/JobMonitorRunner.cs            |  23 ++-
 .../JobMonitor/MonitorState.cs                |  20 ++-
 .../JobMonitor/TestResultUploadPipeline.cs    | 165 ++++++++++++++----
 .../JobMonitorRunnerTests.cs                  |  85 +++++++++
 5 files changed, 260 insertions(+), 43 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
index 98d3ad3d990..e1b8d91a0a9 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/UploadPipeline.md
@@ -21,6 +21,11 @@ work items proceeds directly to finalization.
 
 ## Work-item processing
 
+Terminal work items are admitted as soon as polling observes their immutable
+exit code, even if the containing Helix job is still running. This lets large
+jobs overlap result publication with their remaining execution instead of
+making every completed work item part of the final drain.
+
 Each worker:
 
 1. downloads recognized result files for one work item, retrying only
@@ -41,8 +46,9 @@ throttling guidance compared with 64 workers.
 
 ## Finalization
 
-The last work item queues its session for finalization. If any work item
-failed, the session remains untagged. Otherwise finalization uploads the
+Once the Helix job is complete, the last outstanding work item queues its
+session for finalization. If any work item failed, the session remains
+untagged. Otherwise finalization uploads the
 failed-work-item attachment, marks the run completed, applies the Helix-job
 tag, and only then marks the job durably processed.
 
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
index d6d0f773884..912b83ed6a4 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
@@ -442,13 +442,26 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
             var completedJobNames = new HashSet<string>(
                 completedJobs.Select(j => j.JobName),
                 StringComparer.OrdinalIgnoreCase);
-            // First pass: upload + reconcile for any newly-completed jobs.
+
+            // Upload terminal work items as soon as they are visible, even while their Helix job
+            // is still running. Large jobs can finish hundreds of work items long before their
+            // final job transition; deferring those uploads creates an avoidable final drain.
+            foreach (HelixJobInfo job in stageJobs.Where(job => !_state.IsHelixJobProcessed(job.JobName)))
+            {
+                _uploads.TryEnqueue(
+                    job,
+                    workItemsByJob[job.JobName],
+                    completedJobNames.Contains(job.JobName),
+                    pollNumber);
+            }
+
+            // First pass: reconcile outcomes for any newly-completed jobs.
             foreach (HelixJobInfo job in completedJobs.Where(j => !_state.IsHelixJobProcessed(j.JobName)))
             {
                 ReconcileCompletedJob(
                     job,
                     workItemsByJob[job.JobName],
-                    queueUpload: true,
+                    queueUpload: false,
                     recordOutcomes: authoritativeJobNames.Contains(job.JobName),
                     discoveryPoll: pollNumber);
             }
@@ -555,7 +568,11 @@ private void ReconcileCompletedJob(
 
             if (queueUpload && !alreadyUploadedByPriorAttempt)
             {
-                _uploads.TryEnqueue(helixJob, workItems, discoveryPoll);
+                _uploads.TryEnqueue(
+                    helixJob,
+                    workItems,
+                    isJobComplete: true,
+                    discoveryPoll: discoveryPoll);
             }
 
             if (!alreadyUploadedByPriorAttempt && recordOutcomes)
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
index 5c1ae76bb14..cb1ef6c5cc8 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
@@ -46,6 +46,8 @@ internal sealed class MonitorState
         // (HelixJobChainKey, WorkItemName). See GetHelixJobChainKey for the keying rationale.
         private readonly Dictionary<(string ChainKey, string WorkItemName), bool> _workItemOutcomes
             = new(WorkItemOutcomeKeyComparer.Instance);
+        private readonly HashSet<(string ChainKey, string WorkItemName)> _failedTestWorkItems
+            = new(WorkItemOutcomeKeyComparer.Instance);
 
         // Helix job names whose per-work-item outcomes have already been reconciled into
         // _workItemOutcomes. Prevents the second reconciliation pass from re-processing
@@ -292,8 +294,21 @@ public bool TryRecordWorkItemOutcomes(HelixJobInfo helixJob, IReadOnlyCollection
                     // Within the same Helix job lineage, the latest result overwrites the prior
                     // one for the same work item name. Independent original Helix jobs have
                     // different roots, even when they share an AzDO submitter and queue.
-                    _workItemOutcomes[(chainKey, wi.Name)] = !wi.IsFailed;
-                    TrackFailedWorkItemConsoleInfoLocked(helixJob, chainKey, wi);
+                    var key = (chainKey, wi.Name);
+                    bool passed = !wi.IsFailed && !_failedTestWorkItems.Contains(key);
+                    // This marker only bridges the race where incremental test-result upload
+                    // finishes before the Helix outcome is reconciled. A later incarnation in
+                    // the same logical stream must be able to replace the failure with a pass.
+                    _failedTestWorkItems.Remove(key);
+                    _workItemOutcomes[key] = passed;
+                    if (wi.IsFailed)
+                    {
+                        TrackFailedWorkItemConsoleInfoLocked(helixJob, chainKey, wi);
+                    }
+                    else if (passed)
+                    {
+                        _failedWorkItemConsoleInfo.Remove(key);
+                    }
                 }
 
                 return true;
@@ -338,6 +353,7 @@ public void ObserveTestResults(
 
                     string chainKey = GetHelixJobChainKeyLocked(job);
                     var key = (chainKey, entry.Key.WorkItemName);
+                    _failedTestWorkItems.Add(key);
                     _workItemOutcomes[key] = false;
 
                     // Ensure the final failure report includes test-only failures too.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
index 6797f278185..63446d621e8 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadPipeline.cs
@@ -25,6 +25,7 @@ internal sealed class TestResultUploadPipeline : IAsyncDisposable
     private readonly ConcurrentDictionary<string, JobUploadSession> _sessions =
         new(StringComparer.OrdinalIgnoreCase);
     private readonly ConcurrentDictionary<int, long> _remainingWorkItemsByPoll = [];
+    private readonly ConcurrentDictionary<int, long> _acceptedWorkItemsByPoll = [];
     private readonly ActionQueue<JobUploadRequest> _jobs;
     private readonly ActionQueue<WorkItemUploadRequest> _workItems;
     private readonly ActionQueue<JobUploadSession> _finalizers;
@@ -69,6 +70,7 @@ public TestResultUploadPipeline(
     public bool TryEnqueue(
         HelixJobInfo job,
         IReadOnlyCollection<WorkItemSummary> workItems,
+        bool isJobComplete,
         int discoveryPoll)
     {
         if (Volatile.Read(ref _draining) != 0 || _state.IsHelixJobProcessed(job.JobName))
@@ -76,27 +78,48 @@ public bool TryEnqueue(
             return false;
         }
 
-        var session = new JobUploadSession(job, workItems, discoveryPoll);
-        if (!_sessions.TryAdd(job.JobName, session))
+        bool addedSession = false;
+        JobUploadSession session = _sessions.GetOrAdd(
+            job.JobName,
+            _ =>
+            {
+                addedSession = true;
+                return new JobUploadSession(job);
+            });
+        if (session.IsFinalized)
+        {
+            return false;
+        }
+
+        IReadOnlyList<string> newWorkItems = session.AddWorkItems(
+            workItems.Where(static workItem => workItem.ExitCode.HasValue),
+            isJobComplete);
+        if (newWorkItems.Count == 0 && !session.IsReadyToFinalize)
         {
             return false;
         }
 
+        if (!_jobs.TryEnqueue(new JobUploadRequest(session, newWorkItems, discoveryPoll)))
+        {
+            // Polling is the only producer, the queue is unbounded, and draining is rejected
+            // above, so this indicates a broken pipeline invariant rather than backpressure.
+            throw new InvalidOperationException("The test result upload pipeline stopped accepting jobs before drain began.");
+        }
+
         _remainingWorkItemsByPoll.AddOrUpdate(
             discoveryPoll,
-            workItems.Count,
-            (_, remaining) => remaining + workItems.Count);
-        if (!_jobs.TryEnqueue(new JobUploadRequest(session)))
+            newWorkItems.Count,
+            (_, remaining) => remaining + newWorkItems.Count);
+        _acceptedWorkItemsByPoll.AddOrUpdate(
+            discoveryPoll,
+            newWorkItems.Count,
+            (_, accepted) => accepted + newWorkItems.Count);
+
+        if (addedSession)
         {
-            _sessions.TryRemove(job.JobName, out _);
-            _remainingWorkItemsByPoll.AddOrUpdate(
-                discoveryPoll,
-                0,
-                (_, remaining) => remaining - workItems.Count);
-            return false;
+            _state.TryQueueHelixJobUpload(job.JobName);
         }
 
-        _state.TryQueueHelixJobUpload(job.JobName);
         return true;
     }
 
@@ -116,9 +139,9 @@ public async Task DrainAsync(
             .Where(pair => pair.Key != finalPoll)
             .Sum(static pair => Math.Max(0, pair.Value));
         long remainingWorkItems = finalPollRemainingWorkItems + priorPollBacklog;
-        int finalPollEligibleWorkItems = _sessions.Values
-            .Where(session => session.DiscoveryPoll == finalPoll)
-            .Sum(static session => session.WorkItems.Count);
+        long finalPollEligibleWorkItems = _acceptedWorkItemsByPoll.TryGetValue(finalPoll, out long accepted)
+            ? accepted
+            : 0;
         int remainingFinalizations = _sessions.Values.Count(static session => !session.IsFinalized);
 
         _logger.LogInformation(
@@ -148,7 +171,7 @@ public async Task DrainAsync(
             "Test result pipeline drained in {Elapsed}. {JobCount} job(s), {WorkItemCount} work item(s), "
             + "and {ResultCount} result(s) were processed; {FailedJobCount} job upload(s) remain untagged.",
             DateTimeOffset.UtcNow - startedAt,
-            snapshot.Jobs.Completed,
+            _sessions.Count,
             snapshot.WorkItems.Completed,
             snapshot.UploadedResults,
             snapshot.FailedJobs);
@@ -174,15 +197,16 @@ private async ValueTask ExpandJobAsync(JobUploadRequest request, CancellationTok
         JobUploadSession session = request.Session;
         _state.MarkHelixJobUploadInProgress(session.Job.JobName);
 
-        if (session.WorkItems.Count == 0)
+        foreach (string workItemName in request.WorkItemNames)
         {
-            await _finalizers.EnqueueAsync(session, cancellationToken);
-            return;
+            await _workItems.EnqueueAsync(
+                new WorkItemUploadRequest(session, workItemName, request.DiscoveryPoll),
+                cancellationToken);
         }
 
-        foreach (WorkItemSummary workItem in session.WorkItems)
+        if (session.TryQueueFinalizer())
         {
-            await _workItems.EnqueueAsync(new WorkItemUploadRequest(session, workItem.Name), cancellationToken);
+            await _finalizers.EnqueueAsync(session, cancellationToken);
         }
     }
 
@@ -243,7 +267,7 @@ private async ValueTask ProcessWorkItemAsync(
         finally
         {
             _remainingWorkItemsByPoll.AddOrUpdate(
-                session.DiscoveryPoll,
+                request.DiscoveryPoll,
                 0,
                 static (_, remaining) => remaining - 1);
             if (session.MarkWorkItemFinished())
@@ -290,7 +314,7 @@ await _azdo.CompleteTestRunAsync(
                     "Test result processing completed for job '{JobName}': {WorkItemCount} work item(s), "
                     + "{ResultFileCount} recognized result file(s), and {UploadedCount} test result(s) uploaded.",
                     session.Job.DisplayName,
-                    session.WorkItems.Count,
+                    session.WorkItemCount,
                     session.ResultFileCount,
                     session.UploadedResultCount);
             }
@@ -380,39 +404,55 @@ private void LogUploadFailure(Exception exception, string operation)
             AzdoWarningPrefix,
             operation);
 
-    private sealed record JobUploadRequest(JobUploadSession Session);
+    private sealed record JobUploadRequest(
+        JobUploadSession Session,
+        IReadOnlyList<string> WorkItemNames,
+        int DiscoveryPoll);
 
-    private sealed record WorkItemUploadRequest(JobUploadSession Session, string WorkItemName);
+    private sealed record WorkItemUploadRequest(
+        JobUploadSession Session,
+        string WorkItemName,
+        int DiscoveryPoll);
 
     private sealed class JobUploadSession
     {
         private readonly object _sync = new();
         private readonly HashSet<string> _failedWorkItems = new(StringComparer.OrdinalIgnoreCase);
+        private readonly HashSet<string> _workItems = new(StringComparer.OrdinalIgnoreCase);
         private Task<int> _testRunTask;
-        private int _finishedWorkItems;
+        private int _pendingWorkItems;
+        private int _jobComplete;
+        private int _finalizerQueued;
         private int _finalized;
         private int _failed;
         private long _resultFileCount;
         private long _uploadedResultCount;
 
-        public JobUploadSession(
-            HelixJobInfo job,
-            IReadOnlyCollection<WorkItemSummary> workItems,
-            int discoveryPoll)
+        public JobUploadSession(HelixJobInfo job)
         {
             Job = job;
-            WorkItems = [.. workItems];
-            DiscoveryPoll = discoveryPoll;
         }
 
         public HelixJobInfo Job { get; }
 
-        public IReadOnlyList<WorkItemSummary> WorkItems { get; }
-
-        public int DiscoveryPoll { get; }
+        public int WorkItemCount
+        {
+            get { lock (_sync) { return _workItems.Count; } }
+        }
 
         public bool IsFinalized => Volatile.Read(ref _finalized) != 0;
 
+        public bool IsReadyToFinalize
+        {
+            get
+            {
+                lock (_sync)
+                {
+                    return IsReadyToFinalizeLocked();
+                }
+            }
+        }
+
         public bool HasFailed => Volatile.Read(ref _failed) != 0;
 
         public long UploadedResultCount => Interlocked.Read(ref _uploadedResultCount);
@@ -468,10 +508,63 @@ public void RecordSuccess(
 
         public void RecordFailure() => Interlocked.Exchange(ref _failed, 1);
 
+        public IReadOnlyList<string> AddWorkItems(
+            IEnumerable<WorkItemSummary> workItems,
+            bool isJobComplete)
+        {
+            lock (_sync)
+            {
+                var added = new List<string>();
+                foreach (WorkItemSummary workItem in workItems)
+                {
+                    if (_workItems.Add(workItem.Name))
+                    {
+                        added.Add(workItem.Name);
+                        _pendingWorkItems++;
+                    }
+                }
+
+                if (isJobComplete)
+                {
+                    _jobComplete = 1;
+                }
+
+                return added;
+            }
+        }
+
         public bool MarkWorkItemFinished()
-            => Interlocked.Increment(ref _finishedWorkItems) == WorkItems.Count;
+        {
+            lock (_sync)
+            {
+                _pendingWorkItems--;
+                return TryQueueFinalizerLocked();
+            }
+        }
+
+        public bool TryQueueFinalizer()
+        {
+            lock (_sync)
+            {
+                return TryQueueFinalizerLocked();
+            }
+        }
 
         public void MarkFinalized() => Interlocked.Exchange(ref _finalized, 1);
+
+        private bool TryQueueFinalizerLocked()
+        {
+            if (!IsReadyToFinalizeLocked() || _finalizerQueued != 0)
+            {
+                return false;
+            }
+
+            _finalizerQueued = 1;
+            return true;
+        }
+
+        private bool IsReadyToFinalizeLocked()
+            => _jobComplete != 0 && _pendingWorkItems == 0;
     }
 }
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
index 3fd69d4602c..dd5e7edc332 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
@@ -8,6 +8,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using AwesomeAssertions;
+using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher;
 using Microsoft.DotNet.Helix.Client.Models;
 using Microsoft.DotNet.Helix.JobMonitor;
 using Microsoft.DotNet.Helix.JobMonitor.Models;
@@ -770,6 +771,90 @@ public async Task UploadPipeline_BoundsWorkItemParallelismAndCreatesOneRunPerJob
             azdo.UploadTestResultsCallCount.Should().Be(20);
         }
 
+        [Fact]
+        public async Task UploadPipeline_UploadsTerminalWorkItemsBeforeJobCompletesAndPreservesTestFailures()
+        {
+            var azdo = new FakeAzureDevOpsService()
+                .WithFailedUpload("helix-linux", "workitem-1");
+            var helix = new FakeHelixService();
+
+            azdo.AddTimelineResponse(
+                MonitorJob(),
+                PipelineJob("Test Linux", "inProgress"));
+            azdo.AddTimelineResponse(
+                MonitorJob(),
+                PipelineJob("Test Linux", "completed", "succeeded"));
+
+            helix.AddResponse(
+                jobs: [HelixJob("helix-linux", "running", initialWorkItemCount: 2)],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-linux"] = PassFail(passed: ["workitem-1"]),
+                });
+            helix.AddResponse(
+                jobs: [HelixJob("helix-linux", "finished", initialWorkItemCount: 2)],
+                passFailByJob: new(StringComparer.OrdinalIgnoreCase)
+                {
+                    ["helix-linux"] = PassFail(passed: ["workitem-1", "workitem-2"]),
+                });
+
+            int delayCount = 0;
+            var runner = new JobMonitorRunner(
+                DefaultOptions(),
+                NullLogger.Instance,
+                azdo,
+                helix,
+                async (_, _) =>
+                {
+                    delayCount++;
+                    await azdo.UploadCompleted.Task.WaitAsync(TimeSpan.FromSeconds(5));
+                    azdo.UploadTestResultsCallCount.Should().Be(1);
+                    azdo.CompleteTestRunCallCount.Should().Be(0);
+                });
+
+            int exitCode = await runner.RunAsync(CancellationToken.None);
+
+            exitCode.Should().Be(1);
+            delayCount.Should().Be(1);
+            azdo.CreateTestRunCallCount.Should().Be(1);
+            azdo.UploadTestResultsCallCount.Should().Be(2);
+            azdo.CompleteTestRunCallCount.Should().Be(1);
+        }
+
+        [Fact]
+        public void IncrementalTestFailure_DoesNotStickAcrossPassingIncarnation()
+        {
+            var state = new MonitorState();
+            HelixJobInfo original = HelixJob(
+                "helix-original",
+                "finished",
+                submitterPhaseName: "Test_Linux",
+                queueId: "q",
+                logicalJobName: "tests");
+            HelixJobInfo retry = HelixJob(
+                "helix-retry",
+                "finished",
+                submitterPhaseName: "Test_Linux",
+                queueId: "q",
+                previousHelixJobName: "helix-original",
+                logicalJobName: "tests");
+            state.ObserveJobs([original, retry]);
+
+            state.ObserveTestResult(
+                original.JobName,
+                "workitem",
+                new TestResultUploadSummary(AllPassed: false, UploadedCount: 1));
+            state.TryRecordWorkItemOutcomes(
+                original,
+                [new WorkItemSummary("original/workitem", original.JobName, "workitem", "Finished") { ExitCode = 1 }]);
+            state.TryRecordWorkItemOutcomes(
+                retry,
+                [new WorkItemSummary("retry/workitem", retry.JobName, "workitem", "Finished") { ExitCode = 0 }]);
+
+            state.HasFailedWorkItem.Should().BeFalse();
+            state.SnapshotFailedWorkItemConsoleInfo().Should().BeEmpty();
+        }
+
         [Fact]
         public async Task UploadPipeline_DoesNotDropCompletedJobsWhenBacklogged()
         {

From 69c3bc53c25bafe858c3cb6c504c2d2b2cf32bd5 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Wed, 19 Aug 2026 16:12:11 -0700
Subject: [PATCH 16/21] Avoid post-response rate limit waits

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../Design/Components/TestResults.md          |  4 ++-
 .../JobMonitor/Services/AzureDevOpsService.cs | 13 ++++---
 .../AzureDevOpsServiceTests.cs                | 36 +++++++++++++++++++
 3 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md
index 1edfa3bcfae..fca4e209bf9 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/Components/TestResults.md
@@ -41,4 +41,6 @@ sets the memory floor; handling that case would require spill-to-disk grouping.
   request and its retries.
 - Azure DevOps rate-limit guidance is applied through a service-wide gate so
   concurrent workers slow down together instead of stampeding a throttled
-  endpoint independently.
+  endpoint independently. A response advances the gate for future requests;
+  the request that already received the response returns immediately rather
+  than redundantly adding the advertised delay to its own completion time.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
index 0073975d02b..efb3ca2a3c0 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
@@ -493,16 +493,15 @@ async Task<string> SendOnceAsync()
                         failed: failed,
                         startedAt: requestStartedAt);
                     metricsRecorded = true;
+                    ObserveRateLimit(response, requestUri);
                     if (!response.IsSuccessStatusCode)
                     {
-                        await HonorRateLimitAsync(response, requestUri, cancellationToken);
                         throw new HttpRequestException(
                             $"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}",
                             null,
                             response.StatusCode);
                     }
 
-                    await HonorRateLimitAsync(response, requestUri, cancellationToken);
                     return content;
                 }
                 finally
@@ -567,10 +566,11 @@ async Task<string> SendOnceAsync()
 
         // Honors Azure DevOps rate limiting guidance:
         // https://learn.microsoft.com/azure/devops/integrate/concepts/rate-limits#api-client-experience
-        // If the response carries a Retry-After header (RFC 6585) we wait the specified amount of
-        // time before allowing the next request to be issued. We also log when the service reports
-        // a non-zero X-RateLimit-Delay so callers have visibility into throttling behavior.
-        private async Task HonorRateLimitAsync(HttpResponseMessage response, string requestUri, CancellationToken cancellationToken)
+        // If the response carries a Retry-After header (RFC 6585), advance the shared gate so the
+        // next request waits before being issued. The request that received the response is already
+        // complete and must not wait as well; doing so adds the advertised delay to finalization
+        // even when no further request exists.
+        private void ObserveRateLimit(HttpResponseMessage response, string requestUri)
         {
             TimeSpan? retryAfter = null;
             RetryConditionHeaderValue retryAfterHeader = response.Headers.RetryAfter;
@@ -616,7 +616,6 @@ private async Task HonorRateLimitAsync(HttpResponseMessage response, string requ
                     "Azure DevOps rate limit back-off. Delaying next request by {DelaySeconds:0.###}s (request: {RequestUri}).",
                     delayToApply.TotalSeconds,
                     requestUri);
-                await _rateLimitGate.WaitAsync(cancellationToken);
             }
         }
 
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs
index 2dc32937781..c3816b4292e 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs
@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
@@ -117,6 +118,41 @@ await service.CompleteTestRunAsync(
                 }
         }
 
+        [Fact]
+        public async Task RateLimitDelay_AppliesToNextRequest_NotCompletedRequest()
+        {
+                int requestCount = 0;
+                var handler = new RecordingHttpMessageHandler(_ =>
+                {
+                    var response = new HttpResponseMessage(HttpStatusCode.OK)
+                    {
+                        Content = new StringContent(
+                            Interlocked.Increment(ref requestCount) == 1
+                                ? "{}"
+                                : @"{""records"":[]}")
+                    };
+
+                    if (requestCount == 1)
+                    {
+                        response.Headers.TryAddWithoutValidation("X-RateLimit-Delay", "1");
+                    }
+
+                    return response;
+                });
+                using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler));
+
+                var stopwatch = Stopwatch.StartNew();
+                await service.CompleteTestRunAsync(123, HelixJobGuid, [], CancellationToken.None);
+                TimeSpan completionElapsed = stopwatch.Elapsed;
+
+                await service.GetTimelineRecordsAsync(CancellationToken.None);
+                TimeSpan nextRequestElapsed = stopwatch.Elapsed - completionElapsed;
+
+                completionElapsed.Should().BeLessThan(TimeSpan.FromMilliseconds(500));
+                nextRequestElapsed.Should().BeGreaterThan(TimeSpan.FromMilliseconds(750));
+                handler.Requests.Should().HaveCount(2);
+        }
+
         [Fact]
         public async Task CompleteTestRunAsync_UploadsFailedWorkItemsAttachmentBeforePatch()
         {

From ea3a636095b83f8e886639d0cc7b3e23e6bcd423 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Thu, 20 Aug 2026 07:51:25 -0700
Subject: [PATCH 17/21] Scope Helix job discovery by build

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../JobMonitor/Design/SemanticBehavior.md     |  6 ++++-
 .../JobMonitor/Interfaces/IHelixService.cs    |  8 +++----
 .../JobMonitor/Services/HelixService.cs       | 15 ++++++++++++-
 .../HelixServiceTests.cs                      | 22 +++++++++++++++++--
 4 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
index ee7aa74df0c..1a0a399314a 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
@@ -297,7 +297,11 @@ behaviorally; method names are illustrative.
 - **List jobs for a build** — given the source filter and build ID, return
   all Helix jobs that the submitter recorded for the build. The source
   filter must be derivable from build metadata in lockstep with the
-  submitter (see §5.1). The returned set spans every attempt of the build; the
+  submitter (see §5.1). Discovery sends both the source and
+  `Properties[BuildId]` filters to Helix. Filtering `BuildId` only after
+  retrieving the source's jobs is prohibited: non-PR sources such as
+  `refs/heads/main` are long-lived and can contain thousands of historical
+  jobs. The returned set spans every attempt of the build; the
   runner keeps the whole stage's jobs (all attempts) so the retry pass can
   reconcile previous-attempt work (§2.3), and classifies each job as
   current- or previous-stage-attempt via `System.StageName` /
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs
index 6808ace5d85..bd74546bbe7 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs
@@ -16,10 +16,10 @@ public interface IHelixService
     {
         /// <summary>
         /// Returns Helix jobs associated with the current build/stage.
-        /// Implementations should query Helix using the given <paramref name="source"/>
-        /// filter (which scopes the query to the repo/branch/PR the build is for, mirroring
-        /// what the Helix job submitter records on each submission) and then narrow the
-        /// result to jobs stamped with <paramref name="buildId"/>.
+        /// Implementations should query Helix using both the given <paramref name="source"/>
+        /// and the job property <c>BuildId=<paramref name="buildId"/></c>. The build property
+        /// must be filtered by the service rather than by retrieving every job for a long-lived
+        /// branch source and narrowing the result locally.
         /// </summary>
         Task<IReadOnlyList<HelixJobInfo>> GetJobsForBuildAsync(
             string source,
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
index 6a7924f52cd..9da3aaea30b 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
@@ -59,10 +59,23 @@ public async Task<IReadOnlyList<HelixJobInfo>> GetJobsForBuildAsync(
                 throw new ArgumentException("A non-empty Helix source filter must be provided.", nameof(source));
             }
 
+            if (string.IsNullOrWhiteSpace(buildId))
+            {
+                throw new ArgumentException("A non-empty build ID filter must be provided.", nameof(buildId));
+            }
+
+            IImmutableDictionary<string, string> properties =
+                ImmutableDictionary<string, string>.Empty.Add("BuildId", buildId);
+
             IImmutableList<JobSummary> jobs = await RetryAsync(
-                async () => await _helixApi.Job.ListAsync(source: source, count: 100_000),
+                async () => await _helixApi.Job.ListAsync(
+                    source: source,
+                    properties: properties,
+                    count: 100_000),
                 cancellationToken);
 
+            // Keep the local check as a defensive contract boundary in case Helix returns a
+            // malformed or unexpectedly broad response.
             return
             [
                 ..jobs
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
index 5592d4af965..09bea71a6a7 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
@@ -26,16 +26,18 @@ namespace Microsoft.DotNet.Helix.Sdk.Tests
     public class HelixServiceTests
     {
         [Fact]
-        public async Task GetJobsForBuildAsync_PassesSourceThroughAndFiltersByBuildId()
+        public async Task GetJobsForBuildAsync_PassesSourceAndBuildIdFilters()
         {
             var api = CreateApi();
             string capturedSource = null;
+            IImmutableDictionary<string, string> capturedProperties = null;
             int? capturedCount = null;
             api.Job
                 .Setup(j => j.ListAsync(null, It.IsAny<int?>(), null, null, It.IsAny<IImmutableDictionary<string, string>>(), It.IsAny<string>(), null, It.IsAny<CancellationToken>()))
-                .Callback<string, int?, string, string, IImmutableDictionary<string, string>, string, string, CancellationToken>((_, count, _, _, _, source, _, _) =>
+                .Callback<string, int?, string, string, IImmutableDictionary<string, string>, string, string, CancellationToken>((_, count, _, _, properties, source, _, _) =>
                 {
                     capturedCount = count;
+                    capturedProperties = properties;
                     capturedSource = source;
                 })
                 .ReturnsAsync(ImmutableList.Create(
@@ -64,6 +66,13 @@ public async Task GetJobsForBuildAsync_PassesSourceThroughAndFiltersByBuildId()
                 CancellationToken.None);
 
             Assert.Equal("pr/public/dotnet/runtime/refs/pull/42/merge", capturedSource);
+            Assert.Collection(
+                capturedProperties,
+                property =>
+                {
+                    Assert.Equal("BuildId", property.Key);
+                    Assert.Equal("123", property.Value);
+                });
             Assert.Equal(100_000, capturedCount);
             Assert.Equal(2, jobs.Count);
             Assert.Equal("running-job", jobs[0].JobName);
@@ -85,6 +94,15 @@ await Assert.ThrowsAsync<ArgumentException>(() =>
                 service.GetJobsForBuildAsync(source: "", buildId: "123", CancellationToken.None));
         }
 
+        [Fact]
+        public async Task GetJobsForBuildAsync_RequiresNonEmptyBuildId()
+        {
+            HelixService service = CreateService(CreateApi().Api.Object);
+
+            await Assert.ThrowsAsync<ArgumentException>(() =>
+                service.GetJobsForBuildAsync(source: "ci/public/dotnet/runtime/refs/heads/main", buildId: "", CancellationToken.None));
+        }
+
         [Fact]
         public async Task DownloadTestResultsAsync_FiltersFilesUsesFileSystemAndContinuesAfterDownloadFailure()
         {

From e21a97e769314744e5ea991c79a0df91eafc7ea2 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Thu, 20 Aug 2026 13:33:10 -0700
Subject: [PATCH 18/21] Address job monitor review feedback

Centralize Azure DevOps transport and throttling, clarify shared-state invariants, tighten cancellation and attempt handling, and fail closed if filtered Helix discovery could be truncated.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../JobMonitor/Design/README.md               |  16 -
 .../JobMonitor/Design/SemanticBehavior.md     |  58 ++--
 .../JobMonitor/JobMonitorRunner.cs            |   2 +-
 .../JobMonitor/MonitorState.cs                |  23 +-
 .../JobMonitor/Parallelism/ParallelAsync.cs   |   2 +
 .../Services/AzureDevOpsRateLimitGate.cs      |  11 +-
 .../JobMonitor/Services/AzureDevOpsService.cs | 260 +++++++++-----
 .../JobMonitor/Services/HelixService.cs       |  11 +-
 .../TestResults/AzureDevOpsResultPublisher.cs | 326 ++----------------
 .../IAzureDevOpsResultTransport.cs            |  16 +
 .../TestResults/LocalTestResultsReader.cs     |   2 +
 .../AzureDevOpsResultPublisherTests.cs        | 259 +++++---------
 .../AzureDevOpsServiceTests.cs                | 144 ++++++--
 .../Fakes/FakeAzureDevOpsService.cs           |  19 +
 .../HelixServiceTests.cs                      |  26 +-
 .../JobMonitorRunnerTests.cs                  |  35 ++
 16 files changed, 557 insertions(+), 653 deletions(-)
 create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/TestResults/IAzureDevOpsResultTransport.cs

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md
index eb836961601..18461550cce 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/README.md
@@ -23,19 +23,3 @@ The design is split by concern:
   counts, retries, throughput, rate-limit waits, and pipeline stage timings.
 - [Shutdown](Components/Shutdown.md) describes normal drain, cancellation, and
   crash recovery.
-
-## Performance goals
-
-The monitor is designed for hundreds of Helix jobs, thousands of work items,
-and millions of test results.
-
-1. Polling and status reporting never wait for result downloads or Azure DevOps
-   result uploads.
-2. Every producer/consumer boundary is bounded.
-3. Parallelism is global per stage, not multiplied independently per Helix job.
-4. Result XML is read forward-only; complete XML documents are never retained.
-5. Azure DevOps requests contain up to 1,000 top-level results, independent of
-   nested sub-result count.
-6. Normal drain should contain only the upload tail that could not overlap
-   polling. Runtime validation is used to tune the default parallelism and
-   verify that the tail remains a small fraction of the monitor duration.
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
index 1a0a399314a..2d08a17ca52 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Design/SemanticBehavior.md
@@ -19,7 +19,8 @@ Azure DevOps pipeline stage. Its job is to:
 - Resubmit failed Helix work items once per invocation.
 - Upload Helix work-item test results to Azure DevOps.
 - Return an exit code that reflects whether the monitored pipeline jobs and
-  the latest completed Helix work items all succeeded.
+  the latest completed Helix work items and their processed test results all
+  succeeded.
 
 ## 2. Operating model
 
@@ -585,43 +586,24 @@ Warnings use AzDO `task.logissue type=warning` formatting; the final
 aggregated error uses `task.logissue type=error`. Informational status
 lines are plain logger output.
 
-### 5.9 Test-result upload pipeline
-
-Uploads use a non-dropping lightweight job-expansion channel plus bounded
-work-item and finalization channels. Their in-memory lifecycle distinguishes
-queued, in-progress, durably completed, and failed uploads; only a completed,
-tagged test run is considered durable.
-
-- Completed jobs are queued asynchronously and expanded into a globally
-  bounded work-item pipeline.
-- Test results are downloaded one work item at a time. Transient download
-  failures are safe to retry and use a bounded retry budget.
-- Work-item concurrency is global across all Helix jobs, so total parallelism
-  never multiplies by the number of completed jobs.
-- Test-run creation is single-flight per Helix job even when multiple work-item
-  workers arrive concurrently.
-- Test-run creation and completion/tagging are each attempted once. These
-  lifecycle writes determine the durable upload boundary, so replaying an
-  ambiguous response could create an extra run or incorrectly mark an
-  incompletely uploaded run as processed.
-- Publishing test results and their attachments uses bounded retries for
-  throttling, server errors, and transient transport failures. These Azure
-  DevOps POST APIs do not expose an idempotency key or document deduplication.
-  A timeout or connection failure can occur after the service commits the
-  request but before the response reaches the client, so retrying may create
-  duplicate results or attachments. The design accepts that risk to avoid
-  losing an entire job's test results after a transient failure.
-- Permanent failures and exhausted retries are logged as warnings and make the
-  job session ineligible for completion/tagging without affecting pass/fail.
-- The normal-termination path waits for queued uploads to drain before exiting.
-- The cancellation path does not wait for pending or in-flight uploads. If an
-  upload has not completed and applied its Helix-job tag, it remains untagged;
-  durable-state discovery causes a later invocation to upload it again.
-
-The upload sequence per job is: download work-item results, lazily create one
-test run with the plain `{TestRunName}`, upload work items with bounded global
-parallelism, upload failure metadata, complete the run, and tag it with the
-Helix job name (`helixjob<guid>`).
+### 5.9 Test-result durability
+
+- Terminal work items may publish results before their containing Helix job
+  completes.
+- A Helix job is durably processed only after all admitted results are
+  published and its Azure DevOps test run is completed and tagged with the
+  Helix job name.
+- A monitor that stops before that durable boundary leaves the job eligible
+  for replay by a later invocation.
+- Failed processed test results contribute to the monitor exit code even when
+  the Helix work-item exit code is successful.
+- Result publication may retry transient failures and therefore accepts the
+  documented duplicate-result risk rather than losing an entire result set.
+
+Channel structure, concurrency, retry budgets, and the concrete upload
+sequence are implementation details documented in
+[Upload pipeline](Components/UploadPipeline.md) and
+[Test-result processing](Components/TestResults.md).
 
 ### 5.10 Status logging
 
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
index 912b83ed6a4..95ca6bf8375 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs
@@ -118,7 +118,7 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken),
                     await ExecuteRetryPassAsync(timelineForFirstPoll, cancellationToken);
                 return await RunPollLoopAsync(jobsForFirstPoll, timelineForFirstPoll, cancellationToken);
             }
-            catch (OperationCanceledException)
+            catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
             {
                 _uploads.Cancel();
                 // On cancellation (AzDO job timeout or build cancellation) the agent grants only a
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
index cb1ef6c5cc8..e5e34fc993e 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs
@@ -561,13 +561,30 @@ public void RecordAbandonedWork(HelixJobInfo job, IEnumerable<WorkItemSummary> w
 
         /// <summary>
         /// Parses a stage-attempt string (e.g. the <c>System.StageAttempt</c> property) into a
-        /// comparable integer. Unknown / unparseable values sort as attempt 1 (the first attempt).
+        /// comparable integer. Missing values are legacy metadata and sort as attempt 1. A
+        /// present malformed value violates the Helix submitter contract and is rejected.
         /// </summary>
         public static int ParseStageAttempt(string stageAttempt)
-            => int.TryParse(stageAttempt, out int attempt) ? attempt : 1;
+            => ParseAttempt(stageAttempt, nameof(stageAttempt));
 
         public static int ParseJobAttempt(string jobAttempt)
-            => int.TryParse(jobAttempt, out int attempt) ? attempt : 1;
+            => ParseAttempt(jobAttempt, nameof(jobAttempt));
+
+        private static int ParseAttempt(string value, string parameterName)
+        {
+            if (string.IsNullOrEmpty(value))
+            {
+                return 1;
+            }
+
+            if (int.TryParse(value, out int attempt) && attempt > 0)
+            {
+                return attempt;
+            }
+
+            throw new InvalidOperationException(
+                $"Attempt metadata '{parameterName}' must be a positive integer, but was '{value}'.");
+        }
 
         /// <summary>
         /// From an arbitrary set of Helix jobs return only the leaves of each lineage chain —
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ParallelAsync.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ParallelAsync.cs
index f9b5f074fa7..2721775e4bf 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ParallelAsync.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Parallelism/ParallelAsync.cs
@@ -15,6 +15,8 @@ public static async Task<IReadOnlyDictionary<TKey, TValue>> ToDictionaryAsync<TS
         where TKey : notnull
     {
         TSource[] items = [.. source];
+        // Each worker owns one unique array slot. The dictionary is constructed only after
+        // Parallel.ForEachAsync completes, so neither collection is mutated concurrently.
         var values = new TValue[items.Length];
 
         await Parallel.ForEachAsync(
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
index d73d99ffeee..8648b6026ef 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
@@ -13,7 +13,11 @@ public AzureDevOpsRateLimitGate(JobMonitorMetrics metrics = null)
         _metrics = metrics;
     }
 
-    public void Defer(TimeSpan delay)
+    /// <summary>
+    /// Extends the shared request deadline. A shared deadline coordinates all concurrent upload
+    /// workers; independent delays would allow other workers to continue issuing throttled calls.
+    /// </summary>
+    public void ExtendDeadline(TimeSpan delay)
     {
         if (delay <= TimeSpan.Zero)
         {
@@ -32,15 +36,18 @@ public void Defer(TimeSpan delay)
         }
     }
 
+    /// <summary>Waits until the current shared request deadline has passed.</summary>
     public async Task WaitAsync(CancellationToken cancellationToken)
     {
         long waitStartedAt = 0;
         try
         {
+            // Re-read after every delay because another worker may extend the deadline while this
+            // worker is waiting.
             while (true)
             {
                 long notBeforeTicks = Interlocked.Read(ref _notBeforeUtcTicks);
-                TimeSpan delay = new DateTimeOffset(notBeforeTicks, TimeSpan.Zero) - DateTimeOffset.UtcNow;
+                TimeSpan delay = TimeSpan.FromTicks(notBeforeTicks - DateTimeOffset.UtcNow.UtcTicks);
                 if (delay <= TimeSpan.Zero)
                 {
                     return;
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
index efb3ca2a3c0..344e6ddae87 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs
@@ -5,8 +5,10 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
+using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Net.Sockets;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -21,6 +23,12 @@ namespace Microsoft.DotNet.Helix.JobMonitor
 {
     internal sealed class AzureDevOpsService : IAzureDevOpsService, IDisposable
     {
+        private const int ControlRequestAttemptCount = 5;
+        private const int ResultRequestAttemptCount = 10;
+        private static readonly TimeSpan s_maximumRetryDelay = TimeSpan.FromSeconds(30);
+        private static readonly System.Text.Json.JsonSerializerOptions s_serializerOptions =
+            new(System.Text.Json.JsonSerializerDefaults.Web);
+
         // A test run tag is applied to every completed test run so we can recover the Helix job
         // name on a subsequent monitor attempt. The Helix job name (a GUID) is encoded as
         // "{HelixJobTagPrefix}{guidWithoutDashes}" because Azure DevOps only accepts alphanumeric
@@ -86,6 +94,7 @@ private void InitializeClient()
         {
             string encodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + _options.SystemAccessToken));
             _azdoClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", encodedToken);
+            _azdoClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
             _azdoClient.DefaultRequestHeaders.UserAgent.ParseAdd("dotnet-helix-job-monitor");
             _azdoClient.Timeout = TimeSpan.FromMinutes(5);
         }
@@ -414,25 +423,18 @@ public async Task<TestResultUploadSummary> UploadTestResultsAsync(
             WorkItemTestResults results,
             CancellationToken cancellationToken)
         {
-            var reportingParameters = new AzureDevOpsReportingParameters(
-                new Uri(_options.CollectionUri, UriKind.Absolute),
-                _options.TeamProject,
-                testRunId.ToString(CultureInfo.InvariantCulture),
-                _options.SystemAccessToken,
-                _options.UseFullyQualifiedTestName,
-                _options.TestResultAttachmentMode);
-            var publisher = new AzureDevOpsResultPublisher(
-                reportingParameters,
-                _logger,
-                _azdoClient,
-                _rateLimitGate,
-                _metrics);
-
             if (results.TestResultFiles.Count == 0)
             {
                 return new TestResultUploadSummary(true, 0);
             }
 
+            var publisher = new AzureDevOpsResultPublisher(
+                _options.TestResultAttachmentMode,
+                _options.UseFullyQualifiedTestName,
+                _logger,
+                CreateResultTransport(testRunId),
+                _metrics);
+
             return await publisher.UploadTestResultsWithSummaryAsync(
                 results.TestResultFiles,
                 new
@@ -443,6 +445,9 @@ public async Task<TestResultUploadSummary> UploadTestResultsAsync(
                 cancellationToken);
         }
 
+        internal IAzureDevOpsResultTransport CreateResultTransport(int testRunId)
+            => new AzureDevOpsResultTransport(this, testRunId);
+
         private async Task<JObject> SendAsync(
             HttpMethod method,
             string requestUri,
@@ -454,27 +459,39 @@ private async Task<JObject> SendAsync(
             return string.IsNullOrWhiteSpace(content) ? [] : JObject.Parse(content);
         }
 
-        // Sends a request and returns the raw response body as a string. Used for endpoints
-        // (such as test-run attachment downloads) that do not return JSON, where SendAsync's
-        // JObject parsing would fail or discard the payload.
-        private async Task<string> SendForStringAsync(
+        private Task<string> SendForStringAsync(
             HttpMethod method,
             string requestUri,
             JToken body = null,
             bool retryTransientFailures = true,
             CancellationToken cancellationToken = default)
+            => SendForStringAsync(
+                method,
+                requestUri,
+                body?.ToString(Formatting.None),
+                AzureDevOpsRequestKind.Control,
+                retryTransientFailures,
+                ControlRequestAttemptCount,
+                cancellationToken);
+
+        // All Azure DevOps calls flow through this method so request validation, throttling,
+        // retries, and metrics remain consistent across control, result, and attachment traffic.
+        private async Task<string> SendForStringAsync(
+            HttpMethod method,
+            string requestUri,
+            string serializedBody,
+            AzureDevOpsRequestKind requestKind,
+            bool retryTransientFailures,
+            int attemptCount,
+            CancellationToken cancellationToken)
         {
-            string serializedBody = body?.ToString(Formatting.None);
             int payloadBytes = serializedBody is null ? 0 : Encoding.UTF8.GetByteCount(serializedBody);
-            int attempt = 0;
 
-            async Task<string> SendOnceAsync()
+            async Task<string> SendOnceAsync(int attempt)
             {
                 await _rateLimitGate.WaitAsync(cancellationToken);
-                int currentAttempt = attempt++;
                 long requestStartedAt = JobMonitorMetrics.StartOperation();
                 bool failed = true;
-                bool metricsRecorded = false;
                 using var request = new HttpRequestMessage(method, requestUri);
                 if (serializedBody != null)
                 {
@@ -485,53 +502,53 @@ async Task<string> SendOnceAsync()
                 {
                     using HttpResponseMessage response = await _azdoClient.SendAsync(request, cancellationToken);
                     string content = response.Content != null ? await response.Content.ReadAsStringAsync(cancellationToken) : null;
-                    failed = !response.IsSuccessStatusCode;
-                    _metrics.RecordAzureDevOpsRequest(
-                        AzureDevOpsRequestKind.Control,
-                        payloadBytes,
-                        isRetry: currentAttempt > 0,
-                        failed: failed,
-                        startedAt: requestStartedAt);
-                    metricsRecorded = true;
-                    ObserveRateLimit(response, requestUri);
+                    TimeSpan? rateLimitDelay = GetRateLimitDelay(response);
+                    if (response.StatusCode == HttpStatusCode.TooManyRequests && rateLimitDelay is null)
+                    {
+                        rateLimitDelay = TimeSpan.FromSeconds(30);
+                    }
+
+                    if (rateLimitDelay is { } delay)
+                    {
+                        // The current request has completed. Extend only the shared deadline for
+                        // requests that have not yet started.
+                        _rateLimitGate.ExtendDeadline(delay);
+                    }
+
                     if (!response.IsSuccessStatusCode)
                     {
-                        throw new HttpRequestException(
-                            $"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}",
-                            null,
-                            response.StatusCode);
+                        ThrowForFailure(response, content, requestUri, requestKind, rateLimitDelay);
                     }
 
+                    failed = false;
                     return content;
                 }
                 finally
                 {
-                    if (!metricsRecorded)
-                    {
-                        _metrics.RecordAzureDevOpsRequest(
-                            AzureDevOpsRequestKind.Control,
-                            payloadBytes,
-                            isRetry: currentAttempt > 0,
-                            failed: failed,
-                            startedAt: requestStartedAt);
-                    }
+                    _metrics.RecordAzureDevOpsRequest(
+                        requestKind,
+                        payloadBytes,
+                        isRetry: attempt > 0,
+                        failed: failed,
+                        startedAt: requestStartedAt);
                 }
             }
 
             if (!retryTransientFailures)
             {
-                return await SendOnceAsync();
+                return await SendOnceAsync(0);
             }
 
             string result = null;
             Exception lastException = null;
             var retryHandler = new ExponentialRetry
             {
-                MaxAttempts = 5,
-                DelayBase = 2,
+                MaxAttempts = attemptCount,
+                DelayBase = requestKind == AzureDevOpsRequestKind.Control ? 2 : 3,
                 DelayConstant = 0,
                 MinRandomFactor = 1,
                 MaxRandomFactor = 1,
+                MaximumDelay = s_maximumRetryDelay,
                 RetryDelayCallback = (failedAttempt, delay) =>
                     _logger.LogDebug(
                         "Azure DevOps {Method} request to '{RequestUri}' failed on attempt {Attempt} of {AttemptCount}. "
@@ -539,22 +556,22 @@ async Task<string> SendOnceAsync()
                         method,
                         requestUri,
                         failedAttempt,
-                        5,
+                        attemptCount,
                         delay),
             };
 
             bool succeeded = await retryHandler.RunAsync(
-                async _ =>
+                async attempt =>
                 {
                     try
                     {
-                        result = await SendOnceAsync();
+                        result = await SendOnceAsync(attempt);
                         return RetryResult.Success;
                     }
-                    catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
+                    catch (Exception ex) when (IsTransientException(ex, cancellationToken))
                     {
                         lastException = ex;
-                        return RetryResult.Retry();
+                        return RetryResult.Retry((ex as TransientAzureDevOpsRequestException)?.RetryAfter);
                     }
                 },
                 cancellationToken);
@@ -564,59 +581,128 @@ async Task<string> SendOnceAsync()
                 : throw lastException ?? new InvalidOperationException("Retry failed without completing the Azure DevOps request.");
         }
 
-        // Honors Azure DevOps rate limiting guidance:
-        // https://learn.microsoft.com/azure/devops/integrate/concepts/rate-limits#api-client-experience
-        // If the response carries a Retry-After header (RFC 6585), advance the shared gate so the
-        // next request waits before being issued. The request that received the response is already
-        // complete and must not wait as well; doing so adds the advertised delay to finalization
-        // even when no further request exists.
-        private void ObserveRateLimit(HttpResponseMessage response, string requestUri)
+        internal static bool IsTransientException(Exception exception, CancellationToken cancellationToken)
+            => !cancellationToken.IsCancellationRequested
+                && exception is OperationCanceledException { InnerException: TimeoutException }
+                    or HttpRequestException
+                or TimeoutException
+                or SocketException
+                or IOException;
+
+        internal static TimeSpan? GetRateLimitDelay(HttpResponseMessage response)
         {
-            TimeSpan? retryAfter = null;
+            TimeSpan delay = TimeSpan.Zero;
             RetryConditionHeaderValue retryAfterHeader = response.Headers.RetryAfter;
-            if (retryAfterHeader != null)
+            if (retryAfterHeader?.Delta is { } delta && delta > delay)
             {
-                if (retryAfterHeader.Delta.HasValue)
-                {
-                    retryAfter = retryAfterHeader.Delta.Value;
-                }
-                else if (retryAfterHeader.Date.HasValue)
+                delay = delta;
+            }
+
+            if (retryAfterHeader?.Date is { } date)
+            {
+                TimeSpan datedDelay = TimeSpan.FromTicks(date.UtcTicks - DateTimeOffset.UtcNow.UtcTicks);
+                if (datedDelay > delay)
                 {
-                    TimeSpan delta = retryAfterHeader.Date.Value - DateTimeOffset.UtcNow;
-                    if (delta > TimeSpan.Zero)
-                    {
-                        retryAfter = delta;
-                    }
+                    delay = datedDelay;
                 }
             }
 
-            TimeSpan delayToApply = TimeSpan.Zero;
-
             if (response.Headers.TryGetValues("X-RateLimit-Delay", out IEnumerable<string> delayValues) &&
                 double.TryParse(delayValues.FirstOrDefault(), NumberStyles.Float, CultureInfo.InvariantCulture, out double delaySeconds) &&
                 delaySeconds > 0)
             {
                 TimeSpan rateLimitDelay = TimeSpan.FromSeconds(delaySeconds);
-                delayToApply = rateLimitDelay;
-                _logger.LogDebug(
-                    "Azure DevOps reported X-RateLimit-Delay of {DelaySeconds:0.###}s on request to {RequestUri}.",
-                    delaySeconds,
-                    requestUri);
+                if (rateLimitDelay > delay)
+                {
+                    delay = rateLimitDelay;
+                }
             }
 
-            if (retryAfter.HasValue && retryAfter.Value > TimeSpan.Zero)
+            return delay > TimeSpan.Zero ? delay : null;
+        }
+
+        private static void ThrowForFailure(
+            HttpResponseMessage response,
+            string responseBody,
+            string requestUri,
+            AzureDevOpsRequestKind requestKind,
+            TimeSpan? rateLimitDelay)
+        {
+            responseBody ??= string.Empty;
+            if (responseBody.Contains("It may have been deleted", StringComparison.OrdinalIgnoreCase)
+                || responseBody.Contains("not authorized to access this resource", StringComparison.OrdinalIgnoreCase)
+                || responseBody.Contains("cannot be added or updated for a test run which is in Completed state", StringComparison.OrdinalIgnoreCase)
+                || response.StatusCode == HttpStatusCode.Forbidden
+                || response.StatusCode == HttpStatusCode.Unauthorized)
+            {
+                throw new TerminalError(responseBody);
+            }
+
+            string message = $"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {responseBody}";
+            if ((int)response.StatusCode >= 500 || response.StatusCode == HttpStatusCode.TooManyRequests)
             {
-                delayToApply = delayToApply > retryAfter.Value ? delayToApply : retryAfter.Value;
+                throw new TransientAzureDevOpsRequestException(
+                    message,
+                    response.StatusCode,
+                    rateLimitDelay ?? (response.StatusCode == HttpStatusCode.TooManyRequests ? TimeSpan.FromSeconds(30) : null));
             }
 
-            if (delayToApply > TimeSpan.Zero)
+            if (requestKind == AzureDevOpsRequestKind.Control)
             {
-                _rateLimitGate.Defer(delayToApply);
-                _logger.LogDebug(
-                    "Azure DevOps rate limit back-off. Delaying next request by {DelaySeconds:0.###}s (request: {RequestUri}).",
-                    delayToApply.TotalSeconds,
-                    requestUri);
+                throw new HttpRequestException(message, null, response.StatusCode);
             }
+
+            throw new AzureDevOpsReportingError(message);
+        }
+
+        private sealed class AzureDevOpsResultTransport(
+            AzureDevOpsService service,
+            int testRunId) : IAzureDevOpsResultTransport
+        {
+            public Task<string> PublishResultsAsync(object results, CancellationToken cancellationToken)
+                => service.SendForStringAsync(
+                    HttpMethod.Post,
+                    $"{service._options.CollectionUri}{service._options.TeamProject}/_apis/test/runs/{testRunId}/results?api-version=7.1-preview.6",
+                    System.Text.Json.JsonSerializer.Serialize(results, s_serializerOptions),
+                    AzureDevOpsRequestKind.ResultBatch,
+                    retryTransientFailures: true,
+                    ResultRequestAttemptCount,
+                    cancellationToken);
+
+            public Task UploadAttachmentAsync(
+                long testResultId,
+                long? testSubResultId,
+                string fileName,
+                string stream,
+                CancellationToken cancellationToken)
+            {
+                string query = testSubResultId is long subResultId
+                    ? $"?testSubResultId={subResultId}&api-version=7.1-preview.1"
+                    : "?api-version=7.1-preview.1";
+                var body = new JObject
+                {
+                    ["fileName"] = fileName,
+                    ["stream"] = stream,
+                };
+
+                return service.SendForStringAsync(
+                    HttpMethod.Post,
+                    $"{service._options.CollectionUri}{service._options.TeamProject}/_apis/test/runs/{testRunId}/results/{testResultId}/attachments{query}",
+                    body.ToString(Formatting.None),
+                    AzureDevOpsRequestKind.Attachment,
+                    retryTransientFailures: true,
+                    ResultRequestAttemptCount,
+                    cancellationToken);
+            }
+        }
+
+        private sealed class TransientAzureDevOpsRequestException(
+            string message,
+            HttpStatusCode statusCode,
+            TimeSpan? retryAfter)
+            : HttpRequestException(message, null, statusCode)
+        {
+            public TimeSpan? RetryAfter { get; } = retryAfter;
         }
 
         public void Dispose()
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
index 9da3aaea30b..11569a54b0f 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
@@ -21,6 +21,8 @@ namespace Microsoft.DotNet.Helix.JobMonitor
 {
     internal sealed class HelixService : IHelixService
     {
+        private const int MaximumJobsPerBuildQuery = 1_000;
+
         private readonly ILogger _logger;
         private readonly IHelixApi _helixApi;
         private readonly IBlobClientFactory _blobClientFactory;
@@ -71,9 +73,16 @@ public async Task<IReadOnlyList<HelixJobInfo>> GetJobsForBuildAsync(
                 async () => await _helixApi.Job.ListAsync(
                     source: source,
                     properties: properties,
-                    count: 100_000),
+                    count: MaximumJobsPerBuildQuery),
                 cancellationToken);
 
+            if (jobs.Count >= MaximumJobsPerBuildQuery)
+            {
+                throw new InvalidOperationException(
+                    $"Helix returned {jobs.Count} jobs for build '{buildId}', reaching the query limit of " +
+                    $"{MaximumJobsPerBuildQuery}. Refusing to monitor a potentially truncated result set.");
+            }
+
             // Keep the local check as a defensive contract boundary in case Helix returns a
             // malformed or unexpectedly broad response.
             return
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs
index 96bc0e596d9..6882b48f494 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/AzureDevOpsResultPublisher.cs
@@ -1,22 +1,16 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System.Net;
-using System.Net.Http.Headers;
-using System.Globalization;
 using System.Text;
 using System.Text.Json;
-using System.Net.Sockets;
-using Microsoft.Arcade.Common;
 using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model;
 using Microsoft.DotNet.Helix.JobMonitor;
 using Microsoft.Extensions.Logging;
 
 namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher;
 
-internal sealed class AzureDevOpsResultPublisher : IDisposable
+internal sealed class AzureDevOpsResultPublisher
 {
-    private const int DefaultAttemptCount = 10;
     // Azure DevOps rejects requests containing more than 1,000 top-level TestCaseResult objects.
     // Nested sub-results do not count toward this limit.
     private const int MaximumResultsPerRequest = 1000;
@@ -24,70 +18,25 @@ internal sealed class AzureDevOpsResultPublisher : IDisposable
     // Preserve the legacy bound on the recursive size of one result hierarchy independently
     // from the top-level per-request limit.
     private const int MaximumNodesPerResultHierarchy = 950;
-    private static readonly TimeSpan s_maximumRetryDelay = TimeSpan.FromSeconds(30);
-    private static readonly TimeSpan s_httpClientTimeout = TimeSpan.FromMinutes(5);
-    private static readonly JsonSerializerOptions s_serializerOptions = new(JsonSerializerDefaults.Web)
-    {
-        WriteIndented = false,
-    };
 
-    private readonly AzureDevOpsReportingParameters _azdoParameters;
-    private readonly HttpClient _httpClient;
+    private readonly TestResultAttachmentMode _attachmentMode;
+    private readonly bool _useFullyQualifiedTestName;
     private readonly ILogger _logger;
-    private readonly bool _ownsHttpClient;
-    private readonly AzureDevOpsRateLimitGate _rateLimitGate;
     private readonly JobMonitorMetrics _metrics;
-
-    public AzureDevOpsResultPublisher(
-        AzureDevOpsReportingParameters azdoParameters,
-        ILogger logger)
-    {
-        _azdoParameters = azdoParameters;
-        _httpClient = CreateHttpClient(azdoParameters.AccessToken);
-        _logger = logger;
-        _metrics = new JobMonitorMetrics();
-        _rateLimitGate = new AzureDevOpsRateLimitGate(_metrics);
-        _ownsHttpClient = true;
-    }
+    private readonly IAzureDevOpsResultTransport _transport;
 
     internal AzureDevOpsResultPublisher(
-        AzureDevOpsReportingParameters azdoParameters,
+        TestResultAttachmentMode attachmentMode,
+        bool useFullyQualifiedTestName,
         ILogger logger,
-        HttpClient httpClient,
-        AzureDevOpsRateLimitGate? rateLimitGate = null,
+        IAzureDevOpsResultTransport transport,
         JobMonitorMetrics? metrics = null)
-        : this(
-            azdoParameters,
-            logger,
-            httpClient,
-            rateLimitGate ?? new AzureDevOpsRateLimitGate(),
-            metrics ?? new JobMonitorMetrics(),
-            ownsHttpClient: false)
-    {
-    }
-
-    private AzureDevOpsResultPublisher(
-        AzureDevOpsReportingParameters azdoParameters,
-        ILogger logger,
-        HttpClient httpClient,
-        AzureDevOpsRateLimitGate rateLimitGate,
-        JobMonitorMetrics metrics,
-        bool ownsHttpClient)
     {
-        _azdoParameters = azdoParameters;
-        _httpClient = httpClient;
+        _attachmentMode = attachmentMode;
+        _useFullyQualifiedTestName = useFullyQualifiedTestName;
         _logger = logger;
-        _rateLimitGate = rateLimitGate;
-        _metrics = metrics;
-        _ownsHttpClient = ownsHttpClient;
-    }
-
-    public void Dispose()
-    {
-        if (_ownsHttpClient)
-        {
-            _httpClient.Dispose();
-        }
+        _transport = transport;
+        _metrics = metrics ?? new JobMonitorMetrics();
     }
 
     public async Task<TestResultUploadSummary> UploadTestResultsWithSummaryAsync(List<string> testResultFiles, object resultMetadata, CancellationToken cancellationToken = default)
@@ -96,7 +45,7 @@ public async Task<TestResultUploadSummary> UploadTestResultsWithSummaryAsync(Lis
         bool parseRecorded = false;
         try
         {
-            var testResultReader = new LocalTestResultsReader(_logger, _azdoParameters.TestResultAttachmentMode);
+            var testResultReader = new LocalTestResultsReader(_logger, _attachmentMode);
 
             var parsedResults = new List<IReadOnlyList<TestResult>>(testResultFiles.Count);
             foreach (string file in testResultFiles)
@@ -110,7 +59,7 @@ public async Task<TestResultUploadSummary> UploadTestResultsWithSummaryAsync(Lis
                 return new TestResultUploadSummary(true, 0);
             }
 
-            IReadOnlyList<AggregatedResult> aggregatedResults = new ResultAggregator().Aggregate(parsedResults, _azdoParameters.UseFullyQualifiedTestName);
+            IReadOnlyList<AggregatedResult> aggregatedResults = new ResultAggregator().Aggregate(parsedResults, _useFullyQualifiedTestName);
             _metrics.RecordPipelineOperation(PipelineOperation.ResultParseAndAggregate, parseStartedAt);
             parseRecorded = true;
             if (aggregatedResults.Count == 0)
@@ -183,15 +132,8 @@ private async Task<IReadOnlyList<PublishedTestCase>> PublishResultsAsync(
         var testCaseResults = converted.Select(static c => c.Converted).ToList();
         var originalList = converted.Select(static c => c.Aggregated).ToList();
 
-        using HttpResponseMessage response = await SendWithRetryAsync(
-            HttpMethod.Post,
-            $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results?api-version=7.1-preview.6",
-            testCaseResults,
-            DefaultAttemptCount,
-            AzureDevOpsRequestKind.ResultBatch,
-            cancellationToken);
-
-        IReadOnlyList<PublishedTestCaseResultReference> publishedResults = await ReadPublishedResultsAsync(response, cancellationToken);
+        string response = await _transport.PublishResultsAsync(testCaseResults, cancellationToken);
+        IReadOnlyList<PublishedTestCaseResultReference> publishedResults = ReadPublishedResults(response);
         if (publishedResults.Count == 0)
         {
             _logger.LogWarning("The test run appears to have been closed, aborting test result uploads.");
@@ -259,22 +201,12 @@ private async Task SendAttachmentAsync(
         long? subResultId,
         CancellationToken cancellationToken)
     {
-        var request = new TestRunAttachmentRequest(
+        await _transport.UploadAttachmentAsync(
+            testId,
+            subResultId,
             attachment.Name,
-            Convert.ToBase64String(Encoding.UTF8.GetBytes(attachment.Text)));
-
-        string path = subResultId is long subId
-            ? $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results/{testId}/attachments?testSubResultId={subId}&api-version=7.1-preview.1"
-            : $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results/{testId}/attachments?api-version=7.1-preview.1";
-
-        using HttpResponseMessage response = await SendWithRetryAsync(
-            HttpMethod.Post,
-            path,
-            request,
-            DefaultAttemptCount,
-            AzureDevOpsRequestKind.Attachment,
+            Convert.ToBase64String(Encoding.UTF8.GetBytes(attachment.Text)),
             cancellationToken);
-        _ = response;
     }
 
     private IEnumerable<ConvertedResult> ConvertResults(IEnumerable<AggregatedResult> results, object resultMetadata)
@@ -291,7 +223,7 @@ static string GetResultGroupType(AggregationType aggregationType)
         }
 
         string comment = JsonSerializer.Serialize(resultMetadata) ?? string.Empty;
-        bool useFullyQualifiedName = _azdoParameters.UseFullyQualifiedTestName;
+        bool useFullyQualifiedName = _useFullyQualifiedTestName;
 
         string DisplayNameFor(AggregatedResult result)
             => useFullyQualifiedName
@@ -514,222 +446,8 @@ private static IEnumerable<List<T>> PartitionBySize<T>(
         }
     }
 
-    private static HttpClient CreateHttpClient(string? accessToken)
-    {
-        var client = new HttpClient { Timeout = s_httpClientTimeout };
-        client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
-        if (!string.IsNullOrWhiteSpace(accessToken))
-        {
-            string basicToken = Convert.ToBase64String(Encoding.ASCII.GetBytes($":{accessToken}"));
-            client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", basicToken);
-        }
-
-        return client;
-    }
-
-    private async Task<HttpResponseMessage> SendWithRetryAsync(
-        HttpMethod method,
-        string relativePath,
-        object? payload,
-        int attemptCount,
-        AzureDevOpsRequestKind requestKind,
-        CancellationToken cancellationToken)
-    {
-        byte[]? body = payload is null ? null : JsonSerializer.SerializeToUtf8Bytes(payload, s_serializerOptions);
-
-        HttpResponseMessage? successfulResponse = null;
-        Exception? lastException = null;
-        var retryHandler = new ExponentialRetry
-        {
-            MaxAttempts = attemptCount,
-            DelayBase = 3,
-            DelayConstant = 0,
-            MinRandomFactor = 1,
-            MaxRandomFactor = 1,
-            MaximumDelay = s_maximumRetryDelay,
-            RetryDelayCallback = (failedAttempt, delay) =>
-                _logger.LogDebug(
-                    "Azure DevOps {Method} request to '{RequestPath}' failed on attempt {Attempt} of {AttemptCount}. "
-                    + "Waiting {RetryDelay} before the next attempt.",
-                    method,
-                    relativePath,
-                    failedAttempt,
-                    attemptCount,
-                    delay),
-        };
-
-        bool succeeded = await retryHandler.RunAsync(
-            async attempt =>
-            {
-                await _rateLimitGate.WaitAsync(cancellationToken);
-                long requestStartedAt = JobMonitorMetrics.StartOperation();
-                bool failed = true;
-
-                Uri baseUri = _azdoParameters.CollectionUri.AbsoluteUri.EndsWith('/')
-                    ? _azdoParameters.CollectionUri
-                    : new Uri(_azdoParameters.CollectionUri.AbsoluteUri + '/', UriKind.Absolute);
-
-                using var request = new HttpRequestMessage(method, new Uri(baseUri, relativePath));
-                if (body is not null)
-                {
-                    request.Content = new ByteArrayContent(body);
-                    request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
-                }
-
-                try
-                {
-                    DateTimeOffset logStartedAt = DateTimeOffset.UtcNow;
-                    _logger.LogDebug(
-                        "Sending Azure DevOps {Method} request to '{RequestPath}', attempt {Attempt} of {AttemptCount}.",
-                        method,
-                        relativePath,
-                        attempt + 1,
-                        attemptCount);
-                    HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken);
-                    if (response.IsSuccessStatusCode)
-                    {
-                        if (GetRateLimitDelay(response) is { } rateLimitDelay)
-                        {
-                            _rateLimitGate.Defer(rateLimitDelay);
-                        }
-
-                        _logger.LogDebug(
-                            "Azure DevOps {Method} request to '{RequestPath}' completed with HTTP {StatusCode} "
-                            + "on attempt {Attempt} of {AttemptCount} after {Elapsed}.",
-                            method,
-                            relativePath,
-                            (int)response.StatusCode,
-                            attempt + 1,
-                            attemptCount,
-                            DateTimeOffset.UtcNow - logStartedAt);
-                        failed = false;
-                        successfulResponse = response;
-                        return RetryResult.Success;
-                    }
-
-                    using (response)
-                    {
-                        string responseBody = await response.Content.ReadAsStringAsync(cancellationToken);
-                        bool isTransientStatus = (int)response.StatusCode >= 500
-                            || response.StatusCode == HttpStatusCode.TooManyRequests;
-                        if (isTransientStatus)
-                        {
-                            TimeSpan? retryAfter = GetRateLimitDelay(response);
-                            if (response.StatusCode == HttpStatusCode.TooManyRequests && retryAfter is null)
-                            {
-                                retryAfter = TimeSpan.FromSeconds(30);
-                            }
-                            if (retryAfter is { } delay)
-                            {
-                                _rateLimitGate.Defer(delay);
-                            }
-
-                            _logger.LogDebug(
-                                "Azure DevOps {Method} request to '{RequestPath}' returned HTTP {StatusCode} "
-                                + "on attempt {Attempt} of {AttemptCount} after {Elapsed}. Retrying.",
-                                method,
-                                relativePath,
-                                (int)response.StatusCode,
-                                attempt + 1,
-                                attemptCount,
-                                DateTimeOffset.UtcNow - logStartedAt);
-                            lastException = new AzureDevOpsReportingError(
-                                $"Azure DevOps request failed with status code {(int)response.StatusCode}: {responseBody}");
-                            return RetryResult.Retry(retryAfter);
-                        }
-
-                        if (responseBody.Contains("It may have been deleted", StringComparison.OrdinalIgnoreCase)
-                            || responseBody.Contains("not authorized to access this resource", StringComparison.OrdinalIgnoreCase)
-                            || responseBody.Contains("cannot be added or updated for a test run which is in Completed state", StringComparison.OrdinalIgnoreCase)
-                            || response.StatusCode == HttpStatusCode.Forbidden
-                            || response.StatusCode == HttpStatusCode.Unauthorized)
-                        {
-                            throw new TerminalError(responseBody);
-                        }
-
-                        throw new AzureDevOpsReportingError(
-                            $"Azure DevOps request failed with status code {(int)response.StatusCode}: {responseBody}");
-                    }
-                }
-                catch (Exception ex) when (IsTransientException(ex, cancellationToken))
-                {
-                    lastException = ex;
-                    _logger.LogDebug(
-                        ex,
-                        "Transient Azure DevOps {Method} request failure for '{RequestPath}' on attempt "
-                        + "{Attempt} of {AttemptCount}. Retrying.",
-                        method,
-                        relativePath,
-                        attempt + 1,
-                        attemptCount);
-                    return RetryResult.Retry();
-                }
-                finally
-                {
-                    _metrics.RecordAzureDevOpsRequest(
-                        requestKind,
-                        body?.Length ?? 0,
-                        isRetry: attempt > 0,
-                        failed: failed,
-                        startedAt: requestStartedAt);
-                }
-            },
-            cancellationToken);
-
-        return succeeded && successfulResponse is not null
-            ? successfulResponse
-            : throw lastException ?? new InvalidOperationException("Azure DevOps retry loop exited unexpectedly.");
-    }
-
-    internal static bool IsTransientException(Exception exception, CancellationToken cancellationToken)
-        => !cancellationToken.IsCancellationRequested
-            && exception is OperationCanceledException { InnerException: TimeoutException }
-                or HttpRequestException
-            or TimeoutException
-            or SocketException
-            or IOException;
-
-    internal static TimeSpan? GetRateLimitDelay(HttpResponseMessage response)
-    {
-        TimeSpan delay = TimeSpan.Zero;
-        RetryConditionHeaderValue? retryAfter = response.Headers.RetryAfter;
-        if (retryAfter?.Delta is { } delta && delta > delay)
-        {
-            delay = delta;
-        }
-
-        if (retryAfter?.Date is { } date)
-        {
-            TimeSpan datedDelay = date - DateTimeOffset.UtcNow;
-            if (datedDelay > delay)
-            {
-                delay = datedDelay;
-            }
-        }
-
-        if (response.Headers.TryGetValues("X-RateLimit-Delay", out IEnumerable<string>? delayValues)
-            && double.TryParse(
-                delayValues.FirstOrDefault(),
-                NumberStyles.Float,
-                CultureInfo.InvariantCulture,
-                out double delaySeconds)
-            && delaySeconds > 0)
-        {
-            TimeSpan headerDelay = TimeSpan.FromSeconds(delaySeconds);
-            if (headerDelay > delay)
-            {
-                delay = headerDelay;
-            }
-        }
-
-        return delay > TimeSpan.Zero ? delay : null;
-    }
-
-    private static async Task<IReadOnlyList<PublishedTestCaseResultReference>> ReadPublishedResultsAsync(
-        HttpResponseMessage response,
-        CancellationToken cancellationToken)
+    private static IReadOnlyList<PublishedTestCaseResultReference> ReadPublishedResults(string content)
     {
-        string content = await response.Content.ReadAsStringAsync(cancellationToken);
         if (string.IsNullOrWhiteSpace(content))
         {
             return [];
@@ -780,8 +498,6 @@ private sealed record ConvertedResult(PublishedTestCase Converted, AggregatedRes
 
     private sealed record ChunkPair(PublishedSubResult Converted, AggregatedResult Aggregated);
 
-    private sealed record TestRunAttachmentRequest(string FileName, string Stream);
-
     private sealed record CustomField(string FieldName, object Value);
 
     private sealed record PublishedTestCase
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/IAzureDevOpsResultTransport.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/IAzureDevOpsResultTransport.cs
new file mode 100644
index 00000000000..a74c333740b
--- /dev/null
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/IAzureDevOpsResultTransport.cs
@@ -0,0 +1,16 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher;
+
+internal interface IAzureDevOpsResultTransport
+{
+    Task<string> PublishResultsAsync(object results, CancellationToken cancellationToken);
+
+    Task UploadAttachmentAsync(
+        long testResultId,
+        long? testSubResultId,
+        string fileName,
+        string stream,
+        CancellationToken cancellationToken);
+}
diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs
index 1d32642e9e8..de208da4cc6 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResults/LocalTestResultsReader.cs
@@ -144,6 +144,7 @@ private static async IAsyncEnumerable<TestResult> ReadElementsAsync(
             }
 
             using XmlReader subtree = reader.ReadSubtree();
+            await subtree.ReadAsync();
             yield return convert(XElement.Load(subtree, LoadOptions.PreserveWhitespace));
         }
     }
@@ -163,6 +164,7 @@ private static async Task<Dictionary<string, TestDefinition>> ReadTrxDefinitions
             }
 
             using XmlReader subtree = reader.ReadSubtree();
+            await subtree.ReadAsync();
             XElement unitTest = XElement.Load(subtree);
             string? id = GetAttribute(unitTest, "id");
             XElement? method = unitTest.Descendants().FirstOrDefault(static x => x.Name.LocalName == "TestMethod");
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
index de483430b69..649f2f480e2 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs
@@ -4,10 +4,6 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Net;
-using System.Net.Http;
-using System.Net.Http.Headers;
-using System.Reflection;
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
@@ -39,23 +35,6 @@ public void JobMonitorUploadParallelismDefaultsToFortyEight()
             Assert.Equal(48, new JobMonitorOptions().TestResultUploadParallelism);
         }
 
-        [Fact]
-        public void Constructor_ConfiguresHttpClientTimeoutForLongUploads()
-        {
-            using var publisher = new AzureDevOpsResultPublisher(
-                new AzureDevOpsReportingParameters(
-                    new Uri("https://dev.azure.com/dnceng-public/"),
-                    "public",
-                    "123",
-                    "token"),
-                NullLogger.Instance);
-
-            FieldInfo field = typeof(AzureDevOpsResultPublisher).GetField("_httpClient", BindingFlags.Instance | BindingFlags.NonPublic);
-            var client = Assert.IsType<HttpClient>(field.GetValue(publisher));
-
-            Assert.Equal(TimeSpan.FromMinutes(5), client.Timeout);
-        }
-
         [Theory]
         [InlineData("Passed", true)]
         [InlineData("NotExecuted", true)]
@@ -65,20 +44,17 @@ public void Constructor_ConfiguresHttpClientTimeoutForLongUploads()
         public void ComputeAllPassed_SingleResult_OnlyFailedAndNoneCountAsFailure(string result, bool expectedAllPassed)
         {
             var results = new[] { new AggregatedResult(AggregationType.Single, "Test1", 1, result) };
-
             Assert.Equal(expectedAllPassed, AzureDevOpsResultPublisher.ComputeAllPassed(results));
         }
 
         [Fact]
         public void ComputeAllPassed_InconclusiveDataDrivenRollup_DoesNotFailTheWorkItem()
         {
-            // Mirrors the rollup the aggregator produces for a theory with some passing and some
-            // skipped data rows: no data row failed, but the mix isn't a clean pass or skip either.
-            var results = new[]
-            {
-                new AggregatedResult(AggregationType.Single, "Test1", 1, "Passed"),
-                new AggregatedResult(AggregationType.DataDriven, "Test2", 1, "Inconclusive"),
-            };
+            AggregatedResult[] results =
+            [
+                new(AggregationType.Single, "Test1", 1, "Passed"),
+                new(AggregationType.DataDriven, "Test2", 1, "Inconclusive"),
+            ];
 
             Assert.True(AzureDevOpsResultPublisher.ComputeAllPassed(results));
         }
@@ -86,57 +62,20 @@ public void ComputeAllPassed_InconclusiveDataDrivenRollup_DoesNotFailTheWorkItem
         [Fact]
         public void ComputeAllPassed_AnyFailedResult_FailsTheWorkItem()
         {
-            var results = new[]
-            {
-                new AggregatedResult(AggregationType.Single, "Test1", 1, "Passed"),
-                new AggregatedResult(AggregationType.DataDriven, "Test2", 1, "Failed"),
-            };
+            AggregatedResult[] results =
+            [
+                new(AggregationType.Single, "Test1", 1, "Passed"),
+                new(AggregationType.DataDriven, "Test2", 1, "Failed"),
+            ];
 
             Assert.False(AzureDevOpsResultPublisher.ComputeAllPassed(results));
         }
 
-        [Fact]
-        public void HttpClientTimeoutIsTransient()
-        {
-            Assert.True(AzureDevOpsResultPublisher.IsTransientException(
-                new OperationCanceledException("The request timed out.", new TimeoutException()),
-                CancellationToken.None));
-        }
-
-        [Fact]
-        public void CallerCancellationIsNotTransient()
-        {
-            using var cancellation = new CancellationTokenSource();
-            cancellation.Cancel();
-
-            Assert.False(AzureDevOpsResultPublisher.IsTransientException(
-                new OperationCanceledException("The request timed out.", new TimeoutException()),
-                cancellation.Token));
-        }
-
-        [Fact]
-        public void CancellationWithoutTimeoutIsNotTransient()
-        {
-            Assert.False(AzureDevOpsResultPublisher.IsTransientException(
-                new OperationCanceledException(),
-                CancellationToken.None));
-        }
-
-        [Fact]
-        public void GetRateLimitDelay_UsesLargestAzureDevOpsDelayHeader()
-        {
-            using var response = new HttpResponseMessage(HttpStatusCode.OK);
-            response.Headers.RetryAfter = new RetryConditionHeaderValue(TimeSpan.FromSeconds(2));
-            response.Headers.Add("X-RateLimit-Delay", "3.5");
-
-            Assert.Equal(TimeSpan.FromSeconds(3.5), AzureDevOpsResultPublisher.GetRateLimitDelay(response));
-        }
-
         [Fact]
         public async Task UploadTestResultsWithCountAsync_BatchesByTopLevelResultCount()
         {
-            var handler = new RecordingResultHandler();
-            using var publisher = CreatePublisher(handler);
+            var transport = new RecordingResultTransport();
+            var publisher = CreatePublisher(transport);
             AggregatedResult[] results =
             [
                 CreateDataDrivenResult("First", 600),
@@ -146,15 +85,14 @@ public async Task UploadTestResultsWithCountAsync_BatchesByTopLevelResultCount()
             long uploadedCount = await publisher.UploadTestResultsWithCountAsync(results, new { });
 
             Assert.Equal(2, uploadedCount);
-            Assert.Equal(new[] { 2 }, handler.RequestResultCounts);
+            Assert.Equal(new[] { 2 }, transport.RequestResultCounts);
         }
 
         [Fact]
         public async Task UploadTestResultsWithCountAsync_SplitsMoreThanOneThousandTopLevelResults()
         {
-            var handler = new RecordingResultHandler();
-            var metrics = new JobMonitorMetrics();
-            using var publisher = CreatePublisher(handler, metrics);
+            var transport = new RecordingResultTransport();
+            var publisher = CreatePublisher(transport);
             AggregatedResult[] results =
             [
                 .. Enumerable.Range(0, 1001)
@@ -164,60 +102,46 @@ .. Enumerable.Range(0, 1001)
             long uploadedCount = await publisher.UploadTestResultsWithCountAsync(results, new { });
 
             Assert.Equal(1001, uploadedCount);
-            Assert.Equal(new[] { 1000, 1 }, handler.RequestResultCounts);
-            JobMonitorMetricsSnapshot snapshot = metrics.Snapshot();
-            Assert.Equal(2, snapshot.AzureDevOpsRequests);
-            Assert.Equal(2, snapshot.AzureDevOpsResultRequests);
-            Assert.Equal(0, snapshot.AzureDevOpsControlRequests);
-            Assert.Equal(0, snapshot.AzureDevOpsAttachmentRequests);
-            Assert.Equal(0, snapshot.AzureDevOpsRetries);
-            Assert.Equal(0, snapshot.AzureDevOpsFailedAttempts);
-            Assert.True(snapshot.AzureDevOpsPayloadBytes > 0);
-            Assert.True(snapshot.MaximumAzureDevOpsRequestTime > TimeSpan.Zero);
+            Assert.Equal(new[] { 1000, 1 }, transport.RequestResultCounts);
         }
 
         [Fact]
         public async Task UploadTestResultsWithCountAsync_SplitHierarchiesIncludeRootInNodeLimit()
         {
-            var handler = new RecordingResultHandler();
-            using var publisher = CreatePublisher(handler);
-            AggregatedResult[] results = [CreateDataDrivenResult("Theory", 950)];
+            var transport = new RecordingResultTransport();
+            var publisher = CreatePublisher(transport);
 
-            long uploadedCount = await publisher.UploadTestResultsWithCountAsync(results, new { });
+            long uploadedCount = await publisher.UploadTestResultsWithCountAsync(
+                [CreateDataDrivenResult("Theory", 950)],
+                new { });
 
             Assert.Equal(2, uploadedCount);
-            Assert.Equal(new[] { 2 }, handler.RequestResultCounts);
-            Assert.Equal(new[] { 950, 2 }, handler.RequestHierarchyNodeCounts.Single());
+            Assert.Equal(new[] { 2 }, transport.RequestResultCounts);
+            Assert.Equal(new[] { 950, 2 }, transport.RequestHierarchyNodeCounts.Single());
         }
 
         [Fact]
         public async Task UploadTestResultsWithCountAsync_RecursivelySplitsOversizedNestedHierarchies()
         {
-            var handler = new RecordingResultHandler();
-            using var publisher = CreatePublisher(handler);
+            var transport = new RecordingResultTransport();
+            var publisher = CreatePublisher(transport);
             var nested = CreateDataDrivenResult("Nested", 950);
             AggregatedResult[] results =
             [
-                new AggregatedResult(
-                    AggregationType.DataDriven,
-                    "Outer",
-                    1,
-                    "Passed",
-                    [nested]),
+                new(AggregationType.DataDriven, "Outer", 1, "Passed", [nested]),
             ];
 
             long uploadedCount = await publisher.UploadTestResultsWithCountAsync(results, new { });
 
             Assert.Equal(2, uploadedCount);
-            Assert.Equal(new[] { 2 }, handler.RequestResultCounts);
-            Assert.Equal(new[] { 950, 4 }, handler.RequestHierarchyNodeCounts.Single());
+            Assert.Equal(new[] { 950, 4 }, transport.RequestHierarchyNodeCounts.Single());
         }
 
         [Fact]
         public async Task UploadTestResultsWithCountAsync_DoesNotMaterializeAllConvertedResults()
         {
-            var handler = new BlockingResultHandler();
-            using var publisher = CreatePublisher(handler);
+            var transport = new BlockingResultTransport();
+            var publisher = CreatePublisher(transport);
             int enumerated = 0;
 
             IEnumerable<AggregatedResult> Results()
@@ -230,49 +154,42 @@ IEnumerable<AggregatedResult> Results()
             }
 
             Task<long> upload = publisher.UploadTestResultsWithCountAsync(Results(), new { });
-            await handler.FirstRequestStarted.Task.WaitAsync(TimeSpan.FromSeconds(5));
+            await transport.FirstRequestStarted.Task.WaitAsync(TimeSpan.FromSeconds(5));
 
             Assert.InRange(enumerated, 0, 1001);
-            handler.ReleaseFirstRequest.SetResult();
+            transport.ReleaseFirstRequest.SetResult();
 
             Assert.Equal(2_000, await upload);
-            Assert.Equal(2, handler.RequestResultCounts.Count);
+            Assert.Equal(2, transport.RequestResultCounts.Count);
         }
 
         [Fact]
-        public async Task UploadTestResultsWithCountAsync_RecordsThrottledRetryWait()
+        public async Task UploadTestResultsWithCountAsync_UsesSemanticAttachmentTransport()
         {
-            var handler = new ThrottlingResultHandler();
-            var metrics = new JobMonitorMetrics();
-            using var publisher = CreatePublisher(handler, metrics);
-            AggregatedResult[] results =
-            [
-                new(AggregationType.Single, "Test", 1, "Passed")
-            ];
-
-            Assert.Equal(1, await publisher.UploadTestResultsWithCountAsync(results, new { }));
-
-            JobMonitorMetricsSnapshot snapshot = metrics.Snapshot();
-            Assert.Equal(2, snapshot.AzureDevOpsResultRequests);
-            Assert.Equal(1, snapshot.AzureDevOpsRetries);
-            Assert.Equal(1, snapshot.AzureDevOpsFailedAttempts);
-            Assert.Equal(1, snapshot.RateLimitDeferrals);
-            Assert.True(snapshot.RateLimitDeferredTime >= TimeSpan.FromMilliseconds(50));
-            Assert.True(snapshot.MaximumRateLimitDeferral >= TimeSpan.FromMilliseconds(50));
+            var transport = new RecordingResultTransport();
+            var publisher = CreatePublisher(transport);
+            var result = new AggregatedResult(
+                AggregationType.Single,
+                "Test",
+                1,
+                "Failed",
+                attachments: [new TestResultAttachment("failure.txt", "details")]);
+
+            Assert.Equal(1, await publisher.UploadTestResultsWithCountAsync([result], new { }));
+
+            ResultAttachment attachment = Assert.Single(transport.Attachments);
+            Assert.Equal(1, attachment.TestResultId);
+            Assert.Null(attachment.TestSubResultId);
+            Assert.Equal("failure.txt", attachment.FileName);
+            Assert.Equal("details", System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(attachment.Stream)));
         }
 
-        private static AzureDevOpsResultPublisher CreatePublisher(
-            HttpMessageHandler handler,
-            JobMonitorMetrics metrics = null)
+        private static AzureDevOpsResultPublisher CreatePublisher(IAzureDevOpsResultTransport transport)
             => new(
-                new AzureDevOpsReportingParameters(
-                    new Uri("https://dev.azure.com/dnceng-public/"),
-                    "public",
-                    "123"),
+                TestResultAttachmentMode.Failed,
+                useFullyQualifiedTestName: false,
                 NullLogger.Instance,
-                new HttpClient(handler),
-                new AzureDevOpsRateLimitGate(metrics),
-                metrics);
+                transport);
 
         private static AggregatedResult CreateDataDrivenResult(string name, int subResultCount)
             => new(
@@ -285,45 +202,52 @@ .. Enumerable.Range(0, subResultCount)
                         .Select(i => new AggregatedResult(AggregationType.Single, $"{name}_{i}", 1, "Passed"))
                 ]);
 
-        private class RecordingResultHandler : HttpMessageHandler
+        private class RecordingResultTransport : IAzureDevOpsResultTransport
         {
             public List<int> RequestResultCounts { get; } = [];
             public List<int[]> RequestHierarchyNodeCounts { get; } = [];
+            public List<ResultAttachment> Attachments { get; } = [];
 
-            protected override async Task<HttpResponseMessage> SendAsync(
-                HttpRequestMessage request,
-                CancellationToken cancellationToken)
+            public virtual Task<string> PublishResultsAsync(object results, CancellationToken cancellationToken)
             {
-                using JsonDocument requestBody = JsonDocument.Parse(
-                    await request.Content.ReadAsStringAsync(cancellationToken));
+                using JsonDocument requestBody = JsonDocument.Parse(JsonSerializer.Serialize(results));
                 int resultCount = requestBody.RootElement.GetArrayLength();
                 RequestResultCounts.Add(resultCount);
                 RequestHierarchyNodeCounts.Add(
                     [.. requestBody.RootElement.EnumerateArray().Select(CountHierarchyNodes)]);
 
-                string responseBody = JsonSerializer.Serialize(new
+                return Task.FromResult(JsonSerializer.Serialize(new
                 {
                     value = Enumerable.Range(1, resultCount).Select(id => new { id })
-                });
-                return new HttpResponseMessage(HttpStatusCode.OK)
-                {
-                    Content = new StringContent(responseBody)
-                };
+                }));
+            }
+
+            public Task UploadAttachmentAsync(
+                long testResultId,
+                long? testSubResultId,
+                string fileName,
+                string stream,
+                CancellationToken cancellationToken)
+            {
+                Attachments.Add(new(testResultId, testSubResultId, fileName, stream));
+                return Task.CompletedTask;
             }
 
             private static int CountHierarchyNodes(JsonElement result)
             {
-                if (!result.TryGetProperty("subResults", out JsonElement subResults) ||
-                    subResults.ValueKind != JsonValueKind.Array)
+                if (!result.TryGetProperty("SubResults", out JsonElement subResults) &&
+                    !result.TryGetProperty("subResults", out subResults))
                 {
                     return 1;
                 }
 
-                return 1 + subResults.EnumerateArray().Sum(CountHierarchyNodes);
+                return subResults.ValueKind == JsonValueKind.Array
+                    ? 1 + subResults.EnumerateArray().Sum(CountHierarchyNodes)
+                    : 1;
             }
         }
 
-        private sealed class BlockingResultHandler : RecordingResultHandler
+        private sealed class BlockingResultTransport : RecordingResultTransport
         {
             public TaskCompletionSource FirstRequestStarted { get; } =
                 new(TaskCreationOptions.RunContinuationsAsynchronously);
@@ -331,9 +255,7 @@ private sealed class BlockingResultHandler : RecordingResultHandler
                 new(TaskCreationOptions.RunContinuationsAsynchronously);
             private int _requestCount;
 
-            protected override async Task<HttpResponseMessage> SendAsync(
-                HttpRequestMessage request,
-                CancellationToken cancellationToken)
+            public override async Task<string> PublishResultsAsync(object results, CancellationToken cancellationToken)
             {
                 if (Interlocked.Increment(ref _requestCount) == 1)
                 {
@@ -341,29 +263,14 @@ protected override async Task<HttpResponseMessage> SendAsync(
                     await ReleaseFirstRequest.Task.WaitAsync(cancellationToken);
                 }
 
-                return await base.SendAsync(request, cancellationToken);
-            }
-        }
-
-        private sealed class ThrottlingResultHandler : RecordingResultHandler
-        {
-            private int _requestCount;
-
-            protected override Task<HttpResponseMessage> SendAsync(
-                HttpRequestMessage request,
-                CancellationToken cancellationToken)
-            {
-                if (Interlocked.Increment(ref _requestCount) == 1)
-                {
-                    var response = new HttpResponseMessage(HttpStatusCode.TooManyRequests);
-                    response.Headers.RetryAfter = new RetryConditionHeaderValue(
-                        TimeSpan.FromMilliseconds(50));
-                    return Task.FromResult(response);
-                }
-
-                return base.SendAsync(request, cancellationToken);
+                return await base.PublishResultsAsync(results, cancellationToken);
             }
         }
 
+        private sealed record ResultAttachment(
+            long TestResultId,
+            long? TestSubResultId,
+            string FileName,
+            string Stream);
     }
 }
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs
index c3816b4292e..571c94e71a1 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs
@@ -10,6 +10,8 @@
 using System.Threading;
 using System.Threading.Tasks;
 using AwesomeAssertions;
+using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher;
+using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model;
 using Microsoft.DotNet.Helix.JobMonitor;
 using Microsoft.DotNet.Helix.Sdk.Tests.Fakes;
 using Microsoft.Extensions.Logging.Abstractions;
@@ -121,36 +123,114 @@ await service.CompleteTestRunAsync(
         [Fact]
         public async Task RateLimitDelay_AppliesToNextRequest_NotCompletedRequest()
         {
-                int requestCount = 0;
-                var handler = new RecordingHttpMessageHandler(_ =>
+            int requestCount = 0;
+            var handler = new RecordingHttpMessageHandler(_ =>
+            {
+                var response = new HttpResponseMessage(HttpStatusCode.OK)
                 {
-                    var response = new HttpResponseMessage(HttpStatusCode.OK)
-                    {
-                        Content = new StringContent(
-                            Interlocked.Increment(ref requestCount) == 1
-                                ? "{}"
-                                : @"{""records"":[]}")
-                    };
+                    Content = new StringContent(
+                        Interlocked.Increment(ref requestCount) == 1
+                            ? "{}"
+                            : @"{""records"":[]}")
+                };
 
-                    if (requestCount == 1)
-                    {
-                        response.Headers.TryAddWithoutValidation("X-RateLimit-Delay", "1");
-                    }
+                if (requestCount == 1)
+                {
+                    response.Headers.TryAddWithoutValidation("X-RateLimit-Delay", "1");
+                }
+
+                return response;
+            });
+            using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler));
+
+            var stopwatch = Stopwatch.StartNew();
+            await service.CompleteTestRunAsync(123, HelixJobGuid, [], CancellationToken.None);
+            TimeSpan completionElapsed = stopwatch.Elapsed;
 
-                    return response;
+            await service.GetTimelineRecordsAsync(CancellationToken.None);
+            TimeSpan nextRequestElapsed = stopwatch.Elapsed - completionElapsed;
+
+            completionElapsed.Should().BeLessThan(TimeSpan.FromMilliseconds(500));
+            // If completion had awaited its own guidance, the shared deadline would have expired
+            // and the next request would not observe nearly the full delay.
+            nextRequestElapsed.Should().BeGreaterThan(TimeSpan.FromMilliseconds(750));
+            handler.Requests.Should().HaveCount(2);
+        }
+
+        [Fact]
+        public async Task ResultTransport_RetriesWritesAndRecordsExplicitRequestKinds()
+        {
+            int resultAttempts = 0;
+            int attachmentAttempts = 0;
+            var handler = new RecordingHttpMessageHandler(request =>
+            {
+                bool isAttachment = request.RequestUri.AbsolutePath.EndsWith("/attachments", StringComparison.OrdinalIgnoreCase);
+                int attempt = isAttachment
+                    ? Interlocked.Increment(ref attachmentAttempts)
+                    : Interlocked.Increment(ref resultAttempts);
+
+                if (attempt == 1)
+                {
+                    var throttled = new HttpResponseMessage(HttpStatusCode.TooManyRequests);
+                    throttled.Headers.RetryAfter = new System.Net.Http.Headers.RetryConditionHeaderValue(
+                        TimeSpan.FromMilliseconds(25));
+                    return throttled;
+                }
+
+                return new HttpResponseMessage(HttpStatusCode.OK)
+                {
+                    Content = new StringContent(isAttachment ? "{}" : @"{""value"":[{""id"":1}]}")
+                };
+            });
+            var metrics = new JobMonitorMetrics();
+            using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler), metrics);
+            IAzureDevOpsResultTransport transport = service.CreateResultTransport(123);
+
+            await transport.PublishResultsAsync(new[] { new { outcome = "Passed" } }, CancellationToken.None);
+            await transport.UploadAttachmentAsync(1, null, "log.txt", "YQ==", CancellationToken.None);
+
+            JobMonitorMetricsSnapshot snapshot = metrics.Snapshot();
+            snapshot.AzureDevOpsResultRequests.Should().Be(2);
+            snapshot.AzureDevOpsAttachmentRequests.Should().Be(2);
+            snapshot.AzureDevOpsControlRequests.Should().Be(0);
+            snapshot.AzureDevOpsRetries.Should().Be(2);
+            snapshot.AzureDevOpsFailedAttempts.Should().Be(2);
+        }
+
+        [Fact]
+        public async Task ResultTransport_TerminalCompletedRunResponseDoesNotRetry()
+        {
+            var handler = new RecordingHttpMessageHandler(_ =>
+                new HttpResponseMessage(HttpStatusCode.BadRequest)
+                {
+                    Content = new StringContent("cannot be added or updated for a test run which is in Completed state")
                 });
-                using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler));
+            using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler));
 
-                var stopwatch = Stopwatch.StartNew();
-                await service.CompleteTestRunAsync(123, HelixJobGuid, [], CancellationToken.None);
-                TimeSpan completionElapsed = stopwatch.Elapsed;
+            Func<Task> action = () => service.CreateResultTransport(123)
+                .PublishResultsAsync(new[] { new { outcome = "Passed" } }, CancellationToken.None);
 
-                await service.GetTimelineRecordsAsync(CancellationToken.None);
-                TimeSpan nextRequestElapsed = stopwatch.Elapsed - completionElapsed;
+            await action.Should().ThrowAsync<TerminalError>();
+            handler.Requests.Should().ContainSingle();
+        }
 
-                completionElapsed.Should().BeLessThan(TimeSpan.FromMilliseconds(500));
-                nextRequestElapsed.Should().BeGreaterThan(TimeSpan.FromMilliseconds(750));
-                handler.Requests.Should().HaveCount(2);
+        [Fact]
+        public void AzureDevOpsTransportClassifiesTimeoutsAndRateLimitHeaders()
+        {
+            AzureDevOpsService.IsTransientException(
+                new OperationCanceledException("timeout", new TimeoutException()),
+                CancellationToken.None).Should().BeTrue();
+
+            using var cancellation = new CancellationTokenSource();
+            cancellation.Cancel();
+            AzureDevOpsService.IsTransientException(
+                new OperationCanceledException("timeout", new TimeoutException()),
+                cancellation.Token).Should().BeFalse();
+
+            using var response = new HttpResponseMessage(HttpStatusCode.OK);
+            response.Headers.RetryAfter = new System.Net.Http.Headers.RetryConditionHeaderValue(TimeSpan.FromSeconds(2));
+            response.Headers.Add("X-RateLimit-Delay", "3.5");
+            AzureDevOpsService.GetRateLimitDelay(response).Should().Be(TimeSpan.FromSeconds(3.5));
         }
 
         [Fact]
@@ -191,6 +271,24 @@ public async Task CompleteTestRunAsync_UploadsFailedWorkItemsAttachmentBeforePat
             patchRequest.RequestUri.ToString().Should().Be("https://dev.azure.com/dnceng-public/public/_apis/test/runs/123?api-version=7.1");
         }
 
+        [Fact]
+        public async Task CompleteTestRunAsync_DoesNotRetryAmbiguousFailedWorkItemsAttachmentWrite()
+        {
+            var handler = new RecordingHttpMessageHandler(_ =>
+                new HttpResponseMessage(HttpStatusCode.ServiceUnavailable));
+            using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler));
+
+            Func<Task> action = () => service.CompleteTestRunAsync(
+                123,
+                HelixJobGuid,
+                ["wi-1"],
+                CancellationToken.None);
+
+            await action.Should().ThrowAsync<HttpRequestException>();
+            handler.Requests.Should().ContainSingle()
+                .Which.RequestUri.AbsolutePath.Should().EndWith("/attachments");
+        }
+
         [Fact]
         public async Task CompleteTestRunAsync_SkipsAttachment_WhenNoFailedWorkItems()
         {
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs
index ea70f4585a9..c76c8b647c9 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs
@@ -25,6 +25,7 @@ internal sealed class FakeAzureDevOpsService : IAzureDevOpsService
         private readonly Queue<Exception> _createFailures = [];
         private readonly Queue<Exception> _uploadFailures = [];
         private readonly Queue<Exception> _completeFailures = [];
+        private readonly Queue<Exception> _timelineFailures = [];
         private readonly HashSet<(string JobName, string WorkItemName)> _recordedFailedTests
             = new(FailedTestWorkItemComparer.Instance);
         private readonly HashSet<(string JobName, string WorkItemName)> _uploadFailedTests
@@ -83,6 +84,16 @@ public FakeAzureDevOpsService WithPreviouslyProcessedJob(string jobName)
             return this;
         }
 
+        public FakeAzureDevOpsService FailNextTimeline(Exception exception)
+        {
+            lock (_sync)
+            {
+                _timelineFailures.Enqueue(exception);
+            }
+
+            return this;
+        }
+
         public FakeAzureDevOpsService FailNextCreate(Exception exception = null)
         {
             lock (_sync)
@@ -146,6 +157,14 @@ public FakeAzureDevOpsService WithFailedUpload(string helixJobName, string workI
         // IAzureDevOpsService implementation
         public Task<IReadOnlyList<AzureDevOpsTimelineRecord>> GetTimelineRecordsAsync(CancellationToken cancellationToken)
         {
+            lock (_sync)
+            {
+                if (_timelineFailures.TryDequeue(out Exception failure))
+                {
+                    return Task.FromException<IReadOnlyList<AzureDevOpsTimelineRecord>>(failure);
+                }
+            }
+
             if (_timelineResponses.Count == 0)
             {
                 _timelineCallCount++;
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
index 09bea71a6a7..1811cb2aee3 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs
@@ -73,7 +73,7 @@ public async Task GetJobsForBuildAsync_PassesSourceAndBuildIdFilters()
                     Assert.Equal("BuildId", property.Key);
                     Assert.Equal("123", property.Value);
                 });
-            Assert.Equal(100_000, capturedCount);
+            Assert.Equal(1_000, capturedCount);
             Assert.Equal(2, jobs.Count);
             Assert.Equal("running-job", jobs[0].JobName);
             Assert.Equal("running", jobs[0].Status);
@@ -103,6 +103,30 @@ await Assert.ThrowsAsync<ArgumentException>(() =>
                 service.GetJobsForBuildAsync(source: "ci/public/dotnet/runtime/refs/heads/main", buildId: "", CancellationToken.None));
         }
 
+        [Fact]
+        public async Task GetJobsForBuildAsync_RejectsPotentiallyTruncatedResponse()
+        {
+            var api = CreateApi();
+            api.Job
+                .Setup(j => j.ListAsync(null, 1_000, null, null, It.IsAny<IImmutableDictionary<string, string>>(), It.IsAny<string>(), null, It.IsAny<CancellationToken>()))
+                .ReturnsAsync(ImmutableList.CreateRange(
+                    Enumerable.Range(0, 1_000)
+                        .Select(index => Job(
+                            $"job-{index}",
+                            finished: null,
+                            new JObject { ["BuildId"] = "123" }))));
+
+            HelixService service = CreateService(api.Api.Object);
+
+            InvalidOperationException exception = await Assert.ThrowsAsync<InvalidOperationException>(() =>
+                service.GetJobsForBuildAsync(
+                    source: "ci/public/dotnet/runtime/refs/heads/main",
+                    buildId: "123",
+                    CancellationToken.None));
+
+            Assert.Contains("potentially truncated result set", exception.Message);
+        }
+
         [Fact]
         public async Task DownloadTestResultsAsync_FiltersFilesUsesFileSystemAndContinuesAfterDownloadFailure()
         {
diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
index dd5e7edc332..c979d4048f3 100644
--- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
+++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs
@@ -23,6 +23,28 @@ namespace Microsoft.DotNet.Helix.Sdk.Tests
     [Collection("NonParallel")]
     public class JobMonitorRunnerTests
     {
+        [Theory]
+        [InlineData(null)]
+        [InlineData("")]
+        public void MissingAttemptMetadata_DefaultsToFirstAttempt(string value)
+        {
+            MonitorState.ParseStageAttempt(value).Should().Be(1);
+            MonitorState.ParseJobAttempt(value).Should().Be(1);
+        }
+
+        [Theory]
+        [InlineData("invalid")]
+        [InlineData("0")]
+        [InlineData("-1")]
+        public void MalformedAttemptMetadata_IsRejected(string value)
+        {
+            Action parseStageAttempt = () => MonitorState.ParseStageAttempt(value);
+            Action parseJobAttempt = () => MonitorState.ParseJobAttempt(value);
+
+            parseStageAttempt.Should().Throw<InvalidOperationException>();
+            parseJobAttempt.Should().Throw<InvalidOperationException>();
+        }
+
         /// <summary>
         /// Single pipeline job goes from queued → in progress → completed (succeeded).
         /// No Helix jobs are ever submitted.
@@ -4654,6 +4676,19 @@ public async Task CompletedHelixJob_LogsFailedWorkItemConsoleLinks()
                 message.Contains("Helix job: helix-linux", StringComparison.Ordinal));
         }
 
+        [Fact]
+        public async Task NonCallerOperationCanceledException_Propagates()
+        {
+            var azdo = new FakeAzureDevOpsService()
+                .FailNextTimeline(new OperationCanceledException("Injected service timeout."));
+            var runner = CreateRunner(azdo, new FakeHelixService());
+
+            Func<Task> action = () => runner.RunAsync(CancellationToken.None);
+
+            await action.Should().ThrowAsync<OperationCanceledException>()
+                .WithMessage("Injected service timeout.");
+        }
+
         [Fact]
         public async Task LoopStatus_LogsAggregateHelixJobWorkItemCounts()
         {

From 80e4dbe8476a615cc395ccd9ef0015d9f12986fc Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Thu, 20 Aug 2026 17:00:41 -0700
Subject: [PATCH 19/21] Simplify rate limit deadline updates

Replace the compare-exchange retry loop with a short lock-protected maximum update while preserving asynchronous deadline waits.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../Services/AzureDevOpsRateLimitGate.cs         | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
index 8648b6026ef..466276cfaf6 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
@@ -5,6 +5,7 @@ namespace Microsoft.DotNet.Helix.JobMonitor;
 
 internal sealed class AzureDevOpsRateLimitGate
 {
+    private readonly object _sync = new();
     private readonly JobMonitorMetrics _metrics;
     private long _notBeforeUtcTicks;
 
@@ -26,13 +27,9 @@ public void ExtendDeadline(TimeSpan delay)
 
         _metrics?.RecordRateLimitDeferral(delay);
         long candidate = DateTimeOffset.UtcNow.Add(delay).UtcTicks;
-        long observed;
-        while (candidate > (observed = Interlocked.Read(ref _notBeforeUtcTicks)))
+        lock (_sync)
         {
-            if (Interlocked.CompareExchange(ref _notBeforeUtcTicks, candidate, observed) == observed)
-            {
-                break;
-            }
+            _notBeforeUtcTicks = Math.Max(_notBeforeUtcTicks, candidate);
         }
     }
 
@@ -46,7 +43,12 @@ public async Task WaitAsync(CancellationToken cancellationToken)
             // worker is waiting.
             while (true)
             {
-                long notBeforeTicks = Interlocked.Read(ref _notBeforeUtcTicks);
+                long notBeforeTicks;
+                lock (_sync)
+                {
+                    notBeforeTicks = _notBeforeUtcTicks;
+                }
+
                 TimeSpan delay = TimeSpan.FromTicks(notBeforeTicks - DateTimeOffset.UtcNow.UtcTicks);
                 if (delay <= TimeSpan.Zero)
                 {

From 5bb6b6a3dbd994e74282e23f1a4458333193cfe9 Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Fri, 21 Aug 2026 07:21:17 -0700
Subject: [PATCH 20/21] Fix build-scoped Helix query merge

Preserve the server-side BuildId filter, defensive local validation, and fail-closed query bound after merging the upstream efficiency fix.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../JobMonitor/Services/HelixService.cs          | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
index 52ae6e9f5c1..361afb5c041 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs
@@ -65,8 +65,7 @@ public async Task<IReadOnlyList<HelixJobInfo>> GetJobsForBuildAsync(
             {
                 throw new ArgumentException("A non-empty build ID filter must be provided.", nameof(buildId));
             }
-            
-            var filterProperties = new Dictionary<string, string>()
+            IImmutableDictionary<string, string> filterProperties = new Dictionary<string, string>()
             {
                 ["BuildId"] = buildId,
             }.ToImmutableDictionary();
@@ -74,7 +73,7 @@ public async Task<IReadOnlyList<HelixJobInfo>> GetJobsForBuildAsync(
             IImmutableList<JobSummary> jobs = await RetryAsync(
                 async () => await _helixApi.Job.ListAsync(
                     source: source,
-                    properties: properties,
+                    properties: filterProperties,
                     count: MaximumJobsPerBuildQuery),
                 cancellationToken);
 
@@ -85,7 +84,16 @@ public async Task<IReadOnlyList<HelixJobInfo>> GetJobsForBuildAsync(
                     $"{MaximumJobsPerBuildQuery}. Refusing to monitor a potentially truncated result set.");
             }
 
-            return [..jobs.Select(j => new HelixJobInfo(j))];
+            // Keep the local check as a defensive contract boundary in case Helix returns a
+            // malformed or unexpectedly broad response.
+            return
+            [
+                ..jobs
+                    .Where(j => j.Properties is JObject properties
+                        && properties.TryGetValue("BuildId", out JToken id)
+                        && buildId == id.Value<string>())
+                    .Select(j => new HelixJobInfo(j))
+            ];
         }
 
         public async Task<WorkItemTestResults> DownloadTestResultsAsync(

From 1ec49e6079349e93f7838e16ade8b016b613177e Mon Sep 17 00:00:00 2001
From: "Matt Mitchell (.NET)" <mmitche@microsoft.com>
Date: Fri, 21 Aug 2026 08:29:43 -0700
Subject: [PATCH 21/21] Restore lock-free rate limit deadline updates

The compare-exchange loop is expected to complete in very few iterations and avoids serializing concurrent deadline readers and writers.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 3a3a2b2f-e044-4835-8a3c-0c99a6a29484
---
 .../Services/AzureDevOpsRateLimitGate.cs         | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
index 466276cfaf6..8648b6026ef 100644
--- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
+++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsRateLimitGate.cs
@@ -5,7 +5,6 @@ namespace Microsoft.DotNet.Helix.JobMonitor;
 
 internal sealed class AzureDevOpsRateLimitGate
 {
-    private readonly object _sync = new();
     private readonly JobMonitorMetrics _metrics;
     private long _notBeforeUtcTicks;
 
@@ -27,9 +26,13 @@ public void ExtendDeadline(TimeSpan delay)
 
         _metrics?.RecordRateLimitDeferral(delay);
         long candidate = DateTimeOffset.UtcNow.Add(delay).UtcTicks;
-        lock (_sync)
+        long observed;
+        while (candidate > (observed = Interlocked.Read(ref _notBeforeUtcTicks)))
         {
-            _notBeforeUtcTicks = Math.Max(_notBeforeUtcTicks, candidate);
+            if (Interlocked.CompareExchange(ref _notBeforeUtcTicks, candidate, observed) == observed)
+            {
+                break;
+            }
         }
     }
 
@@ -43,12 +46,7 @@ public async Task WaitAsync(CancellationToken cancellationToken)
             // worker is waiting.
             while (true)
             {
-                long notBeforeTicks;
-                lock (_sync)
-                {
-                    notBeforeTicks = _notBeforeUtcTicks;
-                }
-
+                long notBeforeTicks = Interlocked.Read(ref _notBeforeUtcTicks);
                 TimeSpan delay = TimeSpan.FromTicks(notBeforeTicks - DateTimeOffset.UtcNow.UtcTicks);
                 if (delay <= TimeSpan.Zero)
                 {