diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/RESULTS_FORMAT.md b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/RESULTS_FORMAT.md new file mode 100644 index 00000000000..d2ad49ce72f --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/RESULTS_FORMAT.md @@ -0,0 +1,168 @@ +# Helix test results JSON format + +The Helix test reporter writes a portable, language-neutral JSON file +describing the results of a work item. This file is written alongside +the legacy pickle format (`__test_report.json`) and is intended as the +long-term wire format between the in-work-item reporter and any consumer +(the Python Helix client, an AOT/native Helix client, build-side +post-processing, etc.). + +## File location + +``` +$HELIX_WORKITEM_ROOT/__test_report_v2.json +``` + +If `HELIX_WORKITEM_ROOT` is unset (e.g. ad-hoc local invocation), the +file is written to the current working directory. + +The file is written **unconditionally** by `reporter/run.py`, regardless +of whether the legacy `helix-scripts` Python package is installed on the +machine. When `helix-scripts` is present, the legacy pickle file is +also written; behavior is byte-identical to releases prior to the +introduction of this format. + +## Producing this file directly + +The xUnit / JUnit / TRX parsers shipped with the reporter are +*adapters*: they read a third-party XML format and populate the same +`TestResult` objects that get serialized here. A test runner that does +not produce one of those XML formats can emit this JSON file directly +and skip the parser entirely. The reporter (or any consumer) will +treat directly-emitted JSON identically to JSON produced from XML. + +Recommended use: write `__test_report_v2.json` to `$HELIX_WORKITEM_ROOT` +from your test runner's own reporting hook, then either omit +`EnableAzurePipelinesReporter` or let it run — the reporter is a no-op +on a missing/empty results XML when the JSON file already exists. + +## Schema + +Top level: + +```json +{ + "schema_version": 1, + "azdo": { ... }, + "results": [ { ... }, ... ] +} +``` + +| Field | Type | Required | Description | +|------------------|--------|----------|-------------| +| `schema_version` | int | yes | Major version of this format. Consumers must refuse files with an unknown major version. Current value: `1`. | +| `azdo` | object | yes | Azure DevOps reporting parameters needed by the consumer to publish results. May contain `null` fields when running outside an AzDO pipeline. | +| `results` | array | yes | Per-test results. May be empty (e.g. work item produced no detectable tests). | + +### `azdo` object + +| Field | Type | Required | Description | +|------------------|----------------|----------|-------------| +| `collection_uri` | string \| null | yes | AzDO collection URI, e.g. `https://dev.azure.com/dnceng/`. | +| `team_project` | string \| null | yes | AzDO project name, e.g. `internal`. | +| `test_run_id` | string \| null | yes | AzDO test run ID, as a string. Created by the build before the work item is dispatched. | +| `access_token` | string \| null | no | AzDO bearer token used by the consumer to POST results. **Sensitive.** Consumers MUST treat the entire file as secret while this field is populated. May be omitted when the consumer obtains the token by other means (managed identity, env var). | + +### `results[]` object + +Each entry describes one test case. + +| Field | Type | Required | Description | +|--------------------|----------------|----------|-------------| +| `name` | string | yes | Human-readable test identifier. Typically `{type}.{method}` plus theory data. Used as the display name in AzDO. | +| `kind` | string | yes | Source format: one of `"xunit"`, `"junit"`, `"trx"`, or any custom value used by a direct producer. Informational only. | +| `type` | string \| null | yes | Containing class / module / fixture name. | +| `method` | string \| null | yes | Method / function name within `type`. | +| `duration_seconds` | number | yes | Wall-clock duration of the test, in seconds. `0` if unknown. | +| `result` | string | yes | Outcome. One of: `"Pass"`, `"Fail"`, `"Skip"`. Other values are reserved. | +| `exception_type` | string \| null | yes | Fully-qualified exception type for failed tests, e.g. `"Xunit.Sdk.TrueException"`. `null` for passes/skips or when not available. | +| `failure_message` | string \| null | yes | Failure message produced by the test framework. `null` for passes/skips. May contain newlines. | +| `stack_trace` | string \| null | yes | Captured stack trace for failed tests. `null` for passes/skips or when not available. May contain newlines. | +| `skip_reason` | string \| null | yes | Reason a test was skipped (e.g. xUnit `[Fact(Skip = "...")]`). `null` for non-skipped tests. | +| `ignored` | bool | no | Producer-side hint that the consumer should not report this result (used during local rerun logic). Defaults to `false`. | +| `attachments` | array | yes | Per-test attachments. Empty array if none. | + +### `attachments[]` object + +| Field | Type | Required | Description | +|---------|----------------|----------|-------------| +| `name` | string | yes | Attachment file name as it should appear in AzDO. Should be unique within the test result. | +| `text` | string | yes | Attachment contents as text. Binary attachments must be base64- or otherwise-encoded into a textual form by the producer; this format does not transport binary blobs natively. | + +## Example + +```json +{ + "schema_version": 1, + "azdo": { + "collection_uri": "https://dev.azure.com/dnceng/", + "team_project": "internal", + "test_run_id": "12345", + "access_token": "eyJ0eXAiOiJKV1QiLCJ..." + }, + "results": [ + { + "name": "MyTests.Math.Addition", + "kind": "xunit", + "type": "MyTests.Math", + "method": "Addition", + "duration_seconds": 0.012, + "result": "Pass", + "exception_type": null, + "failure_message": null, + "stack_trace": null, + "skip_reason": null, + "ignored": false, + "attachments": [] + }, + { + "name": "MyTests.Math.Division_ByZero", + "kind": "xunit", + "type": "MyTests.Math", + "method": "Division_ByZero", + "duration_seconds": 0.003, + "result": "Fail", + "exception_type": "System.DivideByZeroException", + "failure_message": "Attempted to divide by zero.", + "stack_trace": " at MyTests.Math.Division_ByZero() in ...", + "skip_reason": null, + "ignored": false, + "attachments": [ + { "name": "stdout.txt", "text": "Computing 1 / 0...\n" } + ] + }, + { + "name": "MyTests.Math.SlowAddition", + "kind": "xunit", + "type": "MyTests.Math", + "method": "SlowAddition", + "duration_seconds": 0, + "result": "Skip", + "exception_type": null, + "failure_message": null, + "stack_trace": null, + "skip_reason": "Disabled pending perf investigation", + "ignored": false, + "attachments": [] + } + ] +} +``` + +## Versioning + +* `schema_version` is a single integer. +* Consumers MUST treat any value other than the version(s) they + understand as an error, not silently degrade. +* Backwards-compatible additions (new optional fields, new optional + outcome strings) do **not** bump the version. +* Breaking changes (renamed fields, changed semantics, changed required + fields) MUST bump the version. + +## Security + +The `azdo.access_token` field, when populated, is a bearer token with +build-scope authority on Azure DevOps. The producer SHOULD ensure the +file is written with permissions that prevent other users on the +machine from reading it; the consumer SHOULD delete or overwrite the +file once results have been published. diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/_helix_compat.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/_helix_compat.py new file mode 100644 index 00000000000..cbc79be6cf6 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/_helix_compat.py @@ -0,0 +1,274 @@ +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. + +"""Minimal stand-ins for the helix.public types used by the reporter. + +The reporter has historically depended on the `helix-scripts` Python package +being installed on the test machine (typically via the helix-prep venv at +`/etc/helix-prep/venv`). Newer Helix client implementations (notably the AOT +client) do not ship that package by default, so the import fails and the +reporter cannot run. + +This module provides API-compatible substitutes for the small surface the +reporter actually consumes: + + - TestResult (data container) + - TestResultAttachment (data container) + - AzureDevOpsReportingParameters (data container) + - JsonReporter (writes a portable JSON results file) + +The constructors mirror the signatures used by the format parsers and by +run.py so they can be swapped in transparently when the real classes are +not importable. + +The JSON file written by JsonReporter is the long-term wire format: it is +language-neutral (no pickle / no class identity coupling) and can be +consumed by either the legacy Python Helix client or a native (e.g. C#) +client without spawning Python. +""" + +import json +import logging +import os +from typing import Iterable, List, Optional + + +# Schema version embedded in every emitted JSON file. Bump when the on-disk +# layout changes in a backwards-incompatible way; consumers should refuse +# files with an unknown major version. +SCHEMA_VERSION = 1 + +# Filename written next to (not replacing) the legacy `__test_report.json` +# pickle file. The pickle file's name is historical and misleading; this +# new file is the real JSON. +JSON_RESULTS_FILENAME = "__test_report_v2.json" + + +def _results_dir(): + """Return the directory the JSON results file is written into. + + Mirrors helix.test_reporting.packing_test_reporter._file_name() closely + enough that a Helix client looking in the same place will find both + files. Falls back to cwd if HELIX_WORKITEM_ROOT is unset (which would + only happen outside a real Helix work item). + """ + return os.environ.get("HELIX_WORKITEM_ROOT") or os.getcwd() + + +def json_results_path(): + """Absolute path to the JSON results file for the current work item.""" + return os.path.join(_results_dir(), JSON_RESULTS_FILENAME) + + +class TestResultAttachment(object): + """API-compatible stand-in for helix.public.TestResultAttachment.""" + + __test__ = False # pytest: do not collect + + def __init__(self, name, text): + self._name = name + self._text = text + + @property + def name(self): + return self._name + + @property + def text(self): + return self._text + + def to_dict(self): + return {"name": self._name, "text": self._text} + + +class TestResult(object): + """API-compatible stand-in for helix.public.TestResult.""" + + __test__ = False # pytest: do not collect + + def __init__(self, + name, + kind, + type_name, + method, + duration, + result, + exception_type, + failure_message, + stack_trace, + skip_reason, + attachments): + self._name = name + self._kind = kind + self._type = type_name + self._method = method + self._duration_seconds = duration + self._result = result + self._exception_type = exception_type + self._failure_message = failure_message + self._stack_trace = stack_trace + self._skip_reason = skip_reason + self._attachments = attachments + self.ignored = False + + @property + def name(self): + return self._name + + @property + def kind(self): + return self._kind + + @property + def type(self): + return self._type + + @property + def method(self): + return self._method + + @property + def duration_seconds(self): + return self._duration_seconds + + @property + def result(self): + return self._result + + @property + def exception_type(self): + return self._exception_type + + @property + def failure_message(self): + return self._failure_message + + @property + def stack_trace(self): + return self._stack_trace + + @property + def skip_reason(self): + return self._skip_reason + + @property + def attachments(self): + return self._attachments + + def to_dict(self): + return { + "name": self._name, + "kind": self._kind, + "type": self._type, + "method": self._method, + "duration_seconds": self._duration_seconds, + "result": self._result, + "exception_type": self._exception_type, + "failure_message": self._failure_message, + "stack_trace": self._stack_trace, + "skip_reason": self._skip_reason, + "ignored": self.ignored, + "attachments": [_attachment_to_dict(a) for a in (self._attachments or [])], + } + + +class AzureDevOpsReportingParameters(object): + """API-compatible stand-in for helix.public.AzureDevOpsReportingParameters.""" + + def __init__(self, collection_uri, team_project, test_run_id, access_token): + self.collection_uri = collection_uri + self.team_project = team_project + self.test_run_id = test_run_id + self.access_token = access_token + + def to_dict(self, include_token=True): + d = { + "collection_uri": self.collection_uri, + "team_project": self.team_project, + "test_run_id": self.test_run_id, + } + if include_token: + d["access_token"] = self.access_token + return d + + +def _attachment_to_dict(a): + """Serialize either our TestResultAttachment or the helix-scripts one.""" + if hasattr(a, "to_dict"): + return a.to_dict() + return {"name": getattr(a, "name", None), "text": getattr(a, "text", None)} + + +def _result_to_dict(r): + """Serialize either our TestResult or the helix-scripts one.""" + if hasattr(r, "to_dict"): + return r.to_dict() + return { + "name": getattr(r, "name", None), + "kind": getattr(r, "kind", None), + "type": getattr(r, "type", None), + "method": getattr(r, "method", None), + "duration_seconds": getattr(r, "duration_seconds", None), + "result": getattr(r, "result", None), + "exception_type": getattr(r, "exception_type", None), + "failure_message": getattr(r, "failure_message", None), + "stack_trace": getattr(r, "stack_trace", None), + "skip_reason": getattr(r, "skip_reason", None), + "ignored": getattr(r, "ignored", False), + "attachments": [_attachment_to_dict(a) for a in (getattr(r, "attachments", None) or [])], + } + + +class JsonReporter(object): + """Writes a portable, schema-versioned JSON file with the test results. + + Always runs alongside the legacy pickle-based PackingTestReporter (when + available) so existing consumers continue to work unchanged. The file + layout is: + + { + "schema_version": 1, + "azdo": { "collection_uri", "team_project", "test_run_id", "access_token" }, + "results": [ { TestResult fields }, ... ] + } + """ + + __test__ = False + + def __init__(self, azdo_parameters, log=None): + self._azdo = azdo_parameters + self._log = log or logging.getLogger(__name__) + + def report_results(self, results): + results = [r for r in (results or []) if r is not None] + path = json_results_path() + payload = { + "schema_version": SCHEMA_VERSION, + "azdo": _azdo_to_dict(self._azdo), + "results": [_result_to_dict(r) for r in results], + } + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + except OSError: + # Directory already exists or cannot be created; let open() raise. + pass + self._log.info("Writing %d test results to '%s' (JSON v%d)", + len(results), path, SCHEMA_VERSION) + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False) + try: + size = os.path.getsize(path) + self._log.info("Wrote %d bytes to '%s'", size, path) + except OSError: + pass + + +def _azdo_to_dict(p): + if hasattr(p, "to_dict"): + return p.to_dict() + return { + "collection_uri": getattr(p, "collection_uri", None), + "team_project": getattr(p, "team_project", None), + "test_run_id": getattr(p, "test_run_id", None), + "access_token": getattr(p, "access_token", None), + } diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/junit.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/junit.py index 629f31b2165..e62531c990b 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/junit.py +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/junit.py @@ -1,6 +1,9 @@ import xml.etree.ElementTree from .result_format import ResultFormat -from helix.public import TestResult, TestResultAttachment +try: + from helix.public import TestResult, TestResultAttachment +except ImportError: + from _helix_compat import TestResult, TestResultAttachment class JUnitFormat(ResultFormat): diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/result_format.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/result_format.py index 8d7cc966311..70a0ae136a3 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/result_format.py +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/result_format.py @@ -1,5 +1,8 @@ from abc import ABCMeta, abstractmethod, abstractproperty -from helix.public import TestResult +try: + from helix.public import TestResult +except ImportError: + from _helix_compat import TestResult from typing import Iterable diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/trx.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/trx.py index cb2afbec916..0464d0518e8 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/trx.py +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/trx.py @@ -1,7 +1,10 @@ import glob import xml.etree.ElementTree from .result_format import ResultFormat -from helix.public import TestResult, TestResultAttachment +try: + from helix.public import TestResult, TestResultAttachment +except ImportError: + from _helix_compat import TestResult, TestResultAttachment class TRXFormat(ResultFormat): diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/xunit.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/xunit.py index bbf8ee6f96e..a09916deca5 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/xunit.py +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/xunit.py @@ -2,7 +2,10 @@ import xml.etree.ElementTree from .result_format import ResultFormat -from helix.public import TestResult, TestResultAttachment +try: + from helix.public import TestResult, TestResultAttachment +except ImportError: + from _helix_compat import TestResult, TestResultAttachment _unescape_char_map = { 'r': '\r', diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py index 00f77cf7cba..9736e54343c 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py @@ -12,7 +12,26 @@ from helpers import get_env from test_results_reader import read_results -from helix.public import DefaultTestReporter, AzureDevOpsReportingParameters, PackingTestReporter +# Bundled, dependency-free shim. Always importable. Provides the JSON writer +# plus stand-ins for TestResult / TestResultAttachment / AzureDevOpsReportingParameters +# used when helix-scripts is not installed (e.g. AOT Helix client). +import _helix_compat + +# Try to use the real helix-scripts types so the legacy pickle (which keys on +# fully-qualified class names) stays byte-identical with previous releases. +# Fall back to the shim when helix-scripts is not present on the machine. +try: + from helix.public import ( + DefaultTestReporter, + AzureDevOpsReportingParameters, + PackingTestReporter, + ) + _HELIX_SCRIPTS_AVAILABLE = True +except ImportError: + DefaultTestReporter = None + PackingTestReporter = None + AzureDevOpsReportingParameters = _helix_compat.AzureDevOpsReportingParameters + _HELIX_SCRIPTS_AVAILABLE = False def process_args() -> Tuple[str, str, str, Optional[str]]: if len(sys.argv) < 4 or len(sys.argv) > 5: @@ -53,16 +72,41 @@ def main(): get_env("HELIX_WORKITEM_UPLOAD_ROOT"), ]) - reporter = DefaultTestReporter( - AzureDevOpsReportingParameters( - collection_uri, - team_project, - test_run_id, - access_token - ) + azdo_parameters = AzureDevOpsReportingParameters( + collection_uri, + team_project, + test_run_id, + access_token, ) - reporter.report_results(all_results) + # 1) Legacy: when helix-scripts is installed, write the pickle file at + # {HELIX_WORKITEM_ROOT}/__test_report.json that the Python Helix client + # (helix.executor) consumes. Behavior here is byte-identical to the + # pre-change reporter so existing consumers see no difference. + if _HELIX_SCRIPTS_AVAILABLE: + try: + reporter = DefaultTestReporter(azdo_parameters) + reporter.report_results(all_results) + except Exception: + log.exception("Legacy pickle reporter failed; continuing to write JSON results") + else: + log.warning( + "helix-scripts not available; skipping legacy pickle reporter. " + "Consumers must read the JSON results file at '%s'.", + _helix_compat.json_results_path(), + ) + + # 2) New: always write a portable JSON results file alongside. This is + # language-neutral and lets non-Python Helix clients (e.g. the AOT + # client) consume results without installing helix-scripts. + try: + _helix_compat.JsonReporter(azdo_parameters, log=log).report_results(all_results) + except Exception: + log.exception("Failed to write JSON results file") + # Don't fail the work item solely because of the new path; the legacy + # pickle file (if it was written above) is still the primary contract. + if not _HELIX_SCRIPTS_AVAILABLE: + raise if __name__ == '__main__': main() diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/test_results_reader/__init__.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/test_results_reader/__init__.py index f80af8dcb82..e73aadad7b3 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/test_results_reader/__init__.py +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/test_results_reader/__init__.py @@ -1,6 +1,13 @@ import logging import os -from helix.public import TestResult, TestResultAttachment +try: + # Preferred: real helix-scripts types (allows pickle round-trip via the + # legacy PackingTestReporter path). + from helix.public import TestResult, TestResultAttachment +except ImportError: + # Fallback: bundled shim used when helix-scripts is not installed on + # the test machine (e.g. AOT Helix client images). + from _helix_compat import TestResult, TestResultAttachment from typing import Iterable, List from formats import all_formats from helpers import get_env