This handler configures the OTel plugin with: - * - *
In production, replace {@code LoggingSpanExporter} with {@code OtlpGrpcSpanExporter} to send spans to an OTLP - * collector (X-Ray, Datadog, etc.). - * - *
Expected trace structure: - * - *
- * durable.invocation - * ├── durable.step:create-greeting [attempt 1] - * ├── durable.step:create-greeting (operation, backfilled) - * ├── durable.step:transform [attempt 1] - * └── durable.step:transform (operation, backfilled) - *- */ -public class OtelExample extends DurableHandler
{@link ExecutionOtelPlugin#ExecutionOtelPlugin()} uses the global provider initialized by the ADOT Java agent. The
- * ExecutionOtelPlugin renders the Workflow span as the trace root with operations as siblings of the invocation span.
- */
-@ExampleTemplate(tracing = true, javaAgent = true)
-public class OtelXRayExecutionStepExample extends DurableHandler Exercises the multi-invocation tracing scenario with the workflow-rooted trace structure. The Workflow span is
- * only exported on the terminal invocation, producing a clean single-execution trace.
- */
-@ExampleTemplate(tracing = true, javaAgent = true)
-public class OtelXRayExecutionWaitExample extends DurableHandler Exports spans through the ADOT Java agent global OpenTelemetry provider. Requires:
- *
- * Expected trace structure in X-Ray:
- *
- * This handler exercises the critical multi-invocation tracing scenario:
- *
- * Exports spans through the ADOT Java agent global OpenTelemetry provider. Requires:
- *
- * Expected trace structure in X-Ray (all under one trace ID — backend propagates same Root):
- *
- * These tests deploy Lambda functions configured with:
- *
- * After invoking the function, the test queries the X-Ray API to verify:
- *
- * Enable with: {@code -Dtest.cloud.enabled=true}
- */
-@EnabledIf("isEnabled")
-class CloudBasedOtelIntegrationTest {
-
- private static final Duration XRAY_INGESTION_DELAY = Duration.ofSeconds(20);
- private static final int XRAY_QUERY_RETRIES = 3;
- private static final Duration XRAY_RETRY_DELAY = Duration.ofSeconds(10);
- private static final ObjectMapper MAPPER = new ObjectMapper();
-
- private static String account;
- private static String region;
- private static String functionNamePrefix;
- private static LambdaClient lambdaClient;
- private static XRayClient xrayClient;
-
- static boolean isEnabled() {
- var enabled = "true".equals(System.getProperty("test.cloud.enabled"));
- if (!enabled) {
- System.out.println("⚠️ OTel X-Ray integration tests disabled. Enable with -Dtest.cloud.enabled=true");
- }
- return enabled;
- }
-
- @BeforeAll
- static void setup() {
- try {
- DefaultCredentialsProvider.builder().build().resolveCredentials();
- } catch (Exception e) {
- throw new IllegalStateException("AWS credentials not available");
- }
-
- account = System.getProperty("test.aws.account");
- region = System.getProperty("test.aws.region");
- functionNamePrefix = System.getProperty("test.function.name.prefix", "");
-
- if (account == null || region == null) {
- try (var sts = StsClient.create()) {
- if (account == null) account = sts.getCallerIdentity().account();
- if (region == null)
- region = sts.serviceClientConfiguration().region().id();
- }
- }
-
- lambdaClient = LambdaClient.builder()
- .credentialsProvider(DefaultCredentialsProvider.builder().build())
- .region(Region.of(region))
- .build();
-
- xrayClient = XRayClient.builder()
- .credentialsProvider(DefaultCredentialsProvider.builder().build())
- .region(Region.of(region))
- .build();
-
- System.out.println("☁️ Running OTel X-Ray integration tests against account " + account + " in " + region);
- }
-
- private static String arn(String functionName) {
- return "arn:aws:lambda:" + region + ":" + account + ":function:" + functionNamePrefix + functionName
- + ":$LATEST";
- }
-
- // ─── Test: Simple Steps (Single Invocation) ──────────────────────────
-
- @Test
- void simpleSteps_producesUnifiedTraceInXRay() throws Exception {
- var startTime = Instant.now();
-
- // 1. Invoke the function (use unique input to avoid stale executions)
- var runner = CloudDurableTestRunner.create(
- arn("otel-xray-step-example"), GreetingRequest.class, String.class, lambdaClient);
- var uniqueInput = "XRay-" + System.currentTimeMillis();
- var result = runner.run(new GreetingRequest(uniqueInput));
-
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus(), "Execution failed: " + result);
- assertEquals("HELLO, " + uniqueInput.toUpperCase() + "!", result.getResult());
-
- // 2. Wait for X-Ray ingestion
- Thread.sleep(XRAY_INGESTION_DELAY.toMillis());
-
- // 3. Query X-Ray for the trace, retrying until durable spans appear
- var durableTrace = queryTraceWithDurableSpans(startTime, "otel-xray-step-example", "create-greeting");
-
- // 5. Verify span structure
- var segmentDocuments =
- durableTrace.segments().stream().map(Segment::document).toList();
- var allSegmentText = String.join("\n", segmentDocuments);
-
- // Verify expected span names appear in the trace
- assertTrue(
- allSegmentText.contains("invocation"),
- "Expected invocation span in trace. Segments: " + summarizeSegments(segmentDocuments));
- assertTrue(allSegmentText.contains("create-greeting"), "Expected create-greeting span in trace");
- assertTrue(allSegmentText.contains("transform"), "Expected transform span in trace");
-
- // Verify all segments share the same trace ID (single unified trace)
- var uniqueTraceIds =
- durableTrace.segments().stream().map(seg -> durableTrace.id()).collect(Collectors.toSet());
- assertEquals(1, uniqueTraceIds.size(), "All segments should belong to a single trace");
-
- System.out.println("✅ Simple steps test passed — "
- + durableTrace.segments().size() + " segments in trace " + durableTrace.id());
- }
-
- // ─── Test: Wait + Resume (Multi-Invocation) ─────────────────────────
-
- @Test
- void waitAndResume_producesUnifiedTraceAcrossInvocations() throws Exception {
- var startTime = Instant.now();
-
- // 1. Invoke the function — will suspend on wait, then resume automatically
- var runner = CloudDurableTestRunner.create(
- arn("otel-xray-wait-example"), GreetingRequest.class, String.class, lambdaClient);
- var uniqueInput = "Wait-" + System.currentTimeMillis();
- var result = runner.run(new GreetingRequest(uniqueInput));
-
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus(), "Execution failed: " + result);
- assertTrue(
- result.getResult().contains("Resumed and completed"),
- "Expected result to contain 'Resumed and completed', got: " + result.getResult());
-
- // 2. Wait for X-Ray ingestion (extra time since multi-invocation takes longer)
- Thread.sleep(XRAY_INGESTION_DELAY.plus(Duration.ofSeconds(5)).toMillis());
-
- // 3. Query X-Ray for the trace, retrying until durable spans appear
- var durableTrace = queryTraceWithDurableSpans(startTime, "otel-xray-wait-example", "before-wait");
-
- // 4. Verify multi-invocation trace structure
- var segmentDocuments =
- durableTrace.segments().stream().map(Segment::document).toList();
- var allSegmentText = String.join("\n", segmentDocuments);
-
- // Verify spans from BOTH invocations appear in the same trace
- assertTrue(allSegmentText.contains("before-wait"), "Expected before-wait span from first invocation");
- assertTrue(allSegmentText.contains("after-wait"), "Expected after-wait span from second invocation");
- assertTrue(allSegmentText.contains("pause"), "Expected wait:pause span in trace");
-
- // Verify multiple invocation spans (one per Lambda invocation)
- var invocationCount = countOccurrences(allSegmentText, "invocation");
- assertTrue(
- invocationCount >= 2,
- "Expected at least 2 invocation spans (multi-invocation), got " + invocationCount);
-
- // Critical assertion: all segments under ONE trace (deterministic ID worked)
- assertEquals(
- 1,
- Set.of(durableTrace.id()).size(),
- "All segments should belong to a single trace — deterministic trace ID must work across invocations");
-
- System.out.println(
- "✅ Wait + resume test passed — " + durableTrace.segments().size() + " segments across "
- + invocationCount + " invocations in trace " + durableTrace.id());
- }
-
- // ─── Helpers ─────────────────────────────────────────────────────────
-
- /** Queries X-Ray for traces with retry logic to handle eventual consistency. */
- private List These verify that the OTel plugin doesn't break execution for map, parallel, and nested context scenarios.
- */
-class OtelXRayExamplesTest {
-
- @Test
- void mapExample_executesSuccessfully() {
- var handler = new OtelXRayExamples.MapExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var result = runner.runUntilComplete(new GreetingRequest("test"));
-
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
- assertEquals("Mapped 3 items", result.getResult(String.class));
- }
-
- @Test
- void parallelExample_executesSuccessfully() {
- var handler = new OtelXRayExamples.ParallelExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var result = runner.runUntilComplete(new GreetingRequest("test"));
-
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
- assertTrue(result.getResult(String.class).contains("Parallel completed: 2 branches"));
- }
-
- @Test
- void nestedContextExample_executesSuccessfully() {
- var handler = new OtelXRayExamples.NestedContextExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var result = runner.runUntilComplete(new GreetingRequest("World"));
-
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
- assertEquals("HELLO, WORLD!", result.getResult(String.class));
- }
-}
diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExecutionStepExampleTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExecutionStepExampleTest.java
deleted file mode 100644
index 8febaf3f0..000000000
--- a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExecutionStepExampleTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable.examples.otel;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import software.amazon.lambda.durable.examples.types.GreetingRequest;
-import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
-
-class OtelXRayExecutionStepExampleTest {
-
- @BeforeEach
- void setUp() {
- OtelXRayExampleTestSupport.installGlobalOpenTelemetry();
- }
-
- @AfterEach
- void tearDown() {
- OtelXRayExampleTestSupport.resetGlobalOpenTelemetry();
- }
-
- @Test
- void testSimpleSteps_succeeds() {
- var handler = new OtelXRayExecutionStepExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var result = runner.runUntilComplete(new GreetingRequest("Alice"));
-
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
- assertEquals("HELLO, ALICE!", result.getResult(String.class));
- }
-
- @Test
- void testReplay_returnsSameResult() {
- var handler = new OtelXRayExecutionStepExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var input = new GreetingRequest("Bob");
- var result1 = runner.runUntilComplete(input);
- var result2 = runner.runUntilComplete(input);
-
- assertEquals(result1.getResult(String.class), result2.getResult(String.class));
- }
-}
diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExecutionWaitExampleTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExecutionWaitExampleTest.java
deleted file mode 100644
index c314c241f..000000000
--- a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayExecutionWaitExampleTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable.examples.otel;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import software.amazon.lambda.durable.examples.types.GreetingRequest;
-import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
-
-class OtelXRayExecutionWaitExampleTest {
-
- @BeforeEach
- void setUp() {
- OtelXRayExampleTestSupport.installGlobalOpenTelemetry();
- }
-
- @AfterEach
- void tearDown() {
- OtelXRayExampleTestSupport.resetGlobalOpenTelemetry();
- }
-
- @Test
- void testFirstInvocation_suspendsOnWait() {
- var handler = new OtelXRayExecutionWaitExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var result = runner.run(new GreetingRequest("Alice"));
-
- assertEquals(ExecutionStatus.PENDING, result.getStatus());
- }
-
- @Test
- void testFullExecution_completesAfterWait() {
- var handler = new OtelXRayExecutionWaitExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var result = runner.runUntilComplete(new GreetingRequest("Alice"));
-
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
- assertTrue(
- result.getResult(String.class).contains("Resumed and completed"),
- "Expected result to contain 'Resumed and completed', got: " + result.getResult(String.class));
- }
-}
diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExampleTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExampleTest.java
deleted file mode 100644
index dffe99080..000000000
--- a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExampleTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable.examples.otel;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import software.amazon.lambda.durable.examples.types.GreetingRequest;
-import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
-
-class OtelXRayStepExampleTest {
-
- @BeforeEach
- void setUp() {
- OtelXRayExampleTestSupport.installGlobalOpenTelemetry();
- }
-
- @AfterEach
- void tearDown() {
- OtelXRayExampleTestSupport.resetGlobalOpenTelemetry();
- }
-
- @Test
- void testSimpleSteps_succeeds() {
- var handler = new OtelXRayStepExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var result = runner.run(new GreetingRequest("Alice"));
-
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
- assertEquals("HELLO, ALICE!", result.getResult(String.class));
- }
-
- @Test
- void testReplay_returnsSameResult() {
- var handler = new OtelXRayStepExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var input = new GreetingRequest("Bob");
- var result1 = runner.run(input);
- var result2 = runner.run(input);
-
- assertEquals(result1.getResult(String.class), result2.getResult(String.class));
- }
-
- @Test
- void testDefaultName() {
- var handler = new OtelXRayStepExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var result = runner.run(new GreetingRequest());
-
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
- assertEquals("HELLO, WORLD!", result.getResult(String.class));
- }
-}
diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExampleTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExampleTest.java
deleted file mode 100644
index ac42d31d9..000000000
--- a/examples/src/test/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExampleTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable.examples.otel;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import software.amazon.lambda.durable.examples.types.GreetingRequest;
-import software.amazon.lambda.durable.model.ExecutionStatus;
-import software.amazon.lambda.durable.testing.LocalDurableTestRunner;
-
-class OtelXRayWaitExampleTest {
-
- @BeforeEach
- void setUp() {
- OtelXRayExampleTestSupport.installGlobalOpenTelemetry();
- }
-
- @AfterEach
- void tearDown() {
- OtelXRayExampleTestSupport.resetGlobalOpenTelemetry();
- }
-
- @Test
- void testFirstInvocation_suspendsOnWait() {
- var handler = new OtelXRayWaitExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var result = runner.run(new GreetingRequest("Alice"));
-
- // First invocation hits the wait and suspends
- assertEquals(ExecutionStatus.PENDING, result.getStatus());
- }
-
- @Test
- void testFullExecution_completesAfterWait() {
- var handler = new OtelXRayWaitExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var result = runner.runUntilComplete(new GreetingRequest("Alice"));
-
- assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
- assertTrue(
- result.getResult(String.class).contains("Resumed and completed"),
- "Expected result to contain 'Resumed and completed', got: " + result.getResult(String.class));
- }
-
- @Test
- void testReplay_returnsSameResult() {
- var handler = new OtelXRayWaitExample();
- var runner = LocalDurableTestRunner.create(GreetingRequest.class, handler);
-
- var input = new GreetingRequest("Bob");
- var result1 = runner.runUntilComplete(input);
- var result2 = runner.runUntilComplete(input);
-
- assertEquals(result1.getResult(String.class), result2.getResult(String.class));
- }
-}
diff --git a/otel-plugin/README.md b/otel-plugin/README.md
index 8e781e217..dcf5d1b4d 100644
--- a/otel-plugin/README.md
+++ b/otel-plugin/README.md
@@ -1,14 +1,16 @@
# AWS Durable Execution SDK - OpenTelemetry Plugin
-OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK for Java. Emits distributed traces that correlate across multiple Lambda invocations of a single durable execution, producing deterministic span and trace IDs so that spans from different invocations are stitched into a single coherent trace.
+OpenTelemetry instrumentation plugin for the AWS Lambda Durable Execution SDK for Java. Emits a deterministic Workflow trace for durable-execution correlation while keeping each Invocation span in the ambient Lambda trace.
## Features
-- **Deterministic Trace IDs**: All invocations of the same durable execution share a single trace, derived from the X-Ray trace header or execution ARN
+- **Deterministic Workflow Traces**: Workflow trace IDs are derived from the execution start time and ARN; stable span IDs are derived from the ARN
+- **Ambient Invocation Traces**: Invocation spans inherit the active Lambda/X-Ray context, or receive a fresh provider-generated root trace ID
+- **Scoped ID Generation**: Unrelated instrumentation scopes retain their provider's normal root trace ID generation
- **Span-per-Operation**: Each durable operation (step, wait, map, etc.) gets its own span with accurate timing
- **Attempt Spans**: Each user function execution (step attempt, child context run) gets a span, including retries
- **Log Correlation**: Injects `trace_id`, `span_id`, and `traceSampled` into SLF4J MDC for end-to-end observability
-- **ADOT Java Agent Integration**: `new InvocationOtelPlugin()` uses the ADOT Java agent's global provider with no handler-side OpenTelemetry initialization
+- **ADOT Java Agent Integration**: `new InvocationOtelPlugin()` late-binds the ADOT Java agent's global provider with no handler-side OpenTelemetry initialization
- **Lambda Layer Discovery**: `DURABLE_EXECUTION_PLUGINS` loads either OTel plugin from a JAR under a layer's `java/lib` directory
## Installation
@@ -48,7 +50,7 @@ If you configure your own `SdkTracerProviderBuilder`, add the OpenTelemetry SDK
### 1. ADOT Lambda Layer
-This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. The `new InvocationOtelPlugin()` constructor uses the global provider initialized by the ADOT Java agent with deterministic span ID generation installed through the plugin's `AutoConfigurationCustomizerProvider` SPI.
+This plugin uses the [AWS Distro for OpenTelemetry (ADOT) Lambda layer](https://aws-otel.github.io/docs/getting-started/lambda) for trace export. The `new InvocationOtelPlugin()` constructor resolves the global provider initialized by the ADOT Java agent at invocation start, with deterministic span ID generation installed through the plugin's `AutoConfigurationCustomizerProvider` SPI. If the provider is not ready, the plugin emits no telemetry for that invocation and retries provider resolution on the next invocation.
The layer ARN follows the format:
@@ -90,7 +92,7 @@ Build the plugin layer ZIP with the OTel plugin JAR at `java/lib/aws-durable-exe
### 2. AWS X-Ray Active Tracing
-Enable active tracing on your Lambda function so the `_X_AMZN_TRACE_ID` environment variable is populated at invocation time. The plugin uses this header to derive deterministic trace IDs that remain consistent across all invocations of the same durable execution.
+Enable active tracing on your Lambda function so the `_X_AMZN_TRACE_ID` environment variable is populated at invocation time. The plugin uses this header to parent Invocation spans to the ambient Lambda/X-Ray trace. The Workflow trace remains independent and deterministic.
**AWS Console:** Lambda > Configuration > Monitoring and operations tools > Active tracing > Enable
@@ -155,23 +157,29 @@ The function's execution role needs the `AWSXRayDaemonWriteAccess` managed polic
## Trace Structure
-The plugin creates spans at four levels:
+With `InvocationOtelPlugin`, the plugin creates two correlated traces:
```
-Workflow (deterministic ID, exported once on terminal invocation)
-Invocation
-├── fetch-data
-│ └── fetch-data attempt 1
-├── cool-down
-└── process
- └── process attempt 1
+Workflow trace:
+Workflow (deterministic trace/span IDs, exported once)
+
+Ambient invocation trace:
+Lambda/X-Ray parent
+└── Invocation
+ ├── fetch-data
+ │ └── fetch-data attempt 1
+ ├── cool-down
+ └── process
+ └── process attempt 1
```
-- **Workflow span** — one logical span per durable execution with a deterministic ID derived from the ARN. Exported only on the terminal invocation (SUCCEEDED/FAILED). Serves as a correlation anchor across invocations.
-- **Invocation span** — one per Lambda invocation
+- **Workflow span** — one logical root per durable execution with a deterministic, X-Ray-compatible trace ID derived from the execution start time and ARN, plus a stable span ID derived from the ARN. Exported only on the terminal invocation (SUCCEEDED/FAILED).
+- **Invocation span** — one per Lambda invocation, parented to ambient context when available
- **Operation span** — one per durable operation, named after your step/wait names
- **Attempt span** — one per user function execution (retries produce additional attempt spans)
+Operation and attempt spans link to the Workflow span. `ExecutionOtelPlugin` reverses that relationship: operations are children of Workflow and link to the current Invocation span.
+
## Span Attributes
### Invocation Span
@@ -255,8 +263,9 @@ new InvocationOtelPlugin(
### ExecutionOtelPlugin
-The `ExecutionOtelPlugin` renders the Workflow span as the trace root with operations as siblings of the invocation
-span. It takes the same `(SdkTracerProviderBuilder, OtelPluginConfig)` constructor:
+The `ExecutionOtelPlugin` renders the Workflow span as the durable trace root with operations beneath it. Invocation
+spans remain in the ambient Lambda trace, and operations link to the Invocation that ran them. It takes the same
+`(SdkTracerProviderBuilder, OtelPluginConfig)` constructor:
```java
// Default: ADOT Java agent global provider, X-Ray context extraction, MDC enabled
@@ -284,8 +293,9 @@ new ExecutionOtelPlugin(
| `instrumentationName(...)` | Instrumentation scope name registered with the tracer | `"aws-durable-execution-sdk-java"` |
> The `tracerProviderBuilder` argument is not used by the no-arg `new InvocationOtelPlugin()` /
-> `new ExecutionOtelPlugin()` constructors; those use the ADOT Java agent's global provider. A `null` passed to any
-> `OtelPluginConfig` builder setter falls back to that option's default.
+> `new ExecutionOtelPlugin()` constructors; those resolve the ADOT Java agent's global provider at invocation start.
+> If it is not ready, all telemetry is disabled for that invocation and resolution is retried on the next invocation.
+> A `null` passed to any `OtelPluginConfig` builder setter falls back to that option's default.
## Known Limitations
@@ -295,7 +305,7 @@ The plugin's spans do not appear as nested subsegments of the Lambda platform se
### Workflow Span
-The Workflow span appears as a separate root segment in the X-Ray trace because it uses `setNoParent()` with a deterministic span ID. This is expected — it serves as a correlation anchor across invocations.
+The Workflow span appears in a separate deterministic trace because it uses `setNoParent()`. Invocation spans remain in the ambient Lambda/X-Ray trace. Links correlate durable operations with the other trace.
## Verification
@@ -304,10 +314,10 @@ After deploying your function with the plugin configured:
1. **Invoke your durable function** — trigger at least one execution that includes multiple steps or a wait/resume cycle.
2. **Check CloudWatch console** — Navigate to CloudWatch > Traces. Enable "Group by nodes" to see:
- - A Workflow span covering the entire execution
- - An Invocation span per Lambda invocation
+ - A deterministic Workflow trace covering the entire execution
+ - Ambient Lambda traces containing one Invocation span per Lambda invocation
- Child spans for each durable operation (named after your step names)
- - All invocations of the same execution grouped under one trace ID
+ - Links between durable Workflow/operation spans and Invocation spans
3. **Check log correlation** — Verify that the Logs section at the bottom of the trace view shows both platform logs and application logs correlated with the trace.
@@ -316,7 +326,7 @@ After deploying your function with the plugin configured:
| Symptom | Likely Cause |
|---------|-------------|
| No traces appear | ADOT layer not added, or `AWS_LAMBDA_EXEC_WRAPPER` not set |
-| Traces appear but are fragmented | X-Ray active tracing not enabled on the Lambda function |
+| Invocation spans are not parented to Lambda | X-Ray active tracing not enabled on the Lambda function |
| Missing spans for some operations | Sampling is configured below 1.0 |
| `_X_AMZN_TRACE_ID` not populated | X-Ray active tracing not enabled |
| Plugin spans missing but Lambda/runtime spans appear | Plugin jar not configured in `OTEL_JAVAAGENT_EXTENSIONS` |
diff --git a/otel-plugin/pom.xml b/otel-plugin/pom.xml
index f6e95fbd9..42b00e8e0 100644
--- a/otel-plugin/pom.xml
+++ b/otel-plugin/pom.xml
@@ -33,19 +33,12 @@
- *
- *
- *
- * invocation
- * ├── create-greeting
- * │ └── create-greeting attempt 1
- * └── transform
- * └── transform attempt 1
- *
- */
-@ExampleTemplate(tracing = true, javaAgent = true)
-public class OtelXRayStepExample extends DurableHandler
- *
- *
- *
- *
- *
- *
- * Trace (single trace ID across both invocations)
- * ├── invocation (invocation 1)
- * │ ├── before-wait
- * │ │ └── before-wait attempt 1
- * │ └── pause (ended as PENDING)
- * └── invocation (invocation 2)
- * ├── pause (completed)
- * └── after-wait
- * └── after-wait attempt 1
- *
- */
-@ExampleTemplate(tracing = true, javaAgent = true)
-public class OtelXRayWaitExample extends DurableHandler
- *
- *
- *
- *
- *
- *
Implementations read trace context from various sources (X-Ray trace header, W3C traceparent, etc.) and return an * {@link ExtractedContext} containing the trace ID and optional parent span ID. * - *
Called once per invocation in {@code onInvocationStart} to establish the parent trace context. + *
Plugins use a valid ambient OpenTelemetry span as the invocation parent when one is available. This extractor is + * consulted only when no ambient span context is available, providing fallback propagation context from the runtime + * environment. */ @FunctionalInterface public interface ContextExtractor { /** - * Extracts trace context from the runtime environment. + * Extracts fallback trace context from the runtime environment. * * @return the extracted context, or {@code null} if no context is available */ diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java index ee15f2c14..709afcbad 100644 --- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java +++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/DeterministicIdGenerator.java @@ -2,42 +2,60 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.otel; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.sdk.trace.IdGenerator; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.concurrent.atomic.AtomicReference; +import java.time.Instant; /** * Generates deterministic trace and span IDs for durable execution observability. * - *
Trace ID resolution order: + *
The durable plugins use short-lived ID overrides around their own {@link SpanBuilder#startSpan()} calls. Outside + * those scopes, generation delegates to the fallback generator so unrelated instrumentation keeps normal root trace ID + * generation. Scoped values are also bridged through thread-keyed system properties because the application plugin and + * Java-agent extension may load this class in different class loaders. * - *
Span IDs for operations are deterministic (derived from execution ARN + operation ID), ensuring the same operation - * produces the same span across invocations. When no pending operation ID is set, falls back to random generation. + *
The existing setter methods remain available for callers that use this class directly. Plugin code does not use
+ * that persistent mode.
*/
public class DeterministicIdGenerator implements IdGenerator {
- private static final IdGenerator RANDOM = IdGenerator.random();
private static final String PROPERTY_PREFIX = "software.amazon.lambda.durable.otel.";
- private static final String EXTRACTED_TRACE_ID_PROPERTY = PROPERTY_PREFIX + "extractedTraceId";
- private static final String DURABLE_EXECUTION_ARN_PROPERTY = PROPERTY_PREFIX + "durableExecutionArn";
- private static final String PENDING_SPAN_OPERATION_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "pendingSpanOperationId.";
- private static final String PENDING_RAW_SPAN_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "pendingRawSpanId.";
+ private static final String SCOPED_TRACE_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "scopedTraceId.";
+ private static final String SCOPED_SPAN_ID_PROPERTY_PREFIX = PROPERTY_PREFIX + "scopedSpanId.";
- private final AtomicReference Trace ID resolution matches {@link InvocationOtelPlugin}: the X-Ray trace ID from {@code _X_AMZN_TRACE_ID} when
- * available (the backend propagates the same Root to all invocations, unifying the trace), else a deterministic trace
- * ID derived from the execution ARN.
+ * The Workflow trace ID is derived from the execution start time and ARN, and is independent of the ambient
+ * Lambda/X-Ray trace. Invocation spans inherit the active ambient context, or extracted upstream context as a fallback.
+ * When using {@link #ExecutionOtelPlugin()}, the plugin resolves the global provider at invocation start. If the
+ * OpenTelemetry Java agent is not initialized yet, telemetry is disabled for that entire invocation and provider
+ * resolution is retried on the next invocation.
*
* Status mapping (parity with the Python/JS references):
*
@@ -79,18 +81,20 @@ public class ExecutionOtelPlugin implements DurableExecutionPlugin {
private static final Logger logger = LoggerFactory.getLogger(ExecutionOtelPlugin.class);
- private final SdkTracerProvider sdkTracerProvider;
- private final Tracer tracer;
+ private volatile SdkTracerProvider sdkTracerProvider;
+ private volatile Tracer tracer;
private final DeterministicIdGenerator idGenerator;
private final ContextExtractor contextExtractor;
private final boolean enableMdc;
private final String workflowSpanName;
- private final ProviderSource providerSource;
+ private final String instrumentationName;
// Per-invocation state
+ private volatile boolean tracingEnabled;
private volatile Span workflowSpan;
private volatile Span invocationSpan;
private volatile String durableExecutionArn;
+ private volatile String workflowTraceId;
// Thread-safe storage for operation spans (keyed by operationId) — open spans that need ending
private final ConcurrentHashMap Uses the provided tracer provider builder. For ADOT Java agent usage, prefer {@link #ExecutionOtelPlugin()}
* with the plugin jar configured through {@code OTEL_JAVAAGENT_EXTENSIONS}.
*
- * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden)
+ * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
*/
public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) {
this(tracerProviderBuilder, OtelPluginConfig.defaults());
@@ -118,8 +122,8 @@ public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) {
/**
* Creates a Workflow-rooted OTel plugin with default settings: X-Ray context extraction and MDC enabled.
*
- * Uses {@code GlobalOpenTelemetry} directly and assumes deterministic ID generation was installed by
- * {@code OtelPluginAutoConfigurationCustomizerProvider}.
+ * Resolves {@code GlobalOpenTelemetry} at invocation start. If the ADOT Java agent has not initialized it yet,
+ * telemetry is disabled for that invocation and resolution is retried on the next invocation.
*/
public ExecutionOtelPlugin() {
this(OtelPluginConfig.defaults());
@@ -138,67 +142,51 @@ public ExecutionOtelPlugin() {
* OtelPluginConfig.builder().enableMdc(false).workflowSpanName("Workflow").build());
* }
*
- * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden)
+ * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
* @param config the plugin configuration
*/
public ExecutionOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) {
- this.idGenerator = new DeterministicIdGenerator();
+ this.idGenerator = DeterministicIdGenerator.installOn(tracerProviderBuilder);
- this.sdkTracerProvider =
- tracerProviderBuilder.setIdGenerator(idGenerator).build();
+ this.sdkTracerProvider = tracerProviderBuilder.build();
this.tracer = sdkTracerProvider.get(config.instrumentationName());
this.contextExtractor = config.contextExtractor();
this.enableMdc = config.enableMdc();
this.workflowSpanName = config.workflowSpanName();
- this.providerSource = ProviderSource.EXPLICIT;
+ this.instrumentationName = config.instrumentationName();
}
/**
* Creates a Workflow-rooted OTel plugin from configuration alone (no caller-supplied tracer provider builder).
*
- * The provider is taken from {@link OtelPluginConfig#providerSource()}: {@link ProviderSource#GLOBAL} uses the
- * ADOT/global provider, otherwise the default {@link ProviderSource#AUTO_OTLP} builds a plugin-owned OTLP/HTTP
- * provider (matching the JavaScript and Python SDK plugins). {@link ProviderSource#EXPLICIT} is rejected here —
- * supply a {@code SdkTracerProviderBuilder} via the two-arg constructor for that.
+ * The config-only constructor uses the ADOT/global provider. Supply a {@code SdkTracerProviderBuilder} via the
+ * two-arg constructor for an application-owned provider.
*
* @param config the plugin configuration
- * @throws IllegalArgumentException if {@code config.providerSource()} is {@link ProviderSource#EXPLICIT}
*/
public ExecutionOtelPlugin(OtelPluginConfig config) {
this.contextExtractor = config.contextExtractor();
this.enableMdc = config.enableMdc();
this.workflowSpanName = config.workflowSpanName();
-
- var setup = OtelPluginSupport.resolveConfiguredProvider(config, "ExecutionOtelPlugin");
- this.providerSource = setup.source();
- this.idGenerator = setup.idGenerator();
- this.sdkTracerProvider = setup.sdkTracerProvider();
- this.tracer = setup.tracer();
- }
-
- /** The tier that produced this plugin's tracer provider. */
- public ProviderSource providerSource() {
- return providerSource;
+ this.instrumentationName = config.instrumentationName();
+ this.idGenerator = OtelPluginSupport.createDefaultIdGenerator();
}
// ─── Invocation hooks ────────────────────────────────────────────────
@Override
public void onInvocationStart(InvocationInfo info) {
- this.durableExecutionArn = info.durableExecutionArn();
+ tracingEnabled = false;
+ if (!bindTracer()) {
+ return;
+ }
- // Set execution ARN for deterministic span/trace ID generation
- idGenerator.setDurableExecutionArn(info.durableExecutionArn());
+ this.durableExecutionArn = info.durableExecutionArn();
- // Extract trace context from the environment (X-Ray header), falling back to the ambient OTel span.
- var extractedContext = contextExtractor.extract();
- if (extractedContext == null) {
- extractedContext = extractCurrentSpanContext();
- }
- if (extractedContext != null) {
- idGenerator.setExtractedTraceId(extractedContext.traceId());
- } else {
- idGenerator.setExtractedTraceId(null);
+ // Prefer the active Java-agent span, then fall back to explicitly extracted upstream context.
+ var invocationParent = extractCurrentSpanContext();
+ if (invocationParent == null) {
+ invocationParent = contextExtractor.extract();
}
// Workflow root span — deterministic span ID from the ARN, no parent. Recreated every invocation with the
@@ -209,14 +197,16 @@ public void onInvocationStart(InvocationInfo info) {
.setNoParent()
.setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn())
.setStartTimestamp(info.executionStartTime() != null ? info.executionStartTime() : Instant.now());
- idGenerator.setNextSpanId(idGenerator.generateWorkflowSpanId());
- workflowSpan = workflowSpanBuilder.startSpan();
+ workflowTraceId =
+ idGenerator.generateTraceIdForExecution(info.durableExecutionArn(), info.executionStartTime());
+ var workflowSpanId = idGenerator.generateWorkflowSpanId(info.durableExecutionArn());
+ workflowSpan = idGenerator.startSpan(workflowSpanBuilder, workflowTraceId, workflowSpanId);
Context parentContext;
- if (extractedContext != null && extractedContext.parentSpanId() != null) {
+ if (invocationParent != null && invocationParent.parentSpanId() != null) {
var parentSpanContext = SpanContext.createFromRemoteParent(
- extractedContext.traceId(),
- extractedContext.parentSpanId(),
+ invocationParent.traceId(),
+ invocationParent.parentSpanId(),
TraceFlags.getSampled(),
TraceState.getDefault());
parentContext = Context.root().with(Span.wrap(parentSpanContext));
@@ -239,13 +229,20 @@ public void onInvocationStart(InvocationInfo info) {
// Inject MDC on the handler thread so handler-level logs (between steps) have trace context.
if (enableMdc) {
- var traceId = idGenerator.generateTraceId();
- MDC.put(MdcSpanEnricher.MDC_TRACE_ID, traceId);
+ MDC.put(
+ MdcSpanEnricher.MDC_TRACE_ID,
+ invocationSpan.getSpanContext().getTraceId());
}
+ tracingEnabled = true;
}
@Override
public void onInvocationEnd(InvocationEndInfo info) {
+ if (!tracingEnabled) {
+ return;
+ }
+ tracingEnabled = false;
+
// Clear invocation-level MDC
if (enableMdc) {
MdcSpanEnricher.clear();
@@ -312,14 +309,11 @@ public void onInvocationEnd(InvocationEndInfo info) {
@Override
public void onOperationStart(OperationInfo info) {
+ if (!tracingEnabled) return;
if (info.id() == null) return;
var parentContext = resolveParentContext(info.parentId());
- // Always use a deterministic span ID keyed by operation ID (regardless of replay) so a suspended-then-resumed
- // operation stitches into a single logical span across invocations.
- idGenerator.setNextSpanOperationId(info.id());
-
var spanBuilder = tracer.spanBuilder(spanName(info.type(), info.subType(), info.name()))
.setParent(parentContext)
.setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn)
@@ -337,7 +331,8 @@ public void onOperationStart(OperationInfo info) {
spanBuilder.setAttribute(DURABLE_OPERATION_SUBTYPE, info.subType());
}
- var span = spanBuilder.startSpan();
+ var operationSpanId = idGenerator.generateSpanIdForOperation(durableExecutionArn, info.id());
+ var span = idGenerator.startSpan(spanBuilder, null, operationSpanId);
// Store the open span — will be ended in onOperationEnd or onInvocationEnd
operationSpans.put(info.id(), span);
@@ -346,6 +341,7 @@ public void onOperationStart(OperationInfo info) {
@Override
public void onOperationEnd(OperationEndInfo info) {
+ if (!tracingEnabled) return;
if (info.id() == null) return;
var span = operationSpans.remove(info.id());
@@ -376,7 +372,6 @@ public void onOperationEnd(OperationEndInfo info) {
// now, using its deterministic span ID (stable across the execution), plus a link to the invocation
// that completed it.
operationContexts.remove(info.id());
- idGenerator.setNextSpanOperationId(info.id());
var parentContext = resolveParentContext(info.parentId());
@@ -397,7 +392,8 @@ public void onOperationEnd(OperationEndInfo info) {
spanBuilder.setAttribute(DURABLE_OPERATION_SUBTYPE, info.subType());
}
- var continuationSpan = spanBuilder.startSpan();
+ var operationSpanId = idGenerator.generateSpanIdForOperation(durableExecutionArn, info.id());
+ var continuationSpan = idGenerator.startSpan(spanBuilder, null, operationSpanId);
if (info.status() != null) {
continuationSpan.setAttribute(DURABLE_OPERATION_STATUS, info.status());
@@ -424,6 +420,8 @@ public void onOperationEnd(OperationEndInfo info) {
@Override
public void onUserFunctionStart(UserFunctionStartInfo info) {
+ if (!tracingEnabled) return;
+
// Skip attempt spans for CONTEXT operations — they are a scoping construct, not a retriable unit of work. Still
// make the operation span current so auto-instrumented calls become children.
if ("CONTEXT".equals(info.type())) {
@@ -479,6 +477,8 @@ public void onUserFunctionStart(UserFunctionStartInfo info) {
@Override
public void onUserFunctionEnd(UserFunctionEndInfo info) {
+ if (!tracingEnabled) return;
+
var key = attemptKey(info.id(), info.attempt());
// Close scope first (must happen on same thread as makeCurrent)
@@ -510,6 +510,24 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) {
// ─── Helpers ─────────────────────────────────────────────────────────
+ private boolean bindTracer() {
+ if (tracer != null) {
+ return true;
+ }
+ synchronized (this) {
+ if (tracer != null) {
+ return true;
+ }
+ var setup = OtelPluginSupport.tryResolveGlobalProvider(instrumentationName, "ExecutionOtelPlugin");
+ if (setup == null) {
+ return false;
+ }
+ sdkTracerProvider = setup.sdkTracerProvider();
+ tracer = setup.tracer();
+ return true;
+ }
+ }
+
private void applyInvocationStatus(Span span, InvocationEndInfo info) {
// Invocation span status mapping:
// SUCCEEDED, PENDING -> OK
@@ -556,10 +574,9 @@ private Context resolveParentContext(String parentId) {
return Context.current().with(Span.wrap(parentSpanContext));
}
// Parent operation from a prior invocation — create a non-recording placeholder with its deterministic ID.
- var deterministicParentSpanId = idGenerator.generateSpanIdForOperation(parentId);
- var traceId = idGenerator.generateTraceId();
+ var deterministicParentSpanId = idGenerator.generateSpanIdForOperation(durableExecutionArn, parentId);
var placeholderContext = SpanContext.create(
- traceId, deterministicParentSpanId, TraceFlags.getSampled(), TraceState.getDefault());
+ workflowTraceId, deterministicParentSpanId, TraceFlags.getSampled(), TraceState.getDefault());
return Context.current().with(Span.wrap(placeholderContext));
}
// No parent operation — hang off the Workflow root span.
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExtractedContext.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExtractedContext.java
index 6aaacd2e9..d381732cb 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExtractedContext.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ExtractedContext.java
@@ -5,9 +5,8 @@
/**
* Trace context extracted from the Lambda runtime environment.
*
- * Contains the trace ID (always present) and an optional parent span ID. When the durable execution backend
- * propagates the same X-Ray Root across all invocations, the trace ID will be consistent, enabling spans from different
- * invocations to be stitched into a single trace.
+ * Contains the trace ID (always present) and an optional parent span ID used to parent an Invocation span to ambient
+ * Lambda/X-Ray context.
*
* @param traceId 32-character lowercase hex trace ID (OTel format, no dashes)
* @param parentSpanId 16-character lowercase hex parent span ID (may be null if no parent available)
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java
index 36e741f37..b51a4ce85 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/InvocationOtelPlugin.java
@@ -50,22 +50,16 @@
*
* Trace ID resolution:
- *
- * The Workflow trace ID is derived from the execution start time and ARN, and is independent of the ambient
+ * Lambda/X-Ray trace. Invocation spans inherit the active ambient context, or extracted upstream context as a fallback.
*
* Requires the ADOT Lambda Layer for trace export. Configure with:
*
@@ -74,16 +68,16 @@
* When using {@link #InvocationOtelPlugin()}, the plugin requires
- * {@code OtelPluginAutoConfigurationCustomizerProvider} to have been installed by the OpenTelemetry Java agent and uses
- * the global provider directly.
+ * When using {@link #InvocationOtelPlugin()}, the plugin resolves the global provider at invocation start. If the
+ * OpenTelemetry Java agent is not initialized yet, telemetry is disabled for that entire invocation and provider
+ * resolution is retried on the next invocation.
*
* X-Ray console limitation: In the X-Ray "Segments Timeline" ungrouped view, the plugin's spans (Invocation,
* operation, attempt) do not appear as nested subsegments of the Lambda platform segment. This is a known limitation of
* the OTLP-to-X-Ray conversion: the ADOT collector cannot attach OTLP-exported spans as subsegments of the Lambda
* service's native X-Ray segment because that segment is created outside the OTLP pipeline. Use the "Group by nodes"
- * view to see the full span hierarchy correctly — it stitches all spans together by trace ID and parent-child
- * relationships regardless of segment boundaries.
+ * view to inspect parent-child relationships within the ambient Invocation trace and the links to the independent
+ * Workflow trace.
*
* Thread-safe: uses {@link ConcurrentHashMap} for span/scope storage since the SDK runs user code on multiple
* threads.
@@ -92,15 +86,16 @@ public class InvocationOtelPlugin implements DurableExecutionPlugin {
private static final Logger logger = LoggerFactory.getLogger(InvocationOtelPlugin.class);
- private final SdkTracerProvider sdkTracerProvider;
- private final Tracer tracer;
+ private volatile SdkTracerProvider sdkTracerProvider;
+ private volatile Tracer tracer;
private final DeterministicIdGenerator idGenerator;
private final ContextExtractor contextExtractor;
private final boolean enableMdc;
private final String workflowSpanName;
- private final ProviderSource providerSource;
+ private final String instrumentationName;
// Per-invocation state
+ private volatile boolean tracingEnabled;
private volatile Span workflowSpan;
private volatile Span invocationSpan;
private volatile String durableExecutionArn;
@@ -133,7 +128,7 @@ public class InvocationOtelPlugin implements DurableExecutionPlugin {
* SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(exporter)));
* }
*
- * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden)
+ * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
*/
public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) {
this(tracerProviderBuilder, OtelPluginConfig.defaults());
@@ -142,8 +137,8 @@ public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder) {
/**
* Creates an OTel plugin with default settings: X-Ray context extraction and MDC enabled.
*
- * Uses {@code GlobalOpenTelemetry} directly and assumes deterministic ID generation was installed by
- * {@code OtelPluginAutoConfigurationCustomizerProvider}.
+ * Resolves {@code GlobalOpenTelemetry} at invocation start. If the ADOT Java agent has not initialized it yet,
+ * telemetry is disabled for that invocation and resolution is retried on the next invocation.
*/
public InvocationOtelPlugin() {
this(OtelPluginConfig.defaults());
@@ -162,71 +157,52 @@ public InvocationOtelPlugin() {
* OtelPluginConfig.builder().enableMdc(false).workflowSpanName("Workflow").build());
* }
*
- * @param tracerProviderBuilder the tracer provider builder (ID generator will be overridden)
+ * @param tracerProviderBuilder the tracer provider builder (its ID generator will be wrapped)
* @param config the plugin configuration
*/
public InvocationOtelPlugin(SdkTracerProviderBuilder tracerProviderBuilder, OtelPluginConfig config) {
- this.idGenerator = new DeterministicIdGenerator();
+ this.idGenerator = DeterministicIdGenerator.installOn(tracerProviderBuilder);
- this.sdkTracerProvider =
- tracerProviderBuilder.setIdGenerator(idGenerator).build();
+ this.sdkTracerProvider = tracerProviderBuilder.build();
this.tracer = sdkTracerProvider.get(config.instrumentationName());
this.contextExtractor = config.contextExtractor();
this.enableMdc = config.enableMdc();
this.workflowSpanName = config.workflowSpanName();
- this.providerSource = ProviderSource.EXPLICIT;
+ this.instrumentationName = config.instrumentationName();
}
/**
* Creates an OTel plugin from configuration alone (no caller-supplied tracer provider builder).
*
- * The provider is taken from {@link OtelPluginConfig#providerSource()}: {@link ProviderSource#GLOBAL} uses the
- * ADOT/global provider, otherwise the default {@link ProviderSource#AUTO_OTLP} builds a plugin-owned OTLP/HTTP
- * provider (matching the JavaScript and Python SDK plugins). {@link ProviderSource#EXPLICIT} is rejected here —
- * supply a {@code SdkTracerProviderBuilder} via the two-arg constructor for that.
+ * The config-only constructor uses the ADOT/global provider. Supply a {@code SdkTracerProviderBuilder} via the
+ * two-arg constructor for an application-owned provider.
*
* @param config the plugin configuration
- * @throws IllegalArgumentException if {@code config.providerSource()} is {@link ProviderSource#EXPLICIT}
*/
public InvocationOtelPlugin(OtelPluginConfig config) {
this.contextExtractor = config.contextExtractor();
this.enableMdc = config.enableMdc();
this.workflowSpanName = config.workflowSpanName();
-
- var setup = OtelPluginSupport.resolveConfiguredProvider(config, "InvocationOtelPlugin");
- this.providerSource = setup.source();
- this.idGenerator = setup.idGenerator();
- this.sdkTracerProvider = setup.sdkTracerProvider();
- this.tracer = setup.tracer();
- }
-
- /** The tier that produced this plugin's tracer provider. */
- public ProviderSource providerSource() {
- return providerSource;
+ this.instrumentationName = config.instrumentationName();
+ this.idGenerator = OtelPluginSupport.createDefaultIdGenerator();
}
// ─── Invocation hooks ────────────────────────────────────────────────
@Override
public void onInvocationStart(InvocationInfo info) {
- this.durableExecutionArn = info.durableExecutionArn();
-
- // Set execution ARN for deterministic span ID generation
- idGenerator.setDurableExecutionArn(info.durableExecutionArn());
-
- // Extract trace context from environment (X-Ray header)
- var extractedContext = contextExtractor.extract();
- if (extractedContext == null) {
- extractedContext = extractCurrentSpanContext();
+ tracingEnabled = false;
+ if (!bindTracer()) {
+ return;
}
- if (extractedContext != null) {
- // Use the X-Ray trace ID — backend propagates same Root across all invocations
- idGenerator.setExtractedTraceId(extractedContext.traceId());
- } else {
- idGenerator.setExtractedTraceId(null);
+ this.durableExecutionArn = info.durableExecutionArn();
+
+ // Prefer the active Java-agent span, then fall back to explicitly extracted upstream context.
+ var invocationParent = extractCurrentSpanContext();
+ if (invocationParent == null) {
+ invocationParent = contextExtractor.extract();
}
- // If no extracted context, idGenerator falls back to ARN-derived trace ID
// Workflow root span — one logical span per durable execution, created unconditionally (independent of the
// X-Ray parent below). Deterministic span ID from the ARN so it is the same across invocations; exported once,
@@ -237,17 +213,19 @@ public void onInvocationStart(InvocationInfo info) {
.setNoParent()
.setAttribute(DURABLE_EXECUTION_ARN, info.durableExecutionArn())
.setStartTimestamp(info.executionStartTime() != null ? info.executionStartTime() : Instant.now());
- idGenerator.setNextSpanId(idGenerator.generateWorkflowSpanId());
- workflowSpan = workflowSpanBuilder.startSpan();
+ var workflowTraceId =
+ idGenerator.generateTraceIdForExecution(info.durableExecutionArn(), info.executionStartTime());
+ var workflowSpanId = idGenerator.generateWorkflowSpanId(info.durableExecutionArn());
+ workflowSpan = idGenerator.startSpan(workflowSpanBuilder, workflowTraceId, workflowSpanId);
// Determine parent context for the invocation span.
Context parentContext;
- if (extractedContext != null && extractedContext.parentSpanId() != null) {
+ if (invocationParent != null && invocationParent.parentSpanId() != null) {
// Reconstruct a remote parent from the extracted trace context (X-Ray header or current span).
// This connects plugin spans to the Lambda service's X-Ray segments.
var parentSpanContext = SpanContext.createFromRemoteParent(
- extractedContext.traceId(),
- extractedContext.parentSpanId(),
+ invocationParent.traceId(),
+ invocationParent.parentSpanId(),
TraceFlags.getSampled(),
TraceState.getDefault());
parentContext = Context.root().with(Span.wrap(parentSpanContext));
@@ -271,13 +249,20 @@ public void onInvocationStart(InvocationInfo info) {
// Inject MDC on the handler thread so handler-level logs (between steps) have trace context.
// This runs on the same thread as context.getLogger() calls in the handler.
if (enableMdc) {
- var traceId = idGenerator.generateTraceId();
- MDC.put(MdcSpanEnricher.MDC_TRACE_ID, traceId);
+ MDC.put(
+ MdcSpanEnricher.MDC_TRACE_ID,
+ invocationSpan.getSpanContext().getTraceId());
}
+ tracingEnabled = true;
}
@Override
public void onInvocationEnd(InvocationEndInfo info) {
+ if (!tracingEnabled) {
+ return;
+ }
+ tracingEnabled = false;
+
// Clear invocation-level MDC (set in onInvocationStart on the handler thread)
if (enableMdc) {
MdcSpanEnricher.clear();
@@ -352,6 +337,7 @@ public void onInvocationEnd(InvocationEndInfo info) {
@Override
public void onOperationStart(OperationInfo info) {
+ if (!tracingEnabled) return;
if (info.id() == null) return;
var parentContext = resolveParentContext(info.parentId());
@@ -363,19 +349,6 @@ public void onOperationStart(OperationInfo info) {
.setAttribute(DURABLE_OPERATION_TYPE, info.type())
.setAttribute(DURABLE_OPERATION_STATUS, info.status() != null ? info.status() : "STARTED");
- if (info.isReplay()) {
- // Operation was already started in a prior invocation — use a random span ID
- // and add a Link to the deterministic span from the original invocation for correlation.
- var deterministicSpanId = idGenerator.generateSpanIdForOperation(info.id());
- var traceId = idGenerator.generateTraceId();
- var linkedSpanContext =
- SpanContext.create(traceId, deterministicSpanId, TraceFlags.getSampled(), TraceState.getDefault());
- spanBuilder.addLink(linkedSpanContext);
- } else {
- // First execution — use deterministic span ID so continuations can link back
- idGenerator.setNextSpanOperationId(info.id());
- }
-
// Link to the Workflow span for execution-level correlation (operation stays parented to the invocation span).
addWorkflowLink(spanBuilder);
@@ -386,7 +359,10 @@ public void onOperationStart(OperationInfo info) {
spanBuilder.setAttribute(DURABLE_OPERATION_SUBTYPE, info.subType());
}
- var span = spanBuilder.startSpan();
+ var span = info.isReplay()
+ ? spanBuilder.startSpan()
+ : idGenerator.startSpan(
+ spanBuilder, null, idGenerator.generateSpanIdForOperation(durableExecutionArn, info.id()));
// Store the open span — will be ended in onOperationEnd or onInvocationEnd
operationSpans.put(info.id(), span);
@@ -396,6 +372,7 @@ public void onOperationStart(OperationInfo info) {
@Override
public void onOperationEnd(OperationEndInfo info) {
+ if (!tracingEnabled) return;
if (info.id() == null) return;
var span = operationSpans.remove(info.id());
@@ -421,18 +398,10 @@ public void onOperationEnd(OperationEndInfo info) {
}
span.end();
} else {
- // Operation was started in a prior invocation — create a continuation span with Link
- // to the deterministic span ID from the original invocation.
- var deterministicSpanId = idGenerator.generateSpanIdForOperation(info.id());
- var traceId = idGenerator.generateTraceId();
- var linkedSpanContext =
- SpanContext.create(traceId, deterministicSpanId, TraceFlags.getSampled(), TraceState.getDefault());
-
var parentContext = resolveParentContext(info.parentId());
var spanBuilder = tracer.spanBuilder(spanName(info.type(), info.subType(), info.name()))
.setParent(parentContext)
- .addLink(linkedSpanContext)
.setAttribute(DURABLE_EXECUTION_ARN, durableExecutionArn)
.setAttribute(DURABLE_OPERATION_ID, info.id())
.setAttribute(DURABLE_OPERATION_TYPE, info.type());
@@ -471,6 +440,8 @@ public void onOperationEnd(OperationEndInfo info) {
@Override
public void onUserFunctionStart(UserFunctionStartInfo info) {
+ if (!tracingEnabled) return;
+
// Skip attempt spans for CONTEXT operations — they are a scoping construct, not a
// retriable unit of work, so attempt number/outcome attributes don't apply.
// The operation span itself provides parent context for auto-instrumented calls.
@@ -529,6 +500,8 @@ public void onUserFunctionStart(UserFunctionStartInfo info) {
@Override
public void onUserFunctionEnd(UserFunctionEndInfo info) {
+ if (!tracingEnabled) return;
+
var key = attemptKey(info.id(), info.attempt());
// Close scope first (must happen on same thread as makeCurrent)
@@ -569,6 +542,24 @@ public void onUserFunctionEnd(UserFunctionEndInfo info) {
// ─── Helpers ─────────────────────────────────────────────────────────
+ private boolean bindTracer() {
+ if (tracer != null) {
+ return true;
+ }
+ synchronized (this) {
+ if (tracer != null) {
+ return true;
+ }
+ var setup = OtelPluginSupport.tryResolveGlobalProvider(instrumentationName, "InvocationOtelPlugin");
+ if (setup == null) {
+ return false;
+ }
+ sdkTracerProvider = setup.sdkTracerProvider();
+ tracer = setup.tracer();
+ return true;
+ }
+ }
+
private void endOpenSpansChildFirst() {
// Attempt spans are children of operation spans.
for (var scope : attemptScopes.values()) {
@@ -599,12 +590,6 @@ private Context resolveParentContext(String parentId) {
if (parentSpanContext != null) {
return Context.current().with(Span.wrap(parentSpanContext));
}
- // Parent operation from a prior invocation — create non-recording placeholder
- var deterministicParentSpanId = idGenerator.generateSpanIdForOperation(parentId);
- var traceId = idGenerator.generateTraceId();
- var placeholderContext = SpanContext.create(
- traceId, deterministicParentSpanId, TraceFlags.getSampled(), TraceState.getDefault());
- return Context.current().with(Span.wrap(placeholderContext));
}
// Fall back to invocation span as parent
if (invocationSpan != null) {
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java
index 33fd6ea66..74c7fc1d3 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginAutoConfigurationCustomizerProvider.java
@@ -5,14 +5,20 @@
import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer;
import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider;
-/** Installs the durable-execution ID generator when the OpenTelemetry Java agent auto-configures the SDK. */
+/** Wraps the Java agent's configured ID generator with scoped durable-execution overrides. */
public final class OtelPluginAutoConfigurationCustomizerProvider implements AutoConfigurationCustomizerProvider {
- private static final DeterministicIdGenerator ID_GENERATOR = new DeterministicIdGenerator();
-
@Override
public void customize(AutoConfigurationCustomizer autoConfiguration) {
OtelPluginAutoConfigurationState.markInstalled();
- autoConfiguration.addTracerProviderCustomizer((builder, config) -> builder.setIdGenerator(ID_GENERATOR));
+ autoConfiguration.addTracerProviderCustomizer((builder, config) -> {
+ DeterministicIdGenerator.installOn(builder);
+ return builder;
+ });
+ }
+
+ @Override
+ public int order() {
+ return Integer.MAX_VALUE;
}
}
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java
index 3cf021ad8..ed4ac9be5 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/OtelPluginConfig.java
@@ -2,8 +2,6 @@
// SPDX-License-Identifier: Apache-2.0
package software.amazon.lambda.durable.otel;
-import java.util.Map;
-
/**
* Immutable configuration for {@link InvocationOtelPlugin} and {@link ExecutionOtelPlugin}.
*
@@ -26,8 +24,8 @@
* }
*
* Defaults: {@code contextExtractor = new XRayContextExtractor()}, {@code enableMdc = true}, {@code workflowSpanName
- * = "Workflow"}, {@code instrumentationName = "aws-durable-execution-sdk-java"}, {@code providerSource =
- * ProviderSource.GLOBAL}. A {@code null} passed to any builder setter falls back to the corresponding default.
+ * = "Workflow"}, {@code instrumentationName = "aws-durable-execution-sdk-java"}. A {@code null} passed to any builder
+ * setter falls back to the corresponding default.
*/
public final class OtelPluginConfig {
@@ -38,9 +36,6 @@ public final class OtelPluginConfig {
private final boolean enableMdc;
private final String workflowSpanName;
private final String instrumentationName;
- private final ProviderSource providerSource;
- private final String otlpEndpoint;
- private final Map {@link ProviderSource#EXPLICIT} is not valid here — it is implied by using a {@code (SdkTracerProviderBuilder,
- * OtelPluginConfig)} constructor and is rejected by the config-only constructors.
- */
- public ProviderSource providerSource() {
- return providerSource;
- }
-
- /** OTLP/HTTP endpoint for the auto-configured provider, or {@code null} to use the OTel default / env var. */
- public String otlpEndpoint() {
- return otlpEndpoint;
- }
-
- /** Extra headers sent by the auto-configured OTLP exporter (never {@code null}). */
- public Map {@link ProviderSource#EXPLICIT} is not accepted through the config-only constructors — supply a
- * {@code SdkTracerProviderBuilder} via the two-arg constructor instead.
- *
- * @param providerSource the provider source, {@link ProviderSource#GLOBAL} or {@link ProviderSource#AUTO_OTLP}
- * @return this builder
- */
- public Builder providerSource(ProviderSource providerSource) {
- this.providerSource = providerSource != null ? providerSource : ProviderSource.GLOBAL;
- return this;
- }
-
- /**
- * Sets the OTLP/HTTP endpoint for the auto-configured provider. When null, the OTel default (or
- * {@code OTEL_EXPORTER_OTLP_ENDPOINT}) is used.
- *
- * @param otlpEndpoint the OTLP/HTTP traces endpoint
- * @return this builder
- */
- public Builder otlpEndpoint(String otlpEndpoint) {
- this.otlpEndpoint = otlpEndpoint;
- return this;
- }
-
- /**
- * Sets extra headers for the auto-configured OTLP exporter (e.g. auth headers for a third-party endpoint).
- *
- * @param otlpHeaders header name/value pairs; null is treated as empty
- * @return this builder
- */
- public Builder otlpHeaders(Map Mirrors the {@code ProviderSource} used by the JavaScript and Python SDK OTel plugins for cross-SDK parity:
- *
- * This is the single knob that selects a plugin's tracer provider. {@link OtelPluginConfig#providerSource()} carries
- * it for the config-only constructors; the {@code (SdkTracerProviderBuilder, OtelPluginConfig)} constructors always
- * report {@link #EXPLICIT}.
- */
-public enum ProviderSource {
- /** Caller-supplied {@code SdkTracerProviderBuilder}; plugin-owned. */
- EXPLICIT,
- /** Globally configured provider (ADOT Java agent); not plugin-owned. */
- GLOBAL,
- /** Auto-configured OTLP/HTTP provider; plugin-owned. */
- AUTO_OTLP
-}
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java
index 8a7fb367c..109f633de 100644
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java
+++ b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/XRayContextExtractor.java
@@ -9,9 +9,9 @@
/**
* Extracts OTel trace context from the AWS X-Ray {@code _X_AMZN_TRACE_ID} environment variable.
*
- * The durable execution backend propagates the same Root trace ID to every invocation of the same execution, so all
- * invocations share one trace. This extractor parses that header and returns the trace ID in OTel format (32 hex chars)
- * along with the parent span ID (16 hex chars).
+ * This extractor parses the Lambda/X-Ray header and returns the trace ID in OTel format (32 hex chars) along with
+ * the parent span ID (16 hex chars). Plugins use it as a fallback parent for Invocation spans; the deterministic
+ * Workflow trace is derived separately from the durable execution ARN.
*
* X-Ray header format: {@code Root=1-5759e988-bd862e3fe1be46a994272793;Parent=53995c3f42cd8ad8;Sampled=1}
*
diff --git a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java
index 2b9389fc7..1afb9ab6f 100644
--- a/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java
+++ b/otel-plugin/src/test/java/software/amazon/lambda/durable/otel/DeterministicIdGeneratorTest.java
@@ -4,12 +4,29 @@
import static org.junit.jupiter.api.Assertions.*;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.trace.IdGenerator;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.data.LinkData;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import io.opentelemetry.sdk.trace.samplers.SamplingResult;
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
class DeterministicIdGeneratorTest {
+ private static final Instant EXECUTION_START_TIME = Instant.parse("2026-08-15T00:00:00Z");
+
private DeterministicIdGenerator generator;
@BeforeEach
@@ -34,6 +51,151 @@ void generateTraceId_withoutArn_returnsRandom() {
assertNotEquals(id1, id2);
}
+ @Test
+ void scopedIds_delegateOutsideScope() {
+ var providerGenerator = new DeterministicIdGenerator();
+ try (var provider =
+ SdkTracerProvider.builder().setIdGenerator(providerGenerator).build()) {
+ var pluginTracer = provider.get("durable-plugin");
+ var unrelatedTracer = provider.get("unrelated-library");
+
+ var before = unrelatedTracer.spanBuilder("before").setNoParent().startSpan();
+ var workflowTraceId = generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME);
+ var workflowSpanId = generator.generateWorkflowSpanId("arn:exec1");
+ var workflow = generator.startSpan(
+ pluginTracer.spanBuilder("Workflow").setNoParent(), workflowTraceId, workflowSpanId);
+ var during = unrelatedTracer.spanBuilder("during").setNoParent().startSpan();
+ var after = unrelatedTracer.spanBuilder("after").setNoParent().startSpan();
+
+ assertEquals(workflowTraceId, workflow.getSpanContext().getTraceId());
+ assertEquals(workflowSpanId, workflow.getSpanContext().getSpanId());
+ assertAllFreshRoots(
+ workflow.getSpanContext(),
+ before.getSpanContext(),
+ during.getSpanContext(),
+ after.getSpanContext());
+
+ before.end();
+ workflow.end();
+ during.end();
+ after.end();
+ }
+ }
+
+ @Test
+ void scopedIds_bridgeAcrossGeneratorInstances() {
+ var agentGenerator = new DeterministicIdGenerator();
+ try (var provider =
+ SdkTracerProvider.builder().setIdGenerator(agentGenerator).build()) {
+ var workflowTraceId = generator.generateTraceIdForExecution("arn:exec1", EXECUTION_START_TIME);
+ var workflowSpanId = generator.generateWorkflowSpanId("arn:exec1");
+ var workflow = generator.startSpan(
+ provider.get("durable-plugin").spanBuilder("Workflow").setNoParent(),
+ workflowTraceId,
+ workflowSpanId);
+
+ assertEquals(workflowTraceId, workflow.getSpanContext().getTraceId());
+ assertEquals(workflowSpanId, workflow.getSpanContext().getSpanId());
+ assertNotEquals(workflowTraceId, agentGenerator.generateTraceId());
+ workflow.end();
+ }
+ }
+
+ @Test
+ void scopedTraceId_isConsumedBeforeSamplerStartsNestedRoot() {
+ var fallbackTraceId = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
+ var fallbackSpanId = "cccccccccccccccc";
+ var agentGenerator = new DeterministicIdGenerator(fixedIds(fallbackTraceId, fallbackSpanId));
+ var tracerReference = new AtomicReference
*
*
- *
- *
+ *
- *
- *
- * @param config the plugin configuration
+ * @param instrumentationName the instrumentation scope name
* @param pluginName the plugin name used in diagnostics/flush logging
- * @return the resolved provider, tracer, ID generator, and source
- * @throws IllegalArgumentException if {@code config.providerSource()} is {@link ProviderSource#EXPLICIT}
+ * @return the resolved provider and tracer, or {@code null} when telemetry must be disabled for this invocation
*/
- static ProviderSetup resolveConfiguredProvider(OtelPluginConfig config, String pluginName) {
- return switch (config.providerSource()) {
- case GLOBAL -> {
- var idGenerator = createDefaultIdGenerator();
- var tracerProvider = getDefaultTracerProvider(pluginName);
- yield new ProviderSetup(
- ProviderSource.GLOBAL,
- getSdkTracerProviderForFlush(tracerProvider, pluginName),
- tracerProvider.get(config.instrumentationName()),
- idGenerator);
- }
- case AUTO_OTLP -> {
- var idGenerator = new DeterministicIdGenerator();
- var sdkTracerProvider = buildAutoOtlpProvider(config, idGenerator, null);
- yield new ProviderSetup(
- ProviderSource.AUTO_OTLP,
- sdkTracerProvider,
- sdkTracerProvider.get(config.instrumentationName()),
- idGenerator);
- }
- case EXPLICIT ->
- throw new IllegalArgumentException(
- "OtelPluginConfig.providerSource(EXPLICIT) requires a caller-supplied SdkTracerProviderBuilder; "
- + "use the (SdkTracerProviderBuilder, OtelPluginConfig) constructor.");
- };
- }
-
- /** Resolves the OTLP/HTTP traces endpoint (config -> env -> exporter default), appending the signal path. */
- private static String resolveOtlpEndpoint(OtelPluginConfig config) {
- if (config.otlpEndpoint() != null && !config.otlpEndpoint().isBlank()) {
- return config.otlpEndpoint();
+ static ProviderSetup tryResolveGlobalProvider(String instrumentationName, String pluginName) {
+ if (!OtelPluginAutoConfigurationState.isInstalled()) {
+ logger.warn(
+ "{} telemetry is disabled for this invocation because "
+ + "OtelPluginAutoConfigurationCustomizerProvider is not installed yet. Provider resolution "
+ + "will be retried on the next invocation. {}",
+ pluginName,
+ javaAgentExtensionsDiagnostic());
+ return null;
}
- var envEndpoint = System.getenv("OTEL_EXPORTER_OTLP_ENDPOINT");
- if (envEndpoint != null && !envEndpoint.isBlank()) {
- var base = envEndpoint.endsWith("/") ? envEndpoint.substring(0, envEndpoint.length() - 1) : envEndpoint;
- return base.endsWith("/v1/traces") ? base : base + "/v1/traces";
+ if (!GlobalOpenTelemetry.isSet()) {
+ logger.warn(
+ "{} telemetry is disabled for this invocation because GlobalOpenTelemetry is not initialized yet. "
+ + "Provider resolution will be retried on the next invocation.",
+ pluginName);
+ return null;
}
- // null -> the OTLP/HTTP exporter's own default (http://localhost:4318/v1/traces)
- return null;
- }
- /** Builds the sampler from {@code OTEL_DURABLE_SAMPLING_RATIO}, falling back to always-on. */
- private static Sampler resolveSampler() {
- var raw = System.getenv("OTEL_DURABLE_SAMPLING_RATIO");
- if (raw != null) {
- try {
- var ratio = Double.parseDouble(raw);
- if (ratio >= 0.0 && ratio <= 1.0) {
- return Sampler.traceIdRatioBased(ratio);
- }
- } catch (NumberFormatException ignored) {
- // fall through to always-on
- }
+ var tracerProvider = GlobalOpenTelemetry.getOrNoop().getTracerProvider();
+ if (tracerProvider == TracerProvider.noop()) {
+ logger.warn(
+ "{} telemetry is disabled for this invocation because GlobalOpenTelemetry contains a no-op tracer "
+ + "provider. Provider resolution will be retried on the next invocation.",
+ pluginName);
+ return null;
}
- return Sampler.alwaysOn();
- }
- /** Builds Lambda resource attributes from AWS_* env vars, merged onto the default resource. */
- private static Resource buildLambdaResource() {
- var functionName = System.getenv("AWS_LAMBDA_FUNCTION_NAME");
- if (functionName == null || functionName.isBlank()) {
- return Resource.getDefault();
- }
- var attributes = Attributes.builder()
- .put(ServiceAttributes.SERVICE_NAME, functionName)
- .put("faas.name", functionName)
- .put("cloud.provider", "aws")
- .put("cloud.platform", "aws_lambda");
- var region = System.getenv("AWS_REGION");
- if (region != null && !region.isBlank()) {
- attributes.put("cloud.region", region);
- }
- var version = System.getenv("AWS_LAMBDA_FUNCTION_VERSION");
- if (version != null && !version.isBlank()) {
- attributes.put("faas.version", version);
- }
- return Resource.getDefault().merge(Resource.create(attributes.build()));
+ logger.info(
+ "{} initialized from existing GlobalOpenTelemetry tracer provider {}; assuming "
+ + "deterministic span IDs were installed through AutoConfigurationCustomizerProvider",
+ pluginName,
+ tracerProvider.getClass().getName());
+ return new ProviderSetup(
+ getSdkTracerProviderForFlush(tracerProvider, pluginName), tracerProvider.get(instrumentationName));
}
/** Extracts trace context from the current OTel span (fallback when X-Ray header is unavailable). */
@@ -212,18 +94,6 @@ static SdkTracerProvider getSdkTracerProviderForFlush(TracerProvider tracerProvi
return null;
}
- private static void validateAutoConfigurationCustomizerProviderInstalled(String pluginName) {
- if (OtelPluginAutoConfigurationState.isInstalled()) {
- return;
- }
- throw new IllegalStateException(
- pluginName + "() requires OtelPluginAutoConfigurationCustomizerProvider to be installed by the "
- + "OpenTelemetry Java agent. Package this plugin jar as an agent extension and set "
- + "OTEL_JAVAAGENT_EXTENSIONS or -Dotel.javaagent.extensions to that jar before constructing "
- + pluginName + "(). "
- + javaAgentExtensionsDiagnostic());
- }
-
private static String javaAgentExtensionsDiagnostic() {
var propertyValue = System.getProperty("otel.javaagent.extensions");
var environmentValue = System.getenv("OTEL_JAVAAGENT_EXTENSIONS");
diff --git a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java b/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java
deleted file mode 100644
index 5081c45c5..000000000
--- a/otel-plugin/src/main/java/software/amazon/lambda/durable/otel/ProviderSource.java
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package software.amazon.lambda.durable.otel;
-
-/**
- * Which of the three resolution tiers produced a plugin's tracer provider.
- *
- *
- *
- *
- *