Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions src/trace/trace-context-service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ describe("TraceContextService", () => {
mockXRayShouldThrow = false;
const traceWrapper = {
traceContext: () => spanContextWrapper,
closeScope: jest.fn(),
};
traceContextService = new TraceContextService(traceWrapper as any, {} as any);
});
Expand Down Expand Up @@ -126,4 +127,54 @@ describe("TraceContextService", () => {
expect(result?.toTraceId()).toBe("newTraceId");
expect(traceContextService.traceSource).toBe("event");
});

it("should not leak dd-trace context from previous invocation when extract is called", async () => {
// Simulate dd-trace having a stale active span from a previous invocation (warm start scenario)
const staleDdTraceContext = {
toTraceId: () => "staleTraceId_999",
toSpanId: () => "staleSpanId_888",
sampleMode: () => 1,
source: TraceSource.DdTrace,
spanContext: {},
};

// Mock tracerWrapper that returns stale context initially, then null after closeScope is called
let traceContextValue: any = staleDdTraceContext;
const mockCloseScopeFn = jest.fn(() => {
// After closeScope is called, traceContext should return null
traceContextValue = null;
});

const mockTracerWrapper = {
traceContext: jest.fn(() => traceContextValue),
closeScope: mockCloseScopeFn,
};

const service = new TraceContextService(mockTracerWrapper as any, {} as any);

// Mock the extractor to return a NEW context for the current invocation
const newEventContext = {
toTraceId: () => "newTraceId_123",
toSpanId: () => "newSpanId_456",
sampleMode: () => 2,
source: TraceSource.Event,
spanContext: {},
};
const mockExtract = jest.fn().mockResolvedValue(newEventContext);
service["traceExtractor"] = { extract: mockExtract } as any;

// Call extract for the new invocation
await service.extract({}, {} as any);

// Verify that closeScope was called to clear the stale context
expect(mockCloseScopeFn).toHaveBeenCalled();

// After the fix: currentTraceHeaders should return the NEW context from the event
// not the stale dd-trace context from the previous invocation
const headers = service.currentTraceHeaders;

expect(headers["x-datadog-trace-id"]).toBe("newTraceId_123");
expect(headers["x-datadog-parent-id"]).toBe("newSpanId_456");
expect(headers["x-datadog-sampling-priority"]).toBe("2");
});
});
9 changes: 4 additions & 5 deletions src/trace/trace-context-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,13 @@ export class TraceContextService {
}

async extract(event: any, context: Context): Promise<SpanContextWrapper | null> {
// Reset trace context from previous invocation to prevent caching
// Reset trace context and close dd-trace scope to prevent stale context from previous invocation due to unfinished spans
this.rootTraceContext = null;
this.tracerWrapper.closeScope();

this.rootTraceContext = await this.traceExtractor?.extract(event, context);

return this.currentTraceContext;
// Return the extracted context, not the current context which may not be related to the event or context
return this.rootTraceContext;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is the fix that codex missed.

}

get currentTraceHeaders(): Partial<DatadogTraceHeaders> {
Expand All @@ -70,8 +71,6 @@ export class TraceContextService {
}

get currentTraceContext(): SpanContextWrapper | null {
if (this.rootTraceContext === null) return null;

const traceContext = this.rootTraceContext;
const currentDatadogContext = this.tracerWrapper.traceContext();
if (currentDatadogContext) {
Expand Down
31 changes: 31 additions & 0 deletions src/trace/tracer-wrapper.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -127,4 +127,35 @@ describe("TracerWrapper", () => {

expect(mockDataStreamsCheckpointer.setConsumeCheckpoint).toHaveBeenCalledWith(eventType, arn, contextJson, false);
});

it("should finish active span when closing scope to prevent context leakage", () => {
const mockFinishFn = jest.fn();
mockSpan = {
context: () => ({
toSpanId: () => "1234",
toTraceId: () => "45678",
_sampling: {
priority: "2",
},
}),
finish: mockFinishFn,
};

const wrapper = new TracerWrapper();
wrapper.closeScope();

expect(mockFinishFn).toHaveBeenCalled();
});

it("should not error when closing scope with no active span", () => {
mockSpan = null;
const wrapper = new TracerWrapper();
expect(() => wrapper.closeScope()).not.toThrow();
});

it("should not error when closing scope with tracer unavailable", () => {
mockNoTracer = true;
const wrapper = new TracerWrapper();
expect(() => wrapper.closeScope()).not.toThrow();
});
});
16 changes: 15 additions & 1 deletion src/trace/tracer-wrapper.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { logDebug } from "../utils";
import { logDebug, logWarning } from "../utils";
import { SpanContextWrapper } from "./span-context-wrapper";
import { TraceSource } from "./trace-context-service";

Expand Down Expand Up @@ -94,6 +94,20 @@ export class TracerWrapper {
return new SpanContextWrapper(span.context(), TraceSource.DdTrace);
}

public closeScope(): void {
try {
const activeSpan = this.currentSpan;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this.currentSpan will check

    if (!this.isTracerAvailable) {
      return;
    }

if (activeSpan && typeof activeSpan.finish === "function") {
logDebug("Detected stale span from previous invocation, finishing it to prevent trace context leakage");
Copy link
Contributor Author

@joeyzhao2018 joeyzhao2018 Nov 19, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Honestly, I was thinking this should be logWarning because this is not expected.
But the customer is actively running into this issue and we simply cannot reproduce, warning logs will spam their logs right away.

Since we are still actively working with them. My proposal is that we keep it at DEBUG level for this release => then check with the customer to understand which span was that leaked span => have a good reproduction case and fix it => update this to WARNING level.

activeSpan.finish();
Comment on lines +99 to +102

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Closing scope still leaves stale dd-trace context active

The new closeScope() only calls finish() on whatever span is currently active, but it never deactivates or clears that span from the tracer’s scope. TracerWrapper.traceContext() still returns the active span unconditionally, and TraceContextService.currentTraceContext (src/trace/trace-context-service.ts:73-79) always prefers that dd-trace span when injecting headers. In the warm-start scenario this change is trying to fix—an unfinished span left active from a prior invocation—closeScope() will finish the span but leave it active, so currentTraceHeaders will still emit the stale trace IDs. Deactivating the scope or skipping finished spans is needed to prevent the leakage the commit is addressing.

Useful? React with 👍 / 👎.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now the extract will return the extracted context and won't use the current context. later the extracted context will be used to create the inferred span or the lambda span and resetting the tracecontext there. Here the finish() is really trying to finish the still active span.

}
} catch (err) {
if (err instanceof Object || err instanceof Error) {
logDebug("Failed to close dd-trace scope", err);
}
}
}

public injectSpan(span: SpanContext): any {
const dest = {};
this.tracer.inject(span, "text_map", dest);
Expand Down
Loading