Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,11 @@ export class LoggingContentGenerator implements ContentGenerator {
outputTokens: response.usageMetadata?.candidatesTokenCount,
cachedInputTokens: response.usageMetadata?.cachedContentTokenCount,
durationMs: Date.now() - startTime,
responseId: response.responseId || undefined,
finishReason:
(response.candidates?.[0]?.finishReason as string) || undefined,
thoughtsTokenCount: response.usageMetadata?.thoughtsTokenCount,
subagentName: subagentNameContext.getStore() || undefined,
});
return response;
} catch (error) {
Expand All @@ -304,6 +309,9 @@ export class LoggingContentGenerator implements ContentGenerator {
error: aborted
? API_CALL_ABORTED_SPAN_STATUS_MESSAGE
: API_CALL_FAILED_SPAN_STATUS_MESSAGE,
errorType: getErrorType(error),
errorStatusCode: getErrorStatus(error),
subagentName: subagentNameContext.getStore() || undefined,
});
await context.with(spanContext, async () => {
this.safelyLogApiError('', durationMs, error, req.model, userPromptId);
Expand Down Expand Up @@ -383,6 +391,9 @@ export class LoggingContentGenerator implements ContentGenerator {
error: aborted
? API_CALL_ABORTED_SPAN_STATUS_MESSAGE
: API_CALL_FAILED_SPAN_STATUS_MESSAGE,
errorType: getErrorType(error),
errorStatusCode: getErrorStatus(error),
subagentName: subagentNameContext.getStore() || undefined,
});
try {
await this.safelyLogOpenAIInteraction(
Expand Down Expand Up @@ -464,6 +475,9 @@ export class LoggingContentGenerator implements ContentGenerator {
let firstModelVersion = '';
let lastUsageMetadata: GenerateContentResponseUsageMetadata | undefined;
let errorOccurred = false;
let lastFinishReason: string | undefined;
let lastError: unknown;
const subagentName = subagentNameContext.getStore();

// TTFT (time to first token): wall-clock from generateContentStream
// dispatch to the first stream chunk containing user-visible content.
Expand Down Expand Up @@ -504,6 +518,8 @@ export class LoggingContentGenerator implements ContentGenerator {
success: false,
durationMs: Date.now() - startTime,
error: 'Stream span timed out (idle)',
responseId: firstResponseId || undefined,
subagentName: subagentName || undefined,
});
spanEndedByTimeout = true;
}, STREAM_IDLE_TIMEOUT_MS);
Expand All @@ -526,6 +542,10 @@ export class LoggingContentGenerator implements ContentGenerator {
if (response.usageMetadata) {
lastUsageMetadata = response.usageMetadata;
}
const candidate = response.candidates?.[0];
if (candidate?.finishReason) {
lastFinishReason = candidate.finishReason as string;
}
// Capture TTFT on the first stream chunk that contains user-visible
// content. hasUserVisibleContent skips role-only / usageMetadata-only
// chunks, so TTFT reflects "model produced something the operator can
Expand Down Expand Up @@ -578,6 +598,7 @@ export class LoggingContentGenerator implements ContentGenerator {
}
} catch (error) {
errorOccurred = true;
lastError = error;
// Same gating as the success path above: if the idle timeout already
// closed the span as failed, do not emit a parallel api_error log
// (the span is the canonical signal). Otherwise we'd produce the
Expand Down Expand Up @@ -626,6 +647,12 @@ export class LoggingContentGenerator implements ContentGenerator {
? API_CALL_ABORTED_SPAN_STATUS_MESSAGE
: API_CALL_FAILED_SPAN_STATUS_MESSAGE
: undefined,
responseId: firstResponseId || undefined,
Comment thread
doudouOUC marked this conversation as resolved.
finishReason: lastFinishReason,
thoughtsTokenCount: lastUsageMetadata?.thoughtsTokenCount,
subagentName: subagentName || undefined,
errorType: lastError ? getErrorType(lastError) : undefined,
errorStatusCode: lastError ? getErrorStatus(lastError) : undefined,
});
}
}
Expand Down
144 changes: 144 additions & 0 deletions packages/core/src/telemetry/session-tracing.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -694,6 +694,150 @@ describe('session-tracing', () => {
});
});

describe('LLM request spans — response metadata & error enrichment', () => {
it('endLLMRequestSpan dual-emits response_id / gen_ai.response.id', () => {
const span = startLLMRequestSpan('m', 'p');
endLLMRequestSpan(span, {
success: true,
responseId: 'chatcmpl-abc123',
});

const attrs = mockSpans[0]!.attributes;
expect(attrs['response_id']).toBe('chatcmpl-abc123');
expect(attrs['gen_ai.response.id']).toBe('chatcmpl-abc123');
});

it('endLLMRequestSpan omits response_id when undefined', () => {
const span = startLLMRequestSpan('m', 'p');
endLLMRequestSpan(span, { success: true });

const attrs = mockSpans[0]!.attributes;
expect(attrs['response_id']).toBeUndefined();
expect(attrs['gen_ai.response.id']).toBeUndefined();
});

it('endLLMRequestSpan dual-emits finish_reason / gen_ai.response.finish_reasons (string vs array)', () => {
const span = startLLMRequestSpan('m', 'p');
endLLMRequestSpan(span, {
success: true,
finishReason: 'STOP',
});

const attrs = mockSpans[0]!.attributes;
expect(attrs['finish_reason']).toBe('STOP');
expect(attrs['gen_ai.response.finish_reasons']).toEqual(['STOP']);
});

it('endLLMRequestSpan omits finish_reason when undefined', () => {
const span = startLLMRequestSpan('m', 'p');
endLLMRequestSpan(span, { success: true });

const attrs = mockSpans[0]!.attributes;
expect(attrs['finish_reason']).toBeUndefined();
expect(attrs['gen_ai.response.finish_reasons']).toBeUndefined();
});

it('endLLMRequestSpan dual-emits thoughts_token_count / gen_ai.usage.reasoning_tokens', () => {
const span = startLLMRequestSpan('m', 'p');
endLLMRequestSpan(span, {
success: true,
thoughtsTokenCount: 42,
});

const attrs = mockSpans[0]!.attributes;
expect(attrs['thoughts_token_count']).toBe(42);
expect(attrs['gen_ai.usage.reasoning_tokens']).toBe(42);
});

it('endLLMRequestSpan emits thoughts_token_count === 0 (no reasoning is meaningful info, not undefined)', () => {
const span = startLLMRequestSpan('m', 'p');
endLLMRequestSpan(span, {
success: true,
thoughtsTokenCount: 0,
});

const attrs = mockSpans[0]!.attributes;
expect(attrs['thoughts_token_count']).toBe(0);
expect(attrs['gen_ai.usage.reasoning_tokens']).toBe(0);
});

it('endLLMRequestSpan omits thoughts_token_count when undefined', () => {
const span = startLLMRequestSpan('m', 'p');
endLLMRequestSpan(span, { success: true });

const attrs = mockSpans[0]!.attributes;
expect(attrs['thoughts_token_count']).toBeUndefined();
expect(attrs['gen_ai.usage.reasoning_tokens']).toBeUndefined();
});

it('endLLMRequestSpan emits subagent_name when present', () => {
const span = startLLMRequestSpan('m', 'p');
endLLMRequestSpan(span, {
success: true,
subagentName: 'Explore-abc123',
});

const attrs = mockSpans[0]!.attributes;
expect(attrs['subagent_name']).toBe('Explore-abc123');
});

it('endLLMRequestSpan omits subagent_name when undefined', () => {
const span = startLLMRequestSpan('m', 'p');
endLLMRequestSpan(span, { success: true });

expect(mockSpans[0]!.attributes['subagent_name']).toBeUndefined();
});

it('endLLMRequestSpan emits error_type and error.type on error spans', () => {
const span = startLLMRequestSpan('m', 'p');
endLLMRequestSpan(span, {
success: false,
error: 'API call failed',
errorType: 'RateLimitError',
errorStatusCode: 429,
});

const attrs = mockSpans[0]!.attributes;
expect(attrs['error_type']).toBe('RateLimitError');
expect(attrs['error.type']).toBe('RateLimitError');
expect(attrs['error_status_code']).toBe(429);
});

it('endLLMRequestSpan omits error_type/error_status_code on success spans', () => {
const span = startLLMRequestSpan('m', 'p');
endLLMRequestSpan(span, { success: true });

const attrs = mockSpans[0]!.attributes;
expect(attrs['error_type']).toBeUndefined();
expect(attrs['error.type']).toBeUndefined();
expect(attrs['error_status_code']).toBeUndefined();
});

it('endLLMRequestSpan emits all new attributes together', () => {
const span = startLLMRequestSpan('m', 'p');
endLLMRequestSpan(span, {
success: true,
inputTokens: 500,
outputTokens: 100,
responseId: 'resp-xyz',
finishReason: 'MAX_TOKENS',
thoughtsTokenCount: 30,
subagentName: 'code-reviewer',
});

const attrs = mockSpans[0]!.attributes;
expect(attrs['response_id']).toBe('resp-xyz');
expect(attrs['gen_ai.response.id']).toBe('resp-xyz');
expect(attrs['finish_reason']).toBe('MAX_TOKENS');
expect(attrs['gen_ai.response.finish_reasons']).toEqual(['MAX_TOKENS']);
expect(attrs['thoughts_token_count']).toBe(30);
expect(attrs['gen_ai.usage.reasoning_tokens']).toBe(30);
expect(attrs['subagent_name']).toBe('code-reviewer');
expect(attrs['input_tokens']).toBe(500);
expect(attrs['output_tokens']).toBe(100);
});
});

describe('tool spans', () => {
it('creates and ends a tool span', () => {
const span = startToolSpan('ReadFile', { 'tool.call_id': 'call-1' });
Expand Down
46 changes: 42 additions & 4 deletions packages/core/src/telemetry/session-tracing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,7 @@ import {
} from './constants.js';
import { clearDetailedSpanState } from './detailed-span-attributes.js';
import { isTelemetrySdkInitialized } from './sdk.js';
import {
getCurrentSessionId,
setSessionContext,
} from './session-context.js';
import { getCurrentSessionId, setSessionContext } from './session-context.js';
import { createDebugLogger } from '../utils/debugLogger.js';

const debugLogger = createDebugLogger('SESSION_TRACING');
Expand Down Expand Up @@ -92,6 +89,22 @@ export interface LLMRequestMetadata {
* in Phase 4a.
*/
retryTotalDelayMs?: number;
/** Provider response ID (e.g. DashScope request_id / OpenAI completion id). */
responseId?: string;
/** Model finish/stop reason (e.g. "STOP", "MAX_TOKENS"). */
finishReason?: string;
/**
* Reasoning/thinking token count. For OpenAI-compatible providers,
* this value is already INCLUDED in outputTokens (candidatesTokenCount).
* Do not sum with outputTokens to avoid double-counting.
*/
thoughtsTokenCount?: number;
/** Subagent name that originated this request, or undefined for main. */
subagentName?: string;
/** Structured error type (e.g. "RateLimitError", "APIConnectionError:ECONNREFUSED"). */
errorType?: string;
/** HTTP status code from the provider error response. */
errorStatusCode?: number;
}

export interface ToolSpanMetadata {
Expand Down Expand Up @@ -565,6 +578,31 @@ export function endLLMRequestSpan(
endAttributes['success'] = metadata.success;
if (metadata.error !== undefined)
endAttributes['error'] = truncateSpanError(metadata.error);
if (metadata.responseId !== undefined) {
endAttributes['response_id'] = metadata.responseId;
endAttributes['gen_ai.response.id'] = metadata.responseId;
}
if (metadata.finishReason !== undefined) {
endAttributes['finish_reason'] = metadata.finishReason;
endAttributes['gen_ai.response.finish_reasons'] = [
metadata.finishReason,
];
}
if (metadata.thoughtsTokenCount !== undefined) {
endAttributes['thoughts_token_count'] = metadata.thoughtsTokenCount;
endAttributes['gen_ai.usage.reasoning_tokens'] =
metadata.thoughtsTokenCount;
}
if (metadata.subagentName !== undefined) {
endAttributes['subagent_name'] = metadata.subagentName;
}
if (metadata.errorType !== undefined) {
endAttributes['error_type'] = metadata.errorType;
endAttributes['error.type'] = metadata.errorType;
}
if (metadata.errorStatusCode !== undefined) {
endAttributes['error_status_code'] = metadata.errorStatusCode;
}
}

spanCtx.span.setAttributes(endAttributes);
Expand Down