diff --git a/packages/cli/cli-v2/src/context/adapter/TaskContextAdapter.ts b/packages/cli/cli-v2/src/context/adapter/TaskContextAdapter.ts index 902350d39c13..47dba5f017fd 100644 --- a/packages/cli/cli-v2/src/context/adapter/TaskContextAdapter.ts +++ b/packages/cli/cli-v2/src/context/adapter/TaskContextAdapter.ts @@ -1,7 +1,6 @@ import { createLogger, LOG_LEVELS, Logger, LogLevel } from "@fern-api/logger"; import { type CaptureExceptionOptions, - type CliError, type CreateInteractiveTaskParams, type Finishable, type InteractiveTaskContext, @@ -9,6 +8,7 @@ import { type Startable, TaskAbortSignal, type TaskContext, + TaskFailOptions, TaskResult } from "@fern-api/task-context"; @@ -56,12 +56,12 @@ export class TaskContextAdapter implements TaskContext { await run(); } - public failAndThrow(message?: string, error?: unknown, options?: { code?: CliError.Code }): never { + public failAndThrow(message?: string, error?: unknown, options?: TaskFailOptions): never { this.failWithoutThrowing(message, error, options); throw new TaskAbortSignal(); } - public failWithoutThrowing(message?: string, error?: unknown, options?: { code?: CliError.Code }): void { + public failWithoutThrowing(message?: string, error?: unknown, options?: TaskFailOptions): void { this.result = TaskResult.Failure; if (error instanceof TaskAbortSignal) { return; @@ -72,7 +72,9 @@ export class TaskContextAdapter implements TaskContext { this.logger.error(fullMessage); } - reportError(this.context, error, { ...options, message }); + if (!options?.skipErrorReporting) { + reportError(this.context, error, { ...options, message }); + } } public getLastFailureMessage(): string | undefined { diff --git a/packages/cli/cli/changes/unreleased/automations-generate-telemetry.yml b/packages/cli/cli/changes/unreleased/automations-generate-telemetry.yml new file mode 100644 index 000000000000..a87a4804b508 --- /dev/null +++ b/packages/cli/cli/changes/unreleased/automations-generate-telemetry.yml @@ -0,0 +1,5 @@ +- summary: | + Emit Tier 1 automation telemetry events for `fern automations generate`, including + per-generator success/failure/skip events and run-level lifecycle events. Per-generator + Fiddle failures no longer emit mislabeled run-level `generation_failed` Sentry events. + type: feat diff --git a/packages/cli/cli/src/cli-context/TaskContextImpl.ts b/packages/cli/cli/src/cli-context/TaskContextImpl.ts index bc6529afadf2..ace90d3809d0 100644 --- a/packages/cli/cli/src/cli-context/TaskContextImpl.ts +++ b/packages/cli/cli/src/cli-context/TaskContextImpl.ts @@ -3,7 +3,6 @@ import { addPrefixToString } from "@fern-api/core-utils"; import { createLogger, LogLevel } from "@fern-api/logger"; import { type CaptureExceptionOptions, - CliError, CreateInteractiveTaskParams, Finishable, InteractiveTaskContext, @@ -11,6 +10,7 @@ import { Startable, TaskAbortSignal, TaskContext, + TaskFailOptions, TaskResult } from "@fern-api/task-context"; @@ -107,13 +107,13 @@ export class TaskContextImpl implements Startable, Finishable, Task public takeOverTerminal: (run: () => void | Promise) => Promise; - public failAndThrow(message?: string, error?: unknown, options?: { code?: CliError.Code }): never { + public failAndThrow(message?: string, error?: unknown, options?: TaskFailOptions): never { this.failWithoutThrowing(message, error, options); this.finish(); throw new TaskAbortSignal(); } - public failWithoutThrowing(message?: string, error?: unknown, options?: { code?: CliError.Code }): void { + public failWithoutThrowing(message?: string, error?: unknown, options?: TaskFailOptions): void { this.result = TaskResult.Failure; if (error instanceof TaskAbortSignal) { return; @@ -122,7 +122,9 @@ export class TaskContextImpl implements Startable, Finishable, Task this.lastFailureMessage = message; } logErrorMessage({ message, error, logger: this.logger }); - reportError(this, error, { ...options, message }); + if (!options?.skipErrorReporting) { + reportError(this, error, { ...options, message }); + } } public getLastFailureMessage(): string | undefined { diff --git a/packages/cli/cli/src/cli-context/__test__/TaskContextImpl.test.ts b/packages/cli/cli/src/cli-context/__test__/TaskContextImpl.test.ts index 7b58d0bfeb36..2a40785f3ce0 100644 --- a/packages/cli/cli/src/cli-context/__test__/TaskContextImpl.test.ts +++ b/packages/cli/cli/src/cli-context/__test__/TaskContextImpl.test.ts @@ -42,6 +42,29 @@ function createInteractiveContext(): { subtask: InteractiveTaskContextImpl; getL return { subtask, getLogs: () => allLogs }; } +describe("TaskContextImpl.skipErrorReporting", () => { + it("marks the task failed without invoking reportError", () => { + const { context } = createContext(); + context.failWithoutThrowing("cli failure", new Error("network"), { skipErrorReporting: true }); + expect(context.getLastFailureMessage()).toBe("cli failure"); + expect(context.getResult()).toBe(TaskResult.Failure); + }); + + it("failAndThrow with skipErrorReporting sets message and throws TaskAbortSignal", () => { + const { subtask } = createInteractiveContext(); + subtask.start(); + try { + subtask.failAndThrow("Fiddle container failed", undefined, { + skipErrorReporting: true + }); + } catch (error) { + expect(error).toBeInstanceOf(TaskAbortSignal); + } + expect(subtask.getLastFailureMessage()).toBe("Fiddle container failed"); + expect(subtask.getResult()).toBe(TaskResult.Failure); + }); +}); + describe("TaskContextImpl.getLastFailureMessage", () => { it("returns the message passed to failWithoutThrowing", () => { const { context } = createContext(); diff --git a/packages/cli/cli/src/commands/automations/generate/GeneratorRunResult.ts b/packages/cli/cli/src/commands/automations/generate/GeneratorRunResult.ts index 3c568c6e149b..80cfcd2594f4 100644 --- a/packages/cli/cli/src/commands/automations/generate/GeneratorRunResult.ts +++ b/packages/cli/cli/src/commands/automations/generate/GeneratorRunResult.ts @@ -1,6 +1,14 @@ import { assertNever } from "@fern-api/core-utils"; - import type { GeneratorSkipReason, PublishTarget, RemoteGeneratorRunRecorder } from "@fern-api/remote-workspace-runner"; +import { resolveErrorCode } from "@fern-api/task-context"; +import type { AutomationTelemetryEmitOptions } from "../../../telemetry/AutomationTelemetryManager.js"; +import { + generatorCompletedAttributes, + generatorFailedAttributes, + generatorSkippedAttributes +} from "../../../telemetry/automationTelemetryAttributes.js"; +import { isAutomationMode } from "../../../telemetry/automationTelemetryContext.js"; +import { AUTOMATION_EVENT_NAMES, type AutomationTelemetryEvent } from "../../../telemetry/automationTelemetryEvent.js"; export type { GeneratorSkipReason, PublishTarget }; @@ -46,6 +54,10 @@ export interface GeneratorRunCounts { skipped: number; } +export interface GeneratorRunCollectorOptions { + emitAutomationEvent?: (event: AutomationTelemetryEvent, options?: AutomationTelemetryEmitOptions) => void; +} + /** Single source of truth for aggregating status counts across results. */ export function countResults(results: readonly GeneratorRunResult[]): GeneratorRunCounts { let succeeded = 0; @@ -117,6 +129,11 @@ export function buildGeneratorsYmlUrl(absolutePath: string | undefined, lineNumb export class GeneratorRunCollector implements RemoteGeneratorRunRecorder { readonly #results: GeneratorRunResult[] = []; + readonly #emitAutomationEvent: GeneratorRunCollectorOptions["emitAutomationEvent"]; + + public constructor(options?: GeneratorRunCollectorOptions) { + this.#emitAutomationEvent = options?.emitAutomationEvent; + } public recordSuccess(args: Parameters[0]): void { this.#results.push({ @@ -136,6 +153,19 @@ export class GeneratorRunCollector implements RemoteGeneratorRunRecorder { generatorsYmlWorkspaceRelativePath: resolveGithubWorkspaceRelativePath(args.generatorsYmlAbsolutePath), generatorsYmlLineNumber: args.generatorsYmlLineNumber ?? null }); + this.emitIfPresent({ + event: AUTOMATION_EVENT_NAMES.GENERATOR_COMPLETED, + attributes: generatorCompletedAttributes({ + generatorName: args.generatorName, + groupName: args.groupName, + apiName: args.apiName, + sdkRepoUrl: args.outputRepoUrl, + version: args.version, + pullRequestUrl: args.pullRequestUrl, + noChangesDetected: args.noChangesDetected, + publishTarget: args.publishTarget + }) + }); } public recordFailure(args: Parameters[0]): void { @@ -156,6 +186,22 @@ export class GeneratorRunCollector implements RemoteGeneratorRunRecorder { generatorsYmlWorkspaceRelativePath: resolveGithubWorkspaceRelativePath(args.generatorsYmlAbsolutePath), generatorsYmlLineNumber: args.generatorsYmlLineNumber ?? null }); + const failureSource = args.failureSource ?? "cli"; + this.emitIfPresent( + { + event: AUTOMATION_EVENT_NAMES.GENERATOR_FAILED, + errorCode: resolveErrorCode(args.cliError), + attributes: generatorFailedAttributes({ + generatorName: args.generatorName, + groupName: args.groupName, + apiName: args.apiName, + sdkRepoUrl: args.outputRepoUrl, + errorMessage: args.errorMessage, + failureSource + }) + }, + failureSource === "cli" ? { error: args.cliError } : undefined + ); } public recordSkipped(args: Parameters[0]): void { @@ -176,6 +222,16 @@ export class GeneratorRunCollector implements RemoteGeneratorRunRecorder { generatorsYmlWorkspaceRelativePath: resolveGithubWorkspaceRelativePath(args.generatorsYmlAbsolutePath), generatorsYmlLineNumber: args.generatorsYmlLineNumber ?? null }); + this.emitIfPresent({ + event: AUTOMATION_EVENT_NAMES.GENERATOR_SKIPPED, + attributes: generatorSkippedAttributes({ + generatorName: args.generatorName, + groupName: args.groupName, + apiName: args.apiName, + sdkRepoUrl: args.outputRepoUrl, + skipReason: args.reason + }) + }); } public results(): readonly GeneratorRunResult[] { @@ -189,4 +245,11 @@ export class GeneratorRunCollector implements RemoteGeneratorRunRecorder { public counts(): GeneratorRunCounts { return countResults(this.#results); } + + private emitIfPresent(event: AutomationTelemetryEvent, options?: AutomationTelemetryEmitOptions): void { + if (!isAutomationMode() || this.#emitAutomationEvent == null) { + return; + } + this.#emitAutomationEvent(event, options); + } } diff --git a/packages/cli/cli/src/commands/automations/generate/__test__/GeneratorRunCollector.test.ts b/packages/cli/cli/src/commands/automations/generate/__test__/GeneratorRunCollector.test.ts index add7d60c2a4c..c802120c5c34 100644 --- a/packages/cli/cli/src/commands/automations/generate/__test__/GeneratorRunCollector.test.ts +++ b/packages/cli/cli/src/commands/automations/generate/__test__/GeneratorRunCollector.test.ts @@ -1,7 +1,118 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; +import { AUTOMATION_EVENT_NAMES } from "../../../../telemetry/automationTelemetryEvent.js"; import { GeneratorRunCollector } from "../GeneratorRunResult.js"; +describe("GeneratorRunCollector telemetry", () => { + const originalAutomationMode = process.env.FERN_AUTOMATION; + + afterEach(() => { + if (originalAutomationMode == null) { + delete process.env.FERN_AUTOMATION; + } else { + process.env.FERN_AUTOMATION = originalAutomationMode; + } + }); + + it("emits generator_completed when automation mode is enabled", () => { + process.env.FERN_AUTOMATION = "true"; + const emitted: unknown[] = []; + const collector = new GeneratorRunCollector({ + emitAutomationEvent: (event) => { + emitted.push(event); + } + }); + + collector.recordSuccess({ + apiName: "foo", + groupName: "python-sdk", + generatorName: "fernapi/fern-python-sdk", + version: "0.1.0", + durationMs: 10, + pullRequestUrl: undefined, + noChangesDetected: undefined, + publishTarget: undefined, + outputRepoUrl: undefined, + generatorsYmlAbsolutePath: undefined, + generatorsYmlLineNumber: undefined + }); + + expect(emitted).toHaveLength(1); + expect(emitted[0]).toMatchObject({ + event: AUTOMATION_EVENT_NAMES.GENERATOR_COMPLETED + }); + }); + + it("emits generator_failed with cli error options", () => { + process.env.FERN_AUTOMATION = "true"; + const emitted: Array<{ event: string; options?: { error?: unknown } }> = []; + const cliError = new Error("network down"); + const collector = new GeneratorRunCollector({ + emitAutomationEvent: (event, options) => { + emitted.push({ event: event.event, options }); + } + }); + + collector.recordFailure({ + apiName: undefined, + groupName: "g", + generatorName: "x", + errorMessage: "network down", + durationMs: 1, + failureSource: "cli", + cliError, + outputRepoUrl: undefined, + generatorsYmlAbsolutePath: undefined, + generatorsYmlLineNumber: undefined + }); + + collector.recordSuccess({ + apiName: undefined, + groupName: "g", + generatorName: "y", + version: "2.0.0", + durationMs: 1, + pullRequestUrl: undefined, + noChangesDetected: undefined, + publishTarget: undefined, + outputRepoUrl: undefined, + generatorsYmlAbsolutePath: undefined, + generatorsYmlLineNumber: undefined + }); + + expect(emitted[0]).toMatchObject({ + event: AUTOMATION_EVENT_NAMES.GENERATOR_FAILED, + options: { error: cliError } + }); + expect(emitted.map((entry) => entry.event)).toEqual([ + AUTOMATION_EVENT_NAMES.GENERATOR_FAILED, + AUTOMATION_EVENT_NAMES.GENERATOR_COMPLETED + ]); + }); + + it("does not emit when automation mode is disabled", () => { + delete process.env.FERN_AUTOMATION; + const emitted: unknown[] = []; + const collector = new GeneratorRunCollector({ + emitAutomationEvent: (event) => { + emitted.push(event); + } + }); + + collector.recordSkipped({ + apiName: undefined, + groupName: "g", + generatorName: "x", + reason: "opted_out", + outputRepoUrl: undefined, + generatorsYmlAbsolutePath: undefined, + generatorsYmlLineNumber: undefined + }); + + expect(emitted).toHaveLength(0); + }); +}); + describe("GeneratorRunCollector", () => { it("is empty on construction", () => { const collector = new GeneratorRunCollector(); diff --git a/packages/cli/cli/src/commands/automations/generate/executeAutomationsGenerate.ts b/packages/cli/cli/src/commands/automations/generate/executeAutomationsGenerate.ts index 8d36ab4e1af8..0644fdcaff55 100644 --- a/packages/cli/cli/src/commands/automations/generate/executeAutomationsGenerate.ts +++ b/packages/cli/cli/src/commands/automations/generate/executeAutomationsGenerate.ts @@ -3,9 +3,12 @@ import { TaskAbortSignal } from "@fern-api/task-context"; import { CliContext } from "../../../cli-context/CliContext.js"; import { loadProjectAndRegisterWorkspacesWithContext } from "../../../cliCommons.js"; +import { generationRunAttributes } from "../../../telemetry/automationTelemetryAttributes.js"; +import { isAutomationMode } from "../../../telemetry/automationTelemetryContext.js"; +import { AUTOMATION_EVENT_NAMES } from "../../../telemetry/automationTelemetryEvent.js"; import { parseGeneratorArg } from "../../generate/filterGenerators.js"; import { generateAPIWorkspaces } from "../../generate/generateAPIWorkspaces.js"; -import { GeneratorRunCollector } from "./GeneratorRunResult.js"; +import { countResults, GeneratorRunCollector } from "./GeneratorRunResult.js"; import { renderGithubAnnotationsForResults } from "./renderGithubAnnotationsForResults.js"; import { renderStdoutSummary, writeResults, writeResultsSync } from "./reportGenerateResults.js"; @@ -33,7 +36,9 @@ export async function executeAutomationsGenerate({ options: AutomationsGenerateOptions; }): Promise { const { generatorName, generatorIndex } = parseGeneratorArg(options.generator); - const collector = new GeneratorRunCollector(); + const collector = new GeneratorRunCollector({ + emitAutomationEvent: (event, emitOptions) => cliContext.emitAutomationTelemetryEvent(event, emitOptions) + }); const { jsonOutputPath } = options; // Sentinel: the happy-path finally skips its write if a signal handler already flushed. @@ -68,6 +73,12 @@ export async function executeAutomationsGenerate({ await withSuppressedLoggerAnnotations(async () => { try { await cliContext.runTask(async () => { + if (isAutomationMode()) { + cliContext.emitAutomationTelemetryEvent({ + event: AUTOMATION_EVENT_NAMES.GENERATION_STARTED + }); + } + await generateAPIWorkspaces({ project: await loadProjectAndRegisterWorkspacesWithContext(cliContext, { commandLineApiWorkspace: options.api, @@ -118,6 +129,12 @@ export async function executeAutomationsGenerate({ // `process.once` self-removes if fired; these are no-ops on the signal path. process.off("SIGINT", flushOnSignal); process.off("SIGTERM", flushOnSignal); + if (isAutomationMode()) { + cliContext.emitAutomationTelemetryEvent({ + event: AUTOMATION_EVENT_NAMES.GENERATION_COMPLETED, + attributes: generationRunAttributes(countResults(collector.results())) + }); + } if (!outputsFlushed) { outputsFlushed = true; await reportFinalOutputs({ collector, jsonOutputPath, taskAborted }); diff --git a/packages/cli/cli/src/telemetry/AutomationTelemetryManager.ts b/packages/cli/cli/src/telemetry/AutomationTelemetryManager.ts index b05c31ee008e..025b1ee004c7 100644 --- a/packages/cli/cli/src/telemetry/AutomationTelemetryManager.ts +++ b/packages/cli/cli/src/telemetry/AutomationTelemetryManager.ts @@ -77,6 +77,9 @@ export class AutomationTelemetryManager { if (!isFailureAutomationEventName(event.event) || event.errorCode == null) { return undefined; } + if (event.attributes?.failure_source === "container") { + return undefined; + } return this.reporter.captureException(error, { tags: toSentryTags(event, context), context: { diff --git a/packages/cli/cli/src/telemetry/__test__/AutomationTelemetryManager.test.ts b/packages/cli/cli/src/telemetry/__test__/AutomationTelemetryManager.test.ts new file mode 100644 index 000000000000..6692234e6802 --- /dev/null +++ b/packages/cli/cli/src/telemetry/__test__/AutomationTelemetryManager.test.ts @@ -0,0 +1,49 @@ +import { CliError } from "@fern-api/task-context"; +import { describe, expect, it, vi } from "vitest"; + +import { AutomationTelemetryManager } from "../AutomationTelemetryManager.js"; +import { AUTOMATION_EVENT_NAMES } from "../automationTelemetryEvent.js"; + +describe("AutomationTelemetryManager", () => { + it("skips CLI Sentry when failure_source is container", () => { + const captureException = vi.fn(); + const manager = new AutomationTelemetryManager({ + instrumentPostHogAutomationEvent: vi.fn(), + captureException + }); + + manager.emit({ + event: AUTOMATION_EVENT_NAMES.GENERATOR_FAILED, + errorCode: CliError.Code.ContainerError, + attributes: { + failure_source: "container", + error_message: "Generator failed in container" + } + }); + + expect(captureException).not.toHaveBeenCalled(); + }); + + it("captures CLI Sentry when failure_source is cli", () => { + const captureException = vi.fn().mockReturnValue("sentry-id"); + const manager = new AutomationTelemetryManager({ + instrumentPostHogAutomationEvent: vi.fn(), + captureException + }); + const error = new Error("network down"); + + manager.emit( + { + event: AUTOMATION_EVENT_NAMES.GENERATOR_FAILED, + errorCode: CliError.Code.NetworkError, + attributes: { + failure_source: "cli", + error_message: "network down" + } + }, + { error } + ); + + expect(captureException).toHaveBeenCalledWith(error, expect.any(Object)); + }); +}); diff --git a/packages/cli/cli/src/telemetry/__test__/automationTelemetryAttributes.test.ts b/packages/cli/cli/src/telemetry/__test__/automationTelemetryAttributes.test.ts new file mode 100644 index 000000000000..e44ae5dbe1e6 --- /dev/null +++ b/packages/cli/cli/src/telemetry/__test__/automationTelemetryAttributes.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; + +import { + deriveGeneratorCompletedOutcome, + generationRunAttributes, + generatorCompletedAttributes, + generatorFailedAttributes +} from "../automationTelemetryAttributes.js"; + +describe("automationTelemetryAttributes", () => { + it("derives pr_created when a pull request URL is present", () => { + expect( + deriveGeneratorCompletedOutcome({ + pullRequestUrl: "https://github.com/acme/sdk/pull/1", + noChangesDetected: false, + publishTarget: undefined + }) + ).toBe("pr_created"); + }); + + it("derives no_diff when Fiddle reported no changes", () => { + expect( + deriveGeneratorCompletedOutcome({ + pullRequestUrl: undefined, + noChangesDetected: true, + publishTarget: undefined + }) + ).toBe("no_diff"); + }); + + it("derives published when a registry target is present", () => { + expect( + deriveGeneratorCompletedOutcome({ + pullRequestUrl: undefined, + noChangesDetected: false, + publishTarget: { + registry: "pypi", + label: "PyPI", + version: "1.0.0", + url: "https://pypi.org/project/acme/1.0.0/" + } + }) + ).toBe("published"); + }); + + it("builds generator_completed attributes with outcome", () => { + expect( + generatorCompletedAttributes({ + generatorName: "fernapi/fern-python-sdk", + groupName: "python-sdk", + apiName: "api", + sdkRepoUrl: "https://github.com/acme/sdk", + version: "1.0.0", + pullRequestUrl: "https://github.com/acme/sdk/pull/1", + noChangesDetected: false, + publishTarget: undefined + }) + ).toMatchObject({ + generator_name: "fernapi/fern-python-sdk", + group_name: "python-sdk", + api_name: "api", + outcome: "pr_created", + version: "1.0.0" + }); + }); + + it("builds generator_failed attributes with failure_source", () => { + expect( + generatorFailedAttributes({ + generatorName: "x", + groupName: "g", + apiName: undefined, + sdkRepoUrl: undefined, + errorMessage: "boom", + failureSource: "cli" + }) + ).toEqual({ + generator_name: "x", + group_name: "g", + error_message: "boom", + failure_source: "cli" + }); + }); + + it("builds generation_completed run aggregates", () => { + expect(generationRunAttributes({ succeeded: 2, failed: 1, skipped: 1 })).toEqual({ + succeeded: 2, + failed: 1, + skipped: 1 + }); + }); +}); diff --git a/packages/cli/cli/src/telemetry/__test__/automationTelemetryEvent.test.ts b/packages/cli/cli/src/telemetry/__test__/automationTelemetryEvent.test.ts new file mode 100644 index 000000000000..31347ba02410 --- /dev/null +++ b/packages/cli/cli/src/telemetry/__test__/automationTelemetryEvent.test.ts @@ -0,0 +1,61 @@ +import { CliError } from "@fern-api/task-context"; +import { describe, expect, it } from "vitest"; + +import { + AUTOMATION_EVENT_NAMES, + failureEventNameForCommand, + isFailureAutomationEventName, + toSentryTags +} from "../automationTelemetryEvent.js"; + +describe("automationTelemetryEvent", () => { + it("marks generate failure events for Sentry gating", () => { + expect(isFailureAutomationEventName(AUTOMATION_EVENT_NAMES.GENERATION_FAILED)).toBe(true); + expect(isFailureAutomationEventName(AUTOMATION_EVENT_NAMES.GENERATOR_FAILED)).toBe(true); + expect(isFailureAutomationEventName(AUTOMATION_EVENT_NAMES.GENERATOR_COMPLETED)).toBe(false); + expect(isFailureAutomationEventName(AUTOMATION_EVENT_NAMES.GENERATOR_SKIPPED)).toBe(false); + expect(isFailureAutomationEventName(AUTOMATION_EVENT_NAMES.GENERATION_STARTED)).toBe(false); + }); + + it("maps automations generate argv to generation_failed", () => { + expect(failureEventNameForCommand(["node", "fern", "automations", "generate"])).toBe( + AUTOMATION_EVENT_NAMES.GENERATION_FAILED + ); + }); + + it("includes generator identity tags in Sentry tags", () => { + expect( + toSentryTags( + { + event: AUTOMATION_EVENT_NAMES.GENERATOR_FAILED, + errorCode: CliError.Code.ContainerError, + attributes: { + generator_name: "fernapi/fern-python-sdk", + group_name: "python-sdk", + api_name: "api", + failure_source: "container" + } + }, + { + action: "generate", + run_id: "run-1", + github_run_id: undefined, + github_run_url: undefined, + org: "acme", + config_repo: "acme/config", + config_commit_sha: undefined, + config_branch: undefined, + config_pr_number: undefined, + trigger: undefined, + cli_version: "1.0.0" + } + ) + ).toMatchObject({ + event: AUTOMATION_EVENT_NAMES.GENERATOR_FAILED, + generator_name: "fernapi/fern-python-sdk", + group_name: "python-sdk", + api_name: "api", + failure_source: "container" + }); + }); +}); diff --git a/packages/cli/cli/src/telemetry/automationTelemetryAttributes.ts b/packages/cli/cli/src/telemetry/automationTelemetryAttributes.ts new file mode 100644 index 000000000000..6feca0dae762 --- /dev/null +++ b/packages/cli/cli/src/telemetry/automationTelemetryAttributes.ts @@ -0,0 +1,95 @@ +import type { PublishTarget } from "@fern-api/remote-workspace-runner"; + +import type { GeneratorRunCounts } from "../commands/automations/generate/GeneratorRunResult.js"; +import type { JsonValue } from "./automationTelemetryEvent.js"; + +export type GeneratorFailureSource = "container" | "cli"; + +export type GeneratorCompletedOutcome = "pr_created" | "no_diff" | "published" | "pushed"; + +export function generatorIdentityAttributes(args: { + generatorName: string; + groupName: string; + apiName: string | undefined; + sdkRepoUrl: string | undefined; +}): Record { + return { + generator_name: args.generatorName, + group_name: args.groupName, + ...(args.apiName != null ? { api_name: args.apiName } : {}), + ...(args.sdkRepoUrl != null ? { sdk_repo_url: args.sdkRepoUrl } : {}) + }; +} + +export function deriveGeneratorCompletedOutcome(args: { + pullRequestUrl: string | undefined; + noChangesDetected: boolean | undefined; + publishTarget: PublishTarget | undefined; +}): GeneratorCompletedOutcome { + if (args.noChangesDetected === true) { + return "no_diff"; + } + if (args.pullRequestUrl != null) { + return "pr_created"; + } + if (args.publishTarget != null) { + return "published"; + } + return "pushed"; +} + +export function generatorCompletedAttributes(args: { + generatorName: string; + groupName: string; + apiName: string | undefined; + sdkRepoUrl: string | undefined; + version: string | null; + pullRequestUrl: string | undefined; + noChangesDetected: boolean | undefined; + publishTarget: PublishTarget | undefined; +}): Record { + return { + ...generatorIdentityAttributes(args), + outcome: deriveGeneratorCompletedOutcome(args), + ...(args.version != null ? { version: args.version } : {}), + ...(args.pullRequestUrl != null ? { pull_request_url: args.pullRequestUrl } : {}), + ...(args.noChangesDetected === true ? { no_diff_skipped: true } : {}), + ...(args.publishTarget != null ? { publish_target: args.publishTarget.registry } : {}) + }; +} + +export function generatorFailedAttributes(args: { + generatorName: string; + groupName: string; + apiName: string | undefined; + sdkRepoUrl: string | undefined; + errorMessage: string; + failureSource: GeneratorFailureSource; +}): Record { + return { + ...generatorIdentityAttributes(args), + error_message: args.errorMessage, + failure_source: args.failureSource + }; +} + +export function generatorSkippedAttributes(args: { + generatorName: string; + groupName: string; + apiName: string | undefined; + sdkRepoUrl: string | undefined; + skipReason: string; +}): Record { + return { + ...generatorIdentityAttributes(args), + skip_reason: args.skipReason + }; +} + +export function generationRunAttributes(counts: GeneratorRunCounts): Record { + return { + succeeded: counts.succeeded, + failed: counts.failed, + skipped: counts.skipped + }; +} diff --git a/packages/cli/cli/src/telemetry/automationTelemetryEvent.ts b/packages/cli/cli/src/telemetry/automationTelemetryEvent.ts index d4592a915af7..a65a0bdcf04f 100644 --- a/packages/cli/cli/src/telemetry/automationTelemetryEvent.ts +++ b/packages/cli/cli/src/telemetry/automationTelemetryEvent.ts @@ -18,10 +18,11 @@ export const AUTOMATION_EVENT_NAMES = { GENERATION_STARTED: "generation_started", GENERATION_COMPLETED: "generation_completed", GENERATION_FAILED: "generation_failed", + GENERATOR_COMPLETED: "generator_completed", + GENERATOR_FAILED: "generator_failed", + GENERATOR_SKIPPED: "generator_skipped", VERIFICATION_FAILED: "verification_failed", - SDK_PR_CREATED: "sdk_pr_created", - UPGRADE_APPLIED: "upgrade_applied", - MAJOR_VERSION_BUMP: "major_version_bump" + UPGRADE_APPLIED: "upgrade_applied" } as const; export type AutomationEventName = (typeof AUTOMATION_EVENT_NAMES)[keyof typeof AUTOMATION_EVENT_NAMES]; @@ -34,10 +35,11 @@ const AUTOMATION_EVENT_FAILURE: Record = { [AUTOMATION_EVENT_NAMES.GENERATION_STARTED]: false, [AUTOMATION_EVENT_NAMES.GENERATION_COMPLETED]: false, [AUTOMATION_EVENT_NAMES.GENERATION_FAILED]: true, + [AUTOMATION_EVENT_NAMES.GENERATOR_COMPLETED]: false, + [AUTOMATION_EVENT_NAMES.GENERATOR_FAILED]: true, + [AUTOMATION_EVENT_NAMES.GENERATOR_SKIPPED]: false, [AUTOMATION_EVENT_NAMES.VERIFICATION_FAILED]: true, - [AUTOMATION_EVENT_NAMES.SDK_PR_CREATED]: false, - [AUTOMATION_EVENT_NAMES.UPGRADE_APPLIED]: false, - [AUTOMATION_EVENT_NAMES.MAJOR_VERSION_BUMP]: false + [AUTOMATION_EVENT_NAMES.UPGRADE_APPLIED]: false }; export function isFailureAutomationEventName(event: AutomationEventName): boolean { @@ -93,6 +95,11 @@ export function toSentryTags( } } + function stringAttribute(attributes: Record | undefined, key: string): string | undefined { + const value = attributes?.[key]; + return typeof value === "string" ? value : undefined; + } + const tags: Record = { surface: "cli", automation_mode: "true" @@ -104,6 +111,10 @@ export function toSentryTags( appendStringIfPresent(tags, "config_repo", context.config_repo); appendStringIfPresent(tags, "trigger", context.trigger); appendStringIfPresent(tags, "error_code", event.errorCode ?? "none"); + appendStringIfPresent(tags, "generator_name", stringAttribute(event.attributes, "generator_name")); + appendStringIfPresent(tags, "group_name", stringAttribute(event.attributes, "group_name")); + appendStringIfPresent(tags, "api_name", stringAttribute(event.attributes, "api_name")); + appendStringIfPresent(tags, "failure_source", stringAttribute(event.attributes, "failure_source")); return tags; } diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/RemoteGeneratorRunRecorder.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/RemoteGeneratorRunRecorder.ts index 245f635b313a..4a9476594351 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/RemoteGeneratorRunRecorder.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/RemoteGeneratorRunRecorder.ts @@ -2,6 +2,8 @@ import type { PublishTarget } from "./publishTarget.js"; export type { PublishTarget }; +export type GeneratorFailureSource = "container" | "cli"; + /** * Why a generator was not executed in automation fan-out mode. * - "local_output": output is configured for local-file-system (can't run remotely). @@ -65,6 +67,9 @@ export interface RemoteGeneratorRunRecorder { args: GeneratorRunIdentity & { errorMessage: string; durationMs: number; + failureSource?: GeneratorFailureSource; + /** Real error object for CLI-path failures — used for Sentry at emit time only. */ + cliError?: unknown; } ): void; diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/RemoteTaskHandler.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/RemoteTaskHandler.ts index 4bf808caa815..1aca470367db 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/RemoteTaskHandler.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/RemoteTaskHandler.ts @@ -207,7 +207,10 @@ export class RemoteTaskHandler { if (s3PreSignedReadUrl != null) { logS3Url(s3PreSignedReadUrl); } - this.context.failAndThrow(message, undefined, { code: CliError.Code.ContainerError }); + this.context.failAndThrow(message, undefined, { + code: CliError.Code.ContainerError, + skipErrorReporting: true + }); }, finished: async (finishedStatus) => { if (finishedStatus.s3PreSignedReadUrlV2 != null) { diff --git a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForAPIWorkspace.ts b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForAPIWorkspace.ts index b5b7aad60eee..07b02e06cd27 100644 --- a/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForAPIWorkspace.ts +++ b/packages/cli/generation/remote-generation/remote-workspace-runner/src/runRemoteGenerationForAPIWorkspace.ts @@ -412,6 +412,7 @@ async function generateOne({ generatorName: generatorInvocation.name, errorMessage: interactiveTaskContext.getLastFailureMessage() ?? "Generator failed", durationMs: Date.now() - startedAt, + failureSource: "container", outputRepoUrl: getOutputRepoUrl(generatorInvocation), generatorsYmlAbsolutePath, generatorsYmlLineNumber: lineNumber @@ -433,6 +434,8 @@ async function generateOne({ generatorName: generatorInvocation.name, errorMessage: message, durationMs: Date.now() - startedAt, + failureSource: "cli", + cliError: error, outputRepoUrl: getOutputRepoUrl(generatorInvocation), generatorsYmlAbsolutePath, generatorsYmlLineNumber: lineNumber @@ -441,7 +444,8 @@ async function generateOne({ // Pass the error object so resolveErrorCode can extract CliError codes, // errno codes, etc. instead of defaulting to InternalError. interactiveTaskContext.failWithoutThrowing(message, error, { - code: resolveErrorCode(error) + code: resolveErrorCode(error), + skipErrorReporting: true }); } } diff --git a/packages/cli/task-context/src/TaskContext.ts b/packages/cli/task-context/src/TaskContext.ts index 850c5e5d5d38..1c64ce8b4921 100644 --- a/packages/cli/task-context/src/TaskContext.ts +++ b/packages/cli/task-context/src/TaskContext.ts @@ -7,11 +7,16 @@ export interface CaptureExceptionOptions { context?: Record | undefined>; } +export interface TaskFailOptions { + code?: CliError.Code; + skipErrorReporting?: boolean; +} + export interface TaskContext { logger: Logger; takeOverTerminal: (run: () => void | Promise) => Promise; - failAndThrow: (message?: string, error?: unknown, options?: { code?: CliError.Code }) => never; - failWithoutThrowing: (message?: string, error?: unknown, options?: { code?: CliError.Code }) => void; + failAndThrow: (message?: string, error?: unknown, options?: TaskFailOptions) => never; + failWithoutThrowing: (message?: string, error?: unknown, options?: TaskFailOptions) => void; captureException: (error: unknown, options?: CaptureExceptionOptions) => string | undefined; getResult: () => TaskResult; /** diff --git a/packages/cli/task-context/src/index.ts b/packages/cli/task-context/src/index.ts index 74a49dc3b53b..33ad2b82971c 100644 --- a/packages/cli/task-context/src/index.ts +++ b/packages/cli/task-context/src/index.ts @@ -10,5 +10,6 @@ export { type PosthogEvent, type Startable, type TaskContext, + type TaskFailOptions, TaskResult } from "./TaskContext.js"; diff --git a/packages/seed/src/TaskContextImpl.ts b/packages/seed/src/TaskContextImpl.ts index ab6f6cc7b8e6..d6772ed0e2f8 100644 --- a/packages/seed/src/TaskContextImpl.ts +++ b/packages/seed/src/TaskContextImpl.ts @@ -12,6 +12,7 @@ import { Startable, TaskAbortSignal, TaskContext, + TaskFailOptions, TaskResult } from "@fern-api/task-context"; import chalk from "chalk"; @@ -84,13 +85,13 @@ export class TaskContextImpl implements Startable, Finishable, Task public takeOverTerminal: (run: () => void | Promise) => Promise; - public failAndThrow(message?: string, error?: unknown, _options?: { code?: CliError.Code }): never { + public failAndThrow(message?: string, error?: unknown, _options?: TaskFailOptions): never { this.failWithoutThrowing(message, error); this.finish(); throw new TaskAbortSignal(); } - public failWithoutThrowing(message?: string, error?: unknown, _options?: { code?: CliError.Code }): void { + public failWithoutThrowing(message?: string, error?: unknown, _options?: TaskFailOptions): void { if (message != null) { this.lastFailureMessage = message; }