feat(cli): instrument fern automations generate telemetry

packages/cli/cli-v2/src/context/adapter/TaskContextAdapter.ts

@@ -1,14 +1,14 @@
 import { createLogger, LOG_LEVELS, Logger, LogLevel } from "@fern-api/logger";
 import {
     type CaptureExceptionOptions,
-    type CliError,
     type CreateInteractiveTaskParams,
     type Finishable,
     type InteractiveTaskContext,
     type PosthogEvent,
     type Startable,
     TaskAbortSignal,
     type TaskContext,
+    TaskFailOptions,
     TaskResult
 } from "@fern-api/task-context";
 
@@ -56,12 +56,12 @@ export class TaskContextAdapter implements TaskContext {
         await run();
     }
 
-    public failAndThrow(message?: string, error?: unknown, options?: { code?: CliError.Code }): never {
+    public failAndThrow(message?: string, error?: unknown, options?: TaskFailOptions): never {
         this.failWithoutThrowing(message, error, options);
         throw new TaskAbortSignal();
     }
 
-    public failWithoutThrowing(message?: string, error?: unknown, options?: { code?: CliError.Code }): void {
+    public failWithoutThrowing(message?: string, error?: unknown, options?: TaskFailOptions): void {
         this.result = TaskResult.Failure;
         if (error instanceof TaskAbortSignal) {
             return;
@@ -72,7 +72,9 @@ export class TaskContextAdapter implements TaskContext {
             this.logger.error(fullMessage);
         }
 
-        reportError(this.context, error, { ...options, message });
+        if (!options?.skipErrorReporting) {
+            reportError(this.context, error, { ...options, message });
+        }
     }
 
     public getLastFailureMessage(): string | undefined {

packages/cli/cli/changes/unreleased/automations-generate-telemetry.yml

@@ -0,0 +1,5 @@
+- summary: |
+    Emit Tier 1 automation telemetry events for `fern automations generate`, including
+    per-generator success/failure/skip events and run-level lifecycle events. Per-generator
+    Fiddle failures no longer emit mislabeled run-level `generation_failed` Sentry events.
+  type: feat

packages/cli/cli/src/cli-context/TaskContextImpl.ts

@@ -3,14 +3,14 @@ import { addPrefixToString } from "@fern-api/core-utils";
 import { createLogger, LogLevel } from "@fern-api/logger";
 import {
     type CaptureExceptionOptions,
-    CliError,
     CreateInteractiveTaskParams,
     Finishable,
     InteractiveTaskContext,
     PosthogEvent,
     Startable,
     TaskAbortSignal,
     TaskContext,
+    TaskFailOptions,
     TaskResult
 } from "@fern-api/task-context";
 
@@ -107,13 +107,13 @@ export class TaskContextImpl implements Startable<TaskContext>, Finishable, Task
 
     public takeOverTerminal: (run: () => void | Promise<void>) => Promise<void>;
 
-    public failAndThrow(message?: string, error?: unknown, options?: { code?: CliError.Code }): never {
+    public failAndThrow(message?: string, error?: unknown, options?: TaskFailOptions): never {
         this.failWithoutThrowing(message, error, options);
         this.finish();
         throw new TaskAbortSignal();
     }
 
-    public failWithoutThrowing(message?: string, error?: unknown, options?: { code?: CliError.Code }): void {
+    public failWithoutThrowing(message?: string, error?: unknown, options?: TaskFailOptions): void {
         this.result = TaskResult.Failure;
         if (error instanceof TaskAbortSignal) {
             return;
@@ -122,7 +122,9 @@ export class TaskContextImpl implements Startable<TaskContext>, Finishable, Task
             this.lastFailureMessage = message;
         }
         logErrorMessage({ message, error, logger: this.logger });
-        reportError(this, error, { ...options, message });
+        if (!options?.skipErrorReporting) {
+            reportError(this, error, { ...options, message });
+        }
     }
 
     public getLastFailureMessage(): string | undefined {

packages/cli/cli/src/cli-context/__test__/TaskContextImpl.test.ts

@@ -42,6 +42,29 @@ function createInteractiveContext(): { subtask: InteractiveTaskContextImpl; getL
     return { subtask, getLogs: () => allLogs };
 }
 
+describe("TaskContextImpl.skipErrorReporting", () => {
+    it("marks the task failed without invoking reportError", () => {
+        const { context } = createContext();
+        context.failWithoutThrowing("cli failure", new Error("network"), { skipErrorReporting: true });
+        expect(context.getLastFailureMessage()).toBe("cli failure");
+        expect(context.getResult()).toBe(TaskResult.Failure);
+    });
+
+    it("failAndThrow with skipErrorReporting sets message and throws TaskAbortSignal", () => {
+        const { subtask } = createInteractiveContext();
+        subtask.start();
+        try {
+            subtask.failAndThrow("Fiddle container failed", undefined, {
+                skipErrorReporting: true
+            });
+        } catch (error) {
+            expect(error).toBeInstanceOf(TaskAbortSignal);
+        }
+        expect(subtask.getLastFailureMessage()).toBe("Fiddle container failed");
+        expect(subtask.getResult()).toBe(TaskResult.Failure);
+    });
+});
+
 describe("TaskContextImpl.getLastFailureMessage", () => {
     it("returns the message passed to failWithoutThrowing", () => {
         const { context } = createContext();

packages/cli/cli/src/commands/automations/generate/GeneratorRunResult.ts

@@ -1,6 +1,14 @@
 import { assertNever } from "@fern-api/core-utils";
-
 import type { GeneratorSkipReason, PublishTarget, RemoteGeneratorRunRecorder } from "@fern-api/remote-workspace-runner";
+import { resolveErrorCode } from "@fern-api/task-context";
+import type { AutomationTelemetryEmitOptions } from "../../../telemetry/AutomationTelemetryManager.js";
+import {
+    generatorCompletedAttributes,
+    generatorFailedAttributes,
+    generatorSkippedAttributes
+} from "../../../telemetry/automationTelemetryAttributes.js";
+import { isAutomationMode } from "../../../telemetry/automationTelemetryContext.js";
+import { AUTOMATION_EVENT_NAMES, type AutomationTelemetryEvent } from "../../../telemetry/automationTelemetryEvent.js";
 
 export type { GeneratorSkipReason, PublishTarget };
 
@@ -46,6 +54,10 @@ export interface GeneratorRunCounts {
     skipped: number;
 }
 
+export interface GeneratorRunCollectorOptions {
+    emitAutomationEvent?: (event: AutomationTelemetryEvent, options?: AutomationTelemetryEmitOptions) => void;
+}
+
 /** Single source of truth for aggregating status counts across results. */
 export function countResults(results: readonly GeneratorRunResult[]): GeneratorRunCounts {
     let succeeded = 0;
@@ -117,6 +129,11 @@ export function buildGeneratorsYmlUrl(absolutePath: string | undefined, lineNumb
 
 export class GeneratorRunCollector implements RemoteGeneratorRunRecorder {
     readonly #results: GeneratorRunResult[] = [];
+    readonly #emitAutomationEvent: GeneratorRunCollectorOptions["emitAutomationEvent"];
+
+    public constructor(options?: GeneratorRunCollectorOptions) {
+        this.#emitAutomationEvent = options?.emitAutomationEvent;
+    }
 
     public recordSuccess(args: Parameters<RemoteGeneratorRunRecorder["recordSuccess"]>[0]): void {
         this.#results.push({
@@ -136,6 +153,19 @@ export class GeneratorRunCollector implements RemoteGeneratorRunRecorder {
             generatorsYmlWorkspaceRelativePath: resolveGithubWorkspaceRelativePath(args.generatorsYmlAbsolutePath),
             generatorsYmlLineNumber: args.generatorsYmlLineNumber ?? null
         });
+        this.emitIfPresent({
+            event: AUTOMATION_EVENT_NAMES.GENERATOR_COMPLETED,
+            attributes: generatorCompletedAttributes({
+                generatorName: args.generatorName,
+                groupName: args.groupName,
+                apiName: args.apiName,
+                sdkRepoUrl: args.outputRepoUrl,
+                version: args.version,
+                pullRequestUrl: args.pullRequestUrl,
+                noChangesDetected: args.noChangesDetected,
+                publishTarget: args.publishTarget
+            })
+        });
     }
 
     public recordFailure(args: Parameters<RemoteGeneratorRunRecorder["recordFailure"]>[0]): void {
@@ -156,6 +186,22 @@ export class GeneratorRunCollector implements RemoteGeneratorRunRecorder {
             generatorsYmlWorkspaceRelativePath: resolveGithubWorkspaceRelativePath(args.generatorsYmlAbsolutePath),
             generatorsYmlLineNumber: args.generatorsYmlLineNumber ?? null
         });
+        const failureSource = args.failureSource ?? "cli";
+        this.emitIfPresent(
+            {
+                event: AUTOMATION_EVENT_NAMES.GENERATOR_FAILED,
+                errorCode: resolveErrorCode(args.cliError),
+                attributes: generatorFailedAttributes({
+                    generatorName: args.generatorName,
+                    groupName: args.groupName,
+                    apiName: args.apiName,
+                    sdkRepoUrl: args.outputRepoUrl,
+                    errorMessage: args.errorMessage,
+                    failureSource
+                })
+            },
+            failureSource === "cli" ? { error: args.cliError } : undefined
+        );
     }
 
     public recordSkipped(args: Parameters<RemoteGeneratorRunRecorder["recordSkipped"]>[0]): void {
@@ -176,6 +222,16 @@ export class GeneratorRunCollector implements RemoteGeneratorRunRecorder {
             generatorsYmlWorkspaceRelativePath: resolveGithubWorkspaceRelativePath(args.generatorsYmlAbsolutePath),
             generatorsYmlLineNumber: args.generatorsYmlLineNumber ?? null
         });
+        this.emitIfPresent({
+            event: AUTOMATION_EVENT_NAMES.GENERATOR_SKIPPED,
+            attributes: generatorSkippedAttributes({
+                generatorName: args.generatorName,
+                groupName: args.groupName,
+                apiName: args.apiName,
+                sdkRepoUrl: args.outputRepoUrl,
+                skipReason: args.reason
+            })
+        });
     }
 
     public results(): readonly GeneratorRunResult[] {
@@ -189,4 +245,11 @@ export class GeneratorRunCollector implements RemoteGeneratorRunRecorder {
     public counts(): GeneratorRunCounts {
         return countResults(this.#results);
     }
+
+    private emitIfPresent(event: AutomationTelemetryEvent, options?: AutomationTelemetryEmitOptions): void {
+        if (!isAutomationMode() || this.#emitAutomationEvent == null) {
+            return;
+        }
+        this.#emitAutomationEvent(event, options);
+    }
 }

packages/cli/cli/src/commands/automations/generate/__test__/GeneratorRunCollector.test.ts

@@ -1,7 +1,118 @@
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it } from "vitest";
 
+import { AUTOMATION_EVENT_NAMES } from "../../../../telemetry/automationTelemetryEvent.js";
 import { GeneratorRunCollector } from "../GeneratorRunResult.js";
 
+describe("GeneratorRunCollector telemetry", () => {
+    const originalAutomationMode = process.env.FERN_AUTOMATION;
+
+    afterEach(() => {
+        if (originalAutomationMode == null) {
+            delete process.env.FERN_AUTOMATION;
+        } else {
+            process.env.FERN_AUTOMATION = originalAutomationMode;
+        }
+    });
+
+    it("emits generator_completed when automation mode is enabled", () => {
+        process.env.FERN_AUTOMATION = "true";
+        const emitted: unknown[] = [];
+        const collector = new GeneratorRunCollector({
+            emitAutomationEvent: (event) => {
+                emitted.push(event);
+            }
+        });
+
+        collector.recordSuccess({
+            apiName: "foo",
+            groupName: "python-sdk",
+            generatorName: "fernapi/fern-python-sdk",
+            version: "0.1.0",
+            durationMs: 10,
+            pullRequestUrl: undefined,
+            noChangesDetected: undefined,
+            publishTarget: undefined,
+            outputRepoUrl: undefined,
+            generatorsYmlAbsolutePath: undefined,
+            generatorsYmlLineNumber: undefined
+        });
+
+        expect(emitted).toHaveLength(1);
+        expect(emitted[0]).toMatchObject({
+            event: AUTOMATION_EVENT_NAMES.GENERATOR_COMPLETED
+        });
+    });
+
+    it("emits generator_failed with cli error options", () => {
+        process.env.FERN_AUTOMATION = "true";
+        const emitted: Array<{ event: string; options?: { error?: unknown } }> = [];
+        const cliError = new Error("network down");
+        const collector = new GeneratorRunCollector({
+            emitAutomationEvent: (event, options) => {
+                emitted.push({ event: event.event, options });
+            }
+        });
+
+        collector.recordFailure({
+            apiName: undefined,
+            groupName: "g",
+            generatorName: "x",
+            errorMessage: "network down",
+            durationMs: 1,
+            failureSource: "cli",
+            cliError,
+            outputRepoUrl: undefined,
+            generatorsYmlAbsolutePath: undefined,
+            generatorsYmlLineNumber: undefined
+        });
+
+        collector.recordSuccess({
+            apiName: undefined,
+            groupName: "g",
+            generatorName: "y",
+            version: "2.0.0",
+            durationMs: 1,
+            pullRequestUrl: undefined,
+            noChangesDetected: undefined,
+            publishTarget: undefined,
+            outputRepoUrl: undefined,
+            generatorsYmlAbsolutePath: undefined,
+            generatorsYmlLineNumber: undefined
+        });
+
+        expect(emitted[0]).toMatchObject({
+            event: AUTOMATION_EVENT_NAMES.GENERATOR_FAILED,
+            options: { error: cliError }
+        });
+        expect(emitted.map((entry) => entry.event)).toEqual([
+            AUTOMATION_EVENT_NAMES.GENERATOR_FAILED,
+            AUTOMATION_EVENT_NAMES.GENERATOR_COMPLETED
+        ]);
+    });
+
+    it("does not emit when automation mode is disabled", () => {
+        delete process.env.FERN_AUTOMATION;
+        const emitted: unknown[] = [];
+        const collector = new GeneratorRunCollector({
+            emitAutomationEvent: (event) => {
+                emitted.push(event);
+            }
+        });
+
+        collector.recordSkipped({
+            apiName: undefined,
+            groupName: "g",
+            generatorName: "x",
+            reason: "opted_out",
+            outputRepoUrl: undefined,
+            generatorsYmlAbsolutePath: undefined,
+            generatorsYmlLineNumber: undefined
+        });
+
+        expect(emitted).toHaveLength(0);
+    });
+});
+
 describe("GeneratorRunCollector", () => {
     it("is empty on construction", () => {
         const collector = new GeneratorRunCollector();