[INC-669] Add tail sampling implemented in OTEL SDK

apps/search/src/instrumentations/otel-node.ts

@@ -4,15 +4,30 @@ import {
   ATTR_SERVICE_INSTANCE_ID,
 } from "@opentelemetry/semantic-conventions/incubating";
 import { createBatchSpanProcessor } from "@saleor/apps-otel/src/batch-span-processor-factory";
+import { DeferredSampler } from "@saleor/apps-otel/src/deferred-sampler";
 import { createHttpInstrumentation } from "@saleor/apps-otel/src/http-instrumentation-factory";
 import { ObservabilityAttributes } from "@saleor/apps-otel/src/observability-attributes";
 import { createServiceInstanceId } from "@saleor/apps-otel/src/service-instance-id-factory";
+import { TailSamplingProcessor } from "@saleor/apps-otel/src/tail-sampling-processor";
 import { registerOTel } from "@vercel/otel";
 
 import pkg from "../../package.json";
 
+const batchProcessor = createBatchSpanProcessor({
+  accessToken: process.env.OTEL_ACCESS_TOKEN,
+});
+
+const tailSamplingProcessor = new TailSamplingProcessor({
+  processor: batchProcessor,
+  slowThresholdMs: 5000,
+  exportErrors: true,
+  exportSlowSpans: true,
+});
+
 registerOTel({
   serviceName: process.env.OTEL_SERVICE_NAME,
+  // Note: DeferredSampler + TailSamplingProcessor must be used together
+  traceSampler: new DeferredSampler(),
   attributes: {
     [ATTR_SERVICE_VERSION]: pkg.version,
     [ATTR_DEPLOYMENT_ENVIRONMENT_NAME]: process.env.ENV,
@@ -23,10 +38,6 @@ registerOTel({
     env: undefined,
     [ObservabilityAttributes.VERCEL_ENV]: process.env.VERCEL_ENV,
   },
-  spanProcessors: [
-    createBatchSpanProcessor({
-      accessToken: process.env.OTEL_ACCESS_TOKEN,
-    }),
-  ],
-  instrumentations: [createHttpInstrumentation()],
+  spanProcessors: [tailSamplingProcessor],
+  instrumentations: [createHttpInstrumentation({ usingDeferredSpanProcessor: true })],
 });

cspell.config.js

@@ -79,6 +79,7 @@ export default {
     "claude",
     "SEPA",
     "mailpit",
+    "traceparent"
   ],
   language: "en-US",
   useGitignore: true,

packages/otel/package.json

@@ -5,7 +5,9 @@
   "scripts": {
     "check-types": "tsc",
     "lint": "eslint .",
-    "lint:fix": "eslint --fix ."
+    "lint:fix": "eslint --fix .",
+    "test": "vitest",
+    "test:ci": "vitest run --coverage"
   },
   "dependencies": {
     "@opentelemetry/exporter-metrics-otlp-http": "catalog:",
@@ -18,24 +20,31 @@
   },
   "devDependencies": {
     "@opentelemetry/api": "catalog:",
+    "@opentelemetry/instrumentation": "catalog:",
     "@opentelemetry/sdk-metrics": "catalog:",
     "@opentelemetry/sdk-trace-node": "catalog:",
     "@opentelemetry/semantic-conventions": "catalog:",
     "@saleor/app-sdk": "link:../../node_modules/@saleor/app-sdk",
     "@saleor/eslint-config-apps": "workspace:*",
     "@saleor/typescript-config-apps": "workspace:*",
     "@types/node": "catalog:",
+    "@vercel/otel": "catalog:",
+    "@vitest/coverage-v8": "catalog:",
     "eslint": "catalog:",
     "next": "catalog:",
     "typescript": "catalog:",
-    "urql": "catalog:"
+    "urql": "catalog:",
+    "vite": "catalog:",
+    "vitest": "catalog:"
   },
   "peerDependencies": {
     "@opentelemetry/api": "catalog:",
+    "@opentelemetry/instrumentation": "catalog:",
     "@opentelemetry/sdk-metrics": "catalog:",
     "@opentelemetry/sdk-trace-node": "catalog:",
     "@opentelemetry/semantic-conventions": "catalog:",
     "@saleor/app-sdk": "catalog:",
+    "@vercel/otel": "catalog:",
     "next": "catalog:",
     "urql": "catalog:"
   }

packages/otel/src/deferred-sampler.test.ts

@@ -0,0 +1,79 @@
+import { ROOT_CONTEXT, SpanKind, trace, TraceFlags } from "@opentelemetry/api";
+import { SamplingDecision } from "@opentelemetry/sdk-trace-node";
+import { describe, expect, it } from "vitest";
+
+import { DeferredSampler, SALEOR_SAMPLING_DECISION_ATTR } from "./deferred-sampler";
+
+describe("DeferredSampler", () => {
+  const traceId = "0af7651916cd43dd8448eb211c80319c";
+  const spanName = "test-span";
+  const spanKind = SpanKind.SERVER;
+  const attributes = {};
+  const links: never[] = [];
+
+  describe("when parent is sampled", () => {
+    it("should return RECORD_AND_SAMPLED decision", () => {
+      const sampler = new DeferredSampler();
+      const parentContext = trace.setSpanContext(ROOT_CONTEXT, {
+        traceId,
+        spanId: "b7ad6b7169203331",
+        traceFlags: TraceFlags.SAMPLED,
+        isRemote: true,
+      });
+
+      const result = sampler.shouldSample(
+        parentContext,
+        traceId,
+        spanName,
+        spanKind,
+        attributes,
+        links,
+      );
+
+      expect(result.decision).toBe(SamplingDecision.RECORD_AND_SAMPLED);
+      expect(result.attributes?.[SALEOR_SAMPLING_DECISION_ATTR]).toBe("sampled");
+    });
+  });
+
+  describe("when parent is not sampled", () => {
+    it("should return RECORD decision (defer to TailSamplingProcessor)", () => {
+      const sampler = new DeferredSampler();
+      const parentContext = trace.setSpanContext(ROOT_CONTEXT, {
+        traceId,
+        spanId: "b7ad6b7169203331",
+        traceFlags: TraceFlags.NONE,
+        isRemote: true,
+      });
+
+      const result = sampler.shouldSample(
+        parentContext,
+        traceId,
+        spanName,
+        spanKind,
+        attributes,
+        links,
+      );
+
+      expect(result.decision).toBe(SamplingDecision.RECORD);
+      expect(result.attributes?.[SALEOR_SAMPLING_DECISION_ATTR]).toBe("not_sampled");
+    });
+  });
+
+  describe("when there is no parent (root span)", () => {
+    it("should return RECORD decision (defer to TailSamplingProcessor)", () => {
+      const sampler = new DeferredSampler();
+
+      const result = sampler.shouldSample(
+        ROOT_CONTEXT,
+        traceId,
+        spanName,
+        spanKind,
+        attributes,
+        links,
+      );
+
+      expect(result.decision).toBe(SamplingDecision.RECORD);
+      expect(result.attributes?.[SALEOR_SAMPLING_DECISION_ATTR]).toBe("none");
+    });
+  });
+});

packages/otel/src/deferred-sampler.ts

@@ -0,0 +1,66 @@
+import { Attributes, Context, Link, SpanKind, trace, TraceFlags } from "@opentelemetry/api";
+import { Sampler, SamplingDecision, SamplingResult } from "@opentelemetry/sdk-trace-node";
+
+/**
+ * Attribute key to store Saleor's original sampling decision.
+ * Used by TailSamplingProcessor to know if Saleor wanted this trace.
+ */
+export const SALEOR_SAMPLING_DECISION_ATTR = "saleor.sampling.decision";
+
+/**
+ * Check if SAMPLED flag is set in traceFlags bitmask.
+ * Uses bitwise AND operation according to OTEL spec
+ */
+function isSampled(traceFlags: number): boolean {
+  return (traceFlags & TraceFlags.SAMPLED) !== 0;
+}
+
+/**
+ * A sampler that defers the final sampling decision to span end.
+ *
+ * - When parent is SAMPLED → return RECORD_AND_SAMPLED (respects parent)
+ * - When parent is NOT SAMPLED or NO parent → return RECORD (defer decision to TailSamplingProcessor)
+ *
+ * This allows the TailSamplingProcessor to make the final decision
+ * based on error status or latency at span end.
+ * Without setting `RECORD` we wouldn't store any span data during runtime
+ */
+export class DeferredSampler implements Sampler {
+  // eslint-disable-next-line @typescript-eslint/max-params -- Required by OpenTelemetry Sampler interface
+  shouldSample(
+    context: Context,
+    _traceId: string,
+    _spanName: string,
+    _spanKind: SpanKind,
+    _attributes: Attributes,
+    _links: Link[],
+  ): SamplingResult {
+    const parentSpanContext = trace.getSpanContext(context);
+    const parentSampled = parentSpanContext && isSampled(parentSpanContext.traceFlags);
+
+    if (parentSampled) {
+      // Parent decided to sample - we MUST sample too
+      return {
+        decision: SamplingDecision.RECORD_AND_SAMPLED,
+        attributes: {
+          [SALEOR_SAMPLING_DECISION_ATTR]: "sampled",
+        },
+      };
+    }
+
+    /*
+     * No parent OR parent decided NOT to sample.
+     * Record span but defer export decision to TailSamplingProcessor.
+     */
+    return {
+      decision: SamplingDecision.RECORD,
+      attributes: {
+        [SALEOR_SAMPLING_DECISION_ATTR]: parentSpanContext ? "not_sampled" : "none",
+      },
+    };
+  }
+
+  toString(): string {
+    return "DeferredSampler";
+  }
+}