RUM-15735: Add profiling aggregation diagnostics telemetry

Author: simaoseica-dd

Datadog/Datadog.xcodeproj/project.pbxproj

@@ -34,6 +34,7 @@
 		1121FD002EE319F000297156 /* ProfilingTelemetryController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1121FCFE2EE319E600297156 /* ProfilingTelemetryController.swift */; };
 		1121FD022EE33B3400297156 /* ConfigurationMetric.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1121FD012EE33B2E00297156 /* ConfigurationMetric.swift */; };
 		1121FD052F00000000297156 /* ProfilingSessionMetric.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1121FD042F00000000297156 /* ProfilingSessionMetric.swift */; };
+		1121FD072F00000100297156 /* AggregationDiagnosticsMetric.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1121FD062F00000100297156 /* AggregationDiagnosticsMetric.swift */; };
 		1124D5302EA6D23C0002E053 /* StartupTypeHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1124D52E2EA6D2370002E053 /* StartupTypeHandler.swift */; };
 		1124D5322EA6D4050002E053 /* RUMAppLaunchManagerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1124D5312EA6D4050002E053 /* RUMAppLaunchManagerTests.swift */; };
 		1124D5382EA6D4390002E053 /* RUMAppLaunchManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1124D5342EA6D4390002E053 /* RUMAppLaunchManager.swift */; };
@@ -1803,6 +1804,7 @@
 		1121FCFE2EE319E600297156 /* ProfilingTelemetryController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfilingTelemetryController.swift; sourceTree = "<group>"; };
 		1121FD012EE33B2E00297156 /* ConfigurationMetric.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConfigurationMetric.swift; sourceTree = "<group>"; };
 		1121FD042F00000000297156 /* ProfilingSessionMetric.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfilingSessionMetric.swift; sourceTree = "<group>"; };
+		1121FD062F00000100297156 /* AggregationDiagnosticsMetric.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AggregationDiagnosticsMetric.swift; sourceTree = "<group>"; };
 		1124D52E2EA6D2370002E053 /* StartupTypeHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StartupTypeHandler.swift; sourceTree = "<group>"; };
 		1124D5312EA6D4050002E053 /* RUMAppLaunchManagerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RUMAppLaunchManagerTests.swift; sourceTree = "<group>"; };
 		1124D5342EA6D4390002E053 /* RUMAppLaunchManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RUMAppLaunchManager.swift; sourceTree = "<group>"; };
@@ -3262,6 +3264,7 @@
 		1121FCFA2EE305D900297156 /* SDKMetrics */ = {
 			isa = PBXGroup;
 			children = (
+				1121FD062F00000100297156 /* AggregationDiagnosticsMetric.swift */,
 				1121FD012EE33B2E00297156 /* ConfigurationMetric.swift */,
 				1121FD042F00000000297156 /* ProfilingSessionMetric.swift */,
 				1121FCFE2EE319E600297156 /* ProfilingTelemetryController.swift */,
@@ -8459,6 +8462,7 @@
 				110DD1B22EEB6BBD002F3B8B /* ProfilingConfiguration.swift in Sources */,
 				D233E7C42E4625D400E7CFDE /* RequestBuilder.swift in Sources */,
 				D233E7C52E4625D400E7CFDE /* ProfileEvent.swift in Sources */,
+				1121FD072F00000100297156 /* AggregationDiagnosticsMetric.swift in Sources */,
 				1121FD002EE319F000297156 /* ProfilingTelemetryController.swift in Sources */,
 				1121FD052F00000000297156 /* ProfilingSessionMetric.swift in Sources */,
 				D233E8022E4B653F00E7CFDE /* dd_pprof.cpp in Sources */,

DatadogProfiling/Mach/dd_profiler.cpp

@@ -27,6 +27,10 @@ static constexpr double DD_PROFILER_MAX_SAMPLE_RATE = 100.0;
 // Maximum queued aggregation batch memory before new batches are dropped.
 static constexpr uint64_t DD_PROFILER_DEFAULT_HARD_LIMIT_BYTES = 64ULL * 1024ULL * 1024ULL;
 
+static dd_profiler_diagnostics_t empty_diagnostics() {
+    return {0, 0, 0};
+}
+
 namespace dd::profiler { class dd_profiler; }
 
 static dd::profiler::dd_profiler* g_dd_profiler = nullptr;
@@ -281,18 +285,14 @@ class dd_profiler {
         return swap_context.flushed_profile;
     }
 
-    void consume_diagnostics(dd_profiler_diagnostics_t* out) {
-        if (!out) {
-            return;
-        }
-
-        out->dropped_batch_count = 0;
-        out->dropped_sample_count = 0;
-        out->max_pending_bytes = 0;
+    dd_profiler_diagnostics_t diagnostics() {
+        dd_profiler_diagnostics_t result = empty_diagnostics();
 
         if (profiler) {
-            profiler->consume_diagnostics(out);
+            profiler->consume_diagnostics(&result);
         }
+
+        return result;
     }
 
 private:
@@ -466,16 +466,12 @@ dd_profiler_status_t dd_profiler_get_status(void) {
     return g_dd_profiler ? g_dd_profiler->status : DD_PROFILER_STATUS_NOT_CREATED;
 }
 
-void dd_profiler_consume_diagnostics(dd_profiler_diagnostics_t* out) {
+dd_profiler_diagnostics_t dd_profiler_diagnostics(void) {
     std::lock_guard<std::mutex> lock(g_dd_profiler_mutex);
-    if (out) {
-        out->dropped_batch_count = 0;
-        out->dropped_sample_count = 0;
-        out->max_pending_bytes = 0;
-    }
     if (g_dd_profiler) {
-        g_dd_profiler->consume_diagnostics(out);
+        return g_dd_profiler->diagnostics();
     }
+    return empty_diagnostics();
 }
 
 bool dd_profiler_is_running() {

DatadogProfiling/Mach/include/dd_profiler.h

@@ -213,13 +213,11 @@ typedef struct profile dd_profile_t;
 dd_profiler_status_t dd_profiler_get_status(void);
 
 /**
- * @brief Returns and resets profiling diagnostics accumulated since the last consume.
+ * @brief Returns and resets profiling diagnostics accumulated since the last call.
  *
- * If `out` is null, this function is a no-op.
- *
- * @param out Output diagnostics structure
+ * @return Profiling diagnostics, or zeroed diagnostics if the profiler does not exist.
  */
-void dd_profiler_consume_diagnostics(dd_profiler_diagnostics_t* out);
+dd_profiler_diagnostics_t dd_profiler_diagnostics(void);
 
 /**
  * @brief Stops profiling if it's currently running

DatadogProfiling/Sources/SDKMetrics/AggregationDiagnosticsMetric.swift

@@ -0,0 +1,74 @@
+/*
+ * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2019-Present Datadog, Inc.
+ */
+
+import Foundation
+
+#if !os(watchOS)
+
+// swiftlint:disable duplicate_imports
+#if swift(>=6.0)
+internal import DatadogMachProfiler
+#else
+@_implementationOnly import DatadogMachProfiler
+#endif
+// swiftlint:enable duplicate_imports
+
+/// Tracks profiling aggregation diagnostics to be added to telemetry metrics.
+internal final class AggregationDiagnosticsMetric {
+    internal enum Constants {
+        /// Namespace for bundling profiling diagnostics.
+        static let diagnosticsKey = "profiling_diagnostics"
+    }
+
+    /// Number of sampled batches dropped during aggregation.
+    let droppedBatchCount: UInt64
+    /// Number of sampled stack traces contained in dropped batches.
+    let droppedSampleCount: UInt64
+    /// Maximum queued aggregation memory observed, in bytes.
+    let maxPendingBytes: UInt64
+
+    init(
+        droppedBatchCount: UInt64 = 0,
+        droppedSampleCount: UInt64 = 0,
+        maxPendingBytes: UInt64 = 0
+    ) {
+        self.droppedBatchCount = droppedBatchCount
+        self.droppedSampleCount = droppedSampleCount
+        self.maxPendingBytes = maxPendingBytes
+    }
+
+    func asMetricAttributes() -> [String: Encodable]? {
+        [Constants.diagnosticsKey: Attributes(aggregation: self)]
+    }
+}
+
+extension AggregationDiagnosticsMetric: Encodable {
+    enum CodingKeys: String, CodingKey {
+        case droppedBatchCount = "dropped_batch_count"
+        case droppedSampleCount = "dropped_sample_count"
+        case maxPendingBytes = "max_pending_bytes"
+    }
+}
+
+extension AggregationDiagnosticsMetric {
+    static func consumeDiagnostics() -> AggregationDiagnosticsMetric {
+        let diagnostics = dd_profiler_diagnostics()
+
+        return AggregationDiagnosticsMetric(
+            droppedBatchCount: diagnostics.dropped_batch_count,
+            droppedSampleCount: diagnostics.dropped_sample_count,
+            maxPendingBytes: diagnostics.max_pending_bytes
+        )
+    }
+
+    /// Container to encode Profiling diagnostics according to the spec.
+    internal struct Attributes: Encodable {
+        /// Diagnostics for the serialized aggregation worker.
+        let aggregation: AggregationDiagnosticsMetric
+    }
+}
+
+#endif

DatadogProfiling/Sources/SDKMetrics/ProfilingTelemetryController.swift

@@ -107,6 +107,7 @@ internal final class ProfilingTelemetryController {
             telemetry.debug("Failed to compute attributes for '\(metric.metricName)'")
             return
         }
+        metricAttributes.merge(AggregationDiagnosticsMetric.consumeDiagnostics().asMetricAttributes() ?? [:]) { $1 }
         metricAttributes.merge(configMetric.asMetricAttributes() ?? [:]) { $1 }
 
         telemetry.metric(name: metric.metricName, attributes: metricAttributes, sampleRate: sampleRate)

DatadogProfiling/Tests/DDProfilerTests.swift

@@ -89,17 +89,15 @@ final class DDProfilerTests: XCTestCase {
             dd_pprof_destroy(profile)
         }
 
-        var diagnostics = dd_profiler_diagnostics_t()
-        dd_profiler_consume_diagnostics(&diagnostics)
+        let diagnostics = dd_profiler_diagnostics()
 
         XCTAssertGreaterThan(diagnostics.dropped_batch_count, 0, "Hard-limit drops should be reported")
         XCTAssertGreaterThan(diagnostics.dropped_sample_count, 0, "Dropped samples should be counted")
         XCTAssertGreaterThan(diagnostics.max_pending_bytes, 0, "Pending-byte high-water mark should be reported")
 
-        var secondRead = dd_profiler_diagnostics_t()
-        dd_profiler_consume_diagnostics(&secondRead)
-        XCTAssertEqual(secondRead.dropped_batch_count, 0, "Consuming diagnostics should reset batch drops")
-        XCTAssertEqual(secondRead.dropped_sample_count, 0, "Consuming diagnostics should reset sample drops")
+        let secondRead = dd_profiler_diagnostics()
+        XCTAssertEqual(secondRead.dropped_batch_count, 0, "Reading diagnostics should reset batch drops")
+        XCTAssertEqual(secondRead.dropped_sample_count, 0, "Reading diagnostics should reset sample drops")
     }
 
     func testDDProfiler_doesNotIncludeProfilerInternalThreadsInProfile() throws {

DatadogProfiling/Tests/SDKMetrics/ProfilingTelemetryControllerTests.swift

@@ -182,6 +182,25 @@ final class ProfilingTelemetryControllerTests: XCTestCase {
         XCTAssertEqual(metricTelemetry.sampleRate, 20.0)
     }
 
+    func testTrackingProfilingSessionMetric_attachesAggregationDiagnostics() throws {
+        // Given
+        let controller = ProfilingTelemetryController(telemetry: telemetry)
+        let diagnosticsKey = AggregationDiagnosticsMetric.Constants.diagnosticsKey
+
+        // When
+        controller.sendProfile(durationNs: 1, fileSize: 2, for: .continuousProfiling)
+        controller.sendProfileNotWritten(for: .customProfiling)
+
+        // Then
+        let diagnostics = telemetry.messages.compactMap { message in
+            message.asMetric?.attributes[diagnosticsKey] as? AggregationDiagnosticsMetric.Attributes
+        }
+        XCTAssertEqual(diagnostics.count, 2)
+        XCTAssertEqual(diagnostics.first?.aggregation.droppedBatchCount, 0)
+        XCTAssertEqual(diagnostics.first?.aggregation.droppedSampleCount, 0)
+        XCTAssertEqual(diagnostics.first?.aggregation.maxPendingBytes, 0)
+    }
+
     func testSendProfile_usesContinuousStartReason_forContinuousOperation() throws {
         // Given
         let controller = ProfilingTelemetryController(telemetry: telemetry)