Handle APM_MULTI_CONFIG, fixup missing exception replay features (#3791)

* Handle APM_MULTI_CONFIG, fixup missing exception replay features

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

# Conflicts:
#	libdatadog

* Update to newest libdatadog

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

---------

Signed-off-by: Bob Weinand <bob.weinand@datadoghq.com>

Author: bwoebi

Cargo.lock

@@ -48,7 +48,7 @@ RUN_TESTS_CMD := DD_SERVICE= DD_ENV= REPORT_EXIT_STATUS=1 TEST_PHP_SRCDIR=$(PROJ
 
 C_FILES = $(shell find components components-rs ext src/dogstatsd zend_abstract_interface -name '*.c' -o -name '*.h' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' )
 TEST_FILES = $(shell find tests/ext -name '*.php*' -o -name '*.inc' -o -name '*.json' -o -name '*.yaml' -o -name 'CONFLICTS' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' )
-RUST_FILES = $(BUILD_DIR)/Cargo.toml $(BUILD_DIR)/Cargo.lock $(shell find components-rs -name '*.c' -o -name '*.rs' -o -name 'Cargo.toml' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' ) $(shell find libdatadog/{build-common,datadog-ffe,datadog-ipc,datadog-ipc-macros,datadog-live-debugger,datadog-live-debugger-ffi,datadog-remote-config,datadog-sidecar,datadog-sidecar-ffi,datadog-sidecar-macros,libdd-alloc,libdd-capabilities,libdd-capabilities-impl,libdd-common,libdd-common-ffi,libdd-crashtracker,libdd-crashtracker-ffi,libdd-data-pipeline,libdd-ddsketch,libdd-dogstatsd-client,libdd-library-config,libdd-library-config-ffi,libdd-log,libdd-libunwind-sys,libdd-shared-runtime,libdd-telemetry,libdd-telemetry-ffi,libdd-tinybytes,libdd-trace-*,spawn_worker,tools/{cc_utils,sidecar_mockgen},libdd-trace-*,Cargo.toml} \( -type l -o -type f \) \( -path "*/src*" -o -path "*/examples*" -o -path "*/libdd-libunwind-sys*" -o -path "*Cargo.toml" -o -path "*/build.rs" -o -path "*/tests/dataservice.rs" -o -path "*/tests/service_functional.rs" \) -not -path "*/datadog-ipc/build.rs" -not -path "*/datadog-sidecar-ffi/build.rs")
+RUST_FILES = $(BUILD_DIR)/Cargo.toml $(BUILD_DIR)/Cargo.lock $(shell find components-rs -name '*.c' -o -name '*.rs' -o -name 'Cargo.toml' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' ) $(shell find libdatadog/{build-common,datadog-ffe,datadog-ipc,datadog-ipc-macros,datadog-live-debugger,datadog-live-debugger-ffi,datadog-remote-config,datadog-sidecar,datadog-sidecar-ffi,datadog-sidecar-macros,libdd-alloc,libdd-capabilities,libdd-capabilities-impl,libdd-common,libdd-common-ffi,libdd-crashtracker,libdd-crashtracker-ffi,libdd-data-pipeline,libdd-ddsketch,libdd-dogstatsd-client,libdd-library-config,libdd-library-config-ffi,libdd-log,libdd-shared-runtime,libdd-telemetry,libdd-telemetry-ffi,libdd-tinybytes,libdd-trace-*,spawn_worker,tools/{cc_utils,sidecar_mockgen},libdd-trace-*,Cargo.toml} \( -type l -o -type f \) \( -path "*/src*" -o -path "*/examples*" -o -path "*Cargo.toml" -o -path "*/build.rs" -o -path "*/tests/dataservice.rs" -o -path "*/tests/service_functional.rs" \) -not -path "*/datadog-ipc/build.rs" -not -path "*/datadog-sidecar-ffi/build.rs")
 ALL_OBJECT_FILES = $(C_FILES) $(RUST_FILES) $(BUILD_DIR)/Makefile
 TEST_OPCACHE_FILES = $(shell find tests/opcache -name '*.php*' -o -name '.gitkeep' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' )
 TEST_STUB_FILES = $(shell find tests/ext -type d -name 'stubs' -exec find '{}' -type f \; | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' )

Makefile

@@ -588,12 +588,20 @@ typedef struct ddog_CaptureConfiguration {
   uint32_t max_field_count;
 } ddog_CaptureConfiguration;
 
+typedef struct ddog_CaptureExpression {
+  ddog_CharSlice name;
+  const struct ddog_ProbeValue *expr;
+  const struct ddog_CaptureConfiguration *capture;
+} ddog_CaptureExpression;
+
 typedef struct ddog_LogProbe {
   const struct ddog_DslString *segments;
   const struct ddog_ProbeCondition *when;
   const struct ddog_CaptureConfiguration *capture;
   bool capture_snapshot;
   uint32_t sampling_snapshots_per_second;
+  const struct ddog_CaptureExpression *capture_expressions;
+  uintptr_t capture_expressions_num;
 } ddog_LogProbe;
 
 typedef struct ddog_SpanProbeTag {

components-rs/common.h

@@ -85,13 +85,6 @@ void ddog_apply_agent_info(struct ddog_AgentInfoReader *reader,
  */
 void ddog_apply_agent_info_concentrator_config(struct ddog_AgentInfoReader *reader);
 
-/**
- * Returns true once the sidecar has received and applied the agent /info response.
- * Used by `dd_trace_internal_fn('await_agent_info')` to block until the concentrator
- * peer-tag keys and span kinds are initialised.
- */
-bool ddog_is_agent_info_ready(void);
-
 bool ddog_shall_log(enum ddog_Log category);
 
 void ddog_set_error_log_level(bool once);
@@ -108,7 +101,8 @@ void ddog_init_remote_config(bool live_debugging_enabled,
                              bool appsec_activation,
                              bool appsec_config);
 
-struct ddog_RemoteConfigState *ddog_init_remote_config_state(const struct ddog_Endpoint *endpoint);
+struct ddog_RemoteConfigState *ddog_init_remote_config_state(const struct ddog_Endpoint *endpoint,
+                                                             bool di_enabled);
 
 const char *ddog_remote_config_get_path(const struct ddog_RemoteConfigState *remote_config);
 
@@ -135,6 +129,14 @@ bool ddog_remote_config_alter_dynamic_config(struct ddog_RemoteConfigState *remo
 void ddog_setup_remote_config(ddog_DynamicConfigUpdate update_config,
                               const struct ddog_LiveDebuggerSetup *setup);
 
+/**
+ * Enable or disable dynamic instrumentation.
+ * When disabling: all installed probe hooks are removed (but kept in `active` for reinstallation).
+ * When enabling: all probes in `active` that have no installed hook are (re-)installed.
+ */
+void ddog_set_dynamic_instrumentation_enabled(struct ddog_RemoteConfigState *remote_config,
+                                              bool enabled);
+
 void ddog_rshutdown_remote_config(struct ddog_RemoteConfigState *remote_config);
 
 void ddog_shutdown_remote_config(struct ddog_RemoteConfigState*);
@@ -174,6 +176,13 @@ bool ddog_exception_hash_limiter_inc(struct ddog_SidecarTransport *connection,
                                      uint64_t hash,
                                      uint32_t granularity_seconds);
 
+/**
+ * Returns true once the agent /info has been received and applied.
+ * Used by the PHP extension to skip stats computation until the concentrator
+ * has been properly initialised with peer-tag keys and span kinds.
+ */
+bool ddog_is_agent_info_ready(void);
+
 /**
  * Look up (or lazily create) the concentrator for `(env, version, service)` and invoke
  * `callback` with a shared reference to it while holding the global read lock.

components-rs/ddtrace.h

@@ -10,7 +10,9 @@
 #include <stdio.h>
 #include "common.h"
 
-void drop_span_decoration_probe(struct ddog_SpanDecorationProbe);
+void ddog_drop_log_probe_capture_expressions(struct ddog_LogProbe);
+
+void ddog_drop_span_decoration_probe(struct ddog_SpanDecorationProbe);
 
 struct ddog_CaptureConfiguration ddog_capture_defaults(void);
 
@@ -54,6 +56,12 @@ ddog_DebuggerCapture *ddog_create_exception_snapshot(struct ddog_Vec_DebuggerPay
                                                      ddog_CharSlice method_name,
                                                      uint64_t timestamp);
 
+/**
+ * Returns a mutable pointer to the last DebuggerPayload in the Vec.
+ * Used to push stack frames to exception replay snapshots after creation.
+ */
+struct ddog_DebuggerPayload *ddog_vec_last_debugger_payload(struct ddog_Vec_DebuggerPayload *buffer);
+
 struct ddog_DebuggerPayload *ddog_create_log_probe_snapshot(const struct ddog_Probe *probe,
                                                             const ddog_CharSlice *message,
                                                             ddog_CharSlice service,
@@ -73,10 +81,25 @@ bool ddog_snapshot_redacted_name(ddog_CharSlice name);
 
 void ddog_snapshot_add_redacted_name(ddog_CharSlice name);
 
+void ddog_snapshot_add_excluded_name(ddog_CharSlice name);
+
 bool ddog_snapshot_redacted_type(ddog_CharSlice name);
 
 void ddog_snapshot_add_redacted_type(ddog_CharSlice name);
 
+void ddog_snapshot_set_throwable(ddog_DebuggerCapture *capture,
+                                 ddog_CharSlice throwable_type,
+                                 ddog_CharSlice message);
+
+void ddog_snapshot_throwable_add_frame(ddog_DebuggerCapture *capture,
+                                       ddog_CharSlice file,
+                                       ddog_CharSlice function,
+                                       int64_t line);
+
+void ddog_snapshot_add_capture_fields(ddog_DebuggerCapture *capture,
+                                      ddog_CharSlice name,
+                                      struct ddog_CaptureValue value);
+
 void ddog_snapshot_add_field(ddog_DebuggerCapture *capture,
                              enum ddog_FieldType type,
                              ddog_CharSlice name,

components-rs/live-debugger.h

@@ -113,6 +113,7 @@ pub struct LiveDebuggerState {
     pub config_id: String,
     pub allow_dfa: Option<Regex>,
     pub deny_dfa: Option<Regex>,
+    pub di_enabled: bool,
 }
 
 #[no_mangle]
@@ -177,14 +178,18 @@ pub unsafe extern "C" fn ddog_init_remote_config(
 #[no_mangle]
 pub unsafe extern "C" fn ddog_init_remote_config_state(
     endpoint: &Endpoint,
+    di_enabled: bool,
 ) -> Box<RemoteConfigState> {
     Box::new(RemoteConfigState {
         manager: RemoteConfigManager::new(ConfigInvariants {
             language: "php".to_string(),
             tracer_version: include_str!("../VERSION").trim().into(),
             endpoint: endpoint.clone(),
         }),
-        live_debugger: LiveDebuggerState::default(),
+        live_debugger: LiveDebuggerState {
+            di_enabled,
+            ..Default::default()
+        },
         dynamic_config: Default::default(),
     })
 }
@@ -414,13 +419,17 @@ fn apply_config(
         match debugger {
             LiveDebuggingData::Probe(probe) => {
                 debug!("Applying live debugger probe {probe:?}");
-                let hook_id = (callbacks.set_probe)(probe.into(), limiter);
-                if hook_id >= 0 {
-                    remote_config
-                        .live_debugger
-                        .spans_map
-                        .insert(probe.id.clone(), hook_id);
+                if remote_config.live_debugger.di_enabled {
+                    let hook_id = (callbacks.set_probe)(probe.into(), limiter);
+                    if hook_id >= 0 {
+                        remote_config
+                            .live_debugger
+                            .spans_map
+                            .insert(probe.id.clone(), hook_id);
+                    }
                 }
+                // If di_enabled is false, probe is stored in `active` but hook is not installed.
+                // It will be installed when di_enabled transitions to true.
             }
             LiveDebuggingData::ServiceConfiguration(config) => {
                 debug!("Applying live debugger service config {config:?}");
@@ -588,6 +597,42 @@ pub unsafe extern "C" fn ddog_setup_remote_config(
     LIVE_DEBUGGER_CALLBACKS = Some(setup.callbacks.clone());
 }
 
+/// Enable or disable dynamic instrumentation.
+/// When disabling: all installed probe hooks are removed (but kept in `active` for reinstallation).
+/// When enabling: all probes in `active` that have no installed hook are (re-)installed.
+#[no_mangle]
+pub extern "C" fn ddog_set_dynamic_instrumentation_enabled(
+    remote_config: &mut RemoteConfigState,
+    enabled: bool,
+) {
+    if remote_config.live_debugger.di_enabled == enabled {
+        return;
+    }
+    remote_config.live_debugger.di_enabled = enabled;
+
+    if let Some(callbacks) = unsafe { &LIVE_DEBUGGER_CALLBACKS } {
+        if !enabled {
+            // Remove all installed probe hooks; keep `active` intact for reinstallation.
+            for (_, hook_id) in remote_config.live_debugger.spans_map.drain() {
+                (callbacks.remove_probe)(hook_id);
+            }
+        } else {
+            // Reinstall all probes currently stored in `active`.
+            for (probe_id, boxed) in remote_config.live_debugger.active.iter() {
+                if let (LiveDebuggingData::Probe(probe), limiter) = &**boxed {
+                    let hook_id = (callbacks.set_probe)(probe.into(), limiter);
+                    if hook_id >= 0 {
+                        remote_config
+                            .live_debugger
+                            .spans_map
+                            .insert(probe_id.clone(), hook_id);
+                    }
+                }
+            }
+        }
+    }
+}
+
 #[no_mangle]
 pub extern "C" fn ddog_rshutdown_remote_config(remote_config: &mut RemoteConfigState) {
     remote_config.live_debugger.spans_map.clear();

components-rs/remote_config.rs

@@ -65,19 +65,6 @@ if test "$PHP_DDTRACE" != "no"; then
     PHP_SUBST(DDTRACE_CARGO)
   fi
 
-  dnl As required by libdd-libunwind-sys
-  if test "$PHP_DDTRACE_RUST_LIBRARY" = "-"; then
-    AC_CHECK_TOOL(DDTRACE_AUTOMAKE, automake, [:])
-    AS_IF([test "$DDTRACE_AUTOMAKE" = ":"], [AC_MSG_ERROR([Please install automake before configuring])])
-    AC_CHECK_TOOL(DDTRACE_LIBTOOL, libtool, [:])
-    AS_IF([test "$DDTRACE_LIBTOOL" = ":"], [
-      AC_CHECK_TOOL(DDTRACE_LIBTOOLIZE, libtoolize, [:])
-      AS_IF([test "$DDTRACE_LIBTOOLIZE" = ":"], [
-        AC_MSG_ERROR([Please install libtool before configuring])
-      ])
-    ])
-  fi
-
   if test "$PHP_DDTRACE_SANITIZE" != "no"; then
     dnl gcc needs -lasan, clang needs -shared-libsan
     ac_cflags=$LDFLAGS

config.m4

@@ -264,6 +264,7 @@ enum ddtrace_sidecar_connection_mode {
     CONFIG(SET, DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS, "", .ini_change = zai_config_system_ini_change) \
     CONFIG(BOOL, DD_APM_TRACING_ENABLED, "true")                                                               \
     CONFIG(SET, DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES, "", .ini_change = zai_config_system_ini_change)     \
+    CONFIG(SET, DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS, "", .ini_change = zai_config_system_ini_change) \
     CONFIG(INT, DD_TRACE_BAGGAGE_MAX_ITEMS, "64")                                                              \
     CONFIG(INT, DD_TRACE_BAGGAGE_MAX_BYTES, "8192")                                                            \
     CONFIG(BOOL, DD_TRACE_INFERRED_PROXY_SERVICES_ENABLED, "false")                                            \

ext/configuration.h

@@ -1675,7 +1675,7 @@ static void dd_initialize_request(void) {
     DDTRACE_G(request_initialized) = true;
 
     if (!DDTRACE_G(remote_config_state) && ddtrace_endpoint) {
-        DDTRACE_G(remote_config_state) = ddog_init_remote_config_state(ddtrace_endpoint);
+        DDTRACE_G(remote_config_state) = ddog_init_remote_config_state(ddtrace_endpoint, get_DD_DYNAMIC_INSTRUMENTATION_ENABLED());
     }
 
     // We need to init RC for the sidecar to write to it immediately

ext/ddtrace.c

@@ -357,7 +357,7 @@ static zend_ulong ddtrace_compute_exception_hash(zend_object *exception) {
     return hash;
 }
 
-static void ddtrace_collect_exception_debug_data(zend_object *exception, zend_string *service_name, uint64_t time, ddog_SpanBytes *span) {
+static void ddtrace_collect_exception_debug_data(zend_object *exception, zend_object *throwable, zend_string *service_name, uint64_t time, ddog_SpanBytes *span) {
     if (!ddtrace_exception_debugging_is_active()) {
         return;
     }
@@ -392,14 +392,14 @@ static void ddtrace_collect_exception_debug_data(zend_object *exception, zend_st
     char *exception_id = zend_arena_alloc(&DDTRACE_G(debugger_capture_arena).arena, uuid_len);
     ddog_snapshot_format_new_uuid((uint8_t(*)[uuid_len])exception_id);
 
-    ddog_add_str_span_meta_CharSlice(span, "_dd.debug.error.exception_capture_id", (ddog_CharSlice){.ptr = exception_id, .len = uuid_len});
+    ddog_add_str_span_meta_CharSlice(span, "_dd.debug.error.exception_id", (ddog_CharSlice){.ptr = exception_id, .len = uuid_len});
 
     memset(&DDTRACE_G(exception_debugger_buffer), 0, sizeof(DDTRACE_G(exception_debugger_buffer)));
 
     zval *frame;
     int frame_num = 0;
     ZEND_HASH_FOREACH_NUM_KEY_VAL(Z_ARR_P(trace), frame_num, frame) {
-        if (get_DD_EXCEPTION_REPLAY_CAPTURE_MAX_FRAMES() >= 0 && get_DD_EXCEPTION_REPLAY_CAPTURE_MAX_FRAMES() < frame_num) {
+        if (get_DD_EXCEPTION_REPLAY_CAPTURE_MAX_FRAMES() >= 0 && get_DD_EXCEPTION_REPLAY_CAPTURE_MAX_FRAMES() <= frame_num) {
             break;
         }
 
@@ -423,6 +423,10 @@ static void ddtrace_collect_exception_debug_data(zend_object *exception, zend_st
         }
 
         ddog_DebuggerCapture *capture = dd_create_frame_and_collect_locals(exception_id, exception_hash, frame_num, class_slice, func_slice, locals, service_name, &capture_config, time, span);
+        zend_string *ex_msg = zai_exception_message(throwable);
+        ddog_snapshot_set_throwable(capture,
+            (ddog_CharSlice){ .ptr = ZSTR_VAL(throwable->ce->name), .len = ZSTR_LEN(throwable->ce->name) },
+            dd_zend_string_to_CharSlice(ex_msg));
         locals = zend_hash_find(Z_ARR_P(frame), key_locals);
 
         zend_string *key;
@@ -431,7 +435,7 @@ static void ddtrace_collect_exception_debug_data(zend_object *exception, zend_st
         if (args && Z_TYPE_P(args) == IS_ARRAY) {
             zend_long idx;
             ZEND_HASH_FOREACH_KEY_VAL(Z_ARR_P(args), idx, key, val) {
-                struct ddog_CaptureValue capture_value = {0};
+                ddog_CaptureValue capture_value = {0};
                 ddtrace_create_capture_value(val, &capture_value, &capture_config, capture_config.max_reference_depth);
                 ddog_CharSlice arg_name;
                 if (key) {
@@ -455,15 +459,19 @@ static void ddtrace_collect_exception_debug_data(zend_object *exception, zend_st
 
         zval *obj = zend_hash_find(Z_ARR_P(frame), ZSTR_KNOWN(ZEND_STR_OBJECT));
         if (obj) {
-            struct ddog_CaptureValue capture_value = {0};
+            ddog_CaptureValue capture_value = {0};
             ddtrace_create_capture_value(obj, &capture_value, &capture_config, capture_config.max_reference_depth);
             ddog_snapshot_add_field(capture, DDOG_FIELD_TYPE_ARG, DDOG_CHARSLICE_C("this"), capture_value);
         }
     } ZEND_HASH_FOREACH_END();
 
-    if (get_DD_EXCEPTION_REPLAY_CAPTURE_MAX_FRAMES() < 0 || get_DD_EXCEPTION_REPLAY_CAPTURE_MAX_FRAMES() > frame_num) {
+    if (get_DD_EXCEPTION_REPLAY_CAPTURE_MAX_FRAMES() < 0 || get_DD_EXCEPTION_REPLAY_CAPTURE_MAX_FRAMES() > frame_num + 1) {
         if (locals && Z_TYPE_P(locals) == IS_ARRAY) {
-            dd_create_frame_and_collect_locals(exception_id, exception_hash, frame_num + 1, DDOG_CHARSLICE_C(""), DDOG_CHARSLICE_C(""), locals, service_name, &capture_config, time, span);
+            ddog_DebuggerCapture *last_capture = dd_create_frame_and_collect_locals(exception_id, exception_hash, frame_num + 1, DDOG_CHARSLICE_C(""), DDOG_CHARSLICE_C(""), locals, service_name, &capture_config, time, span);
+            zend_string *last_ex_msg = zai_exception_message(throwable);
+            ddog_snapshot_set_throwable(last_capture,
+                (ddog_CharSlice){ .ptr = ZSTR_VAL(throwable->ce->name), .len = ZSTR_LEN(throwable->ce->name) },
+                dd_zend_string_to_CharSlice(last_ex_msg));
         }
     }
 
@@ -508,8 +516,9 @@ void ddtrace_exception_to_meta(zend_object *exception, zend_string *service_name
         previous = zai_exception_read_property(exception, ZSTR_KNOWN(ZEND_STR_PREVIOUS));
     }
 
-    // exception is now the innermost exception, i.e. what we need
-    ddtrace_collect_exception_debug_data(exception, service_name, time / 1000000, span);
+    // Capture frames from the innermost (root cause) exception so the stack trace reflects
+    // where the error originated. Use the outermost exception for the throwable type/message so it matches the span error and the exception replay filter.
+    ddtrace_collect_exception_debug_data(exception, exception_root, service_name, time / 1000000, span);
 
     previous = zai_exception_read_property(exception_root, ZSTR_KNOWN(ZEND_STR_PREVIOUS));
     while (Z_TYPE_P(previous) == IS_OBJECT && !Z_IS_RECURSIVE_P(previous) &&