feat(ad): expose Prometheus /metrics endpoint and scrape it from the OTel Collector

Add a Prometheus client library (io.prometheus:prometheus-metrics 1.3.6) to the
ad service, expose `/metrics` on `AD_PROMETHEUS_PORT` (default 9465), and
register a custom counter `demo_ad_served_total{category}` incremented as ads
are returned. Wire a `prometheus/ad` receiver in the OTel Collector to scrape
the endpoint, demonstrating Prometheus metric ingestion via the collector.

Author: cyrille-leclerc

.env

@@ -53,6 +53,7 @@ ACCOUNTING_DOCKERFILE=./src/accounting/Dockerfile
 # Ad Service
 AD_PORT=9555
 AD_ADDR=ad:${AD_PORT}
+AD_PROMETHEUS_PORT=9465
 AD_DOCKERFILE=./src/ad/Dockerfile
 
 # Cart Service

CHANGELOG.md

@@ -7,6 +7,10 @@ the release.
 
 ## Unreleased
 
+* [ad] Expose a Prometheus `/metrics` endpoint (default port `9465`) using the
+  Prometheus Java client library, with a custom counter
+  `demo_ad_served_total{category}`, and scrape it from the OTel Collector via a
+  new `prometheus/ad` receiver to demo Prometheus metric ingestion.
 * [telemetry-docs] Add a new service to provide telemetry documentation based
   on Weaver
   ([#2794](https://github.com/open-telemetry/opentelemetry-demo/pull/2794))

docker-compose.minimal.yml

@@ -35,8 +35,10 @@ services:
     restart: unless-stopped
     ports:
       - "${AD_PORT}"
+      - "${AD_PROMETHEUS_PORT}"
     environment:
       - AD_PORT
+      - AD_PROMETHEUS_PORT
       - FLAGD_HOST
       - FLAGD_PORT
       - OTEL_EXPORTER_OTLP_ENDPOINT=http://${OTEL_COLLECTOR_HOST}:${OTEL_COLLECTOR_PORT_HTTP}
@@ -738,6 +740,7 @@ services:
         condition: service_healthy
     logging: *logging
     environment:
+      - AD_PROMETHEUS_PORT
       - FRONTEND_PROXY_ADDR
       - HOST_FILESYSTEM
       - IMAGE_PROVIDER_HOST

docker-compose.yml

@@ -63,8 +63,10 @@ services:
     restart: unless-stopped
     ports:
       - "${AD_PORT}"
+      - "${AD_PROMETHEUS_PORT}"
     environment:
       - AD_PORT
+      - AD_PROMETHEUS_PORT
       - FLAGD_HOST
       - FLAGD_PORT
       - OTEL_EXPORTER_OTLP_ENDPOINT=http://${OTEL_COLLECTOR_HOST}:${OTEL_COLLECTOR_PORT_HTTP}
@@ -889,6 +891,7 @@ services:
         condition: service_healthy
     logging: *logging
     environment:
+      - AD_PROMETHEUS_PORT
       - FRONTEND_PROXY_ADDR
       - HOST_FILESYSTEM
       - IMAGE_PROVIDER_HOST

src/ad/Dockerfile

@@ -29,4 +29,5 @@ ADD --chmod=644 https://github.com/open-telemetry/opentelemetry-java-instrumenta
 ENV JAVA_TOOL_OPTIONS="-javaagent:/usr/src/app/opentelemetry-javaagent.jar -Xmx200m"
 
 EXPOSE ${AD_PORT}
+EXPOSE ${AD_PROMETHEUS_PORT}
 ENTRYPOINT [ "./build/install/opentelemetry-demo-ad/bin/Ad" ]

src/ad/build.gradle

@@ -24,6 +24,7 @@ def opentelemetryInstrumentationVersion = "2.27.0"
 def grpcVersion = "1.81.0"
 def jacksonVersion = "2.21.3"
 def protocVersion = "4.34.1"
+def prometheusMetricsVersion = "1.3.6"
 
 tasks.withType(JavaCompile).configureEach {
     sourceCompatibility = JavaVersion.VERSION_21
@@ -55,7 +56,9 @@ dependencies {
                 "io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations",
                 "org.apache.logging.log4j:log4j-core:2.25.4",
                 "dev.openfeature.contrib.providers:flagd:0.13.1",
-                'dev.openfeature:sdk:1.20.2'
+                'dev.openfeature:sdk:1.20.2',
+                "io.prometheus:prometheus-metrics-core:${prometheusMetricsVersion}",
+                "io.prometheus:prometheus-metrics-exporter-httpserver:${prometheusMetricsVersion}"
 
         runtimeOnly "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}",
                 "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}",

src/ad/src/main/java/oteldemo/AdService.java

@@ -25,6 +25,8 @@
 import io.opentelemetry.context.Scope;
 import io.opentelemetry.instrumentation.annotations.SpanAttribute;
 import io.opentelemetry.instrumentation.annotations.WithSpan;
+import io.prometheus.metrics.core.metrics.Counter;
+import io.prometheus.metrics.exporter.httpserver.HTTPServer;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -57,6 +59,14 @@ public final class AdService {
 
   private Server server;
   private HealthStatusManager healthMgr;
+  private HTTPServer prometheusServer;
+
+  private static final Counter adsServedCounter =
+      Counter.builder()
+          .name("demo_ad_served_total")
+          .help("Total number of ads served, labeled by category")
+          .labelNames("category")
+          .register();
 
   private static final AdService service = new AdService();
   private static final Tracer tracer = GlobalOpenTelemetry.getTracer("ad");
@@ -81,6 +91,11 @@ private void start() throws IOException {
                     () ->
                         new IllegalStateException(
                             "environment vars: AD_PORT must not be null")));
+    int prometheusPort =
+        Integer.parseInt(Optional.ofNullable(System.getenv("AD_PROMETHEUS_PORT")).orElse("9465"));
+    prometheusServer = HTTPServer.builder().port(prometheusPort).buildAndStart();
+    logger.info(
+        "Prometheus metrics endpoint started, listening on " + prometheusServer.getPort() + "/metrics");
     healthMgr = new HealthStatusManager();
 
     // Create a flagd instance with OpenTelemetry
@@ -118,6 +133,9 @@ private void stop() {
       healthMgr.clearStatus("");
       server.shutdown();
     }
+    if (prometheusServer != null) {
+      prometheusServer.stop();
+    }
   }
 
   private enum AdRequestType {
@@ -231,6 +249,7 @@ public void getAds(AdRequest req, StreamObserver<AdResponse> responseObserver) {
   private Collection<Ad> getAdsByCategory(@SpanAttribute("demo.ad.category") String category) {
     Collection<Ad> ads = adsMap.get(category);
     Span.current().setAttribute("demo.ad.count", ads.size());
+    adsServedCounter.labelValues(category).inc(ads.size());
     return ads;
   }
 
@@ -251,6 +270,7 @@ private List<Ad> getRandomAds() {
         ads.add(Iterables.get(allAds, random.nextInt(allAds.size())));
       }
       span.setAttribute("demo.ad.count", ads.size());
+      adsServedCounter.labelValues("random").inc(ads.size());
 
     } finally {
       span.end();

src/otel-collector/otelcol-config.yml

@@ -57,6 +57,13 @@ receivers:
     endpoint: "valkey-cart:6379"
     username: "valkey"
     collection_interval: 10s
+  prometheus/ad:
+    config:
+      scrape_configs:
+        - job_name: ad
+          scrape_interval: 10s
+          static_configs:
+            - targets: ["ad:${env:AD_PROMETHEUS_PORT}"]
   # Host metrics
   host_metrics:
     root_path: /hostfs
@@ -232,7 +239,7 @@ service:
       processors: [resourcedetection, memory_limiter, transform/sanitize_spans]
       exporters: [otlp_grpc/jaeger, debug, span_metrics]
     metrics:
-      receivers: [docker_stats, http_check/frontend-proxy, host_metrics, nginx, otlp, postgresql, redis, span_metrics, kafkametrics]
+      receivers: [docker_stats, http_check/frontend-proxy, host_metrics, nginx, otlp, postgresql, prometheus/ad, redis, span_metrics, kafkametrics]
       processors: [resourcedetection, memory_limiter]
       exporters: [otlp_http/prometheus, debug]
     logs: