diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/metrics.py b/ooniapi/services/ooniprobe/src/ooniprobe/metrics.py index 8fa444bf..90447300 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/metrics.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/metrics.py @@ -91,6 +91,12 @@ class Metrics: labelnames=["status", "instance"], ) + FASTPATH_INSTANCE_FAILURE = Counter( + "fastpath_instance_failure", + "How many times a measurement submission per fastpath instance failed", + labelnames = ["instance"] + ) + SEND_S3_CNT = Counter( "measurement_s3_upload_count", "How many times ooniprobe sent a measurement to s3 disaggregated by status" diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py index b2b01c43..0c3032f1 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/reports.py @@ -194,13 +194,14 @@ async def receive_measurement( break except Exception as e: + Metrics.FASTPATH_INSTANCE_FAILURE.labels( + instance=fastpath_url + ).inc() log.exception( f"[{i + 1} / {len(fastpath_urls)}] Unable to send measurement to fastpath " f"({fastpath_url}): {e}" ) - Metrics.SEND_FASTPATH_CNT.labels(status="fail", instance="NA").inc() - if success: # Geoip anomaly detection runs only when the measurement was successfully # submitted to the fastpath, so retries don't cause duplicate anomaly entries. @@ -222,6 +223,8 @@ async def receive_measurement( return ReceiveMeasurementResponse(measurement_uid=msmt_uid) + Metrics.SEND_FASTPATH_CNT.labels(status="fail", instance="NA").inc() + # wasn't possible to send msmnt to fastpath, try to send it to s3 ts_prefix = now.strftime("%Y%m%d%H") tn = test_name.replace("_", "") diff --git a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py index 472576af..5d72ad48 100644 --- a/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py +++ b/ooniapi/services/ooniprobe/src/ooniprobe/routers/v1/probe_services.py @@ -998,7 +998,7 @@ async def submit_measurement( f"({fastpath_url}): {e}" ) - Metrics.SEND_FASTPATH_CNT.labels(status="fail", instance="NA").inc() + Metrics.SEND_FASTPATH_CNT.labels(status="fail", instance="NA").inc() if success: # Geoip anomaly detection runs only when the measurement was successfully