[fix] Alerts aren't sent for passive metrics #670

Closes #670

Author: pandafy

openwisp_monitoring/monitoring/base/models.py

@@ -992,39 +992,38 @@ def _is_crossed_by(self, current_value, time=None, retention_policy=None):
         extra_fields = []
         if self.metric.alert_on_related_field:
             extra_fields = [self.metric.alert_field]
-        if time is None:
-            # retrieves latest measurements, ordered by most recent first
-            points = self.metric.read(
-                since=timezone.now() - timedelta(minutes=self._tolerance_search_range),
-                limit=None,
-                order="-time",
-                retention_policy=retention_policy,
-                extra_fields=extra_fields,
-            )
-            # store a list with the results
-            results = [value_crossed]
-            # loop on each measurement starting from the most recent
-            for i, point in enumerate(points, 1):
-                # skip the first point because it was just added before this
-                # check started and its value coincides with ``current_value``
-                if i <= 1:
-                    continue
-                utc_time = utc.localize(datetime.utcfromtimestamp(point["time"]))
-                # did this point cross the threshold? Append to result list
-                results.append(self._value_crossed(point[self.metric.alert_field]))
-                # tolerance is trepassed
-                if self._time_crossed(utc_time):
-                    # if the latest results are consistent, the metric being
-                    # monitored is not flapping and we can confidently return
-                    # wheter the value crosses the threshold or not
-                    if len(set(results)) == 1:
-                        return value_crossed
-                    # otherwise, the results are flapping, the situation has not changed
-                    # we will return a value that will not trigger changes
-                    return not self.metric.is_healthy_tolerant
-                # otherwise keep looking back
+        time = time or timezone.now()
+        # retrieves latest measurements, ordered by most recent first
+        points = self.metric.read(
+            since=timezone.now() - timedelta(minutes=self._tolerance_search_range),
+            limit=None,
+            order="-time",
+            retention_policy=retention_policy,
+            extra_fields=extra_fields,
+        )
+        # store a list with the results
+        results = [value_crossed]
+        # loop on each measurement starting from the most recent
+        for i, point in enumerate(points, 1):
+            # skip the first point because it was just added before this
+            # check started and its value coincides with ``current_value``
+            if i <= 1:
                 continue
-            # the search has not yielded any conclusion
-            # return result based on the current value and time
-            time = timezone.now()
+            utc_time = utc.localize(datetime.utcfromtimestamp(point["time"]))
+            # did this point cross the threshold? Append to result list
+            results.append(self._value_crossed(point[self.metric.alert_field]))
+            # tolerance is trepassed
+            if self._time_crossed(utc_time):
+                # if the latest results are consistent, the metric being
+                # monitored is not flapping and we can confidently return
+                # wheter the value crosses the threshold or not
+                if len(set(results)) == 1:
+                    return value_crossed
+                # otherwise, the results are flapping, the situation has not changed
+                # we will return a value that will not trigger changes
+                return not self.metric.is_healthy_tolerant
+            # otherwise keep looking back
+            continue
+        # the search has not yielded any conclusion
+        # return result based on the current value and time
         return self._time_crossed(time) and value_crossed

openwisp_monitoring/monitoring/tests/test_monitoring_notifications.py

@@ -10,6 +10,7 @@
 )
 
 Metric = load_model("monitoring", "Metric")
+AlertSettings = load_model("monitoring", "AlertSettings")
 Notification = load_model("openwisp_notifications", "Notification")
 Device = load_model("config", "Device")
 Config = load_model("config", "Config")
@@ -358,8 +359,8 @@ def test_object_check_threshold_crossed_historical_data(self):
         self._write_metric(om, 99, time=start_time)
         om.refresh_from_db()
         self.assertEqual(om.is_healthy, False)
-        self.assertEqual(om.is_healthy_tolerant, True)
-        self.assertEqual(Notification.objects.count(), 0)
+        self.assertEqual(om.is_healthy_tolerant, False)
+        self.assertEqual(Notification.objects.count(), 1)
 
     def test_flapping_metric_with_tolerance(self):
         self._create_admin()
@@ -679,3 +680,48 @@ def test_multiple_notifications(self):
             self.assertEqual(Notification.objects.count(), 1)
             n = notification_queryset.first()
             self._check_notification_parameters(n, admin, om, user)
+
+    def test_passive_metric_alert(self):
+        self._get_admin()
+        data = {
+            "type": "DeviceMonitoring",
+            "resources": {
+                "cpus": 1,
+                "load": [0, 0, 0],
+            },
+        }
+        device = self._create_device(organization=self._create_org())
+        self._post_data(device.id, device.key, data)
+        cpu_metric = Metric.objects.get(key="cpu")
+        self.assertEqual(Notification.objects.count(), 0)
+
+        AlertSettings.objects.update(
+            custom_tolerance=5,
+            custom_threshold=90,
+        )
+
+        data["resources"]["load"] = [100, 100, 100]
+
+        with freeze_time(timezone.now() + timedelta(minutes=1)):
+            response = self._post_data(device.id, device.key, data)
+            self.assertEqual(response.status_code, 200)
+            self.assertEqual(Notification.objects.count(), 0)
+            cpu_metric.refresh_from_db()
+            self.assertEqual(cpu_metric.is_healthy, False)
+            self.assertEqual(cpu_metric.is_healthy_tolerant, True)
+
+        with freeze_time(timezone.now() + timedelta(minutes=4)):
+            response = self._post_data(device.id, device.key, data)
+            self.assertEqual(response.status_code, 200)
+            self.assertEqual(Notification.objects.count(), 0)
+            cpu_metric.refresh_from_db()
+            self.assertEqual(cpu_metric.is_healthy, False)
+            self.assertEqual(cpu_metric.is_healthy_tolerant, True)
+
+        with freeze_time(timezone.now() + timedelta(minutes=6)):
+            response = self._post_data(device.id, device.key, data)
+            self.assertEqual(response.status_code, 200)
+            cpu_metric.refresh_from_db()
+            self.assertEqual(cpu_metric.is_healthy, False)
+            self.assertEqual(cpu_metric.is_healthy_tolerant, False)
+            self.assertEqual(Notification.objects.count(), 1)