Add happy-path test for NooBaaReplicationTargetUnreachable alert (RHSTOR-8110)

.secrets.baseline

@@ -661,143 +661,143 @@
         "hashed_secret": "03e227627ab8681281fdb8aa3d799b03f782d672",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2030,
+        "line_number": 2031,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "ef5f3d909f23bd0aa02b4253f98350384f709c86",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2137,
+        "line_number": 2138,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "cb1ae2b504c4615841d8144267a131231d2bd677",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2138,
+        "line_number": 2139,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "1a1e70e87dd0452c42f33ce9bf74aa28134dba6b",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2139,
+        "line_number": 2140,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "7b1ba2f04f2f1604dc4e3caffcadf9fcbce7df5b",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2140,
+        "line_number": 2141,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "0fa3b21ced80146d752888f2b60ec80e0d4b8925",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2145,
+        "line_number": 2146,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "f084f2068494b8d1cd06811dd97d02c3d85f40ee",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2160,
+        "line_number": 2161,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "adfa401a3b0a733d8f00519ac8c6b3893a2e7e8e",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2161,
+        "line_number": 2162,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "898e46bbadc12f87120548bd445eb4210c8407c8",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2169,
+        "line_number": 2170,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "f57ccec6b8f7b12b635ab53d26c3bf7300247341",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2170,
+        "line_number": 2171,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "77b044ea736f8cbe568d1954424186d901f89db9",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2171,
+        "line_number": 2172,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "d64368f12ca17c69568c6a132f17d44d56e60660",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2172,
+        "line_number": 2173,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "8f9ca35156c02cb6ba58c5b51230b9bedc38de4f",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2173,
+        "line_number": 2174,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "9ec53cfd9929c70c3f87c210b6a7b77fb6d79d43",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2767,
+        "line_number": 2768,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "ee977806d7286510da8b9a7492ba58e2484c0ecc",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2920,
+        "line_number": 2925,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "adc1f5c8707f7d7aba3aabe13c15e5ef1151872e",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2921,
+        "line_number": 2926,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "ee46262b2df945e46ea310b925ad087465dbd3f2",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 3646,
+        "line_number": 3647,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "f678cad4ab874d71b559a069d5e34a95fe38a480",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 3647,
+        "line_number": 3648,
         "type": "Secret Keyword",
         "verified_result": null
       }

ocs_ci/ocs/constants.py

@@ -1575,6 +1575,7 @@
 ALERT_BUCKETEXCEEDINGSIZEQUOTASTATE = "NooBaaBucketExceedingSizeQuotaState"
 ALERT_NAMESPACERESOURCEERRORSTATE = "NooBaaNamespaceResourceErrorState"
 ALERT_NAMESPACEBUCKETERRORSTATE = "NooBaaNamespaceBucketErrorState"
+ALERT_NOOBAA_REPLICATION_TARGET_UNREACHABLE = "NooBaaReplicationTargetUnreachable"
 ALERT_NODEDOWN = "CephNodeDown"
 ALERT_CLUSTERNEARFULL = "CephClusterNearFull"
 ALERT_CLUSTERCRITICALLYFULL = "CephClusterCriticallyFull"

tests/functional/object/mcg/test_mcg_replication_target_unreachable.py

@@ -0,0 +1,135 @@
+import logging
+
+import pytest
+
+from ocs_ci.framework.testlib import MCGTest, tier2, polarion_id
+from ocs_ci.framework.pytest_customization.marks import mcg, red_squad
+from ocs_ci.utility.prometheus import PrometheusAPI
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.bucket_utils import (
+    write_random_test_objects_to_bucket,
+    compare_bucket_object_list,
+)
+from ocs_ci.ocs.exceptions import CommandFailed
+
+logger = logging.getLogger(__name__)
+
+
+@mcg
+@red_squad
+class TestMCGReplicationTargetUnreachableAlert(MCGTest):
+    """
+    Test suite for MCG replication target unreachable alert (RHSTOR-8110)
+    """
+
+    @pytest.fixture(autouse=True)
+    def reduce_replication_delay(self, add_env_vars_to_noobaa_core):
+        """
+        Reduce the replication delay to one minute
+
+        Args:
+            add_env_vars_to_noobaa_core (function): A function to add env vars to the noobaa-core pod
+        """
+        new_delay_in_milliseconds = 60 * 1000
+        new_env_var_tuples = [
+            (constants.BUCKET_REPLICATOR_DELAY_PARAM, new_delay_in_milliseconds),
+        ]
+        add_env_vars_to_noobaa_core(new_env_var_tuples)
+
+    @tier2
+    @polarion_id("OCS-XXXX")
+    def test_noobaa_replication_target_unreachable(
+        self,
+        bucket_factory,
+        mcg_obj,
+        awscli_pod_session,
+        test_directory_setup,
+        threading_lock,
+        jira_issue,
+    ):
+        """
+        1. Create source and target buckets with a replication policy
+        2. Write test objects and verify replication works
+        3. Delete the target bucket and wait for alert to fire
+        4. Verify alert properties and bucket names (not hash IDs - DFBUGS-6380)
+        5. Delete the source bucket
+        6. Verify alert clears after source bucket deletion (blocked by DFBUGS-6398)
+        """
+        # 1. Create source and target buckets with a replication policy
+        target_bucket = bucket_factory(1, "OC")[0]
+        replication_policy = ("basic-replication-rule", target_bucket.name, None)
+        source_bucket = bucket_factory(1, "OC", replication_policy=replication_policy)[
+            0
+        ]
+        logger.info(
+            f"Replication configured: {source_bucket.name} -> {target_bucket.name}"
+        )
+
+        # 2. Write test objects and verify replication works
+        write_random_test_objects_to_bucket(
+            awscli_pod_session,
+            source_bucket.name,
+            test_directory_setup.origin_dir,
+            amount=5,
+            mcg_obj=mcg_obj,
+        )
+        assert compare_bucket_object_list(
+            mcg_obj, source_bucket.name, target_bucket.name, timeout=600
+        ), "Replication did not work before target deletion"
+        logger.info("Replication verified successfully")
+
+        # 3. Delete the target bucket and wait for alert to fire
+        target_bucket.delete()
+        logger.info(f"Target bucket deleted: {target_bucket.name}")
+
+        api = PrometheusAPI(threading_lock=threading_lock)
+        alerts = api.wait_for_alert(
+            name=constants.ALERT_NOOBAA_REPLICATION_TARGET_UNREACHABLE,
+            state="firing",
+            timeout=60 * 8,
+            sleep=30,
+        )
+        # 4. Verify alert properties and bucket names (not hash IDs - DFBUGS-6380)
+        alert = next(
+            (
+                a
+                for a in alerts
+                if a.get("labels", {}).get("source_bucket") == source_bucket.name
+            ),
+            None,
+        )
+        assert (
+            alert is not None
+        ), "NooBaaReplicationTargetUnreachable alert not found for source bucket"
+        assert (
+            alert["annotations"]["message"]
+            == "A NooBaa Replication Target Is Unreachable"
+        )
+        assert alert["annotations"]["severity_level"] == "warning"
+
+        description = alert.get("annotations", {}).get("description", "")
+        logger.info(f"Alert description: {description}")
+        assert (
+            f"from bucket {source_bucket.name} to bucket {target_bucket.name}"
+            in description
+        ), f"Expected bucket names in description, got: {description}"
+
+        # 5. Delete the source bucket
+        try:
+            source_bucket.delete(verify=True)
+        except CommandFailed as e:
+            logger.warning(f"Failed to delete source bucket: {e}")
+
+        # 6. Verify alert clears after source bucket deletion (blocked by DFBUGS-6398)
+        if jira_issue("DFBUGS-6398"):
+            pytest.skip("DFBUGS-6398: alert persists after source bucket deletion")
+
+        cleared = api.wait_for_alert(
+            name=constants.ALERT_NOOBAA_REPLICATION_TARGET_UNREACHABLE,
+            state=None,
+            timeout=180,
+            sleep=30,
+        )
+        assert (
+            len(cleared) == 0
+        ), f"{constants.ALERT_NOOBAA_REPLICATION_TARGET_UNREACHABLE} alerts were not cleared"