diff --git a/.secrets.baseline b/.secrets.baseline index 13a4e17c62c..b073232fc62 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -661,7 +661,7 @@ "hashed_secret": "03e227627ab8681281fdb8aa3d799b03f782d672", "is_secret": false, "is_verified": false, - "line_number": 2030, + "line_number": 2031, "type": "Secret Keyword", "verified_result": null }, @@ -669,7 +669,7 @@ "hashed_secret": "ef5f3d909f23bd0aa02b4253f98350384f709c86", "is_secret": false, "is_verified": false, - "line_number": 2137, + "line_number": 2138, "type": "Secret Keyword", "verified_result": null }, @@ -677,7 +677,7 @@ "hashed_secret": "cb1ae2b504c4615841d8144267a131231d2bd677", "is_secret": false, "is_verified": false, - "line_number": 2138, + "line_number": 2139, "type": "Secret Keyword", "verified_result": null }, @@ -685,7 +685,7 @@ "hashed_secret": "1a1e70e87dd0452c42f33ce9bf74aa28134dba6b", "is_secret": false, "is_verified": false, - "line_number": 2139, + "line_number": 2140, "type": "Secret Keyword", "verified_result": null }, @@ -693,7 +693,7 @@ "hashed_secret": "7b1ba2f04f2f1604dc4e3caffcadf9fcbce7df5b", "is_secret": false, "is_verified": false, - "line_number": 2140, + "line_number": 2141, "type": "Secret Keyword", "verified_result": null }, @@ -701,7 +701,7 @@ "hashed_secret": "0fa3b21ced80146d752888f2b60ec80e0d4b8925", "is_secret": false, "is_verified": false, - "line_number": 2145, + "line_number": 2146, "type": "Secret Keyword", "verified_result": null }, @@ -709,7 +709,7 @@ "hashed_secret": "f084f2068494b8d1cd06811dd97d02c3d85f40ee", "is_secret": false, "is_verified": false, - "line_number": 2160, + "line_number": 2161, "type": "Secret Keyword", "verified_result": null }, @@ -717,7 +717,7 @@ "hashed_secret": "adfa401a3b0a733d8f00519ac8c6b3893a2e7e8e", "is_secret": false, "is_verified": false, - "line_number": 2161, + "line_number": 2162, "type": "Secret Keyword", "verified_result": null }, @@ -725,7 +725,7 @@ "hashed_secret": "898e46bbadc12f87120548bd445eb4210c8407c8", "is_secret": false, "is_verified": false, - "line_number": 2169, + "line_number": 2170, "type": "Secret Keyword", "verified_result": null }, @@ -733,7 +733,7 @@ "hashed_secret": "f57ccec6b8f7b12b635ab53d26c3bf7300247341", "is_secret": false, "is_verified": false, - "line_number": 2170, + "line_number": 2171, "type": "Secret Keyword", "verified_result": null }, @@ -741,7 +741,7 @@ "hashed_secret": "77b044ea736f8cbe568d1954424186d901f89db9", "is_secret": false, "is_verified": false, - "line_number": 2171, + "line_number": 2172, "type": "Secret Keyword", "verified_result": null }, @@ -749,7 +749,7 @@ "hashed_secret": "d64368f12ca17c69568c6a132f17d44d56e60660", "is_secret": false, "is_verified": false, - "line_number": 2172, + "line_number": 2173, "type": "Secret Keyword", "verified_result": null }, @@ -757,7 +757,7 @@ "hashed_secret": "8f9ca35156c02cb6ba58c5b51230b9bedc38de4f", "is_secret": false, "is_verified": false, - "line_number": 2173, + "line_number": 2174, "type": "Secret Keyword", "verified_result": null }, @@ -765,7 +765,7 @@ "hashed_secret": "9ec53cfd9929c70c3f87c210b6a7b77fb6d79d43", "is_secret": false, "is_verified": false, - "line_number": 2767, + "line_number": 2768, "type": "Secret Keyword", "verified_result": null }, @@ -773,7 +773,7 @@ "hashed_secret": "ee977806d7286510da8b9a7492ba58e2484c0ecc", "is_secret": false, "is_verified": false, - "line_number": 2920, + "line_number": 2925, "type": "Secret Keyword", "verified_result": null }, @@ -781,7 +781,7 @@ "hashed_secret": "adc1f5c8707f7d7aba3aabe13c15e5ef1151872e", "is_secret": false, "is_verified": false, - "line_number": 2921, + "line_number": 2926, "type": "Secret Keyword", "verified_result": null }, @@ -789,7 +789,7 @@ "hashed_secret": "ee46262b2df945e46ea310b925ad087465dbd3f2", "is_secret": false, "is_verified": false, - "line_number": 3646, + "line_number": 3647, "type": "Secret Keyword", "verified_result": null }, @@ -797,7 +797,7 @@ "hashed_secret": "f678cad4ab874d71b559a069d5e34a95fe38a480", "is_secret": false, "is_verified": false, - "line_number": 3647, + "line_number": 3648, "type": "Secret Keyword", "verified_result": null } diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 72f65ec7a17..4e4e723e1a2 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -1575,6 +1575,7 @@ ALERT_BUCKETEXCEEDINGSIZEQUOTASTATE = "NooBaaBucketExceedingSizeQuotaState" ALERT_NAMESPACERESOURCEERRORSTATE = "NooBaaNamespaceResourceErrorState" ALERT_NAMESPACEBUCKETERRORSTATE = "NooBaaNamespaceBucketErrorState" +ALERT_NOOBAA_REPLICATION_TARGET_UNREACHABLE = "NooBaaReplicationTargetUnreachable" ALERT_NODEDOWN = "CephNodeDown" ALERT_CLUSTERNEARFULL = "CephClusterNearFull" ALERT_CLUSTERCRITICALLYFULL = "CephClusterCriticallyFull" diff --git a/tests/functional/object/mcg/test_mcg_replication_target_unreachable.py b/tests/functional/object/mcg/test_mcg_replication_target_unreachable.py new file mode 100644 index 00000000000..162161abfeb --- /dev/null +++ b/tests/functional/object/mcg/test_mcg_replication_target_unreachable.py @@ -0,0 +1,135 @@ +import logging + +import pytest + +from ocs_ci.framework.testlib import MCGTest, tier2, polarion_id +from ocs_ci.framework.pytest_customization.marks import mcg, red_squad +from ocs_ci.utility.prometheus import PrometheusAPI +from ocs_ci.ocs import constants +from ocs_ci.ocs.bucket_utils import ( + write_random_test_objects_to_bucket, + compare_bucket_object_list, +) +from ocs_ci.ocs.exceptions import CommandFailed + +logger = logging.getLogger(__name__) + + +@mcg +@red_squad +class TestMCGReplicationTargetUnreachableAlert(MCGTest): + """ + Test suite for MCG replication target unreachable alert (RHSTOR-8110) + """ + + @pytest.fixture(autouse=True) + def reduce_replication_delay(self, add_env_vars_to_noobaa_core): + """ + Reduce the replication delay to one minute + + Args: + add_env_vars_to_noobaa_core (function): A function to add env vars to the noobaa-core pod + """ + new_delay_in_milliseconds = 60 * 1000 + new_env_var_tuples = [ + (constants.BUCKET_REPLICATOR_DELAY_PARAM, new_delay_in_milliseconds), + ] + add_env_vars_to_noobaa_core(new_env_var_tuples) + + @tier2 + @polarion_id("OCS-7917") + def test_noobaa_replication_target_unreachable( + self, + bucket_factory, + mcg_obj, + awscli_pod_session, + test_directory_setup, + threading_lock, + jira_issue, + ): + """ + 1. Create source and target buckets with a replication policy + 2. Write test objects and verify replication works + 3. Delete the target bucket and wait for alert to fire + 4. Verify alert properties and bucket names (not hash IDs - DFBUGS-6380) + 5. Delete the source bucket + 6. Verify alert clears after source bucket deletion (blocked by DFBUGS-6398) + """ + # 1. Create source and target buckets with a replication policy + target_bucket = bucket_factory(1, "OC")[0] + replication_policy = ("basic-replication-rule", target_bucket.name, None) + source_bucket = bucket_factory(1, "OC", replication_policy=replication_policy)[ + 0 + ] + logger.info( + f"Replication configured: {source_bucket.name} -> {target_bucket.name}" + ) + + # 2. Write test objects and verify replication works + write_random_test_objects_to_bucket( + awscli_pod_session, + source_bucket.name, + test_directory_setup.origin_dir, + amount=5, + mcg_obj=mcg_obj, + ) + assert compare_bucket_object_list( + mcg_obj, source_bucket.name, target_bucket.name, timeout=600 + ), "Replication did not work before target deletion" + logger.info("Replication verified successfully") + + # 3. Delete the target bucket and wait for alert to fire + target_bucket.delete() + logger.info(f"Target bucket deleted: {target_bucket.name}") + + api = PrometheusAPI(threading_lock=threading_lock) + alerts = api.wait_for_alert( + name=constants.ALERT_NOOBAA_REPLICATION_TARGET_UNREACHABLE, + state="firing", + timeout=60 * 8, + sleep=30, + ) + # 4. Verify alert properties and bucket names (not hash IDs - DFBUGS-6380) + alert = next( + ( + a + for a in alerts + if a.get("labels", {}).get("source_bucket") == source_bucket.name + ), + None, + ) + assert ( + alert is not None + ), "NooBaaReplicationTargetUnreachable alert not found for source bucket" + assert ( + alert["annotations"]["message"] + == "A NooBaa Replication Target Is Unreachable" + ) + assert alert["annotations"]["severity_level"] == "warning" + + description = alert.get("annotations", {}).get("description", "") + logger.info(f"Alert description: {description}") + assert ( + f"from bucket {source_bucket.name} to bucket {target_bucket.name}" + in description + ), f"Expected bucket names in description, got: {description}" + + # 5. Delete the source bucket + try: + source_bucket.delete(verify=True) + except CommandFailed as e: + logger.warning(f"Failed to delete source bucket: {e}") + + # 6. Verify alert clears after source bucket deletion (blocked by DFBUGS-6398) + if jira_issue("DFBUGS-6398"): + pytest.skip("DFBUGS-6398: alert persists after source bucket deletion") + + cleared = api.wait_for_alert( + name=constants.ALERT_NOOBAA_REPLICATION_TARGET_UNREACHABLE, + state=None, + timeout=180, + sleep=30, + ) + assert ( + len(cleared) == 0 + ), f"{constants.ALERT_NOOBAA_REPLICATION_TARGET_UNREACHABLE} alerts were not cleared"