Fix NFS mount failure on IBM Cloud by fixing retry logic and opening port 2049 (#15119)

ocs_ci/framework/pytest_customization/marks.py

@@ -614,6 +614,11 @@ def setup_multicluster_marker(marker_base, push_missing_configs=False):
 # Marker for skipping tests for provider clusters based on OCS version
 skip_for_provider_if_ocs_version = pytest.mark.skip_for_provider_if_ocs_version
 
+# Marker for skipping tests for provider or client clusters based on OCS version
+skip_for_provider_or_client_if_ocs_version = (
+    pytest.mark.skip_for_provider_or_client_if_ocs_version
+)
+
 skipif_rosa = pytest.mark.skipif(
     config.ENV_DATA["platform"].lower() == ROSA_PLATFORM,
     reason="Test will not run on ROSA cluster",

ocs_ci/ocs/constants.py

@@ -3604,6 +3604,8 @@
 NFS_CSI_PLUGIN_LABEL = "app=csi-nfsplugin"
 NFS_CSI_CTRLPLUGIN_LABEL_419 = "app=openshift-storage.nfs.csi.ceph.com-ctrlplugin"
 NFS_CSI_NODEPLUGIN_LABEL_419 = "app=openshift-storage.nfs.csi.ceph.com-nodeplugin"
+NFS_STORAGECLASS_NAME = "ocs-storagecluster-ceph-nfs"
+COPY_NFS_STORAGECLASS_NAME = "ocs-storagecluster-ceph-nfs-copy"
 DAEMONSET_CSI_CEPHFS = "openshift-storage.cephfs.csi.ceph.com-nodeplugin"
 DAEMONSET_CSI_RBD = "openshift-storage.rbd.csi.ceph.com-nodeplugin"
 DAEMONSET_CSI_RBD_CSI_ADDONS = (

ocs_ci/utility/ibmcloud.py

@@ -36,7 +36,6 @@
 from ocs_ci.utility.utils import get_infra_id, get_ocp_version, run_cmd, TimeoutSampler
 from ocs_ci.ocs.node import get_nodes
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -1454,6 +1453,134 @@ def configure_ingress_load_balancer_security_group():
         raise
 
 
+def _get_lb_security_groups(svc_name, namespace):
+    """
+    Look up the IBM Cloud VPC load balancer backing a Kubernetes
+    LoadBalancer Service and return its security groups.
+
+    Args:
+        svc_name (str): Kubernetes Service name
+        namespace (str): Kubernetes namespace
+
+    Returns:
+        list: security group dicts from the VPC LB, empty list on
+            failure
+
+    """
+    rg_name = get_resource_group_name(config.ENV_DATA["cluster_path"])
+
+    svc_ocp = OCP(
+        kind="Service",
+        namespace=namespace,
+        resource_name=svc_name,
+    )
+    svc_data = svc_ocp.get()
+
+    lb_ingress = svc_data.get("status", {}).get("loadBalancer", {}).get("ingress", [])
+    if not lb_ingress:
+        logger.warning(
+            f"No LB ingress on service {svc_name}, cannot configure " f"security group"
+        )
+        return []
+
+    lb_hostname = lb_ingress[0].get("hostname") or lb_ingress[0].get("ip")
+    if not lb_hostname:
+        logger.warning(f"No hostname/IP in LB ingress for service {svc_name}")
+        return []
+
+    logger.debug(f"LB endpoint for {svc_name}: {lb_hostname}")
+
+    cmd = f"ibmcloud is lbs --resource-group-name {rg_name} " f"--output json"
+    out = run_ibmcloud_cmd(cmd)
+    load_balancers = json.loads(out)
+
+    matching_lb = None
+    for lb in load_balancers:
+        if lb.get("hostname") == lb_hostname:
+            matching_lb = lb
+            break
+
+    if not matching_lb:
+        logger.error(f"Could not find IBM Cloud VPC LB with hostname {lb_hostname}")
+        return []
+
+    security_groups = matching_lb.get("security_groups", [])
+    if not security_groups:
+        logger.warning(f"No security groups on LB {matching_lb.get('name')}")
+    return security_groups
+
+
+def configure_nfs_lb_security_group():
+    """
+    Add an inbound TCP rule for port 2049 (NFS) to the security
+    groups attached to the NFS LoadBalancer on IBM Cloud VPC.
+
+    Must be called after the ``rook-ceph-nfs-my-nfs-load-balancer``
+    Service has an ingress address assigned.
+    """
+    svc_name = "rook-ceph-nfs-my-nfs-load-balancer"
+    namespace = constants.OPENSHIFT_STORAGE_NAMESPACE
+    logger.info("Configuring NFS LB security group for port 2049")
+
+    security_groups = _get_lb_security_groups(svc_name, namespace)
+    for sg in security_groups:
+        sg_name = sg.get("name")
+        try:
+            logger.info(f"Adding inbound TCP 2049 to {sg_name}")
+            add_security_group_rule(sg_name, "inbound", "tcp", 2049, 2049)
+        except Exception as e:
+            logger.warning(
+                f"Failed to add port 2049 rule to {sg_name} "
+                f"(may already exist): {e}"
+            )
+
+    logger.info("NFS LB security group configuration done")
+
+
+def remove_nfs_lb_security_group_rules():
+    """
+    Remove inbound TCP 2049 rules from the security groups attached
+    to the NFS LoadBalancer on IBM Cloud VPC.
+
+    Should be called before deleting the NFS LoadBalancer Service so
+    the VPC LB is still present for look-up.
+    """
+    svc_name = "rook-ceph-nfs-my-nfs-load-balancer"
+    namespace = constants.OPENSHIFT_STORAGE_NAMESPACE
+    logger.info("Removing NFS LB security group rules for port 2049")
+
+    security_groups = _get_lb_security_groups(svc_name, namespace)
+    for sg in security_groups:
+        sg_id = sg.get("id")
+        sg_name = sg.get("name")
+        try:
+            cmd = f"ibmcloud is security-group {sg_id} " f"--output json"
+            out = run_ibmcloud_cmd(cmd)
+            sg_detail = json.loads(out)
+        except Exception as e:
+            logger.warning(f"Could not fetch rules for {sg_name}: {e}")
+            continue
+
+        for rule in sg_detail.get("rules", []):
+            if (
+                rule.get("direction") == "inbound"
+                and rule.get("protocol") == "tcp"
+                and rule.get("port_min") == 2049
+                and rule.get("port_max") == 2049
+            ):
+                rule_id = rule.get("id")
+                logger.info(f"Deleting rule {rule_id} from {sg_name}")
+                try:
+                    run_ibmcloud_cmd(
+                        f"ibmcloud is security-group-rule-delete "
+                        f"{sg_id} {rule_id} --force"
+                    )
+                except Exception as e:
+                    logger.warning(f"Failed to delete rule {rule_id}: {e}")
+
+    logger.info("NFS LB security group cleanup done")
+
+
 def create_address_prefix(prefix_name, vpc, zone, cidr):
     """
     Create address prefix in VPC.

ocs_ci/utility/nfs_utils.py

@@ -194,13 +194,22 @@ def create_nfs_load_balancer_service(
     if "hostname" in host_details:
         hostname_add = host_details["hostname"]
         log.info("ingress hostname, %s", hostname_add)
-        return hostname_add
     elif "ip" in host_details:
-        host_ip = host_details["ip"]
-        log.info("ingress host ip, %s", host_ip)
-        return host_ip
+        hostname_add = host_details["ip"]
+        log.info(f"ingress host ip, {hostname_add}")
     else:
         log.error("host details unavailable")
+        return None
+
+    platform = config.ENV_DATA.get("platform", "").lower()
+    if platform == constants.IBMCLOUD_PLATFORM:
+        from ocs_ci.utility.ibmcloud import (
+            configure_nfs_lb_security_group,
+        )
+
+        configure_nfs_lb_security_group()
+
+    return hostname_add
 
 
 def update_etc_hosts_on_nfs_client(con, hostname):
@@ -295,6 +304,14 @@ def delete_nfs_load_balancer_service(
         )
         return
 
+    platform = config.ENV_DATA.get("platform", "").lower()
+    if platform == constants.IBMCLOUD_PLATFORM:
+        from ocs_ci.utility.ibmcloud import (
+            remove_nfs_lb_security_group_rules,
+        )
+
+        remove_nfs_lb_security_group_rules()
+
     log.info("Deleting NFS LoadBalancer service %s", svc_name)
     storage_cluster_obj.exec_oc_cmd(f"delete service {svc_name}")