diff --git a/.secrets.baseline b/.secrets.baseline index b5496ed72d73..a2b1fbd7068b 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -625,7 +625,7 @@ "hashed_secret": "a12337323b638ab044b1166bff4b1a1f83162819", "is_secret": false, "is_verified": false, - "line_number": 825, + "line_number": 826, "type": "Secret Keyword", "verified_result": null }, @@ -633,7 +633,7 @@ "hashed_secret": "fb947972c92f052c0a08866d182be0075a2b601b", "is_secret": false, "is_verified": false, - "line_number": 834, + "line_number": 835, "type": "Secret Keyword", "verified_result": null }, @@ -641,7 +641,7 @@ "hashed_secret": "03e227627ab8681281fdb8aa3d799b03f782d672", "is_secret": false, "is_verified": false, - "line_number": 2014, + "line_number": 2015, "type": "Secret Keyword", "verified_result": null }, @@ -649,7 +649,7 @@ "hashed_secret": "ef5f3d909f23bd0aa02b4253f98350384f709c86", "is_secret": false, "is_verified": false, - "line_number": 2121, + "line_number": 2122, "type": "Secret Keyword", "verified_result": null }, @@ -657,7 +657,7 @@ "hashed_secret": "cb1ae2b504c4615841d8144267a131231d2bd677", "is_secret": false, "is_verified": false, - "line_number": 2122, + "line_number": 2123, "type": "Secret Keyword", "verified_result": null }, @@ -665,7 +665,7 @@ "hashed_secret": "1a1e70e87dd0452c42f33ce9bf74aa28134dba6b", "is_secret": false, "is_verified": false, - "line_number": 2123, + "line_number": 2124, "type": "Secret Keyword", "verified_result": null }, @@ -673,7 +673,7 @@ "hashed_secret": "7b1ba2f04f2f1604dc4e3caffcadf9fcbce7df5b", "is_secret": false, "is_verified": false, - "line_number": 2124, + "line_number": 2125, "type": "Secret Keyword", "verified_result": null }, @@ -681,7 +681,7 @@ "hashed_secret": "0fa3b21ced80146d752888f2b60ec80e0d4b8925", "is_secret": false, "is_verified": false, - "line_number": 2129, + "line_number": 2130, "type": "Secret Keyword", "verified_result": null }, @@ -689,7 +689,7 @@ "hashed_secret": "f084f2068494b8d1cd06811dd97d02c3d85f40ee", "is_secret": false, "is_verified": false, - "line_number": 2144, + "line_number": 2145, "type": "Secret Keyword", "verified_result": null }, @@ -697,7 +697,7 @@ "hashed_secret": "adfa401a3b0a733d8f00519ac8c6b3893a2e7e8e", "is_secret": false, "is_verified": false, - "line_number": 2145, + "line_number": 2146, "type": "Secret Keyword", "verified_result": null }, @@ -705,7 +705,7 @@ "hashed_secret": "898e46bbadc12f87120548bd445eb4210c8407c8", "is_secret": false, "is_verified": false, - "line_number": 2153, + "line_number": 2154, "type": "Secret Keyword", "verified_result": null }, @@ -713,7 +713,7 @@ "hashed_secret": "f57ccec6b8f7b12b635ab53d26c3bf7300247341", "is_secret": false, "is_verified": false, - "line_number": 2154, + "line_number": 2155, "type": "Secret Keyword", "verified_result": null }, @@ -721,7 +721,7 @@ "hashed_secret": "77b044ea736f8cbe568d1954424186d901f89db9", "is_secret": false, "is_verified": false, - "line_number": 2155, + "line_number": 2156, "type": "Secret Keyword", "verified_result": null }, @@ -729,7 +729,7 @@ "hashed_secret": "d64368f12ca17c69568c6a132f17d44d56e60660", "is_secret": false, "is_verified": false, - "line_number": 2156, + "line_number": 2157, "type": "Secret Keyword", "verified_result": null }, @@ -737,7 +737,7 @@ "hashed_secret": "8f9ca35156c02cb6ba58c5b51230b9bedc38de4f", "is_secret": false, "is_verified": false, - "line_number": 2157, + "line_number": 2158, "type": "Secret Keyword", "verified_result": null }, @@ -745,7 +745,7 @@ "hashed_secret": "9ec53cfd9929c70c3f87c210b6a7b77fb6d79d43", "is_secret": false, "is_verified": false, - "line_number": 2740, + "line_number": 2743, "type": "Secret Keyword", "verified_result": null }, @@ -753,7 +753,7 @@ "hashed_secret": "ee977806d7286510da8b9a7492ba58e2484c0ecc", "is_secret": false, "is_verified": false, - "line_number": 2897, + "line_number": 2900, "type": "Secret Keyword", "verified_result": null }, @@ -761,7 +761,7 @@ "hashed_secret": "adc1f5c8707f7d7aba3aabe13c15e5ef1151872e", "is_secret": false, "is_verified": false, - "line_number": 2898, + "line_number": 2901, "type": "Secret Keyword", "verified_result": null }, @@ -769,7 +769,7 @@ "hashed_secret": "ee46262b2df945e46ea310b925ad087465dbd3f2", "is_secret": false, "is_verified": false, - "line_number": 3619, + "line_number": 3622, "type": "Secret Keyword", "verified_result": null }, @@ -777,7 +777,7 @@ "hashed_secret": "f678cad4ab874d71b559a069d5e34a95fe38a480", "is_secret": false, "is_verified": false, - "line_number": 3620, + "line_number": 3623, "type": "Secret Keyword", "verified_result": null } diff --git a/conf/README.md b/conf/README.md index b11c5c62a3cc..a4566f231511 100644 --- a/conf/README.md +++ b/conf/README.md @@ -520,7 +520,8 @@ Configuration specific to external Ceph cluster * `key` - Admin keyring value used for the external Ceph cluster * `external_cluster_details` - base64 encoded data of json output from exporter script * `rgw_secure` - boolean parameter which defines if external Ceph cluster RGW is secured using SSL -* `rgw_cert_ca` - url pointing to CA certificate used to sign certificate for RGW with SSL +* `rgw_cert_ca` - URL for the RGW signing CA when external Ceph is **below 19.0**, or as a **fallback** if cephadm CA fetch fails on 19.0+ +* For external Ceph **19.0 and newer**, ocs-ci runs ``ceph orch certmgr cert get cephadm_root_ca_cert`` via ``cephadm shell`` on the ``_admin`` node (``get_external_cluster_client("_admin")``, falling back to ``node1``) instead of using ``rgw_cert_ca``, unless that command fails * `use_rbd_namespace` - boolean parameter to use RBD namespace in pool * `rbd_namespace` - Name of RBD namespace to use in pool diff --git a/ocs_ci/deployment/helpers/external_cluster_helpers.py b/ocs_ci/deployment/helpers/external_cluster_helpers.py index f1454d5a315c..ae2049feb040 100644 --- a/ocs_ci/deployment/helpers/external_cluster_helpers.py +++ b/ocs_ci/deployment/helpers/external_cluster_helpers.py @@ -312,6 +312,37 @@ def get_admin_keyring(self): config.EXTERNAL_MODE["admin_keyring"]["key"] = client_admin[index + 2] return + def fetch_cephadm_root_ca_cert_pem(self): + """ + Return the cephadm-managed root CA (``cephadm_root_ca_cert``) from this host. + + Used for Ceph 19+ when RGW TLS is signed by the cephadm CA. Runs + ``cephadm shell``; prefixes ``sudo`` when the SSH user is not ``root``. + + Returns: + str: PEM text (with trailing newline). + + Raises: + ExternalClusterExporterRunFailed: If the remote command fails or PEM is empty. + + """ + cmd = ( + "/usr/sbin/cephadm shell -- ceph orch certmgr cert get cephadm_root_ca_cert" + ) + if self.user != "root": + cmd = f"sudo {cmd}" + ret, out, err = self.rhcs_conn.exec_cmd(cmd) + if ret != 0: + raise ExternalClusterExporterRunFailed( + f"cephadm_root_ca_cert fetch failed: {err}" + ) + pem = _normalize_cephadm_certmgr_stdout(out) + if not pem: + raise ExternalClusterExporterRunFailed( + "cephadm_root_ca_cert fetch returned empty output" + ) + return pem if pem.endswith("\n") else f"{pem}\n" + def get_rgw_endpoint_api_port(self): """ Fetches rgw endpoint api port. @@ -1126,16 +1157,93 @@ def generate_exporter_script(use_configmap=False): return external_cluster_details_exporter.name +def _normalize_cephadm_certmgr_stdout(raw_stdout): + text = (raw_stdout or "").strip() + if "BEGIN CERTIFICATE" in text or "BEGIN TRUSTED CERTIFICATE" in text: + return text + if text.startswith("{"): + try: + payload = json.loads(text) + except json.JSONDecodeError: + return text + for key in ("certificate", "cert", "pem", "data", "ca_certificate"): + val = payload.get(key) + if isinstance(val, str) and val.strip(): + return val.strip() + return text + + +def _external_ceph_semantic_version_or_none(): + """ + Parse Ceph version from ``ceph --version`` on the default external SSH host. + + Uses ``cephadm shell`` so the command runs in the cephadm environment; if that + fails (e.g. legacy non-cephadm layout), falls back to host ``ceph --version``. + """ + try: + ext = get_external_cluster_instance() + cmd = "/usr/sbin/cephadm shell -- ceph --version" + if ext.user != "root": + cmd = f"sudo {cmd}" + rc, out, _ = ext.rhcs_conn.exec_cmd(cmd) + if rc != 0 or not (out or "").strip(): + rc, out, _ = ext.rhcs_conn.exec_cmd("ceph --version") + if rc != 0: + return None + m = re.search(r"ceph\s+version\s+(\d+)\.(\d+)", out, re.I) + if not m: + return None + return version.get_semantic_version( + f"{m.group(1)}.{m.group(2)}", only_major_minor=True + ) + except Exception as exc: + logger.debug("Could not determine external Ceph version: %s", exc) + return None + + +def external_rgw_ca_should_use_cephadm_fetch(): + """ + True when external Ceph reports version >= 19.0 (cephadm certmgr path), or if + version is not determined (None) - assuming that from ODF 4.18 we are using Ceph >= 19. + """ + v = _external_ceph_semantic_version_or_none() + if v is None: + return True + return v >= version.get_semantic_version("19.0", only_major_minor=True) + + +def try_embed_rgw_ca_pem_in_mcg_cli_resources(service_ca_data, sts_dict): + """ + If deploy stashed ``embedded_external_rgw_ca_pem``, add it to the service-ca + ConfigMap and add a matching volumeMount on ``sts_dict`` (first container). + + Returns: + bool: True if embedding was applied. + """ + pem = config.EXTERNAL_MODE.get("embedded_external_rgw_ca_pem") + if not pem: + return False + service_ca_data.setdefault("data", {})[constants.EXTERNAL_RGW_CA_CM_KEY] = pem + sts_dict["spec"]["template"]["spec"]["containers"][0]["volumeMounts"].append( + { + "name": "service-ca", + "mountPath": constants.EXTERNAL_RGW_CA_CONTAINER_PATH, + "subPath": constants.EXTERNAL_RGW_CA_CM_KEY, + } + ) + return True + + def get_and_apply_rgw_cert_ca(apply=True): """ - Downloads CA Certificate of RGW if SSL is used and apply it to be trusted - by the OCP cluster + Obtain the RGW TLS CA: for external Ceph **19.0+**, fetch ``cephadm_root_ca_cert`` + from the ``_admin`` node via SSH; otherwise download from ``rgw_cert_ca`` URL. Args: apply (bool): if True, the certificate is applied as trusted CA by the OCP cluster Returns: - str: path to the downloaded RGW Cert CA + str: path to the local RGW CA PEM file """ rgw_cert_ca_path = tempfile.NamedTemporaryFile( @@ -1144,10 +1252,33 @@ def get_and_apply_rgw_cert_ca(apply=True): suffix=".pem", delete=False, ).name - download_file( - config.EXTERNAL_MODE["rgw_cert_ca"], - rgw_cert_ca_path, - ) + config.EXTERNAL_MODE.pop("embedded_external_rgw_ca_pem", None) + + if external_rgw_ca_should_use_cephadm_fetch(): + try: + host, user, password, ssh_key = get_external_cluster_client("_admin") + pem = ExternalCluster( + host, user, password, ssh_key + ).fetch_cephadm_root_ca_cert_pem() + with open(rgw_cert_ca_path, "w", encoding="utf-8") as pem_fd: + pem_fd.write(pem) + config.EXTERNAL_MODE["embedded_external_rgw_ca_pem"] = pem + logger.info( + "Using cephadm_root_ca_cert from external cluster (Ceph >= 19.0)" + ) + except Exception as exc: + logger.warning( + "cephadm CA fetch failed (%s); falling back to rgw_cert_ca URL", exc + ) + download_file( + config.EXTERNAL_MODE["rgw_cert_ca"], + rgw_cert_ca_path, + ) + else: + download_file( + config.EXTERNAL_MODE["rgw_cert_ca"], + rgw_cert_ca_path, + ) # configure the CA cert to be trusted by the OCP cluster if apply: ssl_certs.configure_trusted_ca_bundle(ca_cert_path=rgw_cert_ca_path) @@ -1191,12 +1322,16 @@ def get_rgw_endpoint(): raise ExternalClusterRGWEndPointMissing(err_msg) -def get_external_cluster_client(): +def get_external_cluster_client(role=None): """ - Finding the client role node IP address. + Resolve SSH target for an external RHCS node by role. + + Args: + role (str or None): Node role to match in ``external_cluster_node_roles`` (e.g. + ``client``, ``_admin``). If None, uses ``_admin`` when multicluster else ``client``. Returns: - tuple: IP address, user, password of the client, ssh key + tuple: (ip_address, user, password, ssh_key) Raises: ExternalClusterCephSSHAuthDetailsMissing: In case one of SSH key or password @@ -1212,13 +1347,13 @@ def get_external_cluster_client(): "Either password or SSH key is missing in EXTERNAL_MODE['login'] section!" ) nodes = config.EXTERNAL_MODE["external_cluster_node_roles"] - node_role = None - node_role = "_admin" if config.multicluster else "client" + if role is None: + role = "_admin" if config.multicluster else "client" try: - return get_node_by_role(nodes, node_role, user, password, ssh_key) + return get_node_by_role(nodes, role, user, password, ssh_key) except ExternalClusterNodeRoleNotFound: - logger.warning(f"No {node_role} role defined, using node1 address!") + logger.warning(f"No {role} role defined, using node1 address!") return (nodes["node1"]["ip_address"], user, password, ssh_key) diff --git a/ocs_ci/ocs/awscli_pod.py b/ocs_ci/ocs/awscli_pod.py index e9f503076a8d..ffd41cd0b909 100644 --- a/ocs_ci/ocs/awscli_pod.py +++ b/ocs_ci/ocs/awscli_pod.py @@ -19,6 +19,9 @@ from ocs_ci.utility import templating from ocs_ci.utility.retry import retry from ocs_ci.utility.utils import update_container_with_mirrored_image +from ocs_ci.deployment.helpers.external_cluster_helpers import ( + try_embed_rgw_ca_pem_in_mcg_cli_resources, +) from ocs_ci.utility.ssl_certs import ( create_ocs_ca_bundle, get_root_ca_cert, @@ -49,19 +52,21 @@ def create_awscli_pod(scope_name=None, namespace=None, service_account=None): service_ca_data["metadata"]["namespace"] = namespace s3cli_label_k, s3cli_label_v = constants.S3CLI_APP_LABEL.split("=") service_ca_data["metadata"]["labels"] = {s3cli_label_k: s3cli_label_v} - log.info("Trying to create the AWS CLI service CA") - - service_ca_configmap = create_resource(**service_ca_data) - OCP(namespace=namespace, kind="ConfigMap").wait_for_resource( - resource_name=service_ca_configmap.name, column="DATA", condition="1" - ) - log.info("Creating the AWS CLI StatefulSet") + log.info("Creating the AWS CLI StatefulSet manifest (before service-ca ConfigMap)") awscli_sts_dict = templating.load_yaml(constants.S3CLI_MULTIARCH_STS_YAML) awscli_sts_dict["spec"]["template"]["spec"]["volumes"][0]["configMap"][ "name" ] = service_ca_configmap_name awscli_sts_dict["metadata"]["namespace"] = namespace + try_embed_rgw_ca_pem_in_mcg_cli_resources(service_ca_data, awscli_sts_dict) + + log.info("Trying to create the AWS CLI service CA") + service_ca_configmap = create_resource(**service_ca_data) + OCP(namespace=namespace, kind="ConfigMap").wait_for_resource( + resource_name=service_ca_configmap.name, column="DATA", condition="1" + ) + update_container_with_mirrored_image(awscli_sts_dict) update_container_with_proxy_env(awscli_sts_dict) _add_startup_commands_to_set_ca(awscli_sts_dict) @@ -151,11 +156,19 @@ def _add_startup_commands_to_set_ca(awscli_sts_dict): f"cp {constants.SERVICE_CA_CRT_AWSCLI_PATH} {constants.AWSCLI_CA_BUNDLE_PATH}" ) - # Download and concatenate an additional CA cert if needed if storagecluster_independent_check() and config.EXTERNAL_MODE.get("rgw_secure"): - startup_cmds.append( - f"wget -O - {config.EXTERNAL_MODE['rgw_cert_ca']} >> {constants.AWSCLI_CA_BUNDLE_PATH}" - ) + if config.EXTERNAL_MODE.get("embedded_external_rgw_ca_pem"): + startup_cmds.append( + f"cat {constants.EXTERNAL_RGW_CA_CONTAINER_PATH} >> {constants.AWSCLI_CA_BUNDLE_PATH}" + ) + elif config.EXTERNAL_MODE.get("rgw_cert_ca"): + startup_cmds.append( + f"wget -O - {config.EXTERNAL_MODE['rgw_cert_ca']} >> {constants.AWSCLI_CA_BUNDLE_PATH}" + ) + else: + log.warning( + "rgw_secure is set but neither embedded RGW CA nor rgw_cert_ca URL is available" + ) if config.DEPLOYMENT.get("use_custom_ingress_ssl_cert"): startup_cmds.append( f"cat /cert/ocs-ca-bundle.crt >> {constants.AWSCLI_CA_BUNDLE_PATH}" diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 0f095b57878b..a580d0168007 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -2399,6 +2399,8 @@ SERVICE_MONITORS = "servicemonitors" SERVICE_CA_CRT_AWSCLI_PATH = f"/cert/{SERVICE_CA_CRT}" AWSCLI_CA_BUNDLE_PATH = "/tmp/ca-bundle.crt" +EXTERNAL_RGW_CA_CM_KEY = "rgw-external-ca.pem" +EXTERNAL_RGW_CA_CONTAINER_PATH = f"/cert/{EXTERNAL_RGW_CA_CM_KEY}" AWSCLI_RELAY_POD_NAME = "awscli-relay-pod" JAVAS3_POD_NAME = "java-s3" SCALECLI_SERVICE_CA_CM_NAME = "scalecli-service-ca" diff --git a/tests/conftest.py b/tests/conftest.py index 0c17c77c7588..18d55728b141 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,9 @@ from ocs_ci.deployment.cnv import CNVInstaller from ocs_ci.deployment import factory as dep_factory +from ocs_ci.deployment.helpers.external_cluster_helpers import ( + try_embed_rgw_ca_pem_in_mcg_cli_resources, +) from ocs_ci.deployment.helpers.hypershift_base import HyperShiftBase from ocs_ci.deployment.hub_spoke import ( destroy_aws_hcp_clusters, @@ -3251,18 +3254,22 @@ def nb_stress_cli_pod_fixture(request, scope_name): s3cli_label_k, s3cli_label_v = constants.STRESS_CLI_APP_LABEL.split("=") service_ca_data["metadata"]["labels"] = {s3cli_label_k: s3cli_label_v} - log.info("Trying to create the Stress CLI service CA") - service_ca_configmap = create_resource(**service_ca_data) - OCP(namespace=namespace, kind="ConfigMap").wait_for_resource( - resource_name=service_ca_configmap.name, column="DATA", condition="1" + log.info( + "Creating the Stress CLI StatefulSet manifest (before service-ca ConfigMap)" ) - - log.info("Creating the Stress CLI StatefulSet") stress_cli_sts_dict = templating.load_yaml(constants.STRESS_CLI_STS_YAML) stress_cli_sts_dict["spec"]["template"]["spec"]["volumes"][0]["configMap"][ "name" ] = service_ca_configmap_name stress_cli_sts_dict["metadata"]["namespace"] = namespace + try_embed_rgw_ca_pem_in_mcg_cli_resources(service_ca_data, stress_cli_sts_dict) + + log.info("Trying to create the Stress CLI service CA") + service_ca_configmap = create_resource(**service_ca_data) + OCP(namespace=namespace, kind="ConfigMap").wait_for_resource( + resource_name=service_ca_configmap.name, column="DATA", condition="1" + ) + update_container_with_mirrored_image(stress_cli_sts_dict) update_container_with_proxy_env(stress_cli_sts_dict) stress_cli_sts_obj = create_resource(**stress_cli_sts_dict) @@ -3292,10 +3299,20 @@ def nb_stress_cli_pod_fixture(request, scope_name): "rgw_secure" ): log.info("Concatenating the RGW CA to the Stress CLI pod's CA bundle") - pod_obj.exec_cmd_on_pod( - f"bash -c 'wget -O - {ocsci_config.EXTERNAL_MODE['rgw_cert_ca']} >> " - f"{constants.AWSCLI_CA_BUNDLE_PATH}'" - ) + if ocsci_config.EXTERNAL_MODE.get("embedded_external_rgw_ca_pem"): + pod_obj.exec_cmd_on_pod( + f"bash -c 'cat {constants.EXTERNAL_RGW_CA_CONTAINER_PATH} >> " + f"{constants.AWSCLI_CA_BUNDLE_PATH}'" + ) + elif ocsci_config.EXTERNAL_MODE.get("rgw_cert_ca"): + pod_obj.exec_cmd_on_pod( + f"bash -c 'wget -O - {ocsci_config.EXTERNAL_MODE['rgw_cert_ca']} >> " + f"{constants.AWSCLI_CA_BUNDLE_PATH}'" + ) + else: + log.warning( + "rgw_secure is set but neither embedded RGW CA nor rgw_cert_ca URL is available" + ) def cleanup(): """