From 71f5ea87ad570c1938e11363d5899343f2c335b7 Mon Sep 17 00:00:00 2001 From: Nagachandan-P Date: Thu, 21 May 2026 02:44:31 +0000 Subject: [PATCH 1/5] ib ip assignment with nic name Signed-off-by: Nagachandan-P --- .../doca-ofed/configure-ib-network.sh.j2 | 165 ++++++++++++++---- 1 file changed, 128 insertions(+), 37 deletions(-) diff --git a/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 b/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 index d87720e495..e48928bd23 100644 --- a/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 +++ b/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 @@ -14,66 +14,157 @@ modprobe ib_ipoib || true modprobe ib_umad || true modprobe ib_uverbs || true -ADMIN_NIC_IP="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" NETMASK_BITS="{{ hostvars['localhost']['admin_netmask_bits'] }}" -declare -A IB_IP_MAP=( + +# Get current node's IP from cloud-init metadata +ADMIN_NIC_IP="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + +# Create IB mapping from all nodes (IB_NIC_NAME only) +declare -A ADMIN_IB_NIC_NAME_MAP=( {% for mac, node in hostvars['localhost']['read_mapping_file']['dict'].items() -%} -{% if node.IB_IP is defined and node.IB_IP | trim | length > 0 %} +{% if node.ADMIN_IP is defined and node.ADMIN_IP | trim | length > 0 and node.IB_NIC_NAME is defined and node.IB_NIC_NAME | trim | length > 0 %} + ["{{ node.ADMIN_IP }}"]="{{ node.IB_NIC_NAME }}" +{%- endif %} +{%- endfor %} +) + +declare -A ADMIN_IB_IP_MAP=( +{% for mac, node in hostvars['localhost']['read_mapping_file']['dict'].items() -%} +{% if node.ADMIN_IP is defined and node.ADMIN_IP | trim | length > 0 and node.IB_IP is defined and node.IB_IP | trim | length > 0 %} ["{{ node.ADMIN_IP }}"]="{{ node.IB_IP }}" {%- endif %} {%- endfor %} ) -IB_IP="${IB_IP_MAP[$ADMIN_NIC_IP]:-}" +# Get IB configuration for this node +IB_NIC_NAME="${ADMIN_IB_NIC_NAME_MAP[$ADMIN_NIC_IP]:-}" +IB_IP="${ADMIN_IB_IP_MAP[$ADMIN_NIC_IP]:-}" -if [ -n "$IB_IP" ]; then - echo "Using explicit IB IP : $IB_IP/$NETMASK_BITS" -else - echo "INFO: No explicit IB IP found in mapping file for node with ADMIN_IP: $ADMIN_NIC_IP" - echo "INFO: Skipping IB IP assignment. If IB networking is required for this node, please add IB_IP to the PXE mapping file." - echo "INFO: IB network interface will remain unconfigured." - exit 0 +# Skip if IB_NIC_NAME is empty +if [ -z "$IB_NIC_NAME" ]; then + echo "INFO: IB_NIC_NAME is empty - skipping IB network configuration" + exit 0 fi -MAX_WAIT=120 # total wait time in seconds (2 minutes) -INTERVAL=10 # check every 10 seconds -ELAPSED=0 -IB_NIC="" +echo "Target IB NIC name: $IB_NIC_NAME" +echo "Target IB IP: $IB_IP" -while [[ $ELAPSED -lt $MAX_WAIT ]]; do - for nic in $(ip -o link show | awk -F': ' '{print $2}' | grep '^ib'); do - if ip link show "$nic" | grep -q "UP,LOWER_UP"; then - IB_NIC="$nic" - break 2 - fi - done +# Extract port number from IB_NIC_NAME +# Handles formats: "InfiniBand.PCIe.Slot.1-1" → "1", "InfiniBand.Single-1" → "1" +PORT_NUMBER=$(echo "$IB_NIC_NAME" | grep -oE "[0-9]+$") +if [ -z "$PORT_NUMBER" ]; then + echo "ERROR: Invalid IB_NIC_NAME format: $IB_NIC_NAME" + echo "ERROR: Expected format: 'InfiniBand.PCIe.Slot.X-Y' or 'InfiniBand.Single-Y'" + exit 1 +fi +echo "Extracted port number: $PORT_NUMBER" + +# Check if IB hardware is present +if ! ls /sys/class/infiniband/ >/dev/null 2>&1; then + echo "ERROR: No IB hardware found in /sys/class/infiniband/" + exit 1 +fi - echo "IB interface not ready yet. Waiting..." - sleep $INTERVAL - ELAPSED=$((ELAPSED + INTERVAL)) +# Find mlx5 device - for now, use the first mlx5 device found +MLX5_DEVICE=$(ls /sys/class/infiniband/ | grep mlx5 | head -1) +if [ -z "$MLX5_DEVICE" ]; then + echo "ERROR: No mlx5 device found in /sys/class/infiniband/" + exit 1 +fi + +echo "Found mlx5 device: $MLX5_DEVICE" +echo "Note: Using first mlx5 device - for multi-device environments, consider adding slot-to-device mapping" + +# Check if the specified port exists +PORT_PATH="/sys/class/infiniband/${MLX5_DEVICE}/ports/${PORT_NUMBER}" +if [ ! -d "$PORT_PATH" ]; then + echo "ERROR: Port ${PORT_NUMBER} does not exist for device ${MLX5_DEVICE}" + echo "ERROR: Available ports:" + ls -la /sys/class/infiniband/${MLX5_DEVICE}/ports/ + exit 1 +fi + +# Retrieve GUID from the specified port +GUID_PATH="${PORT_PATH}/gids/0" +if [ ! -f "$GUID_PATH" ]; then + echo "ERROR: GUID file not found at ${GUID_PATH}" + exit 1 +fi + +GUID=$(cat "$GUID_PATH") +echo "Retrieved GUID: $GUID" + +# Convert GUID to individual octets for comparison +# GUID format: fe80:0000:0000:0000:946d:ae03:008c:108c +# Interface format: 94:6d:ae:03:00:8c:10:8c (1-byte groups) +GUID_OCTETS=$(echo "$GUID" | python3 -c " +import sys +guid = sys.stdin.read().strip() +parts = guid.split(':') +if len(parts) >= 8: + # Take last 4 parts and convert each 4-char group to 2-char groups + last_parts = parts[4:8] + octets = [] + for part in last_parts: + octets.append(part[:2]) + octets.append(part[2:]) + print(':'.join(octets)) +") +echo "GUID octets for comparison: $GUID_OCTETS" + +# Find the IB interface by comparing octets +IB_INTERFACE="" +for iface in $(ip link show | grep -E "^[0-9]+: ib" | awk -F: '{print $2}'); do + # Get the full address from the interface and extract the last 8 octets + full_addr=$(ip link show "$iface" | grep "link/infiniband" | sed 's/.*link\/infiniband //' | sed 's/ brd.*//') + # Extract the last 8 octets (fields 13-20) and format with colons for comparison + iface_octets=$(echo "$full_addr" | cut -d: -f13-20 | python3 -c " +import sys +parts = sys.stdin.read().strip().split(':') +if len(parts) >= 8: + formatted = f'{parts[0]}:{parts[1]}:{parts[2]}:{parts[3]}:{parts[4]}:{parts[5]}:{parts[6]}:{parts[7]}' + print(formatted) +") + echo "Interface $iface octets: $iface_octets" + if [ "$iface_octets" = "$GUID_OCTETS" ]; then + IB_INTERFACE="$iface" + break + fi done -if [[ -z "$IB_NIC" ]]; then - echo "No active InfiniBand interface found after ${MAX_WAIT}s. Exiting." - exit 0 +if [ -z "$IB_INTERFACE" ]; then + echo "ERROR: No IB interface found with GUID octets $GUID_OCTETS" + echo "ERROR: Available IB interfaces:" + for iface in $(ip link show | grep -E "^[0-9]+: ib" | awk -F: '{print $2}'); do + full_addr=$(ip link show "$iface" | grep "link/infiniband" | sed 's/.*link\/infiniband //' | sed 's/ brd.*//') + iface_octets=$(echo "$full_addr" | cut -d: -f13-20 | python3 -c " +import sys +parts = sys.stdin.read().strip().split(':') +if len(parts) >= 8: + formatted = f'{parts[0]}{parts[1]}:{parts[2]}{parts[3]}:{parts[4]}{parts[5]}{parts[6]}{parts[7]}' + print(formatted) +") + echo " $iface: $iface_octets" + done + exit 1 fi -echo "Using IB interface: $IB_NIC" +echo "SUCCESS: Found IB interface $IB_INTERFACE" if command -v nmcli >/dev/null 2>&1; then echo "Configuring IB interface using NetworkManager" - nmcli con delete "$IB_NIC" &>/dev/null || true - nmcli con add type infiniband ifname "$IB_NIC" con-name "$IB_NIC" - nmcli con modify "$IB_NIC" ipv4.method manual ipv4.addresses "$IB_IP/$NETMASK_BITS" - nmcli con up "$IB_NIC" + nmcli con delete "$IB_INTERFACE" &>/dev/null || true + nmcli con add type infiniband ifname "$IB_INTERFACE" con-name "$IB_INTERFACE" + nmcli con modify "$IB_INTERFACE" ipv4.method manual ipv4.addresses "$IB_IP/$NETMASK_BITS" + nmcli con up "$IB_INTERFACE" else echo "Configuring IB interface using iproute2" - ip addr flush dev "$IB_NIC" - ip addr add "$IB_IP/$NETMASK_BITS" dev "$IB_NIC" - ip link set "$IB_NIC" up + ip addr flush dev "$IB_INTERFACE" + ip addr add "$IB_IP/$NETMASK_BITS" dev "$IB_INTERFACE" + ip link set "$IB_INTERFACE" up fi -echo "SUCCESS: Assigned $IB_IP/$NETMASK_BITS to $IB_NIC" +echo "SUCCESS: Assigned $IB_IP/$NETMASK_BITS to $IB_INTERFACE" # Configure DNS for InfiniBand network if [ -n "$IB_IP" ]; then From 03b01459c2b337b9e665d7c692109a14391a8b7b Mon Sep 17 00:00:00 2001 From: Nagachandan-P Date: Thu, 21 May 2026 10:58:29 +0000 Subject: [PATCH 2/5] with slot and port Signed-off-by: Nagachandan-P --- .../doca-ofed/configure-ib-network.sh.j2 | 95 ++++++++++++++++--- 1 file changed, 82 insertions(+), 13 deletions(-) diff --git a/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 b/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 index e48928bd23..821e14b7fe 100644 --- a/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 +++ b/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 @@ -49,15 +49,28 @@ fi echo "Target IB NIC name: $IB_NIC_NAME" echo "Target IB IP: $IB_IP" -# Extract port number from IB_NIC_NAME -# Handles formats: "InfiniBand.PCIe.Slot.1-1" → "1", "InfiniBand.Single-1" → "1" -PORT_NUMBER=$(echo "$IB_NIC_NAME" | grep -oE "[0-9]+$") -if [ -z "$PORT_NUMBER" ]; then +# Extract slot and port numbers from IB_NIC_NAME +# Support formats: "InfiniBand.Slot.X-Y", "NIC.InfiniBand.X-Y", "InfiniBand.Single-Y" +SLOT_NUMBER="" +PORT_NUMBER="" + +if [[ "$IB_NIC_NAME" =~ ^(InfiniBand\.PCIe\.Slot\.|InfiniBand\.Slot\.|NIC\.InfiniBand\.)([0-9]+)-([0-9]+)$ ]]; then + SLOT_NUMBER="${BASH_REMATCH[2]}" + PORT_NUMBER="${BASH_REMATCH[3]}" + echo "Extracted slot number: $SLOT_NUMBER" + echo "Extracted port number: $PORT_NUMBER" +elif [[ "$IB_NIC_NAME" =~ ^InfiniBand\.Single-([0-9]+)$ ]]; then + PORT_NUMBER="${BASH_REMATCH[1]}" + echo "Single-port device detected, port number: $PORT_NUMBER" +else echo "ERROR: Invalid IB_NIC_NAME format: $IB_NIC_NAME" - echo "ERROR: Expected format: 'InfiniBand.PCIe.Slot.X-Y' or 'InfiniBand.Single-Y'" + echo "ERROR: Expected formats:" + echo " - 'InfiniBand.PCIe.Slot.X-Y' (slot X, port Y)" + echo " - 'InfiniBand.Slot.X-Y' (slot X, port Y)" + echo " - 'NIC.InfiniBand.X-Y' (slot X, port Y)" + echo " - 'InfiniBand.Single-Y' (single device, port Y)" exit 1 fi -echo "Extracted port number: $PORT_NUMBER" # Check if IB hardware is present if ! ls /sys/class/infiniband/ >/dev/null 2>&1; then @@ -65,15 +78,71 @@ if ! ls /sys/class/infiniband/ >/dev/null 2>&1; then exit 1 fi -# Find mlx5 device - for now, use the first mlx5 device found -MLX5_DEVICE=$(ls /sys/class/infiniband/ | grep mlx5 | head -1) -if [ -z "$MLX5_DEVICE" ]; then - echo "ERROR: No mlx5 device found in /sys/class/infiniband/" - exit 1 +# Count mlx5 devices to determine mapping strategy +MLX5_DEVICE_COUNT=$(ls /sys/class/infiniband/ | grep mlx5 | wc -l) +echo "Found $MLX5_DEVICE_COUNT mlx5 device(s)" + +# Simple slot-to-device mapping function +find_device_by_slot() { + local target_slot="$1" + + for mlx_device in /sys/class/infiniband/mlx5_*; do + [ -e "$mlx_device" ] || continue + device_name=$(basename "$mlx_device") + + # Get PCI address and extract slot number + pci_address=$(basename "$(readlink -f "/sys/class/infiniband/$device_name/device" 2>/dev/null)") + if [ -n "$pci_address" ]; then + # Extract slot from PCI address: 0000:17:00.0 → "17" + pci_slot=$(echo "$pci_address" | cut -d: -f2 | sed 's/^0*//') + target_clean=$(echo "$target_slot" | sed 's/^0*//') + + echo "DEBUG: $device_name → PCI $pci_address → slot $pci_slot" + + if [[ "$pci_slot" == "$target_clean" ]]; then + echo "$device_name" + return 0 + fi + fi + done + return 1 +} + +# Select mlx5 device based on device count and slot information +if [ "$MLX5_DEVICE_COUNT" -gt 1 ] && [ -n "$SLOT_NUMBER" ]; then + echo "INFO: Multiple mlx5 devices detected, using slot-based mapping" + echo "INFO: Looking for device in slot $SLOT_NUMBER" + + MLX5_DEVICE=$(find_device_by_slot "$SLOT_NUMBER") + + if [ -z "$MLX5_DEVICE" ]; then + echo "ERROR: No mlx5 device found for slot $SLOT_NUMBER" + echo "ERROR: Available mlx5 devices:" + for mlx_device in /sys/class/infiniband/mlx5_*; do + [ -e "$mlx_device" ] || continue + device_name=$(basename "$mlx_device") + pci_address=$(basename "$(readlink -f "/sys/class/infiniband/$device_name/device" 2>/dev/null)") + if [ -n "$pci_address" ]; then + pci_slot=$(echo "$pci_address" | cut -d: -f2 | sed 's/^0*//') + echo " $device_name: PCI $pci_address (slot $pci_slot)" + else + echo " $device_name: PCI information not available" + fi + done + exit 1 + fi + + echo "SUCCESS: Found mlx5 device $MLX5_DEVICE for slot $SLOT_NUMBER" +else + echo "INFO: Using device-based mapping (single device or no slot specified)" + MLX5_DEVICE=$(ls /sys/class/infiniband/ | grep mlx5 | head -1) + if [ -z "$MLX5_DEVICE" ]; then + echo "ERROR: No mlx5 device found in /sys/class/infiniband/" + exit 1 + fi fi -echo "Found mlx5 device: $MLX5_DEVICE" -echo "Note: Using first mlx5 device - for multi-device environments, consider adding slot-to-device mapping" +echo "Selected mlx5 device: $MLX5_DEVICE" # Check if the specified port exists PORT_PATH="/sys/class/infiniband/${MLX5_DEVICE}/ports/${PORT_NUMBER}" From e91d44560d6a8f09fbd932b702e8dc764bd58ca8 Mon Sep 17 00:00:00 2001 From: Nagachandan-P Date: Thu, 21 May 2026 11:42:47 +0000 Subject: [PATCH 3/5] input validation Signed-off-by: Nagachandan-P --- .../validation_flows/provision_validation.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/common/library/module_utils/input_validation/validation_flows/provision_validation.py b/common/library/module_utils/input_validation/validation_flows/provision_validation.py index 92a6286c7f..1dacd6644b 100644 --- a/common/library/module_utils/input_validation/validation_flows/provision_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/provision_validation.py @@ -317,6 +317,69 @@ def validate_duplicate_ib_ips_in_mapping_file(pxe_mapping_file_path): raise ValueError(f"Duplicate IB_IP found in PXE mapping file: {'; '.join(duplicates)}") +def validate_ib_nic_name_format_in_mapping_file(pxe_mapping_file_path): + """Validates IB_NIC_NAME format structure in the mapping file. + + Validates that IB_NIC_NAME follows one of the supported formats: + - 'InfiniBand.PCIe.Slot.X-Y' (slot X, port Y) + - 'InfiniBand.Slot.X-Y' (slot X, port Y) + - 'NIC.InfiniBand.X-Y' (slot X, port Y) + - 'InfiniBand.Single-Y' (single device, port Y) + + Only validates format structure, not specific slot/port ranges. + """ + if not pxe_mapping_file_path or not os.path.isfile(pxe_mapping_file_path): + raise ValueError(f"PXE mapping file not found: {pxe_mapping_file_path}") + + with open(pxe_mapping_file_path, "r", encoding="utf-8") as fh: + raw_lines = fh.readlines() + + non_comment_lines = [ln for ln in raw_lines if ln.strip()] + reader = csv.DictReader(non_comment_lines) + + fieldname_map = {fn.strip().upper(): fn for fn in reader.fieldnames} + ib_nic_col = fieldname_map.get("IB_NIC_NAME") + hostname_col = fieldname_map.get("HOSTNAME") + + if not ib_nic_col: + return # No IB_NIC_NAME column to validate + + # Supported IB_NIC_NAME format patterns + slot_pattern = re.compile(r'^(InfiniBand\.PCIe\.Slot\.|InfiniBand\.Slot\.|NIC\.InfiniBand\.)([0-9]+)-([0-9]+)$') + single_pattern = re.compile(r'^InfiniBand\.Single-([0-9]+)$') + + invalid_formats = [] + + for row_idx, row in enumerate(reader, start=2): + ib_nic_name = row.get(ib_nic_col, "").strip() if ib_nic_col and row.get(ib_nic_col) else "" + hostname = "" + if hostname_col: + hostname = row.get(hostname_col, "").strip() if hostname_col and row.get(hostname_col) else "" + + # Skip empty IB_NIC_NAME (already handled by consistency validation) + if not ib_nic_name: + continue + + # Check if format matches supported patterns + if slot_pattern.match(ib_nic_name): + # Valid slot-based format + continue + elif single_pattern.match(ib_nic_name): + # Valid single-device format + continue + else: + # Invalid format + hostname_disp = f" ({hostname})" if hostname else "" + invalid_formats.append(f"'{ib_nic_name}' at CSV row {row_idx}{hostname_disp}") + + if invalid_formats: + raise ValueError( + f"Invalid IB_NIC_NAME format(s) found in PXE mapping file: {'; '.join(invalid_formats)}. " + f"Supported formats are: " + f"'InfiniBand.PCIe.Slot.X-Y', 'InfiniBand.Slot.X-Y', 'NIC.InfiniBand.X-Y', 'InfiniBand.Single-Y'" + ) + + def validate_group_parent_service_tag_consistency_in_mapping_file(pxe_mapping_file_path): """Validates that GROUP_NAME has a consistent PARENT_SERVICE_TAG across the mapping file.""" if not pxe_mapping_file_path or not os.path.isfile(pxe_mapping_file_path): @@ -919,6 +982,7 @@ def validate_provision_config( validate_duplicate_hostnames_in_mapping_file(pxe_mapping_file_path) validate_duplicate_admin_ips_in_mapping_file(pxe_mapping_file_path) validate_duplicate_ib_ips_in_mapping_file(pxe_mapping_file_path) + validate_ib_nic_name_format_in_mapping_file(pxe_mapping_file_path) validate_group_parent_service_tag_consistency_in_mapping_file(pxe_mapping_file_path) validate_functional_groups_separation(pxe_mapping_file_path) validate_parent_service_tag_hierarchy(pxe_mapping_file_path) From f975efe96b5fa72dadc0b2c3f45c2a832b8950b9 Mon Sep 17 00:00:00 2001 From: Nagachandan-P Date: Fri, 22 May 2026 02:11:27 +0000 Subject: [PATCH 4/5] slot matching logic Signed-off-by: Nagachandan-P --- .../doca-ofed/configure-ib-network.sh.j2 | 161 ++++++++++++++---- 1 file changed, 129 insertions(+), 32 deletions(-) diff --git a/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 b/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 index 821e14b7fe..8564c5ca13 100644 --- a/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 +++ b/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 @@ -54,16 +54,20 @@ echo "Target IB IP: $IB_IP" SLOT_NUMBER="" PORT_NUMBER="" +echo "DEBUG: Parsing IB_NIC_NAME format: '$IB_NIC_NAME'" + if [[ "$IB_NIC_NAME" =~ ^(InfiniBand\.PCIe\.Slot\.|InfiniBand\.Slot\.|NIC\.InfiniBand\.)([0-9]+)-([0-9]+)$ ]]; then SLOT_NUMBER="${BASH_REMATCH[2]}" PORT_NUMBER="${BASH_REMATCH[3]}" - echo "Extracted slot number: $SLOT_NUMBER" - echo "Extracted port number: $PORT_NUMBER" + echo "INFO: Successfully parsed slot-based format" + echo "INFO: Extracted slot number: $SLOT_NUMBER" + echo "INFO: Extracted port number: $PORT_NUMBER" elif [[ "$IB_NIC_NAME" =~ ^InfiniBand\.Single-([0-9]+)$ ]]; then PORT_NUMBER="${BASH_REMATCH[1]}" - echo "Single-port device detected, port number: $PORT_NUMBER" + echo "INFO: Successfully parsed single-port format" + echo "INFO: Single-port device detected, port number: $PORT_NUMBER" else - echo "ERROR: Invalid IB_NIC_NAME format: $IB_NIC_NAME" + echo "ERROR: Failed to parse IB_NIC_NAME format: '$IB_NIC_NAME'" echo "ERROR: Expected formats:" echo " - 'InfiniBand.PCIe.Slot.X-Y' (slot X, port Y)" echo " - 'InfiniBand.Slot.X-Y' (slot X, port Y)" @@ -80,12 +84,31 @@ fi # Count mlx5 devices to determine mapping strategy MLX5_DEVICE_COUNT=$(ls /sys/class/infiniband/ | grep mlx5 | wc -l) -echo "Found $MLX5_DEVICE_COUNT mlx5 device(s)" +echo "=== IB NETWORK CONFIGURATION LOGGING STARTED ===" +echo "INFO: Found $MLX5_DEVICE_COUNT mlx5 device(s) on this system" +echo "INFO: Target IB NIC name from PXE mapping: $IB_NIC_NAME" +echo "INFO: Target IB IP from PXE mapping: $IB_IP" + +# List all available mlx5 devices for debugging +echo "DEBUG: Available mlx5 devices on this system:" +for mlx_device in /sys/class/infiniband/mlx5_*; do + [ -e "$mlx_device" ] || continue + device_name=$(basename "$mlx_device") + pci_address=$(basename "$(readlink -f "/sys/class/infiniband/$device_name/device" 2>/dev/null)") + if [ -n "$pci_address" ]; then + pci_slot=$(echo "$pci_address" | cut -d: -f2 | sed 's/^0*//') + echo "DEBUG: Device: $device_name, PCI: $pci_address, Slot: $pci_slot" + else + echo "DEBUG: Device: $device_name, PCI: Not available" + fi +done # Simple slot-to-device mapping function find_device_by_slot() { local target_slot="$1" + echo "DEBUG: Starting slot-based device search for slot '$target_slot'" + for mlx_device in /sys/class/infiniband/mlx5_*; do [ -e "$mlx_device" ] || continue device_name=$(basename "$mlx_device") @@ -97,71 +120,100 @@ find_device_by_slot() { pci_slot=$(echo "$pci_address" | cut -d: -f2 | sed 's/^0*//') target_clean=$(echo "$target_slot" | sed 's/^0*//') - echo "DEBUG: $device_name → PCI $pci_address → slot $pci_slot" + echo "DEBUG: Checking device $device_name: PCI $pci_address → slot $pci_slot (target: $target_clean)" if [[ "$pci_slot" == "$target_clean" ]]; then + echo "SUCCESS: Found matching device! Slot $target_clean matches PCI slot $pci_slot for device $device_name" echo "$device_name" return 0 + else + echo "DEBUG: Slot mismatch - device $device_name has PCI slot $pci_slot, looking for slot $target_clean" fi + else + echo "DEBUG: Could not get PCI address for device $device_name" fi done + + echo "ERROR: No device found matching slot '$target_slot'" return 1 } # Select mlx5 device based on device count and slot information if [ "$MLX5_DEVICE_COUNT" -gt 1 ] && [ -n "$SLOT_NUMBER" ]; then - echo "INFO: Multiple mlx5 devices detected, using slot-based mapping" - echo "INFO: Looking for device in slot $SLOT_NUMBER" + echo "INFO: MULTI-DEVICE MODE: Multiple mlx5 devices detected ($MLX5_DEVICE_COUNT)" + echo "INFO: MULTI-DEVICE MODE: Using slot-based mapping for slot '$SLOT_NUMBER'" MLX5_DEVICE=$(find_device_by_slot "$SLOT_NUMBER") if [ -z "$MLX5_DEVICE" ]; then - echo "ERROR: No mlx5 device found for slot $SLOT_NUMBER" - echo "ERROR: Available mlx5 devices:" + echo "ERROR: SLOT MISMATCH: No mlx5 device found for slot '$SLOT_NUMBER'" + echo "ERROR: SLOT MISMATCH: This indicates the slot number in PXE mapping doesn't match any hardware PCI slot" + echo "ERROR: Available mlx5 devices with their PCI slots:" for mlx_device in /sys/class/infiniband/mlx5_*; do [ -e "$mlx_device" ] || continue device_name=$(basename "$mlx_device") pci_address=$(basename "$(readlink -f "/sys/class/infiniband/$device_name/device" 2>/dev/null)") if [ -n "$pci_address" ]; then pci_slot=$(echo "$pci_address" | cut -d: -f2 | sed 's/^0*//') - echo " $device_name: PCI $pci_address (slot $pci_slot)" + echo " - $device_name: PCI $pci_address (slot $pci_slot)" else - echo " $device_name: PCI information not available" + echo " - $device_name: PCI information not available" fi done + echo "ERROR: SLOT MISMATCH: Please update PXE mapping file with correct PCI slot numbers" exit 1 fi - echo "SUCCESS: Found mlx5 device $MLX5_DEVICE for slot $SLOT_NUMBER" + echo "SUCCESS: SLOT MATCHING: Found mlx5 device $MLX5_DEVICE for slot '$SLOT_NUMBER'" + echo "SUCCESS: SLOT MATCHING: Slot-based device selection successful" else - echo "INFO: Using device-based mapping (single device or no slot specified)" + if [ "$MLX5_DEVICE_COUNT" -eq 1 ]; then + echo "INFO: SINGLE-DEVICE MODE: Only one mlx5 device detected" + echo "INFO: SINGLE-DEVICE MODE: Using fallback device selection (ignoring slot number)" + echo "INFO: SINGLE-DEVICE MODE: This works for single-device setups even with incorrect slot numbers" + else + echo "INFO: FALLBACK MODE: Multiple devices detected but no slot number specified" + echo "INFO: FALLBACK MODE: Using first available device" + fi + MLX5_DEVICE=$(ls /sys/class/infiniband/ | grep mlx5 | head -1) if [ -z "$MLX5_DEVICE" ]; then echo "ERROR: No mlx5 device found in /sys/class/infiniband/" exit 1 fi + + echo "INFO: FALLBACK DEVICE: Selected $MLX5_DEVICE (first available device)" fi -echo "Selected mlx5 device: $MLX5_DEVICE" +echo "INFO: FINAL DEVICE SELECTION: Using mlx5 device '$MLX5_DEVICE'" # Check if the specified port exists PORT_PATH="/sys/class/infiniband/${MLX5_DEVICE}/ports/${PORT_NUMBER}" +echo "DEBUG: Checking if port $PORT_NUMBER exists on device $MLX5_DEVICE" +echo "DEBUG: Port path: $PORT_PATH" + if [ ! -d "$PORT_PATH" ]; then - echo "ERROR: Port ${PORT_NUMBER} does not exist for device ${MLX5_DEVICE}" - echo "ERROR: Available ports:" - ls -la /sys/class/infiniband/${MLX5_DEVICE}/ports/ + echo "ERROR: PORT NOT FOUND: Port ${PORT_NUMBER} does not exist for device ${MLX5_DEVICE}" + echo "ERROR: PORT NOT FOUND: Available ports on device $MLX5_DEVICE:" + ls -la /sys/class/infiniband/${MLX5_DEVICE}/ports/ 2>/dev/null || echo "ERROR: No ports directory found" exit 1 +else + echo "SUCCESS: Port validation passed - port $PORT_NUMBER exists on device $MLX5_DEVICE" fi # Retrieve GUID from the specified port GUID_PATH="${PORT_PATH}/gids/0" +echo "DEBUG: Retrieving GUID from port $PORT_NUMBER on device $MLX5_DEVICE" +echo "DEBUG: GUID path: $GUID_PATH" + if [ ! -f "$GUID_PATH" ]; then - echo "ERROR: GUID file not found at ${GUID_PATH}" + echo "ERROR: GUID NOT FOUND: GUID file not found at ${GUID_PATH}" + echo "ERROR: GUID NOT FOUND: Cannot proceed with interface matching without GUID" exit 1 fi GUID=$(cat "$GUID_PATH") -echo "Retrieved GUID: $GUID" +echo "SUCCESS: Retrieved GUID from hardware: $GUID" # Convert GUID to individual octets for comparison # GUID format: fe80:0000:0000:0000:946d:ae03:008c:108c @@ -179,13 +231,25 @@ if len(parts) >= 8: octets.append(part[2:]) print(':'.join(octets)) ") -echo "GUID octets for comparison: $GUID_OCTETS" +echo "SUCCESS: GUID octets for interface comparison: $GUID_OCTETS" # Find the IB interface by comparing octets +echo "DEBUG: Starting IB interface search using GUID matching" IB_INTERFACE="" + +# List all available IB interfaces first +echo "DEBUG: Available IB interfaces on this system:" +for iface in $(ip link show | grep -E "^[0-9]+: ib" | awk -F: '{print $2}'); do + echo "DEBUG: Found IB interface: $iface" +done + for iface in $(ip link show | grep -E "^[0-9]+: ib" | awk -F: '{print $2}'); do + echo "DEBUG: Checking interface $iface for GUID match" + # Get the full address from the interface and extract the last 8 octets full_addr=$(ip link show "$iface" | grep "link/infiniband" | sed 's/.*link\/infiniband //' | sed 's/ brd.*//') + echo "DEBUG: Interface $iface full address: $full_addr" + # Extract the last 8 octets (fields 13-20) and format with colons for comparison iface_octets=$(echo "$full_addr" | cut -d: -f13-20 | python3 -c " import sys @@ -194,56 +258,89 @@ if len(parts) >= 8: formatted = f'{parts[0]}:{parts[1]}:{parts[2]}:{parts[3]}:{parts[4]}:{parts[5]}:{parts[6]}:{parts[7]}' print(formatted) ") - echo "Interface $iface octets: $iface_octets" + echo "DEBUG: Interface $iface octets: $iface_octets (target: $GUID_OCTETS)" + if [ "$iface_octets" = "$GUID_OCTETS" ]; then + echo "SUCCESS: GUID MATCH FOUND! Interface $iface matches hardware GUID" IB_INTERFACE="$iface" break + else + echo "DEBUG: GUID mismatch - interface $iface does not match target GUID" fi done if [ -z "$IB_INTERFACE" ]; then - echo "ERROR: No IB interface found with GUID octets $GUID_OCTETS" - echo "ERROR: Available IB interfaces:" + echo "ERROR: INTERFACE NOT FOUND: No IB interface found with GUID octets $GUID_OCTETS" + echo "ERROR: INTERFACE NOT FOUND: This indicates a hardware/software mismatch" + echo "ERROR: INTERFACE NOT FOUND: Available IB interfaces with their GUID octets:" for iface in $(ip link show | grep -E "^[0-9]+: ib" | awk -F: '{print $2}'); do full_addr=$(ip link show "$iface" | grep "link/infiniband" | sed 's/.*link\/infiniband //' | sed 's/ brd.*//') iface_octets=$(echo "$full_addr" | cut -d: -f13-20 | python3 -c " import sys parts = sys.stdin.read().strip().split(':') if len(parts) >= 8: - formatted = f'{parts[0]}{parts[1]}:{parts[2]}{parts[3]}:{parts[4]}{parts[5]}{parts[6]}{parts[7]}' + formatted = f'{parts[0]}{parts[1]}:{parts[2]}{parts[3]}:{parts[4]}{parts[5]}:{parts[6]}{parts[7]}' print(formatted) ") - echo " $iface: $iface_octets" + echo " - $iface: $iface_octets" done exit 1 fi -echo "SUCCESS: Found IB interface $IB_INTERFACE" +echo "SUCCESS: INTERFACE SELECTION: Found IB interface $IB_INTERFACE matching hardware GUID" if command -v nmcli >/dev/null 2>&1; then - echo "Configuring IB interface using NetworkManager" + echo "INFO: IP CONFIGURATION: Using NetworkManager to configure IB interface" + echo "DEBUG: IP CONFIGURATION: Target IP $IB_IP/$NETMASK_BITS on interface $IB_INTERFACE" + + echo "DEBUG: IP CONFIGURATION: Removing existing NetworkManager connection for $IB_INTERFACE" nmcli con delete "$IB_INTERFACE" &>/dev/null || true + + echo "DEBUG: IP CONFIGURATION: Creating new NetworkManager connection for $IB_INTERFACE" nmcli con add type infiniband ifname "$IB_INTERFACE" con-name "$IB_INTERFACE" + + echo "DEBUG: IP CONFIGURATION: Setting IP address $IB_IP/$NETMASK_BITS on $IB_INTERFACE" nmcli con modify "$IB_INTERFACE" ipv4.method manual ipv4.addresses "$IB_IP/$NETMASK_BITS" + + echo "DEBUG: IP CONFIGURATION: Bringing up NetworkManager connection for $IB_INTERFACE" nmcli con up "$IB_INTERFACE" + + echo "SUCCESS: IP CONFIGURATION: NetworkManager successfully configured $IB_INTERFACE with IP $IB_IP/$NETMASK_BITS" else - echo "Configuring IB interface using iproute2" + echo "INFO: IP CONFIGURATION: Using iproute2 to configure IB interface (NetworkManager not available)" + echo "DEBUG: IP CONFIGURATION: Target IP $IB_IP/$NETMASK_BITS on interface $IB_INTERFACE" + + echo "DEBUG: IP CONFIGURATION: Flushing existing IP addresses from $IB_INTERFACE" ip addr flush dev "$IB_INTERFACE" + + echo "DEBUG: IP CONFIGURATION: Adding IP address $IB_IP/$NETMASK_BITS to $IB_INTERFACE" ip addr add "$IB_IP/$NETMASK_BITS" dev "$IB_INTERFACE" + + echo "DEBUG: IP CONFIGURATION: Bringing up interface $IB_INTERFACE" ip link set "$IB_INTERFACE" up + + echo "SUCCESS: IP CONFIGURATION: iproute2 successfully configured $IB_INTERFACE with IP $IB_IP/$NETMASK_BITS" fi -echo "SUCCESS: Assigned $IB_IP/$NETMASK_BITS to $IB_INTERFACE" +echo "SUCCESS: FINAL IP ASSIGNMENT: Successfully assigned $IB_IP/$NETMASK_BITS to IB interface $IB_INTERFACE" # Configure DNS for InfiniBand network if [ -n "$IB_IP" ]; then - echo "Configuring DNS for InfiniBand interface" + echo "INFO: DNS CONFIGURATION: Configuring DNS for InfiniBand interface" # Add VAST DNS servers (completely safe - handles empty arrays) {% for dns_server in hostvars['localhost']['ib_network_dns'] %} + echo "DEBUG: DNS CONFIGURATION: Adding DNS server {{ dns_server }}" echo "nameserver {{ dns_server }}" >> /etc/resolv.conf {% endfor %} - echo "SUCCESS: DNS configured for IB network" + echo "SUCCESS: DNS CONFIGURATION: DNS configuration completed for IB network" +else + echo "INFO: DNS CONFIGURATION: No DNS configuration needed (IB IP is empty)" fi + +echo "=== IB NETWORK CONFIGURATION COMPLETED SUCCESSFULLY ===" +echo "SUMMARY: IB interface $IB_INTERFACE configured with IP $IB_IP/$NETMASK_BITS" +echo "SUMMARY: Device used: $MLX5_DEVICE, Port: $PORT_NUMBER" +echo "SUMMARY: Configuration method: $([ -n "$SLOT_NUMBER" ] && echo "Slot-based mapping" || echo "Fallback device selection")" From 50747c9a0d10fa713b181d091682858006ad2390 Mon Sep 17 00:00:00 2001 From: Nagachandan-P Date: Fri, 22 May 2026 02:31:32 +0000 Subject: [PATCH 5/5] no slot matching for single nic Signed-off-by: Nagachandan-P --- .../doca-ofed/configure-ib-network.sh.j2 | 73 +++++++++++++------ 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 b/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 index 8564c5ca13..7644c08d00 100644 --- a/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 +++ b/provision/roles/configure_ochami/templates/doca-ofed/configure-ib-network.sh.j2 @@ -138,17 +138,37 @@ find_device_by_slot() { return 1 } -# Select mlx5 device based on device count and slot information -if [ "$MLX5_DEVICE_COUNT" -gt 1 ] && [ -n "$SLOT_NUMBER" ]; then - echo "INFO: MULTI-DEVICE MODE: Multiple mlx5 devices detected ($MLX5_DEVICE_COUNT)" - echo "INFO: MULTI-DEVICE MODE: Using slot-based mapping for slot '$SLOT_NUMBER'" +# === DEVICE COUNT ANALYSIS AND SELECTION LOGIC === +echo "INFO: DEVICE ANALYSIS: Analyzing mlx5 device count for configuration strategy" + +if [ "$MLX5_DEVICE_COUNT" -eq 0 ]; then + echo "ERROR: DEVICE ANALYSIS: No mlx5 devices found on this system" + echo "ERROR: DEVICE ANALYSIS: Please check hardware and driver installation" + exit 1 - MLX5_DEVICE=$(find_device_by_slot "$SLOT_NUMBER") +elif [ "$MLX5_DEVICE_COUNT" -eq 1 ]; then + echo "INFO: SINGLE-DEVICE MODE: Only one mlx5 device available - no slot-based mapping needed" + echo "INFO: SINGLE-DEVICE MODE: Using available device regardless of slot number in PXE mapping" + echo "INFO: SINGLE-DEVICE MODE: This ensures robustness for single-device deployments" + MLX5_DEVICE=$(ls /sys/class/infiniband/ | grep mlx5 | head -1) if [ -z "$MLX5_DEVICE" ]; then - echo "ERROR: SLOT MISMATCH: No mlx5 device found for slot '$SLOT_NUMBER'" - echo "ERROR: SLOT MISMATCH: This indicates the slot number in PXE mapping doesn't match any hardware PCI slot" - echo "ERROR: Available mlx5 devices with their PCI slots:" + echo "ERROR: DEVICE ANALYSIS: Failed to detect mlx5 device despite count showing 1" + exit 1 + fi + + echo "SUCCESS: SINGLE-DEVICE MODE: Selected device '$MLX5_DEVICE' (only available device)" + +elif [ "$MLX5_DEVICE_COUNT" -gt 1 ]; then + echo "INFO: MULTI-DEVICE MODE: Found $MLX5_DEVICE_COUNT active mlx5 devices - using slot-based mapping" + echo "INFO: MULTI-DEVICE MODE: Slot-based mapping required for precise device selection" + + if [ -z "$SLOT_NUMBER" ]; then + echo "ERROR: MULTI-DEVICE MODE: Multiple devices found but no slot number specified in PXE mapping" + echo "ERROR: MULTI-DEVICE MODE: PXE mapping contains: '$IB_NIC_NAME'" + echo "ERROR: MULTI-DEVICE MODE: Expected format: 'InfiniBand.PCIe.Slot.X-Y' where X is the slot number" + echo "ERROR: MULTI-DEVICE MODE: Please update PXE mapping file with correct slot numbers" + echo "ERROR: MULTI-DEVICE MODE: Available mlx5 devices with their PCI slots:" for mlx_device in /sys/class/infiniband/mlx5_*; do [ -e "$mlx_device" ] || continue device_name=$(basename "$mlx_device") @@ -160,29 +180,34 @@ if [ "$MLX5_DEVICE_COUNT" -gt 1 ] && [ -n "$SLOT_NUMBER" ]; then echo " - $device_name: PCI information not available" fi done - echo "ERROR: SLOT MISMATCH: Please update PXE mapping file with correct PCI slot numbers" exit 1 fi - echo "SUCCESS: SLOT MATCHING: Found mlx5 device $MLX5_DEVICE for slot '$SLOT_NUMBER'" - echo "SUCCESS: SLOT MATCHING: Slot-based device selection successful" -else - if [ "$MLX5_DEVICE_COUNT" -eq 1 ]; then - echo "INFO: SINGLE-DEVICE MODE: Only one mlx5 device detected" - echo "INFO: SINGLE-DEVICE MODE: Using fallback device selection (ignoring slot number)" - echo "INFO: SINGLE-DEVICE MODE: This works for single-device setups even with incorrect slot numbers" - else - echo "INFO: FALLBACK MODE: Multiple devices detected but no slot number specified" - echo "INFO: FALLBACK MODE: Using first available device" - fi + echo "INFO: MULTI-DEVICE MODE: Looking for device matching slot '$SLOT_NUMBER'" + MLX5_DEVICE=$(find_device_by_slot "$SLOT_NUMBER") - MLX5_DEVICE=$(ls /sys/class/infiniband/ | grep mlx5 | head -1) if [ -z "$MLX5_DEVICE" ]; then - echo "ERROR: No mlx5 device found in /sys/class/infiniband/" + echo "ERROR: MULTI-DEVICE MODE: Found $MLX5_DEVICE_COUNT active mlx5 devices but none match slot '$SLOT_NUMBER'" + echo "ERROR: MULTI-DEVICE MODE: Slot '$SLOT_NUMBER' from PXE mapping doesn't match any hardware PCI slot" + echo "ERROR: MULTI-DEVICE MODE: Available mlx5 devices with their PCI slots:" + for mlx_device in /sys/class/infiniband/mlx5_*; do + [ -e "$mlx_device" ] || continue + device_name=$(basename "$mlx_device") + pci_address=$(basename "$(readlink -f "/sys/class/infiniband/$device_name/device" 2>/dev/null)") + if [ -n "$pci_address" ]; then + pci_slot=$(echo "$pci_address" | cut -d: -f2 | sed 's/^0*//') + echo " - $device_name: PCI $pci_address (slot $pci_slot)" + else + echo " - $device_name: PCI information not available" + fi + done + echo "ERROR: MULTI-DEVICE MODE: Please update PXE mapping file with correct PCI slot numbers" + echo "ERROR: MULTI-DEVICE MODE: Use format: 'InfiniBand.PCIe.Slot.-'" exit 1 fi - echo "INFO: FALLBACK DEVICE: Selected $MLX5_DEVICE (first available device)" + echo "SUCCESS: MULTI-DEVICE MODE: Slot '$SLOT_NUMBER' matches device '$MLX5_DEVICE'" + echo "SUCCESS: MULTI-DEVICE MODE: Slot-based device selection completed successfully" fi echo "INFO: FINAL DEVICE SELECTION: Using mlx5 device '$MLX5_DEVICE'" @@ -342,5 +367,5 @@ fi echo "=== IB NETWORK CONFIGURATION COMPLETED SUCCESSFULLY ===" echo "SUMMARY: IB interface $IB_INTERFACE configured with IP $IB_IP/$NETMASK_BITS" echo "SUMMARY: Device used: $MLX5_DEVICE, Port: $PORT_NUMBER" -echo "SUMMARY: Configuration method: $([ -n "$SLOT_NUMBER" ] && echo "Slot-based mapping" || echo "Fallback device selection")" +echo "SUMMARY: Configuration method: $([ "$MLX5_DEVICE_COUNT" -eq 1 ] && echo "Single-device mode (no slot mapping needed)" || echo "Multi-device mode (slot-based mapping)")"