etcd on the local disk support

common/library/module_utils/input_validation/schema/omnia_config.json

@@ -111,6 +111,9 @@
           "deployment": {
             "type": "boolean"
           },
+          "etcd_on_local_disk": {
+            "type": "boolean"
+          },
           "k8s_cni": {
             "enum": ["calico"],
             "description": "K8s CNI plugin to use for this cluster."

input/omnia_config.yml

@@ -192,6 +192,7 @@ slurm_cluster:
 service_k8s_cluster:
   - cluster_name: service_cluster
     deployment: true
+    etcd_on_local_disk: false
     k8s_cni: "calico"
     pod_external_ip_range: "172.16.107.170-172.16.107.200"
     k8s_service_addresses: "10.233.0.0/18"

provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2

@@ -20,6 +20,18 @@
       disable_root: false
 
       write_files:
+        - path: /usr/local/bin/etcd-disk-setup.sh
+          owner: root:root
+          permissions: '{{ file_mode_755 }}'
+          content: |
+            {{ lookup('template', 'templates/k8s-scripts/etcd-disk-setup.sh.j2') | indent(12) }}
+
+        - path: /usr/local/bin/etcd-fstab-update.sh
+          owner: root:root
+          permissions: '{{ file_mode_755 }}'
+          content: |
+            {{ lookup('template', 'templates/k8s-scripts/etcd-fstab-update.sh.j2') | indent(12) }}
+
         - path: /usr/local/bin/doca-install.sh
           owner: root:root
           permissions: '{{ file_mode_755 }}'
@@ -355,7 +367,11 @@
 {% endfor %}
         # K8s NFS mount entries
         - mkdir -p /tmp/crio-storage {{ k8s_client_mount_path }} /var/lib/etcd  /var/lib/kubelet /etc/kubernetes /var/log/pods /var/lib/packages
+{% if etcd_on_local_disk | default(false) %}
+        - /usr/local/bin/etcd-fstab-update.sh
+{% else %}
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/etcd      /var/lib/etcd        nfs noatime,nolock 0 0" >> /etc/fstab
+{% endif %}
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubelet   /var/lib/kubelet     nfs noatime,nolock 0 0" >> /etc/fstab
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubernetes   /etc/kubernetes      nfs noatime,nolock 0 0" >> /etc/fstab
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs   /var/log/pods      nfs noatime,nolock 0 0" >> /etc/fstab
@@ -470,6 +486,9 @@
           if [ ! -f "$MARKER" ]; then
             # FIRST BOOT - CLUSTER INIT
             # -- All the commands below this line should be run ONCE ONLY:
+{% if etcd_on_local_disk | default(false) %}
+            /usr/local/bin/etcd-disk-setup.sh
+{% endif %}
             echo "Initial boot - initializing and setting up service_kube_control_plane_first_x86_64"
             mv /tmp/ipaddress_pool.yaml {{ k8s_client_mount_path }}/metallb/ipaddress_pool.yaml
             mv /tmp/l2advertisement.yaml {{ k8s_client_mount_path }}/metallb/l2advertisement.yaml
@@ -1104,6 +1123,9 @@ PYEOF
             echo "Cluster already initialized. Performing node reboot procedures."
             # CRI and kubelet already enabled above
             # You can log health status etc if you wish:
+{% if etcd_on_local_disk | default(false) %}
+            /usr/local/bin/etcd-fstab-update.sh
+{% endif %}
             mkdir -p $HOME/.kube /root/.kube
             cp -f /etc/kubernetes/admin.conf $HOME/.kube/config
             chown $(id -u):$(id -g) $HOME/.kube/config

provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_x86_64.yaml.j2

@@ -20,6 +20,18 @@
       disable_root: false
 
       write_files:
+        - path: /usr/local/bin/etcd-disk-setup.sh
+          owner: root:root
+          permissions: '{{ file_mode_755 }}'
+          content: |
+            {{ lookup('template', 'templates/k8s-scripts/etcd-disk-setup.sh.j2') | indent(12) }}
+
+        - path: /usr/local/bin/etcd-fstab-update.sh
+          owner: root:root
+          permissions: '{{ file_mode_755 }}'
+          content: |
+            {{ lookup('template', 'templates/k8s-scripts/etcd-fstab-update.sh.j2') | indent(12) }}
+
         - path: /usr/local/bin/doca-install.sh
           owner: root:root
           permissions: '{{ file_mode_755 }}'
@@ -277,7 +289,11 @@
 {% endfor %}
         # K8s NFS mount entries
         - mkdir -p /tmp/crio-storage {{ k8s_client_mount_path }} /var/lib/etcd  /var/lib/kubelet /etc/kubernetes /var/log/pods /var/lib/packages
+{% if etcd_on_local_disk | default(false) %}
+        - /usr/local/bin/etcd-fstab-update.sh
+{% else %}
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/etcd      /var/lib/etcd        nfs noatime,nolock 0 0" >> /etc/fstab
+{% endif %}
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubelet   /var/lib/kubelet     nfs noatime,nolock 0 0" >> /etc/fstab
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubernetes   /etc/kubernetes      nfs noatime,nolock 0 0" >> /etc/fstab
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs   /var/log/pods      nfs noatime,nolock 0 0" >> /etc/fstab
@@ -393,6 +409,9 @@
           if [ ! -f "$MARKER" ]; then
             # Join Kubernetes cluster
             echo "Initial boot - initializing and setting up service_kube_control_plane_x86_64"
+{% if etcd_on_local_disk | default(false) %}
+            /usr/local/bin/etcd-disk-setup.sh
+{% endif %}
             rm -rf /var/lib/etcd/*  /var/lib/kubelet/* /etc/kubernetes/*
             rm -rf /var/lib/etcd/.*  /var/lib/kubelet/.* /etc/kubernetes/.*
             K8S_CLIENT_MOUNT_PATH="{{ k8s_client_mount_path }}"
@@ -524,6 +543,9 @@
             echo "Cluster already initialized. Performing node reboot procedures."
             # CRI and kubelet already enabled above
             # You can log health status etc if you wish:
+{% if etcd_on_local_disk | default(false) %}
+            /usr/local/bin/etcd-fstab-update.sh
+{% endif %}
             mkdir -p $HOME/.kube /root/.kube
             cp -f /etc/kubernetes/admin.conf $HOME/.kube/config
             chown $(id -u):$(id -g) $HOME/.kube/config

provision/roles/configure_ochami/templates/k8s-scripts/etcd-disk-setup.sh.j2

@@ -0,0 +1,172 @@
+#!/bin/bash
+# etcd-fstab-update.sh
+# Wipes BOSS/fallback disk, creates ext4, mounts /var/lib/etcd
+# Every command logged with timestamp + exit code using echo only
+set -uo pipefail
+
+MOUNT="/var/lib/etcd"
+LOG="/var/log/etcd-disk-setup.log"
+exec > >(tee -a "$LOG") 2>&1
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+ts() { date +%F_%T; }
+
+run() {
+    echo "$(ts) [CMD] $*"
+    "$@"
+    local rc=$?
+    echo "$(ts) [RC=$rc] $*"
+    return $rc
+}
+
+runq() {
+    echo "$(ts) [CMD] $* (ok to fail)"
+    "$@" 2>&1 || true
+}
+
+ping_uuid() {
+    local uuid
+    uuid=$(blkid -s UUID -o value "$1" 2>/dev/null || true)
+    echo "$(ts) [UUID] $1 = ${uuid:-NOT_FOUND}"
+}
+
+dump_fstab_entry() {
+    echo "$(ts) [FSTAB] grep for $1:"
+    grep "$1" /etc/fstab || echo "$(ts) [FSTAB] no entry"
+}
+
+# ── START ─────────────────────────────────────────────────────────────────────
+
+echo ""
+echo "$(ts) [INFO] ===== START etcd-disk-setup ====="
+
+# ── Detect root disk ─────────────────────────────────────────────────────────
+
+echo "$(ts) [STEP] Detecting root disk"
+ROOT_DISK=$(lsblk -no PKNAME "$(findmnt -no SOURCE /)" 2>/dev/null | head -1 || true)
+echo "$(ts) [INFO] Root disk: ${ROOT_DISK:-none}"
+
+# ── Scan for BOSS or fallback disk ───────────────────────────────────────────
+
+echo "$(ts) [STEP] Scanning disks"
+DISK=""
+for d in $(lsblk -dn -o NAME | grep -Ev "^(loop|sr|fd|${ROOT_DISK:-NONE})"); do
+    model=$(lsblk -dn -o MODEL "/dev/$d" 2>/dev/null | xargs || true)
+    pci=""
+    case "$d" in nvme*) pci=$(cat "/sys/class/nvme/${d%n*}/address" 2>/dev/null || true) ;; esac
+    sub=""
+    [ -n "$pci" ] && sub=$(lspci -D -s "$pci" -v 2>/dev/null | grep -i Subsystem || true)
+    echo "$(ts) [INFO] Disk /dev/$d model='$model' pci='$pci'"
+    if echo "$model $sub" | grep -qi boss; then
+        DISK="/dev/$d"
+        echo "$(ts) [OK] BOSS disk found: $DISK"
+        break
+    fi
+    [ -z "$DISK" ] && DISK="/dev/$d" && echo "$(ts) [INFO] Fallback candidate: $DISK"
+done
+
+[ -z "$DISK" ] && { echo "$(ts) [ERROR] No disk found"; exit 1; }
+echo "$(ts) [OK] Selected disk: $DISK"
+
+# ── Safety: never touch root disk ────────────────────────────────────────────
+
+[ "$DISK" = "/dev/$ROOT_DISK" ] && { echo "$(ts) [ERROR] Refusing to wipe root disk!"; exit 1; }
+
+# ── Partition naming ─────────────────────────────────────────────────────────
+
+case "$DISK" in
+    *nvme*|*mmcblk*) PART="${DISK}p1" ;;
+    *)               PART="${DISK}1"  ;;
+esac
+echo "$(ts) [INFO] Target partition: $PART"
+
+# ── Unmount existing ─────────────────────────────────────────────────────────
+
+echo "$(ts) [STEP] Unmounting any existing mounts"
+runq umount "$MOUNT"
+runq umount "$PART"
+
+# ── Wipe disk ────────────────────────────────────────────────────────────────
+
+echo "$(ts) [STEP] Wiping disk signatures"
+run wipefs -af "$DISK"
+
+# ── Partition disk ───────────────────────────────────────────────────────────
+
+echo "$(ts) [STEP] Creating GPT partition"
+run parted -s "$DISK" mklabel gpt mkpart primary ext4 1MiB 100%
+run partprobe "$DISK"
+run udevadm settle --timeout=30
+
+# ── Wait for partition ───────────────────────────────────────────────────────
+
+echo "$(ts) [STEP] Waiting for $PART"
+for i in 1 2 3 4 5; do
+    if [ -b "$PART" ]; then
+        echo "$(ts) [OK] $PART ready (attempt $i)"
+        break
+    fi
+    echo "$(ts) [INFO] Attempt $i — sleeping 2s"
+    sleep 2
+done
+[ -b "$PART" ] || { echo "$(ts) [ERROR] $PART not found"; exit 1; }
+
+# ── Format ───────────────────────────────────────────────────────────────────
+
+echo "$(ts) [STEP] Formatting $PART as ext4"
+run mkfs.ext4 -F -L etcd_data "$PART"
+
+# ── Ping UUID ────────────────────────────────────────────────────────────────
+
+echo "$(ts) [STEP] Pinging UUID"
+ping_uuid "$PART"
+UUID=$(blkid -s UUID -o value "$PART" 2>/dev/null || true)
+[ -z "$UUID" ] && { echo "$(ts) [ERROR] UUID is empty"; exit 1; }
+echo "$(ts) [OK] UUID=$UUID"
+
+# ── Update fstab ─────────────────────────────────────────────────────────────
+
+echo "$(ts) [STEP] Updating /etc/fstab"
+run mkdir -p "$MOUNT"
+run cp /etc/fstab "/etc/fstab.bak.$(date +%s)"
+run sed -i "\|[[:space:]]${MOUNT}[[:space:]]|d" /etc/fstab
+FSTAB_LINE="UUID=$UUID $MOUNT ext4 defaults,noatime,nofail 0 2"
+echo "$FSTAB_LINE" >> /etc/fstab
+echo "$(ts) [OK] fstab entry added: $FSTAB_LINE"
+
+# ── Dump fstab entry ─────────────────────────────────────────────────────────
+
+dump_fstab_entry "$MOUNT"
+
+# ── Mount ────────────────────────────────────────────────────────────────────
+
+echo "$(ts) [STEP] Mounting $MOUNT"
+runq systemctl daemon-reload
+run mount "$MOUNT"
+mountpoint -q "$MOUNT" || { echo "$(ts) [ERROR] Mount verification failed"; exit 1; }
+echo "$(ts) [OK] Mounted: $PART -> $MOUNT"
+
+# ── etcd user + perms ────────────────────────────────────────────────────────
+
+echo "$(ts) [STEP] Configuring etcd user/perms"
+getent group etcd >/dev/null || run groupadd --system etcd
+id etcd >/dev/null 2>&1     || run useradd --system -M -s /sbin/nologin -g etcd etcd
+run chown etcd:etcd "$MOUNT"
+run chmod 700 "$MOUNT"
+
+# ── Final Summary ────────────────────────────────────────────────────────────
+
+echo ""
+echo "$(ts) [INFO] ── Final Summary ──"
+echo "$(ts) [INFO] Disk:      $DISK"
+echo "$(ts) [INFO] Partition: $PART"
+echo "$(ts) [INFO] UUID:      $UUID"
+echo "$(ts) [INFO] Mount:     $MOUNT"
+echo ""
+df -h "$MOUNT"
+echo ""
+ping_uuid "$PART"
+dump_fstab_entry "$MOUNT"
+echo ""
+echo "$(ts) [OK] ===== DONE ====="