etcd on the local disk support

common/library/module_utils/input_validation/schema/omnia_config.json

@@ -106,6 +106,9 @@
           "deployment": {
             "type": "boolean"
           },
+          "etcd_on_local_disk": {
+            "type": "boolean"
+          },
           "k8s_cni": {
             "enum": ["calico"],
             "description": "K8s CNI plugin to use for this cluster."

input/omnia_config.yml

@@ -181,6 +181,7 @@ slurm_cluster:
 service_k8s_cluster:
   - cluster_name: service_cluster
     deployment: true
+    etcd_on_local_disk: false
     k8s_cni: "calico"
     pod_external_ip_range: "172.16.107.170-172.16.107.200"
     k8s_service_addresses: "10.233.0.0/18"

provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2

@@ -85,6 +85,18 @@
             logdir /var/log/chrony
             cmdport 0
 
+        - path: /usr/local/bin/etcd-disk-setup.sh
+          owner: root:root
+          permissions: '{{ file_mode_755 }}'
+          content: |
+            {{ lookup('template', 'templates/k8s-configs/etcd-disk-setup.sh.j2') | indent(12) }}
+
+        - path: /usr/local/bin/etcd-fstab-update.sh
+          owner: root:root
+          permissions: '{{ file_mode_755 }}'
+          content: |
+            {{ lookup('template', 'templates/k8s-configs/etcd-fstab-update.sh.j2') | indent(12) }}
+
         - path: /etc/modules-load.d/k8s.conf
           content: |
             br_netfilter
@@ -355,7 +367,14 @@
 {% endfor %}
         # K8s NFS mount entries
         - mkdir -p /tmp/crio-storage {{ k8s_client_mount_path }} /var/lib/etcd  /var/lib/kubelet /etc/kubernetes /var/log/pods /var/lib/packages
+{% if etcd_on_local_disk | default(false) == true %}
+        # etcd_on_local_disk=true → call non-destructive fstab updater every boot
+        # This re-detects the BOSS disk, removes any duplicate/stale /var/lib/etcd
+        # entries, appends one canonical UUID-based entry, and mounts. NEVER wipes.
+        - /usr/local/bin/etcd-fstab-update.sh
+{% else %}
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/etcd      /var/lib/etcd        nfs noatime,nolock 0 0" >> /etc/fstab
+{% endif %}
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubelet   /var/lib/kubelet     nfs noatime,nolock 0 0" >> /etc/fstab
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubernetes   /etc/kubernetes      nfs noatime,nolock 0 0" >> /etc/fstab
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs   /var/log/pods      nfs noatime,nolock 0 0" >> /etc/fstab
@@ -474,6 +493,12 @@
             mv /tmp/ipaddress_pool.yaml {{ k8s_client_mount_path }}/metallb/ipaddress_pool.yaml
             mv /tmp/l2advertisement.yaml {{ k8s_client_mount_path }}/metallb/l2advertisement.yaml
             # Setup Kubernetes cluster
+{% if etcd_on_local_disk | default(false) == true %}
+            # etcd_on_local_disk=true → first-boot DESTRUCTIVE BOSS disk provisioning
+            # Wipes + partitions + formats + writes fstab + mounts /var/lib/etcd.
+            # This block runs ONCE ONLY on first boot.
+            /usr/local/bin/etcd-disk-setup.sh
+{% endif %}
             rm -rf /var/lib/etcd/*  /var/lib/kubelet/* /etc/kubernetes/*
             rm -rf /var/lib/etcd/.*  /var/lib/kubelet/.* /etc/kubernetes/.*
             #!/bin/bash

provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_x86_64.yaml.j2

@@ -81,6 +81,19 @@
             nf_conntrack
             vxlan
           permissions: '0644'
+
+        - path: /usr/local/bin/etcd-disk-setup.sh
+          owner: root:root
+          permissions: '{{ file_mode_755 }}'
+          content: |
+            {{ lookup('template', 'templates/k8s-configs/etcd-disk-setup.sh.j2') | indent(12) }}
+
+        - path: /usr/local/bin/etcd-fstab-update.sh
+          owner: root:root
+          permissions: '{{ file_mode_755 }}'
+          content: |
+            {{ lookup('template', 'templates/k8s-configs/etcd-fstab-update.sh.j2') | indent(12) }}
+
         - path: /etc/sysctl.d/k8s.conf
           content: |
             net.bridge.bridge-nf-call-iptables=1
@@ -277,7 +290,14 @@
 {% endfor %}
         # K8s NFS mount entries
         - mkdir -p /tmp/crio-storage {{ k8s_client_mount_path }} /var/lib/etcd  /var/lib/kubelet /etc/kubernetes /var/log/pods /var/lib/packages
+{% if etcd_on_local_disk | default(false) == true %}
+        # etcd_on_local_disk=true → call non-destructive fstab updater every boot
+        # This re-detects the BOSS disk, removes any duplicate/stale /var/lib/etcd
+        # entries, appends one canonical UUID-based entry, and mounts. NEVER wipes.
+        - /usr/local/bin/etcd-fstab-update.sh
+{% else %}
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/etcd      /var/lib/etcd        nfs noatime,nolock 0 0" >> /etc/fstab
+{% endif %}
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubelet   /var/lib/kubelet     nfs noatime,nolock 0 0" >> /etc/fstab
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/kubernetes   /etc/kubernetes      nfs noatime,nolock 0 0" >> /etc/fstab
         - echo "{{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs   /var/log/pods      nfs noatime,nolock 0 0" >> /etc/fstab
@@ -393,6 +413,12 @@
           if [ ! -f "$MARKER" ]; then
             # Join Kubernetes cluster
             echo "Initial boot - initializing and setting up service_kube_control_plane_x86_64"
+{% if etcd_on_local_disk | default(false) == true %}
+            # etcd_on_local_disk=true → first-boot DESTRUCTIVE BOSS disk provisioning
+            # Wipes + partitions + formats + writes fstab + mounts /var/lib/etcd.
+            # This block runs ONCE ONLY on first boot.
+            /usr/local/bin/etcd-disk-setup.sh
+{% endif %}
             rm -rf /var/lib/etcd/*  /var/lib/kubelet/* /etc/kubernetes/*
             rm -rf /var/lib/etcd/.*  /var/lib/kubelet/.* /etc/kubernetes/.*
             K8S_CLIENT_MOUNT_PATH="{{ k8s_client_mount_path }}"