refactor: retire scripts/ in favor of lib/ and tools/

.env.example

@@ -334,7 +334,7 @@ EXCALIDRAW_DB_PASSWORD=changeme
 # Setup:
 #   1. Set STATUS_PIPELINE_MINIO_ACCESS_KEY/SECRET_KEY and STATUS_PIPELINE_STATUS_SLUG
 #   2. Upload tailscale-acl-policy.json to Tailscale admin (adds tag:homelab-nas + Funnel)
-#   3. Run: scripts/status-pipeline/setup-nas-tailscale.sh (install Tailscale on NAS)
+#   3. Run: tools/status-pipeline/setup-nas-tailscale.sh (install Tailscale on NAS)
 #   4. Run: task status-pipeline:setup (deploys sync script + cron to NAS)
 #   5. Run: task status-pipeline:funnel:on (enables public Funnel endpoint)
 #   6. Set STATUS_PIPELINE_URL below with the Funnel URL

.gitignore

@@ -17,6 +17,12 @@ eggs/
 .eggs/
 lib/
 lib64/
+# Re-include our source dirs that collide with the Python venv-ignore above
+# (lib/ here is the boilerplate venv ignore; ours is committed shared bash)
+!/lib/
+!/lib/**
+!/tests/unit/lib/
+!/tests/unit/lib/**
 parts/
 sdist/
 var/

.pre-commit-config.yaml

@@ -43,7 +43,7 @@ repos:
       hooks:
           - id: ansible-lint
             name: Ansible Lint
-            entry: scripts/dev/run-ansible-lint.sh
+            entry: tools/dev/run-ansible-lint.sh
             language: system
             files: \.
             pass_filenames: true

Taskfile.yml

@@ -9,8 +9,8 @@ dotenv: [".env"]
 includes:
     ansible: ansible/Taskfile.yml
     status-pipeline:
-        taskfile: scripts/status-pipeline/Taskfile.yml
-        dir: scripts/status-pipeline
+        taskfile: tools/status-pipeline/Taskfile.yml
+        dir: tools/status-pipeline
 
 env:
     REGISTRY_NAMESPACE: $GITHUB_USERNAME
@@ -69,7 +69,7 @@ tasks:
             - echo "🧪 Running shell unit tests with coverage..."
             - mkdir -p coverage
             - rm -rf /tmp/bats-run-* 2>/dev/null || true
-            - kcov --bash-method=DEBUG --include-pattern=scripts,stacks --exclude-pattern=/usr,/tmp,/var coverage/unit bats --jobs 1 stacks/apps/*/tests/*.bats tests/unit/scripts/*.bats
+            - kcov --bash-method=DEBUG --include-pattern=lib,stacks --exclude-pattern=/usr,/tmp,/var coverage/unit bats --jobs 1 stacks/apps/*/tests/*.bats tests/unit/lib/*.bats
             - echo "📊 Merging coverage reports..."
             - kcov --merge coverage/merged coverage/unit
             - echo "✅ Shell tests completed [Coverage report](coverage/merged/index.html)"
@@ -96,7 +96,7 @@ tasks:
         desc: Run core tests with faster execution
         cmds:
             - echo "🧪 Running unit tests..."
-            - bats --tap stacks/apps/*/tests/*.bats tests/unit/scripts/*.bats
+            - bats --tap stacks/apps/*/tests/*.bats tests/unit/lib/*.bats
 
     test-local:
         desc: Run local-only tests (requires Docker)
@@ -132,7 +132,7 @@ tasks:
     lint:shell:
         desc: Run shellcheck on shell scripts
         cmds:
-            - find scripts -name "*.sh" -type f -exec uv run shellcheck -x --source-path=scripts {} +
+            - find tools lib -name "*.sh" -type f -exec uv run shellcheck -x --source-path=lib --source-path=tools {} +
 
     lint:yaml:
         desc: Run yamllint on YAML files
@@ -212,7 +212,7 @@ tasks:
             - sh: command -v dockumentor
               msg: "dockumentor not found. Install with: uv tool install dockumentor"
         cmds:
-            - ./scripts/dev/generate-stack-docs.sh
+            - ./tools/dev/generate-stack-docs.sh
 
     secrets:login:
         desc: Login and unlock the configured secret vault

ansible/Taskfile.yml

@@ -148,7 +148,7 @@ tasks:
         desc: "Check cluster health — node states, degraded services, DNS crash history, gossip instability"
         dir: "{{.ROOT_DIR}}"
         cmds:
-            - bash scripts/cluster-health.sh
+            - bash tools/cluster-health.sh
 
     cluster:update-labels:
         desc: Update Docker Swarm node labels from inventory

docs/services/cert-sync-nas.md

@@ -63,7 +63,7 @@ TZ=${TZ:-America/New_York}
 
 - `acme_certs:/acme.sh`
 - `./sync-nas-cert.sh:/scripts/sync-nas-cert.sh:ro`
-- `../../../scripts/common:/scripts/common:ro`
+- `./common:/scripts/common:ro`
 
 
 ---

llms.txt

@@ -31,7 +31,8 @@ stacks/
   monitoring/             # Prometheus + Grafana + Loki
   dns/                    # Technitium DNS server
 docs/                     # MkDocs documentation site
-scripts/                  # Utility scripts (health check, doc generation)
+tools/                    # Invocable tooling (ci, status-pipeline, dev helpers, cluster-health)
+lib/                      # Shared bash sourced by tooling (ssh.sh)
 ```
 
 ## Common Operations

stacks/apps/cert-sync-nas/README.md

@@ -12,7 +12,7 @@ On every container start, the startup script:
 4. Installs a weekly crontab (Sundays at 3 AM) to renew and re-sync
 5. Starts `crond` to keep the container running
 
-`sync-nas-cert.sh` calls `omv_cert_install` from `scripts/common/nas/omv.sh`, which:
+`sync-nas-cert.sh` calls `omv_cert_install` from `common/nas/omv.sh`, which:
 - SCPs the cert and key to `/tmp/` on the NAS
 - SSHes in and runs `omv-rpc CertificateMgmt set` to import the cert
 - Applies dirty config modules and restarts nginx

stacks/apps/cert-sync-nas/common/ssh.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ -z "${SSH_KEY_FILE:-}" ]; then
+    SSH_KEY_FILE="$HOME/.ssh/homelab_rsa"
+    echo "Warning: SSH_KEY_FILE not set, defaulting to $SSH_KEY_FILE" >&2
+    echo "  Add SSH_KEY_FILE=~/.ssh/homelab_rsa to your .env file to suppress this warning." >&2
+fi
+SSH_TIMEOUT="${SSH_TIMEOUT:-5}"
+SSH_EXECUTE_TIMEOUT="${SSH_EXECUTE_TIMEOUT:-300}"
+
+export SSH_KEY_FILE
+export SSH_TIMEOUT
+export SSH_EXECUTE_TIMEOUT
+
+# check if ssh is installed and working (skip in test mode)
+if [ -z "${TEST:-}" ]; then
+  if ! command -v ssh &> /dev/null; then
+    echo "Error: ssh could not be found"
+    exit 1
+  fi
+fi
+
+# SSH wrapper that uses the SSH key file. It's
+# a bit more opinionated than the default ssh command.
+# Args:
+#   $1: SSH user@hostname
+#   $2: Command to run
+# Returns:
+#   None
+ssh_key_auth() {
+    local key_file="$SSH_KEY_FILE"
+    local timeout_duration="${SSH_TIMEOUT:-5}"
+
+    timeout "$timeout_duration" ssh -i "$key_file" \
+        -o StrictHostKeyChecking=accept-new \
+        -o PasswordAuthentication=no \
+        -o PubkeyAuthentication=yes \
+        -o IdentitiesOnly=yes \
+        -o ConnectTimeout=5 \
+        "$1" "$2"
+}
+
+# SSH wrapper that uses ssh-copy-id to copy the SSH key to the remote machine
+# Args:
+#   $1: SSH user@hostname
+# Returns:
+#   None
+ssh_copy_id() {
+    ssh-copy-id -i "$SSH_KEY_FILE" -o PasswordAuthentication=yes "$1"
+}
+
+# Execute command on remote host using SSH key authentication
+# Args:
+#   [--login]: optional flag — run command in a login shell (loads full PATH)
+#   $1: user@hostname
+#   $2: command to execute
+# Returns:
+#   SSH command exit code
+ssh_execute() {
+    local login=false
+    if [[ "${1:-}" == "--login" ]]; then
+        login=true
+        shift
+    fi
+    local user_host="$1"
+    local command="$2"
+    if [[ "$login" == true ]]; then
+        command="bash -l -c '${command//\'/\'\\\'\'}'"
+    fi
+    ssh_key_auth "$user_host" "$command"
+}
+
+# Test SSH connectivity to a host
+# Args:
+#   $1: user@hostname
+# Returns:
+#   0 if connection successful, 1 otherwise
+ssh_test_connection() {
+    local user_host="$1"
+    ssh_key_auth "$user_host" "exit" 2>/dev/null
+}
+
+# Copy SSH key to remote host
+# Args:
+#   $1: user@hostname
+# Returns:
+#   ssh-copy-id exit code
+ssh_copy_key() {
+    local user_host="$1"
+    ssh_copy_id "$user_host"
+}
+
+# Create directory on remote host
+# Args:
+#   $1: user@hostname
+#   $2: directory path
+#   $3: permissions (optional, e.g., 700)
+# Returns:
+#   SSH command exit code
+ssh_create_directory() {
+    local user_host="$1"
+    local dir_path="$2"
+    local permissions="${3:-755}"
+    ssh_execute "$user_host" "mkdir -p '$dir_path' && chmod '$permissions' '$dir_path'"
+}
+
+# Check if command exists on remote host
+# Args:
+#   $1: user@hostname
+#   $2: command name
+# Returns:
+#   0 if command exists, 1 otherwise
+ssh_command_exists() {
+    local user_host="$1"
+    local command="$2"
+    ssh_execute "$user_host" "command -v '$command'" >/dev/null 2>&1
+}
+
+# Copy a file to a remote host via SCP using the SSH key file
+# Args:
+#   $1: source file path
+#   $2: destination (user@host:/path)
+# Returns:
+#   0 on success, non-zero on failure
+scp_copy_file() {
+    local src="$1"
+    local dest="$2"
+    local key_file="$SSH_KEY_FILE"
+    local timeout_duration="${SSH_TIMEOUT:-5}"
+
+    timeout "$timeout_duration" scp -i "$key_file" \
+        -o StrictHostKeyChecking=accept-new \
+        -o PasswordAuthentication=no \
+        -o PubkeyAuthentication=yes \
+        -o IdentitiesOnly=yes \
+        -o ConnectTimeout=5 \
+        "$src" "$dest"
+}
+
+# Execute a local script on a remote host via SSH
+# Args:
+#   $1: user@hostname
+#   $2: path to local script file
+# Returns:
+#   SSH command exit code
+ssh_execute_script() {
+    local user_host="$1"
+    local script_file="$2"
+    local key_file="$SSH_KEY_FILE"
+    local timeout_duration="${SSH_EXECUTE_TIMEOUT:-300}"
+
+    timeout "$timeout_duration" ssh -i "$key_file" \
+        -o StrictHostKeyChecking=accept-new \
+        -o PasswordAuthentication=no \
+        -o PubkeyAuthentication=yes \
+        -o IdentitiesOnly=yes \
+        -o ConnectTimeout=5 \
+        "$user_host" bash < "$script_file"
+}
+
+# Export all SSH functions so they're available to subshells
+export -f ssh_key_auth
+export -f ssh_copy_id
+export -f ssh_execute
+export -f ssh_test_connection
+export -f ssh_copy_key
+export -f ssh_create_directory
+export -f ssh_command_exists
+export -f scp_copy_file
+export -f ssh_execute_script

stacks/apps/cert-sync-nas/docker-compose.yml

@@ -75,13 +75,13 @@ configs:
   cert_sync_script_v1:
     file: ./sync-nas-cert.sh
   cert_sync_omv_v1:
-    file: ../../../scripts/common/nas/omv.sh
+    file: ./common/nas/omv.sh
   cert_sync_install_cert_v3:
-    file: ../../../scripts/common/nas/install-cert-remote.sh
+    file: ./common/nas/install-cert-remote.sh
   cert_sync_cert_sh_v1:
-    file: ../../../scripts/common/cert.sh
+    file: ./common/cert.sh
   cert_sync_ssh_sh_v1:
-    file: ../../../scripts/common/ssh.sh
+    file: ./common/ssh.sh
 
 secrets:
   cert_sync_ssh_key:

tests/unit/scripts/cert_test.bats → stacks/apps/cert-sync-nas/tests/cert_test.bats

@@ -1,13 +1,13 @@
 #!/usr/bin/env bats
 
-# Tests for scripts/common/cert.sh
+# Tests for the cert-sync-nas bundle's cert.sh
 
 load test_helper
 
 setup() {
     export TEST=true
     # shellcheck disable=SC1091
-    source "${BATS_TEST_DIRNAME}/../../../scripts/common/cert.sh"
+    source "${BATS_TEST_DIRNAME}/../common/cert.sh"
 
     TEST_DIR="$(temp_make)"
     export TEST_DIR

tests/unit/scripts/omv_test.bats → stacks/apps/cert-sync-nas/tests/omv_test.bats

@@ -1,18 +1,18 @@
 #!/usr/bin/env bats
 
-# Tests for scripts/common/nas/omv.sh
+# Tests for the cert-sync-nas bundle's nas/omv.sh
 
 load test_helper
 
 setup() {
     export TEST=true
-    local scripts_dir="${BATS_TEST_DIRNAME}/../../../scripts"
+    local bundle_dir="${BATS_TEST_DIRNAME}/../common"
     # shellcheck disable=SC1091
-    source "${scripts_dir}/common/ssh.sh"
+    source "${bundle_dir}/ssh.sh"
     # shellcheck disable=SC1091
-    source "${scripts_dir}/common/cert.sh"
+    source "${bundle_dir}/cert.sh"
     # shellcheck disable=SC1091
-    source "${scripts_dir}/common/nas/omv.sh"
+    source "${bundle_dir}/nas/omv.sh"
 
     TEST_DIR="$(temp_make)"
     export TEST_DIR

stacks/apps/kiwix/setup-nas-downloads.sh

@@ -79,11 +79,11 @@ load_environment() {
     fi
 
     # Source common ssh library
-    if [ -f "$PROJECT_ROOT/scripts/common/ssh.sh" ]; then
-        source "$PROJECT_ROOT/scripts/common/ssh.sh"
+    if [ -f "$PROJECT_ROOT/lib/ssh.sh" ]; then
+        source "$PROJECT_ROOT/lib/ssh.sh"
         log_info "Loaded ssh library"
     else
-        log_error "Could not find scripts/common/ssh.sh"
+        log_error "Could not find lib/ssh.sh"
         log_error "Please ensure you're running this from the project directory"
         exit 1
     fi

stacks/apps/kiwix/tests/setup_nas_downloads_test.bats

@@ -27,8 +27,8 @@ BASE_DOMAIN=test.com
 EOF
 
     # Create mock ssh.sh
-    mkdir -p "$PROJECT_ROOT/scripts/common"
-    cat > "$PROJECT_ROOT/scripts/common/ssh.sh" <<'EOF'
+    mkdir -p "$PROJECT_ROOT/lib"
+    cat > "$PROJECT_ROOT/lib/ssh.sh" <<'EOF'
 #!/bin/bash
 ssh_test_connection() { return 0; }
 ssh_copy_key() { return 0; }
@@ -37,7 +37,7 @@ ssh_create_directory() { return 0; }
 ssh_command_exists() { return 0; }
 export -f ssh_test_connection ssh_copy_key ssh_execute ssh_create_directory ssh_command_exists
 EOF
-    chmod +x "$PROJECT_ROOT/scripts/common/ssh.sh"
+    chmod +x "$PROJECT_ROOT/lib/ssh.sh"
 }
 
 teardown() {
@@ -140,7 +140,7 @@ teardown() {
     [[ "$output" =~ "source" ]] && [[ "$output" =~ ".env" ]]
 }
 
-@test "load_environment should source scripts/common/ssh.sh" {
+@test "load_environment should source lib/ssh.sh" {
     local script="${BATS_TEST_DIRNAME}/../setup-nas-downloads.sh"
     run grep -A20 "^load_environment()" "$script"
     [[ "$output" =~ "source" ]] && [[ "$output" =~ "ssh.sh" ]]