diff --git a/src/airgap/airgap_input.txt b/src/airgap/airgap_input.txt deleted file mode 100644 index c310c63a..00000000 --- a/src/airgap/airgap_input.txt +++ /dev/null @@ -1,201 +0,0 @@ -[platform]: - le-nextgen-signed - ci-scm-signed - template-service-signed - platform-service-signed - pipeline-service-signed - ng-manager-signed - nextgenui-signed - minio - log-service-signed - cv-nextgen-signed - cdcdata-signed - accesscontrol-service-signed - delegate - delegate-proxy-signed - gateway-signed - helm-init-container - manager-signed - mongo - ng-auth-ui-signed - redis - upgrader - postgresql - dashboard-service-signed - policy-mgmt - ui-signed - srm-ui-signed - smp-service-discovery-server-signed - debezium-service-signed - audit-event-streaming-signed - queue-service-signed - service-discovery-collector - ng-dashboard-aggregator-signed - ingress-nginx/controller - defaultbackend-amd64 - redis_exporter - postgres-exporter - mongodb-exporter - statsd-exporter - vault-secret-loader - busybox - -[cdng]: - argocd - gitops-agent - haproxy - gitops-service-signed - gitops-agent-installer-helper - shellcheck - -[ci]: - ti-service-signed - kaniko-acr - ci-addon - artifactory - kaniko - kaniko-ecr - kaniko-gcr - gcs - drone-git - ci-lite-engine - cache - s3 - sto-plugin - ci-manager-signed - buildx-acr - buildx-ecr - buildx-gar - buildx-gcr - buildx - docker - ecr - gar - gcr - acr - traceable-job-runner - harness-cache-server - -[sto]: - ci-addon - ci-lite-engine - stocore-signed - ticket-service-signed - sto-plugin - anchore-job-runner - aqua-security-job-runner - aqua-trivy-job-runner - aws-ecr-job-runner - aws-security-hub-job-runner - bandit-job-runner - blackduckhub-job-runner - brakeman-job-runner - checkmarx-job-runner - fossa-job-runner - github-advanced-security-job-runner - grype-job-runner - nexusiq-job-runner - nikto-job-runner - nmap-job-runner - modelscan-job-runner - osv-job-runner - owasp-dependency-check-job-runner - prowler-job-runner - snyk-job-runner - sonarqube-agent-job-runner - traceable-job-runner - twistlock-job-runner - veracode-agent-job-runner - whitesource-agent-job-runner - wiz-job-runner - zap-job-runner - refid-cache - burp-job-runner - checkov-job-runner - docker-content-trust-job-runner - gitleaks-job-runner - semgrep-job-runner - semgrep-job-runner - shiftleft-job-runner - sysdig-job-runner - -[ff]: - ff-cron-signed - ff-server-analytics-db-migration-signed - ff-server-primary-db-migration-signed - ff-service-signed - ff-pushpin-signed - ff-pushpin-worker-signed - -[ccm]: - batch-processing-signed - ce-anomaly-detection-signed - ce-cloud-info-signed - ce-nextgen-signed - event-service-signed - ng-ce-ui - telescopes-signed - clickhouse - ccm-gcp-smp-signed - -[ce]: - chaos-log-watcher - smp-chaos-k8s-ifs-signed - smp-chaos-linux-infra-controller-signed - smp-chaos-linux-infra-server-signed - smp-chaos-manager-signed - smp-chaos-web-signed - source-probe - smp-chaos-bg-processor-signed - chaos-ddcr - chaos-machine-ifc-signed - chaos-machine-ifs-signed - chaos-ddcr-faults - enterprise-chaos-hub-signed - chaos-event-watcher -[ssca]: - ssca-plugin - slsa-plugin - ssca-cdxgen-plugin - ssca-compliance-plugin - ssca-manager-signed - ssca-ui-signed - ssca-artifact-signing-plugin - component-service-signed - component-analysis-service-signed - -[db-devops]: - db-devops-service-signed - -[code]: - code-api-signed - code-githa-signed - code-gitrpc-signed - code-search-signed - -[iacm]: - drone-git - ci-lite-engine - ci-addon - harness_terraform - iac-server-signed - iacm-manager-signed - harness_terraform_vm - -[idp]: - idp-service-signed - idp-admin-signed - idp-app-signed - drone-git - ci-lite-engine - ci-addon - cookiecutter - createcatalog - createorganisation - createproject - createrepo - createresource - directpush - registercatalog - slacknotify - updatecatalogproperty diff --git a/src/airgap/create-airgap-bundle.sh b/src/airgap/create-airgap-bundle.sh index 7e903747..abc3b88b 100755 --- a/src/airgap/create-airgap-bundle.sh +++ b/src/airgap/create-airgap-bundle.sh @@ -2,132 +2,387 @@ set -e -handle_error() { - echo "Error $1" >&2 +log_info() { echo "[INFO] $(date +%H:%M:%S) $*"; } +log_warn() { echo "[WARN] $(date +%H:%M:%S) $*" >&2; } +log_error() { echo "[ERROR] $(date +%H:%M:%S) $*" >&2; } + +SKOPEO_ARCH="${SKOPEO_ARCH:-amd64}" +SKOPEO_OS="${SKOPEO_OS:-linux}" + +usage() { + echo "Usage: $0 --internal-file [--section | --all]" + echo "" + echo " --internal-file Path to images_internal.txt" + echo " --section Process a single section (e.g., 'ci', 'sto-scanners@1')" + echo " --all Process all sections" + echo " --output-dir Output directory (default: current directory)" exit 1 } -export DOCKER_DEFAULT_PLATFORM=linux/amd64 +INTERNAL_FILE="" +SECTION="" +PROCESS_ALL=false +OUTPUT_DIR="." -# Provide lists of image names -for moduleImageFile in $MODULE_IMAGE_FILES; do - lists+=("$moduleImageFile") +while [ $# -gt 0 ]; do + case "$1" in + --internal-file) INTERNAL_FILE="$2"; shift 2 ;; + --section) SECTION="$2"; shift 2 ;; + --all) PROCESS_ALL=true; shift ;; + --output-dir) OUTPUT_DIR="$2"; shift 2 ;; + *) log_error "Unknown option: $1"; usage ;; + esac done -if [ ${#lists[@]} -le 2 ]; then # validation with 2 to make sure multiple elements are in list - echo "Error: No module image list provided. List: ${lists[@]}" >&2 +if [ -z "$INTERNAL_FILE" ]; then + log_error "Missing required --internal-file" + usage +fi + +if [ ! -f "$INTERNAL_FILE" ]; then + log_error "File not found: $INTERNAL_FILE" exit 1 fi -pull_image() { - i="$1" - if docker pull --quiet "${i}"; then - echo "Image pull success: ${i}" - echo "${i}" >> "${pulled_file}" - else - echo "Failed to pull image '${image}' from list '${list}'" >&2 - failed=1 - fi -} +if [ -z "$SECTION" ] && [ "$PROCESS_ALL" = false ]; then + log_error "Must specify --section or --all" + usage +fi + +MAX_PULL_RETRIES="${MAX_PULL_RETRIES:-3}" +PULL_RETRY_DELAY="${PULL_RETRY_DELAY:-10}" +MAX_PARALLEL_PULLS="${MAX_PARALLEL_PULLS:-6}" +MAX_PARALLEL_BUNDLES="${MAX_PARALLEL_BUNDLES:-4}" -if [ $# -eq 1 ]; then - image_list_file="$1" - if [[ ! -f $image_list_file ]]; then - echo "Error: File $image_list_file not found." +for cmd in skopeo pigz jq; do + if ! command -v "$cmd" &>/dev/null; then + log_error "Required command not found: ${cmd}" exit 1 fi +done - base_name=$(basename "$image_list_file" .txt) - images_file="${base_name}.tgz" +copy_image() { + local image="$1" + local output_tar="$2" + local attempt=0 + local delay="${PULL_RETRY_DELAY}" - failed=0 - # Create a temporary file to store the list of successfully pulled images - pulled_file="$(mktemp)" + # skopeo docker-archive rejects refs whose first path component looks like a + # hostname (contains a '.' or ':', e.g. "docker.io"). Passing the full ref + # "docker.io/harness/foo:1.0" causes skopeo to write RepoTags: null, which + # breaks `docker load` and bundle validation. + # + # Fix: strip only the registry hostname prefix (e.g. "docker.io/", "gcr.io/") + # while keeping the org/name:tag portion intact. This means the archive stores + # "harness/foo:1.0" which docker load restores under that exact ref — preserving + # backward compatibility for customers who retag using the org-qualified name. + # + local archive_tag + local first_segment="${image%%/*}" + if echo "$first_segment" | grep -qE '[.:]'; then + archive_tag="${image#*/}" + else + archive_tag="${image}" + fi - for list in ${lists[*]}; do - if [[ ! -f $list ]]; then - echo "Error: File $list not found." - exit 1 + while [ $attempt -lt "${MAX_PULL_RETRIES}" ]; do + attempt=$((attempt + 1)) + if skopeo copy --quiet \ + --override-arch "${SKOPEO_ARCH}" --override-os "${SKOPEO_OS}" \ + "docker://${image}" "docker-archive:${output_tar}:${archive_tag}" 2>&1; then + log_info "Copied (attempt ${attempt}): ${image}" + return 0 + fi + if [ $attempt -lt "${MAX_PULL_RETRIES}" ]; then + log_warn "Copy attempt ${attempt}/${MAX_PULL_RETRIES} failed for: ${image} — retrying in ${delay}s" + sleep "${delay}" + delay=$((delay * 2)) fi + done - pids=() + log_error "Failed to copy after ${MAX_PULL_RETRIES} attempts: ${image}" + return 1 +} - # Download images in parallel - while IFS= read -r i; do - [ -z "${i}" ] && continue - pull_image "${i}" & - pids+=($!) - done < "${image_list_file}" +# Run at most MAX_PARALLEL_PULLS copies concurrently to avoid registry rate limiting. +copy_images_bounded() { + local staging_dir="$1" + shift + local images=("$@") + local failed=0 + local pids=() + local active=0 + local idx=0 - for pid in ${pids[*]}; do - wait $pid || handle_error "Failed background task with PID: $pid" - done - # Wait for all background tasks to finish - wait + for img in "${images[@]}"; do + copy_image "${img}" "${staging_dir}/image_${idx}.tar" &2 - exit 1 + + for pid in "${pids[@]}"; do + wait "$pid" || failed=1 + done + + return $failed +} + +# Merge individual docker-archive tars into a single archive compressed with pigz. +# Each input tar has its own manifest.json; the merged output contains a combined +# manifest so that `docker load` restores all images at once. +merge_and_compress() { + local tgz_file="$1" + local staging_dir="$2" + + local merge_dir + merge_dir="$(mktemp -d)" + local combined_manifest="[]" + + for tar_file in "${staging_dir}"/*.tar; do + [ -f "$tar_file" ] || continue + + local this_manifest + this_manifest=$(tar -xf "$tar_file" -O manifest.json 2>/dev/null) || { + log_error "Failed to read manifest.json from ${tar_file}" + rm -rf "${merge_dir}" + return 1 + } + # Normalize RepoTags to match docker save format (strip docker.io/ and docker.io/library/) + this_manifest=$(echo "${this_manifest}" | jq -c '[.[] | .RepoTags = [.RepoTags[]? | sub("^docker.io/library/"; "") | sub("^docker.io/"; "")]]') + combined_manifest=$(echo "${combined_manifest}" | jq --argjson new "${this_manifest}" '. + $new') + + # Layers and configs are content-addressed; duplicates overwrite harmlessly. + tar -xf "$tar_file" -C "${merge_dir}" + done + + rm -f "${merge_dir}/manifest.json" + echo "${combined_manifest}" > "${merge_dir}/manifest.json" + + # Put manifest.json first so validation can extract it without decompressing entire archive + ( cd "${merge_dir}" && { echo manifest.json; find . -mindepth 1 ! -path ./manifest.json | sort; } ) \ + | tar -cf - -C "${merge_dir}" -T - \ + | pigz > "${tgz_file}" + rm -rf "${merge_dir}" +} + +create_combined_bundle() { + local section_name="$1" + local bucket_path="$2" + shift 2 + local images=("$@") + + local out_dir="${OUTPUT_DIR}/${bucket_path}" + mkdir -p "${out_dir}" + + local bundle_name + bundle_name=$(echo "${section_name}" | tr '/' '_') + local tgz_file="${out_dir}/${bundle_name}_images.tgz" + + local staging_dir + staging_dir="$(mktemp -d)" + + log_info "Copying ${#images[@]} images for combined bundle: ${section_name} (parallel limit: ${MAX_PARALLEL_PULLS}, retries: ${MAX_PULL_RETRIES})" + + if ! copy_images_bounded "${staging_dir}" "${images[@]}"; then + log_error "Some images failed to copy for section: ${section_name}" + rm -rf "${staging_dir}" + return 1 fi - # Get the list of successfully pulled images - pulled=$(cat "${pulled_file}") - # Save pulled images to a tarball - echo "Creating ${images_file} with $(echo ${pulled} | wc -w | tr -d '[:space:]') images" - docker save $(echo ${pulled}) | gzip --stdout > ${images_file} || handle_error "Failed to create tarball: ${images_file}" + local count + count=$(find "${staging_dir}" -name '*.tar' | wc -l | tr -d '[:space:]') + log_info "Creating ${tgz_file} with ${count} images" - # Remove temporary file - rm "${pulled_file}" || handle_error "Failed to remove temporary file: ${pulled_file}" + merge_and_compress "${tgz_file}" "${staging_dir}" + rm -rf "${staging_dir}" - # Cleanup: Remove the pulled images - #for image in ${pulled}; do - # docker rmi -f ${image} - #done + log_info "Combined bundle created: ${tgz_file}" +} -else -# Loop through each list - for list in ${lists[*]}; do - if [[ ! -f $list ]]; then - echo "Error: File $list not found." - exit 1 +bundle_one_image() { + local out_dir="$1" + local img_name="$2" + shift 2 + local img_tags=("$@") + + if [ ${#img_tags[@]} -eq 0 ]; then + return fi - # Generate image file name from list name - base_name=$(basename "$list" .txt) - images_file="${base_name}.tgz" + local tgz_file="${out_dir}/${img_name}.tgz" + local staging_dir + staging_dir="$(mktemp -d)" - # Create a temporary file to store the list of successfully pulled images - pulled_file="$(mktemp)" + log_info "Copying ${#img_tags[@]} tags for single bundle: ${img_name}" - pids=() + if ! copy_images_bounded "${staging_dir}" "${img_tags[@]}"; then + log_error "Some tags failed to copy for image: ${img_name}" + rm -rf "${staging_dir}" + return 1 + fi - # Download images in parallel - while IFS= read -r i; do - [ -z "${i}" ] && continue - pull_image "${i}" & - pids+=($!) - done < "${list}" + local count + count=$(find "${staging_dir}" -name '*.tar' | wc -l | tr -d '[:space:]') + log_info "Creating ${tgz_file} with ${count} tags" + + merge_and_compress "${tgz_file}" "${staging_dir}" + rm -rf "${staging_dir}" + + log_info "Single bundle created: ${tgz_file}" +} - for pid in ${pids[*]}; do - wait $pid || handle_error "Failed background task with PID: $pid" +create_single_bundles() { + local section_name="$1" + local bucket_path="$2" + shift 2 + local section_lines=("$@") + + local out_dir="${OUTPUT_DIR}/${bucket_path}" + mkdir -p "${out_dir}" + + # Parse all image groups. Each @image= defines a unique bundle name (from images_internal.txt) + local -a group_names=() + local -a group_tags=() + local current_name="" + local current_tags="" + + for line in "${section_lines[@]}"; do + if [[ "$line" =~ ^#\ @image= ]]; then + if [ -n "$current_name" ] && [ -n "$current_tags" ]; then + group_names+=("$current_name") + group_tags+=("$current_tags") + fi + current_name="${line#*@image=}" + current_name="${current_name%% *}" + current_tags="" + elif [ -n "$line" ] && [[ ! "$line" =~ ^# ]]; then + current_tags="${current_tags:+${current_tags}$'\n'}${line}" + fi done - # Wait for all background tasks to finish - wait - # Get the list of successfully pulled images - pulled=$(cat "${pulled_file}") + if [ -n "$current_name" ] && [ -n "$current_tags" ]; then + group_names+=("$current_name") + group_tags+=("$current_tags") + fi - # Save pulled images to a tarball - echo "Creating ${images_file} with $(echo ${pulled} | wc -w | tr -d '[:space:]') images" - docker save $(echo ${pulled}) | gzip --stdout > ${images_file} || handle_error "Failed to create tarball: ${images_file}" + local total=${#group_names[@]} + log_info "Bundling ${total} image groups for: ${section_name} (parallel limit: ${MAX_PARALLEL_BUNDLES})" + local pids=() + local active=0 + local failed=0 - # Remove temporary file - rm "${pulled_file}" || handle_error "Failed to remove temporary file: ${pulled_file}" - Cleanup: Remove the pulled images - for image in ${pulled}; do - docker rmi -f ${image} + for i in "${!group_names[@]}"; do + local name="${group_names[$i]}" + local tags_str="${group_tags[$i]}" + + local -a tags_arr=() + while IFS= read -r t; do + [ -n "$t" ] && tags_arr+=("$t") + done <<< "$tags_str" + + bundle_one_image "$out_dir" "$name" "${tags_arr[@]}" /dev/null && tput colors &>/dev/null 2>&1; then + RED=$'\033[0;31m'; YELLOW=$'\033[1;33m'; GREEN=$'\033[0;32m' + CYAN=$'\033[0;36m'; BOLD=$'\033[1m'; DIM=$'\033[2m'; RESET=$'\033[0m' +else + RED=''; YELLOW=''; GREEN=''; CYAN=''; BOLD=''; DIM=''; RESET='' +fi + +log_info() { printf "${GREEN}[INFO]${RESET} %s %s\n" "$(date +%H:%M:%S)" "$*"; } +log_warn() { printf "${YELLOW}[WARN]${RESET} %s %s\n" "$(date +%H:%M:%S)" "$*" >&2; } +log_error() { printf "${RED}[ERROR]${RESET} %s %s\n" "$(date +%H:%M:%S)" "$*" >&2; } +log_step() { printf "\n${BOLD}${CYAN}┌─ %s${RESET}\n" "$*"; } +log_done() { printf "${GREEN} ✓${RESET} %s\n" "$*"; } +log_skip() { printf "${DIM} ↷ skipped: %s${RESET}\n" "$*" >&2; } + +# ───────────────────────────────────────────────────────────────────────────── +# Usage +# ───────────────────────────────────────────────────────────────────────────── +DEFAULT_BASE_URL="https://storage.googleapis.com/smp-airgap-bundles" + +usage_short() { + cat <&2 + +${BOLD}Required inputs:${RESET} + -v, --version VERSION Harness release version (e.g. 0.37.0) + ${DIM}Uses default GCS bucket: ${DEFAULT_BASE_URL}${RESET} + --url URL Complete base URL to the release directory + ${DIM}(alternative to --version, for self-hosted mirrors)${RESET} + -o, --output-dir PATH Directory to save downloaded bundles ${DIM}(not needed with --list)${RESET} + +${BOLD}Non-interactive mode also requires:${RESET} + -b, --bundles LIST Comma-separated bundle names (modules, sub-bundles, or agents) + ${DIM}e.g. platform,ci,ci-plugins,delegate,delegate-fips${RESET} + ${DIM}Use 'all' to download everything. Run --list to see names.${RESET} + +${BOLD}Quick start:${RESET} + # See what is available: + ./download-airgap-bundles.sh -v 0.37.0 --list + + # Pick interactively and save to a file (no download yet): + ./download-airgap-bundles.sh -v 0.37.0 --generate-selection-file (-g) + + # Then download using that saved file: + ./download-airgap-bundles.sh -v 0.37.0 --output-dir ./bundles --selection-file selection.conf + +Run ${BOLD}./download-airgap-bundles.sh --help${RESET} for full usage and examples. +EOF + exit 1 +} + +usage() { + cat < +if [ -n "$BUNDLE_URL" ]; then + EFFECTIVE_BASE="${BUNDLE_URL%/}" +else + # Normalise: strip any "harness-" prefix so we always control the format + VERSION="${VERSION#harness-}" + EFFECTIVE_BASE="${DEFAULT_BASE_URL}/harness-${VERSION}" +fi + +# ───────────────────────────────────────────────────────────────────────────── +# Download helper +# ───────────────────────────────────────────────────────────────────────────── +get_file_size() { + [ -f "$1" ] && { stat -f%z "$1" 2>/dev/null || stat -c%s "$1" 2>/dev/null || echo 0; } || echo 0 +} + +download_file() { + local url="$1" dest="$2" + # $3 (optional) — name of a variable to receive the error message on failure + local _err_var="${3:-}" + mkdir -p "$(dirname "$dest")" + + # Use a temp file to capture stderr separately so the progress bar + # still streams live to the terminal while errors are also preserved. + local _stderr_tmp + _stderr_tmp=$(mktemp) + + local _rc=0 + if command -v curl &>/dev/null; then + # tee stderr to /dev/tty (progress bar live on terminal) AND the temp + # file (so errors are still capturable on failure). + curl -fSL --progress-bar -o "$dest" "$url" 2> >(tee "$_stderr_tmp" >/dev/tty) + _rc=$? + elif command -v wget &>/dev/null; then + wget -q --show-progress --progress=bar -O "$dest" "$url" 2> >(tee "$_stderr_tmp" >/dev/tty) + _rc=$? + else + log_error "Neither curl nor wget found." + rm -f "$_stderr_tmp" + exit 1 + fi + + if [ "$_rc" -ne 0 ] && [ -n "$_err_var" ]; then + printf -v "$_err_var" '%s' "$(cat "$_stderr_tmp")" + fi + rm -f "$_stderr_tmp" + return "$_rc" +} + +# ───────────────────────────────────────────────────────────────────────────── +# Manifest parsing (Python embedded) +# +# Emits lines in one of three formats: +# MODULE||||| +# CHILD||||| +# AGENT|||| +# +# AGENT lines are emitted for every image in a `single` bundle-type section +# that has a `parent` (i.e. sub-bundles like platform-agents, sto-scanners). +# ───────────────────────────────────────────────────────────────────────────── +parse_manifest() { + local manifest_file="$1" + python3 - "$manifest_file" <<'PYTHON' +import sys +import yaml + +def get_name(entry): + return entry['name'] if isinstance(entry, dict) else str(entry) + +def get_suffixes(entry): + if isinstance(entry, dict): + return entry.get('variants', []) + return [] + +def main(): + with open(sys.argv[1]) as f: + data = yaml.safe_load(f) + + modules = data.get('modules', {}) + + for mod_name, cfg in modules.items(): + parent = cfg.get('parent', '') + bundle_type = cfg.get('bundle_type', 'combined') + bucket_path = cfg.get('bucket_path', mod_name) + description = cfg.get('description', mod_name) + requires = ','.join(cfg.get('requires', [])) + + if parent: + print(f"CHILD|{mod_name}|{parent}|{bucket_path}|{bundle_type}|{description}") + else: + print(f"MODULE|{mod_name}|{requires}|{bucket_path}|{bundle_type}|{description}") + + # Emit AGENT lines for every image in single-type sections + if bundle_type == 'single': + for entry in cfg.get('images', []): + name = get_name(entry) + suffixes = ','.join(get_suffixes(entry)) + print(f"AGENT|{mod_name}|{bucket_path}|{name}|{suffixes}") + +if __name__ == '__main__': + try: + main() + except Exception as e: + sys.stderr.write(f"Parse error: {e}\n") + sys.exit(1) +PYTHON +} + +# ───────────────────────────────────────────────────────────────────────────── +# Manifest query helpers (all operate on the parsed string) +# ───────────────────────────────────────────────────────────────────────────── + +module_requires() { echo "$1" | awk -F'|' -v m="$2" '$1=="MODULE" && $2==m {print $3}'; } +module_bucket() { echo "$1" | awk -F'|' -v m="$2" '$1=="MODULE" && $2==m {print $4}'; } +child_bucket() { echo "$1" | awk -F'|' -v c="$2" '$1=="CHILD" && $2==c {print $4}'; } +child_parent() { echo "$1" | awk -F'|' -v c="$2" '$1=="CHILD" && $2==c {print $3}'; } +children_of() { echo "$1" | awk -F'|' -v p="$2" '$1=="CHILD" && $3==p {print $2}'; } +agents_in_section() { echo "$1" | awk -F'|' -v s="$2" '$1=="AGENT" && $2==s {print $4 "|" $5}'; } +all_agent_sections(){ echo "$1" | awk -F'|' '$1=="AGENT" {print $2}' | sort -u; } + +# ───────────────────────────────────────────────────────────────────────────── +# Dependency resolution (BFS, bash 3.x compatible) +# ───────────────────────────────────────────────────────────────────────────── +resolve_modules() { + local parsed="$1" + local csv="$2" + local seen="" resolved="" + local queue=() + IFS=',' read -ra queue <<< "$csv" + + while [ ${#queue[@]} -gt 0 ]; do + local m="${queue[0]}" + queue=("${queue[@]:1}") + m="${m// /}" + [ -z "$m" ] && continue + echo "$seen" | grep -qF "|${m}|" && continue + seen="${seen}|${m}|" + resolved="${resolved} ${m}" + local reqs + reqs=$(module_requires "$parsed" "$m") + if [ -n "$reqs" ]; then + IFS=',' read -ra rarr <<< "$reqs" + for r in "${rarr[@]}"; do + r="${r// /}" + [ -n "$r" ] && queue+=("$r") + done + fi + done + echo "$resolved" +} + +# ───────────────────────────────────────────────────────────────────────────── +# Build the full list of agent bundle names for a section (base + variants) +# ───────────────────────────────────────────────────────────────────────────── +expand_agent_section() { + local parsed="$1" + local section="$2" + agents_in_section "$parsed" "$section" | while IFS='|' read -r name suffixes; do + echo "$name" + if [ -n "$suffixes" ]; then + IFS=',' read -ra sarr <<< "$suffixes" + for s in "${sarr[@]}"; do + echo "${name}${s}" + done + fi + done +} + +# ───────────────────────────────────────────────────────────────────────────── +# Counters (global) +# ───────────────────────────────────────────────────────────────────────────── +DL_COUNT=0 +DL_SIZE=0 +DL_FAILED=0 + +do_download() { + local url="$1" dest="$2" label="$3" + log_info "Downloading ${BOLD}${label}${RESET}" + printf " ${DIM}URL: %s${RESET}\n" "$url" + local _dl_err="" + if download_file "$url" "$dest" _dl_err; then + local sz + sz=$(get_file_size "$dest") + DL_COUNT=$((DL_COUNT + 1)) + DL_SIZE=$((DL_SIZE + sz)) + log_done "Saved → ${dest} ${DIM}($(( sz / 1024 / 1024 )) MB)${RESET}" + else + log_warn "Failed to download: ${BOLD}${label}${RESET}" + if [ -n "$_dl_err" ]; then + # Indent each line of the error for readability + echo "$_dl_err" | while IFS= read -r _line; do + printf " ${RED}│${RESET} ${DIM}%s${RESET}\n" "$_line" + done + fi + DL_FAILED=$((DL_FAILED + 1)) + fi +} + +# ───────────────────────────────────────────────────────────────────────────── +# Download a combined module bundle +# ───────────────────────────────────────────────────────────────────────────── +download_module() { + local parsed="$1" mod="$2" + local bucket + bucket=$(module_bucket "$parsed" "$mod") + [ -z "$bucket" ] && bucket="$mod" + local url="${EFFECTIVE_BASE}/${bucket}/${mod}_images.tgz" + local dest="${OUTPUT_DIR}/${bucket}/${mod}_images.tgz" + do_download "$url" "$dest" "module: ${mod}" +} + +# ───────────────────────────────────────────────────────────────────────────── +# Download a child (sub-bundle) — combined type: one tgz for the whole section +# ───────────────────────────────────────────────────────────────────────────── +download_child() { + local parsed="$1" child="$2" + local bucket + bucket=$(child_bucket "$parsed" "$child") + [ -z "$bucket" ] && bucket="$child" + local url="${EFFECTIVE_BASE}/${bucket}/${child}_images.tgz" + local dest="${OUTPUT_DIR}/${bucket}/${child}_images.tgz" + do_download "$url" "$dest" "sub-bundle: ${child}" +} + +# ───────────────────────────────────────────────────────────────────────────── +# Download a single agent bundle by its full name (e.g. delegate-fips) +# ───────────────────────────────────────────────────────────────────────────── +download_agent() { + local parsed="$1" agent_name="$2" + local section bucket + section=$(echo "$parsed" | awk -F'|' -v a="$agent_name" ' + $1=="AGENT" { + name=$4 + split($5, sfx, ",") + if (name == a) { print $2; exit } + for (i in sfx) { + if (name sfx[i] == a) { print $2; exit } + } + }') + if [ -z "$section" ]; then + log_warn "Agent '${agent_name}' not found in manifest — skipping" + return + fi + bucket=$(echo "$parsed" | awk -F'|' -v s="$section" '$1=="AGENT" && $2==s {print $3; exit}') + [ -z "$bucket" ] && bucket="$section" + # Normalize: dots used as variant separators in image tags (e.g. delegate.minimal-fips) + # must become dashes in the bundle filename (e.g. delegate-minimal-fips.tgz). + local file_name="${agent_name//./-}" + local url="${EFFECTIVE_BASE}/${bucket}/${file_name}.tgz" + local dest="${OUTPUT_DIR}/${bucket}/${file_name}.tgz" + do_download "$url" "$dest" "agent: ${agent_name}" +} + +# ───────────────────────────────────────────────────────────────────────────── +# Interactive selection helper: parses "1,3-5,all,none" against a numbered list +# Input: a file with lines "idx|value" +# Output: space-separated "value" tokens to process +# ───────────────────────────────────────────────────────────────────────────── +parse_selection() { + local list_file="$1" + local selection="$2" + local total + total=$(wc -l < "$list_file" | tr -d ' ') + local result="" + + selection=$(echo "$selection" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') + [ "$selection" = "none" ] || [ -z "$selection" ] && echo "" && return + if [ "$selection" = "all" ]; then + while IFS='|' read -r _ val; do result="${result} ${val}"; done < "$list_file" + echo "$result"; return + fi + for part in $(echo "$selection" | tr ',' ' '); do + if [[ "$part" =~ ^([0-9]+)-([0-9]+)$ ]]; then + for ((i=${BASH_REMATCH[1]}; i<=${BASH_REMATCH[2]}; i++)); do + val=$(awk -F'|' -v n="$i" '$1==n {print $2}' "$list_file") + [ -n "$val" ] && result="${result} ${val}" + done + else + val=$(awk -F'|' -v n="$part" '$1==n {print $2}' "$list_file") + [ -n "$val" ] && result="${result} ${val}" + fi + done + echo "$result" +} + +prompt() { + local msg="$1" + local total="${2:-}" # optional: total number of items in the list + # All display output goes to stderr so it remains visible when called inside $() + echo "" >&2 + printf " ${BOLD}${CYAN}?${RESET} ${BOLD}%s${RESET}\n" "$msg" >&2 + echo "" >&2 + printf " ${DIM}How to select:${RESET}\n" >&2 + printf " ${DIM} single item → 1${RESET}\n" >&2 + if [ -n "$total" ] && [ "$total" -ge 3 ] 2>/dev/null; then + local mid=$(( total / 2 )) + local hi=$(( total < 5 ? total : 5 )) + printf " ${DIM} multiple → 1,%d,%d${RESET}\n" "$mid" "$hi" >&2 + printf " ${DIM} range → 1-%d${RESET}\n" "$hi" >&2 + else + printf " ${DIM} multiple → 1,2,3${RESET}\n" >&2 + printf " ${DIM} range → 1-4${RESET}\n" >&2 + fi + printf " ${DIM} everything → all${RESET}\n" >&2 + printf " ${DIM} skip / none → none (or press Enter)${RESET}\n" >&2 + echo "" >&2 + printf " ${CYAN}›${RESET} " >&2 + local ans + read -r ans /dev/null || read -r ans + echo "$ans" # only the answer goes to stdout, captured by $() +} + +# Print a confirmation list of items about to be downloaded +print_download_plan() { + local label="$1"; shift + echo "" + printf " ${BOLD}%s to download:${RESET}\n" "$label" + for item in "$@"; do + printf " ${CYAN}•${RESET} %s\n" "$item" + done +} + +# ───────────────────────────────────────────────────────────────────────────── +# print_selection_plan — show what will be downloaded (non-interactive / file mode). +# ───────────────────────────────────────────────────────────────────────────── +print_selection_plan() { + local parsed="$1" + shift + # remaining args: space-separated lists modules_to_dl children_to_dl agents_to_dl + local _mods="$1" _children="$2" _agents="$3" + + echo "" + printf " ${BOLD}Download plan:${RESET}\n" + for _b in $_mods $_children $_agents; do + [ -z "$_b" ] && continue + local _desc + _desc=$(echo "$parsed" | awk -F'|' -v n="$_b" ' + ($1=="MODULE" || $1=="CHILD") && $2==n { print $6; exit }') + [ -z "$_desc" ] && _desc="$_b" + printf " ${CYAN}•${RESET} %-28s ${DIM}%s${RESET}\n" "$_b" "$_desc" + done + echo "" +} + +# ───────────────────────────────────────────────────────────────────────────── +# list_bundles — print every available bundle name and exit. +# ───────────────────────────────────────────────────────────────────────────── +list_bundles() { + local parsed="$1" + local _ver_label="${VERSION:-${BUNDLE_URL}}" + echo "" + printf "${BOLD}${CYAN}╔══════════════════════════════════════════════════════╗${RESET}\n" + printf "${BOLD}${CYAN}║ Available Harness Airgap Bundles ║${RESET}\n" + printf "${BOLD}${CYAN}╚══════════════════════════════════════════════════════╝${RESET}\n" + echo "" + printf "${DIM}Source: %s${RESET}\n" "$_ver_label" + printf "${DIM}Use any NAME below with: -b / --bundles NAME,NAME,...${RESET}\n" + echo "" + + printf "${BOLD}Modules${RESET}\n" + printf " %-24s %-12s %s\n" "NAME" "REQUIRES" "DESCRIPTION" + printf " %s\n" "──────────────────────────────────────────────────────────────" + while IFS='|' read -r _mname _mdesc _mreqs; do + printf " ${GREEN}%-24s${RESET} ${DIM}%-12s${RESET} %s\n" "$_mname" "${_mreqs:-(none)}" "$_mdesc" + done < <(echo "$parsed" | awk -F'|' '$1=="MODULE" {print $2 "|" $6 "|" $3}' | sort -t'|' -k3 -r) + echo "" + + printf "${BOLD}Sub-bundles${RESET}\n" + printf " %-24s %-12s %s\n" "NAME" "PARENT" "DESCRIPTION" + printf " %s\n" "──────────────────────────────────────────────────────────────" + local _prev_parent="" + while IFS='|' read -r _cname _cparent _cdesc _cbtype; do + [ "$_cbtype" = "single" ] && continue + [ "$_cparent" != "$_prev_parent" ] && [ -n "$_prev_parent" ] && echo "" + _prev_parent="$_cparent" + printf " ${YELLOW}%-24s${RESET} ${DIM}%-12s${RESET} %s\n" "$_cname" "$_cparent" "$_cdesc" + done < <(echo "$parsed" | awk -F'|' '$1=="CHILD" {print $2 "|" $3 "|" $6 "|" $5}' | sort -t'|' -k2,2 -k1,1) + echo "" + + printf "${BOLD}Agents ${DIM}(individual image archives)${RESET}\n" + printf " %-24s %s\n" "NAME" "SECTION" + printf " %s\n" "──────────────────────────────────────────────────────────────" + local _prev_sect="" + while IFS='|' read -r _aname _asect; do + [ "$_asect" != "$_prev_sect" ] && [ -n "$_prev_sect" ] && echo "" + _prev_sect="$_asect" + printf " ${CYAN}%-24s${RESET} ${DIM}%s${RESET}\n" "$_aname" "$_asect" + done < <(echo "$parsed" | awk -F'|' '$1=="AGENT" {print $4 "|" $3}' | sort -t'|' -k2,2 -k1,1) + echo "" + + printf "${DIM}Run --generate-selection-file (-g) to pick interactively and save a selection file.${RESET}\n" + printf "${DIM}Run --selection-file (-s) selection.conf to download from a saved file.${RESET}\n" + echo "" +} + +# ───────────────────────────────────────────────────────────────────────────── +# load_selection_file — read a selection.conf and set BUNDLES_CSV. +# Format (order-independent, '#' comments stripped): +# bundles=platform,ci,ci-plugins,delegate +# ───────────────────────────────────────────────────────────────────────────── +load_selection_file() { + local _file="$1" + [ ! -f "$_file" ] && log_error "Selection file not found: ${_file}" && exit 1 + log_info "Loading selections from: ${BOLD}${_file}${RESET}" + while IFS= read -r _line; do + _line="${_line%%#*}" + _line="${_line#"${_line%%[![:space:]]*}"}" + _line="${_line%"${_line##*[![:space:]]}"}" + [ -z "$_line" ] && continue + local _key _val + _key="${_line%%=*}" + _val="${_line#*=}" + case "$_key" in + bundles) [ -z "$BUNDLES_CSV" ] && BUNDLES_CSV="$_val" ;; + esac + done < "$_file" +} + +# ───────────────────────────────────────────────────────────────────────────── +# checkbox_menu — interactive checkbox selector with arrow-key navigation +# +# Usage: +# result=$(checkbox_menu "Title" "Label|value" "Label|value|dep1,dep2" ...) +# +# Item format: "display label | value | dep_value1,dep_value2" +# - The optional 3rd field lists values of items that should be auto-checked +# when this item is toggled on (e.g. required dependencies). +# +# All display goes to /dev/tty so the UI is visible even inside $(). +# Selected values are written to stdout, one per line. +# +# Keys: ↑/↓ navigate Space toggle a all n none Enter confirm +# ───────────────────────────────────────────────────────────────────────────── +checkbox_menu() { + # set -e (errexit) causes false-returning [ ] && cmd expressions to kill the + # script. This is a UI function full of intentional short-circuits, so we + # save the flag and disable it for the duration of the function, then restore. + local _cbm_e=0 + case "$-" in *e*) _cbm_e=1 ;; esac + set +e + + local _title="$1"; shift + # _auto[i]=1 means this item was checked automatically as a dependency + declare -a _labels _values _deps _checked _auto + local _n=0 _i + for _item in "$@"; do + _labels[$_n]="${_item%%|*}" + local _vd="${_item#*|}" # "value" or "value|deps" + if [[ "$_vd" == *"|"* ]]; then + _values[$_n]="${_vd%%|*}" + _deps[$_n]="${_vd#*|}" + else + _values[$_n]="$_vd" + _deps[$_n]="" + fi + _checked[$_n]=0 + _auto[$_n]=0 + _n=$((_n + 1)) + done + if [ "$_n" -eq 0 ]; then + [ "$_cbm_e" = "1" ] && set -e + return + fi + + local _cursor=0 _drawn=0 + # Skip any leading non-selectable header rows + while [ "$_cursor" -lt "$_n" ] && [ -z "${_values[$_cursor]}" ]; do + _cursor=$((_cursor + 1)) + done + local _EL=$'\033[K' # clear to end of line — prevents stale chars when redrawing + + # ── Viewport: clamp list to terminal height so we never overdraw ────────── + local _term_h + _term_h=$(tput lines 2>/dev/null || echo 24) + # Reserve lines: title block (3) + above/below indicators (2) + footer (3) = 8 + local _view_size=$(( _term_h - 8 )) + if [ "$_view_size" -lt 3 ]; then _view_size=3; fi + if [ "$_view_size" -gt "$_n" ]; then _view_size="$_n"; fi + local _view_top=0 + + # ── Auto-check a dep by value (only if not already manually checked) ───── + _cm_autocheck() { + local _dep_val="$1" + for ((_j=0; _j<_n; _j++)); do + if [ "${_values[$_j]}" = "$_dep_val" ] && [ "${_checked[$_j]}" = "0" ]; then + _checked[$_j]=1 + _auto[$_j]=1 + fi + done + } + + # ── When unchecking item at index $1, release any auto-deps that are no + # longer required by another checked item ───────────────────────────── + _cm_release_deps() { + local _idx="$1" + local _dep _k _d _still_needed + + for _dep in ${_deps[$_idx]//,/ }; do + [ -z "$_dep" ] && continue + _still_needed=0 + + # Search every OTHER checked item to see if it still needs this dep + for ((_k=0; _k<_n; _k++)); do + [ "$_k" -eq "$_idx" ] && continue + [ "${_checked[$_k]}" != "1" ] && continue + for _d in ${_deps[$_k]//,/ }; do + if [ "$_d" = "$_dep" ]; then + _still_needed=1 + break 2 # exits both for _d and for ((_k)) at once + fi + done + done + + # If nothing else needs it and it was auto-selected, release it + if [ "$_still_needed" = "0" ]; then + for ((_k=0; _k<_n; _k++)); do + if [ "${_values[$_k]}" = "$_dep" ] && [ "${_auto[$_k]}" = "1" ]; then + _checked[$_k]=0 + _auto[$_k]=0 + fi + done + fi + done + } + + # ── Terminal: raw mode + hidden cursor ─────────────────────────────────── + local _saved_tty="" + _saved_tty=$(stty -g /dev/null) || true + stty -echo -icanon min 1 time 0 /dev/null || true + tput civis >/dev/tty 2>/dev/null || true + + _cm_restore() { + stty "$_saved_tty" /dev/null || true + tput cnorm >/dev/tty 2>/dev/null || true + } + trap '_cm_restore; exit 130' INT + trap '_cm_restore' EXIT + + # ── Draw / redraw the viewport ─────────────────────────────────────────── + # Total drawn height is always _view_size + 5 (constant) so cursor-up + # always lands at the exact right position on redraw. + _cm_draw() { + # Scroll viewport to keep cursor visible + if [ "$_cursor" -lt "$_view_top" ]; then + _view_top="$_cursor" + elif [ "$_cursor" -ge $(( _view_top + _view_size )) ]; then + _view_top=$(( _cursor - _view_size + 1 )) + fi + + if [ "$_drawn" -gt 0 ]; then + printf '\033[%dA' "$_drawn" >/dev/tty + fi + _drawn=0 + + # ── "above" indicator (always 1 line) ──────────────────────────────── + if [ "$_view_top" -gt 0 ]; then + printf " ${DIM} ▲ %d more above${_EL}${RESET}\n" "$_view_top" >/dev/tty + else + printf "${_EL}\n" >/dev/tty + fi + _drawn=1 + + # ── Items in viewport (always _view_size lines, padded if needed) ───── + local _slot _idx _box _row + for (( _slot=0; _slot<_view_size; _slot++ )); do + _idx=$(( _view_top + _slot )) + if [ "$_idx" -ge "$_n" ]; then + printf "${_EL}\n" >/dev/tty # blank padding at bottom + else + local _lbl="${_labels[$_idx]}" + local _val="${_values[$_idx]}" + if [ -z "$_val" ]; then + # Non-selectable header row — no checkbox, dimmed + printf " ${DIM}%s${_EL}${RESET}\n" "$_lbl" >/dev/tty + else + if [ "${_checked[$_idx]}" = "1" ]; then + if [ "${_auto[$_idx]}" = "1" ]; then _box="[↑]"; else _box="[✓]"; fi + else + _box="[ ]" + fi + if [ "$_idx" -eq "$_cursor" ]; then + _row=" ${CYAN}${BOLD}▶ %s %s${_EL}${RESET}\n" + elif [ "${_auto[$_idx]}" = "1" ]; then + _row=" ${DIM} %s %s${_EL}${RESET}\n" + else + _row=" %s %s${_EL}\n" + fi + printf "$_row" "$_box" "$_lbl" >/dev/tty + fi + fi + _drawn=$((_drawn + 1)) + done + + # ── "below" indicator (always 1 line) ──────────────────────────────── + local _below=$(( _n - _view_top - _view_size )) + if [ "$_below" -gt 0 ]; then + printf " ${DIM} ▼ %d more below${_EL}${RESET}\n" "$_below" >/dev/tty + else + printf "${_EL}\n" >/dev/tty + fi + _drawn=$((_drawn + 1)) + + # ── Footer (blank + 4 help lines = 5 lines) ────────────────────────── + printf "\n ${DIM}↑/↓ = navigate.${_EL}${RESET}\n" >/dev/tty + printf " ${DIM}Space = toggle, Enter = confirm.${_EL}${RESET}\n" >/dev/tty + printf " ${DIM}a = all, n = none.${_EL}${RESET}\n" >/dev/tty + printf " ${DIM}[↑] = auto-selected dependency.${_EL}${RESET}\n" >/dev/tty + _drawn=$((_drawn + 5)) + } + + # Header is printed once (not part of the redraw region) + printf "\n${BOLD}${CYAN} %s${RESET}\n\n" "$_title" >/dev/tty + _cm_draw + + # ── Event loop ─────────────────────────────────────────────────────────── + local _key _seq + while true; do + _key="" + IFS= read -rsn1 _key /dev/tty + + # ── Emit selected values to stdout (captured by the caller's $()) ──────── + # Restore set -e before returning so the caller's error handling is intact + [ "$_cbm_e" = "1" ] && set -e + for ((_i=0; _i<_n; _i++)); do + if [ -n "${_values[$_i]}" ] && [ "${_checked[$_i]}" = "1" ]; then + echo "${_values[$_i]}" + fi + done +} + +# ───────────────────────────────────────────────────────────────────────────── +# Main +# ───────────────────────────────────────────────────────────────────────────── +main() { + [ -n "$SELECTION_FILE" ] && load_selection_file "$SELECTION_FILE" + + echo "" + printf "${BOLD}${CYAN}╔══════════════════════════════════════════════════════╗${RESET}\n" + if [ "$LIST_ONLY" = true ]; then + printf "${BOLD}${CYAN}║ Harness Airgap Bundle Lister ║${RESET}\n" + elif [ "$GENERATE_SELECTION" = true ]; then + printf "${BOLD}${CYAN}║ Harness Airgap — Generate Selection File ║${RESET}\n" + else + printf "${BOLD}${CYAN}║ Harness Airgap Bundle Downloader ║${RESET}\n" + fi + printf "${BOLD}${CYAN}╚══════════════════════════════════════════════════════╝${RESET}\n" + echo "" + log_info "Bundle source : ${BOLD}${EFFECTIVE_BASE}${RESET}" + if [ "$LIST_ONLY" = false ] && [ "$GENERATE_SELECTION" = false ]; then + log_info "Output dir : ${BOLD}${OUTPUT_DIR}${RESET}" + mkdir -p "$OUTPUT_DIR" + fi + if [ "$GENERATE_SELECTION" = true ]; then + local _sel_out="${SELECTION_OUTPUT_FILE:-selection.conf}" + log_info "Selection file: ${BOLD}${_sel_out}${RESET} ${DIM}(no download will be performed)${RESET}" + fi + + # ── Fetch / load manifest ───────────────────────────────────────────────── + local tmp_manifest + tmp_manifest=$(mktemp) + trap "rm -f ${tmp_manifest}" EXIT + + local manifest_url="${EFFECTIVE_BASE}/bundle-manifest.yaml" + log_info "Fetching manifest → ${manifest_url}" + local _manifest_err="" + if ! download_file "$manifest_url" "$tmp_manifest" _manifest_err || [ ! -s "$tmp_manifest" ]; then + log_warn "Manifest not found at bundle source." + if [ -n "$_manifest_err" ]; then + echo "$_manifest_err" | while IFS= read -r _line; do + printf " ${RED}│${RESET} ${DIM}%s${RESET}\n" "$_line" + done + fi + if [ -n "$VERSION" ]; then + local github_manifest_url="https://raw.githubusercontent.com/harness/helm-charts/${VERSION}/src/bundle-manifest.yaml" + log_info "Trying GitHub fallback → ${github_manifest_url}" + rm -f "$tmp_manifest"; tmp_manifest=$(mktemp) + local _gh_err="" + if ! download_file "$github_manifest_url" "$tmp_manifest" _gh_err || [ ! -s "$tmp_manifest" ]; then + log_error "Manifest not found on GitHub either." + if [ -n "$_gh_err" ]; then + echo "$_gh_err" | while IFS= read -r _line; do + printf " ${RED}│${RESET} ${DIM}%s${RESET}\n" "$_line" + done + fi + log_error " Checked: ${manifest_url}" + log_error " Checked: ${github_manifest_url}" + exit 1 + fi + log_done "Manifest loaded from GitHub (harness-${VERSION})" + else + log_error "Manifest not found at the provided --url." + log_error " Checked: ${manifest_url}" + log_error "GitHub fallback is only available with --version." + exit 1 + fi + fi + [ ! -s "$tmp_manifest" ] && log_error "Manifest is empty" && exit 1 + log_done "Manifest loaded successfully" + + local parsed + parsed=$(parse_manifest "$tmp_manifest") + + if [ "$LIST_ONLY" = true ]; then + list_bundles "$parsed" + exit 0 + fi + + # ── Build the flat universe of every selectable name ───────────────────── + # Order: platform first (no requires), then modules with deps, each followed + # by their children (combined sub-bundles, then agent sections with variants). + # + # Each checkbox_menu item: "LABEL|VALUE|DEPS" + # Agent-section rows are header-only (non-selectable): VALUE="" so checkbox_menu + # emits nothing for them and they can never be "selected" themselves. + declare -a _universe_items # items for checkbox_menu + declare -a _universe_values # parallel: just the values (for 'all' expansion) + + local _ui=0 + + # Sort modules: empty requires first (platform), then by requires length ascending. + # awk prints "name|requires", sort -t'|' -k2 puts empty string first. + local _mod_order + _mod_order=$(echo "$parsed" | awk -F'|' '$1=="MODULE" {print $2 "|" $3}' | sort -t'|' -k2,2 | awk -F'|' '{print $1}') + + while IFS= read -r _mname; do + [ -z "$_mname" ] && continue + local _mdesc _mreqs _mlabel + _mdesc=$(echo "$parsed" | awk -F'|' -v m="$_mname" '$1=="MODULE" && $2==m {print $6}') + _mreqs=$(echo "$parsed" | awk -F'|' -v m="$_mname" '$1=="MODULE" && $2==m {print $3}') + _mlabel="${_mname} — ${_mdesc}" + [ -n "$_mreqs" ] && _mlabel="${_mlabel} ${DIM}(requires: ${_mreqs})${RESET}" + _universe_items[$_ui]="${_mlabel}|${_mname}|${_mreqs}" + _universe_values[$_ui]="$_mname" + _ui=$((_ui + 1)) + + # Children of this module — combined sub-bundles first, then single (agents section) + while IFS= read -r _cname; do + [ -z "$_cname" ] && continue + local _cdesc _cbtype + _cdesc=$(echo "$parsed" | awk -F'|' -v c="$_cname" '$1=="CHILD" && $2==c {print $6}') + _cbtype=$(echo "$parsed" | awk -F'|' -v c="$_cname" '$1=="CHILD" && $2==c {print $5}') + if [ "$_cbtype" = "single" ]; then + # Agent section: non-selectable header row (VALUE is empty so nothing emitted) + _universe_items[$_ui]=" ↳ ${_cdesc} ${DIM}[agents]${RESET}||" + _universe_values[$_ui]="" # not selectable + _ui=$((_ui + 1)) + # Individual agent variants under this section + while IFS='|' read -r _aname _asuffixes; do + _universe_items[$_ui]=" ↳ ${_aname}|${_aname}|" + _universe_values[$_ui]="$_aname" + _ui=$((_ui + 1)) + if [ -n "$_asuffixes" ]; then + IFS=',' read -ra _sarr <<< "$_asuffixes" + for _s in "${_sarr[@]}"; do + local _vn="${_aname}${_s}" + _universe_items[$_ui]=" ↳ ${_vn} ${DIM}(variant)${RESET}|${_vn}|" + _universe_values[$_ui]="$_vn" + _ui=$((_ui + 1)) + done + fi + done < <(agents_in_section "$parsed" "$_cname") + else + _universe_items[$_ui]=" ↳ ${_cname} — ${_cdesc}|${_cname}|" + _universe_values[$_ui]="$_cname" + _ui=$((_ui + 1)) + fi + done < <(children_of "$parsed" "$_mname") + done <<< "$_mod_order" + + # ── Resolve BUNDLES_CSV or run interactive menu ─────────────────────────── + local selected_values="" # space-separated list of selected names + + if [ -n "$BUNDLES_CSV" ]; then + # Non-interactive / selection-file path + local _bundles_lower + _bundles_lower=$(echo "$BUNDLES_CSV" | tr '[:upper:]' '[:lower:]') + if [ "$_bundles_lower" = "all" ]; then + for _v in "${_universe_values[@]}"; do + [ -n "$_v" ] && selected_values="${selected_values} ${_v}" + done + else + IFS=',' read -ra _req <<< "$BUNDLES_CSV" + for _r in "${_req[@]}"; do + _r="${_r// /}" + [ -n "$_r" ] && selected_values="${selected_values} ${_r}" + done + fi + elif [ "$NON_INTERACTIVE" = false ]; then + # ── Step 1: select modules ──────────────────────────────────────────── + declare -a _mod_items + local _mi=0 + while IFS= read -r _mname; do + [ -z "$_mname" ] && continue + local _mdesc2 _mreqs2 _ml2 + _mdesc2=$(echo "$parsed" | awk -F'|' -v m="$_mname" '$1=="MODULE" && $2==m {print $6}') + _mreqs2=$(echo "$parsed" | awk -F'|' -v m="$_mname" '$1=="MODULE" && $2==m {print $3}') + _ml2="${_mname} — ${_mdesc2}" + [ -n "$_mreqs2" ] && _ml2="${_ml2} ${DIM}(requires: ${_mreqs2})${RESET}" + _mod_items[$_mi]="${_ml2}|${_mname}|${_mreqs2}" + _mi=$((_mi + 1)) + done <<< "$_mod_order" + + local _sel_mods="" + _sel_mods=$(checkbox_menu "Step 1 of 3 — Select modules" "${_mod_items[@]}") + + local _picked_modules="" + while IFS= read -r _v; do + [ -n "$_v" ] && _picked_modules="${_picked_modules} ${_v}" + done <<< "$_sel_mods" + + # Resolve module deps so step 2/3 includes children of auto-added deps + local _resolved_for_menu="" + if [ -n "$(echo "$_picked_modules" | tr -d ' ')" ]; then + local _pm_csv + _pm_csv=$(echo "$_picked_modules" | tr ' ' ',' | sed 's/^,//') + _resolved_for_menu=$(resolve_modules "$parsed" "$_pm_csv") + fi + + for _m in $_resolved_for_menu; do + selected_values="${selected_values} ${_m}" + done + + # ── Step 2: combined sub-bundles from all resolved modules (one menu) ─ + declare -a _combined_items + local _ci=0 + for _m in $_resolved_for_menu; do + while IFS= read -r _cname; do + [ -z "$_cname" ] && continue + local _cbt2 + _cbt2=$(echo "$parsed" | awk -F'|' -v c="$_cname" '$1=="CHILD" && $2==c {print $5}') + [ "$_cbt2" != "combined" ] && continue + local _cdesc2 + _cdesc2=$(echo "$parsed" | awk -F'|' -v c="$_cname" '$1=="CHILD" && $2==c {print $6}') + _combined_items[$_ci]="${_cname} — ${_cdesc2} ${DIM}[${_m}]${RESET}|${_cname}|" + _ci=$((_ci + 1)) + done < <(children_of "$parsed" "$_m") + done + + if [ "$_ci" -gt 0 ]; then + local _sel_combined="" + _sel_combined=$(checkbox_menu "Step 2 of 3 — Select sub-bundles (optional)" "${_combined_items[@]}") + while IFS= read -r _v; do + [ -n "$_v" ] && selected_values="${selected_values} ${_v}" + done <<< "$_sel_combined" + fi + + # ── Step 3: agent variants — one menu per agent section across resolved modules ─ + local _agent_step=1 + local _total_agent_sections=0 + for _m in $_resolved_for_menu; do + while IFS= read -r _cname; do + [ -z "$_cname" ] && continue + local _cbt3 + _cbt3=$(echo "$parsed" | awk -F'|' -v c="$_cname" '$1=="CHILD" && $2==c {print $5}') + [ "$_cbt3" = "single" ] && _total_agent_sections=$((_total_agent_sections + 1)) + done < <(children_of "$parsed" "$_m") + done + + for _m in $_resolved_for_menu; do + while IFS= read -r _cname; do + [ -z "$_cname" ] && continue + local _cbt4 + _cbt4=$(echo "$parsed" | awk -F'|' -v c="$_cname" '$1=="CHILD" && $2==c {print $5}') + [ "$_cbt4" != "single" ] && continue + + local _cdesc4 + _cdesc4=$(echo "$parsed" | awk -F'|' -v c="$_cname" '$1=="CHILD" && $2==c {print $6}') + + declare -a _agent_items + local _ai=0 + while IFS='|' read -r _aname _asuffixes; do + _agent_items[$_ai]="${_aname}|${_aname}|" + _ai=$((_ai + 1)) + if [ -n "$_asuffixes" ]; then + IFS=',' read -ra _sarr <<< "$_asuffixes" + for _s in "${_sarr[@]}"; do + local _vn="${_aname}${_s}" + _agent_items[$_ai]=" ↳ ${_vn} ${DIM}(variant)${RESET}|${_vn}|" + _ai=$((_ai + 1)) + done + fi + done < <(agents_in_section "$parsed" "$_cname") + + if [ "$_ai" -gt 0 ]; then + local _sel_agents="" + _sel_agents=$(checkbox_menu \ + "Step 3 of 3 — ${_cdesc4} [${_m}] (${_agent_step}/${_total_agent_sections}, optional)" \ + "${_agent_items[@]}") + while IFS= read -r _v; do + [ -n "$_v" ] && selected_values="${selected_values} ${_v}" + done <<< "$_sel_agents" + _agent_step=$((_agent_step + 1)) + unset _agent_items + fi + done < <(children_of "$parsed" "$_m") + done + + BUNDLES_CSV=$(echo "$selected_values" | tr ' ' ',' | sed 's/^,//;s/,$//') + fi + + # ── Classify selected names into modules / children / agents ───────────── + # Modules get dependency-resolved; children and agents are taken as-is. + local raw_modules="" children_to_dl="" agents_to_dl="" + + for _sel in $selected_values; do + _sel="${_sel// /}" + [ -z "$_sel" ] && continue + if echo "$parsed" | grep -q "^MODULE|${_sel}|"; then + raw_modules="${raw_modules} ${_sel}" + elif echo "$parsed" | grep -q "^CHILD|${_sel}|"; then + local _btype + _btype=$(echo "$parsed" | awk -F'|' -v c="$_sel" '$1=="CHILD" && $2==c {print $5}') + if [ "$_btype" = "single" ]; then + # This is an agent-section name, not a downloadable bundle itself — skip; + # the user should pick individual agent variants instead. + log_warn "'${_sel}' is an agent section, not a downloadable bundle. Select specific agent names (e.g. delegate, upgrader)." + else + children_to_dl="${children_to_dl} ${_sel}" + fi + else + # Assume agent variant name + agents_to_dl="${agents_to_dl} ${_sel}" + fi + done + + # Resolve module dependencies (adds required modules not yet in the list) + local resolved_modules="" + if [ -n "$raw_modules" ]; then + local _mcsv + _mcsv=$(echo "$raw_modules" | tr ' ' ',' | sed 's/^,//') + resolved_modules=$(resolve_modules "$parsed" "$_mcsv") + fi + + # Show summary + local all_selected="${resolved_modules} ${children_to_dl} ${agents_to_dl}" + if [ -n "$(echo "$all_selected" | tr -d ' ')" ]; then + local _display + _display=$(echo "$all_selected" | tr ' ' ',' | sed 's/^,//;s/,$//') + log_info "Bundles to download: ${BOLD}${_display}${RESET}" + if [ "$NON_INTERACTIVE" = true ] || [ -n "${SELECTION_FILE:-}" ]; then + print_selection_plan "$parsed" "$resolved_modules" "$children_to_dl" "$agents_to_dl" + fi + else + log_skip "Nothing selected" + fi + + # ── Download phase ──────────────────────────────────────────────────────── + if [ "$GENERATE_SELECTION" = false ]; then + log_step "Starting downloads" + + if [ -n "$resolved_modules" ]; then + # shellcheck disable=SC2086 + print_download_plan "Modules" $resolved_modules + log_step "Downloading module bundles" + for mod in $resolved_modules; do + mod="${mod// /}" + [ -z "$mod" ] && continue + echo "$parsed" | grep -q "^MODULE|${mod}|" && download_module "$parsed" "$mod" + done + fi + + if [ -n "$children_to_dl" ]; then + # shellcheck disable=SC2086 + print_download_plan "Sub-bundles" $children_to_dl + log_step "Downloading sub-bundles" + for child in $children_to_dl; do + child="${child// /}" + [ -z "$child" ] && continue + download_child "$parsed" "$child" + done + fi + + if [ -n "$agents_to_dl" ]; then + # shellcheck disable=SC2086 + print_download_plan "Agents" $agents_to_dl + log_step "Downloading agents" + for agent in $agents_to_dl; do + agent="${agent// /}" + [ -z "$agent" ] && continue + download_agent "$parsed" "$agent" + done + fi + + if [ -z "$resolved_modules" ] && [ -z "$children_to_dl" ] && [ -z "$agents_to_dl" ]; then + log_skip "No bundles selected" + fi + fi + + # ── Write selection file ────────────────────────────────────────────────── + if [ "$GENERATE_SELECTION" = true ]; then + local _sel_out="${SELECTION_OUTPUT_FILE:-selection.conf}" + local _abs_sel_out + case "$_sel_out" in + /*) _abs_sel_out="$_sel_out" ;; + *) _abs_sel_out="$PWD/$_sel_out" ;; + esac + + local _bundles_out + _bundles_out=$(echo "$BUNDLES_CSV" | sed 's/^,//;s/,$//') + + cat > "$_abs_sel_out" < --output-dir ./bundles \\ +# --selection-file ${_abs_sel_out} +# ────────────────────────────────────────────────────────────────────────────── +# bundles: comma-separated list of bundle names (modules, sub-bundles, agents), +# or 'all'. Run --list to see all available names. + +bundles=${_bundles_out:-none} +EOF + echo "" + printf "${BOLD}${CYAN}╔══════════════════════════════════════════════════════╗${RESET}\n" + printf "${BOLD}${CYAN}║ Selection File Written ║${RESET}\n" + printf "${BOLD}${CYAN}╚══════════════════════════════════════════════════════╝${RESET}\n" + printf " ${GREEN}✓ Saved to${RESET} : ${BOLD}%s${RESET}\n" "$_abs_sel_out" + printf " ${CYAN} Bundles${RESET} : ${BOLD}%s${RESET}\n" "$_bundles_out" + echo "" + printf " ${DIM}To download, run:${RESET}\n" + printf " ${BOLD}./download-airgap-bundles.sh --version %s --output-dir ./bundles \\\\\n" "${VERSION:-}" + printf " --selection-file %s${RESET}\n" "$_abs_sel_out" + echo "" + exit 0 + fi + + # ── Download summary ────────────────────────────────────────────────────── + local total_mb=$(( DL_SIZE / 1024 / 1024 )) + echo "" + printf "${BOLD}${CYAN}╔══════════════════════════════════════════════════════╗${RESET}\n" + printf "${BOLD}${CYAN}║ Download Summary ║${RESET}\n" + printf "${BOLD}${CYAN}╚══════════════════════════════════════════════════════╝${RESET}\n" + printf " ${GREEN}✓ Downloaded${RESET} : ${BOLD}%d${RESET} bundle(s)\n" "$DL_COUNT" + if [ "$DL_FAILED" -gt 0 ]; then + printf " ${RED}✗ Failed${RESET} : ${BOLD}%d${RESET} bundle(s)\n" "$DL_FAILED" + fi + printf " ${CYAN}≈ Total size${RESET} : ${BOLD}%d MB${RESET}\n" "$total_mb" + printf " ${CYAN} Saved to${RESET} : ${BOLD}%s${RESET}\n" "$OUTPUT_DIR" + echo "" + [ "$DL_FAILED" -gt 0 ] && exit 1 + exit 0 +} + +main "$@" diff --git a/src/airgap/harness-airgap-images.sh b/src/airgap/harness-airgap-images.sh index 19ed91df..ecb5faf9 100755 --- a/src/airgap/harness-airgap-images.sh +++ b/src/airgap/harness-airgap-images.sh @@ -1,319 +1,695 @@ -#!/bin/bash +#!/usr/bin/env bash +# +# harness-airgap-images.sh +# +# Push airgap .tgz bundle(s) to a target container registry. +# +# Uses docker mode by default (docker load/tag/push). +# Use -s to request skopeo mode (daemonless copy). If skopeo/jq are missing, +# script falls back to docker mode automatically. +# +# Usage: +# ./harness-airgap-images.sh -r -d [options] +# ./harness-airgap-images.sh -r -f [options] +# + +set -euo pipefail + +# ───────────────────────────────────────────────────────────────────────────── +# Colours (auto-disabled when stdout is not a terminal) +# ───────────────────────────────────────────────────────────────────────────── +if [ -t 1 ] && command -v tput &>/dev/null && tput colors &>/dev/null 2>&1; then + RED=$'\033[0;31m'; YELLOW=$'\033[1;33m'; GREEN=$'\033[0;32m' + CYAN=$'\033[0;36m'; BOLD=$'\033[1m'; DIM=$'\033[2m'; RESET=$'\033[0m' +else + RED=''; YELLOW=''; GREEN=''; CYAN=''; BOLD=''; DIM=''; RESET='' +fi -# Default values +# ───────────────────────────────────────────────────────────────────────────── +# Logging +# ───────────────────────────────────────────────────────────────────────────── +log_info() { printf "${GREEN}[INFO]${RESET} %s %s\n" "$(date +%H:%M:%S)" "$*"; } +log_warn() { printf "${YELLOW}[WARN]${RESET} %s %s\n" "$(date +%H:%M:%S)" "$*" >&2; } +log_error() { printf "${RED}[ERROR]${RESET} %s %s\n" "$(date +%H:%M:%S)" "$*" >&2; } +log_debug() { [ "$debug" = true ] && printf "${DIM}[DEBUG] %s %s${RESET}\n" "$(date +%H:%M:%S)" "$*" >&2 || true; } +log_step() { printf "\n${BOLD}${CYAN}┌─ %s${RESET}\n" "$*"; } +log_done() { printf "${GREEN} ✓${RESET} %s\n" "$*"; } +log_fail() { printf "${RED} ✗ failed: %s${RESET}\n" "$*" >&2; } + +# ───────────────────────────────────────────────────────────────────────────── +# Defaults +# ───────────────────────────────────────────────────────────────────────────── registry="" tgz_file="" tgz_directory="" -success_count=0 -fail_count=0 -declare -a failed_images -declare -a missing_images -declare -a verified_images debug=false cleanup=false -error_occurred=false create_ecr=false +non_interactive=false +use_skopeo=false +request_skopeo=false + +success_count=0 +fail_count=0 +total_count=0 +declare -a failed_images=() +declare -a verified_images=() +START_TIME=$(date +%s) + +# ───────────────────────────────────────────────────────────────────────────── +# Usage +# ───────────────────────────────────────────────────────────────────────────── +show_help() { + cat < [-f | -d ] [options] + +${BOLD}Required:${RESET} + -r Target registry URL + e.g. artifactory.harness.internal/harness + +${BOLD}Source (one required):${RESET} + -f Single .tgz bundle file to process + -d Directory containing .tgz files (searched recursively) + +${BOLD}Options:${RESET} + -s, --use-skopeo Use skopeo mode (faster daemonless copy). Falls back to docker if unavailable + -c Clean up locally loaded Docker images after pushing (docker mode only) + -e Create ECR repository before push (requires: aws configure + AWS_REGION) + -n Non-interactive mode — skip optional prompts (e.g. ng-dashboard) + -D Enable debug output + -h Show this help + +${BOLD}Environment variables:${RESET} + AWS_REGION AWS region for ECR operations (required with -e) + ECR_NAMESPACE Optional namespace prefix for ECR repository names + +${BOLD}Examples:${RESET} + # Push all bundles in a directory + ./$(basename "$0") -r myregistry.example.com/harness -d ./bundles + + # Push a single bundle + ./$(basename "$0") -r myregistry.example.com/harness -f ./bundles/platform_images.tgz + + # Push using skopeo mode + ./$(basename "$0") -r myregistry.example.com/harness -f ./bundles/platform_images.tgz -s + + # Non-interactive, with cleanup, ECR auto-create + ./$(basename "$0") -r 123456789.dkr.ecr.us-east-1.amazonaws.com/harness \\ + -d ./bundles -n -c -e +EOF + exit 1 +} -# Function to display help message -function show_help { - echo "Usage: $0 -r [-f | -d ]" - echo " -r The Artifactory registry name (e.g., artifactory.harness.internal/platform-staging)." - echo " -f The name of the .tgz file to process (optional if -d is provided)." - echo " -d The directory containing .tgz files to process (optional if -f is provided)." - echo " -D Enable debug mode for detailed logging." - echo " -c Enable cleanup after script execution." - echo " -e Create ECR repository before pushing image. Please run `aws configure` before using this option and export ECR_NAMESPACE and AWS_REGION environment variables" - exit 1 +# ───────────────────────────────────────────────────────────────────────────── +# Helpers +# ───────────────────────────────────────────────────────────────────────────── + +# Extract image RepoTags from a bundle produced by create-airgap-bundle.sh. +# manifest.json is placed first in the archive so tar stops after the first entry. +# Requires jq (already a dependency of create-airgap-bundle.sh and validate_airgap.sh). +extract_image_names_from_tgz() { + local tgz="$1" + local raw="" + for mpath in manifest.json ./manifest.json; do + raw=$(tar -xzOf "$tgz" "$mpath" 2>/dev/null) && [ -n "$raw" ] && break || true + done + [ -z "$raw" ] && return 1 + printf '%s' "$raw" | jq -r '.[].RepoTags[]?' } -check_image_in_registry() { - local image=$1 - if docker manifest inspect "$image" > /dev/null 2>&1; then - debug_log "Image $image is already present in the registry." - return 0 # Image exists - else - debug_log "Image $image is not present in the registry." - return 1 # Image does not exist - fi +extract_image_names_from_tar() { + local tar_file="$1" + local raw="" + for mpath in manifest.json ./manifest.json; do + raw=$(tar -xOf "$tar_file" "$mpath" 2>/dev/null) && [ -n "$raw" ] && break || true + done + [ -z "$raw" ] && return 1 + printf '%s' "$raw" | jq -r '.[].RepoTags[]?' } -create_ecr_repository() { - local repository=$1 - local namespace=${ECR_NAMESPACE} - local awsregion=$AWS_REGION - - if [[ -z "$awsregion" ]]; then - echo "AWS_REGION is not set. Please export AWS_REGION before using -e option." - error_occurred=true - return 1 - fi - - local full_repo_name - if [[ -z "$namespace" ]]; then - full_repo_name="$repository" - else - full_repo_name="$namespace/$repository" - fi - - debug_log "Checking if repository '$full_repo_name' exists in region '$awsregion'..." - if aws ecr describe-repositories --repository-names "$full_repo_name" --region "$awsregion" > /dev/null 2>&1; then - debug_log "Repository $full_repo_name already exists." - else - debug_log "Repository $full_repo_name does not exist. Creating..." - if aws ecr create-repository --repository-name "$full_repo_name" --region "$awsregion"; then - echo "Successfully created repository: $full_repo_name" +# Strip registry host prefix if first path segment looks like a hostname. +# Example: docker.io/harness/foo:1.0 -> harness/foo:1.0 +strip_registry_prefix() { + local ref="$1" + local first="${ref%%/*}" + case "$first" in + *.*|*:*|localhost) printf '%s\n' "${ref#*/}" ;; + *) printf '%s\n' "$ref" ;; + esac +} + +# skopeo docker-archive transport expects an uncompressed tar archive. +# Our bundles are .tgz (gzip-compressed tar), so convert to a temporary .tar. +prepare_skopeo_archive() { + local tgz="$1" + local out_tar="$2" + + if command -v pigz >/dev/null 2>&1; then + pigz -dc "$tgz" >"$out_tar" else - echo "Failed to create repository: $full_repo_name" - error_occurred=true - return 1 + gzip -dc "$tgz" >"$out_tar" fi - fi } +create_ecr_repository() { + local repository="$1" + local region="${AWS_REGION:-}" + local namespace="${ECR_NAMESPACE:-}" -debug_log() { - if [ "$debug" = true ]; then - echo "[DEBUG] $1" - fi -} + if [ -z "$region" ]; then + log_error "AWS_REGION is not set — cannot create ECR repository" + return 1 + fi -cleanup_images() { - echo "Cleaning up Docker images..." - # Iterate through verified_images array to remove images from the local Docker environment - for image in "${verified_images[@]}"; do - # Extracting the image ID from the registry path and tag - local image_id=$(docker images -q "$image") - if [[ ! -z "$image_id" ]]; then - docker rmi "$image_id" || debug_log "Failed to remove Docker image: $image" + local full_repo="${namespace:+${namespace}/}${repository}" + log_debug "Ensuring ECR repository exists: ${full_repo}" + + if aws ecr describe-repositories --repository-names "$full_repo" --region "$region" >/dev/null 2>&1; then + log_debug "ECR repository already exists: ${full_repo}" else - debug_log "Image not found or already removed: $image" + if aws ecr create-repository --repository-name "$full_repo" --region "$region" >/dev/null; then + log_info "Created ECR repository: ${BOLD}${full_repo}${RESET}" + else + log_error "Failed to create ECR repository: ${full_repo}" + return 1 + fi fi - done } -# Parse command-line arguments -while getopts "hr:f:d:Dc:" opt; do - case "$opt" in - h) show_help ;; - r) registry="$OPTARG" ;; - f) tgz_file="$OPTARG" ;; - d) tgz_directory="$OPTARG" ;; - D) debug=true ;; - c) cleanup=true ;; - e) create_ecr=true ;; - *) show_help ;; - esac -done +cleanup_images() { + log_step "Cleaning up local Docker images" + local removed=0 + for image in "${verified_images[@]}"; do + local image_id + image_id=$(docker images -q "$image" 2>/dev/null || true) + if [ -n "$image_id" ]; then + docker rmi "$image_id" >/dev/null 2>&1 && removed=$((removed + 1)) || log_debug "Could not remove: ${image}" + fi + done + log_info "Removed ${removed} local image(s)" +} -# Check for mandatory options -if [ -z "$registry" ]; then - echo "Registry not specified!" - show_help - exit 1 -fi +# ───────────────────────────────────────────────────────────────────────────── +# process_tgz_file_skopeo +# +# Streams images directly from the .tgz archive to the registry using skopeo. +# No docker daemon, no local disk extraction, no docker load. +# ───────────────────────────────────────────────────────────────────────────── +process_tgz_file_skopeo() { + local file="$1" + local idx="$2" + local total="$3" + + local relative_path="${file#${tgz_directory}/}" + [ "$relative_path" = "$file" ] && relative_path=$(basename "$file") + + local file_size + file_size=$(du -sh "$file" 2>/dev/null | awk '{print $1}' || echo "?") + + printf "\n${BOLD}[%d/%d]${RESET} %s ${DIM}(%s)${RESET}\n" "$idx" "$total" "$relative_path" "$file_size" + + # docker-archive transport can not directly consume compressed .tgz. + local skopeo_tar + skopeo_tar=$(mktemp "${TMPDIR:-/tmp}/airgap-skopeo-archive.XXXXXX.tar") + log_info "Preparing skopeo archive (decompressing .tgz → .tar) …" + if ! prepare_skopeo_archive "$file" "$skopeo_tar"; then + log_fail "Failed to prepare skopeo archive for ${relative_path}" + rm -f "$skopeo_tar" + failed_images+=("$relative_path") + fail_count=$((fail_count + 1)) + return 1 + fi -if [ -z "$tgz_file" ] && [ -z "$tgz_directory" ]; then - echo "No .tgz file or directory specified!" - show_help - exit 1 -fi + log_info "Reading archive manifest …" + # Read RepoTags from already prepared .tar to avoid decompressing twice. + local images_raw + images_raw=$(extract_image_names_from_tar "$skopeo_tar") || true + if [ -z "$images_raw" ]; then + log_fail "Could not read image names from ${relative_path}" + rm -f "$skopeo_tar" + failed_images+=("$relative_path") + fail_count=$((fail_count + 1)) + return 1 + fi + + local images=() + while IFS= read -r _img; do + [ -n "$_img" ] && images+=("$_img") + done <&1) - - # Check the exit status of docker load - local exit_status=$? - if [ $exit_status -ne 0 ]; then - echo "Failed to load Docker image from $file" - ((fail_count++)) - error_occurred=true - return 1 - fi - - # Extract the image names and tags - while IFS= read -r line; do - echo "Loading: $line" - if [[ "$line" =~ Loaded\ image:\ (.+) ]]; then - local image_info="${BASH_REMATCH[1]}" - local service_name="${image_info%%:*}" - debug_log "Loaded image for service $service_name: $image_info" - - if ! check_image_in_registry "$registry/$image_info"; then if [ "$create_ecr" = true ]; then - create_ecr_repository "$service_name" + create_ecr_repository "$service_name" || true fi - debug_log "Tagging and pushing image $image_info to $registry" - if docker tag "$image_info" "$registry/$image_info" && docker push "$registry/$image_info"; then - echo "Successfully pushed $image_info to $registry" - ((success_count++)) - verified_images+=("$registry/$image_info") + + # Run skopeo with output going to a temp file so we capture both the + # exit code (in the parent shell) and the output for display. + # Also retry with stripped registry prefix to handle mixed tag formats. + local cmd_tmp cmd_rc=1 + local src_ref="$image_ref" + local alt_src_ref + alt_src_ref=$(strip_registry_prefix "$image_ref") + cmd_tmp=$(mktemp) + + skopeo copy --remove-signatures \ + "docker-archive:${skopeo_tar}:${src_ref}" \ + "docker://${target}" >"$cmd_tmp" 2>&1 || cmd_rc=$? + + if [ "$cmd_rc" -ne 0 ] && [ "$alt_src_ref" != "$src_ref" ]; then + printf " ${DIM}retrying source tag lookup as %s …${RESET}\n" "$alt_src_ref" + cmd_rc=0 + src_ref="$alt_src_ref" + skopeo copy --remove-signatures \ + "docker-archive:${skopeo_tar}:${alt_src_ref}" \ + "docker://${target}" >"$cmd_tmp" 2>&1 || cmd_rc=$? + fi + + while IFS= read -r line; do + printf " ${DIM}%s${RESET}\n" "$line" + done <"$cmd_tmp" + + if [ "$cmd_rc" -eq 0 ]; then + rm -f "$cmd_tmp" + log_done "Copied → ${target}" + success_count=$((success_count + 1)) + bundle_pushed=$((bundle_pushed + 1)) else - debug_log "Failed to tag or push image $image_info" - failed_images+=("$image_info") - ((fail_count++)) - error_occurred=true + # Some registries can still end up with the image present even when + # skopeo exits non-zero near the end of the operation. + local verify_tmp verify_rc=0 + verify_tmp=$(mktemp) + + skopeo inspect "docker://${target}" >"$verify_tmp" 2>&1 || verify_rc=$? + + if [ "$verify_rc" -eq 0 ]; then + rm -f "$verify_tmp" "$cmd_tmp" + log_info "Destination tag resolves, image push successful." + log_done "Copied (verified) → ${target}" + success_count=$((success_count + 1)) + bundle_pushed=$((bundle_pushed + 1)) + else + log_error "skopeo copy exited with code ${cmd_rc} for ${image_ref}" + while IFS= read -r vline; do + [ -n "$vline" ] && printf " ${RED}verify:${RESET} ${DIM}%s${RESET}\n" "$vline" + done <"$verify_tmp" + rm -f "$verify_tmp" "$cmd_tmp" + log_fail "${image_ref} → ${target}" + failed_images+=("$image_ref") + fail_count=$((fail_count + 1)) + bundle_failed=$((bundle_failed + 1)) + fi fi - else - verified_images+=("$image_info") - fi - fi - done <<< "$load_result" # to avoid subshell (success_count) + done + + rm -f "$skopeo_tar" + + # Per-file mini-summary + local parts="" + [ "$bundle_pushed" -gt 0 ] && parts="${parts} ${GREEN}${bundle_pushed} pushed${RESET}" + [ "$bundle_failed" -gt 0 ] && parts="${parts} ${RED}${bundle_failed} failed${RESET}" + [ -n "$parts" ] && printf " └─%s\n" "$parts" } -# Get DockerHub credentials and image details from arguments -read -p "Do you want to install ng-dashboard (yes/no)? " response -if [[ "$response" == "yes" ]]; then - read -p "Enter DockerHub username: " DOCKERHUB_USERNAME - read -sp "Enter DockerHub password: " DOCKERHUB_PASSWORD - echo - read -p "Enter release version: " RELEASE_VERSION - if [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_PASSWORD" ] && [ -n "$RELEASE_VERSION" ]; then - # Log in to DockerHub - echo "Logging in to DockerHub..." - echo $DOCKERHUB_PASSWORD | docker login -u $DOCKERHUB_USERNAME --password-stdin - - # Check if login was successful - if [ $? -ne 0 ]; then - echo "Docker login failed. Please check your credentials." - exit 1 +# ───────────────────────────────────────────────────────────────────────────── +# process_tgz_file_docker +# +# Fallback: docker load → docker tag → docker push (sequential, with output). +# ───────────────────────────────────────────────────────────────────────────── +process_tgz_file_docker() { + local file="$1" + local idx="$2" + local total="$3" + + local relative_path="${file#${tgz_directory}/}" + [ "$relative_path" = "$file" ] && relative_path=$(basename "$file") + + local file_size + file_size=$(du -sh "$file" 2>/dev/null | awk '{print $1}' || echo "?") + + printf "\n${BOLD}[%d/%d]${RESET} %s ${DIM}(%s)${RESET}\n" "$idx" "$total" "$relative_path" "$file_size" + + # ── Disk space check ───────────────────────────────────────────────────── + local required_kb available_kb + required_kb=$(du -ks "$file" | awk '{print $1}') + available_kb=$(df -Pk . | awk 'NR==2 {print $4}') + if [ "$required_kb" -gt "$available_kb" ]; then + log_error "Insufficient disk space to load ${relative_path}" + log_error " Required : $(( required_kb / 1024 )) MB" + log_error " Available: $(( available_kb / 1024 )) MB" + failed_images+=("$relative_path (disk space)") + fail_count=$((fail_count + 1)) + return 1 fi - # Check if harness-${RELEASE_VERSION}.tgz already exists - if [ -f "harness-${RELEASE_VERSION}.tgz" ]; then - echo "Removing existing harness-${RELEASE_VERSION}.tgz..." - rm "harness-${RELEASE_VERSION}.tgz" - fi - - # Check if harness-${RELEASE_VERSION} already exists - if [ -d "harness-${RELEASE_VERSION}" ]; then - echo "Deleting existing target folder: harness-${RELEASE_VERSION}" - rm -rf "harness-${RELEASE_VERSION}" + # ── docker load ────────────────────────────────────────────────────────── + log_info "Loading ${DIM}${relative_path}${RESET} …" + local load_output load_rc=0 + load_output=$(docker load -i "$file" 2>&1) || load_rc=$? + + if [ "$load_rc" -ne 0 ]; then + log_fail "docker load failed for ${relative_path}" + echo "$load_output" | while IFS= read -r line; do + printf " ${RED}│${RESET} ${DIM}%s${RESET}\n" "$line" + done + failed_images+=("$relative_path") + fail_count=$((fail_count + 1)) + return 1 fi - # Download the harness-.tgz file - DOWNLOAD_URL="https://github.com/harness/helm-charts/releases/download/harness-${RELEASE_VERSION}/harness-${RELEASE_VERSION}.tgz" - echo "Downloading $DOWNLOAD_URL..." - curl -L -o "harness-${RELEASE_VERSION}.tgz" "$DOWNLOAD_URL" - # Check if the download was successful - if [ $? -ne 0 ]; then - echo "Failed to download harness-${RELEASE_VERSION}.tgz" - exit 1 - fi - - echo "Successfully downloaded harness-${RELEASE_VERSION}.tgz" - - # Extract the contents of the archive - echo "Extracting harness-${RELEASE_VERSION}.tgz..." - mkdir "harness-${RELEASE_VERSION}" - tar -xzvf "harness-${RELEASE_VERSION}.tgz" -C "harness-${RELEASE_VERSION}" - - #Fetching looker image tag - echo "Searching for 'looker' in images.txt..." - IMAGE_TAG=$(grep "looker" "harness-${RELEASE_VERSION}/harness/images.txt") - - if [ -z "$IMAGE_TAG" ]; then - echo "Image tag for $IMAGE_NAME not found in images.txt" - exit 1 + local loaded_count + loaded_count=$(echo "$load_output" | grep -c "^Loaded image" 2>/dev/null || echo 0) + log_info "Loaded ${BOLD}${loaded_count}${RESET} image(s)" + + # ── docker tag + docker push (sequential, visible output) ──────────────── + local bundle_pushed=0 bundle_failed=0 img_idx=0 + + while IFS= read -r line; do + local image_ref="" + case "$line" in + "Loaded image: "*) image_ref="${line#Loaded image: }" ;; + "Loaded image ID: "*) image_ref="${line#Loaded image ID: }" ;; + esac + [ -z "$image_ref" ] && continue + + img_idx=$((img_idx + 1)) + local target="${registry}/${image_ref}" + local service_name="${image_ref%%:*}" + service_name="${service_name##*/}" + + printf " ${DIM}[%d/%d]${RESET} %s\n" "$img_idx" "$loaded_count" "$image_ref" + + if [ "$create_ecr" = true ]; then + create_ecr_repository "$service_name" || true + fi + + # Tag + printf " ${DIM}tagging → %s${RESET}\n" "$target" + if ! docker tag "$image_ref" "$target" 2>/dev/null; then + log_fail "docker tag failed: ${image_ref} → ${target}" + failed_images+=("$image_ref") + fail_count=$((fail_count + 1)) + bundle_failed=$((bundle_failed + 1)) + continue + fi + + # Push — capture to temp file so exit code is in the parent shell. + printf " ${DIM}pushing → %s …${RESET}\n" "$target" + local push_tmp push_rc=0 + push_tmp=$(mktemp) + docker push "$target" >"$push_tmp" 2>&1 || push_rc=$? + while IFS= read -r pline; do + printf " ${DIM}%s${RESET}\n" "$pline" + done <"$push_tmp" + rm -f "$push_tmp" + + if [ "$push_rc" -eq 0 ]; then + log_done "Pushed → ${target}" + success_count=$((success_count + 1)) + bundle_pushed=$((bundle_pushed + 1)) + verified_images+=("$target") + else + log_fail "${image_ref} → ${target}" + failed_images+=("$image_ref") + fail_count=$((fail_count + 1)) + bundle_failed=$((bundle_failed + 1)) + fi + done <<< "$load_output" + + # Per-file mini-summary + local parts="" + [ "$bundle_pushed" -gt 0 ] && parts="${parts} ${GREEN}${bundle_pushed} pushed${RESET}" + [ "$bundle_failed" -gt 0 ] && parts="${parts} ${RED}${bundle_failed} failed${RESET}" + [ -n "$parts" ] && printf " └─%s\n" "$parts" +} + +# ───────────────────────────────────────────────────────────────────────────── +# process_looker — optional ng-dashboard / Looker image (interactive only) +# ───────────────────────────────────────────────────────────────────────────── +process_looker() { + if [ "$non_interactive" = true ]; then + log_debug "Skipping ng-dashboard (non-interactive mode)" + return fi - # Pull the Docker image from the private repository - echo "Pulling image $IMAGE_TAG..." - docker pull $IMAGE_TAG - - #Push looker image to private registery - debug_log "Tagging and pushing image $IMAGE_TAG to $registry" - looker_image=$(echo "$IMAGE_TAG" | sed 's/^[^\/]*\///') - if ! check_image_in_registry "$registry/$looker_image"; then - if docker tag "$IMAGE_TAG" "$registry/$looker_image" && docker push "$registry/$looker_image"; then - echo "Successfully pushed $looker_image to $registry" - ((success_count++)) - verified_images+=("$registry/$looker_image") - else - debug_log "Failed to tag or push image $looker_image" - failed_images+=("$looker_image") - ((fail_count++)) - error_occurred=true - fi - else - verified_images+=("$looker_image") + printf "\n${BOLD}Optional: ng-dashboard (Looker)${RESET}\n" + read -rp " Do you want to install ng-dashboard? [yes/no]: " response + [[ "$response" != "yes" ]] && return + + read -rp " DockerHub username: " DOCKERHUB_USERNAME + read -rsp " DockerHub password: " DOCKERHUB_PASSWORD + echo + read -rp " Harness release version (e.g. 0.37.0): " RELEASE_VERSION + + if [ -z "$DOCKERHUB_USERNAME" ] || [ -z "$DOCKERHUB_PASSWORD" ] || [ -z "$RELEASE_VERSION" ]; then + log_warn "Incomplete credentials or version — skipping ng-dashboard" + return fi - # Check if pull was successful + echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin if [ $? -ne 0 ]; then - echo "Failed to pull the Docker image. Please check the repository and image details." - exit 1 + log_error "DockerHub login failed — skipping ng-dashboard" + return fi - echo "Successfully pulled the Docker image: $DOCKERHUB_REPOSITORY/$IMAGE" + local tgz_name="harness-${RELEASE_VERSION}.tgz" + local dir_name="harness-${RELEASE_VERSION}" + rm -rf "$tgz_name" "$dir_name" - # Optionally, log out from DockerHub - echo "Logging out from DockerHub..." - docker logout + local url="https://github.com/harness/helm-charts/releases/download/harness-${RELEASE_VERSION}/${tgz_name}" + log_info "Downloading ${url}" + if ! curl -fSL -o "$tgz_name" "$url"; then + log_error "Failed to download ${tgz_name}" + docker logout + return + fi - #Cleaning up the folders - if [ -f "harness-${RELEASE_VERSION}.tgz" ]; then - echo "Removing existing harness-${RELEASE_VERSION}.tgz..." - rm "harness-${RELEASE_VERSION}.tgz" + mkdir "$dir_name" + tar -xzf "$tgz_name" -C "$dir_name" + + local looker_tag + looker_tag=$(grep "looker" "${dir_name}/harness/images.txt" 2>/dev/null || true) + if [ -z "$looker_tag" ]; then + log_warn "Looker image not found in images.txt — skipping" + rm -rf "$tgz_name" "$dir_name" + docker logout + return fi - if [ -d "harness-${RELEASE_VERSION}" ]; then - echo "Deleting existing target folder: harness-${RELEASE_VERSION}" - rm -rf "harness-${RELEASE_VERSION}" + log_info "Pulling ${looker_tag}" + docker pull "$looker_tag" + + local looker_image + looker_image=$(echo "$looker_tag" | sed 's|^[^/]*/||') + local looker_target="${registry}/${looker_image}" + + if docker tag "$looker_tag" "$looker_target" && docker push "$looker_target"; then + log_done "Pushed Looker → ${looker_target}" + success_count=$((success_count + 1)) + verified_images+=("$looker_target") + else + log_fail "Failed to push Looker: ${looker_image}" + failed_images+=("$looker_image") + fail_count=$((fail_count + 1)) fi -else - echo "DOCKERHUB_USERNAME, DOCKERHUB_PASSWORD & RELEASE_VERSION are not set. Will not pull looker image" + + rm -rf "$tgz_name" "$dir_name" + docker logout +} + +# ───────────────────────────────────────────────────────────────────────────── +# Argument parsing +# ───────────────────────────────────────────────────────────────────────────── +# Long option compatibility shim for getopts. +# Supports: +# --use-skopeo -> -s +# --help -> -h +if [ "$#" -gt 0 ]; then + normalized_args=() + for arg in "$@"; do + case "$arg" in + --use-skopeo) normalized_args+=("-s") ;; + --help) normalized_args+=("-h") ;; + *) normalized_args+=("$arg") ;; + esac + done + set -- "${normalized_args[@]}" fi + +while getopts "hr:f:d:Dcsne" opt; do + case "$opt" in + h) show_help ;; + r) registry="$OPTARG" ;; + f) tgz_file="$OPTARG" ;; + d) tgz_directory="$OPTARG" ;; + s) request_skopeo=true ;; + D) debug=true ;; + c) cleanup=true ;; + n) non_interactive=true ;; + e) create_ecr=true ;; + *) show_help ;; + esac +done + +# ───────────────────────────────────────────────────────────────────────────── +# Validation +# ───────────────────────────────────────────────────────────────────────────── +errors=0 +if [ -z "$registry" ]; then + log_error "Registry not specified (-r)" + errors=$((errors + 1)) +fi +if [ -z "$tgz_file" ] && [ -z "$tgz_directory" ]; then + log_error "No source specified — use -f or -d " + errors=$((errors + 1)) +fi +if [ -n "$tgz_file" ] && [ -n "$tgz_directory" ]; then + log_error "-f and -d are mutually exclusive" + errors=$((errors + 1)) +fi +if [ -n "$tgz_file" ] && [ ! -f "$tgz_file" ]; then + log_error "File not found: ${tgz_file}" + errors=$((errors + 1)) +fi +if [ -n "$tgz_directory" ] && [ ! -d "$tgz_directory" ]; then + log_error "Directory not found: ${tgz_directory}" + errors=$((errors + 1)) +fi +if [ "$errors" -gt 0 ]; then + printf "\nRun ${BOLD}$(basename "$0") -h${RESET} for usage.\n" >&2 + exit 1 fi -# Process the specified .tgz file or directory -if [[ -n "$tgz_file" ]]; then - process_tgz_file "$tgz_file" -elif [[ -n "$tgz_directory" ]]; then - for file in "$tgz_directory"/*.tgz; do - process_tgz_file "$file" - done +# ───────────────────────────────────────────────────────────────────────────── +# Detect push method +# ───────────────────────────────────────────────────────────────────────────── +if [ "$request_skopeo" = true ]; then + if command -v skopeo &>/dev/null && command -v jq &>/dev/null; then + use_skopeo=true + else + use_skopeo=false + log_warn "skopeo mode requested (-s), but skopeo/jq not found — falling back to docker mode" + fi fi -# Print statistics -if (( success_count > 0 || fail_count > 0 )); then - echo "Total successful pushes: $success_count" - echo "Total failed pushes: $fail_count" -elif [[ "$error_occurred" = false ]]; then - echo "No new images were pushed." +# ───────────────────────────────────────────────────────────────────────────── +# Banner +# ───────────────────────────────────────────────────────────────────────────── +echo "" +printf "${BOLD}${CYAN}╔══════════════════════════════════════════════════════╗${RESET}\n" +printf "${BOLD}${CYAN}║ Harness Airgap Image Pusher ║${RESET}\n" +printf "${BOLD}${CYAN}╚══════════════════════════════════════════════════════╝${RESET}\n" +echo "" +log_info "Registry : ${BOLD}${registry}${RESET}" +[ -n "$tgz_file" ] && log_info "Source : ${BOLD}${tgz_file}${RESET}" +[ -n "$tgz_directory" ] && log_info "Source : ${BOLD}${tgz_directory}${RESET}" +if [ "$use_skopeo" = true ]; then + log_info "Method : ${BOLD}skopeo${RESET} ${DIM}(copying, no docker load)${RESET}" else - echo "Errors occurred during processing. Please review the error messages above." + log_info "Method : ${BOLD}docker${RESET} ${DIM}(load → tag → push)${RESET}" + if [ "$request_skopeo" = true ]; then + log_info "Install skopeo + jq and re-run with --use-skopeo for faster daemonless uploads" + else + log_info "Tip : Install skopeo + jq and use ${BOLD}--use-skopeo${RESET} for faster daemonless uploads" + fi fi - -if [ ${#verified_images[@]} -gt 0 ]; then - debug_log "Verified images in the registry:" - for image in "${verified_images[@]}"; do - debug_log " - $image" - done +[ "$create_ecr" = true ] && log_info "ECR auto-create enabled (region: ${AWS_REGION:-not set})" +[ "$cleanup" = true ] && log_info "Local image cleanup enabled after push" +[ "$non_interactive" = true ] && log_info "Non-interactive mode" +echo "" + +# ───────────────────────────────────────────────────────────────────────────── +# Optional Looker step +# ───────────────────────────────────────────────────────────────────────────── +process_looker + +# ───────────────────────────────────────────────────────────────────────────── +# Collect files +# ───────────────────────────────────────────────────────────────────────────── +declare -a tgz_files=() + +if [ -n "$tgz_file" ]; then + tgz_files=("$tgz_file") +else + while IFS= read -r _f; do + [ -n "$_f" ] && tgz_files+=("$_f") + done < <(find "$tgz_directory" -name "*.tgz" -type f | sort) fi -if [ ${#failed_images[@]} -gt 0 ]; then - echo "Failed to process images:" - for image in "${failed_images[@]}"; do - debug_log " - $image" - done +total_count=${#tgz_files[@]} + +if [ "$total_count" -eq 0 ]; then + log_error "No .tgz files found${tgz_directory:+ in ${tgz_directory}}" + exit 1 fi -if [ "$cleanup" = true ]; then +log_step "Processing ${total_count} bundle(s)" +[ "$use_skopeo" = true ] && log_info "Runtime mode: ${BOLD}skopeo${RESET}" +[ "$use_skopeo" = true ] || log_info "Runtime mode: ${BOLD}docker${RESET}" +[ -n "$tgz_directory" ] && log_info "Found ${total_count} .tgz file(s) in ${BOLD}${tgz_directory}${RESET}" + +# ───────────────────────────────────────────────────────────────────────────── +# Main processing loop +# ───────────────────────────────────────────────────────────────────────────── +for idx in "${!tgz_files[@]}"; do + if [ "$use_skopeo" = true ]; then + process_tgz_file_skopeo "${tgz_files[$idx]}" "$((idx + 1))" "$total_count" + else + process_tgz_file_docker "${tgz_files[$idx]}" "$((idx + 1))" "$total_count" + fi +done + +# ───────────────────────────────────────────────────────────────────────────── +# Cleanup (docker mode only) +# ───────────────────────────────────────────────────────────────────────────── +if [ "$cleanup" = true ] && [ "$use_skopeo" != true ]; then cleanup_images fi + +# ───────────────────────────────────────────────────────────────────────────── +# Summary +# ───────────────────────────────────────────────────────────────────────────── +END_TIME=$(date +%s) +ELAPSED=$(( END_TIME - START_TIME )) +ELAPSED_FMT=$(printf '%dm %ds' $(( ELAPSED / 60 )) $(( ELAPSED % 60 ))) + +echo "" +printf "${BOLD}${CYAN}╔══════════════════════════════════════════════════════╗${RESET}\n" +printf "${BOLD}${CYAN}║ Summary ║${RESET}\n" +printf "${BOLD}${CYAN}╚══════════════════════════════════════════════════════╝${RESET}\n" +echo "" +log_info "Bundles processed : ${BOLD}${total_count}${RESET}" +log_info "Images pushed : ${BOLD}${GREEN}${success_count}${RESET}" +if [ "$fail_count" -gt 0 ]; then + log_info "Images failed : ${BOLD}${RED}${fail_count}${RESET}" +else + log_info "Images failed : ${BOLD}${fail_count}${RESET}" +fi +log_info "Elapsed : ${BOLD}${ELAPSED_FMT}${RESET}" +echo "" + +if [ "${#failed_images[@]}" -gt 0 ]; then + printf "${RED}${BOLD}Failed images:${RESET}\n" + for img in "${failed_images[@]}"; do + printf " ${RED}✗${RESET} %s\n" "$img" + done + echo "" + exit 1 +fi diff --git a/src/airgap/selection.conf.example b/src/airgap/selection.conf.example new file mode 100644 index 00000000..6b0b21b8 --- /dev/null +++ b/src/airgap/selection.conf.example @@ -0,0 +1,22 @@ +# Harness Airgap Bundle Selection File +# ────────────────────────────────────────────────────────────────────────────── +# Use with: ./download-airgap-bundles.sh --version --output-dir ./bundles \ +# --selection-file selection.conf +# +# Tip: run --generate-selection-file to build this file interactively, or +# run --list to see all available names. +# Lines starting with '#' are ignored. +# ────────────────────────────────────────────────────────────────────────────── + +# List of bundles to download (comma-separated). +# This includes Modules, Plugins, Agents, and Scanners. +# +# - Modules (e.g. platform, ci): Dependencies are auto-resolved. +# - Plugins (e.g. ci-plugins): Must be listed explicitly if needed. +# - Agents (e.g. delegate, upgrader): Must be listed explicitly. +# - Scanners (e.g. grype-job-runner): Must be listed explicitly. +# +# Use 'all' to download everything available in the manifest. +# +# Example: bundles=platform,ci,ci-plugins,delegate,delegate-fips +bundles= diff --git a/src/airgap/upload_all_bundles.sh b/src/airgap/upload_all_bundles.sh index c85d21ef..a5d770b7 100755 --- a/src/airgap/upload_all_bundles.sh +++ b/src/airgap/upload_all_bundles.sh @@ -1,47 +1,120 @@ #!/bin/bash +set -e -upload_to_bucket() { - export GOOGLE_APPLICATION_CREDENTIALS=$1 - gsutil_command=("gsutil" "cp" "$2" "$3") +log_info() { echo "[INFO] $(date +%H:%M:%S) $*"; } +log_warn() { echo "[WARN] $(date +%H:%M:%S) $*" >&2; } +log_error() { echo "[ERROR] $(date +%H:%M:%S) $*" >&2; } +log_debug() { [ "${DEBUG:-false}" = "true" ] && echo "[DEBUG] $(date +%H:%M:%S) $*"; } - if ! "${gsutil_command[@]}"; then - echo "An error occurred while uploading the file: $?" +usage() { + echo "Usage: $0 " + echo "" + echo " service_account_file Path to GCP service account JSON key file" + echo " release_number Release version (e.g., 1.0.0)" + echo "" + echo "Uploads the bundle output directory to gs://smp-airgap-bundles//" + echo "Uploads bundle-manifest.yaml and images.txt." + exit 1 +} + +if [ $# -lt 2 ]; then + log_error "Missing required arguments" + usage +fi + +SERVICE_ACCOUNT_FILE="$1" +RELEASE_NUMBER="$2" +BUCKET="gs://smp-airgap-bundles" +RELEASE_PATH="${BUCKET}/${RELEASE_NUMBER}" + +if [ ! -f "$SERVICE_ACCOUNT_FILE" ]; then + log_error "Service account file not found: $SERVICE_ACCOUNT_FILE" + exit 1 +fi + +# Use Application Default Credentials - ensures gsutil and gcloud storage both use this service account +export GOOGLE_APPLICATION_CREDENTIALS="${SERVICE_ACCOUNT_FILE}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SRC_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" + +# Find output directory: look for platform/ or *.tgz in current dir +OUTPUT_DIR="" +if [ -d "platform" ] || [ -d "ci" ]; then + OUTPUT_DIR="$(pwd)" +elif [ -d "output/platform" ] || [ -d "output/ci" ]; then + OUTPUT_DIR="$(pwd)/output" +else + # Look for any .tgz in current dir or subdirs + if find . -maxdepth 2 -name "*.tgz" -type f 2>/dev/null | head -1 | grep -q .; then + OUTPUT_DIR="$(pwd)" + else + log_error "Could not find output directory (look for platform/, ci/, or *.tgz)" exit 1 fi +fi - echo "File $2 uploaded to $3." -} +log_info "Using output directory: ${OUTPUT_DIR}" -# Define the bucket path -bucket_path="gs://smp-airgap-bundles" +# Find bundle-manifest.yaml and images.txt +BUNDLE_MANIFEST="" +IMAGES_TXT="" -# Parse command line arguments -if [[ $# -ne 2 ]]; then - echo "Usage: $0 " - exit 1 +if [ -f "${OUTPUT_DIR}/bundle-manifest.yaml" ]; then + BUNDLE_MANIFEST="${OUTPUT_DIR}/bundle-manifest.yaml" +elif [ -f "${SRC_DIR}/bundle-manifest.yaml" ]; then + BUNDLE_MANIFEST="${SRC_DIR}/bundle-manifest.yaml" fi -service_account_file=$1 -release_number=$2 +if [ -f "${OUTPUT_DIR}/images.txt" ]; then + IMAGES_TXT="${OUTPUT_DIR}/images.txt" +elif [ -f "${SRC_DIR}/harness/images.txt" ]; then + IMAGES_TXT="${SRC_DIR}/harness/images.txt" +elif [ -f "${OUTPUT_DIR}/harness/images.txt" ]; then + IMAGES_TXT="${OUTPUT_DIR}/harness/images.txt" +fi -# Array of files to upload -for moduleImageZipFile in $MODULE_IMAGE_ZIP_FILES; do - files+=("$moduleImageZipFile") -done +if [ -z "$BUNDLE_MANIFEST" ]; then + log_error "bundle-manifest.yaml not found" + exit 1 +fi -if [ ${#files[@]} -le 2 ]; then # validation with 2 to make sure multiple elements are in list - echo "Error: No module image zip files provided. Files: ${files[@]}" >&2 +if [ -z "$IMAGES_TXT" ]; then + log_error "images.txt not found" exit 1 fi -# Create an empty file and upload it to the destination bucket path -touch empty_file -gsutil cp empty_file "$bucket_path/$release_number/" +log_info "Using service account: ${SERVICE_ACCOUNT_FILE} (GOOGLE_APPLICATION_CREDENTIALS set)" +gcloud auth activate-service-account --key-file="${SERVICE_ACCOUNT_FILE}" --quiet + +# Create release path first (like old script - avoids bucket-list permission requirements) +log_info "Creating release path: ${RELEASE_PATH}/" +touch /tmp/upload_empty +gsutil cp /tmp/upload_empty "${RELEASE_PATH}/.keep" +gsutil rm "${RELEASE_PATH}/.keep" 2>/dev/null || true +rm -f /tmp/upload_empty -# Iterate over the files and upload each one -for file in "${files[@]}"; do - upload_to_bucket "$service_account_file" "$file" "$bucket_path/$release_number/$file" +# Copy bundle hierarchy using gsutil cp (no rsync/ls - avoids storage.buckets.get) +# Build temp dir excluding images_internal.txt +UPLOAD_DIR=$(mktemp -d) +trap "rm -rf ${UPLOAD_DIR}" EXIT +for item in "${OUTPUT_DIR}"/*; do + [ -e "$item" ] || continue + [ "$(basename "$item")" = "images_internal.txt" ] && continue + cp -r "$item" "${UPLOAD_DIR}/" done -# Remove the empty file from the release number directory -gsutil rm "$bucket_path/$release_number/empty_file" +log_info "Uploading bundle hierarchy to ${RELEASE_PATH}/" +gsutil -m cp -r "${UPLOAD_DIR}/." "${RELEASE_PATH}/" + +# Upload manifest and images.txt to release folder root +log_info "Uploading bundle-manifest.yaml and images.txt to ${RELEASE_PATH}/" +gsutil cp "${BUNDLE_MANIFEST}" "${RELEASE_PATH}/bundle-manifest.yaml" +gsutil cp "${IMAGES_TXT}" "${RELEASE_PATH}/images.txt" + +echo "" +log_info "=== UPLOAD SUMMARY ===" +log_info "Bucket path: ${RELEASE_PATH}/" +log_info "bundle-manifest.yaml: ${RELEASE_PATH}/bundle-manifest.yaml" +log_info "images.txt: ${RELEASE_PATH}/images.txt" +log_info "Upload complete." diff --git a/src/airgap/validate_airgap.sh b/src/airgap/validate_airgap.sh old mode 100644 new mode 100755 index 21473a3f..34ec39ac --- a/src/airgap/validate_airgap.sh +++ b/src/airgap/validate_airgap.sh @@ -1,57 +1,341 @@ #!/bin/bash +set -e -for module_name in $MODULES; do - MODULE_NAMES+=("$module_name") +log_info() { echo "[INFO] $(date +%H:%M:%S) $*"; } +log_warn() { echo "[WARN] $(date +%H:%M:%S) $*" >&2; } +log_error() { echo "[ERROR] $(date +%H:%M:%S) $*" >&2; } + +usage() { + echo "Usage: $0 --internal-file [--output-dir ]" + echo "" + echo " --internal-file Path to images_internal.txt" + echo " --output-dir Bundle output directory (default: current directory)" + echo "" + echo "Validates airgap .tgz bundles against expected images from images_internal.txt." + echo "Exits with code 1 if any validation errors are found." + echo "" + echo "Environment:" + echo " VALIDATE_PARALLEL=N Run up to N validations in parallel (default: 8)" + exit 1 +} + +INTERNAL_FILE="" +OUTPUT_DIR="." + +while [ $# -gt 0 ]; do + case "$1" in + --internal-file) INTERNAL_FILE="$2"; shift 2 ;; + --output-dir) OUTPUT_DIR="$2"; shift 2 ;; + -h|--help) usage ;; + *) log_error "Unknown option: $1"; usage ;; + esac done -if [ ${#MODULE_NAMES[@]} -le 2 ]; then # validation with 2 to make sure multiple elements are in list - echo "Error: No module names provided. List: ${MODULE_NAMES[@]}" >&2 +if [ -z "$INTERNAL_FILE" ]; then + log_error "Missing required --internal-file" + usage +fi + +if [ ! -f "$INTERNAL_FILE" ]; then + log_error "File not found: $INTERNAL_FILE" exit 1 fi -abort() { - echo "Error: $1" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" 2>/dev/null && pwd)" || { + log_error "Output directory not found or not accessible: $OUTPUT_DIR" + exit 1 +} + +# Portable extraction of @module=, @type=, @path=, @image= +# Uses || true because grep returns 1 when no match; with set -e that would exit the script +extract_attr() { + echo "$1" | grep -oE "${2}[^ ]+" 2>/dev/null | cut -d= -f2- | head -1 || true +} + +get_repo_tags_from_tgz() { + local tgz="$1" + if [ ! -f "$tgz" ]; then + return 1 + fi + local raw_tags tar_out manifest_path + + # Try manifest.json first (fast when it's first in archive - create-airgap-bundle puts it there) + for manifest_path in "manifest.json" "./manifest.json"; do + tar_out=$(tar -xzOf "$tgz" "$manifest_path" 2>&1) || true + raw_tags=$(echo "$tar_out" | jq -r '.[].RepoTags[]? // empty' 2>/dev/null || true) + [ -n "$raw_tags" ] && break + done + + # Fallback: find manifest.json from archive listing (handles older bundles, varying tar implementations) + if [ -z "$raw_tags" ]; then + manifest_path=$(tar -tzf "$tgz" 2>/dev/null | grep -E 'manifest\.json$' | head -1) + if [ -n "$manifest_path" ]; then + tar_out=$(tar -xzOf "$tgz" "$manifest_path" 2>&1) || true + raw_tags=$(echo "$tar_out" | jq -r '.[].RepoTags[]? // empty' 2>/dev/null || true) + fi + fi + + if [ -z "$raw_tags" ] && [ -n "$tar_out" ]; then + log_info "tar/jq failed for ${tgz}: $(echo "$tar_out" | head -5)" + elif [ -z "$raw_tags" ]; then + log_info "Archive contents of ${tgz}: $(tar -tzf "$tgz" 2>/dev/null | head -10)" + fi + + while IFS= read -r tag; do + [ -n "$tag" ] && strip_registry "$tag" || true + done <<< "$raw_tags" +} + +# Normalise an image ref by stripping the registry hostname prefix if present. +# Bundles are written in docker-save format (harness/foo:tag, defaultbackend-amd64:tag). +# Expected refs from images_internal.txt may include registry (docker.io/, registry.k8s.io/). +# We strip so both sides compare as org/name:tag or name:tag. +# +strip_registry() { + local ref="$1" + local first="${ref%%/*}" + if [[ "$first" =~ [.:] ]]; then + echo "${ref#*/}" + else + echo "${ref}" + fi +} + +# Check if jq is available +if ! command -v jq &>/dev/null; then + log_error "jq is required for validation. Install with: brew install jq (macOS) or apt-get install jq (Linux)" exit 1 +fi + +VALIDATE_PARALLEL="${VALIDATE_PARALLEL:-8}" +[ "$VALIDATE_PARALLEL" -lt 1 ] && VALIDATE_PARALLEL=1 +ERRORS=() +ERRORS_FILE="" + +# Outputs errors to stdout (one per line). Caller collects into ERRORS or file. +validate_combined_bundle() { + local module="$1" + local path="$2" + shift 2 + local expected_images=("$@") + + local bundle_name + bundle_name=$(echo "${module}" | tr '/' '_') + local tgz_path="${OUTPUT_DIR}/${path}/${bundle_name}_images.tgz" + + if [ ! -f "$tgz_path" ]; then + log_info "Combined bundle path checked: ${tgz_path}" + echo "Combined bundle not found: ${tgz_path}" + return + fi + + if [ ${#expected_images[@]} -eq 0 ]; then + log_warn "Combined bundle ${module}: no expected images in manifest (section may be misconfigured)" + fi + + local actual_tags + actual_tags=$(get_repo_tags_from_tgz "$tgz_path") + if [ -z "$actual_tags" ]; then + echo "Could not extract RepoTags from ${tgz_path} (invalid or empty manifest)" + return + fi + + local expected_norm + expected_norm=$(for exp in "${expected_images[@]}"; do strip_registry "$exp"; done | sort -u) + local missing_norm + missing_norm=$(comm -23 <(echo "$expected_norm") <(echo "$actual_tags" | sort -u) 2>/dev/null) || true + + if [ -n "$missing_norm" ]; then + log_info "Combined ${module}: expected=$(echo "$expected_norm" | tr '\n' ' '), actual=$(echo "$actual_tags" | tr '\n' ' ')" + while IFS= read -r m; do + [ -n "$m" ] && echo "Combined bundle ${module}: missing expected image: ${m}" + done <<< "$missing_norm" + fi } -for module_name in "${MODULE_NAMES[@]}"; do - echo "Validating ${module_name}..." - TGZ_FILE="${module_name}_images.tgz" - TXT_FILE="${module_name}_images.txt" +# Outputs errors to stdout (one per line). Caller collects into ERRORS or file. +validate_single_image() { + local path="$1" + local image_name="$2" + shift 2 + local expected_tags=("$@") + + if [ ${#expected_tags[@]} -eq 0 ]; then + log_warn "Single bundle ${path}/${image_name}: no expected tags in manifest (section may be misconfigured)" + return + fi - [[ ! -f "${TGZ_FILE}" ]] && abort "${TGZ_FILE} not found!" + local tgz_path="${OUTPUT_DIR}/${path}/${image_name}.tgz" - # Extract the image tags and format them - IMAGES_LIST=$(tar -xzOf "${TGZ_FILE}" manifest.json | jq -r '.[].RepoTags | join("\n")') + if [ ! -f "$tgz_path" ]; then + log_info "Single bundle path checked: ${tgz_path}" + echo "Single bundle not found: ${tgz_path}" + return + fi - [[ ! -f "${TXT_FILE}" ]] && abort "${TXT_FILE} not found!" + local actual_tags + actual_tags=$(get_repo_tags_from_tgz "$tgz_path") + if [ -z "$actual_tags" ]; then + echo "Could not extract RepoTags from ${tgz_path} (invalid or empty manifest)" + return + fi - echo "Matching images for ${TGZ_FILE} and ${TXT_FILE}:" + local expected_norm + expected_norm=$(for exp in "${expected_tags[@]}"; do strip_registry "$exp"; done | sort -u) + local missing_norm + missing_norm=$(comm -23 <(echo "$expected_norm") <(echo "$actual_tags" | sort -u) 2>/dev/null) || true - mismatched=false + if [ -n "$missing_norm" ]; then + log_info "Single ${path}/${image_name}: expected=$(echo "$expected_norm" | tr '\n' ' '), actual=$(echo "$actual_tags" | tr '\n' ' ')" + while IFS= read -r m; do + [ -n "$m" ] && echo "Single bundle ${path}/${image_name}: missing expected tag: ${m}" + done <<< "$missing_norm" + fi +} - # Count the number of images in .tgz and .txt - num_images_tgz=$(echo "$IMAGES_LIST" | wc -l) - num_images_txt=$(wc -l < "${TXT_FILE}") +# Collect validation tasks +declare -a TASK_TYPE TASK_MODULE TASK_PATH TASK_NAME TASK_EXPECTED - if [ "$num_images_tgz" -ne "$num_images_txt" ]; then - abort "Number of images in ${TGZ_FILE} does not match ${TXT_FILE}" +run_one_validation() { + local type="$1" + local module="$2" + local path="$3" + local name="$4" + shift 4 + local expected=("$@") + local out + if [ "$type" = "combined" ]; then + out=$(validate_combined_bundle "$module" "$path" "${expected[@]}" 2>/dev/null) || true + else + out=$(validate_single_image "$path" "$name" "${expected[@]}" 2>/dev/null) || true fi + [ -n "$out" ] && echo "$out" + true +} + +# Parse and collect tasks +in_section=false +section_module="" +section_type="" +section_path="" +section_images=() +current_image_name="" +declare -a current_image_tags_arr + +while IFS= read -r line; do + if [[ "$line" =~ ^#\ @module= ]]; then + if [ "$in_section" = true ]; then + if [ "$section_type" = "combined" ]; then + TASK_TYPE+=("combined") + TASK_MODULE+=("$section_module") + TASK_PATH+=("$section_path") + TASK_NAME+=("") + TASK_EXPECTED+=("${section_images[*]}") + fi + fi - while IFS= read -r line; do - line_without_prefix="${line#docker.io/}" + section_module=$(extract_attr "$line" "@module=") + section_type=$(extract_attr "$line" "@type=") + section_path=$(extract_attr "$line" "@path=") + section_images=() + current_image_name="" + current_image_tags_arr=() + in_section=true - if [[ "$IMAGES_LIST" =~ "$line" || "$IMAGES_LIST" =~ "$line_without_prefix" ]]; then - echo "$line" + elif [[ "$line" =~ ^#\ @image= ]]; then + if [ "$section_type" = "single" ] && [ -n "$current_image_name" ] && [ ${#current_image_tags_arr[@]} -gt 0 ]; then + TASK_TYPE+=("single") + TASK_MODULE+=("$section_module") + TASK_PATH+=("$section_path") + TASK_NAME+=("$current_image_name") + TASK_EXPECTED+=("${current_image_tags_arr[*]}") + fi + current_image_name="${line#*@image=}" + current_image_name="${current_image_name%% *}" + current_image_tags_arr=() + + elif [ "$in_section" = true ] && [ -n "$line" ] && [[ ! "$line" =~ ^# ]]; then + if [ "$section_type" = "combined" ]; then + section_images+=("$line") else - mismatched=true - echo "Mismatch found: $line" + current_image_tags_arr+=("$line") fi - done < "${TXT_FILE}" + fi +done < "$INTERNAL_FILE" + +if [ "$in_section" = true ]; then + if [ "$section_type" = "combined" ]; then + TASK_TYPE+=("combined") + TASK_MODULE+=("$section_module") + TASK_PATH+=("$section_path") + TASK_NAME+=("") + TASK_EXPECTED+=("${section_images[*]}") + elif [ "$section_type" = "single" ] && [ -n "$current_image_name" ] && [ ${#current_image_tags_arr[@]} -gt 0 ]; then + TASK_TYPE+=("single") + TASK_MODULE+=("$section_module") + TASK_PATH+=("$section_path") + TASK_NAME+=("$current_image_name") + TASK_EXPECTED+=("${current_image_tags_arr[*]}") + fi +fi + +# Run validations in parallel +ERRORS_FILE=$(mktemp) +trap "rm -f $ERRORS_FILE" EXIT + +export -f run_one_validation validate_combined_bundle validate_single_image get_repo_tags_from_tgz strip_registry log_info 2>/dev/null || true +export OUTPUT_DIR - if $mismatched; then - abort "Images in ${TGZ_FILE} and ${TXT_FILE} do not match" +running=0 +total=${#TASK_TYPE[@]} +declare -a pids=() + +for ((i=0; i> "$ERRORS_FILE" 2>&1 & + pids+=($!) + + running=$((running + 1)) + if [ "$running" -ge "$VALIDATE_PARALLEL" ]; then + wait "${pids[0]}" 2>/dev/null || true + pids=("${pids[@]:1}") + running=$((running - 1)) + fi done +for pid in "${pids[@]}"; do wait "$pid" 2>/dev/null || true; done + +# Collect errors +while IFS= read -r line; do + [ -n "$line" ] && ERRORS+=("$line") +done < "$ERRORS_FILE" + +# Report summary +echo "" +log_info "=== VALIDATION SUMMARY ===" +if [ ${#ERRORS[@]} -gt 0 ]; then + log_error "Validation FAILED with ${#ERRORS[@]} error(s):" + for i in "${!ERRORS[@]}"; do + log_error " [$((i+1))] ${ERRORS[$i]}" + done + log_error "See actual vs expected above for each failed bundle." + exit 1 +else + log_info "Validation PASSED: ${total} bundle(s) validated successfully." + exit 0 +fi diff --git a/src/bundle-manifest.yaml b/src/bundle-manifest.yaml new file mode 100644 index 00000000..198a6f41 --- /dev/null +++ b/src/bundle-manifest.yaml @@ -0,0 +1,340 @@ +version: "1.0" + +global: + variants: [] + +modules: + platform: + description: "Core Platform Services (Required for all installations)" + bundle_type: combined + requires: [] + bucket_path: "platform" + images: + - busybox + - ci-scm-signed + - template-service-signed + - platform-service-signed + - pipeline-service-signed + - ng-manager-signed + - nextgenui-signed + - minio + - log-service-signed + - cdcdata-signed + - accesscontrol-service-signed + - delegate-proxy-signed + - gateway-signed + - helm-init-container + - manager-signed + - mongo + - ng-auth-ui-signed + - redis + - postgresql + - policy-mgmt + - smp-service-discovery-server-signed + - debezium-service-signed + - audit-event-streaming-signed + - queue-service-signed + - service-discovery-collector + - ng-dashboard-aggregator-signed + - harness/controller + - defaultbackend-amd64 + - redis_exporter + - postgres-exporter + - mongodb-exporter + - vault-secret-loader + - ui-signed + - name: delegate + variants: [".minimal", ".minimal-fips"] + variants_only: true + + platform-agents: + description: "Platform Agents" + bundle_type: single + parent: platform + bucket_path: "platform/agents" + images: + - name: delegate + variants: [".minimal", ".minimal-fips", "-fips"] + - name: upgrader + variants: ["-fips"] + + dashboard: + description: "Dashboard" + bundle_type: combined + parent: platform + bucket_path: "dashboard" + exclude: + - looker-signed + images: + - dashboard-service-signed + - statsd-exporter + + cdng: + description: "Continuous Deployment" + bundle_type: combined + requires: [platform] + bucket_path: "cdng" + images: + - gitops-service-signed + - cv-nextgen-signed + - le-nextgen-signed + - srm-ui-signed + + cdng-agents: + description: "CD Agents" + bundle_type: combined + parent: cdng + bucket_path: "cdng/agents" + images: + - argocd + - gitops-agent + - haproxy + - shellcheck + - gitops-agent-installer-helper + + ci: + description: "Continuous Integration" + bundle_type: combined + requires: [platform] + bucket_path: "ci" + images: + - ci-manager-signed + - ti-service-signed + - ci-addon + - ci-lite-engine + + ci-plugins: + description: "CI Build Plugins" + bundle_type: combined + parent: ci + bucket_path: "ci/plugins" + images: + - drone-git + - harness-cache-server + - kaniko + - kaniko-acr + - kaniko-ecr + - kaniko-gcr + - artifactory + - gcs + - cache + - s3 + - buildx + - buildx-acr + - buildx-ecr + - buildx-gar + - buildx-gcr + - docker + - ecr + - gar + - gcr + - acr + + sto: + description: "Security Testing Orchestration" + bundle_type: combined + requires: [platform, ci] + bucket_path: "sto" + images: + - stocore-signed + - ticket-service-signed + - refid-cache + - name: sto-plugin + variants: ["-fips"] + + sto-scanners: + description: "STO Security Scanners" + bundle_type: single + parent: sto + bucket_path: "sto/scanners" + images: + - anchore-job-runner + - aqua-security-job-runner + - name: aqua-trivy-job-runner + variants: ["-fips"] + - aws-ecr-job-runner + - aws-security-hub-job-runner + - name: bandit-job-runner + variants: ["-fips"] + - blackduckhub-job-runner + - brakeman-job-runner + - burp-job-runner + - checkmarx-job-runner + - checkov-job-runner + - docker-content-trust-job-runner + - fossa-job-runner + - github-advanced-security-job-runner + - gitleaks-job-runner + - name: grype-job-runner + variants: ["-fips"] + - modelscan-job-runner + - nexusiq-job-runner + - nikto-job-runner + - nmap-job-runner + - name: osv-job-runner + variants: ["-fips"] + - owasp-dependency-check-job-runner + - prowler-job-runner + - name: semgrep-job-runner + variants: ["-fips"] + - shiftleft-job-runner + - snyk-job-runner + - name: sonarqube-agent-job-runner + variants: ["-fips"] + - sysdig-job-runner + - traceable-job-runner + - twistlock-job-runner + - veracode-agent-job-runner + - whitesource-agent-job-runner + - wiz-job-runner + - zap-job-runner + + ff: + description: "Feature Flags" + bundle_type: combined + requires: [platform] + bucket_path: "ff" + images: + - ff-cron-signed + - ff-server-analytics-db-migration-signed + - ff-server-primary-db-migration-signed + - ff-service-signed + - ff-pushpin-signed + - ff-pushpin-worker-signed + + ccm: + description: "Cloud Cost Management" + bundle_type: combined + requires: [platform] + bucket_path: "ccm" + images: + - batch-processing-signed + - ce-anomaly-detection-signed + - ce-cloud-info-signed + - ce-nextgen-signed + - event-service-signed + - ng-ce-ui + - telescopes-signed + - clickhouse + - ccm-gcp-smp-signed + + ce: + description: "Chaos Engineering" + bundle_type: combined + requires: [platform] + bucket_path: "ce" + images: + - smp-chaos-k8s-ifs-signed + - smp-chaos-linux-infra-controller-signed + - smp-chaos-linux-infra-server-signed + - smp-chaos-manager-signed + - smp-chaos-web-signed + - source-probe + - smp-chaos-bg-processor-signed + - chaos-machine-ifc-signed + - chaos-machine-ifs-signed + - enterprise-chaos-hub-signed + + ce-plugins: + description: "Chaos Engineering Plugins" + bundle_type: combined + parent: ce + bucket_path: "ce/plugins" + images: + - chaos-log-watcher + - chaos-ddcr + - chaos-ddcr-faults + - chaos-event-watcher + + ssca: + description: "Supply Chain Security" + bundle_type: combined + requires: [platform] + bucket_path: "ssca" + images: + - ssca-manager-signed + - ssca-ui-signed + - component-service-signed + - component-analysis-service-signed + + ssca-plugins: + description: "SCS Plugins" + bundle_type: combined + parent: ssca + bucket_path: "ssca/plugins" + images: + - ssca-plugin + - slsa-plugin + - ssca-cdxgen-plugin + - ssca-compliance-plugin + - ssca-artifact-signing-plugin + + dbdevops: + description: "Database DevOps" + bundle_type: combined + requires: [platform] + bucket_path: "dbdevops" + images: + - db-devops-service-signed + + code: + description: "Code Repository" + bundle_type: combined + requires: [platform] + bucket_path: "code" + images: + - code-api-signed + - code-githa-signed + - code-gitrpc-signed + - code-search-signed + + iacm: + description: "Infrastructure as Code Management" + bundle_type: combined + requires: [platform] + bucket_path: "iacm" + images: + - iac-server-signed + - iacm-manager-signed + + iacm-plugins: + description: "IACM Plugins" + bundle_type: combined + parent: iacm + bucket_path: "iacm/plugins" + images: + - ci-addon + - ci-lite-engine + - drone-git + - harness_terraform + - harness_terraform_vm + + idp: + description: "Internal Developer Portal" + bundle_type: combined + requires: [platform] + bucket_path: "idp" + images: + - idp-service-signed + - idp-admin-signed + - idp-app-signed + + idp-plugins: + description: "IDP Plugins" + bundle_type: combined + parent: idp + bucket_path: "idp/plugins" + images: + - ci-addon + - ci-lite-engine + - drone-git + - cookiecutter + - createcatalog + - createorganisation + - createproject + - createrepo + - createresource + - directpush + - registercatalog + - slacknotify + - updatecatalogproperty diff --git a/src/generate-image-list.sh b/src/generate-image-list.sh index 25e58e0f..1af9d25f 100755 --- a/src/generate-image-list.sh +++ b/src/generate-image-list.sh @@ -1,13 +1,20 @@ #!/bin/bash +set -o errexit +set -o pipefail + +log_info() { echo "[INFO] $(date +%H:%M:%S) $*"; } +log_error() { echo "[ERROR] $(date +%H:%M:%S) $*" >&2; } + usage () { echo "Usage: $0 [options]" echo "" echo "Options:" - echo " -i, --image-gen-input ${SCRIPT_DIR}/generate-image.yaml Path to generate-image.yaml file" - echo " -o, --output-dir ${SCRIPT_DIR}/harness Path to directory in which images.txt is to be generated" - echo " -d, --harness-dir ${SCRIPT_DIR}/harness Path to harness directory" - echo " -h, --help Show this help message" + echo " -i, --image-gen-input Path to generate-image.yaml overrides file" + echo " -o, --output-dir Output directory for generated files" + echo " -d, --harness-dir Path to harness chart directory" + echo " -k, --keep-transient Keep images_raw.txt and images_internal.txt" + echo " -h, --help Show this help message" echo "" } @@ -15,7 +22,7 @@ SCRIPT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) HARNESS_DIR=${SCRIPT_DIR}/harness IMAGE_GEN_INPUT_FILE=${SCRIPT_DIR}/generate-image.yaml OUTPUT_DIR=${SCRIPT_DIR}/harness - +KEEP_TRANSIENT=false while [ $# -gt 0 ]; do key="$1" @@ -32,6 +39,10 @@ while [ $# -gt 0 ]; do HARNESS_DIR="$2" shift 2 ;; + -k|--keep-transient) + KEEP_TRANSIENT=true + shift + ;; -h|--help) usage exit @@ -44,53 +55,96 @@ while [ $# -gt 0 ]; do esac done +if ! command -v python3 &>/dev/null; then + log_error "python3 not found - cannot resolve bundle manifest" + exit 1 +fi +CHART_YAML="${HARNESS_DIR}/Chart.yaml" +if [ ! -f "${CHART_YAML}" ]; then + log_error "Chart.yaml not found at ${CHART_YAML}" + exit 1 +fi -helm template ${HARNESS_DIR} -f ${IMAGE_GEN_INPUT_FILE} | grep -i image | grep \/ | grep -v imagePullPolicy | grep -v "#" | awk '{$1=$1};1' | sort -u | sed 's/^[^:]*: //g' | sed -e "s/^'//" -e "s/'$//"| sed -e 's/^"//' -e 's/"$//' > ${OUTPUT_DIR}/images_tmp.txt +log_info "Auto-detecting module conditions from Chart.yaml" +AUTO_ENABLE_FLAGS="" +while IFS= read -r condition; do + condition=$(echo "$condition" | xargs) + [ -z "$condition" ] && continue + AUTO_ENABLE_FLAGS="${AUTO_ENABLE_FLAGS} --set ${condition}=true" +done < <(grep "condition:" "${CHART_YAML}" | awk -F'condition:' '{print $2}') -# Remove duplicates based on image name and tag -awk -F: '{ print $1 ":" $2 }' ${OUTPUT_DIR}/images_tmp.txt | sort -u > ${OUTPUT_DIR}/images.txt +if [ -n "${AUTO_ENABLE_FLAGS}" ]; then + log_info "Auto-enabled module flags:${AUTO_ENABLE_FLAGS}" +fi -rm ${OUTPUT_DIR}/images_tmp.txt -# Add minimal images -IMAGES=("docker.io/harness/delegate:[0-9.]+") -SUFFIX=(".minimal" ".minimal-fips" "-fips") -for i in "${!IMAGES[@]}" -do - MATCHES=$(grep -oE "${IMAGES[i]}" "${OUTPUT_DIR}/images.txt") - if [ -n "$MATCHES" ]; then - for j in "${!SUFFIX[@]}" - do - echo "$MATCHES" | sed "s/$/${SUFFIX[j]}/" | tee -a ${OUTPUT_DIR}/images.txt - done +# Auto-flip all enabled:false and create:false keys from values.yaml to true. +# Covers sub-component flags (metrics exporters, ingress, monitoring, etc.) +# not listed as Chart.yaml conditions. See smp-tools.py auto-enable-flags for BLOCKLIST. +VALUES_YAML="${HARNESS_DIR}/values.yaml" +VALUES_FLAGS="" +if [ -f "${VALUES_YAML}" ]; then + log_info "Scanning ${VALUES_YAML} for disabled flags to enable" + while IFS= read -r flag; do + [ -z "$flag" ] && continue + VALUES_FLAGS="${VALUES_FLAGS} ${flag}" + done < <(python3 "${SCRIPT_DIR}/smp-tools.py" auto-enable-flags "${VALUES_YAML}") + if [ -n "${VALUES_FLAGS}" ]; then + log_info "Auto-enabled values flags:${VALUES_FLAGS}" fi -done +fi + +AUTO_ENABLE_FLAGS="${AUTO_ENABLE_FLAGS}${VALUES_FLAGS}" + -IMAGES=("harness/aqua-trivy-job-runner:([0-9.]+|latest)" - "harness/bandit-job-runner:([0-9.]+|latest)" - "harness/grype-job-runner:([0-9.]+|latest)" - "harness/osv-job-runner:([0-9.]+|latest)" - "harness/sonarqube-agent-job-runner:([0-9.]+|latest)" - "harness/semgrep-job-runner:([0-9.]+|latest)" - "harness/gitleaks-trivy-job-runner:([0-9.]+|latest)" - "harness/upgrader:([0-9.]+|latest)") -SUFFIX=("-fips") - -for i in "${!IMAGES[@]}" -do - MATCHES=$(grep -oE "${IMAGES[i]}" "${OUTPUT_DIR}/images.txt") - if [ -n "$MATCHES" ]; then - for j in "${!SUFFIX[@]}" - do - echo "$MATCHES" | sed "s/$/${SUFFIX[j]}/" | tee -a "${OUTPUT_DIR}/images.txt" - done +OVERRIDE_FLAGS="" +if [ -f "${IMAGE_GEN_INPUT_FILE}" ]; then + log_info "Extracting --set flags from ${IMAGE_GEN_INPUT_FILE}" + while IFS= read -r flag; do + [ -z "$flag" ] && continue + OVERRIDE_FLAGS="${OVERRIDE_FLAGS} ${flag}" + done < <(python3 "${SCRIPT_DIR}/smp-tools.py" auto-enable-flags --mode true "${IMAGE_GEN_INPUT_FILE}") + if [ -n "${OVERRIDE_FLAGS}" ]; then + log_info "Override flags from ${IMAGE_GEN_INPUT_FILE}:${OVERRIDE_FLAGS}" fi -done +fi + +log_info "Running helm template to extract images" +helm template ${HARNESS_DIR} ${AUTO_ENABLE_FLAGS} ${OVERRIDE_FLAGS} \ + | grep -i image | grep \/ | grep -v imagePullPolicy | grep -v "#" \ + | awk '{$1=$1};1' | sort -u \ + | sed 's/^[^:]*: //g' \ + | sed -e "s/^'//" -e "s/'$//" \ + | sed -e 's/^"//' -e 's/"$//' \ + > ${OUTPUT_DIR}/images_tmp.txt + +awk -F: '{ print $1 ":" $2 }' ${OUTPUT_DIR}/images_tmp.txt | sort -u > ${OUTPUT_DIR}/images_raw.txt +rm ${OUTPUT_DIR}/images_tmp.txt + +sed -i '' -e '/index\.docker\.io\/chaosnative:/d' -e '/^$/d' ${OUTPUT_DIR}/images_raw.txt 2>/dev/null || \ + sed -i -e '/index\.docker\.io\/chaosnative:/d' -e '/^$/d' ${OUTPUT_DIR}/images_raw.txt + +IMAGE_COUNT=$(wc -l < ${OUTPUT_DIR}/images_raw.txt | tr -d '[:space:]') +log_info "Generated images_raw.txt with ${IMAGE_COUNT} base images (no variants)" + +log_info "Resolving bundle manifest to generate images.txt and images_internal.txt" +python3 ${SCRIPT_DIR}/smp-tools.py bundle-images \ + --manifest ${SCRIPT_DIR}/bundle-manifest.yaml \ + --raw-images ${OUTPUT_DIR}/images_raw.txt \ + --output-dir ${OUTPUT_DIR} + +log_info "Running bundle manifest validation" +python3 ${SCRIPT_DIR}/smp-tools.py validate-bundle \ + --manifest ${SCRIPT_DIR}/bundle-manifest.yaml -# Remove duplicates one more time in case minimal images have added any -awk -F: '{ print $1 ":" $2 }' ${OUTPUT_DIR}/images.txt | sort -u > ${OUTPUT_DIR}/images_tmp.txt -mv ${OUTPUT_DIR}/images_tmp.txt ${OUTPUT_DIR}/images.txt -sed -i '' -e '/index\.docker\.io\/chaosnative:/d' -e '/^$/d' ${OUTPUT_DIR}/images.txt +if [ "${KEEP_TRANSIENT}" = false ]; then + log_info "Cleaning up transient files" + rm -f ${OUTPUT_DIR}/images_raw.txt + rm -f ${OUTPUT_DIR}/images_internal.txt +else + log_info "Keeping transient files (--keep-transient)" +fi +log_info "Image generation complete" exit 0 diff --git a/src/generate-image.yaml b/src/generate-image.yaml index 6060afdc..e69de29b 100644 --- a/src/generate-image.yaml +++ b/src/generate-image.yaml @@ -1,66 +0,0 @@ -global: - monitoring: - enabled: true - ccm: - enabled: true - cd: - enabled: true - cg: - enabled: true - chaos: - enabled: true - ci: - enabled: true - database: - clickhouse: - enabled: true - ff: - enabled: true - lwd: - autocud: - enabled: false - enabled: false - ng: - enabled: true - ngcustomdashboard: - enabled: true - opa: - enabled: true - servicediscoverymanager: - enabled: true - srm: - enabled: true - sto: - enabled: true - ssca: - enabled: true - dbops: - enabled: true - code: - enabled: true - iacm: - enabled: true - idp: - enabled: true - externalSecretsLoader: - enabled: true -platform: - bootstrap: - networking: - nginx: - create: true - defaultbackend: - create: true - database: - mongodb: - metrics: - enabled: true - redis: - metrics: - enabled: true - timescaledb: - prometheus: - enabled: true - postgresql: - metrics: - enabled: true diff --git a/src/get_modules.sh b/src/get_modules.sh old mode 100644 new mode 100755 index 09852cf8..43d91095 --- a/src/get_modules.sh +++ b/src/get_modules.sh @@ -1,25 +1,20 @@ #!/bin/bash -# Check if HELM_CHARTS_DIR is set if [[ -z "$HELM_CHARTS_DIR" ]]; then echo "Error: HELM_CHARTS_DIR environment variable is not set" exit 1 fi -# Check if Chart.yaml exists if [[ ! -f "$HELM_CHARTS_DIR/src/harness/Chart.yaml" ]]; then echo "Error: Chart.yaml not found at $HELM_CHARTS_DIR/src/harness/Chart.yaml" exit 1 fi -# Initialize empty arrays and flags MODULES=() MODULE_IMAGE_FILES=() MODULE_IMAGE_ZIP_FILES=() -platform_added=false while read -r value; do - # Remove leading/trailing whitespace from value value=$(echo "$value" | xargs) module="" if [[ $value == "harness" || $value == "harness-common" ]]; then @@ -35,7 +30,7 @@ while read -r value; do else module="$value" fi - + if [[ -n "$module" ]]; then MODULES+=("$module") MODULE_IMAGE_FILES+=("${module}_images.txt") @@ -43,15 +38,20 @@ while read -r value; do fi done < <(grep "name:" "$HELM_CHARTS_DIR/src/harness/Chart.yaml" | awk -F 'name:' '{print $2}') -# Get unique entries get_unique() { printf "%s\n" "$@" | /usr/bin/sort -ur | tr '\n' ' ' | sed 's/ $//'; } -# Get unique values and export as space-separated strings export MODULES=$(get_unique "${MODULES[@]}") export MODULE_IMAGE_FILES=$(get_unique "${MODULE_IMAGE_FILES[@]}") export MODULE_IMAGE_ZIP_FILES=$(get_unique "${MODULE_IMAGE_ZIP_FILES[@]}") -# Print the arrays (optional, for verification) echo "Modules: $MODULES" echo "Image files: $MODULE_IMAGE_FILES" echo "Image zip files: $MODULE_IMAGE_ZIP_FILES" + +INTERNAL_FILE="$HELM_CHARTS_DIR/src/harness/images_internal.txt" +if [ -f "$INTERNAL_FILE" ]; then + export BUNDLE_SECTIONS=$(grep '# @module=' "$INTERNAL_FILE" | sed 's/.*@module=\([^ ]*\).*/\1/' | tr '\n' ' ' | sed 's/ $//') + if [ -n "$BUNDLE_SECTIONS" ]; then + echo "Bundle sections: $BUNDLE_SECTIONS" + fi +fi diff --git a/src/harness/Chart.lock b/src/harness/Chart.lock index 275820e6..1aaecc50 100644 --- a/src/harness/Chart.lock +++ b/src/harness/Chart.lock @@ -4,42 +4,42 @@ dependencies: version: 1.4.22 - name: ccm repository: https://harness.github.io/helm-charts - version: 0.37.0 + version: 0.38.0 - name: cd repository: https://harness.github.io/helm-charts - version: 0.37.0 + version: 0.38.0 - name: chaos repository: https://harness.github.io/helm-charts - version: 0.37.0 + version: 0.38.0 - name: ci repository: https://harness.github.io/helm-charts - version: 0.37.0 + version: 0.38.1 - name: ff repository: https://harness.github.io/helm-charts - version: 0.37.0 + version: 0.38.0 - name: platform repository: https://harness.github.io/helm-charts - version: 0.37.0 + version: 0.38.1 - name: srm repository: https://harness.github.io/helm-charts - version: 0.37.0 + version: 0.38.0 - name: sto repository: https://harness.github.io/helm-charts - version: 0.37.0 + version: 0.38.0 - name: ssca repository: https://harness.github.io/helm-charts - version: 0.37.0 + version: 0.38.0 - name: db-devops repository: https://harness.github.io/helm-charts - version: 0.37.0 + version: 0.38.0 - name: code repository: https://harness.github.io/helm-charts - version: 0.37.0 + version: 0.38.0 - name: iacm repository: https://harness.github.io/helm-charts - version: 0.37.0 + version: 0.38.0 - name: idp repository: https://harness.github.io/helm-charts - version: 0.37.0 -digest: sha256:df4dab071db8d30604a6825054ca805f87413b8c6d58f5718dc840a98c110a30 -generated: "2026-02-24T17:05:56.844334762Z" + version: 0.38.0 +digest: sha256:dd0ad4ea2e283765da3278b60eeef68084ac96cd9223ad40b7d7e9069b8f9fd2 +generated: "2026-03-13T06:03:37.075462592Z" diff --git a/src/harness/Chart.yaml b/src/harness/Chart.yaml index 853409b6..73ccdb6d 100644 --- a/src/harness/Chart.yaml +++ b/src/harness/Chart.yaml @@ -19,14 +19,14 @@ dependencies: - condition: global.ci.enabled name: ci repository: https://harness.github.io/helm-charts - version: 0.38.0 + version: 0.38.1 - condition: global.ff.enabled name: ff repository: https://harness.github.io/helm-charts version: 0.38.0 - name: platform repository: https://harness.github.io/helm-charts - version: 0.38.0 + version: 0.38.1 - condition: global.cd.enabled name: srm repository: https://harness.github.io/helm-charts @@ -59,4 +59,4 @@ description: Helm Chart for deploying Harness. kubeVersion: '>=1.27.0-0' name: harness type: application -version: 0.38.0 +version: 0.38.1 diff --git a/src/harness/README.md b/src/harness/README.md index 13bb810f..b719be74 100644 --- a/src/harness/README.md +++ b/src/harness/README.md @@ -4,7 +4,7 @@ This readme provides the basic instructions to deploy Harness using a Helm chart Helm Chart for deploying Harness. -![Version: 0.37.0](https://img.shields.io/badge/Version-0.37.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.80917](https://img.shields.io/badge/AppVersion-1.0.80917-informational?style=flat-square) +![Version: 0.38.1](https://img.shields.io/badge/Version-0.38.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.80917](https://img.shields.io/badge/AppVersion-1.0.80917-informational?style=flat-square) For full release notes, go to [Self-Managed Enterprise Edition release notes](https://developer.harness.io/release-notes/self-managed-enterprise-edition). @@ -81,101 +81,117 @@ This command removes the Kubernetes components that are associated with the char If your cluster is in an air-gapped environment, your deployment requires the following images: ``` +## Core Platform Services (Required for all installations) docker.io/busybox:1.37.0 -docker.io/haproxy:lts-alpine3.23 -docker.io/harness/accesscontrol-service-signed:1.195.0 -docker.io/harness/argocd:v3.2.5 -docker.io/harness/audit-event-streaming-signed:1.74.0 -docker.io/harness/batch-processing-signed:1.77.14 -docker.io/harness/ccm-gcp-smp-signed:100079 -docker.io/harness/cdcdata-signed:1.49.7 -docker.io/harness/ce-anomaly-detection-signed:1.21.0 -docker.io/harness/ce-cloud-info-signed:1.13.2 -docker.io/harness/ce-nextgen-signed:1.79.22 -docker.io/harness/chaos-ddcr-faults:1.74.0 -docker.io/harness/chaos-ddcr:1.74.0 -docker.io/harness/chaos-event-watcher:1.74.0 -docker.io/harness/chaos-log-watcher:1.74.0 -docker.io/harness/chaos-machine-ifc-signed:1.74.0 -docker.io/harness/chaos-machine-ifs-signed:1.74.0 -docker.io/harness/ci-manager-signed:1.120.5 -docker.io/harness/ci-scm-signed:1.44.0 -docker.io/harness/clickhouse:24.12.4-debian-12-r1 -docker.io/harness/code-api-signed:1.73.3 -docker.io/harness/code-githa-signed:1.73.1 -docker.io/harness/code-gitrpc-signed:1.73.1 -docker.io/harness/code-search-signed:1.73.1 -docker.io/harness/component-analysis-service-signed:1.1.7 -docker.io/harness/component-service-signed:1.6.30 -docker.io/harness/cv-nextgen-signed:1.55.1 -docker.io/harness/dashboard-service-signed:1.98.0 -docker.io/harness/db-devops-service-signed:1.78.3 +docker.io/harness/ci-scm-signed:1.45.1 +docker.io/harness/template-service-signed:1.134.0 +docker.io/harness/platform-service-signed:1.110.0 +docker.io/harness/pipeline-service-signed:1.172.8 +docker.io/harness/ng-manager-signed:1.132.4 +docker.io/harness/nextgenui-signed:1.119.5 +docker.io/harness/minio:RELEASE.2025-10-15T17-29-55Z-jammy +docker.io/harness/log-service-signed:1.41.1 +docker.io/harness/cdcdata-signed:1.51.2 +docker.io/harness/accesscontrol-service-signed:1.208.0 +docker.io/harness/delegate-proxy-signed:1.9.0 +docker.io/harness/gateway-signed:1.62.5 +docker.io/harness/helm-init-container:1.8.0 +docker.io/harness/manager-signed:1.131.5 +docker.io/harness/mongo:7.0.28-jammy +docker.io/harness/ng-auth-ui-signed:1.38.2 +docker.io/harness/redis:7.4.8-jammy +docker.io/harness/postgresql:14.20-debian +docker.io/harness/policy-mgmt:1.38.1 +docker.io/harness/smp-service-discovery-server-signed:0.56.0 docker.io/harness/debezium-service-signed:1.25.1 -docker.io/harness/delegate-proxy-signed:1.6.0 +docker.io/harness/audit-event-streaming-signed:1.77.0 +docker.io/harness/queue-service-signed:1.9.0 +docker.io/harness/service-discovery-collector:0.56.0 +docker.io/harness/ng-dashboard-aggregator-signed:1.96.0 +docker.io/harness/controller:1.14.3-jammy +registry.k8s.io/defaultbackend-amd64:1.5 +docker.io/harness/redis_exporter:1.80.2-jammy +docker.io/prometheuscommunity/postgres-exporter:v0.19.1 +docker.io/harness/mongodb-exporter:0.47.1-jammy +harness/vault-secret-loader:1.0.2 +docker.io/harness/ui-signed:1.32.4 +docker.io/harness/delegate:26.02.88404.minimal +docker.io/harness/delegate:26.02.88404.minimal-fips + +### Platform Agents docker.io/harness/delegate:26.02.88404 -docker.io/harness/delegate:26.02.88404-fips docker.io/harness/delegate:26.02.88404.minimal docker.io/harness/delegate:26.02.88404.minimal-fips -docker.io/harness/enterprise-chaos-hub-signed:1.74.4 -docker.io/harness/event-service-signed:1.14.1 -docker.io/harness/ff-cron-signed:1.1154.1 -docker.io/harness/ff-pushpin-signed:1.1135.0 -docker.io/harness/ff-pushpin-worker-signed:1.1135.0 -docker.io/harness/ff-server-analytics-db-migration-signed:1.1154.1 -docker.io/harness/ff-server-primary-db-migration-signed:1.1154.1 -docker.io/harness/ff-service-signed:1.1154.1 -docker.io/harness/gateway-signed:1.60.3 -docker.io/harness/gitops-agent-installer-helper:v0.0.9 -docker.io/harness/gitops-agent:v0.109.0 -docker.io/harness/gitops-service-signed:1.50.4 -docker.io/harness/helm-init-container:1.7.0 -docker.io/harness/iac-server-signed:1.302.0 -docker.io/harness/iacm-manager-signed:1.128.1 -docker.io/harness/idp-admin-signed:1.36.3 -docker.io/harness/idp-app-signed:1.36.11 -docker.io/harness/idp-service-signed:1.36.15 -docker.io/harness/le-nextgen-signed:1.13.0 -docker.io/harness/log-service-signed:1.41.0 -docker.io/harness/looker-signed:1.8.9 -docker.io/harness/manager-signed:1.128.4 -docker.io/harness/minio:2025.7.18-debian-12-r0 -docker.io/harness/mongo:7.0.28-debian-12-r1 -docker.io/harness/mongodb-exporter:0.40.0-debian-12-r40 -docker.io/harness/nextgenui-signed:1.116.11 -docker.io/harness/ng-auth-ui-signed:1.38.2 -docker.io/harness/ng-ce-ui:1.75.7 -docker.io/harness/ng-dashboard-aggregator-signed:1.93.0 -docker.io/harness/ng-manager-signed:1.129.7 -docker.io/harness/pipeline-service-signed:1.169.5 -docker.io/harness/platform-service-signed:1.107.0 -docker.io/harness/policy-mgmt:1.36.5 -docker.io/harness/postgresql:14.20-alpine3.23 -docker.io/harness/queue-service-signed:1.8.1 -docker.io/harness/redis:7.4.7-alpine -docker.io/harness/refid-cache:latest -docker.io/harness/service-discovery-collector:0.54.0 -docker.io/harness/smp-chaos-bg-processor-signed:1.74.4 -docker.io/harness/smp-chaos-k8s-ifs-signed:1.74.0 -docker.io/harness/smp-chaos-linux-infra-controller-signed:1.74.0 -docker.io/harness/smp-chaos-linux-infra-server-signed:1.74.0 -docker.io/harness/smp-chaos-manager-signed:1.74.4 -docker.io/harness/smp-chaos-web-signed:1.74.1 -docker.io/harness/smp-service-discovery-server-signed:0.54.0 -docker.io/harness/source-probe:main-latest -docker.io/harness/srm-ui-signed:1.16.0 -docker.io/harness/ssca-manager-signed:1.50.14 -docker.io/harness/ssca-ui-signed:0.38.5 -docker.io/harness/statsd-exporter:1.0-prometheus-busybox-1 -docker.io/harness/stocore-signed:1.180.4 -docker.io/harness/telescopes-signed:1.6.0 -docker.io/harness/template-service-signed:1.131.3 -docker.io/harness/ti-service-signed:1.60.5 -docker.io/harness/ticket-service-signed:1.4.5 -docker.io/harness/ui-signed:1.32.3 -docker.io/harness/upgrader:1.10.0 +docker.io/harness/delegate:26.02.88404-fips +docker.io/harness/upgrader:1.11.0 +docker.io/harness/upgrader:1.11.0-fips + +### Dashboard +docker.io/harness/looker-signed:1.10.1 +docker.io/harness/dashboard-service-signed:1.102.0 +docker.io/harness/statsd-exporter:3.0-prometheus-busybox-2 + +## Continuous Deployment +docker.io/harness/gitops-service-signed:1.51.3 +docker.io/harness/cv-nextgen-signed:1.57.2 +docker.io/harness/le-nextgen-signed:1.13.1 +docker.io/harness/srm-ui-signed:1.16.2 + +### CD Agents +docker.io/harness/argocd:v3.3.0 +docker.io/harness/gitops-agent:v0.110.1 +docker.io/haproxy:lts-alpine3.23 docker.io/koalaman/shellcheck:v0.5.0 -docker.io/oliver006/redis_exporter:v1.80.2 -docker.io/prometheuscommunity/postgres-exporter:v0.15.0 +docker.io/harness/gitops-agent-installer-helper:v0.0.11-rebuilt + +## Continuous Integration +docker.io/harness/ci-manager-signed:1.123.5 +docker.io/harness/ti-service-signed:1.61.3 +harness/ci-addon:1.18.10 +harness/ci-addon:1.18.6 +harness/ci-addon:rootless-1.18.10 +harness/ci-addon:rootless-1.18.6 +harness/ci-lite-engine:1.18.10 +harness/ci-lite-engine:1.18.6 +harness/ci-lite-engine:rootless-1.18.10 +harness/ci-lite-engine:rootless-1.18.6 + +### CI Build Plugins +harness/drone-git:1.7.16-rootless +harness/harness-cache-server:1.7.13 +plugins/kaniko:1.13.4 +plugins/kaniko-acr:1.13.5 +plugins/kaniko-ecr:1.13.4 +plugins/kaniko-gcr:1.13.4 +plugins/artifactory:1.8.3 +plugins/artifactory:1.8.4 +plugins/gcs:1.6.7 +plugins/gcs:1.6.8 +plugins/cache:1.9.18 +plugins/cache:1.9.21 +plugins/cache:1.9.24 +plugins/s3:1.5.6 +plugins/s3:1.5.8 +plugins/buildx:1.3.13 +plugins/buildx-acr:1.4.4 +plugins/buildx-ecr:1.4.3 +plugins/buildx-gar:1.4.3 +plugins/buildx-gcr:1.4.3 +plugins/docker:21.2.2 +plugins/ecr:21.2.2 +plugins/gar:21.2.2 +plugins/gcr:21.2.2 +plugins/acr:21.2.2 + +## Security Testing Orchestration +docker.io/harness/stocore-signed:1.182.0 +docker.io/harness/ticket-service-signed:1.8.0 +docker.io/harness/refid-cache:latest +harness/refid-cache:latest +harness/sto-plugin:latest +harness/sto-plugin:latest-fips + +### STO Security Scanners harness/anchore-job-runner:latest harness/aqua-security-job-runner:latest harness/aqua-trivy-job-runner:latest @@ -189,35 +205,12 @@ harness/brakeman-job-runner:latest harness/burp-job-runner:latest harness/checkmarx-job-runner:latest harness/checkov-job-runner:latest -harness/ci-addon:1.18.2 -harness/ci-addon:1.18.6 -harness/ci-addon:1.18.7 -harness/ci-addon:rootless-1.18.2 -harness/ci-addon:rootless-1.18.6 -harness/ci-addon:rootless-1.18.7 -harness/ci-lite-engine:1.18.2 -harness/ci-lite-engine:1.18.6 -harness/ci-lite-engine:1.18.7 -harness/ci-lite-engine:rootless-1.18.2 -harness/ci-lite-engine:rootless-1.18.6 -harness/ci-lite-engine:rootless-1.18.7 -harness/cookiecutter:1.19.0 -harness/createcatalog:1.19.0 -harness/createorganisation:1.19.0 -harness/createproject:1.19.0 -harness/createrepo:1.19.0 -harness/createresource:1.19.0 -harness/directpush:1.19.0 harness/docker-content-trust-job-runner:latest -harness/drone-git:1.7.10-rootless -harness/drone-git:1.7.15-rootless harness/fossa-job-runner:latest harness/github-advanced-security-job-runner:latest harness/gitleaks-job-runner:latest harness/grype-job-runner:latest harness/grype-job-runner:latest-fips -harness/harness-cache-server:1.7.10 -harness/harness-cache-server:1.7.8 harness/modelscan-job-runner:latest harness/nexusiq-job-runner:latest harness/nikto-job-runner:latest @@ -226,65 +219,128 @@ harness/osv-job-runner:latest harness/osv-job-runner:latest-fips harness/owasp-dependency-check-job-runner:latest harness/prowler-job-runner:latest -harness/refid-cache:latest -harness/registercatalog:1.19.0 harness/semgrep-job-runner:latest harness/semgrep-job-runner:latest-fips harness/shiftleft-job-runner:latest -harness/slacknotify:1.19.0 -harness/slsa-plugin:0.52.1 harness/snyk-job-runner:latest harness/sonarqube-agent-job-runner:latest harness/sonarqube-agent-job-runner:latest-fips -harness/ssca-artifact-signing-plugin:0.52.1 -harness/ssca-cdxgen-plugin:0.52.1 -harness/ssca-compliance-plugin:0.52.1 -harness/ssca-plugin:0.52.1 -harness/sto-plugin:latest harness/sysdig-job-runner:latest harness/traceable-job-runner:latest harness/twistlock-job-runner:latest -harness/updatecatalogproperty:1.19.0 -harness/upgrader:1.10.0-fips -harness/vault-secret-loader:1.0.2 harness/veracode-agent-job-runner:latest harness/whitesource-agent-job-runner:latest harness/wiz-job-runner:latest harness/zap-job-runner:latest -plugins/acr:21.2.2 -plugins/artifactory:1.8.3 -plugins/buildx-acr:1.4.3 -plugins/buildx-ecr:1.4.3 -plugins/buildx-gar:1.4.3 -plugins/buildx-gcr:1.4.3 -plugins/buildx:1.3.13 -plugins/cache:1.9.18 -plugins/docker:21.2.2 -plugins/ecr:21.2.2 -plugins/gar:21.2.2 -plugins/gcr:21.2.2 -plugins/gcs:1.6.7 + +## Feature Flags +docker.io/harness/ff-cron-signed:1.1160.0 +docker.io/harness/ff-server-analytics-db-migration-signed:1.1160.0 +docker.io/harness/ff-server-primary-db-migration-signed:1.1160.0 +docker.io/harness/ff-service-signed:1.1160.0 +docker.io/harness/ff-pushpin-signed:1.1138.0 +docker.io/harness/ff-pushpin-worker-signed:1.1138.0 + +## Cloud Cost Management +docker.io/harness/batch-processing-signed:1.78.5 +docker.io/harness/ce-anomaly-detection-signed:1.22.0 +docker.io/harness/ce-cloud-info-signed:1.14.4 +docker.io/harness/ce-nextgen-signed:1.80.6 +docker.io/harness/event-service-signed:1.15.2 +docker.io/harness/ng-ce-ui:1.76.3 +docker.io/harness/telescopes-signed:1.7.2 +docker.io/harness/clickhouse:25.12.5-jammy +docker.io/harness/ccm-gcp-smp-signed:100079 + +## Chaos Engineering +docker.io/harness/smp-chaos-k8s-ifs-signed:1.76.0 +docker.io/harness/smp-chaos-linux-infra-controller-signed:1.76.0 +docker.io/harness/smp-chaos-linux-infra-server-signed:1.76.1 +docker.io/harness/smp-chaos-manager-signed:1.76.2 +docker.io/harness/smp-chaos-web-signed:1.76.2 +docker.io/harness/source-probe:main-latest +docker.io/harness/smp-chaos-bg-processor-signed:1.76.2 +docker.io/harness/chaos-machine-ifc-signed:1.76.0 +docker.io/harness/chaos-machine-ifs-signed:1.76.0 +docker.io/harness/enterprise-chaos-hub-signed:1.76.2 + +### Chaos Engineering Plugins +docker.io/harness/chaos-log-watcher:1.76.0 +docker.io/harness/chaos-ddcr:1.76.0 +docker.io/harness/chaos-ddcr-faults:1.76.0 +docker.io/harness/chaos-event-watcher:1.76.0 + +## Supply Chain Security +docker.io/harness/ssca-manager-signed:1.52.13 +docker.io/harness/ssca-ui-signed:0.39.2 +docker.io/harness/component-service-signed:1.7.1 +docker.io/harness/component-analysis-service-signed:1.2.1 + +### SCS Plugins +harness/ssca-plugin:0.53.5 +harness/slsa-plugin:0.53.2 +harness/ssca-cdxgen-plugin:0.53.5 +harness/ssca-compliance-plugin:0.53.4 +harness/ssca-artifact-signing-plugin:0.53.2 + +## Database DevOps +docker.io/harness/db-devops-service-signed:1.81.0 + +## Code Repository +docker.io/harness/code-api-signed:1.76.1 +docker.io/harness/code-githa-signed:1.76.0 +docker.io/harness/code-gitrpc-signed:1.76.0 +docker.io/harness/code-search-signed:1.76.0 + +## Infrastructure as Code Management +docker.io/harness/iac-server-signed:1.329.0 +docker.io/harness/iacm-manager-signed:1.134.0 + +### IACM Plugins +harness/ci-addon:1.18.10 +harness/ci-addon:1.18.6 +harness/ci-addon:rootless-1.18.10 +harness/ci-addon:rootless-1.18.6 +harness/ci-lite-engine:1.18.10 +harness/ci-lite-engine:1.18.6 +harness/ci-lite-engine:rootless-1.18.10 +harness/ci-lite-engine:rootless-1.18.6 +harness/drone-git:1.7.16-rootless plugins/harness_terraform:latest plugins/harness_terraform_vm:latest -plugins/kaniko-acr:1.13.4 -plugins/kaniko-ecr:1.13.3 -plugins/kaniko-ecr:1.13.4 -plugins/kaniko-gcr:1.13.3 -plugins/kaniko-gcr:1.13.4 -plugins/kaniko:1.13.3 -plugins/kaniko:1.13.4 -plugins/s3:1.5.4 -plugins/s3:1.5.6 -plugins/s3:1.5.7 -registry.k8s.io/defaultbackend-amd64:1.5 -registry.k8s.io/ingress-nginx/controller:v1.14.0 + +## Internal Developer Portal +docker.io/harness/idp-service-signed:1.37.5 +docker.io/harness/idp-admin-signed:1.37.2 +docker.io/harness/idp-app-signed:1.37.9 + +### IDP Plugins +harness/ci-addon:1.18.10 +harness/ci-addon:1.18.6 +harness/ci-addon:rootless-1.18.10 +harness/ci-addon:rootless-1.18.6 +harness/ci-lite-engine:1.18.10 +harness/ci-lite-engine:1.18.6 +harness/ci-lite-engine:rootless-1.18.10 +harness/ci-lite-engine:rootless-1.18.6 +harness/drone-git:1.7.16-rootless +harness/cookiecutter:1.20.0 +harness/createcatalog:1.20.0 +harness/createorganisation:1.20.0 +harness/createproject:1.20.0 +harness/createrepo:1.20.0 +harness/createresource:1.20.0 +harness/directpush:1.20.0 +harness/registercatalog:1.20.0 +harness/slacknotify:1.20.0 +harness/updatecatalogproperty:1.20.0 ``` ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| -| ccm.batch-processing | object | `{"awsAccountTagsCollectionJobConfig":{"enabled":true},"cliProxy":{"enabled":false,"host":"localhost","password":"","port":80,"protocol":"http","username":""},"cloudProviderConfig":{"CLUSTER_DATA_GCS_BACKUP_BUCKET":"placeHolder","CLUSTER_DATA_GCS_BUCKET":"placeHolder","DATA_PIPELINE_CONFIG_GCS_BASE_PATH":"placeHolder","GCP_PROJECT_ID":"placeHolder","S3_SYNC_CONFIG_BUCKET_NAME":"placeHolder","S3_SYNC_CONFIG_REGION":"placeHolder"},"postgres":{"image":{"repository":"harness/postgresql","tag":"14.20-alpine3.23"}},"stackDriverLoggingEnabled":false}` | Set ccm.batch-processing.clickhouse.enabled to true for AWS infrastructure | +| ccm.batch-processing | object | `{"awsAccountTagsCollectionJobConfig":{"enabled":true},"cliProxy":{"enabled":false,"host":"localhost","password":"","port":80,"protocol":"http","username":""},"cloudProviderConfig":{"CLUSTER_DATA_GCS_BACKUP_BUCKET":"placeHolder","CLUSTER_DATA_GCS_BUCKET":"placeHolder","DATA_PIPELINE_CONFIG_GCS_BASE_PATH":"placeHolder","GCP_PROJECT_ID":"placeHolder","S3_SYNC_CONFIG_BUCKET_NAME":"placeHolder","S3_SYNC_CONFIG_REGION":"placeHolder"},"postgres":{"image":{"repository":"harness/postgresql","tag":"14.20-debian"}},"stackDriverLoggingEnabled":false}` | Set ccm.batch-processing.clickhouse.enabled to true for AWS infrastructure | | ccm.batch-processing.awsAccountTagsCollectionJobConfig | object | `{"enabled":true}` | Set ccm.batch-processing.awsAccountTagsCollectionJobConfig.enabled to false for AWS infrastructure | | ccm.batch-processing.cliProxy | object | `{"enabled":false,"host":"localhost","password":"","port":80,"protocol":"http","username":""}` | Set ccm.batch-processing.cliProxy.protocol to http or https depending on the proxy configuration | | ccm.batch-processing.stackDriverLoggingEnabled | bool | `false` | Set ccm.batch-processing.stackDriverLoggingEnabled to true for GCP infrastructure | @@ -292,7 +348,7 @@ registry.k8s.io/ingress-nginx/controller:v1.14.0 | ccm.ce-nextgen.stackDriverLoggingEnabled | bool | `false` | Set ccm.nextgen-ce.stackDriverLoggingEnabled to true for GCP infrastructure | | ccm.cloud-info.proxy | object | `{"httpsProxyEnabled":false,"httpsProxyUrl":"http://localhost"}` | Set ccm.cloud-info.proxy.httpsProxyUrl to proxy url(ex: http://localhost:8080, if http proxy is running on localhost port 8080) | | ccm.event-service | object | `{"stackDriverLoggingEnabled":false}` | Set ccm.event-service.stackDriverLoggingEnabled to true for GCP infrastructure | -| cd | object | `{"gitops":{"upgraderImage":{"image":{"tag":"1.10.0"}}}}` | Config for CD services | +| cd | object | `{"gitops":{"agentRedisImage":{"image":{"repository":"harness/redis","tag":"7.4.8-jammy"}},"upgraderImage":{"image":{"tag":"1.11.0"}}}}` | Config for CD services | | chaos.chaos-common.installLinuxCRDs | bool | `false` | | | chaos.chaos-k8s-ifs.nodeSelector | object | `{}` | | | chaos.chaos-k8s-ifs.tolerations | list | `[]` | | @@ -310,32 +366,32 @@ registry.k8s.io/ingress-nginx/controller:v1.14.0 | chaos.chaos-web.tolerations | list | `[]` | | | ci | object | `{"ci-manager":{"affinity":{},"config":{"ENV":"SMP","REFID_CACHE_IMAGE":"harness/refid-cache:latest"},"nodeSelector":{},"tolerations":[]},"ti-service":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]}}` | Install the Continuous Integration (CI) manager pod | | code.code-api.autoai.postgres.image.repository | string | `"harness/postgresql"` | | -| code.code-api.autoai.postgres.image.tag | string | `"14.20-alpine3.23"` | | +| code.code-api.autoai.postgres.image.tag | string | `"14.20-debian"` | | | code.code-api.postgres.image.repository | string | `"harness/postgresql"` | | -| code.code-api.postgres.image.tag | string | `"14.20-alpine3.23"` | | +| code.code-api.postgres.image.tag | string | `"14.20-debian"` | | | code.code-githa.postgres.image.repository | string | `"harness/postgresql"` | | -| code.code-githa.postgres.image.tag | string | `"14.20-alpine3.23"` | | +| code.code-githa.postgres.image.tag | string | `"14.20-debian"` | | | code.code-gitrpc.postgres.image.repository | string | `"harness/postgresql"` | | -| code.code-gitrpc.postgres.image.tag | string | `"14.20-alpine3.23"` | | +| code.code-gitrpc.postgres.image.tag | string | `"14.20-debian"` | | | code.code-search.postgres.image.repository | string | `"harness/postgresql"` | | -| code.code-search.postgres.image.tag | string | `"14.20-alpine3.23"` | | +| code.code-search.postgres.image.tag | string | `"14.20-debian"` | | | enabled | bool | `false` | | | ff.ff-psql-migrations.postgres.image.repository | string | `"harness/postgresql"` | | -| ff.ff-psql-migrations.postgres.image.tag | string | `"14.20-alpine3.23"` | | +| ff.ff-psql-migrations.postgres.image.tag | string | `"14.20-debian"` | | | ff.ff-pushpin-service.postgres.image.repository | string | `"harness/postgresql"` | | -| ff.ff-pushpin-service.postgres.image.tag | string | `"14.20-alpine3.23"` | | +| ff.ff-pushpin-service.postgres.image.tag | string | `"14.20-debian"` | | | ff.ff-pushpin-service.waitForInitContainer.image.tag | string | `"1.2.0"` | | | ff.ff-service.ff-admin-server.postgres.image.repository | string | `"harness/postgresql"` | | -| ff.ff-service.ff-admin-server.postgres.image.tag | string | `"14.20-alpine3.23"` | | +| ff.ff-service.ff-admin-server.postgres.image.tag | string | `"14.20-debian"` | | | ff.ff-service.ff-admin-server.secrets.default.PLATFORM_AUTH_KEY | string | `"secret"` | | | ff.ff-service.ff-client-server.postgres.image.repository | string | `"harness/postgresql"` | | -| ff.ff-service.ff-client-server.postgres.image.tag | string | `"14.20-alpine3.23"` | | +| ff.ff-service.ff-client-server.postgres.image.tag | string | `"14.20-debian"` | | | ff.ff-service.ff-client-server.secrets.default.PLATFORM_AUTH_KEY | string | `"secret"` | | | ff.ff-service.ff-metrics-server.postgres.image.repository | string | `"harness/postgresql"` | | -| ff.ff-service.ff-metrics-server.postgres.image.tag | string | `"14.20-alpine3.23"` | | +| ff.ff-service.ff-metrics-server.postgres.image.tag | string | `"14.20-debian"` | | | ff.ff-service.ff-metrics-server.secrets.default.PLATFORM_AUTH_KEY | string | `"secret"` | | | ff.ff-timescale-migrations.postgres.image.repository | string | `"harness/postgresql"` | | -| ff.ff-timescale-migrations.postgres.image.tag | string | `"14.20-alpine3.23"` | | +| ff.ff-timescale-migrations.postgres.image.tag | string | `"14.20-debian"` | | | global.airgap | string | `"false"` | Airgap functionality. Disabled by default | | global.autoscaling | object | `{"enabled":true}` | Enable to set auto-scaling globally | | global.awsServiceEndpointUrls | object | `{"cloudwatchEndPointUrl":"https://monitoring.us-east-2.amazonaws.com","ecsEndPointUrl":"https://ecs.us-east-2.amazonaws.com","enabled":false,"endPointRegion":"us-east-2","stsEndPointUrl":"https://sts.us-east-2.amazonaws.com"}` | Set global.awsServiceEndpointUrls.cloudwatchEndPointUrl to set cloud watch endpoint url | @@ -443,23 +499,23 @@ registry.k8s.io/ingress-nginx/controller:v1.14.0 | global.waitForInitContainer.image.pullPolicy | string | `"Always"` | | | global.waitForInitContainer.image.registry | string | `"docker.io"` | | | global.waitForInitContainer.image.repository | string | `"harness/helm-init-container"` | | -| global.waitForInitContainer.image.tag | string | `"1.7.0"` | | +| global.waitForInitContainer.image.tag | string | `"1.8.0"` | | | iacm.iac-server.affinity | object | `{}` | | | iacm.iac-server.autoscaling.enabled | bool | `false` | | | iacm.iac-server.createDb.image.repository | string | `"harness/postgresql"` | | -| iacm.iac-server.createDb.image.tag | string | `"14.20-alpine3.23"` | | +| iacm.iac-server.createDb.image.tag | string | `"14.20-debian"` | | | iacm.iac-server.nodeSelector | object | `{}` | | | iacm.iac-server.postgres.image.repository | string | `"harness/postgresql"` | | -| iacm.iac-server.postgres.image.tag | string | `"14.20-alpine3.23"` | | +| iacm.iac-server.postgres.image.tag | string | `"14.20-debian"` | | | iacm.iac-server.tolerations | list | `[]` | | | iacm.iacm-manager.affinity | object | `{}` | | | iacm.iacm-manager.autoscaling.enabled | bool | `false` | | | iacm.iacm-manager.nodeSelector | object | `{}` | | | iacm.iacm-manager.tolerations | list | `[]` | | | idp.idp-app-ui.postgres.image.repository | string | `"harness/postgresql"` | | -| idp.idp-app-ui.postgres.image.tag | string | `"14.20-alpine3.23"` | | -| platform | object | `{"access-control":{"affinity":{},"config":{"ENV":"SMP"},"mongoHosts":[],"mongoSSL":{"enabled":false},"nodeSelector":{},"postgresql":{"image":{"repository":"harness/postgresql","tag":"14.20-alpine3.23"}},"tolerations":[]},"bootstrap":{"database":{"clickhouse":{"enabled":false},"minio":{"affinity":{},"nodeSelector":{},"tolerations":[]},"mongodb":{"affinity":{},"arbiter":{"affinity":{},"nodeSelector":{},"tolerations":[]},"metrics":{"enabled":false},"nodeSelector":{},"podAnnotations":{"prometheus.io/path":"/metrics","prometheus.io/port":"9216","prometheus.io/scrape":"false"},"tolerations":[]},"mongodbupgrades":{"mongoFCVUpgrade":{"affinity":{},"enabled":true,"ignoreFailure":false,"nodeSelector":{},"resources":{},"tolerations":[],"ttlSecondsAfterFinished":900}},"postgresql":{"image":{"repository":"harness/postgresql","tag":"14.20-alpine3.23"},"metrics":{"enabled":false},"podAnnotations":{"prometheus.io/path":"/metrics","prometheus.io/port":"9187","prometheus.io/scrape":"false"}},"redis":{"affinity":{},"metrics":{"enabled":false},"nodeSelector":{},"podAnnotations":{"prometheus.io/path":"/metrics","prometheus.io/port":"9121","prometheus.io/scrape":"false"},"tolerations":[]},"timescaledb":{"affinity":{},"curlImage":{"tag":"8.17.0"},"nodeSelector":{},"persistentVolumes":{"data":{"enabled":true,"size":"100Gi"},"wal":{"enabled":true,"size":"1Gi"}},"podAnnotations":{"prometheus.io/path":"/metrics","prometheus.io/port":"9187","prometheus.io/scrape":"false"},"prometheus":{"enabled":false},"tolerations":[]}},"harness-secrets":{"enabled":true},"networking":{"defaultbackend":{"create":false,"resources":{"limits":{"memory":"20Mi"},"requests":{"cpu":"10m","memory":"20Mi"}}},"nginx":{"affinity":{},"controller":{"annotations":{}},"create":false,"healthNodePort":"","healthPort":"","httpNodePort":"","httpsNodePort":"","loadBalancerEnabled":false,"loadBalancerIP":"","nodeSelector":{},"resources":{"limits":{"memory":"512Mi"},"requests":{"cpu":"0.5","memory":"512Mi"}},"tolerations":[]}}},"change-data-capture":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"delegate-proxy":{"affinity":{},"nodeSelector":{},"tolerations":[]},"gateway":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"harness-manager":{"affinity":{},"config":{"ENV":"SMP"},"featureFlags":{"ADDITIONAL":""},"immutable_delegate_docker_image":{"image":{"digest":"","registry":"docker.io","repository":"harness/delegate","tag":"26.02.88404"}},"nodeSelector":{},"shutdownHooksEnabled":true,"tolerations":{},"upgrader_docker_image":{"image":{"tag":"1.10.0"}}},"log-service":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"looker":{"affinity":{},"nodeSelector":{},"tolerations":[]},"next-gen-ui":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"ng-auth-ui":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"ng-custom-dashboards":{"affinity":{},"nodeSelector":{},"tolerations":[]},"ng-manager":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"shutdownHooksEnabled":true,"tolerations":[]},"pipeline-service":{"affinity":{},"config":{"ENV":"SMP","PUBLISH_ADVISER_EVENT_FOR_CUSTOM_ADVISERS":"true"},"nodeSelector":{},"shutdownHooksEnabled":true,"tolerations":[]},"platform-service":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"scm-service":{"affinity":{},"nodeSelector":{},"tolerations":[]},"template-service":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"ui":{"affinity":{},"nodeSelector":{},"tolerations":[]}}` | Config for platform-level services (always deployed by default to support all services) | -| platform.access-control | object | `{"affinity":{},"config":{"ENV":"SMP"},"mongoHosts":[],"mongoSSL":{"enabled":false},"nodeSelector":{},"postgresql":{"image":{"repository":"harness/postgresql","tag":"14.20-alpine3.23"}},"tolerations":[]}` | Access control settings (taints, tolerations, and so on) | +| idp.idp-app-ui.postgres.image.tag | string | `"14.20-debian"` | | +| platform | object | `{"access-control":{"affinity":{},"config":{"ENV":"SMP"},"mongoHosts":[],"mongoSSL":{"enabled":false},"nodeSelector":{},"postgresql":{"image":{"repository":"harness/postgresql","tag":"14.20-debian"}},"tolerations":[]},"bootstrap":{"database":{"clickhouse":{"enabled":false},"minio":{"affinity":{},"nodeSelector":{},"tolerations":[]},"mongodb":{"affinity":{},"arbiter":{"affinity":{},"nodeSelector":{},"tolerations":[]},"metrics":{"enabled":false},"nodeSelector":{},"podAnnotations":{"prometheus.io/path":"/metrics","prometheus.io/port":"9216","prometheus.io/scrape":"false"},"tolerations":[]},"mongodbupgrades":{"mongoFCVUpgrade":{"affinity":{},"enabled":true,"ignoreFailure":false,"nodeSelector":{},"resources":{},"tolerations":[],"ttlSecondsAfterFinished":900}},"postgresql":{"image":{"repository":"harness/postgresql","tag":"14.20-debian"},"metrics":{"enabled":false},"podAnnotations":{"prometheus.io/path":"/metrics","prometheus.io/port":"9187","prometheus.io/scrape":"false"}},"redis":{"affinity":{},"metrics":{"enabled":false},"nodeSelector":{},"podAnnotations":{"prometheus.io/path":"/metrics","prometheus.io/port":"9121","prometheus.io/scrape":"false"},"tolerations":[]},"timescaledb":{"affinity":{},"curlImage":{"tag":"8.17.0"},"nodeSelector":{},"persistentVolumes":{"data":{"enabled":true,"size":"100Gi"},"wal":{"enabled":true,"size":"1Gi"}},"podAnnotations":{"prometheus.io/path":"/metrics","prometheus.io/port":"9187","prometheus.io/scrape":"false"},"prometheus":{"enabled":false},"tolerations":[]}},"harness-secrets":{"enabled":true},"networking":{"defaultbackend":{"create":false,"resources":{"limits":{"memory":"20Mi"},"requests":{"cpu":"10m","memory":"20Mi"}}},"nginx":{"affinity":{},"controller":{"annotations":{}},"create":false,"healthNodePort":"","healthPort":"","httpNodePort":"","httpsNodePort":"","loadBalancerEnabled":false,"loadBalancerIP":"","nodeSelector":{},"resources":{"limits":{"memory":"512Mi"},"requests":{"cpu":"0.5","memory":"512Mi"}},"tolerations":[]}}},"change-data-capture":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"delegate-proxy":{"affinity":{},"nodeSelector":{},"tolerations":[]},"gateway":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"harness-manager":{"affinity":{},"config":{"ENV":"SMP"},"featureFlags":{"ADDITIONAL":""},"immutable_delegate_docker_image":{"image":{"digest":"","registry":"docker.io","repository":"harness/delegate","tag":"26.02.88404"}},"nodeSelector":{},"shutdownHooksEnabled":true,"tolerations":{},"upgrader_docker_image":{"image":{"tag":"1.11.0"}}},"log-service":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"looker":{"affinity":{},"nodeSelector":{},"tolerations":[]},"next-gen-ui":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"ng-auth-ui":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"ng-custom-dashboards":{"affinity":{},"nodeSelector":{},"tolerations":[]},"ng-manager":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"shutdownHooksEnabled":true,"tolerations":[]},"pipeline-service":{"affinity":{},"config":{"ENV":"SMP","PUBLISH_ADVISER_EVENT_FOR_CUSTOM_ADVISERS":"true"},"nodeSelector":{},"shutdownHooksEnabled":true,"tolerations":[]},"platform-service":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"scm-service":{"affinity":{},"nodeSelector":{},"tolerations":[]},"template-service":{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]},"ui":{"affinity":{},"nodeSelector":{},"tolerations":[]}}` | Config for platform-level services (always deployed by default to support all services) | +| platform.access-control | object | `{"affinity":{},"config":{"ENV":"SMP"},"mongoHosts":[],"mongoSSL":{"enabled":false},"nodeSelector":{},"postgresql":{"image":{"repository":"harness/postgresql","tag":"14.20-debian"}},"tolerations":[]}` | Access control settings (taints, tolerations, and so on) | | platform.access-control.mongoHosts | list | `[]` | - replica3.host.com:27017 | | platform.access-control.mongoSSL | object | `{"enabled":false}` | enable mongoSSL for external database connections | | platform.bootstrap.networking.defaultbackend.create | bool | `false` | Create will deploy a default backend into your cluster | @@ -468,7 +524,7 @@ registry.k8s.io/ingress-nginx/controller:v1.14.0 | platform.change-data-capture | object | `{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]}` | change-data-capture settings (taints, tolerations, and so on) | | platform.delegate-proxy | object | `{"affinity":{},"nodeSelector":{},"tolerations":[]}` | delegate proxy settings (taints, tolerations, and so on) | | platform.gateway | object | `{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]}` | gateway settings (taints, tolerations, and so on) | -| platform.harness-manager | object | `{"affinity":{},"config":{"ENV":"SMP"},"featureFlags":{"ADDITIONAL":""},"immutable_delegate_docker_image":{"image":{"digest":"","registry":"docker.io","repository":"harness/delegate","tag":"26.02.88404"}},"nodeSelector":{},"shutdownHooksEnabled":true,"tolerations":{},"upgrader_docker_image":{"image":{"tag":"1.10.0"}}}` | harness-manager (taints, tolerations, and so on) | +| platform.harness-manager | object | `{"affinity":{},"config":{"ENV":"SMP"},"featureFlags":{"ADDITIONAL":""},"immutable_delegate_docker_image":{"image":{"digest":"","registry":"docker.io","repository":"harness/delegate","tag":"26.02.88404"}},"nodeSelector":{},"shutdownHooksEnabled":true,"tolerations":{},"upgrader_docker_image":{"image":{"tag":"1.11.0"}}}` | harness-manager (taints, tolerations, and so on) | | platform.harness-manager.featureFlags | object | `{"ADDITIONAL":""}` | Feature Flags | | platform.harness-manager.featureFlags.ADDITIONAL | string | `""` | Additional Feature Flag (placeholder to add any other featureFlags) | | platform.log-service | object | `{"affinity":{},"config":{"ENV":"SMP"},"nodeSelector":{},"tolerations":[]}` | log-service (taints, tolerations, and so on) | @@ -501,8 +557,8 @@ registry.k8s.io/ingress-nginx/controller:v1.14.0 | srm.le-nextgen.affinity | object | `{}` | | | srm.le-nextgen.nodeSelector | object | `{}` | | | srm.le-nextgen.tolerations | list | `[]` | | -| sto | object | `{"sto-core":{"affinity":{},"autoscaling":{"enabled":false},"migrationPostgres":{"image":{"repository":"harness/postgresql","tag":"14.20-alpine3.23"}},"nodeSelector":{},"postgres":{"image":{"repository":"harness/postgresql","tag":"14.20-alpine3.23"}},"tolerations":[]},"ticket-service":{"postgres":{"image":{"repository":"harness/postgresql","tag":"14.20-alpine3.23"}}}}` | Config for Security Test Orchestration (STO) | -| sto.ticket-service | object | `{"postgres":{"image":{"repository":"harness/postgresql","tag":"14.20-alpine3.23"}}}` | Install the STO core | +| sto | object | `{"sto-core":{"affinity":{},"autoscaling":{"enabled":false},"migrationPostgres":{"image":{"repository":"harness/postgresql","tag":"14.20-debian"}},"nodeSelector":{},"postgres":{"image":{"repository":"harness/postgresql","tag":"14.20-debian"}},"tolerations":[]},"ticket-service":{"postgres":{"image":{"repository":"harness/postgresql","tag":"14.20-debian"}}}}` | Config for Security Test Orchestration (STO) | +| sto.ticket-service | object | `{"postgres":{"image":{"repository":"harness/postgresql","tag":"14.20-debian"}}}` | Install the STO core | | upgrades.versionLookups.enabled | bool | `true` | | ---------------------------------------------- diff --git a/src/harness/images.txt b/src/harness/images.txt index cab5be60..4cf09b63 100644 --- a/src/harness/images.txt +++ b/src/harness/images.txt @@ -1,98 +1,114 @@ +## Core Platform Services (Required for all installations) docker.io/busybox:1.37.0 -docker.io/haproxy:lts-alpine3.23 -docker.io/harness/accesscontrol-service-signed:1.195.0 -docker.io/harness/argocd:v3.2.5 -docker.io/harness/audit-event-streaming-signed:1.74.0 -docker.io/harness/batch-processing-signed:1.77.14 -docker.io/harness/ccm-gcp-smp-signed:100079 -docker.io/harness/cdcdata-signed:1.49.7 -docker.io/harness/ce-anomaly-detection-signed:1.21.0 -docker.io/harness/ce-cloud-info-signed:1.13.2 -docker.io/harness/ce-nextgen-signed:1.79.22 -docker.io/harness/chaos-ddcr-faults:1.74.0 -docker.io/harness/chaos-ddcr:1.74.0 -docker.io/harness/chaos-event-watcher:1.74.0 -docker.io/harness/chaos-log-watcher:1.74.0 -docker.io/harness/chaos-machine-ifc-signed:1.74.0 -docker.io/harness/chaos-machine-ifs-signed:1.74.0 -docker.io/harness/ci-manager-signed:1.120.5 -docker.io/harness/ci-scm-signed:1.44.0 -docker.io/harness/clickhouse:24.12.4-debian-12-r1 -docker.io/harness/code-api-signed:1.73.3 -docker.io/harness/code-githa-signed:1.73.1 -docker.io/harness/code-gitrpc-signed:1.73.1 -docker.io/harness/code-search-signed:1.73.1 -docker.io/harness/component-analysis-service-signed:1.1.7 -docker.io/harness/component-service-signed:1.6.30 -docker.io/harness/cv-nextgen-signed:1.55.1 -docker.io/harness/dashboard-service-signed:1.98.0 -docker.io/harness/db-devops-service-signed:1.78.3 +docker.io/harness/ci-scm-signed:1.45.1 +docker.io/harness/template-service-signed:1.134.0 +docker.io/harness/platform-service-signed:1.110.0 +docker.io/harness/pipeline-service-signed:1.172.8 +docker.io/harness/ng-manager-signed:1.132.4 +docker.io/harness/nextgenui-signed:1.119.5 +docker.io/harness/minio:RELEASE.2025-10-15T17-29-55Z-jammy +docker.io/harness/log-service-signed:1.41.1 +docker.io/harness/cdcdata-signed:1.51.2 +docker.io/harness/accesscontrol-service-signed:1.208.0 +docker.io/harness/delegate-proxy-signed:1.9.0 +docker.io/harness/gateway-signed:1.62.5 +docker.io/harness/helm-init-container:1.8.0 +docker.io/harness/manager-signed:1.131.5 +docker.io/harness/mongo:7.0.28-jammy +docker.io/harness/ng-auth-ui-signed:1.38.2 +docker.io/harness/redis:7.4.8-jammy +docker.io/harness/postgresql:14.20-debian +docker.io/harness/policy-mgmt:1.38.1 +docker.io/harness/smp-service-discovery-server-signed:0.56.0 docker.io/harness/debezium-service-signed:1.25.1 -docker.io/harness/delegate-proxy-signed:1.6.0 +docker.io/harness/audit-event-streaming-signed:1.77.0 +docker.io/harness/queue-service-signed:1.9.0 +docker.io/harness/service-discovery-collector:0.56.0 +docker.io/harness/ng-dashboard-aggregator-signed:1.96.0 +docker.io/harness/controller:1.14.3-jammy +registry.k8s.io/defaultbackend-amd64:1.5 +docker.io/harness/redis_exporter:1.80.2-jammy +docker.io/prometheuscommunity/postgres-exporter:v0.19.1 +docker.io/harness/mongodb-exporter:0.47.1-jammy +harness/vault-secret-loader:1.0.2 +docker.io/harness/ui-signed:1.32.4 +docker.io/harness/delegate:26.02.88404.minimal +docker.io/harness/delegate:26.02.88404.minimal-fips + +### Platform Agents docker.io/harness/delegate:26.02.88404 -docker.io/harness/delegate:26.02.88404-fips docker.io/harness/delegate:26.02.88404.minimal docker.io/harness/delegate:26.02.88404.minimal-fips -docker.io/harness/enterprise-chaos-hub-signed:1.74.4 -docker.io/harness/event-service-signed:1.14.1 -docker.io/harness/ff-cron-signed:1.1154.1 -docker.io/harness/ff-pushpin-signed:1.1135.0 -docker.io/harness/ff-pushpin-worker-signed:1.1135.0 -docker.io/harness/ff-server-analytics-db-migration-signed:1.1154.1 -docker.io/harness/ff-server-primary-db-migration-signed:1.1154.1 -docker.io/harness/ff-service-signed:1.1154.1 -docker.io/harness/gateway-signed:1.60.3 -docker.io/harness/gitops-agent-installer-helper:v0.0.9 -docker.io/harness/gitops-agent:v0.109.0 -docker.io/harness/gitops-service-signed:1.50.4 -docker.io/harness/helm-init-container:1.7.0 -docker.io/harness/iac-server-signed:1.302.0 -docker.io/harness/iacm-manager-signed:1.128.1 -docker.io/harness/idp-admin-signed:1.36.3 -docker.io/harness/idp-app-signed:1.36.11 -docker.io/harness/idp-service-signed:1.36.15 -docker.io/harness/le-nextgen-signed:1.13.0 -docker.io/harness/log-service-signed:1.41.0 -docker.io/harness/looker-signed:1.8.9 -docker.io/harness/manager-signed:1.128.4 -docker.io/harness/minio:2025.7.18-debian-12-r0 -docker.io/harness/mongo:7.0.28-debian-12-r1 -docker.io/harness/mongodb-exporter:0.40.0-debian-12-r40 -docker.io/harness/nextgenui-signed:1.116.11 -docker.io/harness/ng-auth-ui-signed:1.38.2 -docker.io/harness/ng-ce-ui:1.75.7 -docker.io/harness/ng-dashboard-aggregator-signed:1.93.0 -docker.io/harness/ng-manager-signed:1.129.7 -docker.io/harness/pipeline-service-signed:1.169.5 -docker.io/harness/platform-service-signed:1.107.0 -docker.io/harness/policy-mgmt:1.36.5 -docker.io/harness/postgresql:14.20-alpine3.23 -docker.io/harness/queue-service-signed:1.8.1 -docker.io/harness/redis:7.4.7-alpine -docker.io/harness/refid-cache:latest -docker.io/harness/service-discovery-collector:0.54.0 -docker.io/harness/smp-chaos-bg-processor-signed:1.74.4 -docker.io/harness/smp-chaos-k8s-ifs-signed:1.74.0 -docker.io/harness/smp-chaos-linux-infra-controller-signed:1.74.0 -docker.io/harness/smp-chaos-linux-infra-server-signed:1.74.0 -docker.io/harness/smp-chaos-manager-signed:1.74.4 -docker.io/harness/smp-chaos-web-signed:1.74.1 -docker.io/harness/smp-service-discovery-server-signed:0.54.0 -docker.io/harness/source-probe:main-latest -docker.io/harness/srm-ui-signed:1.16.0 -docker.io/harness/ssca-manager-signed:1.50.14 -docker.io/harness/ssca-ui-signed:0.38.5 -docker.io/harness/statsd-exporter:1.0-prometheus-busybox-1 -docker.io/harness/stocore-signed:1.180.4 -docker.io/harness/telescopes-signed:1.6.0 -docker.io/harness/template-service-signed:1.131.3 -docker.io/harness/ti-service-signed:1.60.5 -docker.io/harness/ticket-service-signed:1.4.5 -docker.io/harness/ui-signed:1.32.3 -docker.io/harness/upgrader:1.10.0 +docker.io/harness/delegate:26.02.88404-fips +docker.io/harness/upgrader:1.11.0 +docker.io/harness/upgrader:1.11.0-fips + +### Dashboard +docker.io/harness/looker-signed:1.10.1 +docker.io/harness/dashboard-service-signed:1.102.0 +docker.io/harness/statsd-exporter:3.0-prometheus-busybox-2 + +## Continuous Deployment +docker.io/harness/gitops-service-signed:1.51.3 +docker.io/harness/cv-nextgen-signed:1.57.2 +docker.io/harness/le-nextgen-signed:1.13.1 +docker.io/harness/srm-ui-signed:1.16.2 + +### CD Agents +docker.io/harness/argocd:v3.3.0 +docker.io/harness/gitops-agent:v0.110.1 +docker.io/haproxy:lts-alpine3.23 docker.io/koalaman/shellcheck:v0.5.0 -docker.io/oliver006/redis_exporter:v1.80.2 -docker.io/prometheuscommunity/postgres-exporter:v0.15.0 +docker.io/harness/gitops-agent-installer-helper:v0.0.11-rebuilt + +## Continuous Integration +docker.io/harness/ci-manager-signed:1.123.5 +docker.io/harness/ti-service-signed:1.61.3 +harness/ci-addon:1.18.10 +harness/ci-addon:1.18.6 +harness/ci-addon:rootless-1.18.10 +harness/ci-addon:rootless-1.18.6 +harness/ci-lite-engine:1.18.10 +harness/ci-lite-engine:1.18.6 +harness/ci-lite-engine:rootless-1.18.10 +harness/ci-lite-engine:rootless-1.18.6 + +### CI Build Plugins +harness/drone-git:1.7.16-rootless +harness/harness-cache-server:1.7.13 +plugins/kaniko:1.13.4 +plugins/kaniko-acr:1.13.5 +plugins/kaniko-ecr:1.13.4 +plugins/kaniko-gcr:1.13.4 +plugins/artifactory:1.8.3 +plugins/artifactory:1.8.4 +plugins/gcs:1.6.7 +plugins/gcs:1.6.8 +plugins/cache:1.9.18 +plugins/cache:1.9.21 +plugins/cache:1.9.24 +plugins/s3:1.5.6 +plugins/s3:1.5.8 +plugins/buildx:1.3.13 +plugins/buildx-acr:1.4.4 +plugins/buildx-ecr:1.4.3 +plugins/buildx-gar:1.4.3 +plugins/buildx-gcr:1.4.3 +plugins/docker:21.2.2 +plugins/ecr:21.2.2 +plugins/gar:21.2.2 +plugins/gcr:21.2.2 +plugins/acr:21.2.2 + +## Security Testing Orchestration +docker.io/harness/stocore-signed:1.182.0 +docker.io/harness/ticket-service-signed:1.8.0 +docker.io/harness/refid-cache:latest +harness/refid-cache:latest +harness/sto-plugin:latest +harness/sto-plugin:latest-fips + +### STO Security Scanners harness/anchore-job-runner:latest harness/aqua-security-job-runner:latest harness/aqua-trivy-job-runner:latest @@ -106,35 +122,12 @@ harness/brakeman-job-runner:latest harness/burp-job-runner:latest harness/checkmarx-job-runner:latest harness/checkov-job-runner:latest -harness/ci-addon:1.18.2 -harness/ci-addon:1.18.6 -harness/ci-addon:1.18.7 -harness/ci-addon:rootless-1.18.2 -harness/ci-addon:rootless-1.18.6 -harness/ci-addon:rootless-1.18.7 -harness/ci-lite-engine:1.18.2 -harness/ci-lite-engine:1.18.6 -harness/ci-lite-engine:1.18.7 -harness/ci-lite-engine:rootless-1.18.2 -harness/ci-lite-engine:rootless-1.18.6 -harness/ci-lite-engine:rootless-1.18.7 -harness/cookiecutter:1.19.0 -harness/createcatalog:1.19.0 -harness/createorganisation:1.19.0 -harness/createproject:1.19.0 -harness/createrepo:1.19.0 -harness/createresource:1.19.0 -harness/directpush:1.19.0 harness/docker-content-trust-job-runner:latest -harness/drone-git:1.7.10-rootless -harness/drone-git:1.7.15-rootless harness/fossa-job-runner:latest harness/github-advanced-security-job-runner:latest harness/gitleaks-job-runner:latest harness/grype-job-runner:latest harness/grype-job-runner:latest-fips -harness/harness-cache-server:1.7.10 -harness/harness-cache-server:1.7.8 harness/modelscan-job-runner:latest harness/nexusiq-job-runner:latest harness/nikto-job-runner:latest @@ -143,55 +136,118 @@ harness/osv-job-runner:latest harness/osv-job-runner:latest-fips harness/owasp-dependency-check-job-runner:latest harness/prowler-job-runner:latest -harness/refid-cache:latest -harness/registercatalog:1.19.0 harness/semgrep-job-runner:latest harness/semgrep-job-runner:latest-fips harness/shiftleft-job-runner:latest -harness/slacknotify:1.19.0 -harness/slsa-plugin:0.52.1 harness/snyk-job-runner:latest harness/sonarqube-agent-job-runner:latest harness/sonarqube-agent-job-runner:latest-fips -harness/ssca-artifact-signing-plugin:0.52.1 -harness/ssca-cdxgen-plugin:0.52.1 -harness/ssca-compliance-plugin:0.52.1 -harness/ssca-plugin:0.52.1 -harness/sto-plugin:latest harness/sysdig-job-runner:latest harness/traceable-job-runner:latest harness/twistlock-job-runner:latest -harness/updatecatalogproperty:1.19.0 -harness/upgrader:1.10.0-fips -harness/vault-secret-loader:1.0.2 harness/veracode-agent-job-runner:latest harness/whitesource-agent-job-runner:latest harness/wiz-job-runner:latest harness/zap-job-runner:latest -plugins/acr:21.2.2 -plugins/artifactory:1.8.3 -plugins/buildx-acr:1.4.3 -plugins/buildx-ecr:1.4.3 -plugins/buildx-gar:1.4.3 -plugins/buildx-gcr:1.4.3 -plugins/buildx:1.3.13 -plugins/cache:1.9.18 -plugins/docker:21.2.2 -plugins/ecr:21.2.2 -plugins/gar:21.2.2 -plugins/gcr:21.2.2 -plugins/gcs:1.6.7 + +## Feature Flags +docker.io/harness/ff-cron-signed:1.1160.0 +docker.io/harness/ff-server-analytics-db-migration-signed:1.1160.0 +docker.io/harness/ff-server-primary-db-migration-signed:1.1160.0 +docker.io/harness/ff-service-signed:1.1160.0 +docker.io/harness/ff-pushpin-signed:1.1138.0 +docker.io/harness/ff-pushpin-worker-signed:1.1138.0 + +## Cloud Cost Management +docker.io/harness/batch-processing-signed:1.78.5 +docker.io/harness/ce-anomaly-detection-signed:1.22.0 +docker.io/harness/ce-cloud-info-signed:1.14.4 +docker.io/harness/ce-nextgen-signed:1.80.6 +docker.io/harness/event-service-signed:1.15.2 +docker.io/harness/ng-ce-ui:1.76.3 +docker.io/harness/telescopes-signed:1.7.2 +docker.io/harness/clickhouse:25.12.5-jammy +docker.io/harness/ccm-gcp-smp-signed:100079 + +## Chaos Engineering +docker.io/harness/smp-chaos-k8s-ifs-signed:1.76.0 +docker.io/harness/smp-chaos-linux-infra-controller-signed:1.76.0 +docker.io/harness/smp-chaos-linux-infra-server-signed:1.76.1 +docker.io/harness/smp-chaos-manager-signed:1.76.2 +docker.io/harness/smp-chaos-web-signed:1.76.2 +docker.io/harness/source-probe:main-latest +docker.io/harness/smp-chaos-bg-processor-signed:1.76.2 +docker.io/harness/chaos-machine-ifc-signed:1.76.0 +docker.io/harness/chaos-machine-ifs-signed:1.76.0 +docker.io/harness/enterprise-chaos-hub-signed:1.76.2 + +### Chaos Engineering Plugins +docker.io/harness/chaos-log-watcher:1.76.0 +docker.io/harness/chaos-ddcr:1.76.0 +docker.io/harness/chaos-ddcr-faults:1.76.0 +docker.io/harness/chaos-event-watcher:1.76.0 + +## Supply Chain Security +docker.io/harness/ssca-manager-signed:1.52.13 +docker.io/harness/ssca-ui-signed:0.39.2 +docker.io/harness/component-service-signed:1.7.1 +docker.io/harness/component-analysis-service-signed:1.2.1 + +### SCS Plugins +harness/ssca-plugin:0.53.5 +harness/slsa-plugin:0.53.2 +harness/ssca-cdxgen-plugin:0.53.5 +harness/ssca-compliance-plugin:0.53.4 +harness/ssca-artifact-signing-plugin:0.53.2 + +## Database DevOps +docker.io/harness/db-devops-service-signed:1.81.0 + +## Code Repository +docker.io/harness/code-api-signed:1.76.1 +docker.io/harness/code-githa-signed:1.76.0 +docker.io/harness/code-gitrpc-signed:1.76.0 +docker.io/harness/code-search-signed:1.76.0 + +## Infrastructure as Code Management +docker.io/harness/iac-server-signed:1.329.0 +docker.io/harness/iacm-manager-signed:1.134.0 + +### IACM Plugins +harness/ci-addon:1.18.10 +harness/ci-addon:1.18.6 +harness/ci-addon:rootless-1.18.10 +harness/ci-addon:rootless-1.18.6 +harness/ci-lite-engine:1.18.10 +harness/ci-lite-engine:1.18.6 +harness/ci-lite-engine:rootless-1.18.10 +harness/ci-lite-engine:rootless-1.18.6 +harness/drone-git:1.7.16-rootless plugins/harness_terraform:latest plugins/harness_terraform_vm:latest -plugins/kaniko-acr:1.13.4 -plugins/kaniko-ecr:1.13.3 -plugins/kaniko-ecr:1.13.4 -plugins/kaniko-gcr:1.13.3 -plugins/kaniko-gcr:1.13.4 -plugins/kaniko:1.13.3 -plugins/kaniko:1.13.4 -plugins/s3:1.5.4 -plugins/s3:1.5.6 -plugins/s3:1.5.7 -registry.k8s.io/defaultbackend-amd64:1.5 -registry.k8s.io/ingress-nginx/controller:v1.14.0 + +## Internal Developer Portal +docker.io/harness/idp-service-signed:1.37.5 +docker.io/harness/idp-admin-signed:1.37.2 +docker.io/harness/idp-app-signed:1.37.9 + +### IDP Plugins +harness/ci-addon:1.18.10 +harness/ci-addon:1.18.6 +harness/ci-addon:rootless-1.18.10 +harness/ci-addon:rootless-1.18.6 +harness/ci-lite-engine:1.18.10 +harness/ci-lite-engine:1.18.6 +harness/ci-lite-engine:rootless-1.18.10 +harness/ci-lite-engine:rootless-1.18.6 +harness/drone-git:1.7.16-rootless +harness/cookiecutter:1.20.0 +harness/createcatalog:1.20.0 +harness/createorganisation:1.20.0 +harness/createproject:1.20.0 +harness/createrepo:1.20.0 +harness/createresource:1.20.0 +harness/directpush:1.20.0 +harness/registercatalog:1.20.0 +harness/slacknotify:1.20.0 +harness/updatecatalogproperty:1.20.0 diff --git a/src/harness/values.yaml b/src/harness/values.yaml index 868d2ffd..412ae8cf 100644 --- a/src/harness/values.yaml +++ b/src/harness/values.yaml @@ -254,7 +254,7 @@ global: registry: docker.io repository: harness/helm-init-container pullPolicy: Always - tag: "1.7.0" + tag: "1.8.0" digest: "" imagePullSecrets: [] fileLogging: @@ -317,7 +317,7 @@ ccm: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" # -- Set ccm.batch-processing.awsAccountTagsCollectionJobConfig.enabled to false for AWS infrastructure awsAccountTagsCollectionJobConfig: enabled: true @@ -400,9 +400,13 @@ enabled: false # -- Config for CD services cd: gitops: + agentRedisImage: + image: + repository: harness/redis + tag: "7.4.8-jammy" upgraderImage: image: - tag: 1.10.0 + tag: 1.11.0 # -- Config for platform-level services (always deployed by default to support all services) platform: # -- Access control settings (taints, tolerations, and so on) @@ -410,7 +414,7 @@ platform: postgresql: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" config: ENV: "SMP" affinity: {} @@ -461,7 +465,7 @@ platform: postgresql: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" metrics: enabled: false podAnnotations: @@ -553,7 +557,7 @@ platform: harness-manager: upgrader_docker_image: image: - tag: 1.10.0 + tag: 1.11.0 config: ENV: "SMP" shutdownHooksEnabled: true @@ -660,16 +664,16 @@ sto: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" sto-core: migrationPostgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" affinity: {} autoscaling: enabled: false @@ -684,7 +688,7 @@ ff: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" ff-client-server: secrets: default: @@ -692,7 +696,7 @@ ff: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" ff-metrics-server: secrets: default: @@ -700,7 +704,7 @@ ff: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" ff-pushpin-service: waitForInitContainer: image: @@ -708,27 +712,27 @@ ff: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" ff-psql-migrations: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" ff-timescale-migrations: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" iacm: iac-server: createDb: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" affinity: {} autoscaling: enabled: false @@ -748,33 +752,33 @@ code: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" autoai: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" code-search: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" code-githa: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" code-gitrpc: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" idp: idp-app-ui: postgres: image: repository: "harness/postgresql" - tag: "14.20-alpine3.23" + tag: "14.20-debian" postmigrationCheck: enabled: true image: diff --git a/src/modules/ccm/Chart.lock b/src/modules/ccm/Chart.lock index a82688e5..e0d4d42f 100644 --- a/src/modules/ccm/Chart.lock +++ b/src/modules/ccm/Chart.lock @@ -1,24 +1,24 @@ dependencies: - name: event-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.14.1 + version: 1.15.2 - name: anomaly-detection repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.21.0 + version: 1.22.0 - name: cloud-info repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.13.2 + version: 1.14.4 - name: telescopes repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.6.0 + version: 1.7.2 - name: batch-processing repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.77.14 + version: 1.78.5 - name: ng-ce-ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.75.7 + version: 1.76.3 - name: ce-nextgen repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.79.22 -digest: sha256:99578ea74eec633cb6527eb71ee1a14c3ec448adf83e9c8b3cb36945339f1eae -generated: "2026-02-24T16:50:06.406369775Z" + version: 1.80.6 +digest: sha256:90b5b82e8f74343de91a41c951614741db1e48aa8c291acd8fe4af235b83c910 +generated: "2026-03-08T16:13:41.652904567Z" diff --git a/src/modules/ccm/Chart.yaml b/src/modules/ccm/Chart.yaml index e001b9c0..3013c91d 100644 --- a/src/modules/ccm/Chart.yaml +++ b/src/modules/ccm/Chart.yaml @@ -3,25 +3,25 @@ appVersion: 1.546.0 dependencies: - name: event-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.14.1 + version: 1.15.2 - name: anomaly-detection repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.21.0 + version: 1.22.0 - name: cloud-info repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.13.2 + version: 1.14.4 - name: telescopes repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.6.0 + version: 1.7.2 - name: batch-processing repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.77.14 + version: 1.78.5 - name: ng-ce-ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.75.7 + version: 1.76.3 - name: ce-nextgen repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.79.22 + version: 1.80.6 description: A Helm chart for Harness Cloud Cost Management (CCM) module name: ccm type: application diff --git a/src/modules/cd/Chart.lock b/src/modules/cd/Chart.lock index 88809a4e..ac8dc91c 100644 --- a/src/modules/cd/Chart.lock +++ b/src/modules/cd/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: gitops repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.50.4 -digest: sha256:be602fea728252f67fb84b1f063451f9352a8341f06d6af35502c18fa9efd73f -generated: "2026-02-24T16:50:08.075113838Z" + version: 1.51.3 +digest: sha256:57f37e8d624ebf2ae0d1bfabf4528f879fb5ee94afc37508f3b0736bcf8d2cf9 +generated: "2026-03-08T16:13:50.970026919Z" diff --git a/src/modules/cd/Chart.yaml b/src/modules/cd/Chart.yaml index d54276db..ccf1247d 100644 --- a/src/modules/cd/Chart.yaml +++ b/src/modules/cd/Chart.yaml @@ -4,7 +4,7 @@ dependencies: - condition: global.cd.enabled name: gitops repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.50.4 + version: 1.51.3 description: Helm chart for Harness CD name: cd type: application diff --git a/src/modules/chaos/Chart.lock b/src/modules/chaos/Chart.lock index 23928d56..f06aed8c 100644 --- a/src/modules/chaos/Chart.lock +++ b/src/modules/chaos/Chart.lock @@ -1,27 +1,27 @@ dependencies: - name: chaos-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.4 + version: 1.76.2 - name: chaos-web repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.1 + version: 1.76.2 - name: chaos-k8s-ifs repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.0 + version: 1.76.0 - name: chaos-linux-ifs repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.0 + version: 1.76.1 - name: chaos-linux-ifc repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.0 + version: 1.76.0 - name: chaos-crd repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts version: 1.73.0 - name: chaos-machine-ifs repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.0 + version: 1.76.0 - name: chaos-machine-ifc repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.0 -digest: sha256:78c1bd143471bae8db245b6356925c688648fe2446bd3e7c535414d356724212 -generated: "2026-02-24T16:49:56.006361394Z" + version: 1.76.0 +digest: sha256:520d040d4eaeffd0ca60d9adaf1e7e1a2f39c8ccb99b2378c70d20fbe68296c3 +generated: "2026-03-08T16:13:51.329702966Z" diff --git a/src/modules/chaos/Chart.yaml b/src/modules/chaos/Chart.yaml index fdc64339..04ca720c 100644 --- a/src/modules/chaos/Chart.yaml +++ b/src/modules/chaos/Chart.yaml @@ -3,28 +3,28 @@ appVersion: 0.6.0 dependencies: - name: chaos-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.4 + version: 1.76.2 - name: chaos-web repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.1 + version: 1.76.2 - name: chaos-k8s-ifs repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.0 + version: 1.76.0 - name: chaos-linux-ifs repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.0 + version: 1.76.1 - name: chaos-linux-ifc repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.0 + version: 1.76.0 - name: chaos-crd repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts version: 1.73.0 - name: chaos-machine-ifs repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.0 + version: 1.76.0 - name: chaos-machine-ifc repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.0 + version: 1.76.0 description: A Helm chart for harness chaos module name: chaos type: application diff --git a/src/modules/ci/Chart.lock b/src/modules/ci/Chart.lock index 271286a5..0b81835e 100644 --- a/src/modules/ci/Chart.lock +++ b/src/modules/ci/Chart.lock @@ -1,9 +1,9 @@ dependencies: - name: ci-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.120.5 + version: 1.123.5 - name: ti-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.60.5 -digest: sha256:cc141b8a6b234b8669a78ccce257d9a32f546c836c30cad674cc0def1f105fb5 -generated: "2026-02-24T16:49:57.873576659Z" + version: 1.61.3 +digest: sha256:9f616ec521df010e411fe50c9ae6b3eb5511a1279fb5f1cbaf9fde4d9f11cf6e +generated: "2026-03-13T05:48:41.272710084Z" diff --git a/src/modules/ci/Chart.yaml b/src/modules/ci/Chart.yaml index 6d405e0e..8576c77e 100644 --- a/src/modules/ci/Chart.yaml +++ b/src/modules/ci/Chart.yaml @@ -3,12 +3,12 @@ appVersion: 1.16.0 dependencies: - name: ci-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.120.5 + version: 1.123.5 - condition: global.ti.enabled name: ti-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.60.5 + version: 1.61.3 description: A Helm chart for Kubernetes name: ci type: application -version: 0.38.0 +version: 0.38.1 diff --git a/src/modules/code/Chart.lock b/src/modules/code/Chart.lock index 8658a6d8..531f067c 100644 --- a/src/modules/code/Chart.lock +++ b/src/modules/code/Chart.lock @@ -1,15 +1,15 @@ dependencies: - name: code-api repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.73.3 + version: 1.76.1 - name: code-gitrpc repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.73.1 + version: 1.76.0 - name: code-githa repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.73.1 + version: 1.76.0 - name: code-search repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.73.1 -digest: sha256:f675b7e9569e32cff720afe1f0b2143955544c613c40f49285d2afdb05eb3ce4 -generated: "2026-02-24T16:49:58.433509797Z" + version: 1.76.0 +digest: sha256:c196e32fda7b2d254f9edf20c5d465a79097180cf1f3ac53005ce6c9bc07f6ae +generated: "2026-03-08T16:13:43.243234569Z" diff --git a/src/modules/code/Chart.yaml b/src/modules/code/Chart.yaml index 12a98f7d..f296ee35 100644 --- a/src/modules/code/Chart.yaml +++ b/src/modules/code/Chart.yaml @@ -4,19 +4,19 @@ dependencies: - condition: global.code.enabled name: code-api repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.73.3 + version: 1.76.1 - condition: global.code.enabled name: code-gitrpc repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.73.1 + version: 1.76.0 - condition: global.code.enabled name: code-githa repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.73.1 + version: 1.76.0 - condition: global.code.enabled name: code-search repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.73.1 + version: 1.76.0 description: A Helm chart for Harness CODE name: code type: application diff --git a/src/modules/db-devops/Chart.lock b/src/modules/db-devops/Chart.lock index 761de681..287cc528 100644 --- a/src/modules/db-devops/Chart.lock +++ b/src/modules/db-devops/Chart.lock @@ -1,6 +1,6 @@ dependencies: - name: db-devops-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.78.3 -digest: sha256:565c0f0684a7c56edc7c949c3db22861305e6d15ba6c5002fd7317a8b2fced8e -generated: "2026-02-24T16:50:08.492611245Z" + version: 1.81.0 +digest: sha256:a4979f570ba77d5cc437586464fba1b0a684565e7625a63150fced46c06173b9 +generated: "2026-03-08T16:13:44.050453726Z" diff --git a/src/modules/db-devops/Chart.yaml b/src/modules/db-devops/Chart.yaml index f64e5226..c924bcf3 100644 --- a/src/modules/db-devops/Chart.yaml +++ b/src/modules/db-devops/Chart.yaml @@ -4,7 +4,7 @@ dependencies: - condition: global.dbops.enabled name: db-devops-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.78.3 + version: 1.81.0 description: A Helm chart for Harness DB devops name: db-devops type: application diff --git a/src/modules/ff/Chart.lock b/src/modules/ff/Chart.lock index f5edef9c..7e048fe0 100644 --- a/src/modules/ff/Chart.lock +++ b/src/modules/ff/Chart.lock @@ -1,9 +1,9 @@ dependencies: - name: ff-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.1154.1 + version: 1.1160.0 - name: ff-pushpin-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.1135.0 -digest: sha256:615eef54e8a8d5d3051d1dbc69e1d86d4ca68e404ee2d3a22a6f71d38453e2ce -generated: "2026-02-24T16:49:59.386268294Z" + version: 1.1138.0 +digest: sha256:fa599fe701d294580a51884c0d4c9b0d02c61215ee04b234856fe987e5e2a20d +generated: "2026-03-08T16:13:44.364706707Z" diff --git a/src/modules/ff/Chart.yaml b/src/modules/ff/Chart.yaml index 444e9251..a6c4deaa 100644 --- a/src/modules/ff/Chart.yaml +++ b/src/modules/ff/Chart.yaml @@ -3,10 +3,10 @@ appVersion: 0.0.1 dependencies: - name: ff-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.1154.1 + version: 1.1160.0 - name: ff-pushpin-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.1135.0 + version: 1.1138.0 description: A Helm chart for harness Feature Flags module name: ff type: application diff --git a/src/modules/iacm/Chart.lock b/src/modules/iacm/Chart.lock index 9e3b675d..29f033e1 100644 --- a/src/modules/iacm/Chart.lock +++ b/src/modules/iacm/Chart.lock @@ -1,9 +1,9 @@ dependencies: - name: iac-server repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.302.0 + version: 1.329.0 - name: iacm-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.128.1 -digest: sha256:adf069b1553a437afdc26441dfeadc1a979fd0da85b2b07b00f75721ecd22c5c -generated: "2026-02-24T16:49:53.377375088Z" + version: 1.134.0 +digest: sha256:bc19c35aee4b5a9b6024c3f03a7582d6998bf01f60d96ee420e5842c1ca80ae4 +generated: "2026-03-08T16:13:39.89900114Z" diff --git a/src/modules/iacm/Chart.yaml b/src/modules/iacm/Chart.yaml index c9fc2e8e..29909093 100644 --- a/src/modules/iacm/Chart.yaml +++ b/src/modules/iacm/Chart.yaml @@ -3,10 +3,10 @@ appVersion: 0.0.79001 dependencies: - name: iac-server repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.302.0 + version: 1.329.0 - name: iacm-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.128.1 + version: 1.134.0 description: Helm chart for Harness IACM name: iacm type: application diff --git a/src/modules/idp/Chart.lock b/src/modules/idp/Chart.lock index 6eb612a9..cb2c1cec 100644 --- a/src/modules/idp/Chart.lock +++ b/src/modules/idp/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: idp-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.36.15 + version: 1.37.5 - name: idp-admin repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.36.3 + version: 1.37.2 - name: idp-app-ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.36.11 -digest: sha256:fc9fbc7f0bf8e471e04a89fef7b4c20e261df130087c639c70b6cc03096a214c -generated: "2026-02-24T16:49:54.151408856Z" + version: 1.37.9 +digest: sha256:1418f4d5a7865196428447001f150c0615950d64c80ea830f27aa01e387b7a0a +generated: "2026-03-08T16:13:44.858159498Z" diff --git a/src/modules/idp/Chart.yaml b/src/modules/idp/Chart.yaml index a629296a..af2680de 100644 --- a/src/modules/idp/Chart.yaml +++ b/src/modules/idp/Chart.yaml @@ -4,15 +4,15 @@ dependencies: - condition: global.idp.enabled name: idp-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.36.15 + version: 1.37.5 - condition: global.idp.enabled name: idp-admin repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.36.3 + version: 1.37.2 - condition: global.idp.enabled name: idp-app-ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.36.11 + version: 1.37.9 description: A Helm chart for Harness IDP name: idp type: application diff --git a/src/modules/platform/Chart.lock b/src/modules/platform/Chart.lock index a1e060d6..32f9323b 100644 --- a/src/modules/platform/Chart.lock +++ b/src/modules/platform/Chart.lock @@ -1,72 +1,72 @@ dependencies: - name: access-control repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.195.0 + version: 1.208.0 - name: bootstrap repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.65.0 + version: 1.71.1 - name: change-data-capture repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.49.7 + version: 1.51.2 - name: debezium-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts version: 1.25.1 - name: delegate-proxy repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.6.0 + version: 1.9.0 - name: gateway repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.60.3 + version: 1.62.5 - name: harness-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.128.4 + version: 1.131.5 - name: log-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.41.0 + version: 1.41.1 - name: next-gen-ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.116.11 + version: 1.119.5 - name: ng-auth-ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts version: 1.38.2 - name: ng-dashboard-aggregator repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.93.0 + version: 1.96.0 - name: ng-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.129.7 + version: 1.132.4 - name: pipeline-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.169.5 + version: 1.172.8 - name: platform-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.107.0 + version: 1.110.0 - name: policy-mgmt repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.36.5 + version: 1.38.1 - name: scm-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.44.0 + version: 1.45.1 - name: template-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.131.3 + version: 1.134.0 - name: ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.32.3 + version: 1.32.4 - name: ng-custom-dashboards repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.98.0 + version: 1.102.0 - name: looker repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.8.9 + version: 1.10.1 - name: service-discovery-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 0.54.0 + version: 0.56.0 - name: audit-event-streaming repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.0 + version: 1.77.0 - name: queue-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.8.1 -digest: sha256:bda84c4df4fbec331ba06d9fd4ef4dabef39603b03c6c8c436b0f1055765629a -generated: "2026-02-24T16:50:00.035866506Z" + version: 1.9.0 +digest: sha256:09b9220f5fd59251491b445f96684595a3de380f0eeca70e43a95ffbce8e3a40 +generated: "2026-03-13T05:48:32.297253897Z" diff --git a/src/modules/platform/Chart.yaml b/src/modules/platform/Chart.yaml index e501d8d6..605995ab 100644 --- a/src/modules/platform/Chart.yaml +++ b/src/modules/platform/Chart.yaml @@ -4,91 +4,91 @@ dependencies: - condition: global.ng.enabled name: access-control repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.195.0 + version: 1.208.0 - name: bootstrap repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.65.0 + version: 1.71.1 - condition: global.cdc.enabled name: change-data-capture repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.49.7 + version: 1.51.2 - condition: global.ssca.enabled name: debezium-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts version: 1.25.1 - name: delegate-proxy repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.6.0 + version: 1.9.0 - name: gateway repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.60.3 + version: 1.62.5 - name: harness-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.128.4 + version: 1.131.5 - condition: global.ng.enabled name: log-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.41.0 + version: 1.41.1 - condition: global.ng.enabled name: next-gen-ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.116.11 + version: 1.119.5 - name: ng-auth-ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts version: 1.38.2 - condition: global.ng.enabled name: ng-dashboard-aggregator repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.93.0 + version: 1.96.0 - condition: global.ng.enabled name: ng-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.129.7 + version: 1.132.4 - condition: global.ng.enabled name: pipeline-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.169.5 + version: 1.172.8 - condition: global.ng.enabled name: platform-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.107.0 + version: 1.110.0 - condition: global.ng.enabled name: policy-mgmt repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.36.5 + version: 1.38.1 - condition: global.ng.enabled name: scm-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.44.0 + version: 1.45.1 - condition: global.ng.enabled name: template-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.131.3 + version: 1.134.0 - condition: global.cg.enabled name: ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.32.3 + version: 1.32.4 - condition: global.ngcustomdashboard.enabled name: ng-custom-dashboards repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.98.0 + version: 1.102.0 - condition: global.ngcustomdashboard.enabled name: looker repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.8.9 + version: 1.10.1 - condition: global.servicediscoverymanager.enabled name: service-discovery-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 0.54.0 + version: 0.56.0 - condition: global.ng.enabled name: audit-event-streaming repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.74.0 + version: 1.77.0 - condition: global.ng.enabled name: queue-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.8.1 + version: 1.9.0 description: Helm chart for Harness Platform name: platform type: application -version: 0.38.0 +version: 0.38.1 diff --git a/src/modules/srm/Chart.lock b/src/modules/srm/Chart.lock index c36007e4..dda4b144 100644 --- a/src/modules/srm/Chart.lock +++ b/src/modules/srm/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: cv-nextgen repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.55.1 + version: 1.57.2 - name: le-nextgen repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.13.0 + version: 1.13.1 - name: srm-ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.16.0 -digest: sha256:6990265ae41b0f8257bc9d8241c6145186ab62e1edc882b7732aa2b0327c8fd0 -generated: "2026-02-24T16:49:55.02680814Z" + version: 1.16.2 +digest: sha256:dc7f37c4c83e63d81016bfb27ccd7f30fc25dc79dad6864b81cfb39c446d2a38 +generated: "2026-03-08T16:13:49.876362384Z" diff --git a/src/modules/srm/Chart.yaml b/src/modules/srm/Chart.yaml index 0c022485..ef3d18be 100644 --- a/src/modules/srm/Chart.yaml +++ b/src/modules/srm/Chart.yaml @@ -4,15 +4,15 @@ dependencies: - condition: global.ng.enabled name: cv-nextgen repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.55.1 + version: 1.57.2 - condition: global.ng.enabled name: le-nextgen repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.13.0 + version: 1.13.1 - condition: global.ng.enabled name: srm-ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.16.0 + version: 1.16.2 description: Helm chart for Harness SRM name: srm type: application diff --git a/src/modules/ssca/Chart.lock b/src/modules/ssca/Chart.lock index 2120b756..5dd3ae7f 100644 --- a/src/modules/ssca/Chart.lock +++ b/src/modules/ssca/Chart.lock @@ -1,15 +1,15 @@ dependencies: - name: ssca-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.50.14 + version: 1.52.13 - name: ssca-ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 0.38.5 + version: 0.39.2 - name: component-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.6.30 + version: 1.7.1 - name: component-analysis-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.1.7 -digest: sha256:539b1676734af2df7cccb09cfbcfe3c73d6f3179cc9164d94320ee9fe7b75c5c -generated: "2026-02-24T16:50:04.882434832Z" + version: 1.2.1 +digest: sha256:6228bc9aa1a9b2c89cf38f533ec2a8e3c64972ae6e51c558b2cf890e548e298f +generated: "2026-03-08T16:13:40.577174427Z" diff --git a/src/modules/ssca/Chart.yaml b/src/modules/ssca/Chart.yaml index 69f8db6c..4f47375c 100644 --- a/src/modules/ssca/Chart.yaml +++ b/src/modules/ssca/Chart.yaml @@ -4,19 +4,19 @@ dependencies: - condition: global.ssca.enabled name: ssca-manager repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.50.14 + version: 1.52.13 - condition: global.ssca.enabled name: ssca-ui repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 0.38.5 + version: 0.39.2 - condition: global.ssca.enabled name: component-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.6.30 + version: 1.7.1 - condition: global.ssca.enabled name: component-analysis-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.1.7 + version: 1.2.1 description: A Helm chart for Harness SSCA name: ssca type: application diff --git a/src/modules/sto/Chart.lock b/src/modules/sto/Chart.lock index e2ec8d2a..2480c0a3 100644 --- a/src/modules/sto/Chart.lock +++ b/src/modules/sto/Chart.lock @@ -1,9 +1,9 @@ dependencies: - name: sto-core repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.180.4 + version: 1.182.0 - name: ticket-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.4.5 -digest: sha256:9256a0593266f9b19521cf1bd679a65a4923f579139fdd2794aab104eaf0856d -generated: "2026-02-24T16:50:05.899504627Z" + version: 1.8.0 +digest: sha256:facfae14bad79fe341556bd7730b62fbd34f1e5deeed08b15eb3d1585a69d76a +generated: "2026-03-08T16:13:50.537437524Z" diff --git a/src/modules/sto/Chart.yaml b/src/modules/sto/Chart.yaml index 36f80573..6c8b792a 100644 --- a/src/modules/sto/Chart.yaml +++ b/src/modules/sto/Chart.yaml @@ -3,10 +3,10 @@ appVersion: 0.0.79001 dependencies: - name: sto-core repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.180.4 + version: 1.182.0 - name: ticket-service repository: oci://us-west1-docker.pkg.dev/gcr-prod/harness-helm-artifacts - version: 1.4.5 + version: 1.8.0 description: Helm chart for Harness STO name: sto type: application diff --git a/src/smp-tools.py b/src/smp-tools.py new file mode 100644 index 00000000..59aec6d8 --- /dev/null +++ b/src/smp-tools.py @@ -0,0 +1,837 @@ +#!/usr/bin/env python3 +""" +SMP tools: unified CLI for bundle image generation, Helm flag extraction, and manifest validation. + +Subcommands: + bundle-images Resolve bundle-manifest.yaml to produce images.txt and images_internal.txt + auto-enable-flags Scan a Helm YAML file and emit --set =true flags for enabled/create keys + validate-bundle Validate bundle-manifest.yaml against images_raw.txt, images.txt, etc. + +Usage: + python3 smp-tools.py bundle-images --manifest bundle-manifest.yaml --raw-images images_raw.txt --output-dir . + python3 smp-tools.py bundle-images --manifest bundle-manifest.yaml --internal-only --images-txt images.txt --output-dir . + python3 smp-tools.py auto-enable-flags values.yaml + python3 smp-tools.py auto-enable-flags --mode true generate-image.yaml + python3 smp-tools.py validate-bundle --manifest bundle-manifest.yaml +""" + +import argparse +import logging +import math +import os +import re +import sys + +try: + import yaml +except ImportError: + print("[ERROR] PyYAML is required. Install with: pip install pyyaml", file=sys.stderr) + sys.exit(1) + +logging.basicConfig(format='[%(levelname)-5s] %(message)s', level=logging.INFO) +log = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# auto-enable-flags +# --------------------------------------------------------------------------- + +AUTO_ENABLE_BLOCKLIST = { + "global.lwd.enabled", + "global.lwd.autocud.enabled", +} + +FLAG_KEYS = {"enabled", "create"} + + +def collect_flags(data, target_value, path=""): + """Walk the YAML tree and yield --set flags for matching keys.""" + if not isinstance(data, dict): + return + for key, value in data.items(): + dotted = f"{path}.{key}" if path else key + if key in FLAG_KEYS and value is target_value: + if "." not in dotted or dotted in AUTO_ENABLE_BLOCKLIST: + continue + yield f"--set {dotted}=true" + elif isinstance(value, dict): + yield from collect_flags(value, target_value, dotted) + + +def cmd_auto_enable_flags(args): + """Emit --set flags from a Helm values YAML file.""" + try: + with open(args.values_yaml) as f: + data = yaml.safe_load(f) + except FileNotFoundError: + print(f"[ERROR] File not found: {args.values_yaml}", file=sys.stderr) + sys.exit(1) + + target_value = args.mode == "true" + if data: + for flag in collect_flags(data, target_value): + print(flag) + + +# --------------------------------------------------------------------------- +# bundle-images +# --------------------------------------------------------------------------- + +def load_manifest(path): + with open(path) as f: + return yaml.safe_load(f) + + +def load_raw_images(path): + with open(path) as f: + return [line.strip() for line in f if line.strip() and not line.startswith('#')] + + +def find_matching_images(short_name, raw_images): + """Find all images in raw_images matching a short name like 'ci-manager-signed'.""" + pattern = f"/{short_name}:" + return [img for img in raw_images if pattern in img] + + +def get_image_name(entry): + """Return the short name from a plain string or dict image entry.""" + return entry['name'] if isinstance(entry, dict) else entry + + +def get_image_variants(entry, section_variants, global_variants): + """ + Resolve variant suffixes using cascading: inline entry > section > global. + + Image entries may be: + - plain string: inherit from section or global + - dict with 'variants': use those directly + - dict with 'extra_variants': append to inherited list + - dict with 'variants_only: true': same as 'variants' but the base image is + suppressed from output (only the variant images are emitted). The base tag + is still resolved from images_raw.txt so that the versioned variant tags can + be constructed correctly. + """ + if isinstance(entry, dict): + if 'variants' in entry: + return entry['variants'] + if 'extra_variants' in entry: + parent = section_variants.get('variants', []) if section_variants else global_variants.get('variants', []) + return list(set(parent + entry['extra_variants'])) + + if section_variants: + return section_variants.get('variants', []) + + return global_variants.get('variants', []) + + +def process_section(name, config, raw_images, global_variants): + """ + Process a single module section uniformly. + Returns (customer_lines, internal_lines, resolved_images, errors). + + For single bundle-type sections each variant (base tag + suffix) gets its own + @image= group marker so the pipeline treats them as independent bundles. + + Excluded images (module-level exclude list) appear in customer_lines (images.txt) + but are omitted from internal_lines so they are never bundled. + """ + bundle_type = config.get('bundle_type', 'combined') + bucket_path = config.get('bucket_path', name) + parent = config.get('parent') + requires = config.get('requires', []) + section_variants = config.get('variants') + images_list = config.get('images', []) + exclude_list = set(config.get('exclude', [])) + + customer_lines = [] + internal_lines = [] + resolved_images = [] + errors = [] + + if parent: + internal_header = f"# @module={name} @type={bundle_type} @path={bucket_path} @parent={parent}" + else: + requires_str = ','.join(requires) if requires else '' + internal_header = f"# @module={name} @type={bundle_type} @path={bucket_path} @requires={requires_str}" + + internal_lines.append(internal_header) + + for short_name in exclude_list: + for match in find_matching_images(short_name, raw_images): + customer_lines.append(match) + resolved_images.append(match) + + for image_entry in images_list: + short_name = get_image_name(image_entry) + matches = find_matching_images(short_name, raw_images) + + if not matches: + errors.append(f"Image '{short_name}' in section '{name}' not found in images_raw.txt") + continue + + variants = get_image_variants(image_entry, section_variants, global_variants) + # When variants_only=True the base image is used only as a tag anchor; it is + # never written to customer or internal output. Requires 'variants' to be set. + variants_only = isinstance(image_entry, dict) and image_entry.get('variants_only', False) + + # For single bundles, use only the first match as base; variants are constructed from it. + # Each @image= gets a unique name (short_name for base, short_name+variant for variants). + match_iter = [matches[0]] if bundle_type == 'single' and matches else matches + + for match in match_iter: + if not variants_only: + customer_lines.append(match) + resolved_images.append(match) + + if bundle_type == 'single': + if not variants_only: + # Base image gets its own group + internal_lines.append(f"# @image={short_name}") + internal_lines.append(match) + + # Each variant is a separate group so they bundle independently. + # Normalize variant for bundle name: .minimal -> -minimal, -fips -> -fips + for variant in variants: + variant_image = f"{match}{variant}" + suffix = variant.lstrip('.-').replace('.', '-') + internal_lines.append(f"# @image={short_name}-{suffix}") + internal_lines.append(variant_image) + customer_lines.append(variant_image) + resolved_images.append(variant_image) + else: + if not variants_only: + internal_lines.append(match) + for variant in variants: + variant_image = f"{match}{variant}" + customer_lines.append(variant_image) + resolved_images.append(variant_image) + internal_lines.append(variant_image) + + return customer_lines, internal_lines, resolved_images, errors + + +PIPELINE_BATCH_SIZE = 12 + + +def count_pulls(internal_lines): + """Count the number of pull operations (non-comment, non-empty lines) in a section.""" + return sum(1 for l in internal_lines if l.strip() and not l.startswith('#')) + + +def split_single_section_into_batches(name, internal_lines, batch_size): + """ + Split a single-type section's internal lines into batches of batch_size image groups. + Returns list of (batch_name, batch_lines) tuples. + """ + header = internal_lines[0] + groups = [] + current_group = [] + + for line in internal_lines[1:]: + if line.startswith('# @image='): + if current_group: + groups.append(current_group) + current_group = [line] + elif line.strip(): + current_group.append(line) + + if current_group: + groups.append(current_group) + + if len(groups) <= batch_size: + return [(name, internal_lines)] + + num_batches = math.ceil(len(groups) / batch_size) + batches = [] + for b in range(num_batches): + batch_groups = groups[b * batch_size:(b + 1) * batch_size] + batch_header = header.replace(f"@module={name}", f"@module={name}@{b + 1}") + batch_lines = [batch_header] + for g in batch_groups: + batch_lines.extend(g) + batches.append((f"{name}@{b + 1}", batch_lines)) + return batches + + +def sort_and_batch_internal_sections(sections_data): + """ + Takes list of (name, config, internal_lines) and returns + sorted, batched internal text (heaviest pull count first). + """ + entries = [] + for name, config, internal_lines in sections_data: + bundle_type = config.get('bundle_type', 'combined') + if bundle_type == 'single': + batches = split_single_section_into_batches(name, internal_lines, PIPELINE_BATCH_SIZE) + else: + batches = [(name, internal_lines)] + for batch_name, batch_lines in batches: + pulls = count_pulls(batch_lines) + entries.append((batch_name, batch_lines, pulls)) + + entries.sort(key=lambda x: -x[2]) + + all_lines = [] + for batch_name, batch_lines, pulls in entries: + all_lines.extend(batch_lines) + all_lines.append('') + + return '\n'.join(all_lines).rstrip() + '\n' + + +def bundle_generate(manifest, raw_images): + """ + Main generation logic. Returns (customer_text, internal_text, all_resolved, all_excluded, all_errors). + """ + global_cfg = manifest.get('global', {}) + global_variants_list = global_cfg.get('variants', []) + global_variants = {'variants': global_variants_list} + modules = manifest.get('modules', {}) + + all_customer_lines = [] + all_resolved = [] + all_excluded = [] + all_errors = [] + sections_data = [] + + total = len(modules) + + for idx, (name, config) in enumerate(modules.items(), 1): + parent = config.get('parent') + log.info(f"[{idx}/{total}] Processing: {name}") + + customer, internal, resolved, errors = process_section( + name, config, raw_images, global_variants + ) + + header_prefix = "###" if parent else "##" + header = f"{header_prefix} {config.get('description', name)}" + all_customer_lines.append(header) + all_customer_lines.extend(customer) + all_customer_lines.append('') + + sections_data.append((name, config, internal)) + all_resolved.extend(resolved) + all_errors.extend(errors) + + customer_text = '\n'.join(all_customer_lines).rstrip() + '\n' + internal_text = sort_and_batch_internal_sections(sections_data) + + return customer_text, internal_text, all_resolved, all_excluded, all_errors + + +def bundle_generate_internal_only(manifest, images_txt_path): + """ + Regenerate images_internal.txt from committed images.txt + manifest. + Parses section headers in images.txt to map images back to modules. + Output is sorted by pull count (heaviest first) with single-type batching. + """ + modules = manifest.get('modules', {}) + + with open(images_txt_path) as f: + images_txt_content = f.read() + + sections = [] + current_desc = None + current_images = [] + + for line in images_txt_content.splitlines(): + if line.startswith('### ') or line.startswith('## '): + if current_desc is not None: + sections.append((current_desc, current_images)) + current_desc = line.lstrip('#').strip() + current_images = [] + elif line.strip(): + current_images.append(line.strip()) + + if current_desc is not None: + sections.append((current_desc, current_images)) + + desc_to_module = {} + excluded_short_names = set() + for mod_name, mod_config in modules.items(): + desc_to_module[mod_config.get('description', mod_name)] = (mod_name, mod_config) + excluded_short_names.update(mod_config.get('exclude', [])) + + def is_image_excluded(img_ref): + for short in excluded_short_names: + if f"/{short}:" in img_ref: + return True + return False + + sections_data = [] + for desc, imgs in sections: + if desc not in desc_to_module: + continue + + mod_name, mod_config = desc_to_module[desc] + bundle_type = mod_config.get('bundle_type', 'combined') + bucket_path = mod_config.get('bucket_path', mod_name) + parent = mod_config.get('parent') + non_excluded_imgs = [img for img in imgs if not is_image_excluded(img)] + + if parent: + header = f"# @module={mod_name} @type={bundle_type} @path={bucket_path} @parent={parent}" + else: + requires = ','.join(mod_config.get('requires', [])) + header = f"# @module={mod_name} @type={bundle_type} @path={bucket_path} @requires={requires}" + + internal_lines = [header] + if bundle_type == 'single': + # Build short_name -> variants from manifest (sort by length desc to match longest first) + variants_by_short = {} + for entry in mod_config.get('images', []): + short = entry['name'] if isinstance(entry, dict) else entry + variants = entry.get('variants', []) if isinstance(entry, dict) else [] + variants_by_short[short] = sorted(variants, key=len, reverse=True) + + for img in non_excluded_imgs: + tag_part = img.rsplit(':', 1)[0] if ':' in img else img + base_name = tag_part.rsplit('/', 1)[-1] if '/' in tag_part else tag_part + tag = img.rsplit(':', 1)[1] if ':' in img else 'latest' + variants = variants_by_short.get(base_name, []) + # Match tag to variant: base uses base_name, variants use base_name-variant_suffix + bundle_name = base_name + for v in variants: + if tag.endswith(v): + # Normalize .minimal -> -minimal, -fips -> -fips + suffix = v.lstrip('.-').replace('.', '-') + bundle_name = f"{base_name}-{suffix}" + break + internal_lines.append(f"# @image={bundle_name}") + internal_lines.append(img) + else: + internal_lines.extend(non_excluded_imgs) + + sections_data.append((mod_name, mod_config, internal_lines)) + + return sort_and_batch_internal_sections(sections_data) + + +def cmd_bundle_images(args): + """Resolve bundle manifest against raw images.""" + manifest = load_manifest(args.manifest) + + if args.internal_only: + images_txt = args.images_txt + if not images_txt: + if args.output_dir: + images_txt = os.path.join(args.output_dir, 'images.txt') + else: + log.error("--images-txt or --output-dir required for --internal-only mode") + sys.exit(1) + + log.info(f"Regenerating images_internal.txt from {images_txt}") + internal_text = bundle_generate_internal_only(manifest, images_txt) + + output_dir = args.output_dir or os.path.dirname(images_txt) + internal_path = os.path.join(output_dir, 'images_internal.txt') + with open(internal_path, 'w') as f: + f.write(internal_text) + log.info(f"Written {internal_path}") + return + + if not args.raw_images: + log.error("--raw-images is required when not using --internal-only") + sys.exit(1) + + if not args.output_dir: + log.error("--output-dir is required when not using --internal-only") + sys.exit(1) + + raw_images = load_raw_images(args.raw_images) + log.info(f"Loaded {len(raw_images)} base images from {args.raw_images}") + + customer_text, internal_text, resolved, excluded, errors = bundle_generate(manifest, raw_images) + + if errors: + log.warning(f"Encountered {len(errors)} resolution errors:") + for err in errors: + log.warning(f" {err}") + + images_path = os.path.join(args.output_dir, 'images.txt') + internal_path = os.path.join(args.output_dir, 'images_internal.txt') + + with open(images_path, 'w') as f: + f.write(customer_text) + + with open(internal_path, 'w') as f: + f.write(internal_text) + + unique_base = set() + unique_all = set() + for img in resolved: + unique_all.add(img) + base = img.rsplit(':', 1)[0] if ':' in img else img + unique_base.add(base) + + log.info(f"Written {images_path} ({customer_text.count(chr(10))} lines)") + log.info(f"Written {internal_path}") + log.info(f"Resolved {len(unique_all)} total image references ({len(unique_base)} unique base images)") + + if excluded: + log.info(f"Excluded from bundle ({len(excluded)} images, still in images.txt):") + for img in excluded: + log.info(f" {img}") + + unmatched = set(raw_images) - set(r for r in resolved if r in raw_images) + if unmatched: + log.warning(f"{len(unmatched)} images in images_raw.txt not mapped to any module:") + for img in sorted(unmatched): + log.warning(f" {img}") + + +# --------------------------------------------------------------------------- +# validate-bundle +# --------------------------------------------------------------------------- + +def _load_validate_lines(path, skip_comments=True, skip_headers=True): + if not path or not os.path.exists(path): + return [] + with open(path) as f: + return [ + line.strip() for line in f + if line.strip() + and (not skip_comments or not line.strip().startswith('#')) + and (not skip_headers or not line.strip().startswith('##')) + ] + + +def _load_lines_with_headers(path): + if not path or not os.path.exists(path): + return [], [] + lines = [] + headers = [] + with open(path) as f: + for line in f: + stripped = line.strip() + if stripped.startswith('##'): + headers.append(stripped) + elif stripped: + lines.append(stripped) + return lines, headers + + +def _get_all_short_names(manifest): + names = set() + for mod_config in manifest.get('modules', {}).values(): + for entry in mod_config.get('images', []): + names.add(get_image_name(entry)) + names.update(mod_config.get('exclude', [])) + return names + + +def _get_excluded_short_names(manifest): + excluded = set() + for mod_config in manifest.get('modules', {}).values(): + excluded.update(mod_config.get('exclude', [])) + return excluded + + +def _get_root_modules(modules): + return {k for k, v in modules.items() if not v.get('parent')} + + +def _get_child_modules(modules): + return {k for k, v in modules.items() if v.get('parent')} + + +def _get_all_bucket_paths(modules): + return [v.get('bucket_path', k) for k, v in modules.items()] + + +def _parse_chart_yaml_modules(chart_yaml_path): + chart = load_manifest(chart_yaml_path) + result = set() + name_map = {'chaos': 'ce', 'db-devops': 'dbdevops', 'cd': 'cdng', 'srm': 'platform'} + for dep in chart.get('dependencies', []): + name = dep.get('name', '') + if name in ('harness', 'harness-common'): + continue + result.add(name_map.get(name, name)) + return result + + +def _scan_chart_templates(harness_dir): + found_images = set() + image_pattern = re.compile(r'image:\s*["\']?([a-zA-Z0-9_./-]+:[a-zA-Z0-9_.-]+)') + for subdir in ('charts', 'templates'): + path = os.path.join(harness_dir, subdir) + if not os.path.isdir(path): + continue + for root, _, files in os.walk(path): + for fname in files: + if not fname.endswith(('.yaml', '.yml', '.tpl')): + continue + try: + with open(os.path.join(root, fname)) as f: + for line in f: + m = image_pattern.search(line) + if m and '{{' not in m.group(1): + found_images.add(m.group(1)) + except (IOError, UnicodeDecodeError): + pass + return found_images + + +def cmd_validate_bundle(args): + """Validate bundle-manifest.yaml against images_raw.txt, images.txt, images_internal.txt, and chart templates.""" + script_dir = os.path.dirname(os.path.abspath(__file__)) + harness_dir = os.path.join(script_dir, 'harness') + + if not args.raw_images and os.path.exists(os.path.join(harness_dir, 'images_raw.txt')): + args.raw_images = os.path.join(harness_dir, 'images_raw.txt') + if not args.images_txt and os.path.exists(os.path.join(harness_dir, 'images.txt')): + args.images_txt = os.path.join(harness_dir, 'images.txt') + if not args.internal_txt and os.path.exists(os.path.join(harness_dir, 'images_internal.txt')): + args.internal_txt = os.path.join(harness_dir, 'images_internal.txt') + if not args.chart_yaml and os.path.exists(os.path.join(harness_dir, 'Chart.yaml')): + args.chart_yaml = os.path.join(harness_dir, 'Chart.yaml') + if not args.harness_dir and os.path.isdir(harness_dir): + args.harness_dir = harness_dir + + errors = [] + warnings = [] + + manifest = load_manifest(args.manifest) + modules = manifest.get('modules', {}) + total_checks = 11 + check_num = 0 + + raw_images = _load_validate_lines(args.raw_images) if args.raw_images else [] + images_txt_lines, images_txt_headers = _load_lines_with_headers(args.images_txt) if args.images_txt else ([], []) + internal_lines = _load_validate_lines(args.internal_txt) if args.internal_txt else [] + + all_short_names = _get_all_short_names(manifest) + excluded_names = _get_excluded_short_names(manifest) + root_modules = _get_root_modules(modules) + + # Check 1: Every image in images_raw.txt maps to a manifest entry + check_num += 1 + log.info(f"[{check_num}/{total_checks}] Checking image coverage in manifest...") + if raw_images: + for img in raw_images: + matched = any(f"/{sn}:" in img for sn in all_short_names) + if not matched: + errors.append(f"Image '{img}' from images_raw.txt not mapped to any module in manifest") + elif any(f"/{ex}:" in img for ex in excluded_names): + log.info(f" Image '{img}' is in manifest but excluded from bundle") + + # Check 2: Every manifest short name resolves to at least one raw image + check_num += 1 + log.info(f"[{check_num}/{total_checks}] Checking manifest references resolve...") + if raw_images: + for short_name in all_short_names: + if short_name in excluded_names: + continue + if not any(f"/{short_name}:" in img for img in raw_images): + errors.append(f"Manifest image '{short_name}' not found in images_raw.txt") + + # Check 3: Chart.yaml modules have manifest entries + check_num += 1 + log.info(f"[{check_num}/{total_checks}] Checking Chart.yaml modules have manifest entries...") + if args.chart_yaml and os.path.exists(args.chart_yaml): + chart_modules = _parse_chart_yaml_modules(args.chart_yaml) + for cm in chart_modules: + if cm not in root_modules: + errors.append(f"Chart.yaml module '{cm}' has no root entry in bundle-manifest.yaml") + + # Check 4: Orphaned manifest modules + check_num += 1 + log.info(f"[{check_num}/{total_checks}] Checking for orphaned manifest modules...") + if args.chart_yaml and os.path.exists(args.chart_yaml): + chart_modules = _parse_chart_yaml_modules(args.chart_yaml) + for mod_name in root_modules: + if mod_name not in chart_modules: + warnings.append(f"Manifest root module '{mod_name}' not found in Chart.yaml dependencies") + + # Check 5: Dependency references are valid + check_num += 1 + log.info(f"[{check_num}/{total_checks}] Checking dependency references...") + for mod_name, mod_config in modules.items(): + for req in mod_config.get('requires', []): + if req not in modules: + errors.append(f"Module '{mod_name}' requires '{req}' which doesn't exist in manifest") + parent = mod_config.get('parent') + if parent and parent not in modules: + errors.append(f"Module '{mod_name}' has parent '{parent}' which doesn't exist in manifest") + + # Check 6: No circular dependencies + check_num += 1 + log.info(f"[{check_num}/{total_checks}] Checking for circular dependencies...") + + def detect_cycle(mod, visited, stack): + visited.add(mod) + stack.add(mod) + for req in modules.get(mod, {}).get('requires', []): + if req in stack: + return [mod, req] + if req not in visited: + cycle = detect_cycle(req, visited, stack) + if cycle: + return cycle + stack.discard(mod) + return None + + visited = set() + for mod_name in modules: + if mod_name not in visited: + cycle = detect_cycle(mod_name, visited, set()) + if cycle: + errors.append(f"Circular dependency detected: {' -> '.join(cycle)}") + + # Check 7: Unique bucket paths + check_num += 1 + log.info(f"[{check_num}/{total_checks}] Checking bucket path uniqueness...") + paths = _get_all_bucket_paths(modules) + seen = set() + for p in paths: + if p in seen: + errors.append(f"Duplicate bucket_path '{p}'") + seen.add(p) + + # Check 8: Section header count matches + check_num += 1 + log.info(f"[{check_num}/{total_checks}] Checking section counts...") + if images_txt_headers: + actual_sections = len(images_txt_headers) + expected_sections = len(modules) + if actual_sections != expected_sections: + errors.append(f"images.txt has {actual_sections} sections but manifest defines {expected_sections}") + + # Check 9: Image count consistency (images.txt vs images_internal.txt) + check_num += 1 + log.info(f"[{check_num}/{total_checks}] Checking image count consistency...") + if images_txt_lines and internal_lines: + excluded_count = sum( + 1 for img in images_txt_lines + if any(f"/{ex}:" in img for ex in excluded_names) + ) + bundled_in_txt = len(images_txt_lines) - excluded_count + diff = bundled_in_txt - len(internal_lines) + if diff < 0: + errors.append(f"images_internal.txt has more images ({len(internal_lines)}) than bundled images in images.txt ({bundled_in_txt})") + elif diff > 0 and excluded_names: + log.info(f" images.txt has {diff} more images than images_internal.txt (expected: excluded images)") + elif diff > 0: + warnings.append(f"images.txt has {len(images_txt_lines)} images but images_internal.txt has {len(internal_lines)}") + + # Check 10: Chart template scan for unlisted images + check_num += 1 + log.info(f"[{check_num}/{total_checks}] Scanning chart templates for unlisted images...") + if args.harness_dir and os.path.isdir(args.harness_dir): + chart_images = _scan_chart_templates(args.harness_dir) + if chart_images and raw_images: + raw_set = set(raw_images) + for ci in chart_images: + if ci not in raw_set: + warnings.append(f"Chart template image '{ci}' not in images_raw.txt (may be template-conditional)") + + # Check 11: Variant definitions reference existing images + check_num += 1 + log.info(f"[{check_num}/{total_checks}] Checking variant definitions...") + for mod_name, mod_config in modules.items(): + for entry in mod_config.get('images', []): + if isinstance(entry, dict) and ('variants' in entry or 'extra_variants' in entry): + if entry['name'] not in all_short_names: + errors.append(f"Variant defined for '{entry['name']}' in '{mod_name}' but image not in manifest") + + # Duplication info + image_module_map = {} + for mod_name, mod_config in modules.items(): + if mod_config.get('parent'): + continue + for entry in mod_config.get('images', []): + short = get_image_name(entry) + image_module_map.setdefault(short, []).append(mod_name) + for img, mods in image_module_map.items(): + if len(mods) > 1: + warnings.append(f"Image '{img}' appears in {len(mods)} modules ({', '.join(mods)}) - duplicated in bundles") + + # Summary + root_count = len(root_modules) + child_count = len(_get_child_modules(modules)) + combined_count = sum(1 for m in modules.values() if m.get('bundle_type') == 'combined') + + print() + print("=== SUMMARY ===") + print(f"Modules: {root_count} | Sub-bundles: {child_count} | Total sections: {len(modules)}") + if raw_images: + print(f"Base images (raw): {len(raw_images)}") + if images_txt_lines: + print(f"Images in images.txt: {len(images_txt_lines)}") + if excluded_names: + print(f"Excluded from bundle: {len(excluded_names)} ({', '.join(sorted(excluded_names))})") + print(f"Combined bundles: {combined_count}") + print() + + if warnings: + print(f"=== WARNINGS ({len(warnings)}) ===") + for i, w in enumerate(warnings, 1): + print(f" [W{i}] {w}") + print() + + if errors: + print(f"=== ERRORS ({len(errors)}) ===") + for i, e in enumerate(errors, 1): + print(f" [E{i}] {e}") + print() + print("Validation FAILED") + sys.exit(1) + + print("=== ERRORS (0) ===") + print("Validation PASSED") + + +# --------------------------------------------------------------------------- +# main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser( + description="SMP tools: bundle image generation and Helm flag extraction", + epilog="Use 'smp-tools.py --help' for subcommand-specific help.", + ) + subparsers = parser.add_subparsers(dest="command", required=True, help="Subcommand to run") + + # bundle-images + bp = subparsers.add_parser("bundle-images", help="Resolve bundle manifest to produce images.txt and images_internal.txt") + bp.add_argument("--manifest", required=True, help="Path to bundle-manifest.yaml") + bp.add_argument("--raw-images", help="Path to images_raw.txt") + bp.add_argument("--output-dir", help="Output directory for images.txt and images_internal.txt") + bp.add_argument("--internal-only", action="store_true", + help="Regenerate only images_internal.txt from existing images.txt") + bp.add_argument("--images-txt", help="Path to existing images.txt (for --internal-only mode)") + bp.set_defaults(func=cmd_bundle_images) + + # auto-enable-flags + ap = subparsers.add_parser("auto-enable-flags", help="Emit --set flags from a Helm values YAML file") + ap.add_argument("values_yaml", help="Path to the YAML file to scan") + ap.add_argument( + "--mode", + choices=["false", "true"], + default="false", + help=( + "'false' (default): emit flags for every enabled/create that is False " + "(use with values.yaml to flip disabled features on). " + "'true': emit flags for every enabled/create that is True " + "(use with generate-image.yaml to forward explicit overrides as --set flags)." + ), + ) + ap.set_defaults(func=cmd_auto_enable_flags) + + # validate-bundle + vp = subparsers.add_parser("validate-bundle", help="Validate bundle-manifest.yaml against images_raw.txt, images.txt, images_internal.txt, and chart templates") + vp.add_argument("--manifest", required=True, help="Path to bundle-manifest.yaml") + vp.add_argument("--raw-images", help="Path to images_raw.txt") + vp.add_argument("--images-txt", help="Path to images.txt") + vp.add_argument("--internal-txt", help="Path to images_internal.txt") + vp.add_argument("--chart-yaml", help="Path to Chart.yaml") + vp.add_argument("--harness-dir", help="Path to harness chart directory (for template scanning)") + vp.set_defaults(func=cmd_validate_bundle) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main()