docker-compose.yml

services:
  postgres:
    image: postgres:16
    container_name: polaris-postgres
    environment:
      POSTGRES_DB: polaris_db
      POSTGRES_USER: polaris_user
      POSTGRES_PASSWORD: polaris_password
    volumes:
      - postgres_data:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U polaris_user -d polaris_db"]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - opendata_network
    restart: unless-stopped

  minio:
    image: minio/minio
    ports:
      - "${MINIO_API_PORT:-9000}:9000"
      - "${MINIO_CONSOLE_PORT:-9001}:9001"
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
      MINIO_DOMAIN: minio
    volumes:
      - minio_data:/data
    command: server /data --console-address ":9001"
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:9000/minio/health/live"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s
    networks:
      opendata_network:
        aliases:
          - warehouse.minio
          - minio

  mc:
    depends_on:
      minio:
        condition: service_healthy
    image: minio/mc
    volumes:
      - ./minio-config:/config
    networks:
      - opendata_network
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-miniouser}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-miniouser}
    entrypoint: /bin/sh
    command: /config/setup-minio.sh

  polaris:
    image: apache/polaris:latest
    platform: linux/amd64
    ports:
      - "8181:8181"
      - "8182:8182"
    depends_on:
      minio:
        condition: service_healthy
      postgres:
        condition: service_healthy
    networks:
      - opendata_network
    environment:
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      AWS_REGION: ${AWS_REGION:-us-east-1}
      AWS_ENDPOINT_URL_S3: http://minio:9000
      AWS_ENDPOINT_URL_STS: http://minio:9000
      POLARIS_BOOTSTRAP_CREDENTIALS: default,admin,password
      polaris.realm-context.realms: default
      polaris.features.DROP_WITH_PURGE_ENABLED: "true"
      POLARIS_PERSISTENCE_TYPE: relational-jdbc
      QUARKUS_DATASOURCE_DB_KIND: postgresql
      QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/polaris_db
      QUARKUS_DATASOURCE_USERNAME: polaris_user
      QUARKUS_DATASOURCE_PASSWORD: polaris_password
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8182/q/health"]
      interval: 10s
      timeout: 5s
      retries: 20
      start_period: 60s

  polaris-bootstrap:
    image: apache/polaris-admin-tool:latest
    platform: linux/amd64
    depends_on:
      postgres:
        condition: service_healthy
    networks:
      - opendata_network
    environment:
      POLARIS_PERSISTENCE_TYPE: relational-jdbc
      QUARKUS_DATASOURCE_DB_KIND: postgresql
      QUARKUS_DATASOURCE_JDBC_URL: jdbc:postgresql://postgres:5432/polaris_db
      QUARKUS_DATASOURCE_USERNAME: polaris_user
      QUARKUS_DATASOURCE_PASSWORD: polaris_password
      POLARIS_BOOTSTRAP_CREDENTIALS: default,admin,password
    command:
      [
        "bootstrap",
        "--realm",
        "default",
        "--credential",
        "default,admin,password",
        "--print-credentials",
      ]

  polaris-init:
    image: python:3.11-slim
    depends_on:
      polaris:
        condition: service_healthy
      mc:
        condition: service_completed_successfully
    volumes:
      - ./polaris-config:/config
      - ./dagster-workspace/projects/streamify:/opt/streamify
    networks:
      - opendata_network
    environment:
      POLARIS_HOST: polaris
      MINIO_HOST: minio
    entrypoint: /bin/bash
    command:
      - -c
      - |
        pip install requests > /dev/null 2>&1
        python /config/setup_polaris.py

  zookeeper:
    image: confluentinc/cp-zookeeper:7.5.0
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    networks:
      - opendata_network
    volumes:
      - zookeeper_data:/var/lib/zookeeper/data
      - zookeeper_logs:/var/lib/zookeeper/log

  kafka:
    image: confluentinc/cp-kafka:8.3.0
    container_name: kafka
    ports:
      - "9092:9092"
      - "9093:9093"
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:9093'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093'
      KAFKA_LISTENERS: 'PLAINTEXT://kafka:9092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9093'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
      CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk'
    networks:
      - opendata_network
    volumes:
      - kafka_data:/var/lib/kafka/data
    healthcheck:
      test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server kafka:9092"]
      interval: 10s
      timeout: 10s
      retries: 5
      start_period: 30s

  schema-registry:
    image: confluentinc/cp-schema-registry:8.3.0
    hostname: schema-registry
    container_name: schema-registry
    depends_on:
      kafka:
        condition: service_healthy
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka:9092'
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
    networks:
      - opendata_network

  ksqldb-server:
    image: confluentinc/cp-ksqldb-server:8.3.0
    hostname: ksqldb-server
    container_name: ksqldb-server
    depends_on:
      kafka:
        condition: service_healthy
      schema-registry:
        condition: service_started
    ports:
      - "8088:8088"
    volumes:
      - ./scripts/ksql-queries.sql:/etc/ksql/queries.sql
    environment:
      KSQL_CONFIG_DIR: "/etc/ksql"
      KSQL_BOOTSTRAP_SERVERS: kafka:9092
      KSQL_HOST_NAME: ksqldb-server
      KSQL_LISTENERS: http://0.0.0.0:8088
      KSQL_CACHE_MAX_BYTES_BUFFERING: 0
      KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: 1
      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
      KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
      # KSQL_KSQL_QUERIES_FILE: "/etc/ksql/queries.sql"  # only for backfill
    networks:
      - opendata_network
  ksqldb-cli:
    image: confluentinc/cp-ksqldb-server:8.3.0
    container_name: ksqldb-cli
    depends_on:
      - ksqldb-server
    entrypoint: /bin/sh
    tty: true
    networks:
      - opendata_network

  # ---------------------------------------------------------------------------
  # Spark 4.0.2 — Scala 2.13, Java 21
  # Iceberg 1.10.1 (iceberg-spark-runtime-4.0_2.13) — latest stable for Spark 4
  # Hadoop AWS 3.4.1 — matches Spark 4.x bundled Hadoop
  # ---------------------------------------------------------------------------

  spark-master:
    image: apache/spark:4.0.2
    command: /opt/spark/sbin/start-master.sh -h spark-master
    environment:
      SPARK_NO_DAEMONIZE: "true"
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-minioadmin}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-minioadmin}
      AWS_REGION: ${AWS_REGION:-us-east-1}
      AWS_ENDPOINT_URL: http://minio:9000
    ports:
      - "8080:8080"
      - "7077:7077"
    networks:
      - opendata_network
    volumes:
      - ./data:/data
      - ./dagster-workspace/projects/streamify:/opt/streamify

  spark-worker:
    image: apache/spark:4.0.2
    user: root
    command: /opt/spark/sbin/start-worker.sh spark://spark-master:7077
    depends_on:
      - spark-master
    environment:
      SPARK_NO_DAEMONIZE: "true"
      SPARK_WORKER_MEMORY: 12G
      SPARK_WORKER_CORES: 8
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      AWS_REGION: ${AWS_REGION:-us-east-1}
      AWS_ENDPOINT_URL: http://minio:9000
    networks:
      - opendata_network
    volumes:
      - ./data:/data
      - ./dagster-workspace/projects/streamify:/opt/streamify

  spark-connect:
    image: apache/spark:4.0.2
    user: root
    entrypoint: /bin/bash
    command:
      - -c
      - |
        mkdir -p /root/.ivy2
        chmod -R 777 /root/.ivy2

        # Spark 4.x / Scala 2.13 package coordinates.
        # spark-connect is built into Spark 4 — no separate artifact needed.
        PACKAGES="\
        org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.1,\
        org.apache.iceberg:iceberg-aws-bundle:1.10.1,\
        org.apache.hadoop:hadoop-aws:3.4.1,\
        org.apache.spark:spark-sql-kafka-0-10_2.13:4.0.2,\
        io.dataflint:dataflint-spark4_2.13:0.8.8"

        # 1. Pre-download packages via a throwaway SparkPi run so the ivy cache
        #    is warm before the Connect server starts.
        echo "==> Downloading packages into ivy cache..."
        /opt/spark/bin/spark-submit \
          --packages "$$PACKAGES" \
          --conf spark.jars.ivy=/root/.ivy2 \
          --class org.apache.spark.examples.SparkPi \
          /opt/spark/examples/jars/spark-examples_*.jar 1

        # 2. Start Spark Connect server.
        # jars are distributed to executors via Spark's built-in --packages
        # mechanism (uploaded AFTER Spark's own jars, so no Netty/Netty conflict).
        echo "==> Starting Spark Connect server..."
        /opt/spark/sbin/start-connect-server.sh \
          --packages "$$PACKAGES" \
          --conf spark.plugins=io.dataflint.spark.SparkDataflintPlugin \
          --conf spark.dataflint.iceberg.autoCatalogDiscovery=true \
          --conf spark.driver.host=spark-connect \
          --conf spark.driver.memory=4g \
          --conf spark.executor.memory=8g \
          --conf spark.executor.cores=8 \
          --conf spark.jars.ivy=/root/.ivy2 \
          --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
          --conf spark.sql.catalog.lakehouse=org.apache.iceberg.spark.SparkCatalog \
          --conf spark.sql.catalog.lakehouse.type=rest \
          --conf spark.sql.catalog.lakehouse.uri=http://polaris:8181/api/catalog \
          --conf spark.sql.catalog.lakehouse.warehouse=lakehouse \
          --conf spark.sql.catalog.lakehouse.credential=$${POLARIS_CLIENT_ID}:$${POLARIS_CLIENT_SECRET} \
          --conf spark.sql.catalog.lakehouse.scope=PRINCIPAL_ROLE:ALL \
          --conf spark.sql.catalog.lakehouse.token-refresh-enabled=true \
          --conf spark.sql.catalog.lakehouse.s3.endpoint=http://minio:9000 \
          --conf spark.sql.catalog.lakehouse.s3.access-key-id=minioadmin \
          --conf spark.sql.catalog.lakehouse.s3.secret-access-key=minioadmin \
          --conf spark.sql.catalog.lakehouse.s3.path-style-access=true \
          --conf spark.sql.defaultCatalog=lakehouse \
          --conf spark.hadoop.fs.s3a.access.key=minioadmin \
          --conf spark.hadoop.fs.s3a.secret.key=minioadmin \
          --conf spark.hadoop.fs.s3a.endpoint=http://minio:9000 \
          --conf spark.hadoop.fs.s3a.path.style.access=true \
          --conf spark.hadoop.fs.s3a.connection.ssl.enabled=false \
          --conf spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem \
          --master spark://spark-master:7077
    depends_on:
      - spark-master
    environment:
      SPARK_NO_DAEMONIZE: "true"
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
      AWS_REGION: ${AWS_REGION:-us-east-1}
      AWS_ENDPOINT_URL: http://minio:9000
      POLARIS_CLIENT_ID: ${POLARIS_CLIENT_ID}
      POLARIS_CLIENT_SECRET: ${POLARIS_CLIENT_SECRET}
    ports:
      - "15002:15002"
      - "4041:4040"
    networks:
      - opendata_network
    volumes:
      - ./data:/data
      - ./dagster-workspace/projects/streamify:/opt/streamify
      - spark_ivy_cache:/root/.ivy2

  eventsim:
    build:
      context: ./eventsim
      dockerfile: Dockerfile
    container_name: eventsim
    depends_on:
      kafka:
        condition: service_healthy
    networks:
      - opendata_network
    environment:
      - JAVA_OPTS=-Xmx4G
    entrypoint: ["/bin/bash", "-c"]
    command:
      - |
        /opt/eventsim/eventsim.sh \
          -c /opt/eventsim/examples/example-config.json \
          --from 1 \
          --end-time 2035-01-01T00:00:00 \
          --nusers 3000 \
          --growth-rate 0.01 \
          --userid 1 \
          --kafkaBrokerList kafka:9092 \
          --randomseed $$(shuf -i 1-10000 -n 1) \
          --continuous
    restart: unless-stopped
    mem_limit: 6g
    memswap_limit: 8g

  kafdrop:
    image: obsidiandynamics/kafdrop:latest
    ports:
      - "9002:9000"
    environment:
      KAFKA_BROKERCONNECT: kafka:9092
      JVM_OPTS: "-Xms32M -Xmx64M"
      SERVER_SERVLET_CONTEXTPATH: "/"
    networks:
      - opendata_network
    depends_on:
      kafka:
        condition: service_healthy
    restart: unless-stopped

  polaris-console:
    build:
      context: ./polaris-console/console
      dockerfile: docker/Dockerfile
    ports:
      - "3002:80"
    environment:
      VITE_POLARIS_API_URL: http://polaris:8181
      VITE_POLARIS_REALM: default
      VITE_OAUTH_TOKEN_URL: http://polaris:8181/api/catalog/v1/oauth/tokens
      VITE_POLARIS_REALM_HEADER_NAME: realm
    networks:
      - opendata_network
    depends_on:
      polaris:
        condition: service_healthy
    restart: unless-stopped

  redis:
    image: redis:7-alpine
    container_name: redis
    ports:
      - "6379:6379"
    networks:
      - opendata_network
    restart: unless-stopped

  clickhouse:
    image: clickhouse/clickhouse-server:latest
    container_name: clickhouse
    ports:
      - "8123:8123"
      - "9009:9000"
    environment:
      - CLICKHOUSE_DB=streamify
      - CLICKHOUSE_USER=default
      - CLICKHOUSE_PASSWORD=
    networks:
      - opendata_network
    restart: unless-stopped

networks:
  opendata_network:
    driver: bridge

volumes:
  postgres_data:
  minio_data:
  zookeeper_data:
  zookeeper_logs:
  kafka_data:
  # ivy2 cache for spark-connect package downloads (driver-side only)
  spark_ivy_cache: