Trendyol-Data-Streaming-Case-Study/kafka_connect/docker-compose-1.yml at main · AhmetFurkanDEMIR/Trendyol-Data-Streaming-Case-Study · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
version: '3.8'

services:
  kafka-connect:
    image: confluentinc/cp-kafka-connect:7.5.0
    container_name: kafka-connect
    hostname: kafka-connect
    user: "0:0"  # Run as root to access mounted SSL certificates
    ports:
      - "8083:8083"
      - "9404:9404"  # JMX Exporter
    environment:
      # Connect Worker Configuration
      CONNECT_BOOTSTRAP_SERVERS: "${KAFKA_BOOTSTRAP_SERVERS}"
      CONNECT_REST_PORT: 8083
      CONNECT_REST_ADVERTISED_HOST_NAME: "ec2-3-255-139-80.eu-west-1.compute.amazonaws.com"

      # Cluster Configuration
      CONNECT_GROUP_ID: "trendyol-connect-cluster"
      CONNECT_CONFIG_STORAGE_TOPIC: "trendyol-kafka-connect-configs"
      CONNECT_OFFSET_STORAGE_TOPIC: "trendyol-kafka-connect-offsets"
      CONNECT_STATUS_STORAGE_TOPIC: "trendyol-kafka-connect-status"

      # Topic Replication Factors
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 3
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 3
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 3

      # Converters
      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false"
      CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"

      # Internal Converters
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"

      # Security Configuration (SASL_SSL)
      CONNECT_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-SASL_SSL}"
      CONNECT_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-SCRAM-SHA-512}"
      CONNECT_SASL_JAAS_CONFIG: "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"${KAFKA_SASL_USERNAME:-admin}\" password=\"${KAFKA_SASL_PASSWORD}\";"
      CONNECT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
      CONNECT_SSL_TRUSTSTORE_LOCATION: "/etc/kafka/secrets/kafka_connect.truststore.jks"
      CONNECT_SSL_TRUSTSTORE_PASSWORD: "${SSL_TRUSTSTORE_PASSWORD:-confluenttruststorepass}"
      CONNECT_SSL_KEYSTORE_LOCATION: "/etc/kafka/secrets/kafka_connect.keystore.jks"
      CONNECT_SSL_KEYSTORE_PASSWORD: "${SSL_KEYSTORE_PASSWORD:-confluenttruststorepass}"
      CONNECT_SSL_KEY_PASSWORD: "${SSL_KEY_PASSWORD:-confluenttruststorepass}"

      # Admin Client Security
      CONNECT_ADMIN_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-SASL_SSL}"
      CONNECT_ADMIN_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-SCRAM-SHA-512}"
      CONNECT_ADMIN_SASL_JAAS_CONFIG: "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"${KAFKA_SASL_USERNAME:-admin}\" password=\"${KAFKA_SASL_PASSWORD}\";"
      CONNECT_ADMIN_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
      CONNECT_ADMIN_SSL_TRUSTSTORE_LOCATION: "/etc/kafka/secrets/kafka_connect.truststore.jks"
      CONNECT_ADMIN_SSL_TRUSTSTORE_PASSWORD: "${SSL_TRUSTSTORE_PASSWORD:-confluenttruststorepass}"
      CONNECT_ADMIN_SSL_KEYSTORE_LOCATION: "/etc/kafka/secrets/kafka_connect.keystore.jks"
      CONNECT_ADMIN_SSL_KEYSTORE_PASSWORD: "${SSL_KEYSTORE_PASSWORD:-confluenttruststorepass}"
      CONNECT_ADMIN_SSL_KEY_PASSWORD: "${SSL_KEY_PASSWORD:-confluenttruststorepass}"

      # Producer Security
      CONNECT_PRODUCER_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-SASL_SSL}"
      CONNECT_PRODUCER_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-SCRAM-SHA-512}"
      CONNECT_PRODUCER_SASL_JAAS_CONFIG: "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"${KAFKA_SASL_USERNAME:-admin}\" password=\"${KAFKA_SASL_PASSWORD}\";"
      CONNECT_PRODUCER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
      CONNECT_PRODUCER_SSL_TRUSTSTORE_LOCATION: "/etc/kafka/secrets/kafka_connect.truststore.jks"
      CONNECT_PRODUCER_SSL_TRUSTSTORE_PASSWORD: "${SSL_TRUSTSTORE_PASSWORD:-confluenttruststorepass}"
      CONNECT_PRODUCER_SSL_KEYSTORE_LOCATION: "/etc/kafka/secrets/kafka_connect.keystore.jks"
      CONNECT_PRODUCER_SSL_KEYSTORE_PASSWORD: "${SSL_KEYSTORE_PASSWORD:-confluenttruststorepass}"
      CONNECT_PRODUCER_SSL_KEY_PASSWORD: "${SSL_KEY_PASSWORD:-confluenttruststorepass}"

      # Consumer Security
      CONNECT_CONSUMER_SECURITY_PROTOCOL: "${KAFKA_SECURITY_PROTOCOL:-SASL_SSL}"
      CONNECT_CONSUMER_SASL_MECHANISM: "${KAFKA_SASL_MECHANISM:-SCRAM-SHA-512}"
      CONNECT_CONSUMER_SASL_JAAS_CONFIG: "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"${KAFKA_SASL_USERNAME:-admin}\" password=\"${KAFKA_SASL_PASSWORD}\";"
      CONNECT_CONSUMER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
      CONNECT_CONSUMER_SSL_TRUSTSTORE_LOCATION: "/etc/kafka/secrets/kafka_connect.truststore.jks"
      CONNECT_CONSUMER_SSL_TRUSTSTORE_PASSWORD: "${SSL_TRUSTSTORE_PASSWORD:-confluenttruststorepass}"
      CONNECT_CONSUMER_SSL_KEYSTORE_LOCATION: "/etc/kafka/secrets/kafka_connect.keystore.jks"
      CONNECT_CONSUMER_SSL_KEYSTORE_PASSWORD: "${SSL_KEYSTORE_PASSWORD:-confluenttruststorepass}"
      CONNECT_CONSUMER_SSL_KEY_PASSWORD: "${SSL_KEY_PASSWORD:-confluenttruststorepass}"

      # Plugin Path
      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components,/etc/kafka-connect/plugins"

      # Logging
      CONNECT_LOG4J_ROOT_LOGLEVEL: INFO
      CONNECT_LOG4J_LOGGERS: "org.reflections=ERROR"

      # JMX Configuration
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: localhost
      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=localhost -Dcom.sun.management.jmxremote.rmi.port=9101"

      # JMX Exporter for Prometheus + SSL Certificate Verification Disable
      KAFKA_OPTS: "-javaagent:/usr/share/jmx_exporter/jmx_prometheus_javaagent.jar=9404:/etc/kafka-connect/jmx-exporter-config.yml -Dssl.endpoint.identification.algorithm="

    volumes:
      - ./plugins:/etc/kafka-connect/plugins
      - ./jmx_prometheus_javaagent.jar:/usr/share/jmx_exporter/jmx_prometheus_javaagent.jar
      - ./jmx-exporter-config.yml:/etc/kafka-connect/jmx-exporter-config.yml
      - ./connectors:/etc/kafka-connect/connectors
      - /var/ssl/private:/etc/kafka/secrets:ro

    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8083/"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s

    restart: unless-stopped

    networks:
      - kafka-network

networks:
  kafka-network:
    driver: bridge