Trendyol-Data-Streaming-Case-Study/kafka_connect/ansible_kafka_connect.yml at main · AhmetFurkanDEMIR/Trendyol-Data-Streaming-Case-Study · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
---
- name: Deploy Kafka Connect with Docker (Dynamic from S3 + Environment Variables)
  hosts: kafka_connect
  become: yes
  gather_facts: yes

  vars:
    kafka_connect_dir: /home/ubuntu/kafka_connect

  tasks:
    - name: Install required system packages
      apt:
        name:
          - unzip
          - curl
          - jq
          - docker.io
          - docker-compose
        state: present
        update_cache: yes

    - name: Stop Confluent Kafka Connect service (if exists)
      systemd:
        name: confluent-kafka-connect
        state: stopped
      ignore_errors: yes

    - name: Disable Confluent Kafka Connect service (if exists)
      systemd:
        name: confluent-kafka-connect
        enabled: no
      ignore_errors: yes

    - name: Fix SSL certificate directory permissions
      file:
        path: /var/ssl/private
        mode: '0755'

    - name: Fix SSL certificate files permissions
      shell: chmod 644 /var/ssl/private/*.jks

    - name: Create Kafka Connect directory
      file:
        path: "{{ kafka_connect_dir }}"
        state: directory
        owner: ubuntu
        group: ubuntu
        mode: '0755'

    - name: Copy Kafka Connect files
      synchronize:
        src: "{{ playbook_dir }}/"
        dest: "{{ kafka_connect_dir }}/"
        delete: yes
        rsync_opts:
          - "--exclude=.git"
          - "--exclude=inventory.yml"
        private_key: "{{ ansible_ssh_private_key_file }}"
      delegate_to: localhost
      become: no

    - name: Create .env file on remote server
      template:
        src: "{{ playbook_dir }}/.env"
        dest: "{{ kafka_connect_dir }}/.env"
        owner: ubuntu
        group: ubuntu
        mode: '0600'

    - name: Ensure scripts directory exists
      file:
        path: "{{ kafka_connect_dir }}/scripts"
        state: directory
        owner: ubuntu
        group: ubuntu
        mode: '0755'

    - name: Copy all management scripts
      copy:
        src: "{{ playbook_dir }}/scripts/"
        dest: "{{ kafka_connect_dir }}/scripts/"
        owner: ubuntu
        group: ubuntu
        mode: '0755'

    - name: Run Kafka Connect setup script (Node 1)
      command: "{{ kafka_connect_dir }}/scripts/setup-kafka-connect-1.sh"
      args:
        chdir: "{{ kafka_connect_dir }}"
      register: setup_result
      when: inventory_hostname == 'kafka-connect-node-1'

    - name: Run Kafka Connect setup script (Node 2)
      command: "{{ kafka_connect_dir }}/scripts/setup-kafka-connect-2.sh"
      args:
        chdir: "{{ kafka_connect_dir }}"
      register: setup_result
      when: inventory_hostname == 'kafka-connect-node-2'

    - name: Display setup results
      debug:
        var: setup_result.stdout_lines

    - name: Wait for Kafka Connect to be healthy
      uri:
        url: http://localhost:8083/
        status_code: 200
      register: result
      until: result.status == 200
      retries: 30
      delay: 2

    - name: Get connector plugins
      uri:
        url: http://localhost:8083/connector-plugins
        return_content: yes
      register: plugins

    - name: Display available connector plugins
      debug:
        msg: "{{ plugins.json | map(attribute='class') | list }}"

    - name: Get running connectors
      uri:
        url: http://localhost:8083/connectors
        return_content: yes
      register: connectors

    - name: Display running connectors
      debug:
        msg: "Running connectors: {{ connectors.json }}"

    - name: Get connector status (if exists)
      uri:
        url: http://localhost:8083/connectors/http-source-topics-connector/status
        return_content: yes
      register: connector_status
      ignore_errors: yes

    - name: Display connector status
      debug:
        msg: "Connector State: {{ connector_status.json.connector.state | default('NOT FOUND') }}"
      when: connector_status.status == 200

    - name: Final summary
      debug:
        msg:
          - "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          - "✅ Kafka Connect Deployment Complete!"
          - "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          - "Server: {{ ansible_host }}"
          - "Connect API: http://{{ ansible_host }}:8083"
          - "JMX Metrics: http://{{ ansible_host }}:9404/metrics"
          - ""
          - "Useful commands:"
          - "  • Check status: curl http://{{ ansible_host }}:8083/connectors"
          - "  • View logs: ssh ubuntu@{{ ansible_host }} 'cd {{ kafka_connect_dir }} && sudo docker compose logs -f'"
          - "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"