Trendyol-Data-Streaming-Case-Study/terraform/modules/compute/scripts/init_kafka_node.sh at main · AhmetFurkanDEMIR/Trendyol-Data-Streaming-Case-Study · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
#!/bin/bash
#
# Kafka Node Initialization Script
#

set -e

############################
# VARIABLES
############################

PROJECT_NAME="trendyol-kafka"
ENVIRONMENT="prod"

LOG_FILE="/var/log/kafka-init.log"

############################
# LOG FUNCTION
############################

log() {
  echo "[$(date +'%Y-%m-%d %H:%M:%S')] $@" | tee -a "$LOG_FILE"
}

log "===== Kafka Node Initialization Started ====="
log "Node Type: ${node_type}"
log "Node ID: ${node_id}"

############################
# SYSTEM SETUP
############################

log "Updating system packages..."
apt-get update -y >> "$LOG_FILE" 2>&1
apt-get upgrade -y >> "$LOG_FILE" 2>&1

log "Installing base utilities..."
apt-get install -y \
  curl wget git vim htop net-tools jq \
  ntp ntpdate systemd-container >> "$LOG_FILE" 2>&1

############################
# CLOUDWATCH AGENT
############################

log "Installing CloudWatch Agent..."
wget -q https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb \
  -O /tmp/cw-agent.deb
dpkg -i /tmp/cw-agent.deb >> "$LOG_FILE" 2>&1

log "Creating CloudWatch Agent config..."

cat <<EOF > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
{
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [
          {
            "file_path": "/var/log/**/*.log",
            "log_group_name": "/aws/ec2/trendyol-kafka/prod",
            "log_stream_name": "{instance_id}/var-log"
          },
          {
            "file_path": "/var/log/kafka/**/*.log",
            "log_group_name": "/aws/ec2/trendyol-kafka/prod",
            "log_stream_name": "{instance_id}/kafka"
          }
        ]
      }
    }
  }
}
EOF

log "Starting CloudWatch Agent..."
/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
  -a fetch-config \
  -m ec2 \
  -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json \
  -s >> "$LOG_FILE" 2>&1

log "===== Kafka Node Initialization Completed ====="
log "CloudWatch logging ACTIVE"
log "Node is ready for Kafka installation and configuration."