Deploy Full Kafka Infrastructure #14
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Full Kafka Infrastructure | |
| on: | |
| workflow_dispatch: # Manuel tetikleme | |
| jobs: | |
| terraform-apply: | |
| if: ${{ !inputs.skip_terraform }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: eu-west-1 | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v2 | |
| with: | |
| terraform_version: 1.5.0 | |
| - name: Terraform Init | |
| run: | | |
| cd terraform | |
| terraform init | |
| - name: Terraform Plan | |
| run: | | |
| cd terraform | |
| terraform plan -out=tfplan | |
| - name: Terraform Apply | |
| run: | | |
| cd terraform | |
| terraform apply -auto-approve tfplan | |
| - name: Save Terraform outputs | |
| run: | | |
| cd terraform | |
| terraform output -json > terraform_outputs.json | |
| - name: Upload Terraform outputs | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: terraform-outputs | |
| path: terraform/terraform_outputs.json | |
| deploy-kafka-cluster: | |
| if: ${{ !inputs.skip_kafka_cluster }} | |
| needs: terraform-apply | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Deploy Kafka Cluster | |
| run: | | |
| docker run --rm \ | |
| -v ${{ github.workspace }}:/workspace \ | |
| -w /workspace \ | |
| -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| -e SSH_PRIVATE_KEY="${{ secrets.EC2 }}" \ | |
| -e ANSIBLE_CONFIG=ansible_kafka_cluster/ansible.cfg \ | |
| -e ADMIN_PASSWORD="${{ secrets.ADMIN_PASSWORD }}" \ | |
| python:3.11-slim bash -c " | |
| set -e | |
| # Sistem paketlerini kur | |
| apt-get update && apt-get install -y awscli jq ssh git python3-pip | |
| pip3 install ansible==9.2.0 | |
| ansible-galaxy collection install confluent.platform:8.1.1 | |
| # SSH key oluştur ve izinleri ayarla | |
| mkdir -p ~/.ssh | |
| echo \"\$SSH_PRIVATE_KEY\" > ~/.ssh/trendyol-kafka-key.pem | |
| chmod 600 ~/.ssh/trendyol-kafka-key.pem | |
| # Inventory oluştur | |
| bash ansible_kafka_cluster/generate-ansible-inventory.sh | |
| # Test ping | |
| ansible all -i ansible_kafka_cluster/inventory.yml -m ping | |
| cat ansible_kafka_cluster/cp-ansible-inventory.yml | |
| # Playbook çalıştır | |
| ansible-playbook -i ansible_kafka_cluster/cp-ansible-inventory.yml confluent.platform.all | |
| " | |
| deploy-kafka-api: | |
| needs: deploy-kafka-cluster | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Deploy Kafka REST API | |
| run: | | |
| docker run --rm \ | |
| -v ${{ github.workspace }}:/workspace \ | |
| -w /workspace \ | |
| -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| -e SSH_PRIVATE_KEY="${{ secrets.EC2 }}" \ | |
| -e ADMIN_PASSWORD="${{ secrets.ADMIN_PASSWORD }}" \ | |
| python:3.11-slim bash -c " | |
| set -e | |
| # Sistem paketlerini kur | |
| apt-get update && apt-get install -y awscli jq ssh git python3-pip ansible | |
| # SSH key oluştur ve izinleri ayarla | |
| mkdir -p ~/.ssh | |
| echo \"\$SSH_PRIVATE_KEY\" > ~/.ssh/trendyol-kafka-key.pem | |
| chmod 600 ~/.ssh/trendyol-kafka-key.pem | |
| # Inventory oluştur | |
| bash kafka_rest_api/generate-inventory.sh | |
| # Test ping | |
| ansible all -i kafka_rest_api/inventory.ini -m ping | |
| # Playbook çalıştır | |
| ansible-playbook -i kafka_rest_api/inventory.ini kafka_rest_api/docker_up.yaml -vvv | |
| " | |
| deploy-kafka-connect: | |
| if: ${{ !inputs.skip_kafka_connect }} | |
| needs: deploy-kafka-api | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Deploy Kafka Connect | |
| run: | | |
| docker run --rm \ | |
| -v ${{ github.workspace }}:/workspace \ | |
| -w /workspace \ | |
| -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| -e SSH_PRIVATE_KEY="${{ secrets.EC2 }}" \ | |
| -e KAFKA_SECURITY_PROTOCOL="${{ secrets.KAFKA_SECURITY_PROTOCOL }}" \ | |
| -e KAFKA_SASL_MECHANISM="${{ secrets.KAFKA_SASL_MECHANISM }}" \ | |
| -e KAFKA_SASL_USERNAME="${{ secrets.KAFKA_SASL_USERNAME }}" \ | |
| -e ADMIN_PASSWORD="${{ secrets.ADMIN_PASSWORD }}" \ | |
| -e SSL_KEYSTORE_PASSWORD="${{ secrets.SSL_KEYSTORE_PASSWORD }}" \ | |
| -e SSL_TRUSTSTORE_PASSWORD="${{ secrets.SSL_TRUSTSTORE_PASSWORD }}" \ | |
| -e JWT_TOKEN="${{ secrets.JWT_TOKEN }}" \ | |
| python:3.11-slim bash -c " | |
| set -e | |
| # Sistem paketlerini kur | |
| apt-get update && apt-get install -y awscli jq ssh git python3-pip rsync | |
| pip3 install ansible==9.2.0 | |
| # SSH key oluştur ve izinleri ayarla | |
| mkdir -p ~/.ssh | |
| echo \"\$SSH_PRIVATE_KEY\" > ~/.ssh/trendyol-kafka-key.pem | |
| chmod 600 ~/.ssh/trendyol-kafka-key.pem | |
| # Inventory ve .env oluştur (S3'ten çeker, env var'larla merge eder) | |
| bash kafka_connect/generate-inventory.sh | |
| # Test ping | |
| ansible all -i kafka_connect/inventory.yml -m ping | |
| cat kafka_connect/inventory.yml | |
| # Playbook çalıştır | |
| ansible-playbook -i kafka_connect/inventory.yml kafka_connect/ansible_kafka_connect.yml | |
| " | |
| deploy-observability: | |
| needs: [terraform-apply, deploy-kafka-cluster, deploy-kafka-api, deploy-kafka-connect] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Deploy Observability Stack | |
| run: | | |
| docker run --rm \ | |
| -v ${{ github.workspace }}:/workspace \ | |
| -w /workspace \ | |
| -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| -e SSH_PRIVATE_KEY="${{ secrets.EC2 }}" \ | |
| -e ANSIBLE_CONFIG=observability/ansible.cfg \ | |
| -e GRAFANA_ADMIN_PASSWORD="${{ secrets.ADMIN_PASSWORD }}" \ | |
| python:3.11-slim bash -c ' | |
| set -e | |
| apt-get update && apt-get install -y awscli jq ssh git python3-pip curl | |
| pip3 install ansible==9.2.0 | |
| mkdir -p ~/.ssh | |
| echo "$SSH_PRIVATE_KEY" > ~/.ssh/trendyol-kafka-key.pem | |
| chmod 600 ~/.ssh/trendyol-kafka-key.pem | |
| bash observability/deploy.sh | |
| ' | |
| deployment-summary: | |
| needs: [terraform-apply, deploy-kafka-cluster, deploy-kafka-api, deploy-kafka-connect, deploy-observability] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: eu-west-1 | |
| - name: Get Infrastructure Details | |
| run: | | |
| # S3'ten Terraform state'i çek | |
| aws s3 cp s3://trendyol-kafka-terraform-state/prod/kafka-infrastructure/terraform.tfstate terraform_state.json | |
| # IP'leri çıkar | |
| BROKER_IPS=$(jq -r '.outputs.kafka_broker_public_dns.value[]' terraform_state.json | head -4) | |
| CONNECT_IP=$(jq -r '.outputs.kafka_connect_public_dns.value' terraform_state.json) | |
| echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" | |
| echo "🎉 Deployment Summary" | |
| echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" | |
| echo "" | |
| echo "📊 Kafka Cluster:" | |
| echo "$BROKER_IPS" | while read broker; do | |
| echo " • $broker:9092" | |
| done | |
| echo "" | |
| echo "🌐 Kafka REST API:" | |
| echo " • API: http://$CONNECT_IP:2020" | |
| echo " • Docs: http://$CONNECT_IP:2020/docs" | |
| echo "" | |
| echo "🔌 Kafka Connect:" | |
| echo " • API: http://$CONNECT_IP:8083" | |
| echo " • JMX: http://$CONNECT_IP:9404/metrics" | |
| echo "" | |
| echo "🔐 Credentials:" | |
| echo " • Username: admin" | |
| echo " • Protocol: SASL_SSL" | |
| echo "" | |
| echo "📝 Useful Commands:" | |
| echo " • List topics: kafka-topics --bootstrap-server $CONNECT_IP:9092 --list" | |
| echo " • List connectors: curl http://$CONNECT_IP:8083/connectors" | |
| echo "" | |
| echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" | |
| - name: Check Deployment Status | |
| run: | | |
| echo "Job Results:" | |
| echo " Terraform: ${{ needs.terraform-apply.result }}" | |
| echo " Kafka Cluster: ${{ needs.deploy-kafka-cluster.result }}" | |
| echo " Kafka REST API: ${{ needs.deploy-kafka-api.result }}" | |
| echo " Kafka Connect: ${{ needs.deploy-kafka-connect.result }}" | |
| echo " Observability: ${{ needs.deploy-observability.result }}" |