deploy-all.yaml

name: Deploy Full Kafka Infrastructure

on:
  workflow_dispatch:  # Manuel tetikleme

jobs:
  terraform-apply:
    if: ${{ !inputs.skip_terraform }}
    runs-on: ubuntu-latest
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-1
          
      - name: Setup Terraform
        uses: hashicorp/setup-terraform@v2
        with:
          terraform_version: 1.5.0
          
      - name: Terraform Init
        run: |
          cd terraform
          terraform init
          
      - name: Terraform Plan
        run: |
          cd terraform
          terraform plan -out=tfplan
          
      - name: Terraform Apply
        run: |
          cd terraform
          terraform apply -auto-approve tfplan
          
      - name: Save Terraform outputs
        run: |
          cd terraform
          terraform output -json > terraform_outputs.json
          
      - name: Upload Terraform outputs
        uses: actions/upload-artifact@v4
        with:
          name: terraform-outputs
          path: terraform/terraform_outputs.json
          
  deploy-kafka-cluster:
    if: ${{ !inputs.skip_kafka_cluster }}
    needs: terraform-apply
    runs-on: ubuntu-latest
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        
      - name: Deploy Kafka Cluster
        run: |
          docker run --rm \
            -v ${{ github.workspace }}:/workspace \
            -w /workspace \
            -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
            -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
            -e SSH_PRIVATE_KEY="${{ secrets.EC2 }}" \
            -e ANSIBLE_CONFIG=ansible_kafka_cluster/ansible.cfg \
            -e ADMIN_PASSWORD="${{ secrets.ADMIN_PASSWORD }}" \
            python:3.11-slim bash -c "
              set -e

              # Sistem paketlerini kur
              apt-get update && apt-get install -y awscli jq ssh git python3-pip

              pip3 install ansible==9.2.0
              ansible-galaxy collection install confluent.platform:8.1.1

              # SSH key oluştur ve izinleri ayarla
              mkdir -p ~/.ssh
              echo \"\$SSH_PRIVATE_KEY\" > ~/.ssh/trendyol-kafka-key.pem
              chmod 600 ~/.ssh/trendyol-kafka-key.pem

              # Inventory oluştur
              bash ansible_kafka_cluster/generate-ansible-inventory.sh

              # Test ping
              ansible all -i ansible_kafka_cluster/inventory.yml -m ping

              cat ansible_kafka_cluster/cp-ansible-inventory.yml

              # Playbook çalıştır
              ansible-playbook -i ansible_kafka_cluster/cp-ansible-inventory.yml confluent.platform.all
            "
            
  deploy-kafka-api:
    needs: deploy-kafka-cluster
    runs-on: ubuntu-latest
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        
      - name: Deploy Kafka REST API
        run: |
          docker run --rm \
            -v ${{ github.workspace }}:/workspace \
            -w /workspace \
            -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
            -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
            -e SSH_PRIVATE_KEY="${{ secrets.EC2 }}" \
            -e ADMIN_PASSWORD="${{ secrets.ADMIN_PASSWORD }}" \
            python:3.11-slim bash -c "
              set -e

              # Sistem paketlerini kur
              apt-get update && apt-get install -y awscli jq ssh git python3-pip ansible

              # SSH key oluştur ve izinleri ayarla
              mkdir -p ~/.ssh
              echo \"\$SSH_PRIVATE_KEY\" > ~/.ssh/trendyol-kafka-key.pem
              chmod 600 ~/.ssh/trendyol-kafka-key.pem

              # Inventory oluştur
              bash kafka_rest_api/generate-inventory.sh

              # Test ping
              ansible all -i kafka_rest_api/inventory.ini -m ping

              # Playbook çalıştır
              ansible-playbook -i kafka_rest_api/inventory.ini kafka_rest_api/docker_up.yaml -vvv
            "
            
  deploy-kafka-connect:
    if: ${{ !inputs.skip_kafka_connect }}
    needs: deploy-kafka-api
    runs-on: ubuntu-latest
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        
      - name: Deploy Kafka Connect
        run: |
          docker run --rm \
            -v ${{ github.workspace }}:/workspace \
            -w /workspace \
            -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
            -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
            -e SSH_PRIVATE_KEY="${{ secrets.EC2 }}" \
            -e KAFKA_SECURITY_PROTOCOL="${{ secrets.KAFKA_SECURITY_PROTOCOL }}" \
            -e KAFKA_SASL_MECHANISM="${{ secrets.KAFKA_SASL_MECHANISM }}" \
            -e KAFKA_SASL_USERNAME="${{ secrets.KAFKA_SASL_USERNAME }}" \
            -e ADMIN_PASSWORD="${{ secrets.ADMIN_PASSWORD }}" \
            -e SSL_KEYSTORE_PASSWORD="${{ secrets.SSL_KEYSTORE_PASSWORD }}" \
            -e SSL_TRUSTSTORE_PASSWORD="${{ secrets.SSL_TRUSTSTORE_PASSWORD }}" \
            -e JWT_TOKEN="${{ secrets.JWT_TOKEN }}" \
            python:3.11-slim bash -c "
              set -e

              # Sistem paketlerini kur
              apt-get update && apt-get install -y awscli jq ssh git python3-pip rsync

              pip3 install ansible==9.2.0

              # SSH key oluştur ve izinleri ayarla
              mkdir -p ~/.ssh
              echo \"\$SSH_PRIVATE_KEY\" > ~/.ssh/trendyol-kafka-key.pem
              chmod 600 ~/.ssh/trendyol-kafka-key.pem

              # Inventory ve .env oluştur (S3'ten çeker, env var'larla merge eder)
              bash kafka_connect/generate-inventory.sh

              # Test ping
              ansible all -i kafka_connect/inventory.yml -m ping

              cat kafka_connect/inventory.yml

              # Playbook çalıştır
              ansible-playbook -i kafka_connect/inventory.yml kafka_connect/ansible_kafka_connect.yml
            "
            

  deploy-observability:
    needs: [terraform-apply, deploy-kafka-cluster, deploy-kafka-api, deploy-kafka-connect]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Deploy Observability Stack
        run: |
          docker run --rm \
            -v ${{ github.workspace }}:/workspace \
            -w /workspace \
            -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
            -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
            -e SSH_PRIVATE_KEY="${{ secrets.EC2 }}" \
            python:3.11-slim bash -c "
              set -e
              apt-get update && apt-get install -y awscli jq ssh git python3-pip ansible
              mkdir -p ~/.ssh
              echo \"$SSH_PRIVATE_KEY\" > ~/.ssh/trendyol-kafka-key.pem
              chmod 600 ~/.ssh/trendyol-kafka-key.pem
              bash observability/generate-inventory.sh
              ansible all -i observability/inventory.yml -m ping
              ansible-playbook -i observability/inventory.yml observability/ansible_observability.yaml
            "

  deployment-summary:
    needs: [terraform-apply, deploy-kafka-cluster, deploy-kafka-api, deploy-kafka-connect, deploy-observability]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-1
      - name: Get Infrastructure Details
        run: |
          # S3'ten Terraform state'i çek
          aws s3 cp s3://trendyol-kafka-terraform-state/prod/kafka-infrastructure/terraform.tfstate terraform_state.json
          # IP'leri çıkar
          BROKER_IPS=$(jq -r '.outputs.kafka_broker_public_dns.value[]' terraform_state.json | head -4)
          CONNECT_IP=$(jq -r '.outputs.kafka_connect_public_dns.value' terraform_state.json)
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo "🎉 Deployment Summary"
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo ""
          echo "📊 Kafka Cluster:"
          echo "$BROKER_IPS" | while read broker; do
            echo "  • $broker:9092"
          done
          echo ""
          echo "🌐 Kafka REST API:"
          echo "  • API: http://$CONNECT_IP:2020"
          echo "  • Docs: http://$CONNECT_IP:2020/docs"
          echo ""
          echo "🔌 Kafka Connect:"
          echo "  • API: http://$CONNECT_IP:8083"
          echo "  • JMX: http://$CONNECT_IP:9404/metrics"
          echo ""
          echo "🔐 Credentials:"
          echo "  • Username: admin"
          echo "  • Protocol: SASL_SSL"
          echo ""
          echo "📝 Useful Commands:"
          echo "  • List topics: kafka-topics --bootstrap-server $CONNECT_IP:9092 --list"
          echo "  • List connectors: curl http://$CONNECT_IP:8083/connectors"
          echo ""
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
      - name: Check Deployment Status
        run: |
          echo "Job Results:"
          echo "  Terraform: ${{ needs.terraform-apply.result }}"
          echo "  Kafka Cluster: ${{ needs.deploy-kafka-cluster.result }}"
          echo "  Kafka REST API: ${{ needs.deploy-kafka-api.result }}"
          echo "  Kafka Connect: ${{ needs.deploy-kafka-connect.result }}"
          echo "  Observability: ${{ needs.deploy-observability.result }}"