upload-python-package.yml

# ====================================================================================
# Purpose:
#   This workflow builds and uploads the job-builder-util-transforms Python package to Google Cloud Storage (GCS).
#   It is designed to distribute the utility transforms package for use in Dataflow jobs.
#   The workflow:
#     1. Sets up Python and installs build dependencies.
#     2. Builds the Python package (wheel and sdist).
#     3. Uploads the built artifacts and Beam YAML provider listing to a specified GCS bucket.
#
# How to Run:
#   This workflow is triggered manually via GitHub Actions `workflow_dispatch`.
#   Required Inputs:
#     - bucket_path: The GCS bucket destination (e.g., gs://my-bucket/path).
# ====================================================================================

name: Build and Upload Python Package for Job Builder

on:
  workflow_dispatch:
    inputs:
      bucket_path:
        description: 'GCS bucket path (e.g., gs://my-bucket/path)'
        required: true
      version:
        description: 'Version number for the package (e.g., 0.1.0)'
        required: true
      only_upload_listing:
        description: 'If true, skips uploading the Python package to GCS and only uploads the provider listing'
        required: false
        type: boolean
        default: false
      date_override:
        description: 'Optional date string to use instead of generating one'
        required: false
        type: string

permissions:
  contents: read

jobs:
  build-and-upload:
    runs-on: [self-hosted, release]

    steps:
      - name: Checkout Code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      # TODO(BEAM-2.76.0): Remove Java setup and background expansion service steps once Beam is upgraded to 2.76.0,
      # as the official 2.76.0 expansion service shadowJar will natively package the Delta Lake IO classes.
      # See PR#4038 for further details on code removal.
      - name: Set up Java
        uses: actions/setup-java@v5
        with:
          distribution: 'temurin'
          java-version: '17'

      - name: Build Java Dependencies
        run: mvn package -pl yaml -am -DskipTests

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: '3.11'

      - name: Install build dependencies
        run: |
          python -m pip install --upgrade pip
          pip install poetry

      - name: Install project dependencies
        working-directory: python/src/main/python/job-builder-util-transforms
        run: poetry install

      - name: Run Tests
        working-directory: python/src/main/python/job-builder-util-transforms
        env:
          LOCAL_TEST_LOOPBACK: '1'
          BEAM_SERVICE_OVERRIDES: '{"sdks:java:io:expansion-service:shadowJar": "localhost:8097"}'
        run: |
          # Start the expansion service in the background from the shaded jar
          java -cp "$GITHUB_WORKSPACE/yaml/target/template-yaml-1.0-SNAPSHOT-shaded.jar" org.apache.beam.sdk.expansion.service.ExpansionService 8097 --alsoStartLoopbackWorker=true &
          # Wait 15 seconds for the service to start up
          sleep 15
          # Execute pytest
          poetry run pytest ../../../test/python/job-builder-util-transforms/

      - name: Build Package
        working-directory: python/src/main/python/job-builder-util-transforms
        run: poetry build

      - name: Upload to GCS
        env:
          BUCKET_PATH: ${{ github.event.inputs.bucket_path }}
          VERSION: ${{ github.event.inputs.version }}
          ONLY_UPLOAD_LISTING: ${{ github.event.inputs.only_upload_listing }}
          DATE_OVERRIDE: ${{ github.event.inputs.date_override }}
        run: |
          DATE="${DATE_OVERRIDE:-$(date +'%Y-%m-%d')}"
          GCS_PACKAGE_PATH="$BUCKET_PATH/$DATE/job_builder_util_transforms-$VERSION.tar.gz"
          if [ "$ONLY_UPLOAD_LISTING" != "true" ]; then
            gcloud storage cp python/src/main/python/job-builder-util-transforms/dist/job_builder_util_transforms-*.tar.gz "$GCS_PACKAGE_PATH"
            gcloud storage cp yaml/target/template-yaml-1.0-SNAPSHOT-shaded.jar "$BUCKET_PATH/$DATE/expansion-service-custom-$VERSION.jar"
          fi
          PACKAGE_PATH_NO_GS="${GCS_PACKAGE_PATH#gs://}"
          sed "s|<package_path>|$PACKAGE_PATH_NO_GS|g" python/src/main/python/job-builder-util-transforms/provider_listing_template.yaml > "python/src/main/python/job-builder-util-transforms/yaml_provider_listing_$VERSION.yaml"
          gcloud storage cp "python/src/main/python/job-builder-util-transforms/yaml_provider_listing_$VERSION.yaml" "$BUCKET_PATH/$DATE/yaml_provider_listing_$VERSION.yaml"