snapshot-docker-no-tests #139
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: snapshot-docker-no-tests | |
| # This workflow is triggered on demand and on merge. Run the complete workflow test, publish the snapshot and build the docker image | |
| # and trigger the API workflow. | |
| on: | |
| workflow_dispatch: | |
| workflow_run: | |
| workflows: ["snapshot-jar-only"] | |
| types: | |
| - completed | |
| jobs: | |
| snapshot: | |
| if: >- | |
| (github.event_name == 'workflow_dispatch') || | |
| (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') || | |
| (github.event_name == 'pull_request_target' && github.event.pull_request.merged == true) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Slack event | |
| if: always() # Pick up events even if the job fails or is canceled. | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: ${{ job.status }} | |
| fields: repo,message,commit,author,action,eventName,ref,workflow,job,took,pullRequest # selectable (default: repo,message) | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_STARLAKE_CORE_WEBHOOK_URL }} # required | |
| - name: Env | |
| run: | | |
| echo "SONATYPE_USERNAME=${{ secrets.SONATYPE_USERNAME }}" >> $GITHUB_ENV | |
| echo "SONATYPE_PASSWORD=${{ secrets.SONATYPE_PASSWORD }}" >> $GITHUB_ENV | |
| echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV | |
| echo "REDSHIFT_DATABASE=${{ secrets.REDSHIFT_DATABASE }}" >> $GITHUB_ENV | |
| echo "REDSHIFT_HOST=${{ secrets.REDSHIFT_HOST }}" >> $GITHUB_ENV | |
| echo "REDSHIFT_PASSWORD=${{ secrets.REDSHIFT_PASSWORD }}" >> $GITHUB_ENV | |
| echo "REDSHIFT_USER=${{ secrets.REDSHIFT_USER }}" >> $GITHUB_ENV | |
| echo "REDSHIFT_ROLE=${{ secrets.REDSHIFT_ROLE }}" >> $GITHUB_ENV | |
| echo "SNOWFLAKE_ACCOUNT=${{ secrets.SNOWFLAKE_ACCOUNT }}" >> $GITHUB_ENV | |
| echo "SNOWFLAKE_DB=${{ secrets.SNOWFLAKE_DB }}" >> $GITHUB_ENV | |
| echo "SNOWFLAKE_PASSWORD=${{ secrets.SNOWFLAKE_PASSWORD }}" >> $GITHUB_ENV | |
| echo "SNOWFLAKE_USER=${{ secrets.SNOWFLAKE_USER }}" >> $GITHUB_ENV | |
| echo "SNOWFLAKE_WAREHOUSE=${{ secrets.SNOWFLAKE_WAREHOUSE }}" >> $GITHUB_ENV | |
| echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" >> $GITHUB_ENV | |
| echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV | |
| echo "AWS_ACCOUNT_ID=${{ secrets.AWS_ACCOUNT_ID }}" >> $GITHUB_ENV | |
| echo "GCP_PROJECT=${{ secrets.GCP_PROJECT }}" >> $GITHUB_ENV | |
| echo "TEMPORARY_GCS_BUCKET=${{ secrets.TEMPORARY_GCS_BUCKET }}" >> $GITHUB_ENV | |
| - uses: smartsquaregmbh/delete-old-packages@v0.3.1 | |
| with: | |
| version-pattern: "^\\d+\\.\\d+\\.\\d+-SNAPSHOT$" # Remove all previous SNAPSHOT versions | |
| keep: 0 | |
| names: | | |
| ai.starlake.starlake-core_2.13 | |
| - uses: actions/checkout@v4 | |
| - name: Install graphviz | |
| run: sudo apt-get install -y graphviz | |
| - name: Set up Zulu 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' # See 'Supported distributions' for available options | |
| java-version: '17' | |
| - name: Setup sbt launcher | |
| uses: sbt/setup-sbt@v1 | |
| - id: 'auth' | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| credentials_json: '${{ secrets.GCP_SERVICE_ACCOUNT }}' | |
| cleanup_credentials: false | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v1' | |
| - name: Use gcloud CLI | |
| run: gcloud info | |
| - name: Publish Snapshot on Sonatype Spark 3 for scala 2.13 | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ secrets.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ secrets.SONATYPE_PASSWORD }} sbt ++2.13 update clean compile publish | |
| docker-build: | |
| needs: snapshot | |
| name: Snapshot Docker Image | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: [amd64, arm64] | |
| permissions: | |
| packages: write | |
| contents: read | |
| attestations: write | |
| id-token: write | |
| steps: | |
| - name: Checkout | |
| id: vcs | |
| uses: actions/checkout@v4 | |
| - name: Setup sbt launcher | |
| uses: sbt/setup-sbt@v1 | |
| - name: Env | |
| run: | | |
| cat .versions >> $GITHUB_ENV | |
| platform=${{ matrix.platform }} | |
| echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV | |
| source .versions | |
| FIRST_LINE=$(head -n 1 version.sbt) | |
| SL_VERSION=$(echo "$FIRST_LINE" | sed -E 's/.*version := "([0-9]+\.[0-9]+\.?.*)"/\1/') | |
| echo "SL_VERSION=${SL_VERSION}" >> $GITHUB_ENV | |
| SL_MAJOR_MINOR_VERSION=$(echo "${SL_VERSION}" | cut -d'.' -f1-2) | |
| echo "SL_MAJOR_MINOR_VERSION=${SL_MAJOR_MINOR_VERSION}" >> $GITHUB_ENV | |
| SL_MAJOR_VERSION=$(echo "${SL_VERSION}" | cut -d'.' -f1) | |
| echo "SL_MAJOR_VERSION=${SL_MAJOR_VERSION}" >> $GITHUB_ENV | |
| REGISTRY_IMAGE=starlakeai/starlake | |
| echo "REGISTRY_IMAGE=${REGISTRY_IMAGE}" >> $GITHUB_ENV | |
| REGISTRY_IMAGE_LATEST=${REGISTRY_IMAGE}:${SL_VERSION:-latest} | |
| echo "REGISTRY_IMAGE_LATEST=${REGISTRY_IMAGE_LATEST}" >> $GITHUB_ENV | |
| echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV | |
| echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV | |
| echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV | |
| - name: Prepare | |
| run: | | |
| ./scripts/docker-prepare.sh | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY_IMAGE }} | |
| labels: | | |
| org.opencontainers.image.vendor=starlakeai | |
| org.opencontainers.image.licenses=Apache-2.0 | |
| org.opencontainers.image.title=starlakeai/starlake-${{ env.SL_MAJOR_MINOR_VERSION }} | |
| org.opencontainers.image.description="A declarative text based tool that enables analysts and engineers to extract, load, transform and orchestrate their data pipelines." | |
| org.opencontainers.image.url=https://starlake.ai | |
| # generate Docker tags based on the following events/attributes | |
| tags: | | |
| type=schedule | |
| type=semver,pattern={{version}},value=${{ env.SL_VERSION }} | |
| type=semver,pattern={{major}}.{{minor}},value=${{ env.SL_MAJOR_MINOR_VERSION }} | |
| type=semver,pattern={{major}},value=${{ env.SL_MAJOR_VERSION }} | |
| # type=sha | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ env.DOCKERHUB_USERNAME }} | |
| password: ${{ env.DOCKERHUB_TOKEN }} | |
| - name: Build and export to Docker | |
| id: build | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: "./distrib/docker" | |
| tags: ${{ env.REGISTRY_IMAGE }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-args: | | |
| BUILD_DATE=${{ env.BUILD_DATE }} | |
| VCS_REF=${{ steps.vcs.outputs.commit }} | |
| SL_VERSION=${{ env.SL_VERSION }} | |
| platforms: linux/${{ matrix.platform }} | |
| outputs: type=image,push-by-digest=true,name-canonical=true,push=true | |
| - name: Export digest | |
| run: | | |
| mkdir -p ${{ runner.temp }}/digests | |
| digest="${{ steps.build.outputs.digest }}" | |
| touch "${{ runner.temp }}/digests/${digest#sha256:}" | |
| - name: Upload digest | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: digests-${{ env.PLATFORM_PAIR }} | |
| path: ${{ runner.temp }}/digests/* | |
| if-no-files-found: error | |
| retention-days: 1 | |
| merge: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - docker-build | |
| steps: | |
| - name: Checkout | |
| id: vcs | |
| uses: actions/checkout@v4 | |
| - name: Env | |
| run: | | |
| cat .versions >> $GITHUB_ENV | |
| source .versions | |
| FIRST_LINE=$(head -n 1 version.sbt) | |
| SL_VERSION=$(echo "$FIRST_LINE" | sed -E 's/.*version := "([0-9]+\.[0-9]+\.?.*)"/\1/') | |
| echo "SL_VERSION=${SL_VERSION}" >> $GITHUB_ENV | |
| SL_MAJOR_MINOR_VERSION=$(echo "${SL_VERSION}" | cut -d'.' -f1-2) | |
| echo "SL_MAJOR_MINOR_VERSION=${SL_MAJOR_MINOR_VERSION}" >> $GITHUB_ENV | |
| SL_MAJOR_VERSION=$(echo "${SL_VERSION}" | cut -d'.' -f1) | |
| echo "SL_MAJOR_VERSION=${SL_MAJOR_VERSION}" >> $GITHUB_ENV | |
| REGISTRY_IMAGE=starlakeai/starlake | |
| echo "REGISTRY_IMAGE=${REGISTRY_IMAGE}" >> $GITHUB_ENV | |
| REGISTRY_IMAGE_LATEST=${REGISTRY_IMAGE}:${SL_VERSION:-latest} | |
| echo "REGISTRY_IMAGE_LATEST=${REGISTRY_IMAGE_LATEST}" >> $GITHUB_ENV | |
| echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV | |
| echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV | |
| echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV | |
| - name: Download digests | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ${{ runner.temp }}/digests | |
| pattern: digests-* | |
| merge-multiple: true | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ env.DOCKERHUB_USERNAME }} | |
| password: ${{ env.DOCKERHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY_IMAGE }} | |
| tags: | | |
| type=semver,pattern={{version}},value=v${{ env.SL_VERSION }} | |
| - name: Create manifest list and push | |
| working-directory: ${{ runner.temp }}/digests | |
| run: | | |
| docker buildx imagetools create -t ${{ env.REGISTRY_IMAGE }}:${{ env.SL_VERSION }} $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) | |
| - name: Inspect image | |
| run: | | |
| docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ env.SL_VERSION }} |