diff --git a/.github/workflows/cache_comparison.yaml b/.github/workflows/cache_comparison.yaml index cafaa498334..c13fb13269b 100644 --- a/.github/workflows/cache_comparison.yaml +++ b/.github/workflows/cache_comparison.yaml @@ -11,7 +11,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 10 - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -23,6 +23,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - env: BASE_REF: ${{ github.event.inputs.base_ref }} BUILD_COMMIT: ${{ github.event.inputs.build_commit }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2529a3cb718..68801299133 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,6 +4,442 @@ jobs: + build_wheels_cp314_linux_arm64: + container: + image: quay.io/pypa/manylinux_2_28_aarch64:latest + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + PANTS_REMOTE_CACHE_READ: 'false' + PANTS_REMOTE_CACHE_WRITE: 'false' + PY: python3.14 + if: github.repository_owner == 'pantsbuild' + name: Build wheels (Linux-ARM64, cp314) + needs: + - release_info + permissions: + attestations: write + contents: write + id-token: write + runs-on: + - depot-ubuntu-24.04-arm-8 + steps: + - name: Free up disk space + run: |- + df -h + rm -rf /mnt/host-root/usr/share/dotnet || true + rm -rf /mnt/host-root/usr/local/lib/android || true + rm -rf /mnt/host-root/opt/ghc || true + rm -rf /mnt/host-root/usr/local/.ghcup || true + df -h + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 10 + ref: ${{ needs.release_info.outputs.build-ref }} + - name: Configure Git + run: git config --global safe.directory "$GITHUB_WORKSPACE" + - name: Install rustup + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -v -y --default-toolchain none + echo "${HOME}/.cargo/bin" >> $GITHUB_PATH + - name: Install Rust toolchain + run: | + # Set the default toolchain. Installs the toolchain if it is not already installed. + rustup default 1.96.0 + cargo version + - name: Expose Pythons + run: | + echo "/opt/python/cp37-cp37m/bin" >> $GITHUB_PATH + echo "/opt/python/cp38-cp38/bin" >> $GITHUB_PATH + echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH + echo "/opt/python/cp310-cp310/bin" >> $GITHUB_PATH + echo "/opt/python/cp311-cp311/bin" >> $GITHUB_PATH + echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH + echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314t/bin" >> $GITHUB_PATH + - name: Install Protoc + uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 23.x + - name: Install Python headers + run: echo 'no-op' + - name: Pin the distribution interpreter + run: echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=['$(dirname "$(command -v python3.14)")']" >> "$GITHUB_ENV" + - env: + HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} + PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} + name: Build wheels + run: ./pants run src/python/pants_release/release.py -- build-wheels + - env: + HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} + PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} + name: Build Pants PEX + run: ./pants package src/python/pants:pants-pex@scie_pbs_free_threaded=cp314 + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: logs-wheels-and-pex-Linux-ARM64 + overwrite: 'true' + path: .pants.d/workdir/*.log + - if: needs.release_info.outputs.is-release == 'true' + name: Attest the pantsbuild.pants wheel + uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + with: + subject-path: dist/deploy/wheels/pantsbuild.pants/**/pantsbuild_pants-*.whl + - if: needs.release_info.outputs.is-release == 'true' + name: Rename the Pants Pex to its final name for upload + run: | + PANTS_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import pants.version;print(pants.version.VERSION)") + PY_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import sys;print(f'cp{sys.version_info[0]}{sys.version_info[1]}{sys.abiflags}')") + PLAT=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import os;print(f'{os.uname().sysname.lower()}_{os.uname().machine.lower()}')") + PEX_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT.pex + PEX_SCIE_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT + + mv dist/src.python.pants/pants-pex.pex dist/src.python.pants/$PEX_FILENAME + mv dist/src.python.pants/pants-pex dist/src.python.pants/$PEX_SCIE_FILENAME + echo "PEX_FILENAME=$PEX_FILENAME" | tee -a "$GITHUB_ENV" + echo "PEX_SCIE_FILENAME=$PEX_SCIE_FILENAME" | tee -a "$GITHUB_ENV" + - if: needs.release_info.outputs.is-release == 'true' + name: Attest the Pants Pex artifact + uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + with: + subject-path: dist/src.python.pants/* + - if: needs.release_info.outputs.is-release == 'true' + name: Upload Wheel and Pex + run: | + curl -L --fail \ + -X POST \ + -H "Authorization: Bearer ${{ github.token }}" \ + -H "Content-Type: application/octet-stream" \ + ${{ needs.release_info.outputs.release-asset-upload-url }}?name=$PEX_FILENAME \ + --data-binary "@dist/src.python.pants/$PEX_FILENAME" + + curl -L --fail \ + -X POST \ + -H "Authorization: Bearer ${{ github.token }}" \ + -H "Content-Type: application/octet-stream" \ + ${{ needs.release_info.outputs.release-asset-upload-url }}?name=$PEX_SCIE_FILENAME \ + --data-binary "@dist/src.python.pants/$PEX_SCIE_FILENAME" + + WHL=$(find dist/deploy/wheels/pantsbuild.pants -type f -name "pantsbuild_pants-*.whl") + curl -L --fail \ + -X POST \ + -H "Authorization: Bearer ${{ github.token }}" \ + -H "Content-Type: application/octet-stream" \ + "${{ needs.release_info.outputs.release-asset-upload-url }}?name=$(basename $WHL)" \ + --data-binary "@$WHL"; + timeout-minutes: 90 + build_wheels_cp314_linux_x86_64: + container: + image: quay.io/pypa/manylinux_2_28_x86_64:latest + volumes: + - /:/mnt/host-root + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + PANTS_REMOTE_CACHE_READ: 'false' + PANTS_REMOTE_CACHE_WRITE: 'false' + PY: python3.14 + if: github.repository_owner == 'pantsbuild' + name: Build wheels (Linux-x86_64, cp314) + needs: + - release_info + permissions: + attestations: write + contents: write + id-token: write + runs-on: + - depot-ubuntu-22.04-8 + steps: + - name: Free up disk space + run: |- + df -h + rm -rf /mnt/host-root/usr/share/dotnet || true + rm -rf /mnt/host-root/usr/local/lib/android || true + rm -rf /mnt/host-root/opt/ghc || true + rm -rf /mnt/host-root/usr/local/.ghcup || true + df -h + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 10 + ref: ${{ needs.release_info.outputs.build-ref }} + - name: Configure Git + run: git config --global safe.directory "$GITHUB_WORKSPACE" + - name: Install rustup + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -v -y --default-toolchain none + echo "${HOME}/.cargo/bin" >> $GITHUB_PATH + - name: Install Rust toolchain + run: | + # Set the default toolchain. Installs the toolchain if it is not already installed. + rustup default 1.96.0 + cargo version + - name: Expose Pythons + run: | + echo "/opt/python/cp37-cp37m/bin" >> $GITHUB_PATH + echo "/opt/python/cp38-cp38/bin" >> $GITHUB_PATH + echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH + echo "/opt/python/cp310-cp310/bin" >> $GITHUB_PATH + echo "/opt/python/cp311-cp311/bin" >> $GITHUB_PATH + echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH + echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314t/bin" >> $GITHUB_PATH + - name: Install Protoc + uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 23.x + - name: Install Go 1.25.3 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + cache: false + go-version: 1.25.3 + - name: Install Go 1.24.9 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + cache: false + go-version: 1.24.9 + - name: Pin the distribution interpreter + run: echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=['$(dirname "$(command -v python3.14)")']" >> "$GITHUB_ENV" + - env: + HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} + PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} + PANTS_PROCESS_EXECUTION_LOCAL_PARALLELISM: '1' + name: Build wheels + run: ./pants run src/python/pants_release/release.py -- build-wheels + - env: + HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} + PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} + PANTS_PROCESS_EXECUTION_LOCAL_PARALLELISM: '1' + name: Build Pants PEX + run: ./pants package src/python/pants:pants-pex@scie_pbs_free_threaded=cp314 + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: logs-wheels-and-pex-Linux-x86_64 + overwrite: 'true' + path: .pants.d/workdir/*.log + - if: needs.release_info.outputs.is-release == 'true' + name: Attest the pantsbuild.pants wheel + uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + with: + subject-path: dist/deploy/wheels/pantsbuild.pants/**/pantsbuild_pants-*.whl + - if: needs.release_info.outputs.is-release == 'true' + name: Rename the Pants Pex to its final name for upload + run: | + PANTS_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import pants.version;print(pants.version.VERSION)") + PY_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import sys;print(f'cp{sys.version_info[0]}{sys.version_info[1]}{sys.abiflags}')") + PLAT=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import os;print(f'{os.uname().sysname.lower()}_{os.uname().machine.lower()}')") + PEX_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT.pex + PEX_SCIE_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT + + mv dist/src.python.pants/pants-pex.pex dist/src.python.pants/$PEX_FILENAME + mv dist/src.python.pants/pants-pex dist/src.python.pants/$PEX_SCIE_FILENAME + echo "PEX_FILENAME=$PEX_FILENAME" | tee -a "$GITHUB_ENV" + echo "PEX_SCIE_FILENAME=$PEX_SCIE_FILENAME" | tee -a "$GITHUB_ENV" + - if: needs.release_info.outputs.is-release == 'true' + name: Attest the Pants Pex artifact + uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + with: + subject-path: dist/src.python.pants/* + - if: needs.release_info.outputs.is-release == 'true' + name: Upload Wheel and Pex + run: | + curl -L --fail \ + -X POST \ + -H "Authorization: Bearer ${{ github.token }}" \ + -H "Content-Type: application/octet-stream" \ + ${{ needs.release_info.outputs.release-asset-upload-url }}?name=$PEX_FILENAME \ + --data-binary "@dist/src.python.pants/$PEX_FILENAME" + + curl -L --fail \ + -X POST \ + -H "Authorization: Bearer ${{ github.token }}" \ + -H "Content-Type: application/octet-stream" \ + ${{ needs.release_info.outputs.release-asset-upload-url }}?name=$PEX_SCIE_FILENAME \ + --data-binary "@dist/src.python.pants/$PEX_SCIE_FILENAME" + + WHL=$(find dist/deploy/wheels/pantsbuild.pants -type f -name "pantsbuild_pants-*.whl") + curl -L --fail \ + -X POST \ + -H "Authorization: Bearer ${{ github.token }}" \ + -H "Content-Type: application/octet-stream" \ + "${{ needs.release_info.outputs.release-asset-upload-url }}?name=$(basename $WHL)" \ + --data-binary "@$WHL"; + - if: needs.release_info.outputs.is-release == 'true' + name: Attest the pantsbuild.pants.testutil wheel + uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + with: + subject-path: dist/deploy/wheels/pantsbuild.pants/**/pantsbuild_pants_testutil*.whl + - if: needs.release_info.outputs.is-release == 'true' + name: Upload testutil Wheel + run: | + WHL=$(find dist/deploy/wheels/pantsbuild.pants -type f -name "pantsbuild_pants_testutil*.whl") + curl -L --fail \ + -X POST \ + -H "Authorization: Bearer ${{ github.token }}" \ + -H "Content-Type: application/octet-stream" \ + "${{ needs.release_info.outputs.release-asset-upload-url }}?name=$(basename $WHL)" \ + --data-binary "@$WHL"; + timeout-minutes: 90 + build_wheels_cp314_macos14_arm64: + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: false + PANTS_REMOTE_CACHE_READ: 'false' + PANTS_REMOTE_CACHE_WRITE: 'false' + PY: python3.14 + if: github.repository_owner == 'pantsbuild' + name: Build wheels (macOS14-ARM64, cp314) + needs: + - release_info + permissions: + attestations: write + contents: write + id-token: write + runs-on: + - depot-macos-14 + steps: + - name: Free up disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + with: + android: true + docker-images: false + dotnet: true + haskell: true + large-packages: false + swap-storage: false + tool-cache: false + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 10 + ref: ${{ needs.release_info.outputs.build-ref }} + - name: Set up Python 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: |- + 3.9 + 3.10 + 3.11 + 3.12 + 3.13 + 3.14 + 3.14t + - name: Install Protoc + uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 23.x + - name: Set rustup profile + run: rustup set profile default + - name: Cache Rust toolchain + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: macOS14-ARM64-rustup-${{ hashFiles('src/rust/rust-toolchain') }}-v2 + path: | + ~/.rustup/toolchains/1.96.0-* + ~/.rustup/update-hashes + ~/.rustup/settings.toml + - name: Cache Cargo + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 + with: + cache-bin: 'false' + shared-key: engine + workspaces: src/rust + - name: Install Protoc + uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 23.x + - name: Install Go 1.25.3 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + cache: false + go-version: 1.25.3 + - name: Install Go 1.24.9 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + cache: false + go-version: 1.24.9 + - name: Pin the distribution interpreter + run: echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=['$(dirname "$(command -v python3.14)")']" >> "$GITHUB_ENV" + - env: + ARCHFLAGS: -arch arm64 + HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} + PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} + _PYTHON_HOST_PLATFORM: macosx-14.0-arm64 + name: Build wheels + run: ./pants run src/python/pants_release/release.py -- build-wheels + - env: + ARCHFLAGS: -arch arm64 + HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} + PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} + _PYTHON_HOST_PLATFORM: macosx-14.0-arm64 + name: Build Pants PEX + run: ./pants package src/python/pants:pants-pex@scie_pbs_free_threaded=cp314 + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: logs-wheels-and-pex-macOS14-ARM64 + overwrite: 'true' + path: .pants.d/workdir/*.log + - if: needs.release_info.outputs.is-release == 'true' + name: Attest the pantsbuild.pants wheel + uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + with: + subject-path: dist/deploy/wheels/pantsbuild.pants/**/pantsbuild_pants-*.whl + - if: needs.release_info.outputs.is-release == 'true' + name: Rename the Pants Pex to its final name for upload + run: | + PANTS_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import pants.version;print(pants.version.VERSION)") + PY_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import sys;print(f'cp{sys.version_info[0]}{sys.version_info[1]}{sys.abiflags}')") + PLAT=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import os;print(f'{os.uname().sysname.lower()}_{os.uname().machine.lower()}')") + PEX_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT.pex + PEX_SCIE_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT + + mv dist/src.python.pants/pants-pex.pex dist/src.python.pants/$PEX_FILENAME + mv dist/src.python.pants/pants-pex dist/src.python.pants/$PEX_SCIE_FILENAME + echo "PEX_FILENAME=$PEX_FILENAME" | tee -a "$GITHUB_ENV" + echo "PEX_SCIE_FILENAME=$PEX_SCIE_FILENAME" | tee -a "$GITHUB_ENV" + - if: needs.release_info.outputs.is-release == 'true' + name: Attest the Pants Pex artifact + uses: actions/attest-build-provenance@977bb373ede98d70efdf65b84cb5f73e068dcc2a # v3.0.0 + with: + subject-path: dist/src.python.pants/* + - if: needs.release_info.outputs.is-release == 'true' + name: Upload Wheel and Pex + run: | + curl -L --fail \ + -X POST \ + -H "Authorization: Bearer ${{ github.token }}" \ + -H "Content-Type: application/octet-stream" \ + ${{ needs.release_info.outputs.release-asset-upload-url }}?name=$PEX_FILENAME \ + --data-binary "@dist/src.python.pants/$PEX_FILENAME" + + curl -L --fail \ + -X POST \ + -H "Authorization: Bearer ${{ github.token }}" \ + -H "Content-Type: application/octet-stream" \ + ${{ needs.release_info.outputs.release-asset-upload-url }}?name=$PEX_SCIE_FILENAME \ + --data-binary "@dist/src.python.pants/$PEX_SCIE_FILENAME" + + WHL=$(find dist/deploy/wheels/pantsbuild.pants -type f -name "pantsbuild_pants-*.whl") + curl -L --fail \ + -X POST \ + -H "Authorization: Bearer ${{ github.token }}" \ + -H "Content-Type: application/octet-stream" \ + "${{ needs.release_info.outputs.release-asset-upload-url }}?name=$(basename $WHL)" \ + --data-binary "@$WHL"; + timeout-minutes: 90 build_wheels_linux_arm64: container: image: quay.io/pypa/manylinux_2_28_aarch64:latest @@ -11,8 +447,9 @@ jobs: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' + PY: python3.14t if: github.repository_owner == 'pantsbuild' - name: Build wheels (Linux-ARM64) + name: Build wheels (Linux-ARM64, cp314t) needs: - release_info permissions: @@ -56,6 +493,7 @@ jobs: echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH echo "/opt/python/cp314-cp314/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314t/bin" >> $GITHUB_PATH - name: Install Protoc uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 with: @@ -63,6 +501,8 @@ jobs: version: 23.x - name: Install Python headers run: echo 'no-op' + - name: Pin the distribution interpreter + run: echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=['$(dirname "$(command -v python3.14t)")']" >> "$GITHUB_ENV" - env: HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} @@ -72,7 +512,7 @@ jobs: HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} name: Build Pants PEX - run: ./pants package src/python/pants:pants-pex + run: ./pants package src/python/pants:pants-pex@scie_pbs_free_threaded=cp314t - continue-on-error: true if: always() name: Upload pants.log @@ -90,7 +530,7 @@ jobs: name: Rename the Pants Pex to its final name for upload run: | PANTS_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import pants.version;print(pants.version.VERSION)") - PY_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import sys;print(f'cp{sys.version_info[0]}{sys.version_info[1]}')") + PY_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import sys;print(f'cp{sys.version_info[0]}{sys.version_info[1]}{sys.abiflags}')") PLAT=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import os;print(f'{os.uname().sysname.lower()}_{os.uname().machine.lower()}')") PEX_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT.pex PEX_SCIE_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT @@ -138,8 +578,9 @@ jobs: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' + PY: python3.14t if: github.repository_owner == 'pantsbuild' - name: Build wheels (Linux-x86_64) + name: Build wheels (Linux-x86_64, cp314t) needs: - release_info permissions: @@ -183,6 +624,7 @@ jobs: echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH echo "/opt/python/cp314-cp314/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314t/bin" >> $GITHUB_PATH - name: Install Protoc uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 with: @@ -198,6 +640,8 @@ jobs: with: cache: false go-version: 1.24.9 + - name: Pin the distribution interpreter + run: echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=['$(dirname "$(command -v python3.14t)")']" >> "$GITHUB_ENV" - env: HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} @@ -209,7 +653,7 @@ jobs: PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} PANTS_PROCESS_EXECUTION_LOCAL_PARALLELISM: '1' name: Build Pants PEX - run: ./pants package src/python/pants:pants-pex + run: ./pants package src/python/pants:pants-pex@scie_pbs_free_threaded=cp314t - continue-on-error: true if: always() name: Upload pants.log @@ -227,7 +671,7 @@ jobs: name: Rename the Pants Pex to its final name for upload run: | PANTS_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import pants.version;print(pants.version.VERSION)") - PY_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import sys;print(f'cp{sys.version_info[0]}{sys.version_info[1]}')") + PY_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import sys;print(f'cp{sys.version_info[0]}{sys.version_info[1]}{sys.abiflags}')") PLAT=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import os;print(f'{os.uname().sysname.lower()}_{os.uname().machine.lower()}')") PEX_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT.pex PEX_SCIE_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT @@ -286,8 +730,9 @@ jobs: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: false PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' + PY: python3.14t if: github.repository_owner == 'pantsbuild' - name: Build wheels (macOS14-ARM64) + name: Build wheels (macOS14-ARM64, cp314t) needs: - release_info permissions: @@ -312,7 +757,7 @@ jobs: with: fetch-depth: 10 ref: ${{ needs.release_info.outputs.build-ref }} - - name: Set up Python 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -322,6 +767,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Install Protoc uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 with: @@ -358,6 +804,8 @@ jobs: with: cache: false go-version: 1.24.9 + - name: Pin the distribution interpreter + run: echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=['$(dirname "$(command -v python3.14t)")']" >> "$GITHUB_ENV" - env: ARCHFLAGS: -arch arm64 HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} @@ -371,7 +819,7 @@ jobs: PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} _PYTHON_HOST_PLATFORM: macosx-14.0-arm64 name: Build Pants PEX - run: ./pants package src/python/pants:pants-pex + run: ./pants package src/python/pants:pants-pex@scie_pbs_free_threaded=cp314t - continue-on-error: true if: always() name: Upload pants.log @@ -389,7 +837,7 @@ jobs: name: Rename the Pants Pex to its final name for upload run: | PANTS_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import pants.version;print(pants.version.VERSION)") - PY_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import sys;print(f'cp{sys.version_info[0]}{sys.version_info[1]}')") + PY_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import sys;print(f'cp{sys.version_info[0]}{sys.version_info[1]}{sys.abiflags}')") PLAT=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import os;print(f'{os.uname().sysname.lower()}_{os.uname().machine.lower()}')") PEX_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT.pex PEX_SCIE_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT @@ -436,6 +884,9 @@ jobs: - build_wheels_linux_x86_64 - build_wheels_linux_arm64 - build_wheels_macos14_arm64 + - build_wheels_cp314_linux_x86_64 + - build_wheels_cp314_linux_arm64 + - build_wheels_cp314_macos14_arm64 - release_info runs-on: ubuntu-22.04 steps: @@ -445,7 +896,7 @@ jobs: fetch-depth: '0' fetch-tags: true ref: ${{ needs.release_info.outputs.build-ref }} - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -457,6 +908,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Install Protoc uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ca96205a748..5d42deacbca 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -38,7 +38,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 10 - - name: Set up Python 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -49,6 +49,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Install Protoc uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 with: @@ -146,7 +147,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 10 - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -158,6 +159,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Install Protoc uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 with: @@ -264,7 +266,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 10 - - name: Set up Python 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -274,6 +276,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Install Protoc uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 with: @@ -346,6 +349,286 @@ jobs: name: Test Rust run: ./cargo test --locked --tests -- --nocapture timeout-minutes: 60 + build_wheels_cp314_linux_arm64: + container: + image: quay.io/pypa/manylinux_2_28_aarch64:latest + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + MODE: debug + PANTS_REMOTE_CACHE_READ: 'false' + PANTS_REMOTE_CACHE_WRITE: 'false' + PY: python3.14 + if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true' || needs.classify_changes.outputs.ci_config == 'true')) && (needs.classify_changes.outputs.no_code != 'true') + name: Build wheels (Linux-ARM64, cp314) + needs: + - classify_changes + permissions: + attestations: write + contents: write + id-token: write + runs-on: + - depot-ubuntu-24.04-arm-8 + steps: + - name: Free up disk space + run: |- + df -h + rm -rf /mnt/host-root/usr/share/dotnet || true + rm -rf /mnt/host-root/usr/local/lib/android || true + rm -rf /mnt/host-root/opt/ghc || true + rm -rf /mnt/host-root/usr/local/.ghcup || true + df -h + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 10 + - name: Configure Git + run: git config --global safe.directory "$GITHUB_WORKSPACE" + - name: Install rustup + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -v -y --default-toolchain none + echo "${HOME}/.cargo/bin" >> $GITHUB_PATH + - name: Install Rust toolchain + run: | + # Set the default toolchain. Installs the toolchain if it is not already installed. + rustup default 1.96.0 + cargo version + - name: Expose Pythons + run: | + echo "/opt/python/cp37-cp37m/bin" >> $GITHUB_PATH + echo "/opt/python/cp38-cp38/bin" >> $GITHUB_PATH + echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH + echo "/opt/python/cp310-cp310/bin" >> $GITHUB_PATH + echo "/opt/python/cp311-cp311/bin" >> $GITHUB_PATH + echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH + echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314t/bin" >> $GITHUB_PATH + - name: Install Protoc + uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 23.x + - name: Install Python headers + run: echo 'no-op' + - name: Pin the distribution interpreter + run: echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=['$(dirname "$(command -v python3.14)")']" >> "$GITHUB_ENV" + - env: + HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} + PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} + name: Build wheels + run: ./pants run src/python/pants_release/release.py -- build-wheels + - env: + HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} + PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} + name: Build Pants PEX + run: ./pants package src/python/pants:pants-pex@scie_pbs_free_threaded=cp314 + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: logs-wheels-and-pex-Linux-ARM64 + overwrite: 'true' + path: .pants.d/workdir/*.log + timeout-minutes: 90 + build_wheels_cp314_linux_x86_64: + container: + image: quay.io/pypa/manylinux_2_28_x86_64:latest + volumes: + - /:/mnt/host-root + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + MODE: debug + PANTS_REMOTE_CACHE_READ: 'false' + PANTS_REMOTE_CACHE_WRITE: 'false' + PY: python3.14 + if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true' || needs.classify_changes.outputs.ci_config == 'true')) && (needs.classify_changes.outputs.no_code != 'true') + name: Build wheels (Linux-x86_64, cp314) + needs: + - classify_changes + permissions: + attestations: write + contents: write + id-token: write + runs-on: + - depot-ubuntu-22.04-8 + steps: + - name: Free up disk space + run: |- + df -h + rm -rf /mnt/host-root/usr/share/dotnet || true + rm -rf /mnt/host-root/usr/local/lib/android || true + rm -rf /mnt/host-root/opt/ghc || true + rm -rf /mnt/host-root/usr/local/.ghcup || true + df -h + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 10 + - name: Configure Git + run: git config --global safe.directory "$GITHUB_WORKSPACE" + - name: Install rustup + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -v -y --default-toolchain none + echo "${HOME}/.cargo/bin" >> $GITHUB_PATH + - name: Install Rust toolchain + run: | + # Set the default toolchain. Installs the toolchain if it is not already installed. + rustup default 1.96.0 + cargo version + - name: Expose Pythons + run: | + echo "/opt/python/cp37-cp37m/bin" >> $GITHUB_PATH + echo "/opt/python/cp38-cp38/bin" >> $GITHUB_PATH + echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH + echo "/opt/python/cp310-cp310/bin" >> $GITHUB_PATH + echo "/opt/python/cp311-cp311/bin" >> $GITHUB_PATH + echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH + echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314t/bin" >> $GITHUB_PATH + - name: Install Protoc + uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 23.x + - name: Install Go 1.25.3 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + cache: false + go-version: 1.25.3 + - name: Install Go 1.24.9 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + cache: false + go-version: 1.24.9 + - name: Pin the distribution interpreter + run: echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=['$(dirname "$(command -v python3.14)")']" >> "$GITHUB_ENV" + - env: + HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} + PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} + PANTS_PROCESS_EXECUTION_LOCAL_PARALLELISM: '1' + name: Build wheels + run: ./pants run src/python/pants_release/release.py -- build-wheels + - env: + HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} + PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} + PANTS_PROCESS_EXECUTION_LOCAL_PARALLELISM: '1' + name: Build Pants PEX + run: ./pants package src/python/pants:pants-pex@scie_pbs_free_threaded=cp314 + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: logs-wheels-and-pex-Linux-x86_64 + overwrite: 'true' + path: .pants.d/workdir/*.log + timeout-minutes: 90 + build_wheels_cp314_macos14_arm64: + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: false + MODE: debug + PANTS_REMOTE_CACHE_READ: 'false' + PANTS_REMOTE_CACHE_WRITE: 'false' + PY: python3.14 + if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true' || needs.classify_changes.outputs.ci_config == 'true')) && (needs.classify_changes.outputs.no_code != 'true') + name: Build wheels (macOS14-ARM64, cp314) + needs: + - classify_changes + permissions: + attestations: write + contents: write + id-token: write + runs-on: + - depot-macos-14 + steps: + - name: Free up disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + with: + android: true + docker-images: false + dotnet: true + haskell: true + large-packages: false + swap-storage: false + tool-cache: false + - name: Check out code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 10 + - name: Set up Python 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: |- + 3.9 + 3.10 + 3.11 + 3.12 + 3.13 + 3.14 + 3.14t + - name: Install Protoc + uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 23.x + - name: Set rustup profile + run: rustup set profile default + - name: Cache Rust toolchain + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: macOS14-ARM64-rustup-${{ hashFiles('src/rust/rust-toolchain') }}-v2 + path: | + ~/.rustup/toolchains/1.96.0-* + ~/.rustup/update-hashes + ~/.rustup/settings.toml + - name: Cache Cargo + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 + with: + cache-bin: 'false' + shared-key: engine + workspaces: src/rust + - name: Install Protoc + uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 23.x + - name: Install Go 1.25.3 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + cache: false + go-version: 1.25.3 + - name: Install Go 1.24.9 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + cache: false + go-version: 1.24.9 + - name: Pin the distribution interpreter + run: echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=['$(dirname "$(command -v python3.14)")']" >> "$GITHUB_ENV" + - env: + ARCHFLAGS: -arch arm64 + HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} + PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} + _PYTHON_HOST_PLATFORM: macosx-14.0-arm64 + name: Build wheels + run: ./pants run src/python/pants_release/release.py -- build-wheels + - env: + ARCHFLAGS: -arch arm64 + HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} + PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} + _PYTHON_HOST_PLATFORM: macosx-14.0-arm64 + name: Build Pants PEX + run: ./pants package src/python/pants:pants-pex@scie_pbs_free_threaded=cp314 + - continue-on-error: true + if: always() + name: Upload pants.log + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: logs-wheels-and-pex-macOS14-ARM64 + overwrite: 'true' + path: .pants.d/workdir/*.log + timeout-minutes: 90 build_wheels_linux_arm64: container: image: quay.io/pypa/manylinux_2_28_aarch64:latest @@ -354,8 +637,9 @@ jobs: MODE: debug PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' + PY: python3.14t if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true' || needs.classify_changes.outputs.ci_config == 'true')) && (needs.classify_changes.outputs.no_code != 'true') - name: Build wheels (Linux-ARM64) + name: Build wheels (Linux-ARM64, cp314t) needs: - classify_changes permissions: @@ -398,6 +682,7 @@ jobs: echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH echo "/opt/python/cp314-cp314/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314t/bin" >> $GITHUB_PATH - name: Install Protoc uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 with: @@ -405,6 +690,8 @@ jobs: version: 23.x - name: Install Python headers run: echo 'no-op' + - name: Pin the distribution interpreter + run: echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=['$(dirname "$(command -v python3.14t)")']" >> "$GITHUB_ENV" - env: HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} @@ -414,7 +701,7 @@ jobs: HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} name: Build Pants PEX - run: ./pants package src/python/pants:pants-pex + run: ./pants package src/python/pants:pants-pex@scie_pbs_free_threaded=cp314t - continue-on-error: true if: always() name: Upload pants.log @@ -434,8 +721,9 @@ jobs: MODE: debug PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' + PY: python3.14t if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true' || needs.classify_changes.outputs.ci_config == 'true')) && (needs.classify_changes.outputs.no_code != 'true') - name: Build wheels (Linux-x86_64) + name: Build wheels (Linux-x86_64, cp314t) needs: - classify_changes permissions: @@ -478,6 +766,7 @@ jobs: echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH echo "/opt/python/cp314-cp314/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314t/bin" >> $GITHUB_PATH - name: Install Protoc uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 with: @@ -493,6 +782,8 @@ jobs: with: cache: false go-version: 1.24.9 + - name: Pin the distribution interpreter + run: echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=['$(dirname "$(command -v python3.14t)")']" >> "$GITHUB_ENV" - env: HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} @@ -504,7 +795,7 @@ jobs: PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} PANTS_PROCESS_EXECUTION_LOCAL_PARALLELISM: '1' name: Build Pants PEX - run: ./pants package src/python/pants:pants-pex + run: ./pants package src/python/pants:pants-pex@scie_pbs_free_threaded=cp314t - continue-on-error: true if: always() name: Upload pants.log @@ -520,8 +811,9 @@ jobs: MODE: debug PANTS_REMOTE_CACHE_READ: 'false' PANTS_REMOTE_CACHE_WRITE: 'false' + PY: python3.14t if: ((github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.release == 'true' || needs.classify_changes.outputs.ci_config == 'true')) && (needs.classify_changes.outputs.no_code != 'true') - name: Build wheels (macOS14-ARM64) + name: Build wheels (macOS14-ARM64, cp314t) needs: - classify_changes permissions: @@ -545,7 +837,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 10 - - name: Set up Python 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -555,6 +847,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Install Protoc uses: benjyw/setup-protoc@fa4f90fddf8838036cbdeadf1262b9755f6cfb85 with: @@ -591,6 +884,8 @@ jobs: with: cache: false go-version: 1.24.9 + - name: Pin the distribution interpreter + run: echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=['$(dirname "$(command -v python3.14t)")']" >> "$GITHUB_ENV" - env: ARCHFLAGS: -arch arm64 HONEYCOMB_API_KEY: ${{ secrets.HONEYCOMB_API_KEY || '--DISABLED--' }} @@ -604,7 +899,7 @@ jobs: PANTS_OPENTELEMETRY_ENABLED: ${{ vars.OPENTELEMETRY_ENABLED || 'False' }} _PYTHON_HOST_PLATFORM: macosx-14.0-arm64 name: Build Pants PEX - run: ./pants package src/python/pants:pants-pex + run: ./pants package src/python/pants:pants-pex@scie_pbs_free_threaded=cp314t - continue-on-error: true if: always() name: Upload pants.log @@ -777,7 +1072,7 @@ jobs: echo "PANTS_REMOTE_STORE_ADDRESS=grpc://localhost:9092" >> "$GITHUB_ENV" echo "PANTS_REMOTE_CACHE_READ=true" >> "$GITHUB_ENV" echo "PANTS_REMOTE_CACHE_WRITE=${CACHE_WRITE}" >> "$GITHUB_ENV" - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -789,6 +1084,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -837,6 +1133,9 @@ jobs: - bootstrap_pants_linux_arm64 - bootstrap_pants_linux_x86_64 - bootstrap_pants_macos14_arm64 + - build_wheels_cp314_linux_arm64 + - build_wheels_cp314_linux_x86_64 + - build_wheels_cp314_macos14_arm64 - build_wheels_linux_arm64 - build_wheels_linux_x86_64 - build_wheels_macos14_arm64 @@ -903,7 +1202,7 @@ jobs: with: cache: false go-version: 1.24.9 - - name: Set up Python 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -914,6 +1213,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -1031,7 +1331,7 @@ jobs: curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" chmod +x "${HOME}/.thrift/thrift" echo "${HOME}/.thrift" >> $GITHUB_PATH - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -1043,6 +1343,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -1160,7 +1461,7 @@ jobs: curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" chmod +x "${HOME}/.thrift/thrift" echo "${HOME}/.thrift" >> $GITHUB_PATH - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -1172,6 +1473,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -1289,7 +1591,7 @@ jobs: curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" chmod +x "${HOME}/.thrift/thrift" echo "${HOME}/.thrift" >> $GITHUB_PATH - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -1301,6 +1603,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -1418,7 +1721,7 @@ jobs: curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" chmod +x "${HOME}/.thrift/thrift" echo "${HOME}/.thrift" >> $GITHUB_PATH - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -1430,6 +1733,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -1547,7 +1851,7 @@ jobs: curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" chmod +x "${HOME}/.thrift/thrift" echo "${HOME}/.thrift" >> $GITHUB_PATH - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -1559,6 +1863,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -1676,7 +1981,7 @@ jobs: curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" chmod +x "${HOME}/.thrift/thrift" echo "${HOME}/.thrift" >> $GITHUB_PATH - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -1688,6 +1993,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -1805,7 +2111,7 @@ jobs: curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" chmod +x "${HOME}/.thrift/thrift" echo "${HOME}/.thrift" >> $GITHUB_PATH - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -1817,6 +2123,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -1934,7 +2241,7 @@ jobs: curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" chmod +x "${HOME}/.thrift/thrift" echo "${HOME}/.thrift" >> $GITHUB_PATH - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -1946,6 +2253,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -2063,7 +2371,7 @@ jobs: curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" chmod +x "${HOME}/.thrift/thrift" echo "${HOME}/.thrift" >> $GITHUB_PATH - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -2075,6 +2383,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -2192,7 +2501,7 @@ jobs: curl --fail -L https://binaries.pantsbuild.org/bin/thrift/linux/x86_64/0.15.0/thrift -o "${HOME}/.thrift/thrift" chmod +x "${HOME}/.thrift/thrift" echo "${HOME}/.thrift" >> $GITHUB_PATH - - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -2204,6 +2513,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: @@ -2290,7 +2600,7 @@ jobs: with: cache: false go-version: 1.24.9 - - name: Set up Python 3.9, 3.10, 3.11, 3.12, 3.13, 3.14 + - name: Set up Python 3.9, 3.10, 3.11, 3.12, 3.13, 3.14, 3.14t uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: |- @@ -2300,6 +2610,7 @@ jobs: 3.12 3.13 3.14 + 3.14t - name: Download native binaries uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: diff --git a/build-support/bin/rust/calculate_engine_hash.sh b/build-support/bin/rust/calculate_engine_hash.sh index a51fd36e5c4..36ab4a56624 100644 --- a/build-support/bin/rust/calculate_engine_hash.sh +++ b/build-support/bin/rust/calculate_engine_hash.sh @@ -33,7 +33,7 @@ function calculate_current_hash() { echo "${MODE_FLAG}" uname -mps # the engine only depends on the implementation and major.minor version, not the patch - "${PY}" -c 'import sys; print(sys.implementation.name, sys.version_info.major, sys.version_info.minor)' + "${PY}" -c 'import sys; print(sys.implementation.name, sys.version_info.major, sys.version_info.minor, sys.abiflags)' git ls-files --cached --others --exclude-standard \ "${NATIVE_ROOT}" \ "${REPO_ROOT}/build-support/bin/rust" | diff --git a/build-support/common.sh b/build-support/common.sh index c73e095a8f9..8f2f7be7dee 100644 --- a/build-support/common.sh +++ b/build-support/common.sh @@ -52,7 +52,16 @@ function determine_python() { which "${PY}" && return 0 fi - version='3.14' + repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. && pwd -P)" + if [[ -f "${repo_root}/.python-version" ]] && command -v uv &> /dev/null; then + interpreter_path="$(uv python find --project "${repo_root}" --no-python-downloads 2> /dev/null || true)" + if [[ -n "${interpreter_path}" ]]; then + echo "${interpreter_path}" + return 0 + fi + fi + + version='3.14t' interpreter_path="$(command -v "python${version}")" if [[ -z "${interpreter_path}" ]]; then echo "pants: Failed to find a Python ${version} interpreter" 1>&2 && return 1 diff --git a/build-support/pants_venv b/build-support/pants_venv index 2eba50b162b..cd2b0e50aab 100755 --- a/build-support/pants_venv +++ b/build-support/pants_venv @@ -14,13 +14,15 @@ REQUIREMENTS=( platform=$(uname -mps) function venv_dir() { - # Include the entire version string in order to differentiate e.g. PyPy from CPython. + # Include the implementation and version to differentiate e.g. PyPy from CPython, and the + # abiflags to differentiate free-threaded from GIL builds (`--version` prints "Python 3.14.3" + # for both, but their extension ABIs are incompatible). # Fingerprinting uname and python output avoids shebang length limits and any odd chars. # Include REPO_ROOT so that different checkouts or worktrees don't clobber each other. venv_fingerprint=$( ( echo "${platform}" echo "${REPO_ROOT}" - ${PY} --version + ${PY} -c 'import sys; print(sys.implementation.name, ".".join(map(str, sys.version_info[:3])), sys.abiflags)' ) | fingerprint_data) # NB: We house these outside the working copy to avoid needing to gitignore them, but also to diff --git a/docs/notes/2.34.x.md b/docs/notes/2.34.x.md new file mode 100644 index 00000000000..2ed1becb17b --- /dev/null +++ b/docs/notes/2.34.x.md @@ -0,0 +1,25 @@ +# 2.34.x Release Series + +Pants is a fast, scalable, user-friendly build system for codebases of all sizes. + +Pants is an open-source project that is not owned or controlled by any one company or organization, and does incur some expenses. These expenses are managed by Pants Build, a non-profit that was established for this purpose. This non-profit's only source of revenue is [sponsorship](https://www.pantsbuild.org/sponsorship) by individuals and companies that use Pants. + +We offer [formal sponsorship tiers for companies](https://www.pantsbuild.org/sponsorship), as well as individual sponsorships via [GitHub](https://github.com/sponsors/pantsbuild). + +## What's New + +### Highlights + +- Pants now runs on free-threaded CPython 3.14t by default. The Pants engine and Pants' own Python code are thread-safe without the GIL. + Each release also ships a regular (GIL) CPython 3.14 distribution as an escape hatch: set `[GLOBAL].pants_free_threaded = false` in `pants.toml` (or the `PANTS_FREE_THREADED` env var) to select it. + Release artifacts are named with the full ABI tag (e.g. `cp314t`). Launching via `scie-pants` requires a launcher release with free-threaded interpreter support; older launchers will fail to locate the release asset and must be updated. + +### Deprecations + +The `cache_factory` argument to `@memoized` is deprecated and will be removed in 2.35.0.dev0. Memoization caches are now engine-managed for thread safety under free-threaded Python, so custom cache factories (e.g. LRU caches) are no longer honored; the argument is silently ignored and can be removed. + +### General + +## Full Changelog + +For the full changelog, see the individual GitHub pull requests in this milestone: https://github.com/pantsbuild/pants/milestone/2.34.x diff --git a/pants b/pants index 547d8b2e127..4284ac4eff3 100755 --- a/pants +++ b/pants @@ -4,8 +4,8 @@ # This bootstrap script runs pants from the live sources in this repo. # -# The script defaults to running with Python 3.11. To use another Python version, -# prefix the script with `PY=python3.14`. +# The script uses `PY` if set, otherwise a uv-managed `.python-version` pin when present, +# otherwise `python3.14` from PATH. set -eo pipefail diff --git a/src/python/pants/BUILD b/src/python/pants/BUILD index aa7639d72af..fcfadd30c7d 100644 --- a/src/python/pants/BUILD +++ b/src/python/pants/BUILD @@ -67,6 +67,9 @@ pex_binary( scie_bind_resource_path=["PANTS_NATIVE_CLIENT=pants/bin/native_client"], scie_exe="{scie.env.PANTS_NATIVE_CLIENT}", scie_env=["NO_SCIE_WARNING=true", "_PANTS_SERVER_EXE={scie.env.VIRTUAL_ENV}/bin/pants"], + # The scie interpreter must match the wheels: each release lane builds one + # parametrization with the corresponding CPython (PY=python3.14t or python3.14). + scie_pbs_free_threaded=parametrize(cp314t=True, cp314=False), ) # NB: we use `dummy.c` to avoid clang/gcc complaining `error: no input files` when building diff --git a/src/python/pants/backend/nfpm/native_libs/elfdeps/rules_integration_test.py b/src/python/pants/backend/nfpm/native_libs/elfdeps/rules_integration_test.py index 35d6eea5322..2b0418c4f67 100644 --- a/src/python/pants/backend/nfpm/native_libs/elfdeps/rules_integration_test.py +++ b/src/python/pants/backend/nfpm/native_libs/elfdeps/rules_integration_test.py @@ -28,7 +28,7 @@ # The tests build a pex with wheels for the current platform. # These vars are used to choose the expected platform-specific test results. _PY_VERSION = ".".join(map(str, sys.version_info[:3])) -_PY_TAG = "".join(map(str, sys.version_info[:2])) +_PY_TAG = "".join(map(str, sys.version_info[:2])) + sys.abiflags _PY_OS = platform.system() # Linux _PY_ARCH_TAG = platform.machine() # x86_64 _ELF_BITS_MARKER = "(64bit)" if platform.architecture() == ("64bit", "ELF") else "" diff --git a/src/python/pants/backend/nfpm/native_libs/rules_integration_test.py b/src/python/pants/backend/nfpm/native_libs/rules_integration_test.py index 06340644796..068a0a62a70 100644 --- a/src/python/pants/backend/nfpm/native_libs/rules_integration_test.py +++ b/src/python/pants/backend/nfpm/native_libs/rules_integration_test.py @@ -37,7 +37,7 @@ from pants.testutil.rule_runner import RuleRunner from pants.util.frozendict import FrozenDict -_PY_TAG = "".join(map(str, sys.version_info[:2])) +_PY_TAG = "".join(map(str, sys.version_info[:2])) + sys.abiflags _PY_OS = platform.system() # Linux _PY_ARCH_TAG = platform.machine() # x86_64 diff --git a/src/python/pants/base/build_root.py b/src/python/pants/base/build_root.py index c6b3098cd56..6813637e266 100644 --- a/src/python/pants/base/build_root.py +++ b/src/python/pants/base/build_root.py @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os +import threading from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path @@ -37,6 +38,7 @@ def find_buildroot(self) -> str: def __init__(self) -> None: self._root_dir: str | None = None + self._lock = threading.RLock() @property def pathlib_path(self) -> Path: @@ -45,15 +47,16 @@ def pathlib_path(self) -> Path: @property def path(self) -> str: """Returns the build root for the current workspace.""" - if self._root_dir is None: - # Do not remove/change this env var without coordinating with `pantsbuild/scie-pants` as - # it is being used when bootstrapping Pants. - override_buildroot = os.environ.get("PANTS_BUILDROOT_OVERRIDE", None) - if override_buildroot: - self._root_dir = override_buildroot - else: - self._root_dir = os.path.realpath(self.find_buildroot()) - return self._root_dir + with self._lock: + if self._root_dir is None: + # Do not remove/change this env var without coordinating with + # `pantsbuild/scie-pants` as it is being used when bootstrapping Pants. + override_buildroot = os.environ.get("PANTS_BUILDROOT_OVERRIDE", None) + if override_buildroot: + self._root_dir = override_buildroot + else: + self._root_dir = os.path.realpath(self.find_buildroot()) + return self._root_dir @path.setter def path(self, root_dir: str) -> None: @@ -61,11 +64,13 @@ def path(self, root_dir: str) -> None: path = os.path.realpath(root_dir) if not os.path.exists(path): raise ValueError(f"Build root does not exist: {root_dir}") - self._root_dir = path + with self._lock: + self._root_dir = path def reset(self) -> None: """Clears the last calculated build root for the current workspace.""" - self._root_dir = None + with self._lock: + self._root_dir = None def __str__(self) -> str: return f"BuildRoot({self._root_dir})" @@ -75,9 +80,10 @@ def temporary(self, path: str) -> Iterator[None]: """Establishes a temporary build root, restoring the prior build root on exit.""" if path is None: raise ValueError("Can only temporarily establish a build root given a path.") - prior = self._root_dir - self._root_dir = path - try: - yield - finally: - self._root_dir = prior + with self._lock: + prior = self._root_dir + self._root_dir = path + try: + yield + finally: + self._root_dir = prior diff --git a/src/python/pants/base/build_root_test.py b/src/python/pants/base/build_root_test.py index add8c606700..13baf9912d3 100644 --- a/src/python/pants/base/build_root_test.py +++ b/src/python/pants/base/build_root_test.py @@ -2,7 +2,9 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os +import time from collections.abc import Generator +from concurrent.futures import ThreadPoolExecutor import pytest @@ -59,6 +61,27 @@ def test_temporary(tmp_build_root) -> None: assert original_path == build_root.path +def test_concurrent_temporary_and_reset(tmp_build_root) -> None: + build_root, original_path, _ = tmp_build_root + paths = [os.path.realpath(safe_mkdtemp()) for _ in range(8)] + + def use_path(path: str) -> None: + with build_root.temporary(path): + time.sleep(0.01) + assert build_root.path == path + build_root.path = path + build_root.reset() + + try: + with ThreadPoolExecutor(max_workers=len(paths)) as executor: + list(executor.map(use_path, paths)) + build_root.reset() + assert build_root.path == original_path + finally: + for path in paths: + safe_rmtree(path) + + def test_singleton(tmp_build_root) -> None: _, _, new_path = tmp_build_root assert BuildRoot().path == BuildRoot().path diff --git a/src/python/pants/base/exception_sink.py b/src/python/pants/base/exception_sink.py index 95d2c726e92..b622ab8e1b7 100644 --- a/src/python/pants/base/exception_sink.py +++ b/src/python/pants/base/exception_sink.py @@ -133,6 +133,7 @@ class ExceptionSink: # (and lets faulthandler figure out signal safety). _pid_specific_error_fileobj = None _shared_error_fileobj = None + _lock = threading.RLock() def __new__(cls, *args, **kwargs): raise TypeError( @@ -145,15 +146,15 @@ class ExceptionSinkError(Exception): @classmethod def install(cls, log_location: str, pantsd_instance: bool) -> None: """Setup global state for this process, such as signal handlers and sys.excepthook.""" + with cls._lock: + # Set the log location for writing logs before bootstrap options are parsed. + cls.reset_log_location(log_location) - # Set the log location for writing logs before bootstrap options are parsed. - cls.reset_log_location(log_location) + # NB: Mutate process-global state! + sys.excepthook = ExceptionSink.log_exception - # NB: Mutate process-global state! - sys.excepthook = ExceptionSink.log_exception - - # Setup a default signal handler. - cls.reset_signal_handler(SignalHandler(pantsd_instance=pantsd_instance)) + # Setup a default signal handler. + cls.reset_signal_handler(SignalHandler(pantsd_instance=pantsd_instance)) # All reset_* methods are ~idempotent! @classmethod @@ -170,42 +171,47 @@ def reset_log_location(cls, new_log_location: str) -> None: :raises: :class:`ExceptionSink.ExceptionSinkError` if the directory does not exist or is not writable. """ - # We could no-op here if the log locations are the same, but there's no reason not to have the - # additional safety of re-acquiring file descriptors each time (and erroring out early if the - # location is no longer writable). - try: - safe_mkdir(new_log_location) - except Exception as e: - raise cls.ExceptionSinkError( - f"The provided log location path at '{new_log_location}' is not writable or could not be created: {str(e)}.", - e, - ) + with cls._lock: + # We could no-op here if the log locations are the same, but there's no reason not to + # have the additional safety of re-acquiring file descriptors each time (and erroring out + # early if the location is no longer writable). + try: + safe_mkdir(new_log_location) + except Exception as e: + raise cls.ExceptionSinkError( + f"The provided log location path at '{new_log_location}' is not writable or " + f"could not be created: {str(e)}.", + e, + ) - pid = os.getpid() - pid_specific_log_path = cls.exceptions_log_path(for_pid=pid, in_dir=new_log_location) - shared_log_path = cls.exceptions_log_path(in_dir=new_log_location) - assert pid_specific_log_path != shared_log_path - try: - pid_specific_error_stream = cls.open_pid_specific_error_stream(pid_specific_log_path) - shared_error_stream = safe_open(shared_log_path, mode="a") - except Exception as e: - raise cls.ExceptionSinkError( - f"Error opening fatal error log streams for log location '{new_log_location}': {str(e)}" - ) + pid = os.getpid() + pid_specific_log_path = cls.exceptions_log_path(for_pid=pid, in_dir=new_log_location) + shared_log_path = cls.exceptions_log_path(in_dir=new_log_location) + assert pid_specific_log_path != shared_log_path + try: + pid_specific_error_stream = cls.open_pid_specific_error_stream( + pid_specific_log_path + ) + shared_error_stream = safe_open(shared_log_path, mode="a") + except Exception as e: + raise cls.ExceptionSinkError( + f"Error opening fatal error log streams for log location " + f"'{new_log_location}': {str(e)}" + ) - # NB: mutate process-global state! - if faulthandler.is_enabled(): - logger.debug("re-enabling faulthandler") - # Call Py_CLEAR() on the previous error stream: - # https://github.com/vstinner/faulthandler/blob/master/faulthandler.c - faulthandler.disable() - # Send a stacktrace to this file if interrupted by a fatal error. - faulthandler.enable(file=pid_specific_error_stream, all_threads=True) + # NB: mutate process-global state! + if faulthandler.is_enabled(): + logger.debug("re-enabling faulthandler") + # Call Py_CLEAR() on the previous error stream: + # https://github.com/vstinner/faulthandler/blob/master/faulthandler.c + faulthandler.disable() + # Send a stacktrace to this file if interrupted by a fatal error. + faulthandler.enable(file=pid_specific_error_stream, all_threads=True) - # NB: mutate the class variables! - cls._log_dir = new_log_location - cls._pid_specific_error_fileobj = pid_specific_error_stream - cls._shared_error_fileobj = shared_error_stream + # NB: mutate the class variables! + cls._log_dir = new_log_location + cls._pid_specific_error_fileobj = pid_specific_error_stream + cls._shared_error_fileobj = shared_error_stream @classmethod def open_pid_specific_error_stream(cls, path): @@ -244,23 +250,26 @@ def _log_exception(cls, msg): pid = os.getpid() fatal_error_log_entry = cls._format_exception_message(msg, pid) - # We care more about this log than the shared log, so write to it first. - try: - cls._try_write_with_flush(cls._pid_specific_error_fileobj, fatal_error_log_entry) - except Exception as e: - logger.error( - f"Error logging the message '{msg}' to the pid-specific file handle for {cls._log_dir} at pid {pid}:\n{e}" - ) + with cls._lock: + # We care more about this log than the shared log, so write to it first. + try: + cls._try_write_with_flush(cls._pid_specific_error_fileobj, fatal_error_log_entry) + except Exception as e: + logger.error( + f"Error logging the message '{msg}' to the pid-specific file handle for " + f"{cls._log_dir} at pid {pid}:\n{e}" + ) - # Write to the shared log. - try: - # TODO: we should probably guard this against concurrent modification by other pants - # subprocesses somehow. - cls._try_write_with_flush(cls._shared_error_fileobj, fatal_error_log_entry) - except Exception as e: - logger.error( - f"Error logging the message '{msg}' to the shared file handle for {cls._log_dir} at pid {pid}:\n{e}" - ) + # Write to the shared log. + try: + # TODO: we should probably guard this against concurrent modification by other pants + # subprocesses somehow. + cls._try_write_with_flush(cls._shared_error_fileobj, fatal_error_log_entry) + except Exception as e: + logger.error( + f"Error logging the message '{msg}' to the shared file handle for " + f"{cls._log_dir} at pid {pid}:\n{e}" + ) @classmethod def _try_write_with_flush(cls, fileobj, payload): @@ -280,17 +289,18 @@ def reset_signal_handler(cls, signal_handler: SignalHandler) -> SignalHandler: the previously-registered signal handler. """ - for signum, handler in signal_handler.signal_handler_mapping.items(): - signal.signal(signum, handler) - # Retry any system calls interrupted by any of the signals we just installed handlers for - # (instead of having them raise EINTR). siginterrupt(3) says this is the default behavior on - # Linux and OSX. - signal.siginterrupt(signum, False) + with cls._lock: + for signum, handler in signal_handler.signal_handler_mapping.items(): + signal.signal(signum, handler) + # Retry any system calls interrupted by any of the signals we just installed handlers + # for (instead of having them raise EINTR). siginterrupt(3) says this is the default + # behavior on Linux and OSX. + signal.siginterrupt(signum, False) - previous_signal_handler = cls._signal_handler - cls._signal_handler = signal_handler + previous_signal_handler = cls._signal_handler + cls._signal_handler = signal_handler - return previous_signal_handler + return previous_signal_handler @classmethod @contextmanager @@ -299,11 +309,12 @@ def trapped_signals(cls, new_signal_handler: SignalHandler) -> Iterator[None]: NB: This method calls signal.signal(), which will crash if not called from the main thread! """ - previous_signal_handler = cls.reset_signal_handler(new_signal_handler) - try: - yield - finally: - cls.reset_signal_handler(previous_signal_handler) + with cls._lock: + previous_signal_handler = cls.reset_signal_handler(new_signal_handler) + try: + yield + finally: + cls.reset_signal_handler(previous_signal_handler) @classmethod @contextmanager @@ -317,11 +328,12 @@ def ignoring_sigint(cls) -> Iterator[None]: to the server, so we don't want the server process to ignore it. """ - try: - cls._signal_handler._toggle_ignoring_sigint(True) - yield - finally: - cls._signal_handler._toggle_ignoring_sigint(False) + with cls._lock: + try: + cls._signal_handler._toggle_ignoring_sigint(True) + yield + finally: + cls._signal_handler._toggle_ignoring_sigint(False) @classmethod def _iso_timestamp_for_now(cls): diff --git a/src/python/pants/base/exception_sink_integration_test.py b/src/python/pants/base/exception_sink_integration_test.py index 484ba8e1a62..3de7195a557 100644 --- a/src/python/pants/base/exception_sink_integration_test.py +++ b/src/python/pants/base/exception_sink_integration_test.py @@ -128,13 +128,18 @@ def test_dumps_traceback_on_sigabrt(self): # Check that the logs show an abort signal and the beginning of a traceback. pid_specific_log_file, shared_log_file = get_log_file_paths(ctx.workdir, pid) - regex_str = """\ -Fatal Python error: Aborted - -Thread [^\n]+ \\(most recent call first\\): -""" - - assert re.search(regex_str, read_file(pid_specific_log_file)) + # On free-threaded CPython, faulthandler can't enumerate all threads, so it emits a + # different header than the GIL-enabled per-thread traceback format. + regex_str = ( + "Fatal Python error: Aborted\n\n" + "(?:Thread [^\n]+ \\(most recent call first\\):" + "|\n" + "Stack \\(most recent call first\\):)\n" + ) + + pid_specific_log_contents = read_file(pid_specific_log_file) + assert re.search(regex_str, pid_specific_log_contents) + assert ' File "' in pid_specific_log_contents # faulthandler.enable() only allows use of a single logging file at once for fatal tracebacks. assert "" == read_file(shared_log_file) diff --git a/src/python/pants/core/subsystems/python_bootstrap.py b/src/python/pants/core/subsystems/python_bootstrap.py index b48364809c3..0c29c7e39d7 100644 --- a/src/python/pants/core/subsystems/python_bootstrap.py +++ b/src/python/pants/core/subsystems/python_bootstrap.py @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) -_PBS_URL_TEMPLATE = "https://github.com/astral-sh/python-build-standalone/releases/download/20251014/cpython-3.14.0+20251014-{}-install_only_stripped.tar.gz" +_PBS_URL_TEMPLATE = "https://github.com/astral-sh/python-build-standalone/releases/download/20260610/cpython-3.14.6+20260610-{}-freethreaded-install_only_stripped.tar.gz" class PythonBootstrapSubsystem(Subsystem): @@ -49,23 +49,23 @@ class PythonBootstrapSubsystem(Subsystem): default={ "linux_arm64": ( _PBS_URL_TEMPLATE.format("aarch64-unknown-linux-gnu"), - "7dbb43b742c040835a277318355fb359b41e509dbf4fbb614da38005a9290e16", - 29297561, + "3d82c130c647988c0cb83fbf6b69637024dfa184da0b4d25c1498a3e9655b2f1", + 30304472, ), "linux_x86_64": ( _PBS_URL_TEMPLATE.format("x86_64-unknown-linux-gnu"), - "493c477b4a88bb1ea2f6c6f57fa0e88ffbe55d9e7b1405c4699f2d41c04eb154", - 34580370, + "d4872b127c641aadb0846a0a4f2c3f6377393a34e89de7668274ec2434d50c00", + 36178201, ), "macos_arm64": ( _PBS_URL_TEMPLATE.format("aarch64-apple-darwin"), - "057476264b07222a2baeff68a733647f91a9d61c94f79beba46a44eb42101749", - 16923076, + "f47f0d5b02fc634385bdc630dd293678cfbdbc009de2b04403ce04c92473b483", + 26071097, ), "macos_x86_64": ( _PBS_URL_TEMPLATE.format("x86_64-apple-darwin"), - "56dcb0cdafabac9d6d976690fb05d9ee92d20ce798c3aabe9049259ebe7d3e0d", - 16905036, + "f8daa286f7315afc2fc1d51e3f6169ef133bd030f344d182f5f24849439aa4be", + 26138236, ), }, help=softwrap( diff --git a/src/python/pants/engine/internals/native_engine.pyi b/src/python/pants/engine/internals/native_engine.pyi index 604fa7ccd5e..92a49eed2ea 100644 --- a/src/python/pants/engine/internals/native_engine.pyi +++ b/src/python/pants/engine/internals/native_engine.pyi @@ -121,6 +121,12 @@ class FrozenOrderedSet(AbstractSet[T_co], Hashable): def issuperset(self, other: Iterable[T_co]) -> bool: ... def isdisjoint(self, other: Iterable[T_co]) -> bool: ... +class LockMap: + def get_or_insert(self, key: Hashable, compute: Callable[[], Any]) -> Any: ... + def put(self, key: Hashable, value: Any) -> None: ... + def forget(self, key: Hashable) -> None: ... + def clear(self) -> None: ... + # ------------------------------------------------------------------------------ # Address # ------------------------------------------------------------------------------ diff --git a/src/python/pants/engine/internals/rule_visitor.py b/src/python/pants/engine/internals/rule_visitor.py index 346061afe12..91aa2d4a286 100644 --- a/src/python/pants/engine/internals/rule_visitor.py +++ b/src/python/pants/engine/internals/rule_visitor.py @@ -7,6 +7,7 @@ import itertools import logging import sys +import threading from collections.abc import Callable, Iterator, Sequence from contextlib import contextmanager from dataclasses import dataclass @@ -52,6 +53,7 @@ def _node_str(node: Any) -> str: PANTS_RULE_DESCRIPTORS_MODULE_KEY = "__pants_rule_descriptors__" +_rule_descriptors_lock = threading.RLock() @dataclass(frozen=True) @@ -83,16 +85,17 @@ def get_module_scope_rules(module: ModuleType) -> tuple[RuleDescriptor, ...]: Note that we don't support recursive rules defined dynamically in inner scopes. """ - descriptors = getattr(module, PANTS_RULE_DESCRIPTORS_MODULE_KEY, None) - if descriptors is None: - descriptors = [] - for node in ast.iter_child_nodes(ast.parse(inspect.getsource(module))): - if isinstance(node, ast.AsyncFunctionDef) and isinstance(node.returns, ast.Name): - descriptors.append(RuleDescriptor(module.__name__, node.name, node.returns.id)) - descriptors = tuple(descriptors) - setattr(module, PANTS_RULE_DESCRIPTORS_MODULE_KEY, descriptors) - - return descriptors + with _rule_descriptors_lock: + descriptors = getattr(module, PANTS_RULE_DESCRIPTORS_MODULE_KEY, None) + if descriptors is None: + descriptors = [] + for node in ast.iter_child_nodes(ast.parse(inspect.getsource(module))): + if isinstance(node, ast.AsyncFunctionDef) and isinstance(node.returns, ast.Name): + descriptors.append(RuleDescriptor(module.__name__, node.name, node.returns.id)) + descriptors = tuple(descriptors) + setattr(module, PANTS_RULE_DESCRIPTORS_MODULE_KEY, descriptors) + + return descriptors class _TypeStack: diff --git a/src/python/pants/engine/internals/rule_visitor_test.py b/src/python/pants/engine/internals/rule_visitor_test.py index 15ad9eb43a3..38c6baf6b68 100644 --- a/src/python/pants/engine/internals/rule_visitor_test.py +++ b/src/python/pants/engine/internals/rule_visitor_test.py @@ -4,13 +4,20 @@ from __future__ import annotations import importlib.util +import inspect import sys import textwrap +import time from collections.abc import Iterable +from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from pathlib import Path -from pants.engine.internals.rule_visitor import collect_awaitables +from pants.engine.internals.rule_visitor import ( + PANTS_RULE_DESCRIPTORS_MODULE_KEY, + collect_awaitables, + get_module_scope_rules, +) from pants.engine.rules import implicitly, rule # The visitor inspects the module for definitions. @@ -185,6 +192,34 @@ async def mutually_recursive_rule_2(arg: int) -> str: assert_byname_awaitables(module.mutually_recursive_rule_2, [(int, [], 1)]) +def test_get_module_scope_rules_concurrent_init(tmp_path: Path, monkeypatch) -> None: + rule_code = textwrap.dedent(""" + async def first_rule(arg: int) -> str: + return str(arg) + + async def second_rule(arg: str) -> int: + return int(arg) + """) + + with temporary_module(tmp_path, rule_code) as module: + source_calls = 0 + original_getsource = inspect.getsource + + def slow_getsource(obj) -> str: + nonlocal source_calls + source_calls += 1 + time.sleep(0.01) + return original_getsource(obj) + + monkeypatch.setattr(inspect, "getsource", slow_getsource) + with ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(lambda _: get_module_scope_rules(module), range(8))) + + assert all(result is results[0] for result in results) + assert getattr(module, PANTS_RULE_DESCRIPTORS_MODULE_KEY) is results[0] + assert source_calls == 1 + + def test_rule_helpers_free_functions() -> None: async def rule(): _top_helper(1) diff --git a/src/python/pants/goal/run_tracker.py b/src/python/pants/goal/run_tracker.py index b6dc4202788..2d1709c47d5 100644 --- a/src/python/pants/goal/run_tracker.py +++ b/src/python/pants/goal/run_tracker.py @@ -6,6 +6,7 @@ import logging import platform import socket +import threading import time import uuid from hashlib import sha256 @@ -57,6 +58,7 @@ def __init__(self, args: tuple[str, ...], options: Options): """ :API: public """ + self._lock = threading.RLock() self._has_started: bool = False self._has_ended: bool = False @@ -77,7 +79,8 @@ def __init__(self, args: tuple[str, ...], options: Options): self.run_logs_file = info_dir / self.run_id / "logs" self.run_logs_file.parent.mkdir(exist_ok=True, parents=True) - native_engine.set_per_run_log_path(str(self.run_logs_file)) + with self._lock: + native_engine.set_per_run_log_path(str(self.run_logs_file)) # Initialized in `start()`. self._run_start_time: float | None = None @@ -97,30 +100,31 @@ def active_standard_backends(self) -> list[str]: def start(self, run_start_time: float, specs: list[str]) -> None: """Start tracking this pants run.""" - if self._has_started: - raise AssertionError("RunTracker.start must not be called multiple times.") - self._has_started = True - - # Initialize the run. - self._run_start_time = run_start_time - - datetime = time.strftime("%A %b %d, %Y %H:%M:%S", time.localtime(run_start_time)) - cmd_line = " ".join(("pants",) + self._args[1:]) - - self._run_info.update( - { - "id": self.run_id, - "timestamp": run_start_time, - "datetime": datetime, - "user": getuser(), - "machine": socket.gethostname(), - "buildroot": get_buildroot(), - "path": get_buildroot(), - "version": VERSION, - "cmd_line": cmd_line, - "specs_from_command_line": specs, - } - ) + with self._lock: + if self._has_started: + raise AssertionError("RunTracker.start must not be called multiple times.") + self._has_started = True + + # Initialize the run. + self._run_start_time = run_start_time + + datetime = time.strftime("%A %b %d, %Y %H:%M:%S", time.localtime(run_start_time)) + cmd_line = " ".join(("pants",) + self._args[1:]) + + self._run_info.update( + { + "id": self.run_id, + "timestamp": run_start_time, + "datetime": datetime, + "user": getuser(), + "machine": socket.gethostname(), + "buildroot": get_buildroot(), + "path": get_buildroot(), + "version": VERSION, + "cmd_line": cmd_line, + "specs_from_command_line": specs, + } + ) def get_anonymous_telemetry_data(self, unhashed_repo_id: str) -> dict[str, str | list[str]]: def maybe_hash_with_repo_id_prefix(s: str) -> str: @@ -128,17 +132,21 @@ def maybe_hash_with_repo_id_prefix(s: str) -> str: # If the repo_id is the empty string we return a blank string. return sha256(qualified_str.encode()).hexdigest() if unhashed_repo_id else "" + with self._lock: + run_info = dict(self._run_info) + run_total_duration = self._run_total_duration + return { - "run_id": str(self._run_info.get("id", uuid.uuid4())), - "timestamp": str(self._run_info.get("timestamp")), + "run_id": str(run_info.get("id", uuid.uuid4())), + "timestamp": str(run_info.get("timestamp")), # Note that this method is called after the StreamingWorkunitHandler.session() ends, # i.e., after end_run() has been called, so duration will be set. - "duration": str(self._run_total_duration), - "outcome": str(self._run_info.get("outcome")), + "duration": str(run_total_duration), + "outcome": str(run_info.get("outcome")), "platform": platform.platform(), "python_implementation": platform.python_implementation(), "python_version": platform.python_version(), - "pants_version": str(self._run_info.get("version")), + "pants_version": str(run_info.get("version")), # Note that if repo_id is the empty string then these three fields will be empty. "repo_id": maybe_hash_with_repo_id_prefix(""), "machine_id": maybe_hash_with_repo_id_prefix(str(uuid.getnode())), @@ -151,18 +159,22 @@ def maybe_hash_with_repo_id_prefix(s: str) -> str: } def set_pantsd_scheduler_metrics(self, metrics: dict[str, int]) -> None: - self._pantsd_metrics = metrics + with self._lock: + self._pantsd_metrics = metrics @property def pantsd_scheduler_metrics(self) -> dict[str, int]: - return dict(self._pantsd_metrics) # defensive copy + with self._lock: + return dict(self._pantsd_metrics) # defensive copy def run_information(self) -> dict[str, Any]: """Basic information about this run.""" - return self._run_info + with self._lock: + return dict(self._run_info) def has_ended(self) -> bool: - return self._has_ended + with self._lock: + return self._has_ended def end_run(self, exit_code: ExitCode) -> None: """This pants run is over, so stop tracking it. @@ -170,23 +182,25 @@ def end_run(self, exit_code: ExitCode) -> None: Note: If end_run() has been called once, subsequent calls are no-ops. """ - if self.has_ended(): - return - self._has_ended = True + with self._lock: + if self._has_ended: + return + self._has_ended = True - if self._run_start_time is None: - raise Exception("RunTracker.end_run() called without calling .start()") + if self._run_start_time is None: + raise Exception("RunTracker.end_run() called without calling .start()") - duration = time.time() - self._run_start_time - self._run_total_duration = duration + duration = time.time() - self._run_start_time + self._run_total_duration = duration - outcome_str = "SUCCESS" if exit_code == PANTS_SUCCEEDED_EXIT_CODE else "FAILURE" - self._run_info["outcome"] = outcome_str + outcome_str = "SUCCESS" if exit_code == PANTS_SUCCEEDED_EXIT_CODE else "FAILURE" + self._run_info["outcome"] = outcome_str - native_engine.set_per_run_log_path(None) + native_engine.set_per_run_log_path(None) def get_cumulative_timings(self) -> list[dict[str, Any]]: - return [{"label": "main", "timing": self._run_total_duration}] + with self._lock: + return [{"label": "main", "timing": self._run_total_duration}] def get_options_to_record(self) -> dict: recorded_options = {} diff --git a/src/python/pants/goal/run_tracker_test.py b/src/python/pants/goal/run_tracker_test.py index 20293d60409..02521411049 100644 --- a/src/python/pants/goal/run_tracker_test.py +++ b/src/python/pants/goal/run_tracker_test.py @@ -4,6 +4,7 @@ import datetime import re import time +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import pytest @@ -144,3 +145,17 @@ def test_anonymous_telemetry_with_no_repo_id(tmp_path: Path) -> None: for key in ("repo_id", "machine_id", "user_id"): assert telemetry.get(key) == "" + + +def test_end_run_concurrent_calls_are_idempotent(tmp_path: Path) -> None: + buildroot = tmp_path.as_posix() + with environment_as(PANTS_BUILDROOT_OVERRIDE=buildroot): + ob = create_options_bootstrapper([]) + run_tracker = RunTracker(ob.args, ob.bootstrap_options) + run_tracker.start(run_start_time=time.time(), specs=[]) + + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(lambda _: run_tracker.end_run(PANTS_SUCCEEDED_EXIT_CODE), range(8))) + + assert run_tracker.has_ended() + assert run_tracker.run_information()["outcome"] == "SUCCESS" diff --git a/src/python/pants/init/logging.py b/src/python/pants/init/logging.py index ab535ad4315..e2b7f1f1c15 100644 --- a/src/python/pants/init/logging.py +++ b/src/python/pants/init/logging.py @@ -7,6 +7,7 @@ import locale import logging import sys +import threading from collections.abc import Iterator from contextlib import contextmanager from io import BufferedReader, TextIOWrapper @@ -27,6 +28,9 @@ logging.addLevelName(logging.WARNING, "WARN") logging.addLevelName(pants_logging.TRACE, "TRACE") +_python_logging_lock = threading.RLock() +_stdio_init_lock = threading.RLock() + class _NativeHandler(Handler): """This class is installed as a Python logging module handler (using the logging.addHandler @@ -108,43 +112,44 @@ def trace_fn(self, message, *args, **kwargs): if self.isEnabledFor(LogLevel.TRACE.level): self._log(LogLevel.TRACE.level, message, *args, **kwargs) - logging.Logger.trace = trace_fn # type: ignore[attr-defined] - logger = logging.getLogger(None) - - def clear_logging_handlers(): - handlers = tuple(logger.handlers) - for handler in handlers: - logger.removeHandler(handler) - return handlers - - def set_logging_handlers(handlers): - for handler in handlers: + with _python_logging_lock: + logging.Logger.trace = trace_fn # type: ignore[attr-defined] + logger = logging.getLogger(None) + + def clear_logging_handlers(): + handlers = tuple(logger.handlers) + for handler in handlers: + logger.removeHandler(handler) + return handlers + + def set_logging_handlers(handlers): + for handler in handlers: + logger.addHandler(handler) + + # Remove existing handlers, and restore them afterward. + handlers = clear_logging_handlers() + try: + # This routes warnings through our loggers instead of straight to raw stderr. + logging.captureWarnings(True) + handler = _NativeHandler() + exc_formatter = _ExceptionFormatter(level, print_stacktrace=print_stacktrace) + handler.setFormatter(exc_formatter) logger.addHandler(handler) + level.set_level_for(logger) - # Remove existing handlers, and restore them afterward. - handlers = clear_logging_handlers() - try: - # This routes warnings through our loggers instead of straight to raw stderr. - logging.captureWarnings(True) - handler = _NativeHandler() - exc_formatter = _ExceptionFormatter(level, print_stacktrace=print_stacktrace) - handler.setFormatter(exc_formatter) - logger.addHandler(handler) - level.set_level_for(logger) - - for key, level in log_levels_by_target.items(): - level.set_level_for(logging.getLogger(key)) - - if logger.isEnabledFor(LogLevel.TRACE.level): - http.client.HTTPConnection.debuglevel = 1 - requests_logger = logging.getLogger("requests.packages.urllib3") - LogLevel.TRACE.set_level_for(requests_logger) - requests_logger.propagate = True + for key, level in log_levels_by_target.items(): + level.set_level_for(logging.getLogger(key)) - yield - finally: - clear_logging_handlers() - set_logging_handlers(handlers) + if logger.isEnabledFor(LogLevel.TRACE.level): + http.client.HTTPConnection.debuglevel = 1 + requests_logger = logging.getLogger("requests.packages.urllib3") + LogLevel.TRACE.set_level_for(requests_logger) + requests_logger.propagate = True + + yield + finally: + clear_logging_handlers() + set_logging_handlers(handlers) @contextmanager @@ -192,39 +197,40 @@ def initialize_stdio_raw( else: literal_filters.append(filt) - # Set the pants log destination. - log_path = str(pants_log_path(PurePath(pants_workdir))) - safe_mkdir_for(log_path) - - # Initialize thread-local stdio, and replace sys.std* with proxies. - original_stdin, original_stdout, original_stderr = sys.stdin, sys.stdout, sys.stderr - try: - raw_stdin, sys.stdout, sys.stderr = native_engine.stdio_initialize( - global_level.level, - log_show_rust_3rdparty, - show_target, - {k: v.level for k, v in log_levels_by_target.items()}, - tuple(literal_filters), - tuple(regex_filters), - log_path, - ) - sys.stdin = TextIOWrapper( - BufferedReader(raw_stdin), - # NB: We set the default encoding explicitly to bypass logic in the TextIOWrapper - # constructor that would poke the underlying file (which is not valid until a - # `stdio_destination` is set). - encoding=locale.getpreferredencoding(False), - ) + with _stdio_init_lock: + # Set the pants log destination. + log_path = str(pants_log_path(PurePath(pants_workdir))) + safe_mkdir_for(log_path) + + # Initialize thread-local stdio, and replace sys.std* with proxies. + original_stdin, original_stdout, original_stderr = sys.stdin, sys.stdout, sys.stderr + try: + raw_stdin, sys.stdout, sys.stderr = native_engine.stdio_initialize( + global_level.level, + log_show_rust_3rdparty, + show_target, + {k: v.level for k, v in log_levels_by_target.items()}, + tuple(literal_filters), + tuple(regex_filters), + log_path, + ) + sys.stdin = TextIOWrapper( + BufferedReader(raw_stdin), + # NB: We set the default encoding explicitly to bypass logic in the TextIOWrapper + # constructor that would poke the underlying file (which is not valid until a + # `stdio_destination` is set). + encoding=locale.getpreferredencoding(False), + ) - sys.__stdin__, sys.__stdout__, sys.__stderr__ = sys.stdin, sys.stdout, sys.stderr # type: ignore[misc,assignment] - # Install a Python logger that will route through the Rust logger. - with _python_logging_setup( - global_level, log_levels_by_target, print_stacktrace=print_stacktrace - ): - yield - finally: - sys.stdin, sys.stdout, sys.stderr = original_stdin, original_stdout, original_stderr - sys.__stdin__, sys.__stdout__, sys.__stderr__ = sys.stdin, sys.stdout, sys.stderr # type: ignore[misc,assignment] + sys.__stdin__, sys.__stdout__, sys.__stderr__ = sys.stdin, sys.stdout, sys.stderr # type: ignore[misc,assignment] + # Install a Python logger that will route through the Rust logger. + with _python_logging_setup( + global_level, log_levels_by_target, print_stacktrace=print_stacktrace + ): + yield + finally: + sys.stdin, sys.stdout, sys.stderr = original_stdin, original_stdout, original_stderr + sys.__stdin__, sys.__stdout__, sys.__stderr__ = sys.stdin, sys.stdout, sys.stderr # type: ignore[misc,assignment] def pants_log_path(workdir: PurePath) -> PurePath: diff --git a/src/python/pants/option/bootstrap_options.py b/src/python/pants/option/bootstrap_options.py index 75894f90ec2..042200aad5f 100644 --- a/src/python/pants/option/bootstrap_options.py +++ b/src/python/pants/option/bootstrap_options.py @@ -903,6 +903,17 @@ class BootstrapOptions: """ ), ) + pants_free_threaded = BoolOption( + advanced=True, + default=True, + help=softwrap( + f""" + Run Pants on a free-threaded CPython interpreter. Like `pants_version`, this is + consumed by the `{bin_name()}` launcher to select which Pants distribution to + install and has no effect once Pants is running. + """ + ), + ) pants_bin_name = StrOption( advanced=True, default="pants", diff --git a/src/python/pants/util/memo.py b/src/python/pants/util/memo.py index 37ce68cb78e..f5219ccd327 100644 --- a/src/python/pants/util/memo.py +++ b/src/python/pants/util/memo.py @@ -7,6 +7,7 @@ from contextlib import contextmanager from typing import Any, TypeVar +from pants.engine.internals import native_engine from pants.util.meta import T, classproperty FuncType = Callable[..., Any] @@ -62,14 +63,13 @@ def per_instance(*args, **kwargs): return equal_args(*instance_and_rest, **kwargs) -def memoized(func: F | None = None, key_factory=equal_args, cache_factory=dict) -> F: +def memoized(func: F | None = None, key_factory=equal_args, cache_factory=None) -> F: """Memoizes the results of a function call. By default, exactly one result is memoized for each unique combination of function arguments. - Note that memoization is not thread-safe and the default result cache will grow without bound; - so care must be taken to only apply this decorator to functions with single threaded access and - an expected reasonably small set of unique call parameters. + Note that the default result cache will grow without bound, so care must be taken to only apply + this decorator to functions with an expected reasonably small set of unique call parameters. Note that the wrapped function comes equipped with 3 helper function attributes: @@ -84,19 +84,17 @@ def memoized(func: F | None = None, key_factory=equal_args, cache_factory=dict) :API: public :param func: The function to wrap. Only generally passed by the python runtime and should be - omitted when passing a custom `key_factory` or `cache_factory`. + omitted when passing a custom `key_factory`. :param key_factory: A function that can form a cache key from the arguments passed to the wrapped, memoized function; by default uses simple parameter-set equality; ie `equal_args`. - :param cache_factory: A no-arg callable that produces a mapping object to use for the memoized - method's value cache. By default the `dict` constructor, but could be a - a factory for an LRU cache for example. + :param cache_factory: Deprecated and ignored. Memoization caches are now engine-managed. :raises: `ValueError` if the wrapper is applied to anything other than a function. :returns: A wrapped function that memoizes its results or else a function wrapper that does this. """ if func is None: # We're being applied as a decorator factory; ie: the user has supplied args, like so: - # >>> @memoized(cache_factory=lru_cache) + # >>> @memoized(key_factory=create_key) # ... def expensive_operation(user): # ... pass # So we return a decorator with the user-supplied args curried in for the python decorator @@ -113,41 +111,53 @@ def memoized(func: F | None = None, key_factory=equal_args, cache_factory=dict) if not inspect.isfunction(func): raise ValueError("The @memoized decorator must be applied innermost of all decorators.") + if cache_factory is not None: + from pants.base.deprecated import warn_or_error + + warn_or_error( + removal_version="2.35.0.dev0", + entity=f"the cache_factory argument to @memoized (on {func.__qualname__})", + hint=( + "Memoization caches are now engine-managed for thread safety under " + "free-threaded Python, so custom cache factories (e.g. LRU caches) are no " + "longer honored. The argument is silently ignored and can be removed." + ), + ) + key_func = key_factory or equal_args - memoized_results = cache_factory() if cache_factory else {} + cache = native_engine.LockMap() @functools.wraps(func) def memoize(*args, **kwargs): key = key_func(*args, **kwargs) - if key in memoized_results: - return memoized_results[key] - result = func(*args, **kwargs) - memoized_results[key] = result - return result + return cache.get_or_insert(key, lambda: func(*args, **kwargs)) @contextmanager def put(*args, **kwargs): key = key_func(*args, **kwargs) - yield functools.partial(memoized_results.__setitem__, key) + + def setter(value): + cache.put(key, value) + + yield setter memoize.put = put # type: ignore[attr-defined] def forget(*args, **kwargs): key = key_func(*args, **kwargs) - if key in memoized_results: - del memoized_results[key] + cache.forget(key) memoize.forget = forget # type: ignore[attr-defined] def clear(): - memoized_results.clear() + cache.clear() memoize.clear = clear # type: ignore[attr-defined] return memoize # type: ignore[return-value] -def memoized_method(func: F | None = None, key_factory=per_instance, cache_factory=dict) -> F: +def memoized_method(func: F | None = None, key_factory=per_instance, cache_factory=None) -> F: """A convenience wrapper for memoizing instance methods. Typically you'd expect a memoized instance method to hold a cached value per class instance; @@ -173,10 +183,10 @@ def memoized_method(func: F | None = None, key_factory=per_instance, cache_facto :API: public :param func: The function to wrap. Only generally passed by the python runtime and should be - omitted when passing a custom `key_factory` or `cache_factory`. + omitted when passing a custom `key_factory`. :param key_factory: A function that can form a cache key from the arguments passed to the wrapped, memoized function; by default `per_instance`. - :param kwargs: Any extra keyword args accepted by `memoized`. + :param cache_factory: Deprecated and ignored. Memoization caches are now engine-managed. :raises: `ValueError` if the wrapper is applied to anything other than a function. :returns: A wrapped function that memoizes its results or else a function wrapper that does this. """ @@ -184,7 +194,7 @@ def memoized_method(func: F | None = None, key_factory=per_instance, cache_facto def memoized_property( - func: Callable[..., T] | None = None, key_factory=per_instance, cache_factory=dict + func: Callable[..., T] | None = None, key_factory=per_instance, cache_factory=None ) -> T: """A convenience wrapper for memoizing properties. @@ -239,10 +249,10 @@ def memoized_property( :API: public :param func: The property getter method to wrap. Only generally passed by the python runtime and - should be omitted when passing a custom `key_factory` or `cache_factory`. + should be omitted when passing a custom `key_factory`. :param key_factory: A function that can form a cache key from the arguments passed to the wrapped, memoized function; by default `per_instance`. - :param kwargs: Any extra keyword args accepted by `memoized`. + :param cache_factory: Deprecated and ignored. Memoization caches are now engine-managed. :raises: `ValueError` if the wrapper is applied to anything other than a function. :returns: A read-only property that memoizes its calculated value and un-caches its value when `del`ed. @@ -255,25 +265,23 @@ def memoized_property( # TODO[13244]: fix type hint issue when using @memoized_classmethod and friends -def memoized_classmethod(func: F | None = None, key_factory=per_instance, cache_factory=dict) -> F: +def memoized_classmethod(func: F | None = None, key_factory=per_instance, cache_factory=None) -> F: return classmethod( # type: ignore[return-value] memoized_method(func, key_factory=key_factory, cache_factory=cache_factory) ) def memoized_classproperty( - func: Callable[..., T] | None = None, key_factory=per_instance, cache_factory=dict + func: Callable[..., T] | None = None, key_factory=per_instance, cache_factory=None ) -> T: return classproperty( memoized_classmethod(func, key_factory=key_factory, cache_factory=cache_factory) ) -def testable_memoized_property( - func: Callable[..., T] | None = None, key_factory=per_instance, cache_factory=dict -) -> T: +def testable_memoized_property(func: Callable[..., T] | None = None, key_factory=per_instance) -> T: """A variant of `memoized_property` that allows for setting of properties (for tests, etc).""" - getter = memoized_method(func=func, key_factory=key_factory, cache_factory=cache_factory) + getter = memoized_method(func=func, key_factory=key_factory) def setter(self, val): with getter.put(self) as putter: diff --git a/src/python/pants/util/memo_test.py b/src/python/pants/util/memo_test.py index 2fed5e0b629..5aef9596f27 100644 --- a/src/python/pants/util/memo_test.py +++ b/src/python/pants/util/memo_test.py @@ -1,6 +1,9 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import threading +from concurrent.futures import ThreadPoolExecutor + import pytest from pants.util.memo import ( @@ -100,27 +103,20 @@ def square(num): # TODO[13244] -def test_cache_factory(): - class SingleEntryMap(dict): - def __setitem__(self, key, value): - self.clear() - return super().__setitem__(key, value) - +def test_cache_factory_deprecated(caplog): calculations = [] - @memoized(cache_factory=SingleEntryMap) + @memoized(cache_factory=dict) def square(num): calculations.append(num) return num * num + assert len(caplog.records) == 1 + assert "the cache_factory argument to @memoized" in caplog.text + assert 4 == square(2) assert 4 == square(2) - assert 9 == square(3) - assert 9 == square(3) - assert 4 == square(2) - assert 4 == square(2) - - assert [2, 3, 2] == calculations + assert [2] == calculations # TODO[13244] @@ -171,6 +167,92 @@ def square(num): assert [2, 3, 2, 3] == calculations +def test_concurrent_same_key_calls_wait_for_cached_value() -> None: + ready = threading.Barrier(8) + started = threading.Event() + unblock = threading.Event() + call_lock = threading.Lock() + calls = 0 + + @memoized + def calculate(num): + nonlocal calls + with call_lock: + calls += 1 + started.set() + assert unblock.wait(timeout=5) + return calls + num + + def call_calculate(): + ready.wait() + return calculate(42) + + with ThreadPoolExecutor(max_workers=8) as executor: + futures = [executor.submit(call_calculate) for _ in range(8)] + assert started.wait(timeout=5) + unblock.set() + results = [future.result(timeout=5) for future in futures] + + assert len(set(results)) == 1 + assert calculate(42) == results[0] + assert calls == 1 + + +def test_concurrent_different_key_calls_compute_concurrently() -> None: + ready = threading.Barrier(8) + started = threading.Barrier(8) + done = threading.Event() + + @memoized + def calculate(num): + ready.wait() + started.wait() + assert done.wait(timeout=5) + return num + + with ThreadPoolExecutor(max_workers=8) as executor: + futures = [executor.submit(calculate, num) for num in range(8)] + done.set() + assert [future.result(timeout=5) for future in futures] == list(range(8)) + + +def test_exception_not_cached() -> None: + calls = 0 + + @memoized + def calculate(): + nonlocal calls + calls += 1 + raise ValueError(calls) + + for expected in (1, 2): + with pytest.raises(ValueError, match=str(expected)): + calculate() + + +def test_concurrent_cache_helpers() -> None: + @memoized + def calculate(num): + return num + 100 + + def put(num) -> None: + with getattr(calculate, "put")(num) as putter: + putter(num) + + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(put, range(32))) + assert [calculate(num) for num in range(32)] == list(range(32)) + + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(getattr(calculate, "forget"), range(16))) + assert [calculate(num) for num in range(16)] == [num + 100 for num in range(16)] + assert [calculate(num) for num in range(16, 32)] == list(range(16, 32)) + + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(lambda _: getattr(calculate, "clear")(), range(8))) + assert calculate(31) == 131 + + class _Called: def __init__(self, increment): self._calls = 0 diff --git a/src/python/pants/util/meta.py b/src/python/pants/util/meta.py index fcb4817fdd2..778fce7aee1 100644 --- a/src/python/pants/util/meta.py +++ b/src/python/pants/util/meta.py @@ -1,6 +1,7 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import threading from abc import ABC, abstractmethod from collections.abc import Callable from typing import Any, TypeVar @@ -19,11 +20,14 @@ class Unicorn(metaclass=SingletonMetaclass): pass """ + _instance_lock = threading.RLock() + def __call__(cls, *args: Any, **kwargs: Any) -> Any: # TODO: convert this into an `@memoized_classproperty`! - if not hasattr(cls, "instance"): - cls.instance = super().__call__(*args, **kwargs) - return cls.instance + with SingletonMetaclass._instance_lock: + if not hasattr(cls, "instance"): + cls.instance = super().__call__(*args, **kwargs) + return cls.instance class _ClassPropertyDescriptor: diff --git a/src/python/pants/util/meta_test.py b/src/python/pants/util/meta_test.py index e7facd8b6f5..f56793ef328 100644 --- a/src/python/pants/util/meta_test.py +++ b/src/python/pants/util/meta_test.py @@ -1,7 +1,9 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import time from abc import ABC, abstractmethod +from concurrent.futures import ThreadPoolExecutor import pytest @@ -16,6 +18,22 @@ class One(metaclass=SingletonMetaclass): assert One() is One() +def test_singleton_concurrent_construction() -> None: + constructions = 0 + + class One(metaclass=SingletonMetaclass): + def __init__(self) -> None: + nonlocal constructions + time.sleep(0.01) + constructions += 1 + + with ThreadPoolExecutor(max_workers=8) as executor: + instances = list(executor.map(lambda _: One(), range(8))) + + assert len({id(instance) for instance in instances}) == 1 + assert constructions == 1 + + class WithProp: _value = "val0" diff --git a/src/python/pants_release/generate_github_workflows.py b/src/python/pants_release/generate_github_workflows.py index 9dc6d36f1fd..d5a19dd429b 100644 --- a/src/python/pants_release/generate_github_workflows.py +++ b/src/python/pants_release/generate_github_workflows.py @@ -108,7 +108,7 @@ class Platform(Enum): # We don't specify a patch version so that we get the latest, which comes pre-installed: # https://github.com/actions/setup-python#available-versions-of-python # NOTE: The last entry becomes the default -_BASE_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] +_BASE_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] PYTHON_VERSIONS_PER_PLATFORM: dict[Platform, list[str]] = { Platform.LINUX_X86_64: _BASE_PYTHON_VERSIONS, @@ -988,8 +988,11 @@ def build_wheels_job( platform: Platform, for_deploy_ref: str | None, needs: list[str] | None, + free_threaded: bool = True, ) -> Jobs: helper = Helper(platform) + abi = "cp314t" if free_threaded else "cp314" + interpreter = "python3.14t" if free_threaded else "python3.14" # For manylinux compatibility, we build Linux wheels in a container rather than directly # on the Ubuntu runner. As a result, we have custom steps here to check out # the code, install rustup and expose Pythons. @@ -1021,6 +1024,7 @@ def build_wheels_job( echo "/opt/python/cp312-cp312/bin" >> $GITHUB_PATH echo "/opt/python/cp313-cp313/bin" >> $GITHUB_PATH echo "/opt/python/cp314-cp314/bin" >> $GITHUB_PATH + echo "/opt/python/cp314-cp314t/bin" >> $GITHUB_PATH """ ), }, @@ -1041,9 +1045,9 @@ def build_wheels_job( IS_PANTS_OWNER if for_deploy_ref else f"({IS_PANTS_OWNER}) && ({DONT_SKIP_WHEELS})" ) return { - helper.job_name("build_wheels"): { + helper.job_name("build_wheels" if free_threaded else "build_wheels_cp314"): { "if": if_condition, - "name": f"Build wheels ({str(platform.value)})", + "name": f"Build wheels ({str(platform.value)}, {abi})", "runs-on": helper.runs_on(), "permissions": { "id-token": "write", @@ -1056,6 +1060,9 @@ def build_wheels_job( "env": { "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION": bool(container), **DISABLE_REMOTE_CACHE_ENV, + # Pin the interpreter the Pants venv and native engine build with, which + # decides the ABI of the wheels and PEX this job produces. + "PY": interpreter, # If we're not deploying these wheels, build in debug mode, which allows for # incremental compilation across wheels. If this becomes too slow in CI, most likely # the answer will be to adjust the `opt-level` for the relevant Cargo profile rather @@ -1070,6 +1077,15 @@ def build_wheels_job( if platform == Platform.LINUX_ARM64 else install_go() ), + { + # Interpreter constraints cannot distinguish the two ABIs of CPython 3.14, + # so restrict the interpreter search to this lane's interpreter. + "name": "Pin the distribution interpreter", + "run": ( + f'echo "PANTS_PYTHON_BOOTSTRAP_SEARCH_PATH=' + f'[\'$(dirname "$(command -v {interpreter})")\']" >> "$GITHUB_ENV"' + ), + }, { "name": "Build wheels", "run": "./pants run src/python/pants_release/release.py -- build-wheels", @@ -1077,7 +1093,7 @@ def build_wheels_job( }, { "name": "Build Pants PEX", - "run": "./pants package src/python/pants:pants-pex", + "run": f"./pants package src/python/pants:pants-pex@scie_pbs_free_threaded={abi}", "env": helper.platform_env(), }, helper.upload_log_artifacts(name="wheels-and-pex"), @@ -1097,7 +1113,7 @@ def build_wheels_job( "run": dedent( """\ PANTS_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import pants.version;print(pants.version.VERSION)") - PY_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import sys;print(f'cp{sys.version_info[0]}{sys.version_info[1]}')") + PY_VER=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import sys;print(f'cp{sys.version_info[0]}{sys.version_info[1]}{sys.abiflags}')") PLAT=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import os;print(f'{os.uname().sysname.lower()}_{os.uname().machine.lower()}')") PEX_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT.pex PEX_SCIE_FILENAME=pants.$PANTS_VER-$PY_VER-$PLAT @@ -1196,6 +1212,9 @@ def build_wheels_jobs(*, for_deploy_ref: str | None = None, needs: list[str] | N **build_wheels_job(Platform.LINUX_X86_64, for_deploy_ref, needs), **build_wheels_job(Platform.LINUX_ARM64, for_deploy_ref, needs), **build_wheels_job(Platform.MACOS14_ARM64, for_deploy_ref, needs), + **build_wheels_job(Platform.LINUX_X86_64, for_deploy_ref, needs, free_threaded=False), + **build_wheels_job(Platform.LINUX_ARM64, for_deploy_ref, needs, free_threaded=False), + **build_wheels_job(Platform.MACOS14_ARM64, for_deploy_ref, needs, free_threaded=False), } diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index 9980a44a307..2d582986eeb 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -1144,6 +1144,7 @@ dependencies = [ "mock", "nailgun", "num_enum", + "once_cell", "options", "options_pants_ng", "pantsd", diff --git a/src/rust/engine/Cargo.toml b/src/rust/engine/Cargo.toml index b809fe46731..2aee7359088 100644 --- a/src/rust/engine/Cargo.toml +++ b/src/rust/engine/Cargo.toml @@ -44,6 +44,7 @@ log = { workspace = true } logging = { path = "../logging" } nailgun = { path = "../nailgun" } num_enum = { workspace = true } +once_cell = { workspace = true } options = { path = "../options" } options_pants_ng = { path = "../pants_ng/options" } pantsd = { path = "../pantsd" } diff --git a/src/rust/engine/src/externs/interface.rs b/src/rust/engine/src/externs/interface.rs index 860a5e12571..d855da76d15 100644 --- a/src/rust/engine/src/externs/interface.rs +++ b/src/rust/engine/src/externs/interface.rs @@ -62,12 +62,13 @@ use crate::{ TypeId, Types, Value, externs, nodes, }; -#[pymodule] +#[pymodule(gil_used = false)] fn native_engine(py: Python, m: &Bound<'_, PyModule>) -> PyO3Result<()> { intrinsics::register(py, m)?; externs::register(py, m)?; externs::address::register(py, m)?; externs::fs::register(m)?; + externs::memo::register(m)?; externs::nailgun::register(py, m)?; externs::options::register(m)?; externs::pants_ng::register(m)?; diff --git a/src/rust/engine/src/externs/memo.rs b/src/rust/engine/src/externs/memo.rs new file mode 100644 index 00000000000..d3d41c3b204 --- /dev/null +++ b/src/rust/engine/src/externs/memo.rs @@ -0,0 +1,421 @@ +// Copyright 2026 Pants project contributors (see CONTRIBUTORS.md). +// Licensed under the Apache License, Version 2.0 (see LICENSE). + +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use once_cell::sync::OnceCell; +use parking_lot::RwLock; +use pyo3::gc::{PyTraverseError, PyVisit}; +use pyo3::intern; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +/// A per-decorated-function memoization cache with exactly-once computation per key. +/// +/// Keys are arbitrary hashable Python objects, interned to a local integer id via a dict owned by +/// this `LockMap`. The engine's global `Interns` table is deliberately not used: it never evicts, +/// while memo keys must be released by `forget`/`clear`/property `del` (a `per_instance` key holds +/// a strong reference to `self`). Ids are assigned with `dict.setdefault`, a single dict operation +/// and so atomic on free-threaded builds; a lost race wastes only an integer. +#[pyclass(frozen, module = "pants.engine.internals.native_engine")] +pub struct LockMap { + ids: Py, + /// `ids.setdefault`, bound once: the hot path calls it per memoized invocation. + ids_setdefault: Py, + next_id: AtomicU64, + map: RwLock>>>>, +} + +#[pymethods] +impl LockMap { + #[new] + fn __new__(py: Python<'_>) -> PyResult { + let ids = PyDict::new(py); + let ids_setdefault = ids.getattr(intern!(py, "setdefault"))?.unbind(); + Ok(Self { + ids: ids.unbind(), + ids_setdefault, + next_id: AtomicU64::new(0), + map: RwLock::new(HashMap::new()), + }) + } + + fn get_or_insert( + &self, + py: Python<'_>, + key: Py, + compute: Py, + ) -> PyResult> { + let id = self.intern(py, &key)?; + let value = py.detach(move || { + self.get_or_insert_with(id, || Python::attach(|py| compute.call0(py))) + })?; + self.reconcile(py, &key, id); + Ok(value) + } + + fn put(&self, py: Python<'_>, key: Py, value: Py) -> PyResult<()> { + let id = self.intern(py, &key)?; + py.detach(move || { + let cell = OnceCell::new(); + let _ = cell.set(value); + self.map.write().insert(id, Arc::new(cell)); + }); + self.reconcile(py, &key, id); + Ok(()) + } + + fn forget(&self, py: Python<'_>, key: Py) -> PyResult<()> { + let id: Option = self + .ids + .bind(py) + .call_method1(intern!(py, "pop"), (key, py.None()))? + .extract()?; + if let Some(id) = id { + py.detach(move || { + self.map.write().remove(&id); + }); + } + Ok(()) + } + + fn clear(&self, py: Python<'_>) -> PyResult<()> { + self.ids.bind(py).call_method0(intern!(py, "clear"))?; + py.detach(|| { + self.map.write().clear(); + }); + Ok(()) + } + + fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.ids)?; + visit.call(&self.ids_setdefault)?; + // gc must not block: skip the values if a writer holds the lock this pass. + if let Some(map) = self.map.try_read() { + for cell in map.values() { + if let Some(value) = cell.get() { + visit.call(value)?; + } + } + } + Ok(()) + } + + fn __clear__(&self) { + // The id dict is its own gc-tracked object (broken by its tp_clear); ours is the values. + self.map.write().clear(); + } +} + +impl LockMap { + /// Return the stable local id for `key`, assigning the next id if unseen. `setdefault` makes + /// the check-and-assign a single atomic dict operation. + fn intern(&self, py: Python<'_>, key: &Py) -> PyResult { + let candidate = self.next_id.fetch_add(1, Ordering::Relaxed); + self.ids_setdefault + .bind(py) + .call1((key, candidate))? + .extract() + } + + /// A writer's intern-then-populate is not atomic against `forget`/`clear`, which could strand + /// a just-populated entry under an id no longer reachable from `ids`. After populating, + /// re-assert the binding: if the key still (or again) maps to our id, the entry is reachable + /// and a later `forget` can release it. If it now maps to a different id (a concurrent + /// forget-then-reintern won), drop our entry rather than leak it. + fn reconcile(&self, py: Python<'_>, key: &Py, id: u64) { + let current: Option = self.intern(py, key).ok().filter(|current| *current != id); + if current.is_some() { + py.detach(|| { + self.map.write().remove(&id); + }); + } + } + + fn get_or_insert_with( + &self, + id: u64, + compute: impl FnOnce() -> PyResult>, + ) -> PyResult> { + let cell = self.cell_for(id); + let value = cell.get_or_try_init(compute)?; + Python::attach(|py| Ok(value.clone_ref(py))) + } + + fn cell_for(&self, id: u64) -> Arc>> { + if let Some(cell) = self.map.read().get(&id).cloned() { + return cell; + } + + self.map + .write() + .entry(id) + .or_insert_with(|| Arc::new(OnceCell::new())) + .clone() + } +} + +pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::LockMap; + use pyo3::exceptions::PyValueError; + use pyo3::prelude::*; + use pyo3::types::PyDictMethods; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Barrier, Condvar, Mutex}; + use std::thread; + use std::time::Duration; + + fn new_lockmap() -> LockMap { + Python::attach(|py| LockMap::__new__(py).unwrap()) + } + + fn key(py: Python<'_>, value: i64) -> Py { + value.into_pyobject(py).unwrap().into_any().unbind() + } + + fn py_value(py: Python<'_>, value: i64) -> Py { + value.into_pyobject(py).unwrap().into_any().unbind() + } + + fn i64_value(value: Py) -> i64 { + Python::attach(|py| value.extract(py).unwrap()) + } + + fn intern(cache: &LockMap, key: &Py) -> u64 { + Python::attach(|py| cache.intern(py, key).unwrap()) + } + + fn wait_for_all(started: &Arc<(Mutex, Condvar)>, total: usize) { + let (lock, cvar) = &**started; + let mut count = lock.lock().unwrap(); + *count += 1; + cvar.notify_all(); + while *count < total { + let result = cvar.wait_timeout(count, Duration::from_secs(5)).unwrap(); + count = result.0; + assert!( + !result.1.timed_out(), + "timed out waiting for concurrent initialization" + ); + } + } + + #[test] + fn same_key_concurrent_calls_initialize_once() { + Python::initialize(); + + let cache = Arc::new(new_lockmap()); + let key = Python::attach(|py| key(py, 1)); + let ready = Arc::new(Barrier::new(8)); + let calls = Arc::new(AtomicUsize::new(0)); + + let handles: Vec<_> = (0..8) + .map(|_| { + let cache = Arc::clone(&cache); + let key = Python::attach(|py| key.clone_ref(py)); + let ready = Arc::clone(&ready); + let calls = Arc::clone(&calls); + thread::spawn(move || { + ready.wait(); + // Each thread interns independently, covering the concurrent id-assignment + // path as well as the once-only computation. + let id = intern(&cache, &key); + let value = cache + .get_or_insert_with(id, || { + calls.fetch_add(1, Ordering::SeqCst); + thread::sleep(Duration::from_millis(50)); + Python::attach(|py| Ok(py_value(py, 99))) + }) + .unwrap(); + i64_value(value) + }) + }) + .collect(); + + let results: Vec<_> = handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect(); + assert_eq!(vec![99; 8], results); + assert_eq!(1, calls.load(Ordering::SeqCst)); + } + + #[test] + fn different_keys_initialize_independently() { + Python::initialize(); + + let cache = Arc::new(new_lockmap()); + let started = Arc::new((Mutex::new(0), Condvar::new())); + let keys: Vec<_> = Python::attach(|py| (0..8).map(|value| key(py, value)).collect()); + + let handles: Vec<_> = keys + .into_iter() + .enumerate() + .map(|(value, key)| { + let cache = Arc::clone(&cache); + let started = Arc::clone(&started); + thread::spawn(move || { + let id = intern(&cache, &key); + let value = cache + .get_or_insert_with(id, || { + wait_for_all(&started, 8); + Python::attach(|py| Ok(py_value(py, value as i64))) + }) + .unwrap(); + i64_value(value) + }) + }) + .collect(); + + let results: Vec<_> = handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect(); + assert_eq!((0..8).collect::>(), results); + } + + #[test] + fn failed_initialization_is_not_cached() { + Python::initialize(); + + let cache = new_lockmap(); + let key = Python::attach(|py| key(py, 1)); + let calls = AtomicUsize::new(0); + + let id = intern(&cache, &key); + assert!( + cache + .get_or_insert_with(id, || { + calls.fetch_add(1, Ordering::SeqCst); + Err(PyValueError::new_err("bad")) + }) + .is_err() + ); + + let value = cache + .get_or_insert_with(id, || { + calls.fetch_add(1, Ordering::SeqCst); + Python::attach(|py| Ok(py_value(py, 42))) + }) + .unwrap(); + + assert_eq!(42, i64_value(value)); + assert_eq!(2, calls.load(Ordering::SeqCst)); + } + + #[test] + fn equal_keys_share_an_id() { + Python::initialize(); + + let cache = new_lockmap(); + // Equal-but-not-identical keys (Python smallint caching is bypassed via tuples). + let (a, b) = Python::attach(|py| { + let a = (1_i64, "x").into_pyobject(py).unwrap().into_any().unbind(); + let b = (1_i64, "x").into_pyobject(py).unwrap().into_any().unbind(); + (a, b) + }); + assert_eq!(intern(&cache, &a), intern(&cache, &b)); + } + + #[test] + fn put_forget_and_clear() { + Python::initialize(); + + let cache = new_lockmap(); + let key = Python::attach(|py| key(py, 1)); + let calls = AtomicUsize::new(0); + + Python::attach(|py| cache.put(py, key.clone_ref(py), py_value(py, 10)).unwrap()); + let id = intern(&cache, &key); + let value = cache + .get_or_insert_with(id, || { + calls.fetch_add(1, Ordering::SeqCst); + Python::attach(|py| Ok(py_value(py, 20))) + }) + .unwrap(); + assert_eq!(10, i64_value(value)); + assert_eq!(0, calls.load(Ordering::SeqCst)); + + Python::attach(|py| cache.forget(py, key.clone_ref(py)).unwrap()); + let id = intern(&cache, &key); + let value = cache + .get_or_insert_with(id, || { + calls.fetch_add(1, Ordering::SeqCst); + Python::attach(|py| Ok(py_value(py, 20))) + }) + .unwrap(); + assert_eq!(20, i64_value(value)); + assert_eq!(1, calls.load(Ordering::SeqCst)); + + Python::attach(|py| cache.put(py, key.clone_ref(py), py_value(py, 30)).unwrap()); + Python::attach(|py| cache.clear(py).unwrap()); + let id = intern(&cache, &key); + let value = cache + .get_or_insert_with(id, || { + calls.fetch_add(1, Ordering::SeqCst); + Python::attach(|py| Ok(py_value(py, 40))) + }) + .unwrap(); + assert_eq!(40, i64_value(value)); + assert_eq!(2, calls.load(Ordering::SeqCst)); + } + + #[test] + fn reconcile_drops_entry_when_forget_won() { + Python::initialize(); + + let cache = new_lockmap(); + let key = Python::attach(|py| key(py, 1)); + + // Writer interned id, then a concurrent forget popped the binding before the writer + // populated the map: reconcile must not leave the entry stranded under the dead id. + let id = intern(&cache, &key); + Python::attach(|py| cache.forget(py, key.clone_ref(py)).unwrap()); + cache + .get_or_insert_with(id, || Python::attach(|py| Ok(py_value(py, 7)))) + .unwrap(); + Python::attach(|py| cache.reconcile(py, &key, id)); + assert!(!cache.map.read().contains_key(&id)); + + // No interference: reconcile keeps the entry when the binding still points at our id. + let id = intern(&cache, &key); + cache + .get_or_insert_with(id, || Python::attach(|py| Ok(py_value(py, 8)))) + .unwrap(); + Python::attach(|py| cache.reconcile(py, &key, id)); + assert!(cache.map.read().contains_key(&id)); + } + + #[test] + fn forget_and_clear_release_keys() { + Python::initialize(); + + let cache = new_lockmap(); + let (key_a, key_b) = Python::attach(|py| (key(py, 1), key(py, 2))); + + Python::attach(|py| { + cache + .put(py, key_a.clone_ref(py), py_value(py, 10)) + .unwrap(); + cache + .put(py, key_b.clone_ref(py), py_value(py, 20)) + .unwrap(); + assert_eq!(2, cache.ids.bind(py).len()); + + cache.forget(py, key_a.clone_ref(py)).unwrap(); + assert_eq!(1, cache.ids.bind(py).len()); + + cache.clear(py).unwrap(); + assert_eq!(0, cache.ids.bind(py).len()); + }); + assert!(cache.map.read().is_empty()); + } +} diff --git a/src/rust/engine/src/externs/mod.rs b/src/rust/engine/src/externs/mod.rs index 2fa22022bea..f7378ddebb7 100644 --- a/src/rust/engine/src/externs/mod.rs +++ b/src/rust/engine/src/externs/mod.rs @@ -37,6 +37,7 @@ pub mod fs; mod interface; #[cfg(test)] mod interface_tests; +pub mod memo; pub mod nailgun; mod options; mod pants_ng; diff --git a/src/rust/engine/src/externs/scheduler.rs b/src/rust/engine/src/externs/scheduler.rs index 433db2bf1f7..0e6ddee55c2 100644 --- a/src/rust/engine/src/externs/scheduler.rs +++ b/src/rust/engine/src/externs/scheduler.rs @@ -1,22 +1,104 @@ // Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). // Licensed under the Apache License, Version 2.0 (see LICENSE). +use std::cell::Cell; use std::time::Duration; +use parking_lot::RwLock; use pyo3::exceptions::PyException; use pyo3::ffi; use pyo3::prelude::*; pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; + // LIFO: registering at import time makes this run last among `atexit` callbacks. + m.py() + .import("atexit")? + .call_method1("register", (wrap_pyfunction!(python_exiting, m)?,))?; Ok(()) } -// NB: This exists because we need the PyInterpreterState to pass to PyThreadState_New, -// however PyInterpreterState_Get wasn't added until Py 3.9. They vary in implementation, but because -// we don't have any sub-interpreters they should both return the same object. -unsafe extern "C" { - pub fn PyInterpreterState_Main() -> *mut ffi::PyInterpreterState; +/// Whether engine threads may still attach to the interpreter. `python_exiting` flips this from +/// `atexit`, which `Py_FinalizeEx` runs before marking the runtime finalizing and after all +/// later-registered callbacks (such as executor shutdowns). The thread hooks attach under the +/// read lock, so they can never race finalization and hang or touch a freed state. +static PYTHON_ALIVE: RwLock = RwLock::new(true); + +#[pyfunction] +fn python_exiting(py: Python) { + py.detach(|| { + *PYTHON_ALIVE.write() = false; + }); +} + +/// Runs `f`, which may attach to the interpreter, under the `PYTHON_ALIVE` gate. Skips `f` +/// once Python is shutting down, or when the interpreter is not initialized. +fn with_python_alive(f: impl FnOnce()) { + let alive = PYTHON_ALIVE.read(); + // SAFETY: `Py_IsInitialized` only reads a runtime flag and may be called from any thread at + // any time, even before initialization or after finalization. + if *alive && unsafe { ffi::Py_IsInitialized() != 0 } { + f() + } +} + +thread_local! { + /// This thread's detached `PyThreadState` and the `PyGILState_STATE` returned by the + /// `PyGILState_Ensure` call that created it. + static THREAD_STATE: Cell> = + const { Cell::new(None) }; +} + +/// Create a thread state for this thread up front and keep it until `thread_state_destroy`. +/// +/// Each `Python::attach` on a thread with no thread state creates one via `PyGILState_Ensure` +/// and destroys it again on release. That resets the debug trace function between calls (see +/// https://github.com/PyO3/pyo3/issues/2495), and on free-threaded builds the teardown is +/// expensive: `PyThreadState_Clear` abandons the thread's mimalloc heaps, which every other +/// thread then pays to reclaim on allocation. +/// +/// NB: The state must be created with `PyGILState_Ensure`: the gilstate machinery does not know +/// about states created with `PyThreadState_New`, and would still create and destroy its own on +/// every attach. `PyEval_SaveThread` then detaches, leaving the thread parked without a state +/// attached (which would otherwise block free-threaded stop-the-world pauses). +fn thread_state_create() { + with_python_alive(|| { + // SAFETY: A fresh runtime thread is not attached and has no gilstate yet: + // `PyGILState_Ensure` creates this thread's state and attaches, `PyEval_SaveThread` + // detaches and returns it, and both stay valid until the matching `PyGILState_Release` + // in `thread_state_destroy`. + unsafe { + let gilstate = ffi::PyGILState_Ensure(); + let tstate = ffi::PyEval_SaveThread(); + THREAD_STATE.set(Some((tstate, gilstate))); + } + if std::env::var("PANTS_DEBUG").is_ok() { + Python::attach(|py| { + let _ = py.eval(c"__import__('debugpy').debug_this_thread()", None, None); + }); + } + }); +} + +/// Release the thread state created by `thread_state_create`. Tokio recycles blocking-pool +/// threads after an idle timeout, so thread states must die with their threads or they +/// accumulate for the life of the process. Re-attaching and then releasing the last gilstate +/// reference has CPython clear and delete the state. +fn thread_state_destroy() { + let Some((tstate, gilstate)) = THREAD_STATE.take() else { + return; + }; + // Leak the state if Python is shutting down. + with_python_alive(|| { + // SAFETY: `tstate` and `gilstate` come from the paired calls in `thread_state_create` on + // this same thread, which is detached here; the `PYTHON_ALIVE` read lock keeps the + // interpreter alive across the attach. Releasing the last gilstate reference has CPython + // clear and delete the state. + unsafe { + ffi::PyEval_RestoreThread(tstate); + ffi::PyGILState_Release(gilstate); + } + }); } #[pyclass] @@ -27,20 +109,12 @@ pub struct PyExecutor(pub task_executor::Executor); impl PyExecutor { #[new] fn __new__(core_threads: usize, max_threads: usize) -> PyResult { - task_executor::Executor::new_owned(core_threads, max_threads, || { - // NB: We need a PyThreadState object which lives throughout the lifetime of this thread - // as the debug trace object is attached to it. Otherwise the PyThreadState is - // constructed/destroyed with each `with_gil` call (inside PyGILState_Ensure/PyGILState_Release). - // - // Constructing (and leaking) a ThreadState object allocates and associates it with the current - // thread, and the Python runtime won't wipe the trace function between calls. - // See https://github.com/PyO3/pyo3/issues/2495 - let _ = - unsafe { ffi::PyThreadState_New(Python::attach(|_| PyInterpreterState_Main())) }; - Python::attach(|py| { - let _ = py.eval(c"__import__('debugpy').debug_this_thread()", None, None); - }); - }) + task_executor::Executor::new_owned( + core_threads, + max_threads, + thread_state_create, + thread_state_destroy, + ) .map(PyExecutor) .map_err(PyException::new_err) } @@ -60,10 +134,24 @@ impl PyExecutor { impl Drop for PyExecutor { fn drop(&mut self) { - if !self.0.is_shutdown() { - // This can lead to hangs, since `Drop` will run on an arbitrary thread under arbitrary - // locks. See #18211. - log::warn!("Executor was not shut down explicitly."); + if self.0.is_shutdown() { + return; + } + log::warn!("Executor was not shut down explicitly."); + // Dropping the Runtime joins its threads, and each thread attaches to Python in + // `thread_state_destroy` before exiting. `Drop` runs attached (via garbage collection), + // so shut down with this thread detached or the joins deadlock on the GIL. + // + // Shutting down from within a runtime would panic, and attaching from a detached thread + // acquires the GIL, which can deadlock under arbitrary Python-side locks (see #18211): + // leak it in both cases. + // + // When already attached, `Python::attach` acquires nothing and `py.detach` only releases. + // + // SAFETY: `PyGILState_Check` may be called from any thread at any time. + if tokio::runtime::Handle::try_current().is_err() && unsafe { ffi::PyGILState_Check() } == 1 + { + Python::attach(|py| py.detach(|| self.0.shutdown(Duration::from_secs(3)))); } } } diff --git a/src/rust/engine/src/interning.rs b/src/rust/engine/src/interning.rs index 0f09e25c3c7..8b98a7de611 100644 --- a/src/rust/engine/src/interning.rs +++ b/src/rust/engine/src/interning.rs @@ -54,13 +54,13 @@ impl Interns { let (id, type_id): (u64, TypeId) = { let v = v.bind(py); let keys = self.keys.bind(py); - let id: u64 = if let Some(key) = keys.get_item(v)? { - key.extract()? - } else { - let id = self.id_generator.fetch_add(1, atomic::Ordering::Relaxed); - keys.set_item(v, id)?; - id - }; + // `setdefault` is a single dict operation, so the check-and-assign is atomic on + // free-threaded builds (a separate get-then-set lets two threads mint distinct ids + // for equal values, breaking Key's id-based equality). A lost race only wastes an id. + let candidate = self.id_generator.fetch_add(1, atomic::Ordering::Relaxed); + let id: u64 = keys + .call_method1(pyo3::intern!(py, "setdefault"), (v, candidate))? + .extract()?; (id, TypeId::new(&v.get_type())) }; diff --git a/src/rust/fs/store/benches/store.rs b/src/rust/fs/store/benches/store.rs index 980e71da71b..d550a36463f 100644 --- a/src/rust/fs/store/benches/store.rs +++ b/src/rust/fs/store/benches/store.rs @@ -22,7 +22,7 @@ use tempfile::TempDir; use store::{OneOffStoreFileByDigest, Snapshot, SnapshotOps, Store, SubsetParams}; fn executor() -> Executor { - Executor::new_owned(num_cpus::get(), num_cpus::get() * 4, || ()).unwrap() + Executor::new_owned(num_cpus::get(), num_cpus::get() * 4, || (), || ()).unwrap() } pub fn criterion_benchmark_materialize(c: &mut Criterion) { diff --git a/src/rust/task_executor/src/lib.rs b/src/rust/task_executor/src/lib.rs index e60e6c95edd..775c73e0572 100644 --- a/src/rust/task_executor/src/lib.rs +++ b/src/rust/task_executor/src/lib.rs @@ -2,7 +2,6 @@ // Licensed under the Apache License, Version 2.0 (see LICENSE). use std::collections::HashMap; -use std::env; use std::future::Future; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -75,13 +74,15 @@ impl Executor { /// need thread configurability, but also want to know reliably when the Runtime will shutdown /// (which, because it is static, will only be at the entire process' exit). /// - pub fn new_owned( + pub fn new_owned( num_worker_threads: usize, max_threads: usize, on_thread_start: F, + on_thread_stop: G, ) -> Result where F: Fn() + Send + Sync + 'static, + G: Fn() + Send + Sync + 'static, { let mut runtime_builder = Builder::new_multi_thread(); @@ -90,9 +91,12 @@ impl Executor { .max_blocking_threads(max_threads - num_worker_threads) .enable_all(); - if env::var("PANTS_DEBUG").is_ok() { - runtime_builder.on_thread_start(on_thread_start); - }; + // NB: These run on every runtime-managed thread, including blocking-pool threads, which + // tokio expires after an idle keep-alive and respawns on demand. Any per-thread state + // created in `on_thread_start` must be torn down in `on_thread_stop`: a leak here is per + // thread created, not per pool slot, and so grows without bound in a long-lived pantsd. + runtime_builder.on_thread_start(on_thread_start); + runtime_builder.on_thread_stop(on_thread_stop); let runtime = runtime_builder .build() diff --git a/testprojects/src/python/native/pyproject.toml b/testprojects/src/python/native/pyproject.toml index 9b469b2e960..cb7d6ee4a53 100644 --- a/testprojects/src/python/native/pyproject.toml +++ b/testprojects/src/python/native/pyproject.toml @@ -1,4 +1,5 @@ [build-system] -# Using the oldest `setuptools` that allows `package_dists_integration_test.py` to pass on Python 3.14 -requires = ["setuptools==66.1.0", "wheel==0.37.0", "ansicolors==1.1.8"] +# Using the oldest `setuptools` that allows `package_dists_integration_test.py` to pass on Python 3.14. +# wheel>=0.44 is required so that `bdist_wheel` can emit free-threaded (cp314t) ABI tags. +requires = ["setuptools==66.1.0", "wheel==0.45.1", "ansicolors==1.1.8"] build-backend = "setuptools.build_meta"