diff --git a/.github/workflows/build-llvm.yml b/.github/workflows/build-llvm.yml index 914a81314..471717b5c 100644 --- a/.github/workflows/build-llvm.yml +++ b/.github/workflows/build-llvm.yml @@ -4,23 +4,9 @@ on: workflow_dispatch: inputs: llvm_version: - description: "LLVM version to build (e.g., 21.1.8)" + description: "LLVM version to build (e.g., 22.1.4)" required: true type: string - skip_upload: - description: "Skip upload and PR creation (build-only mode)" - required: false - type: boolean - default: false - skip_pr: - description: "Skip PR creation (upload only, no PR)" - required: false - type: boolean - default: false - pull_request: - # if you want to run this workflow, change the branch name to main, - # if you want to turn off it, change it to non existent branch. - branches: [main-turn-off] jobs: build: @@ -121,20 +107,48 @@ jobs: with: environments: ${{ matrix.pixi_env || 'package' }} + - name: Free Disk Space (macOS) + if: runner.os == 'macOS' + run: | + echo "=== Before cleanup ===" + df -h / + + # Remove iOS/tvOS/watchOS/visionOS simulators + if [ -d "/Library/Developer/CoreSimulator" ]; then + sudo rm -rf /Library/Developer/CoreSimulator + echo "Removed CoreSimulator" + fi + + # Remove unnecessary Xcode platforms (keep macOS only) + XCODE_PATH="$(xcode-select -p)" + PLATFORMS_DIR="${XCODE_PATH}/Platforms" + for platform in iPhoneOS.platform iPhoneSimulator.platform AppleTVOS.platform AppleTVSimulator.platform WatchOS.platform WatchSimulator.platform XROS.platform XRSimulator.platform; do + if [ -d "${PLATFORMS_DIR}/${platform}" ]; then + sudo rm -rf "${PLATFORMS_DIR}/${platform}" + echo "Removed ${platform}" + fi + done + + # Remove Android SDK + sudo rm -rf ~/Library/Android + + # Remove .NET/PowerShell + sudo rm -rf /usr/local/share/dotnet + sudo rm -rf /usr/local/share/powershell + + # Remove Haskell + sudo rm -rf ~/.ghcup + + echo "=== After cleanup ===" + df -h / + - name: Clone llvm-project shell: bash run: | - VERSION="${{ inputs.llvm_version || '21.1.8' }}" + VERSION="${{ inputs.llvm_version }}" echo "Cloning LLVM ${VERSION}..." git clone --branch "llvmorg-${VERSION}" --depth 1 https://github.com/llvm/llvm-project.git .llvm - - name: Validate distribution components - shell: bash - run: | - python3 scripts/validate-llvm-components.py \ - --llvm-src=.llvm \ - --components-file=scripts/llvm-components.json - - name: Build LLVM (install-distribution) shell: bash run: | @@ -150,158 +164,30 @@ jobs: --build-dir=build \ ${EXTRA_ARGS} - - name: Build clice using installed LLVM - if: ${{ !matrix.target_triple }} - shell: bash - run: | - pixi run cmake-config ${{ matrix.llvm_mode }} ON -- \ - "-DCLICE_ENABLE_LTO=${{ matrix.lto }}" \ - "-DLLVM_INSTALL_PATH=.llvm/build-install" - pixi run cmake-build ${{ matrix.llvm_mode }} - - - name: Build clice using installed LLVM (cross-compile) - if: ${{ matrix.target_triple }} - shell: bash - run: | - ENV="${{ matrix.pixi_env || 'package' }}" - pixi run -e "$ENV" cmake-config ${{ matrix.llvm_mode }} ON -- \ - "-DCLICE_ENABLE_LTO=${{ matrix.lto }}" \ - "-DCLICE_TARGET_TRIPLE=${{ matrix.target_triple }}" \ - "-DLLVM_INSTALL_PATH=.llvm/build-install" - pixi run -e "$ENV" cmake-build ${{ matrix.llvm_mode }} - - - name: Verify cross-compiled binary architecture - if: ${{ matrix.target_triple && runner.os != 'Windows' }} - shell: bash - run: | - BINARY="build/${{ matrix.llvm_mode }}/bin/clice" - echo "Binary info:" - file "$BINARY" - case "${{ matrix.target_triple }}" in - aarch64-linux-gnu) file "$BINARY" | grep -q "aarch64" ;; - x86_64-apple-darwin) file "$BINARY" | grep -q "x86_64" ;; - esac - - - name: Upload cross-compiled clice for functional test - if: ${{ matrix.target_triple && matrix.lto == 'OFF' }} - uses: actions/upload-artifact@v4 - with: - name: cross-clice-${{ matrix.target_triple }}-${{ matrix.llvm_mode }} - path: | - build/${{ matrix.llvm_mode }}/bin/ - build/${{ matrix.llvm_mode }}/lib/ - if-no-files-found: error - retention-days: 1 - - - name: Run tests - if: ${{ !matrix.target_triple }} - shell: bash - run: pixi run test ${{ matrix.llvm_mode }} - - # Prune is only supported for native builds (requires linking clice to test). - # Cross-compiled targets reuse the native prune manifest of the same OS. - - name: Prune LLVM static libraries (Debug/RelWithDebInfo no LTO) - if: (!matrix.target_triple) && (matrix.llvm_mode == 'Debug' || (matrix.llvm_mode == 'RelWithDebInfo' && matrix.lto == 'OFF')) - shell: bash - run: | - MANIFEST="pruned-libs-${{ matrix.os }}.json" - echo "LLVM_PRUNED_MANIFEST=${MANIFEST}" >> "${GITHUB_ENV}" - python3 scripts/prune-llvm-bin.py \ - --action discover \ - --install-dir ".llvm/build-install/lib" \ - --build-dir "build/${{ matrix.llvm_mode }}" \ - --max-attempts 60 \ - --sleep-seconds 60 \ - --manifest "${MANIFEST}" - - - name: Upload pruned-libs manifest - if: (!matrix.target_triple) && matrix.llvm_mode == 'RelWithDebInfo' && matrix.lto == 'OFF' - uses: actions/upload-artifact@v4 - with: - name: llvm-pruned-libs-${{ matrix.os }} - path: ${{ env.LLVM_PRUNED_MANIFEST }} - if-no-files-found: error - compression-level: 0 - - - name: Apply pruned-libs manifest (RelWithDebInfo + LTO, native only) - if: (!matrix.target_triple) && matrix.llvm_mode == 'RelWithDebInfo' && matrix.lto == 'ON' - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - MANIFEST="pruned-libs-${{ matrix.os }}.json" - python3 scripts/prune-llvm-bin.py \ - --action apply \ - --manifest "${MANIFEST}" \ - --install-dir ".llvm/build-install/lib" \ - --build-dir "build/${{ matrix.llvm_mode }}" \ - --gh-run-id "${{ github.run_id }}" \ - --gh-artifact "llvm-pruned-libs-${{ matrix.os }}" \ - --gh-download-dir "artifacts" \ - --max-attempts 60 \ - --sleep-seconds 60 - - # For cross-compiled LTO builds, apply the native prune manifest. - # The unused library set is arch-independent (same API surface). - - name: Apply pruned-libs manifest (cross-compile + LTO) - if: matrix.target_triple && matrix.lto == 'ON' - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - MANIFEST="pruned-libs-${{ matrix.os }}.json" - python3 scripts/prune-llvm-bin.py \ - --action apply \ - --manifest "${MANIFEST}" \ - --install-dir ".llvm/build-install/lib" \ - --build-dir "build/${{ matrix.llvm_mode }}" \ - --gh-run-id "${{ github.run_id }}" \ - --gh-artifact "llvm-pruned-libs-${{ matrix.os }}" \ - --gh-download-dir "artifacts" \ - --max-attempts 60 \ - --sleep-seconds 60 - - name: Package LLVM install directory shell: bash run: | - MODE_TAG="releasedbg" - if [[ "${{ matrix.llvm_mode }}" == "Debug" ]]; then - MODE_TAG="debug" - fi - - # Determine arch/platform/toolchain from target triple or runner OS + # Determine platform/arch from target triple or runner OS if [[ -n "${{ matrix.target_triple }}" ]]; then case "${{ matrix.target_triple }}" in - x86_64-apple-darwin) - ARCH="x64"; PLATFORM="macos"; TOOLCHAIN="clang" ;; - aarch64-linux-gnu) - ARCH="aarch64"; PLATFORM="linux"; TOOLCHAIN="gnu" ;; - aarch64-pc-windows-msvc) - ARCH="aarch64"; PLATFORM="windows"; TOOLCHAIN="msvc" ;; + x86_64-apple-darwin) PLATFORM="macos"; ARCH="x64" ;; + aarch64-linux-gnu) PLATFORM="linux"; ARCH="aarch64" ;; + aarch64-pc-windows-msvc) PLATFORM="windows"; ARCH="aarch64" ;; esac else - ARCH="x64" - PLATFORM="linux" - TOOLCHAIN="gnu" + PLATFORM="linux"; ARCH="x64" if [[ "${{ matrix.os }}" == windows-* ]]; then PLATFORM="windows" - TOOLCHAIN="msvc" elif [[ "${{ matrix.os }}" == macos-* ]]; then - ARCH="arm64" - PLATFORM="macos" - TOOLCHAIN="clang" + PLATFORM="macos"; ARCH="arm64" fi fi - SUFFIX="" - if [[ "${{ matrix.lto }}" == "ON" ]]; then - SUFFIX="-lto" - fi - if [[ "${{ matrix.llvm_mode }}" == "Debug" && "${{ matrix.os }}" != windows-* ]]; then - SUFFIX="${SUFFIX}-asan" - fi + NAME_ARGS="--platform $PLATFORM --arch $ARCH --mode ${{ matrix.llvm_mode }}" + if [[ "${{ matrix.lto }}" == "ON" ]]; then NAME_ARGS+=" --lto"; fi + if [[ "${{ matrix.llvm_mode }}" == "Debug" && "${{ matrix.os }}" != windows-* ]]; then NAME_ARGS+=" --asan"; fi - ARCHIVE="${ARCH}-${PLATFORM}-${TOOLCHAIN}-${MODE_TAG}${SUFFIX}.tar.xz" + ARCHIVE=$(python3 scripts/release-llvm.py artifact-name $NAME_ARGS) set -eo pipefail tar -C .llvm -cf - build-install | xz -T0 -9 -c > "${ARCHIVE}" @@ -313,134 +199,3 @@ jobs: name: ${{ env.LLVM_INSTALL_ARCHIVE }} path: ${{ env.LLVM_INSTALL_ARCHIVE }} if-no-files-found: error - - test-cross: - needs: build - strategy: - fail-fast: false - matrix: - include: - - os: macos-15-intel - llvm_mode: RelWithDebInfo - target_triple: x86_64-apple-darwin - - os: ubuntu-24.04-arm - llvm_mode: RelWithDebInfo - target_triple: aarch64-linux-gnu - - os: windows-11-arm - llvm_mode: RelWithDebInfo - target_triple: aarch64-pc-windows-msvc - runs-on: ${{ matrix.os }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - uses: ./.github/actions/setup-pixi - with: - environments: test-run - - - name: Download cross-compiled clice - uses: actions/download-artifact@v4 - with: - name: cross-clice-${{ matrix.target_triple }}-${{ matrix.llvm_mode }} - path: build/${{ matrix.llvm_mode }}/ - - - name: Make binaries executable - if: runner.os != 'Windows' - run: chmod +x build/${{ matrix.llvm_mode }}/bin/* - - - name: Run tests - run: pixi run -e test-run test ${{ matrix.llvm_mode }} - - upload: - needs: build - if: ${{ !cancelled() && inputs.llvm_version && !inputs.skip_upload }} - runs-on: ubuntu-24.04 - permissions: - contents: read - steps: - - uses: actions/checkout@v4 - - - name: Download all build artifacts - env: - GH_TOKEN: ${{ github.token }} - run: scripts/download-llvm.sh "${{ github.run_id }}" - - - name: Upload to clice-llvm - env: - GH_TOKEN: ${{ secrets.UPLOAD_LLVM }} - TARGET_REPO: clice-io/clice-llvm - run: python3 scripts/upload-llvm.py "${{ inputs.llvm_version }}" "${TARGET_REPO}" "${{ github.run_id }}" - - - name: Save manifest for update-clice job - uses: actions/upload-artifact@v4 - with: - name: llvm-manifest-final - path: artifacts/llvm-manifest.json - if-no-files-found: error - compression-level: 0 - - update-clice: - needs: upload - if: ${{ !inputs.skip_pr }} - runs-on: ubuntu-24.04 - permissions: - contents: write - pull-requests: write - steps: - - uses: actions/checkout@v4 - - - name: Download manifest - uses: actions/download-artifact@v4 - with: - name: llvm-manifest-final - path: . - - - name: Update manifest and version - run: | - python3 scripts/update-llvm-version.py \ - --version "${{ inputs.llvm_version }}" \ - --manifest-src llvm-manifest.json \ - --manifest-dest config/llvm-manifest.json \ - --package-cmake cmake/package.cmake - - - name: Create or update PR - env: - GH_TOKEN: ${{ github.token }} - run: | - VERSION="${{ inputs.llvm_version }}" - BRANCH="chore/update-llvm-${VERSION}" - RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - RELEASE_URL="https://github.com/clice-io/clice-llvm/releases/tag/${VERSION}" - - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git checkout -b "${BRANCH}" - git add config/llvm-manifest.json cmake/package.cmake - git commit -m "chore: update LLVM to ${VERSION}" - git push --force-with-lease origin "${BRANCH}" - - # Check if PR already exists for this branch - EXISTING_PR=$(gh pr list --head "${BRANCH}" --json number --jq '.[0].number // empty') - - BODY="$(cat < Auto-generated by build-llvm workflow - EOF - )" - - if [[ -n "${EXISTING_PR}" ]]; then - echo "Updating existing PR #${EXISTING_PR}" - gh pr edit "${EXISTING_PR}" --body "${BODY}" - else - gh pr create \ - --title "chore: update LLVM to ${VERSION}" \ - --body "${BODY}" \ - --base main - fi diff --git a/.github/workflows/release-llvm.yml b/.github/workflows/release-llvm.yml new file mode 100644 index 000000000..f2e00f1d3 --- /dev/null +++ b/.github/workflows/release-llvm.yml @@ -0,0 +1,227 @@ +name: release llvm + +on: + workflow_dispatch: + inputs: + source_run_id: + description: "Run ID from build-llvm containing LLVM artifacts" + required: true + type: string + llvm_version: + description: "LLVM version (e.g. 22.1.4)" + required: true + type: string + +env: + SOURCE_RUN_ID: ${{ inputs.source_run_id }} + LLVM_VERSION: ${{ inputs.llvm_version }} + +jobs: + discover: + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + artifact: x64-linux-gnu-releasedbg.tar.xz + - os: macos-15 + artifact: arm64-macos-clang-releasedbg.tar.xz + - os: windows-2025 + artifact: x64-windows-msvc-releasedbg.tar.xz + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - uses: ./.github/actions/setup-pixi + with: + environments: package + + - name: Free Disk Space (macOS) + if: runner.os == 'macOS' + run: | + if [ -d "/Library/Developer/CoreSimulator" ]; then + sudo rm -rf /Library/Developer/CoreSimulator + fi + XCODE_PATH="$(xcode-select -p)" + PLATFORMS_DIR="${XCODE_PATH}/Platforms" + for p in iPhoneOS.platform iPhoneSimulator.platform AppleTVOS.platform AppleTVSimulator.platform WatchOS.platform WatchSimulator.platform XROS.platform XRSimulator.platform; do + [ -d "${PLATFORMS_DIR}/${p}" ] && sudo rm -rf "${PLATFORMS_DIR}/${p}" + done + sudo rm -rf ~/Library/Android /usr/local/share/dotnet /usr/local/share/powershell ~/.ghcup + + - name: Download LLVM artifact + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + gh run download "${{ env.SOURCE_RUN_ID }}" \ + -n "${{ matrix.artifact }}" -D .llvm-download + mkdir -p .llvm + tar -xf ".llvm-download/${{ matrix.artifact }}" -C .llvm + rm -rf .llvm-download + + - name: Build clice + shell: bash + run: | + pixi run cmake-config RelWithDebInfo ON -- \ + "-DLLVM_INSTALL_PATH=.llvm/build-install" + pixi run cmake-build RelWithDebInfo + + - name: Discover unused libraries + shell: bash + run: | + python3 scripts/release-llvm.py discover \ + --install-dir .llvm/build-install/lib \ + --build-dir build/RelWithDebInfo \ + --manifest pruned-libs.json \ + --skip-pattern "*Tidy*Module*" + + - name: Print prune results + shell: bash + run: | + python3 -c " + import json + data = json.load(open('pruned-libs.json')) + removed = data['removed'] + total = data.get('total_saved_bytes', 0) + print(f'Libraries prunable: {len(removed)}') + print(f'Space saved: {total / 1048576:.1f} MB') + for entry in sorted(removed, key=lambda e: e['size'], reverse=True)[:20]: + print(f' {entry[\"size\"] / 1048576:>7.1f} MB {entry[\"name\"]}') + " + + - name: Upload manifest + uses: actions/upload-artifact@v4 + with: + name: prune-manifest-${{ matrix.os }} + path: pruned-libs.json + + create-release: + needs: discover + runs-on: ubuntu-24.04 + steps: + - name: Create or reuse release + env: + GH_TOKEN: ${{ secrets.UPLOAD_LLVM }} + run: | + TAG="${{ env.LLVM_VERSION }}" + REPO="clice-io/clice-llvm" + if gh release view "$TAG" --repo "$REPO" &>/dev/null; then + echo "Release $TAG already exists, will overwrite assets" + else + gh release create "$TAG" \ + --repo "$REPO" \ + --title "$TAG" \ + --notes "LLVM ${TAG} — build run https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + fi + + repackage: + needs: [discover, create-release] + runs-on: ubuntu-24.04 + permissions: + contents: read + actions: read + strategy: + fail-fast: false + matrix: + artifact: + - aarch64-linux-gnu-releasedbg-lto.tar.xz + - aarch64-linux-gnu-releasedbg.tar.xz + - aarch64-windows-msvc-releasedbg-lto.tar.xz + - aarch64-windows-msvc-releasedbg.tar.xz + - arm64-macos-clang-debug-asan.tar.xz + - arm64-macos-clang-releasedbg-lto.tar.xz + - arm64-macos-clang-releasedbg.tar.xz + - x64-linux-gnu-debug-asan.tar.xz + - x64-linux-gnu-releasedbg-lto.tar.xz + - x64-linux-gnu-releasedbg.tar.xz + - x64-macos-clang-releasedbg-lto.tar.xz + - x64-macos-clang-releasedbg.tar.xz + - x64-windows-msvc-releasedbg-lto.tar.xz + - x64-windows-msvc-releasedbg.tar.xz + steps: + - uses: actions/checkout@v4 + + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + + - name: Download prune manifests + uses: actions/download-artifact@v4 + with: + pattern: prune-manifest-* + path: manifests + + - name: Repackage + env: + GH_TOKEN: ${{ github.token }} + run: | + python3 scripts/release-llvm.py repackage \ + --source-run-id "${{ env.SOURCE_RUN_ID }}" \ + --manifests-dir manifests \ + --output-dir artifacts \ + --max-parallel 1 \ + --version "${{ env.LLVM_VERSION }}" \ + --artifacts "${{ matrix.artifact }}" + + - name: Upload to release + env: + GH_TOKEN: ${{ secrets.UPLOAD_LLVM }} + run: | + gh release upload "${{ env.LLVM_VERSION }}" \ + "artifacts/${{ matrix.artifact }}" \ + --repo clice-io/clice-llvm \ + --clobber + + - name: Upload metadata + uses: actions/upload-artifact@v4 + with: + name: metadata-${{ matrix.artifact }} + path: artifacts/${{ matrix.artifact }}.meta.json + compression-level: 0 + retention-days: 1 + + finalize: + needs: repackage + runs-on: ubuntu-24.04 + steps: + - name: Download all metadata + uses: actions/download-artifact@v4 + with: + pattern: metadata-* + path: metadata + merge-multiple: true + + - name: Build manifest + run: | + python3 -c " + import json, pathlib + entries = [json.loads(p.read_text()) for p in sorted(pathlib.Path('metadata').glob('*.meta.json'))] + manifest = {'version': entries[0]['version'], 'artifacts': {}} + for entry in entries: + filename = entry.pop('filename') + entry.pop('version', None) + manifest['artifacts'][filename] = entry + with open('llvm-manifest.json', 'w') as f: + json.dump(manifest, f, indent=2) + f.write('\n') + print(f'Manifest: {len(manifest[\"artifacts\"])} artifacts') + " + + - name: Upload manifest to release + env: + GH_TOKEN: ${{ secrets.UPLOAD_LLVM }} + run: | + gh release upload "${{ env.LLVM_VERSION }}" \ + llvm-manifest.json \ + --repo clice-io/clice-llvm \ + --clobber + + - name: Save manifest artifact + uses: actions/upload-artifact@v4 + with: + name: llvm-manifest-final + path: llvm-manifest.json + if-no-files-found: error + compression-level: 0 diff --git a/.github/workflows/upload-llvm.yml b/.github/workflows/upload-llvm.yml deleted file mode 100644 index 6ad5def10..000000000 --- a/.github/workflows/upload-llvm.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: upload-llvm - -permissions: - contents: write - -on: - pull_request: - # if you want to run this workflow, change the branch name to main, - # if you want to turn off it, change it to non existent branch. - branches: [main-turn-off] - - workflow_dispatch: - inputs: - workflow_id: - description: "Workflow run ID to pull artifacts from" - required: true - type: string - version: - description: "Release version/tag to publish (e.g., v1.2.3)" - required: true - type: string - -jobs: - upload: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - - name: Download artifacts from workflow - env: - GH_TOKEN: ${{ github.token }} - run: scripts/download-llvm.sh "${{ inputs.workflow_id }}" - - - name: Recreate release with artifacts - env: - GH_TOKEN: ${{ secrets.UPLOAD_LLVM }} - TARGET_REPO: clice-io/clice-llvm - run: python3 scripts/upload-llvm.py "${{ inputs.version }}" "${TARGET_REPO}" "${{ inputs.workflow_id }}" diff --git a/cmake/llvm.cmake b/cmake/llvm.cmake index 694857ef5..a1235d3de 100644 --- a/cmake/llvm.cmake +++ b/cmake/llvm.cmake @@ -1,131 +1,200 @@ include_guard() -function(setup_llvm LLVM_VERSION) - find_package(Python3 COMPONENTS Interpreter REQUIRED) - - set(LLVM_SETUP_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/.llvm/setup-llvm.json") - set(LLVM_SETUP_SCRIPT "${PROJECT_SOURCE_DIR}/scripts/setup-llvm.py") - set(LLVM_SETUP_ARGS - "--version" "${LLVM_VERSION}" - "--build-type" "${CMAKE_BUILD_TYPE}" - "--binary-dir" "${CMAKE_CURRENT_BINARY_DIR}" - "--manifest" "${PROJECT_SOURCE_DIR}/config/llvm-manifest.json" - "--output" "${LLVM_SETUP_OUTPUT}" - ) +function(_detect_llvm_artifact_name OUT_FILENAME) + if(DEFINED CLICE_TARGET_TRIPLE) + if(CLICE_TARGET_TRIPLE MATCHES "^aarch64") + set(_ARCH "aarch64") + elseif(CLICE_TARGET_TRIPLE MATCHES "^x86_64") + set(_ARCH "x64") + else() + message(FATAL_ERROR "Unsupported arch in CLICE_TARGET_TRIPLE: ${CLICE_TARGET_TRIPLE}") + endif() - if(CLICE_ENABLE_LTO) - list(APPEND LLVM_SETUP_ARGS "--enable-lto") + if(CLICE_TARGET_TRIPLE MATCHES "linux") + set(_PLATFORM "linux") + set(_TOOLCHAIN "gnu") + elseif(CLICE_TARGET_TRIPLE MATCHES "darwin") + set(_PLATFORM "macos") + set(_TOOLCHAIN "clang") + elseif(CLICE_TARGET_TRIPLE MATCHES "windows") + set(_PLATFORM "windows") + set(_TOOLCHAIN "msvc") + else() + message(FATAL_ERROR "Unsupported platform in CLICE_TARGET_TRIPLE: ${CLICE_TARGET_TRIPLE}") + endif() + else() + if(WIN32) + set(_PLATFORM "windows") + set(_TOOLCHAIN "msvc") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|ARM64") + set(_ARCH "aarch64") + else() + set(_ARCH "x64") + endif() + elseif(APPLE) + set(_PLATFORM "macos") + set(_TOOLCHAIN "clang") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|ARM64") + set(_ARCH "arm64") + else() + set(_ARCH "x64") + endif() + else() + set(_PLATFORM "linux") + set(_TOOLCHAIN "gnu") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|ARM64") + set(_ARCH "aarch64") + else() + set(_ARCH "x64") + endif() + endif() endif() - if(DEFINED LLVM_INSTALL_PATH AND NOT LLVM_INSTALL_PATH STREQUAL "") - list(APPEND LLVM_SETUP_ARGS "--install-path" "${LLVM_INSTALL_PATH}") + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(_MODE "debug") + else() + set(_MODE "releasedbg") endif() - if(DEFINED CLICE_OFFLINE_BUILD AND CLICE_OFFLINE_BUILD) - list(APPEND LLVM_SETUP_ARGS "--offline") + set(_SUFFIX "") + if(CLICE_ENABLE_LTO) + string(APPEND _SUFFIX "-lto") + endif() + if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT WIN32) + string(APPEND _SUFFIX "-asan") endif() - if(DEFINED CLICE_TARGET_TRIPLE) - if(CLICE_TARGET_TRIPLE MATCHES "linux") - list(APPEND LLVM_SETUP_ARGS "--target-platform" "Linux") - elseif(CLICE_TARGET_TRIPLE MATCHES "darwin") - list(APPEND LLVM_SETUP_ARGS "--target-platform" "macosx") - elseif(CLICE_TARGET_TRIPLE MATCHES "windows") - list(APPEND LLVM_SETUP_ARGS "--target-platform" "Windows") + set(${OUT_FILENAME} "${_ARCH}-${_PLATFORM}-${_TOOLCHAIN}-${_MODE}${_SUFFIX}.tar.xz" PARENT_SCOPE) +endfunction() + +function(_download_and_extract _URL _SHA256 _DEST _LABEL) + set(_DOWNLOAD_PATH "${CMAKE_CURRENT_BINARY_DIR}/${_LABEL}") + if(EXISTS "${_DOWNLOAD_PATH}") + file(SHA256 "${_DOWNLOAD_PATH}" _EXISTING_HASH) + if(NOT _EXISTING_HASH STREQUAL "${_SHA256}") + message(STATUS "Hash mismatch for cached ${_LABEL}, re-downloading") + file(REMOVE "${_DOWNLOAD_PATH}") endif() + endif() - if(CLICE_TARGET_TRIPLE MATCHES "^aarch64") - list(APPEND LLVM_SETUP_ARGS "--target-arch" "arm64") - elseif(CLICE_TARGET_TRIPLE MATCHES "^x86_64") - list(APPEND LLVM_SETUP_ARGS "--target-arch" "x64") + if(NOT EXISTS "${_DOWNLOAD_PATH}") + message(STATUS "Downloading ${_LABEL}") + file(DOWNLOAD "${_URL}" "${_DOWNLOAD_PATH}" + EXPECTED_HASH SHA256=${_SHA256} + SHOW_PROGRESS + STATUS _DL_STATUS) + list(GET _DL_STATUS 0 _DL_CODE) + if(NOT _DL_CODE EQUAL 0) + list(GET _DL_STATUS 1 _DL_MSG) + file(REMOVE "${_DOWNLOAD_PATH}") + message(FATAL_ERROR "Failed to download ${_LABEL}: ${_DL_MSG}") endif() endif() execute_process( - COMMAND "${Python3_EXECUTABLE}" "${LLVM_SETUP_SCRIPT}" ${LLVM_SETUP_ARGS} - RESULT_VARIABLE LLVM_SETUP_RESULT - OUTPUT_VARIABLE LLVM_SETUP_STDOUT - ERROR_VARIABLE LLVM_SETUP_STDERR - ECHO_OUTPUT_VARIABLE - ECHO_ERROR_VARIABLE - COMMAND_ERROR_IS_FATAL ANY - ) + COMMAND "${CMAKE_COMMAND}" -E tar xf "${_DOWNLOAD_PATH}" + WORKING_DIRECTORY "${_DEST}" + RESULT_VARIABLE _TAR_RESULT) + if(NOT _TAR_RESULT EQUAL 0) + message(FATAL_ERROR "Failed to extract ${_LABEL}") + endif() +endfunction() - file(READ "${LLVM_SETUP_OUTPUT}" LLVM_SETUP_JSON) - string(JSON LLVM_INSTALL_PATH GET "${LLVM_SETUP_JSON}" install_path) - string(JSON LLVM_CMAKE_DIR GET "${LLVM_SETUP_JSON}" cmake_dir) - set(LLVM_INSTALL_PATH "${LLVM_INSTALL_PATH}" CACHE PATH "Path to LLVM installation" FORCE) - set(LLVM_CMAKE_DIR "${LLVM_CMAKE_DIR}" CACHE PATH "Path to LLVM CMake files" FORCE) +function(_download_llvm LLVM_VERSION) + _detect_llvm_artifact_name(_FILENAME) - get_filename_component(LLVM_INSTALL_PATH "${LLVM_INSTALL_PATH}" ABSOLUTE) + set(_MANIFEST_PATH "${PROJECT_SOURCE_DIR}/config/llvm-manifest.json") + file(READ "${_MANIFEST_PATH}" _MANIFEST_JSON) - if(NOT EXISTS "${LLVM_INSTALL_PATH}") - message(FATAL_ERROR "Error: The specified LLVM_INSTALL_PATH does not exist: ${LLVM_INSTALL_PATH}") + string(JSON _SHA256 ERROR_VARIABLE _ERR + GET "${_MANIFEST_JSON}" artifacts "${_FILENAME}" sha256) + if(_ERR) + message(FATAL_ERROR "No matching LLVM artifact in manifest for: ${_FILENAME}") endif() - # set llvm include and lib path - add_library(llvm-libs INTERFACE IMPORTED) + set(_BASE_URL "https://github.com/clice-io/clice-llvm/releases/download/${LLVM_VERSION}") + set(_INSTALL_ROOT "${CMAKE_CURRENT_BINARY_DIR}/.llvm") + set(_VERSION_STAMP "${_INSTALL_ROOT}/.llvm-version") - # add to include directories - target_include_directories(llvm-libs INTERFACE "${LLVM_INSTALL_PATH}/include") + set(_NEED_INSTALL TRUE) + if(EXISTS "${_INSTALL_ROOT}/lib/cmake/llvm/LLVMConfig.cmake" AND EXISTS "${_VERSION_STAMP}") + file(READ "${_VERSION_STAMP}" _CACHED_VERSION) + string(STRIP "${_CACHED_VERSION}" _CACHED_VERSION) + if(_CACHED_VERSION STREQUAL "${LLVM_VERSION}") + set(_NEED_INSTALL FALSE) + else() + message(STATUS "LLVM version changed (${_CACHED_VERSION} -> ${LLVM_VERSION}), reinstalling") + file(REMOVE_RECURSE "${_INSTALL_ROOT}") + endif() + endif() - if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND NOT WIN32) - target_link_directories(llvm-libs INTERFACE "${LLVM_INSTALL_PATH}/lib") - target_link_libraries(llvm-libs INTERFACE - LLVMSupport - LLVMFrontendOpenMP - LLVMOption - LLVMTargetParser - clangAST - clangASTMatchers - clangBasic - clangDriver - clangFormat - clangFrontend - clangLex - clangSema - clangSerialization - clangTidy - clangTidyUtils - clangTidyAndroidModule - clangTidyAbseilModule - clangTidyAlteraModule - clangTidyBoostModule - clangTidyBugproneModule - clangTidyCERTModule - clangTidyConcurrencyModule - clangTidyCppCoreGuidelinesModule - clangTidyDarwinModule - clangTidyFuchsiaModule - clangTidyGoogleModule - clangTidyHICPPModule - clangTidyLinuxKernelModule - clangTidyLLVMModule - clangTidyLLVMLibcModule - clangTidyMiscModule - clangTidyModernizeModule - clangTidyObjCModule - clangTidyOpenMPModule - clangTidyPerformanceModule - clangTidyPortabilityModule - clangTidyReadabilityModule - clangTidyZirconModule - clangTooling - clangToolingCore - clangToolingInclusions - clangToolingInclusionsStdlib - clangToolingSyntax - ) + if(_NEED_INSTALL) + file(MAKE_DIRECTORY "${_INSTALL_ROOT}") + + _download_and_extract( + "${_BASE_URL}/${_FILENAME}" "${_SHA256}" "${_INSTALL_ROOT}" "${_FILENAME}") + + if(EXISTS "${_INSTALL_ROOT}/build-install") + file(GLOB _NESTED "${_INSTALL_ROOT}/build-install/*") + foreach(_ENTRY ${_NESTED}) + get_filename_component(_NAME "${_ENTRY}" NAME) + file(RENAME "${_ENTRY}" "${_INSTALL_ROOT}/${_NAME}") + endforeach() + file(REMOVE_RECURSE "${_INSTALL_ROOT}/build-install") + endif() + + file(WRITE "${_VERSION_STAMP}" "${LLVM_VERSION}\n") + endif() + + set(LLVM_INSTALL_PATH "${_INSTALL_ROOT}" PARENT_SCOPE) +endfunction() + +function(setup_llvm LLVM_VERSION) + if(DEFINED LLVM_INSTALL_PATH AND NOT LLVM_INSTALL_PATH STREQUAL "") + get_filename_component(LLVM_INSTALL_PATH "${LLVM_INSTALL_PATH}" ABSOLUTE) + if(NOT EXISTS "${LLVM_INSTALL_PATH}") + message(FATAL_ERROR "LLVM_INSTALL_PATH does not exist: ${LLVM_INSTALL_PATH}") + endif() + elseif(DEFINED CLICE_OFFLINE_BUILD AND CLICE_OFFLINE_BUILD) + message(FATAL_ERROR "LLVM_INSTALL_PATH must be set in offline mode") else() - file(GLOB LLVM_LIBRARIES CONFIGURE_DEPENDS "${LLVM_INSTALL_PATH}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}LLVM[a-zA-Z]*${CMAKE_STATIC_LIBRARY_SUFFIX}") - file(GLOB CLANG_LIBRARIES CONFIGURE_DEPENDS "${LLVM_INSTALL_PATH}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}clang[a-zA-Z]*${CMAKE_STATIC_LIBRARY_SUFFIX}") - # TODO: find a better way to find out whether zlib and zstd are needed - # Currently link if present in the LLVM lib directory - file(GLOB OTHER_REQUIRED_LIBS CONFIGURE_DEPENDS - "${LLVM_INSTALL_PATH}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}z${CMAKE_STATIC_LIBRARY_SUFFIX}" - "${LLVM_INSTALL_PATH}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}zstd${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - target_link_libraries(llvm-libs INTERFACE ${LLVM_LIBRARIES} ${CLANG_LIBRARIES} ${OTHER_REQUIRED_LIBS}) + _download_llvm("${LLVM_VERSION}") + endif() + + set(LLVM_INSTALL_PATH "${LLVM_INSTALL_PATH}" CACHE PATH "Path to LLVM installation" FORCE) + + find_package(LLVM REQUIRED CONFIG + PATHS "${LLVM_INSTALL_PATH}/lib/cmake/llvm" NO_DEFAULT_PATH) + find_package(Clang REQUIRED CONFIG + PATHS "${LLVM_INSTALL_PATH}/lib/cmake/clang" NO_DEFAULT_PATH) + + llvm_map_components_to_libnames(LLVM_RESOLVED + support frontendopenmp option targetparser) + + add_library(llvm-libs INTERFACE IMPORTED) + target_link_libraries(llvm-libs INTERFACE + ${LLVM_RESOLVED} + clangAST clangASTMatchers clangBasic clangDriver clangOptions + clangFormat clangFrontend clangLex clangSema clangSerialization + clangTidy clangTidyUtils + clangTidyAbseilModule clangTidyAlteraModule clangTidyAndroidModule + clangTidyBoostModule clangTidyBugproneModule clangTidyCERTModule + clangTidyConcurrencyModule clangTidyCppCoreGuidelinesModule + clangTidyCustomModule clangTidyDarwinModule clangTidyFuchsiaModule + clangTidyGoogleModule clangTidyHICPPModule clangTidyLinuxKernelModule + clangTidyLLVMModule clangTidyLLVMLibcModule clangTidyMiscModule + clangTidyModernizeModule clangTidyMPIModule clangTidyObjCModule + clangTidyOpenMPModule clangTidyPerformanceModule + clangTidyPortabilityModule clangTidyReadabilityModule + clangTidyZirconModule + clangTooling clangToolingCore + clangToolingInclusions clangToolingInclusionsStdlib clangToolingSyntax + ) + + target_include_directories(llvm-libs INTERFACE + ${LLVM_INCLUDE_DIRS} ${CLANG_INCLUDE_DIRS}) + + if(NOT BUILD_SHARED_LIBS) target_compile_definitions(llvm-libs INTERFACE CLANG_BUILD_STATIC=1) endif() endfunction() diff --git a/cmake/package.cmake b/cmake/package.cmake index f3ef7b958..3ed1169e3 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -1,7 +1,7 @@ include_guard() include(${CMAKE_CURRENT_LIST_DIR}/llvm.cmake) -setup_llvm("21.1.8") +setup_llvm("22.1.4") # install dependencies include(FetchContent) diff --git a/config/llvm-manifest.json b/config/llvm-manifest.json index fd9eefeb9..9c355b348 100644 --- a/config/llvm-manifest.json +++ b/config/llvm-manifest.json @@ -1,142 +1,117 @@ -[ - { - "version": "21.1.8", - "filename": "aarch64-linux-gnu-releasedbg-lto.tar.xz", - "sha256": "f3444ee840b50933c23656cbee7c4d010e752ac55ca66095b97f7c0e997b13b5", - "lto": true, - "asan": false, - "platform": "linux", - "arch": "arm64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "aarch64-linux-gnu-releasedbg.tar.xz", - "sha256": "b9012bf059e4d8673fb564b5780e5fc78c6a2e47f5cc6a39f444d1879b42dd2a", - "lto": false, - "asan": false, - "platform": "linux", - "arch": "arm64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "aarch64-windows-msvc-releasedbg-lto.tar.xz", - "sha256": "8870d16141ba7f9ea12f5147b8d91329abbbaa4376cd4576667dd323d896dd08", - "lto": true, - "asan": false, - "platform": "windows", - "arch": "arm64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "aarch64-windows-msvc-releasedbg.tar.xz", - "sha256": "ad394e79ec85dd40f942671bb0342ffe54a103eb2baabacb773999d57d80134b", - "lto": false, - "asan": false, - "platform": "windows", - "arch": "arm64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "arm64-macos-clang-debug-asan.tar.xz", - "sha256": "b02d20e4f7294ee33f49a09dfdd765b3b44135e003ef50e3a760aeee39e3f993", - "lto": false, - "asan": true, - "platform": "macosx", - "arch": "arm64", - "build_type": "Debug" - }, - { - "version": "21.1.8", - "filename": "arm64-macos-clang-releasedbg-lto.tar.xz", - "sha256": "e40c21eb0d0b91d9d4ab31212a5cb01ea46707f5c29839414567857e4147604d", - "lto": true, - "asan": false, - "platform": "macosx", - "arch": "arm64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "arm64-macos-clang-releasedbg.tar.xz", - "sha256": "e1b01de34f0edfd41c118e4981a93afb35556ae369597e864f4a393db623b926", - "lto": false, - "asan": false, - "platform": "macosx", - "arch": "arm64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "x64-linux-gnu-debug-asan.tar.xz", - "sha256": "76bb82d822b5377fb5e0fac8abcfba125142e6a0acc02bb36d1fa1532a268646", - "lto": false, - "asan": true, - "platform": "linux", - "arch": "x64", - "build_type": "Debug" - }, - { - "version": "21.1.8", - "filename": "x64-linux-gnu-releasedbg-lto.tar.xz", - "sha256": "32f5edddec1e689124f045b586fb402ae30febc05203af7391b088bc8494cd53", - "lto": true, - "asan": false, - "platform": "linux", - "arch": "x64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "x64-linux-gnu-releasedbg.tar.xz", - "sha256": "8ba3c84f23a2a81a86c54780754a61adf99048aa2ac0dc9b9708d0f842d553de", - "lto": false, - "asan": false, - "platform": "linux", - "arch": "x64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "x64-macos-clang-releasedbg-lto.tar.xz", - "sha256": "97e81d6296896d7237f118f728d05291707b9e4e5791e07ce4be8aee0517505d", - "lto": true, - "asan": false, - "platform": "macosx", - "arch": "x64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "x64-macos-clang-releasedbg.tar.xz", - "sha256": "53c13f8e1082fa2fe2f9c05303de48cb3133bf5f24271f4b3062f1dec578159c", - "lto": false, - "asan": false, - "platform": "macosx", - "arch": "x64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "x64-windows-msvc-releasedbg-lto.tar.xz", - "sha256": "16bcf0e4cbc3d2b1204edd619a3837004dacea28eeff0a101c8d0212f936427d", - "lto": true, - "asan": false, - "platform": "windows", - "arch": "x64", - "build_type": "RelWithDebInfo" - }, - { - "version": "21.1.8", - "filename": "x64-windows-msvc-releasedbg.tar.xz", - "sha256": "81d31fad05e200726c8178314b0b2045c947483dddd8cb974f4c376ae5f441fa", - "lto": false, - "asan": false, - "platform": "windows", - "arch": "x64", - "build_type": "RelWithDebInfo" +{ + "version": "22.1.4", + "artifacts": { + "aarch64-linux-gnu-releasedbg-lto.tar.xz": { + "sha256": "dc25d1a6ca10d6f1faae8163f2ce328c0930a916bc4dc9763d16e9e57e87997f", + "lto": true, + "asan": false, + "platform": "linux", + "arch": "arm64", + "build_type": "RelWithDebInfo" + }, + "aarch64-linux-gnu-releasedbg.tar.xz": { + "sha256": "3b32f625730b67eb3303e51b1d430a6ec62fff74ea971e82534eec18a081850e", + "lto": false, + "asan": false, + "platform": "linux", + "arch": "arm64", + "build_type": "RelWithDebInfo" + }, + "aarch64-windows-msvc-releasedbg-lto.tar.xz": { + "sha256": "23170fe0632aa2b3a9c6f98e9989e4eadae54d1d2dd2676c5e9acff0670cb2f0", + "lto": true, + "asan": false, + "platform": "windows", + "arch": "arm64", + "build_type": "RelWithDebInfo" + }, + "aarch64-windows-msvc-releasedbg.tar.xz": { + "sha256": "67b86792734bdd755d2b4c6ada01ef5b7337954bb5ade510a37c2b79e858b1b1", + "lto": false, + "asan": false, + "platform": "windows", + "arch": "arm64", + "build_type": "RelWithDebInfo" + }, + "arm64-macos-clang-debug-asan.tar.xz": { + "sha256": "e42dd49fa93506c21a7f1d4f0890da16271b8af64acfc9b908242000613baf35", + "lto": false, + "asan": true, + "platform": "macosx", + "arch": "arm64", + "build_type": "Debug" + }, + "arm64-macos-clang-releasedbg-lto.tar.xz": { + "sha256": "f881ac42de055cd9e719a14408897fc935ddd9d98e5f2656ad26b6d220f4a0e8", + "lto": true, + "asan": false, + "platform": "macosx", + "arch": "arm64", + "build_type": "RelWithDebInfo" + }, + "arm64-macos-clang-releasedbg.tar.xz": { + "sha256": "65bd4b9a458e5587872761ff22abb443610b0e6bcbc1964e6e4225776fe1e19a", + "lto": false, + "asan": false, + "platform": "macosx", + "arch": "arm64", + "build_type": "RelWithDebInfo" + }, + "x64-linux-gnu-debug-asan.tar.xz": { + "sha256": "d2196f3dedb8e1ea9179d2066b0a7c2ca4ff50328f630b20ffbb3bf9b58b8b57", + "lto": false, + "asan": true, + "platform": "linux", + "arch": "x64", + "build_type": "Debug" + }, + "x64-linux-gnu-releasedbg-lto.tar.xz": { + "sha256": "dd36d45ad4e14bf8d3a66177598c140e3d5fff67e68e944cd944c55ae513c426", + "lto": true, + "asan": false, + "platform": "linux", + "arch": "x64", + "build_type": "RelWithDebInfo" + }, + "x64-linux-gnu-releasedbg.tar.xz": { + "sha256": "be92fd64d515db037765603c6ae899369fe2a2d42cc6b36c5bfa11d07d4b67ec", + "lto": false, + "asan": false, + "platform": "linux", + "arch": "x64", + "build_type": "RelWithDebInfo" + }, + "x64-macos-clang-releasedbg-lto.tar.xz": { + "sha256": "c8aa85561037a4e53d15ae97cdc9d81afe5e1f9e198560ab4ba7da23a6be4f91", + "lto": true, + "asan": false, + "platform": "macosx", + "arch": "x64", + "build_type": "RelWithDebInfo" + }, + "x64-macos-clang-releasedbg.tar.xz": { + "sha256": "9fd6bf02f347a45f688478ceda363388dc828edc6c8ad292f7d329e9dd010070", + "lto": false, + "asan": false, + "platform": "macosx", + "arch": "x64", + "build_type": "RelWithDebInfo" + }, + "x64-windows-msvc-releasedbg-lto.tar.xz": { + "sha256": "b3e7a70941af2f307bdeff166398bc319acc9d6fcf6df1cc0ba61424e0548376", + "lto": true, + "asan": false, + "platform": "windows", + "arch": "x64", + "build_type": "RelWithDebInfo" + }, + "x64-windows-msvc-releasedbg.tar.xz": { + "sha256": "ef67d91f49f586eba0fcd0722eed62524abdab921b4e71859aa8706689fd36b8", + "lto": false, + "asan": false, + "platform": "windows", + "arch": "x64", + "build_type": "RelWithDebInfo" + } } -] +} diff --git a/pixi.toml b/pixi.toml index 8c05d9e38..e4177ab12 100644 --- a/pixi.toml +++ b/pixi.toml @@ -191,15 +191,6 @@ depends-on = [ [tasks.build-llvm] cmd = ["python3", "scripts/build-llvm.py"] -[tasks.upload-llvm] -args = ["workflow_id", "llvm_version", { "arg" = "branch", default = "main" }] -cmd = """ -gh workflow run upload-llvm.yml \ - --ref {{ branch }} \ - --field workflow_id={{workflow_id}} \ - --field version={{llvm_version}} -""" - [tasks.delete-artifacts] args = ["file_name"] cmd = ["scripts/delete-artifacts.bash", "{{ file_name }}"] diff --git a/scripts/build-llvm.py b/scripts/build-llvm.py index f769d9068..f8ccf07a7 100644 --- a/scripts/build-llvm.py +++ b/scripts/build-llvm.py @@ -4,7 +4,6 @@ import shutil import argparse import os -import json from pathlib import Path @@ -157,9 +156,16 @@ def main(): print(f"Toolchain: {toolchain_file}") print("---------------------") - components_path = Path(__file__).resolve().parent / "llvm-components.json" - with components_path.open() as f: - llvm_distribution_components = json.load(f)["components"] + llvm_distribution_components = [ + "llvm-libraries", + "clang-libraries", + "llvm-headers", + "clang-headers", + "clang-tidy-headers", + "clang-resource-headers", + "cmake-exports", + "clang-cmake-exports", + ] components_joined = ";".join(llvm_distribution_components) cmake_args = [ diff --git a/scripts/download-llvm.sh b/scripts/download-llvm.sh deleted file mode 100755 index 3a640e031..000000000 --- a/scripts/download-llvm.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -if [[ $# -ne 1 ]]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -WORKFLOW_ID="$1" - -mkdir -p artifacts -gh run download "${WORKFLOW_ID}" --dir artifacts - -echo "Downloaded artifacts:" -find artifacts -maxdepth 4 -type f -print diff --git a/scripts/llvm-components.json b/scripts/llvm-components.json deleted file mode 100644 index 6578f04f6..000000000 --- a/scripts/llvm-components.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "components": [ - "LLVMDemangle", - "LLVMSupport", - "LLVMCore", - "LLVMOption", - "LLVMBinaryFormat", - "LLVMMC", - "LLVMMCParser", - "LLVMObject", - "LLVMProfileData", - "LLVMBitReader", - "LLVMBitstreamReader", - "LLVMRemarks", - "LLVMObjectYAML", - "LLVMAggressiveInstCombine", - "LLVMInstCombine", - "LLVMIRReader", - "LLVMTextAPI", - "LLVMSymbolize", - "LLVMDebugInfoDWARF", - "LLVMDebugInfoDWARFLowLevel", - "LLVMDebugInfoCodeView", - "LLVMDebugInfoGSYM", - "LLVMDebugInfoPDB", - "LLVMDebugInfoBTF", - "LLVMDebugInfoMSF", - "LLVMAsmParser", - "LLVMTargetParser", - "LLVMTransformUtils", - "LLVMAnalysis", - "LLVMScalarOpts", - "LLVMFrontendHLSL", - "LLVMFrontendOpenMP", - "LLVMFrontendOffloading", - "LLVMFrontendAtomic", - "LLVMFrontendDirective", - "LLVMWindowsDriver", - "clangIndex", - "clangAPINotes", - "clangAST", - "clangASTMatchers", - "clangBasic", - "clangDriver", - "clangFormat", - "clangFrontend", - "clangLex", - "clangParse", - "clangSema", - "clangSerialization", - "clangRewrite", - "clangAnalysis", - "clangEdit", - "clangSupport", - "clangStaticAnalyzerCore", - "clangStaticAnalyzerFrontend", - "clangTidy", - "clangTidyUtils", - "clangTidyAndroidModule", - "clangTidyAbseilModule", - "clangTidyAlteraModule", - "clangTidyBoostModule", - "clangTidyBugproneModule", - "clangTidyCERTModule", - "clangTidyConcurrencyModule", - "clangTidyCppCoreGuidelinesModule", - "clangTidyDarwinModule", - "clangTidyFuchsiaModule", - "clangTidyGoogleModule", - "clangTidyHICPPModule", - "clangTidyLinuxKernelModule", - "clangTidyLLVMModule", - "clangTidyLLVMLibcModule", - "clangTidyMiscModule", - "clangTidyModernizeModule", - "clangTidyObjCModule", - "clangTidyOpenMPModule", - "clangTidyPerformanceModule", - "clangTidyPortabilityModule", - "clangTidyReadabilityModule", - "clangTidyZirconModule", - "clangTooling", - "clangToolingCore", - "clangToolingInclusions", - "clangToolingInclusionsStdlib", - "clangToolingSyntax", - "clangToolingRefactoring", - "clangTransformer", - "clangCrossTU", - "clangAnalysisFlowSensitive", - "clangAnalysisFlowSensitiveModels", - "clangStaticAnalyzerCheckers", - "clangIncludeCleaner", - "llvm-headers", - "clang-headers", - "clang-tidy-headers", - "clang-resource-headers" - ] -} diff --git a/scripts/prune-llvm-bin.py b/scripts/prune-llvm-bin.py deleted file mode 100644 index 9a392156f..000000000 --- a/scripts/prune-llvm-bin.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python3 -""" -Prune unused LLVM binaries/libraries from the install tree. - -Two modes: - - discover: iteratively delete candidates and rebuild to confirm they are unused, - then write the deletion list to a manifest JSON file. - - apply: read the manifest and delete the listed files without rebuilding - (useful for LTO builds that trust the non-LTO pruning result). -""" - -import argparse -import json -import shutil -import subprocess -import time -from datetime import datetime, timezone -from pathlib import Path -from typing import Iterable, List, Optional - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Prune unused LLVM artifacts.") - parser.add_argument( - "--action", - choices=["discover", "apply"], - default="discover", - help="discover: probe deletable files and record them; apply: delete using manifest", - ) - parser.add_argument( - "--install-dir", - type=Path, - default=Path(".llvm/build-install/lib"), - help="Path to the LLVM install lib directory (default: .llvm/build-install/lib)", - ) - parser.add_argument( - "--build-dir", - type=Path, - default=Path("build"), - help="CMake build directory used for validation builds (default: build)", - ) - parser.add_argument( - "--manifest", - type=Path, - default=Path("pruned-libs.json"), - help="Path to manifest JSON for recording or applying deletions", - ) - parser.add_argument( - "--gh-run-id", - type=str, - help="GitHub run ID to download manifest from when applying", - ) - parser.add_argument( - "--gh-artifact", - type=str, - default="llvm-pruned-libs", - help="Artifact name/pattern that contains the manifest", - ) - parser.add_argument( - "--gh-download-dir", - type=Path, - default=Path("artifacts"), - help="Directory to place downloaded artifacts", - ) - parser.add_argument( - "--max-attempts", - type=int, - default=30, - help="Maximum attempts when waiting for manifest download", - ) - parser.add_argument( - "--sleep-seconds", - type=int, - default=60, - help="Seconds to sleep between manifest download attempts", - ) - return parser.parse_args() - - -def _print_failure_output( - result: subprocess.CalledProcessError, max_lines: int = 50 -) -> None: - stdout = getattr(result, "stdout", "") or "" - stderr = getattr(result, "stderr", "") or "" - combined = stdout + stderr - if not combined: - return - print("Build output (last lines):") - lines = combined.splitlines() - for line in lines[-max_lines:]: - print(line) - - -def run_build(build_dir: Path) -> bool: - cmd = ["cmake", "--build", str(build_dir)] - print(f"Running: {' '.join(cmd)}") - try: - subprocess.run(cmd, check=True, capture_output=True, text=True) - return True - except subprocess.CalledProcessError as exc: - _print_failure_output(exc) - return False - - -def candidate_files(install_dir: Path) -> Iterable[Path]: - if not install_dir.is_dir(): - raise FileNotFoundError(f"lib dir not found: {install_dir}") - for path in sorted(install_dir.iterdir()): - if not path.is_file(): - continue - if path.suffix.lower() in {".a", ".lib"}: - yield path - else: - print(f"Skipping non-static file: {path.name}") - - -def try_delete(path: Path, build_dir: Path) -> bool: - backup = path.with_suffix(path.suffix + ".bak") - print(f"Testing deletion: {path}") - shutil.move(path, backup) - success = run_build(build_dir) - if success: - backup.unlink(missing_ok=True) - print(f"Safe to delete: {path.name}") - return True - shutil.move(backup, path) - print(f"Required; restored: {path.name}") - return False - - -def discover(install_dir: Path, build_dir: Path) -> List[str]: - deletable: List[str] = [] - for path in candidate_files(install_dir): - if try_delete(path, build_dir): - deletable.append(path.name) - return deletable - - -def write_manifest( - manifest: Path, removed: List[str], install_dir: Path, build_dir: Path -) -> None: - data = { - "generated_at": datetime.now(timezone.utc).isoformat(), - "install_dir": str(install_dir), - "build_dir": str(build_dir), - "removed": removed, - } - manifest.write_text(json.dumps(data, indent=2)) - print(f"Wrote manifest with {len(removed)} entries to {manifest}") - - -def apply_manifest(manifest: Path, install_dir: Path) -> None: - if not manifest.is_file(): - raise FileNotFoundError(f"Manifest not found: {manifest}") - data = json.loads(manifest.read_text()) - removed = data.get("removed", []) - if not isinstance(removed, list): - raise ValueError("Manifest missing 'removed' list") - for name in removed: - target = install_dir / name - if target.exists(): - print(f"Deleting {target}") - target.unlink() - else: - print(f"Already absent: {target}") - - -def find_manifest(download_dir: Path, manifest_name: str) -> Optional[Path]: - for path in download_dir.rglob(manifest_name): - if path.is_file(): - return path - return None - - -def wait_and_download_manifest( - run_id: str, - artifact: str, - download_dir: Path, - manifest_name: str, - max_attempts: int, - sleep_seconds: int, -) -> Path: - download_dir.mkdir(parents=True, exist_ok=True) - for attempt in range(1, max_attempts + 1): - print( - f"[{attempt}/{max_attempts}] Downloading manifest via gh run download " - f"(run={run_id}, artifact={artifact})" - ) - cmd = [ - "gh", - "run", - "download", - str(run_id), - "--pattern", - artifact, - "--dir", - str(download_dir), - ] - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode == 0: - found = find_manifest(download_dir, manifest_name) - if found: - print(f"Found manifest at {found}") - return found - print("Download succeeded but manifest not found; retrying...") - else: - print("gh run download failed; stderr follows:") - print(result.stderr) - if attempt < max_attempts: - time.sleep(sleep_seconds) - raise RuntimeError("Manifest could not be downloaded within the allotted attempts") - - -def ensure_manifest( - manifest: Path, - run_id: Optional[str], - artifact: str, - download_dir: Path, - max_attempts: int, - sleep_seconds: int, -) -> Path: - if manifest.exists(): - return manifest - if not run_id: - raise FileNotFoundError( - f"Manifest {manifest} missing and no gh run ID provided for download" - ) - downloaded = wait_and_download_manifest( - run_id=run_id, - artifact=artifact, - download_dir=download_dir, - manifest_name=manifest.name, - max_attempts=max_attempts, - sleep_seconds=sleep_seconds, - ) - if downloaded != manifest: - shutil.copy(downloaded, manifest) - return manifest - - -def main() -> None: - args = parse_args() - install_dir = args.install_dir - build_dir = args.build_dir - if args.action == "discover": - deletable = discover(install_dir, build_dir) - write_manifest(args.manifest, deletable, install_dir, build_dir) - else: - manifest = ensure_manifest( - manifest=args.manifest, - run_id=args.gh_run_id, - artifact=args.gh_artifact, - download_dir=args.gh_download_dir, - max_attempts=args.max_attempts, - sleep_seconds=args.sleep_seconds, - ) - apply_manifest(manifest, install_dir) - - -if __name__ == "__main__": - main() diff --git a/scripts/release-llvm.py b/scripts/release-llvm.py new file mode 100644 index 000000000..caa75336c --- /dev/null +++ b/scripts/release-llvm.py @@ -0,0 +1,549 @@ +#!/usr/bin/env python3 +""" +LLVM release pipeline utilities. + +Actions: + discover: Iteratively probe which static libs can be removed by deleting + and rebuilding, then write the result to a manifest JSON. + apply: Read a manifest and replace listed libs with empty archives. + repackage: Download all LLVM build artifacts, apply pruning, and repackage + them in parallel. +""" + +import argparse +import concurrent.futures +import fnmatch +import hashlib +import json +import shutil +import subprocess +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterable, List, Optional + +PLATFORM_INFO = { + "linux": {"toolchain": "gnu", "arm_arch": "aarch64"}, + "macos": {"toolchain": "clang", "arm_arch": "arm64"}, + "windows": {"toolchain": "msvc", "arm_arch": "aarch64"}, +} + + +def build_artifact_name( + platform: str, + arch: str, + mode: str, + *, + lto: bool = False, + asan: bool = False, +) -> str: + info = PLATFORM_INFO.get(platform) + if not info: + raise ValueError(f"Unknown platform: {platform}") + toolchain = info["toolchain"] + mode_tag = "debug" if mode == "Debug" else "releasedbg" + suffix = "" + if lto: + suffix += "-lto" + if asan: + suffix += "-asan" + return f"{arch}-{platform}-{toolchain}-{mode_tag}{suffix}.tar.xz" + + +ARTIFACTS = [ + build_artifact_name(p, a, m, lto=l, asan=s) + for p, a, m, l, s in [ + ("linux", "aarch64", "RelWithDebInfo", True, False), + ("linux", "aarch64", "RelWithDebInfo", False, False), + ("windows", "aarch64", "RelWithDebInfo", True, False), + ("windows", "aarch64", "RelWithDebInfo", False, False), + ("macos", "arm64", "Debug", False, True), + ("macos", "arm64", "RelWithDebInfo", True, False), + ("macos", "arm64", "RelWithDebInfo", False, False), + ("linux", "x64", "Debug", False, True), + ("linux", "x64", "RelWithDebInfo", True, False), + ("linux", "x64", "RelWithDebInfo", False, False), + ("macos", "x64", "RelWithDebInfo", True, False), + ("macos", "x64", "RelWithDebInfo", False, False), + ("windows", "x64", "RelWithDebInfo", True, False), + ("windows", "x64", "RelWithDebInfo", False, False), + ] +] + +MANIFEST_DIRS = { + "linux": "prune-manifest-ubuntu-24.04", + "macos": "prune-manifest-macos-15", + "windows": "prune-manifest-windows-2025", +} + +ARCHIVE_MAGIC = b"!\n" + + +def _is_shared_lib(path: Path) -> bool: + return ".so" in path.suffixes or ".dylib" in path.suffixes + + +def _replace_with_empty_archive(path: Path) -> None: + if _is_shared_lib(path): + path.write_bytes(b"") + else: + path.write_bytes(ARCHIVE_MAGIC) + + +def _remove_binaries(build_dir: Path) -> None: + bin_dir = build_dir / "bin" + if not bin_dir.is_dir(): + return + for f in bin_dir.iterdir(): + if f.is_file() and f.suffix in {"", ".exe"}: + f.unlink() + + +def _run_build(build_dir: Path) -> bool: + try: + subprocess.run( + ["cmake", "--build", str(build_dir)], + check=True, + capture_output=True, + text=True, + ) + return True + except subprocess.CalledProcessError as exc: + combined = (exc.stdout or "") + (exc.stderr or "") + if combined: + print("Build output (last lines):") + for line in combined.splitlines()[-50:]: + print(line) + return False + + +def _manifest_for(artifact: str, manifests_dir: Path) -> Optional[Path]: + for platform, dirname in MANIFEST_DIRS.items(): + if platform in artifact: + return manifests_dir / dirname / "pruned-libs.json" + return None + + +# ── discover ───────────────────────────────────────────────────────── + + +def _candidate_files( + install_dir: Path, skip_patterns: Optional[List[str]] = None +) -> Iterable[Path]: + if not install_dir.is_dir(): + raise FileNotFoundError(f"lib dir not found: {install_dir}") + for path in sorted(install_dir.iterdir()): + if not path.is_file(): + continue + if path.suffix.lower() not in {".a", ".lib"}: + print(f"Skipping non-static file: {path.name}") + continue + if skip_patterns and any(fnmatch.fnmatch(path.name, p) for p in skip_patterns): + print(f"Skipping (never-prune): {path.name}") + continue + yield path + + +def _nullify_shared_libs(install_dir: Path) -> List[dict]: + nullified: List[dict] = [] + for path in sorted(install_dir.iterdir()): + if not path.is_file() or not _is_shared_lib(path): + continue + size = path.stat().st_size + if size > 0: + print(f"Nullifying shared lib: {path.name} ({size} bytes)") + path.write_bytes(b"") + nullified.append({"name": path.name, "size": size}) + return nullified + + +def _try_delete(path: Path, build_dir: Path) -> Optional[int]: + size = path.stat().st_size + backup = path.with_suffix(path.suffix + ".bak") + print(f"Testing deletion: {path}") + shutil.move(path, backup) + _remove_binaries(build_dir) + if _run_build(build_dir): + backup.unlink(missing_ok=True) + print(f"Safe to delete: {path.name} ({size} bytes)") + return size + shutil.move(backup, path) + print(f"Required; restored: {path.name}") + return None + + +def discover( + install_dir: Path, + build_dir: Path, + skip_patterns: Optional[List[str]] = None, +) -> List[dict]: + nullified = _nullify_shared_libs(install_dir) + deletable: List[dict] = [] + for path in _candidate_files(install_dir, skip_patterns): + size = _try_delete(path, build_dir) + if size is not None: + deletable.append({"name": path.name, "size": size}) + return nullified + deletable + + +def _write_manifest( + manifest: Path, removed: List[dict], install_dir: Path, build_dir: Path +) -> None: + total_size = sum(e["size"] for e in removed) + data = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "install_dir": str(install_dir), + "build_dir": str(build_dir), + "total_saved_bytes": total_size, + "removed": removed, + } + manifest.write_text(json.dumps(data, indent=2)) + print(f"Wrote manifest with {len(removed)} entries to {manifest}") + print(f"Total space saved: {total_size / 1048576:.1f} MB") + + +# ── apply ──────────────────────────────────────────────────────────── + + +def apply_manifest(manifest: Path, install_dir: Path) -> None: + if not manifest.is_file(): + raise FileNotFoundError(f"Manifest not found: {manifest}") + data = json.loads(manifest.read_text()) + removed = data.get("removed", []) + if not isinstance(removed, list): + raise ValueError("Manifest missing 'removed' list") + for entry in removed: + name = entry["name"] if isinstance(entry, dict) else entry + target = install_dir / name + if target.exists(): + _replace_with_empty_archive(target) + else: + print(f"Already absent: {target}") + + +# ── metadata ───────────────────────────────────────────────────────── + + +def sha256sum(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def build_metadata_entry(path: Path, version: str) -> dict: + name = path.name.lower() + if "windows" in name: + platform = "windows" + elif "linux" in name: + platform = "linux" + elif "macos" in name: + platform = "macosx" + else: + platform = "unknown" + + if name.startswith("aarch64-") or name.startswith("arm64-"): + arch = "arm64" + elif name.startswith("x64-") or name.startswith("x86_64-"): + arch = "x64" + else: + arch = "unknown" + + return { + "version": version, + "filename": path.name, + "sha256": sha256sum(path), + "lto": "-lto" in name, + "asan": "-asan" in name, + "platform": platform, + "arch": arch, + "build_type": "Debug" if "debug" in name else "RelWithDebInfo", + } + + +# ── repackage ──────────────────────────────────────────────────────── + + +def _compress_tar_xz( + source_dir: Path, + output_path: Path, + xz_level: str, + label: str, +) -> int: + print(f"[{label}] Compressing (xz {xz_level})...", flush=True) + with output_path.open("wb") as out: + tar = subprocess.Popen( + ["tar", "-C", str(source_dir), "-cf", "-", "."], + stdout=subprocess.PIPE, + ) + xz = subprocess.Popen( + ["xz", "-T0", xz_level, "-c"], + stdin=tar.stdout, + stdout=out, + ) + tar.stdout.close() + xz.communicate() + tar.wait() + if tar.returncode != 0 or xz.returncode != 0: + raise RuntimeError( + f"tar/xz failed (tar={tar.returncode}, xz={xz.returncode})" + ) + return output_path.stat().st_size + + +def _process_artifact( + artifact: str, + source_run_id: str, + manifests_dir: Path, + output_dir: Path, + version: Optional[str] = None, +) -> None: + workdir = Path(tempfile.mkdtemp(prefix="repackage-")) + try: + dl_dir = workdir / "dl" + content_dir = workdir / "content" + + print(f"[{artifact}] Downloading...", flush=True) + subprocess.run( + ["gh", "run", "download", source_run_id, "-n", artifact, "-D", str(dl_dir)], + check=True, + ) + + print(f"[{artifact}] Extracting...", flush=True) + content_dir.mkdir() + subprocess.run( + [ + "tar", + "--strip-components=1", + "-xf", + str(dl_dir / artifact), + "-C", + str(content_dir), + ], + check=True, + ) + shutil.rmtree(dl_dir) + + if "debug" in artifact or "asan" in artifact: + print(f"[{artifact}] Debug/ASAN — skipping prune", flush=True) + else: + manifest = _manifest_for(artifact, manifests_dir) + if not manifest: + raise RuntimeError(f"No manifest mapping for artifact: {artifact}") + if not manifest.is_file(): + raise FileNotFoundError(f"Prune manifest missing: {manifest}") + print(f"[{artifact}] Pruning...", flush=True) + apply_manifest(manifest, content_dir / "lib") + + output_path = output_dir / artifact + file_size = _compress_tar_xz(content_dir, output_path, "-9e", artifact) + + print(f"[{artifact}] Done ({file_size / 1048576:.1f} MB)", flush=True) + + if version: + meta = build_metadata_entry(output_path, version) + meta_path = output_dir / f"{artifact}.meta.json" + meta_path.write_text(json.dumps(meta, indent=2)) + finally: + shutil.rmtree(workdir, ignore_errors=True) + + +def repackage( + source_run_id: str, + manifests_dir: Path, + output_dir: Path, + max_parallel: int = 3, + artifacts: Optional[List[str]] = None, + version: Optional[str] = None, +) -> None: + targets = artifacts if artifacts else ARTIFACTS + output_dir.mkdir(parents=True, exist_ok=True) + + failed: List[str] = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=max_parallel) as pool: + futures = { + pool.submit( + _process_artifact, a, source_run_id, manifests_dir, output_dir, version + ): a + for a in targets + } + for future in concurrent.futures.as_completed(futures): + artifact = futures[future] + try: + future.result() + except Exception as exc: + print(f"[{artifact}] FAILED: {exc}", flush=True) + failed.append(artifact) + + if failed: + print(f"\nFailed artifacts ({len(failed)}):") + for name in failed: + print(f" {name}") + sys.exit(1) + + print(f"\nAll {len(targets)} artifacts repackaged:") + for path in sorted(output_dir.iterdir()): + if path.is_file() and path.suffix != ".json": + print(f" {path.stat().st_size / 1048576:>8.1f} MB {path.name}") + + +def _find_manifest(download_dir: Path, manifest_name: str) -> Optional[Path]: + for path in download_dir.rglob(manifest_name): + if path.is_file(): + return path + return None + + +def _wait_and_download_manifest( + run_id: str, + artifact: str, + download_dir: Path, + manifest_name: str, + max_attempts: int, + sleep_seconds: int, +) -> Path: + import time + + download_dir.mkdir(parents=True, exist_ok=True) + for attempt in range(1, max_attempts + 1): + print( + f"[{attempt}/{max_attempts}] Downloading manifest " + f"(run={run_id}, artifact={artifact})" + ) + result = subprocess.run( + [ + "gh", + "run", + "download", + str(run_id), + "--pattern", + artifact, + "--dir", + str(download_dir), + ], + capture_output=True, + text=True, + ) + if result.returncode == 0: + found = _find_manifest(download_dir, manifest_name) + if found: + print(f"Found manifest at {found}") + return found + print("Download succeeded but manifest not found; retrying...") + else: + print(f"gh run download failed: {result.stderr.strip()}") + if attempt < max_attempts: + time.sleep(sleep_seconds) + raise RuntimeError("Manifest could not be downloaded within the allotted attempts") + + +def _ensure_manifest( + manifest: Path, + run_id: Optional[str], + artifact: str, + download_dir: Path, + max_attempts: int, + sleep_seconds: int, +) -> Path: + if manifest.exists(): + return manifest + if not run_id: + raise FileNotFoundError( + f"Manifest {manifest} missing and no gh run ID provided" + ) + downloaded = _wait_and_download_manifest( + run_id, + artifact, + download_dir, + manifest.name, + max_attempts, + sleep_seconds, + ) + if downloaded != manifest: + shutil.copy(downloaded, manifest) + return manifest + + +# ── CLI ────────────────────────────────────────────────────────────── + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="LLVM release pipeline utilities.") + sub = parser.add_subparsers(dest="action", required=True) + + dp = sub.add_parser("discover", help="Probe which static libs can be removed") + dp.add_argument("--install-dir", type=Path, required=True) + dp.add_argument("--build-dir", type=Path, required=True) + dp.add_argument("--manifest", type=Path, required=True) + dp.add_argument("--skip-pattern", action="append", default=[]) + + ap = sub.add_parser("apply", help="Replace listed libs with empty archives") + ap.add_argument("--manifest", type=Path, required=True) + ap.add_argument("--install-dir", type=Path, required=True) + ap.add_argument("--gh-run-id", type=str) + ap.add_argument("--gh-artifact", type=str, default="llvm-pruned-libs") + ap.add_argument("--gh-download-dir", type=Path, default=Path("artifacts")) + ap.add_argument("--max-attempts", type=int, default=30) + ap.add_argument("--sleep-seconds", type=int, default=60) + + rp = sub.add_parser("repackage", help="Download, prune, and repackage artifacts") + rp.add_argument("--source-run-id", type=str, required=True) + rp.add_argument("--manifests-dir", type=Path, required=True) + rp.add_argument("--output-dir", type=Path, required=True) + rp.add_argument("--max-parallel", type=int, default=3) + rp.add_argument("--artifacts", nargs="*") + rp.add_argument("--version", type=str) + + np = sub.add_parser( + "artifact-name", help="Print the artifact filename for given params" + ) + np.add_argument("--platform", required=True, choices=["linux", "macos", "windows"]) + np.add_argument("--arch", required=True, choices=["x64", "arm64", "aarch64"]) + np.add_argument("--mode", required=True, choices=["Debug", "RelWithDebInfo"]) + np.add_argument("--lto", action="store_true") + np.add_argument("--asan", action="store_true") + + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + if args.action == "discover": + removed = discover(args.install_dir, args.build_dir, args.skip_pattern) + _write_manifest(args.manifest, removed, args.install_dir, args.build_dir) + elif args.action == "apply": + manifest = _ensure_manifest( + manifest=args.manifest, + run_id=args.gh_run_id, + artifact=args.gh_artifact, + download_dir=args.gh_download_dir, + max_attempts=args.max_attempts, + sleep_seconds=args.sleep_seconds, + ) + apply_manifest(manifest, args.install_dir) + elif args.action == "repackage": + repackage( + source_run_id=args.source_run_id, + manifests_dir=args.manifests_dir, + output_dir=args.output_dir, + max_parallel=args.max_parallel, + artifacts=args.artifacts, + version=args.version, + ) + elif args.action == "artifact-name": + print( + build_artifact_name( + args.platform, + args.arch, + args.mode, + lto=args.lto, + asan=args.asan, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/setup-llvm.py b/scripts/setup-llvm.py deleted file mode 100644 index 8c530c422..000000000 --- a/scripts/setup-llvm.py +++ /dev/null @@ -1,405 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import hashlib -import json -import os -import shutil -import subprocess -import sys -import tarfile -from pathlib import Path -from urllib.error import HTTPError, URLError -from urllib.request import Request, urlopen - - -PRIVATE_CLANG_FILES = [ - "Sema/CoroutineStmtBuilder.h", - "Sema/TypeLocBuilder.h", - "Sema/TreeTransform.h", -] - - -def log(message: str) -> None: - print(f"[setup-llvm] {message}", flush=True) - - -def read_manifest(path: Path) -> list[dict]: - with path.open("r", encoding="utf-8") as handle: - return json.load(handle) - - -def detect_platform() -> str: - plat = sys.platform - if plat.startswith("win"): - return "Windows" - if plat == "darwin": - return "macosx" - if plat.startswith("linux"): - return "Linux" - raise RuntimeError(f"Unsupported platform: {plat}") - - -def detect_arch() -> str: - import platform - - machine = platform.machine().lower() - if machine in ("x86_64", "amd64"): - return "x64" - if machine in ("aarch64", "arm64"): - return "arm64" - raise RuntimeError(f"Unsupported architecture: {machine}") - - -def pick_artifact( - manifest: list[dict], - version: str, - build_type: str, - is_lto: bool, - platform: str, - arch: str, -) -> dict: - base_version = version.split("+", 1)[0] - saw_missing_arch = False - for entry in manifest: - if entry.get("version") != version: - continue - if entry.get("platform") != platform.lower(): - continue - entry_arch = entry.get("arch") - if entry_arch is None: - saw_missing_arch = True - continue - if entry_arch != arch: - continue - if entry.get("build_type") != build_type: - continue - if bool(entry.get("lto")) != is_lto: - continue - return entry - if saw_missing_arch: - raise RuntimeError( - f"Manifest contains entries without an 'arch' field for version={base_version}, " - f"platform={platform}. The manifest format changed to require explicit " - f"architectures; regenerate it via scripts/update-llvm-version.py." - ) - raise RuntimeError( - f"No matching LLVM artifact in manifest for version={base_version}, platform={platform}, " - f"arch={arch}, build_type={build_type}, lto={is_lto}" - ) - - -def sha256sum(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def download(url: str, dest: Path, token: str | None) -> None: - log(f"Start download: {url} -> {dest}") - dest.parent.mkdir(parents=True, exist_ok=True) - headers = {"User-Agent": "clice-setup-llvm"} - if token: - headers["Authorization"] = f"Bearer {token}" - request = Request(url, headers=headers) - try: - with urlopen(request) as response, dest.open("wb") as handle: - total_bytes = response.length - if total_bytes is None: - header_len = response.getheader("Content-Length") - if header_len and header_len.isdigit(): - total_bytes = int(header_len) - downloaded = 0 - next_percent = 10 - next_unknown_mark = 10 * 1024 * 1024 # 10MB steps when size is unknown - while True: - chunk = response.read(1024 * 512) - if not chunk: - break - handle.write(chunk) - downloaded += len(chunk) - if total_bytes: - percent = int(downloaded * 100 / total_bytes) - while percent >= next_percent and next_percent <= 100: - log( - f"Download progress: {next_percent}% " - f"({downloaded / 1024 / 1024:.1f}MB/" - f"{total_bytes / 1024 / 1024:.1f}MB)" - ) - next_percent += 10 - else: - if downloaded >= next_unknown_mark: - log( - f"Downloaded {downloaded / 1024 / 1024:.1f}MB (size unknown)" - ) - next_unknown_mark += 10 * 1024 * 1024 - if total_bytes and next_percent <= 100: - log("Download progress: 100% (size verified by server)") - log(f"Finished download: {dest} ({downloaded / 1024 / 1024:.1f}MB)") - except HTTPError as err: - raise RuntimeError(f"HTTP error {err.code} while downloading {url}") from err - except URLError as err: - raise RuntimeError(f"Failed to download {url}: {err.reason}") from err - - -def ensure_download( - url: str, dest: Path, expected_sha256: str, token: str | None -) -> None: - if dest.exists(): - current = sha256sum(dest) - if current == expected_sha256: - return - dest.unlink() - download(url, dest, token) - current = sha256sum(dest) - if current != expected_sha256: - dest.unlink(missing_ok=True) - raise RuntimeError( - f"SHA256 mismatch for {dest.name}: expected {expected_sha256}, got {current}" - ) - - -def extract_archive(archive: Path, dest_dir: Path) -> None: - log(f"Extracting {archive.name} to {dest_dir}") - dest_dir.mkdir(parents=True, exist_ok=True) - name = archive.name.lower() - if name.endswith(".tar.xz") or name.endswith(".tar.gz") or name.endswith(".tar"): - with tarfile.open(archive, "r:*") as tar: - tar.extractall(path=dest_dir) - log("Extraction complete") - return - raise RuntimeError(f"Unsupported archive format: {archive}") - - -def flatten_install_dir(dest_dir: Path) -> None: - # Some archives add an extra root directory (llvm-install, build-install, etc.). - for name in ("llvm-install", "build-install"): - nested = dest_dir / name - if not nested.is_dir(): - continue - log(f"Flattening nested install directory: {nested}") - for entry in nested.iterdir(): - target = dest_dir / entry.name - if target.exists(): - raise RuntimeError( - f"Cannot flatten {nested}: target already exists: {target}" - ) - shutil.move(str(entry), str(target)) - nested.rmdir() - break - - -def parse_version_tuple(text: str) -> tuple[int, ...]: - digits = [] - current = "" - for ch in text: - if ch.isdigit(): - current += ch - else: - if current: - digits.append(int(current)) - current = "" - if ch in {".", "-"}: - continue - if current: - digits.append(int(current)) - return tuple(digits) - - -def system_llvm_ok(required_version: str, build_type: str) -> Path | None: - if build_type.lower().startswith("debug"): - return None - llvm_config = shutil.which("llvm-config") - if not llvm_config: - return None - try: - version = subprocess.check_output([llvm_config, "--version"], text=True).strip() - prefix = subprocess.check_output([llvm_config, "--prefix"], text=True).strip() - except (subprocess.CalledProcessError, OSError): - return None - required = parse_version_tuple(required_version.split("+", 1)[0]) - found = parse_version_tuple(version) - if not found or found < required: - return None - return Path(prefix) - - -def github_api(url: str, token: str | None) -> dict: - headers = { - "Accept": "application/vnd.github+json", - "User-Agent": "clice-setup-llvm", - } - if token: - headers["Authorization"] = f"Bearer {token}" - request = Request(url, headers=headers) - with urlopen(request) as response: - return json.load(response) - - -def lookup_llvm_commit(version: str, token: str | None) -> str | None: - tag_version = version.split("+", 1)[0] - tag = f"llvmorg-{tag_version}" - ref_url = f"https://api.github.com/repos/llvm/llvm-project/git/ref/tags/{tag}" - try: - ref = github_api(ref_url, token) - except Exception: - return None - obj = ref.get("object") or {} - obj_type = obj.get("type") - obj_sha = obj.get("sha") - if obj_type == "commit": - return obj_sha - if obj_type == "tag" and obj_sha: - tag_url = f"https://api.github.com/repos/llvm/llvm-project/git/tags/{obj_sha}" - try: - tag_info = github_api(tag_url, token) - except Exception: - return None - return tag_info.get("object", {}).get("sha") or tag_info.get("sha") - return None - - -def ensure_private_headers( - install_path: Path, work_dir: Path, version: str, token: str | None, offline: bool -) -> None: - missing = [] - for rel in PRIVATE_CLANG_FILES: - if (install_path / "include" / "clang" / rel).exists(): - continue - if (work_dir / "include" / "clang" / rel).exists(): - continue - missing.append(rel) - if not missing or offline: - return - commit = lookup_llvm_commit(version, token) - if not commit: - return - for rel in missing: - dest = work_dir / "include" / "clang" / rel - dest.parent.mkdir(parents=True, exist_ok=True) - url = f"https://raw.githubusercontent.com/llvm/llvm-project/{commit}/clang/lib/{rel}" - log(f"Fetching private header: {url}") - download(url, dest, token) - - -def main() -> None: - parser = argparse.ArgumentParser(description="Setup LLVM dependencies for CMake") - parser.add_argument("--version", required=True) - parser.add_argument("--build-type", required=True) - parser.add_argument("--binary-dir", required=True) - parser.add_argument("--manifest", required=True) - parser.add_argument("--install-path") - parser.add_argument("--enable-lto", action="store_true") - parser.add_argument("--offline", action="store_true") - parser.add_argument( - "--target-platform", - help="Override platform for cross-compilation (e.g. macosx, linux, windows)", - ) - parser.add_argument( - "--target-arch", - help="Override architecture for cross-compilation (e.g. x64, arm64)", - ) - parser.add_argument("--output", required=True) - args = parser.parse_args() - - log( - "Args: " - f"version={args.version}, build_type={args.build_type}, " - f"binary_dir={args.binary_dir}, install_path={args.install_path or '(auto)'}, " - f"enable_lto={args.enable_lto}, offline={args.offline}" - ) - token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") - build_type = args.build_type - platform_name = args.target_platform if args.target_platform else detect_platform() - arch_name = args.target_arch if args.target_arch else detect_arch() - log( - f"Platform: {platform_name}, arch: {arch_name}, normalized build type: {build_type}" - ) - manifest = read_manifest(Path(args.manifest)) - - binary_dir = Path(args.binary_dir).resolve() - install_root = binary_dir / ".llvm" - - install_path: Path | None = None - needs_install = False - if args.install_path: - candidate = Path(args.install_path) - if candidate.exists(): - log(f"Using provided LLVM install at {candidate}") - else: - log( - f"Provided LLVM install path does not exist; will install to {candidate}" - ) - needs_install = True - install_path = candidate - else: - detected = system_llvm_ok(args.version, build_type) - if detected: - log(f"Found suitable system LLVM at {detected}") - install_path = detected - - artifact = None - if install_path is None: - needs_install = True - artifact = pick_artifact( - manifest, - args.version, - build_type, - args.enable_lto, - platform_name, - arch_name, - ) - log(f"Selected artifact: {artifact.get('filename')} for download") - filename = artifact["filename"] - url_version = args.version.replace("+", "%2B") - url = f"https://github.com/clice-io/clice-llvm/releases/download/{url_version}/{filename}" - download_path = binary_dir / filename - ensure_download(url, download_path, artifact["sha256"], token) - extract_archive(download_path, install_root) - flatten_install_dir(install_root) - install_path = install_root - elif needs_install: - artifact = pick_artifact( - manifest, - args.version, - build_type, - args.enable_lto, - platform_name, - arch_name, - ) - log(f"Selected artifact: {artifact.get('filename')} for download") - filename = artifact["filename"] - url_version = args.version.replace("+", "%2B") - url = f"https://github.com/clice-io/clice-llvm/releases/download/{url_version}/{filename}" - download_path = binary_dir / filename - ensure_download(url, download_path, artifact["sha256"], token) - target_dir = install_path.resolve() - extract_archive(download_path, target_dir) - flatten_install_dir(target_dir) - install_path = target_dir - else: - install_path = install_path.resolve() - log(f"Using existing LLVM install at {install_path}") - - cmake_dir = install_path / "lib" / "cmake" / "llvm" - ensure_private_headers(install_path, binary_dir, args.version, token, args.offline) - - output = Path(args.output) - output.parent.mkdir(parents=True, exist_ok=True) - with output.open("w", encoding="utf-8") as handle: - json.dump( - { - "install_path": str(install_path), - "cmake_dir": str(cmake_dir), - "artifact": artifact or {}, - }, - handle, - indent=2, - ) - handle.write("\n") - - -if __name__ == "__main__": - main() diff --git a/scripts/update-llvm-version.py b/scripts/update-llvm-version.py index c18f84355..e19876fb2 100755 --- a/scripts/update-llvm-version.py +++ b/scripts/update-llvm-version.py @@ -16,8 +16,10 @@ def copy_manifest(src: Path, dest: Path) -> None: print(f"Error: {src} is not valid JSON: {err}", file=sys.stderr) sys.exit(1) - if not isinstance(data, list) or len(data) == 0: - print(f"Error: {src} must be a non-empty JSON array", file=sys.stderr) + if not isinstance(data, dict) or "artifacts" not in data: + print( + f"Error: {src} must be a JSON object with 'artifacts' key", file=sys.stderr + ) sys.exit(1) dest.parent.mkdir(parents=True, exist_ok=True) @@ -25,7 +27,7 @@ def copy_manifest(src: Path, dest: Path) -> None: json.dump(data, handle, indent=2) handle.write("\n") - print(f"Copied manifest: {src} -> {dest} ({len(data)} entries)") + print(f"Copied manifest: {src} -> {dest} ({len(data['artifacts'])} artifacts)") def update_package_cmake(path: Path, version: str) -> None: @@ -77,25 +79,26 @@ def check_package_cmake(path: Path) -> None: def check_manifest(path: Path) -> None: - """Verify the manifest is a well-formed non-empty array with required fields.""" + """Verify the manifest is well-formed with required fields.""" try: data = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as err: print(f"Error: {path} is not valid JSON: {err}", file=sys.stderr) sys.exit(1) - if not isinstance(data, list) or len(data) == 0: - print(f"Error: {path} must be a non-empty JSON array", file=sys.stderr) + if not isinstance(data, dict) or "artifacts" not in data: + print( + f"Error: {path} must be a JSON object with 'artifacts' key", file=sys.stderr + ) + sys.exit(1) + artifacts = data["artifacts"] + if not artifacts: + print(f"Error: {path} has no artifacts", file=sys.stderr) sys.exit(1) - required = ("version", "platform", "arch", "build_type", "filename", "sha256") - for idx, entry in enumerate(data): - missing = [k for k in required if k not in entry] - if missing: - print( - f"Error: {path} entry {idx} is missing fields: {missing}", - file=sys.stderr, - ) + for name, entry in artifacts.items(): + if "sha256" not in entry: + print(f"Error: {path} artifact {name} missing 'sha256'", file=sys.stderr) sys.exit(1) - print(f"OK: {path} has {len(data)} well-formed entries") + print(f"OK: {path} has {len(artifacts)} well-formed artifacts") def main() -> None: diff --git a/scripts/upload-llvm.py b/scripts/upload-llvm.py deleted file mode 100644 index 5a0db3e5d..000000000 --- a/scripts/upload-llvm.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 - -import hashlib -import json -import os -import subprocess -import sys -from pathlib import Path - - -def sha256sum(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def parse_platform(name: str) -> str: - lowered = name.lower() - if "windows" in lowered: - return "windows" - if "linux" in lowered: - return "linux" - if "macos" in lowered: - return "macosx" - raise ValueError(f"Unable to determine platform from filename: {name}") - - -def parse_arch(name: str) -> str: - lowered = name.lower() - if lowered.startswith("aarch64-") or lowered.startswith("arm64-"): - return "arm64" - if lowered.startswith("x64-") or lowered.startswith("x86_64-"): - return "x64" - raise ValueError(f"Unable to determine arch from filename: {name}") - - -def parse_build_type(name: str) -> str: - lowered = name.lower() - if "debug" in lowered: - return "Debug" - return "RelWithDebInfo" - - -def build_metadata_entry(path: Path, version: str) -> dict: - filename = path.name - return { - "version": version, - "filename": filename, - "sha256": sha256sum(path), - "lto": "-lto" in filename.lower(), - "asan": "-asan" in filename.lower(), - "platform": parse_platform(filename), - "arch": parse_arch(filename), - "build_type": parse_build_type(filename), - } - - -def main() -> None: - if len(sys.argv) != 4: - print( - "Usage: upload-llvm.py ", file=sys.stderr - ) - sys.exit(1) - - tag, target_repo, workflow_id = sys.argv[1:] - artifacts_dir = Path("artifacts") - - if not artifacts_dir.is_dir(): - print(f"Artifacts directory not found: {artifacts_dir}", file=sys.stderr) - sys.exit(1) - - artifact_files = sorted( - p - for p in artifacts_dir.rglob("*") - if p.is_file() and p.suffix.lower() != ".json" - ) - - if not artifact_files: - print("No artifacts found to upload.", file=sys.stderr) - sys.exit(1) - - version_without_prefix = tag.lstrip("vV") - metadata = [ - build_metadata_entry(path, version_without_prefix) for path in artifact_files - ] - - json_path = artifacts_dir / "llvm-manifest.json" - with json_path.open("w", encoding="utf-8") as handle: - json.dump(metadata, handle, indent=2) - handle.write("\n") - - assets = [str(path) for path in artifact_files] - assets.append(str(json_path)) - - env = os.environ.copy() - - print(f"Checking for existing release {tag} in {target_repo}...") - view_result = subprocess.run( - ["gh", "release", "view", tag, "--repo", target_repo], - env=env, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - if view_result.returncode == 0: - print(f"Deleting existing release {tag} in {target_repo}...") - subprocess.run( - ["gh", "release", "delete", tag, "--repo", target_repo, "-y"], - env=env, - check=True, - ) - - print(f"Creating release {tag} in {target_repo} with {len(assets)} assets...") - # fmt: off - args = [ - "gh", "release", "create", tag, *assets, - "--repo", target_repo, - "--title", tag, - "--notes", f"Artifacts build from workflow run https://github.com/clice-io/clice/actions/runs/{workflow_id}", - "--latest", - ] - # fmt: on - subprocess.run(args, env=env, check=True) - - -if __name__ == "__main__": - main() diff --git a/scripts/validate-llvm-components.py b/scripts/validate-llvm-components.py deleted file mode 100755 index 26c804ffe..000000000 --- a/scripts/validate-llvm-components.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python3 -""" -Validate the LLVM distribution component list against the actual LLVM source tree. - -Scans the LLVM source for CMake library targets and compares them against -a components JSON file to detect stale or misspelled entries. -""" - -import argparse -import difflib -import json -import re -import sys -from pathlib import Path - - -# CMake function calls that define library targets. -# The captured group uses [^\s)]+ to grab the target name without -# trailing parentheses or whitespace. -LLVM_LIB_PATTERNS = [ - re.compile(r"add_llvm_component_library\(\s*([^\s)]+)"), - re.compile(r"add_llvm_library\(\s*([^\s)]+)"), -] - -CLANG_LIB_PATTERNS = [ - re.compile(r"add_clang_library\(\s*([^\s)]+)"), -] - -# Header-only / custom install targets. -HEADER_PATTERNS = [ - re.compile(r"add_llvm_install_targets\(\s*([^\s)]+)"), - re.compile(r"add_custom_target\(\s*([^\s)]+)"), - re.compile(r"add_library\(\s*([^\s)]+)"), -] - -# Targets we recognise as header-only distribution components. -KNOWN_HEADER_TARGETS = { - "llvm-headers", - "clang-headers", - "clang-tidy-headers", - "clang-resource-headers", -} - - -def scan_targets(directory: Path, patterns: list[re.Pattern]) -> set[str]: - """Recursively scan *directory* for CMakeLists.txt files and extract target names.""" - targets: set[str] = set() - if not directory.is_dir(): - return targets - for cmake_file in directory.rglob("CMakeLists.txt"): - text = cmake_file.read_text(errors="replace") - for pattern in patterns: - for match in pattern.finditer(text): - targets.add(match.group(1)) - return targets - - -def scan_header_targets(llvm_src: Path) -> set[str]: - """Scan for well-known header / custom-install targets across the tree.""" - found: set[str] = set() - for cmake_file in llvm_src.rglob("CMakeLists.txt"): - text = cmake_file.read_text(errors="replace") - for pattern in HEADER_PATTERNS: - for match in pattern.finditer(text): - name = match.group(1) - if name in KNOWN_HEADER_TARGETS: - found.add(name) - return found - - -def collect_source_targets(llvm_src: Path) -> set[str]: - """Return the full set of library / header targets found in the LLVM source tree.""" - targets: set[str] = set() - targets |= scan_targets(llvm_src / "llvm" / "lib", LLVM_LIB_PATTERNS) - targets |= scan_targets(llvm_src / "clang" / "lib", CLANG_LIB_PATTERNS) - targets |= scan_targets(llvm_src / "clang-tools-extra", CLANG_LIB_PATTERNS) - targets |= scan_header_targets(llvm_src) - return targets - - -def load_components(path: Path) -> list[str]: - with path.open("r", encoding="utf-8") as handle: - data = json.load(handle) - if isinstance(data, dict): - data = data.get("components", []) - if not isinstance(data, list) or not data: - print(f"Error: no component list found in {path}", file=sys.stderr) - sys.exit(1) - return data - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Validate LLVM distribution components against the source tree." - ) - parser.add_argument( - "--llvm-src", - required=True, - help="Path to the llvm-project source root", - ) - parser.add_argument( - "--components-file", - required=True, - help="Path to llvm-components.json", - ) - args = parser.parse_args() - - llvm_src = Path(args.llvm_src).expanduser().resolve() - components_file = Path(args.components_file).expanduser().resolve() - - if not llvm_src.is_dir(): - print(f"Error: LLVM source directory not found: {llvm_src}") - sys.exit(1) - - if not (llvm_src / "llvm" / "CMakeLists.txt").exists(): - print(f"Error: {llvm_src} does not look like an llvm-project root.") - sys.exit(1) - - if not components_file.is_file(): - print(f"Error: components file not found: {components_file}") - sys.exit(1) - - components = load_components(components_file) - source_targets = collect_source_targets(llvm_src) - - print(f"Found {len(source_targets)} targets in LLVM source tree") - print(f"Components file lists {len(components)} entries") - - # Check for components that are missing from the source tree. - missing: list[tuple[str, list[str]]] = [] - for name in components: - if name not in source_targets: - suggestions = difflib.get_close_matches( - name, source_targets, n=3, cutoff=0.6 - ) - missing.append((name, suggestions)) - - if missing: - print(f"\nError: {len(missing)} component(s) not found in the source tree:\n") - for name, suggestions in missing: - print(f" - {name}") - if suggestions: - print(f" Did you mean: {', '.join(suggestions)}?") - sys.exit(1) - - # Warn about source targets not present in the component list. - component_set = set(components) - new_targets = sorted(source_targets - component_set - KNOWN_HEADER_TARGETS) - # Filter to targets that follow LLVM/Clang naming conventions to reduce noise. - noteworthy = [t for t in new_targets if t.startswith(("LLVM", "clang", "Clang"))] - if noteworthy: - print( - f"\nWarning: {len(noteworthy)} target(s) in source not listed in components:" - ) - for name in noteworthy: - print(f" + {name}") - - print("\nAll components validated successfully.") - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/src/command/argument_parser.cpp b/src/command/argument_parser.cpp index 55543b9fc..447c39bbc 100644 --- a/src/command/argument_parser.cpp +++ b/src/command/argument_parser.cpp @@ -8,15 +8,15 @@ #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" #include "clang/Driver/Driver.h" -#include "clang/Driver/Options.h" #include "clang/Driver/Types.h" +#include "clang/Options/OptionUtils.h" +#include "clang/Options/Options.h" namespace clice { namespace { namespace opt = llvm::opt; -namespace driver = clang::driver; /// Access private members of OptTable via the Thief pattern. bool enable_dash_dash_parsing(const opt::OptTable& table); @@ -35,7 +35,7 @@ struct Thief { template struct Thief<&opt::OptTable::DashDashParsing, &opt::OptTable::GroupedShortOptions>; -auto& option_table = driver::getDriverOptTable(); +auto& option_table = clang::getDriverOptTable(); } // namespace @@ -45,7 +45,7 @@ std::unique_ptr ArgumentParser::parse_one(unsigned& index) { return option_table.ParseOneArg(*this, index, opt::Visibility(visibility_mask)); } -using ID = clang::driver::options::ID; +using ID = clang::options::ID; bool is_discarded_option(unsigned id) { switch(id) { @@ -165,7 +165,7 @@ llvm::StringRef resource_dir() { if(exe.empty()) { return std::string{}; } - return clang::driver::Driver::GetResourcesPath(exe); + return clang::GetResourcesPath(exe); }(); return dir; } @@ -246,7 +246,7 @@ std::string print_argv(llvm::ArrayRef args) { } unsigned default_visibility(llvm::StringRef driver) { - namespace options = clang::driver::options; + namespace options = clang::options; auto name = llvm::sys::path::filename(driver); name.consume_back(".exe"); diff --git a/src/command/search_config.cpp b/src/command/search_config.cpp index 34ea21407..0488d78e6 100644 --- a/src/command/search_config.cpp +++ b/src/command/search_config.cpp @@ -6,11 +6,11 @@ #include "llvm/ADT/StringSet.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" -#include "clang/Driver/Options.h" +#include "clang/Options/Options.h" namespace clice { -using ID = clang::driver::options::ID; +using ID = clang::options::ID; SearchConfig extract_search_config(llvm::ArrayRef arguments, llvm::StringRef directory) { @@ -26,7 +26,7 @@ SearchConfig extract_search_config(llvm::ArrayRef arguments, auto make_absolute = [&](llvm::StringRef path) -> std::string { llvm::SmallString<256> abs_path(path); if(!llvm::sys::path::is_absolute(abs_path)) { - llvm::sys::fs::make_absolute(directory, abs_path); + llvm::sys::path::make_absolute(directory, abs_path); } llvm::sys::path::remove_dots(abs_path, true); return abs_path.str().str(); diff --git a/src/command/toolchain.cpp b/src/command/toolchain.cpp index b9dbc9ec4..b470b81e1 100644 --- a/src/command/toolchain.cpp +++ b/src/command/toolchain.cpp @@ -18,8 +18,8 @@ #include "llvm/TargetParser/Host.h" #include "clang/Driver/Compilation.h" #include "clang/Driver/Driver.h" -#include "clang/Driver/Options.h" #include "clang/Driver/Tool.h" +#include "clang/Options/Options.h" #ifndef _WIN32 @@ -476,7 +476,7 @@ std::vector query_clang_toolchain(const QueryParams& params) { // producing cc1 flags we don't recognize. Filter them out here. // Long-term we should unify the command pipeline so the driver // version always matches the embedded LLVM. - auto& table = clang::driver::getDriverOptTable(); + auto& table = clang::getDriverOptTable(); auto cc1_args = llvm::ArrayRef(args).drop_front(2); unsigned missing_index = 0, missing_count = 0; auto parsed = table.ParseArgs(cc1_args, missing_index, missing_count); diff --git a/src/compile/compilation.cpp b/src/compile/compilation.cpp index ffc5bf386..ceff44a29 100644 --- a/src/compile/compilation.cpp +++ b/src/compile/compilation.cpp @@ -7,6 +7,7 @@ #include "support/logging.h" #include "llvm/Support/Error.h" +#include "clang/Driver/CreateInvocationFromArgs.h" #include "clang/Frontend/MultiplexConsumer.h" #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Lex/PreprocessorOptions.h" @@ -247,13 +248,14 @@ CompilationStatus CompilationUnitRef::Self::run_clang( self.instance = std::make_unique(std::move(invocation)); auto& instance = *self.instance; - instance.createDiagnostics(*params.vfs, diagnostic_consumer.release(), true); + instance.createDiagnostics(diagnostic_consumer.release(), true); if(auto remapping = clang::createVFSFromCompilerInvocation(instance.getInvocation(), instance.getDiagnostics(), params.vfs)) { - instance.createFileManager(std::move(remapping)); + instance.setVirtualFileSystem(std::move(remapping)); } + instance.createFileManager(); if(!instance.createTarget()) { return CompilationStatus::SetupFail; diff --git a/src/index/usr_generation.cpp b/src/index/usr_generation.cpp index 2812c237c..c98acd68a 100644 --- a/src/index/usr_generation.cpp +++ b/src/index/usr_generation.cpp @@ -504,14 +504,14 @@ bool USRGenerator::GenLoc(const Decl* D, bool IncludeOffset) { static void printQualifier(llvm::raw_ostream& Out, const LangOptions& LangOpts, - NestedNameSpecifier* NNS) { + NestedNameSpecifier NNS) { // FIXME: Encode the qualifier, don't just print it. PrintingPolicy PO(LangOpts); PO.SuppressTagKeyword = true; PO.SuppressUnwrittenScope = true; PO.ConstantArraySizeAsWritten = false; PO.AnonymousTagLocations = false; - NNS->print(Out, PO); + NNS.print(Out, PO); } void USRGenerator::VisitType(QualType T) { @@ -740,8 +740,9 @@ void USRGenerator::VisitType(QualType T) { return; } if(const InjectedClassNameType* InjT = T->getAs()) { - T = InjT->getInjectedSpecializationType(); - continue; + Out << '$'; + VisitTagDecl(InjT->getDecl()); + return; } if(const auto* VT = T->getAs()) { Out << (T->isExtVectorType() ? ']' : '['); diff --git a/src/semantic/ast_utility.cpp b/src/semantic/ast_utility.cpp index d09c6144d..daa9c995f 100644 --- a/src/semantic/ast_utility.cpp +++ b/src/semantic/ast_utility.cpp @@ -200,10 +200,6 @@ llvm::StringRef identifier_of(const clang::NamedDecl& D) { } llvm::StringRef identifier_of(clang::QualType type) { - if(const auto* ET = llvm::dyn_cast(type)) { - return identifier_of(ET->getNamedType()); - } - if(const auto* BT = llvm::dyn_cast(type)) { clang::PrintingPolicy PP(clang::LangOptions{}); PP.adjustForCPlusPlus(); @@ -312,12 +308,6 @@ auto decl_of(clang::QualType type) -> const clang::NamedDecl* { return nullptr; } - // Strip type-sugar that wraps the underlying type without adding a decl - // (e.g. ElaboratedType for "struct Foo" vs plain "Foo"). - if(auto ET = type->getAs()) { - type = ET->getNamedType(); - } - if(auto TST = type->getAs()) { auto decl = TST->getTemplateName().getAsTemplateDecl(); if(type->isDependentType()) { @@ -409,8 +399,8 @@ std::string display_name_of(const clang::NamedDecl* decl) { // Handle 'using namespace'. They all have the same name - . if(auto* UD = llvm::dyn_cast(decl)) { out << "using namespace "; - if(auto* Qual = UD->getQualifier()) - Qual->print(out, policy); + if(auto Qual = UD->getQualifier()) + Qual.print(out, policy); UD->getNominatedNamespaceAsWritten()->printName(out); return out.str(); } @@ -437,8 +427,8 @@ std::string display_name_of(const clang::NamedDecl* decl) { } // Print nested name qualifier if it was written in the source code. - if(auto* qualifier = get_qualifier_loc(decl).getNestedNameSpecifier()) { - qualifier->print(out, policy); + if(auto qualifier = get_qualifier_loc(decl).getNestedNameSpecifier()) { + qualifier.print(out, policy); } // Print the name itself. diff --git a/src/semantic/filtered_ast_visitor.h b/src/semantic/filtered_ast_visitor.h index 2f09035a8..dfa8ab591 100644 --- a/src/semantic/filtered_ast_visitor.h +++ b/src/semantic/filtered_ast_visitor.h @@ -107,7 +107,7 @@ class FilteredASTVisitor : public clang::RecursiveASTVisitor { return true; } - bool TraverseTypeLoc(clang::TypeLoc loc) { + bool TraverseTypeLoc(clang::TypeLoc loc, bool TraverseQualifier = true) { CHECK_DERIVED_IMPL(TraverseTypeLoc); if(!loc) { @@ -116,10 +116,10 @@ class FilteredASTVisitor : public clang::RecursiveASTVisitor { /// FIXME: Workaround for `QualifiedTypeLoc`. if(auto QL = loc.getAs()) { - return Base::TraverseTypeLoc(QL.getUnqualifiedLoc()); + return Base::TraverseTypeLoc(QL.getUnqualifiedLoc(), TraverseQualifier); } - return Base::TraverseTypeLoc(loc); + return Base::TraverseTypeLoc(loc, TraverseQualifier); } bool TraverseAttr(clang::Attr* attr) { @@ -132,9 +132,8 @@ class FilteredASTVisitor : public clang::RecursiveASTVisitor { return Base::TraverseAttr(attr); } - /// We don't want to node withou location information. - constexpr bool TraverseNestedNameSpecifier - [[gnu::always_inline]] (clang::NestedNameSpecifier*) { + /// We don't want to node without location information. + constexpr bool TraverseNestedNameSpecifier [[gnu::always_inline]] (clang::NestedNameSpecifier) { CHECK_DERIVED_IMPL(TraverseNestedNameSpecifier); return true; } diff --git a/src/semantic/resolver.cpp b/src/semantic/resolver.cpp index 120e523dd..2851f8385 100644 --- a/src/semantic/resolver.cpp +++ b/src/semantic/resolver.cpp @@ -161,13 +161,13 @@ struct InstantiationStack { }; /// Helper to extract underlying type from a Decl. -static clang::QualType get_decl_type(clang::Decl* decl) { +static clang::QualType get_decl_type(clang::Decl* decl, clang::ASTContext& context) { if(!decl) return clang::QualType(); if(auto* TND = llvm::dyn_cast(decl)) return TND->getUnderlyingType(); if(auto* RD = llvm::dyn_cast(decl)) - return clang::QualType(RD->getTypeForDecl(), 0); + return context.getCanonicalTagType(RD); return clang::QualType(); } @@ -205,14 +205,11 @@ class SubstituteOnly : public clang::TreeTransform { /// Desugar dependent typedefs to expose template parameters for substitution. clang::QualType TransformTypedefType(clang::TypeLocBuilder& TLB, clang::TypedefTypeLoc TL) { - if(auto* TND = TL.getTypedefNameDecl()) { + if(auto* TND = TL.getDecl()) { auto underlying = TND->getUnderlyingType(); if(underlying->isDependentType()) { auto type = TransformType(underlying); if(!type.isNull()) { - if(auto ET = llvm::dyn_cast(type)) { - type = ET->getNamedType(); - } TLB.pushTrivial(context, type, {}); return type; } @@ -221,25 +218,9 @@ class SubstituteOnly : public clang::TreeTransform { return Base::TransformTypedefType(TLB, TL); } - clang::QualType TransformElaboratedType(clang::TypeLocBuilder& TLB, - clang::ElaboratedTypeLoc TL) { - clang::QualType type = TransformType(TL.getNamedTypeLoc().getType()); - if(type.isNull()) { - return Base::TransformElaboratedType(TLB, TL); - } - TLB.pushTrivial(context, type, {}); - return type; - } - clang::QualType TransformInjectedClassNameType(clang::TypeLocBuilder& TLB, clang::InjectedClassNameTypeLoc TL) { - auto ICT = TL.getTypePtr(); - clang::QualType type = TransformType(ICT->getInjectedSpecializationType()); - if(type.isNull()) { - return Base::TransformInjectedClassNameType(TLB, TL); - } - TLB.pushTrivial(context, type, {}); - return type; + return Base::TransformInjectedClassNameType(TLB, TL); } using Base::TransformTemplateSpecializationType; @@ -496,24 +477,25 @@ class PseudoInstantiator : public clang::TreeTransform { } if(auto TST = type->getAs()) { - TD = TST->getTemplateName().getAsTemplateDecl(); - args = TST->template_arguments(); - } else if(auto DTST = type->getAs()) { - // If this DTST was already resolved (possibly to itself when unresolvable), - // skip the redundant lookup. - if(resolved.count(DTST)) { - return lookup_result(); - } + auto TN = TST->getTemplateName(); + if(auto* DTN = TN.getAsDependentTemplateName()) { + // This was formerly DependentTemplateSpecializationType. + if(resolved.count(TST)) { + return lookup_result(); + } - auto& template_name = DTST->getDependentTemplateName(); - auto name = template_name.getName().getIdentifier(); - if(!name) { - return {}; - } + auto identifier = DTN->getName().getIdentifier(); + if(!identifier) { + return {}; + } - if(auto decl = preferred(lookup(template_name.getQualifier(), name))) { - TD = decl; - args = DTST->template_arguments(); + if(auto decl = preferred(lookup(DTN->getQualifier(), identifier))) { + TD = decl; + args = TST->template_arguments(); + } + } else { + TD = TN.getAsTemplateDecl(); + args = TST->template_arguments(); } } @@ -536,45 +518,27 @@ class PseudoInstantiator : public clang::TreeTransform { return lookup_result(); } - lookup_result lookup(const clang::NestedNameSpecifier* NNS, clang::DeclarationName name) { + lookup_result lookup(clang::NestedNameSpecifier NNS, clang::DeclarationName name) { if(!NNS) { return lookup_result(); } - if(auto iter = resolved.find(NNS); iter != resolved.end()) { + if(auto iter = resolved.find(NNS.getAsVoidPointer()); iter != resolved.end()) { return lookup(iter->second, name); } // Handle each NestedNameSpecifier kind: - // - Identifier: dependent name in NNS chain (e.g. `base::type::inner`), resolve recursively - // - TypeSpec: concrete or dependent type used as qualifier (e.g. `vector::`) - // - Global/Namespace/NamespaceAlias/Super: not dependent, cannot resolve further - switch(NNS->getKind()) { - case clang::NestedNameSpecifier::Identifier: { - auto stack_size = stack.data.size(); - auto* decl = preferred(lookup(NNS->getPrefix(), NNS->getAsIdentifier())); - auto type = get_decl_type(decl); - if(!type.isNull()) { - type = substitute(type); - } - while(stack.data.size() > stack_size) { - stack.pop(); - } - if(!type.isNull()) { - resolved.try_emplace(NNS, type); - return lookup(type, name); - } - return {}; - } - - case clang::NestedNameSpecifier::TypeSpec: { - return lookup(clang::QualType(NNS->getAsType(), 0), name); + // - Type: concrete or dependent type used as qualifier (e.g. `vector::`) + // - Global/Namespace/MicrosoftSuper/Null: not dependent, cannot resolve further + switch(NNS.getKind()) { + case clang::NestedNameSpecifier::Kind::Type: { + return lookup(clang::QualType(NNS.getAsType(), 0), name); } - case clang::NestedNameSpecifier::Global: - case clang::NestedNameSpecifier::Namespace: - case clang::NestedNameSpecifier::NamespaceAlias: - case clang::NestedNameSpecifier::Super: { + case clang::NestedNameSpecifier::Kind::Null: + case clang::NestedNameSpecifier::Kind::Global: + case clang::NestedNameSpecifier::Kind::Namespace: + case clang::NestedNameSpecifier::Kind::MicrosoftSuper: { return {}; } } @@ -708,14 +672,14 @@ class PseudoInstantiator : public clang::TreeTransform { /// /// TODO: Replace with a general mechanism for resolving well-known standard /// library patterns, or improve the resolver to handle these chains naturally. - clang::QualType hole(clang::NestedNameSpecifier* NNS, + clang::QualType hole(clang::NestedNameSpecifier NNS, const clang::IdentifierInfo* member, TemplateArguments arguments) { - if(NNS->getKind() != clang::NestedNameSpecifier::TypeSpec) { + if(NNS.getKind() != clang::NestedNameSpecifier::Kind::Type) { return clang::QualType(); } - auto TST = NNS->getAsType()->getAs(); + auto TST = NNS.getAsType()->getAs(); if(!TST) { return clang::QualType(); } @@ -738,17 +702,18 @@ class PseudoInstantiator : public clang::TreeTransform { return clang::QualType(); auto T = arguments[0].getAsType(); - auto prefix = - clang::NestedNameSpecifier::Create(context, nullptr, Alloc.getTypePtr()); + clang::NestedNameSpecifier prefix(Alloc.getTypePtr()); auto rebind = sema.getPreprocessor().getIdentifierInfo("rebind"); - auto DTST = context.getDependentTemplateSpecializationType( - clang::ElaboratedTypeKeyword::None, - clang::DependentTemplateStorage(prefix, rebind, false), - arguments); + clang::TemplateName DTN = context.getDependentTemplateName({prefix, rebind, false}); + auto DTST = + context.getTemplateSpecializationType(clang::ElaboratedTypeKeyword::None, + DTN, + arguments, + {}); - prefix = clang::NestedNameSpecifier::Create(context, prefix, DTST.getTypePtr()); + prefix = clang::NestedNameSpecifier(DTST.getTypePtr()); auto other = sema.getPreprocessor().getIdentifierInfo("other"); auto DNT = context.getDependentNameType(clang::ElaboratedTypeKeyword::Typename, @@ -770,9 +735,11 @@ class PseudoInstantiator : public clang::TreeTransform { for(auto& arg: replaceArguments) { canonicalArguments.emplace_back(context.getCanonicalTemplateArgument(arg)); } - auto result = context.getTemplateSpecializationType(TST->getTemplateName(), - replaceArguments, - canonicalArguments); + auto result = + context.getTemplateSpecializationType(clang::ElaboratedTypeKeyword::None, + TST->getTemplateName(), + replaceArguments, + canonicalArguments); LOG_DEBUG( "{}" "hole: 'allocator_traits::rebind_alloc' → '{}'", pad(), @@ -859,7 +826,9 @@ class PseudoInstantiator : public clang::TreeTransform { clang::QualType TransformDependentNameType(clang::TypeLocBuilder& TLB, clang::DependentNameTypeLoc TL, - bool DeducedTSTContext = false) { + bool DeducedTSTContext = false, + clang::QualType ObjectType = clang::QualType(), + clang::NamedDecl* UnqualLookup = nullptr) { auto* DNT = TL.getTypePtr(); LOG_DEBUG("{}" "resolve '{}'", pad(), clang::QualType(DNT, 0).getAsString()); ++indent; @@ -897,10 +866,10 @@ class PseudoInstantiator : public clang::TreeTransform { return original; } - auto* NNS = NNSLoc.getNestedNameSpecifier(); + auto NNS = NNSLoc.getNestedNameSpecifier(); auto stack_size = stack.data.size(); auto* decl = preferred(lookup(NNS, DNT->getIdentifier())); - auto type = get_decl_type(decl); + auto type = get_decl_type(decl, context); clang::QualType result; if(!type.isNull()) { @@ -963,43 +932,50 @@ class PseudoInstantiator : public clang::TreeTransform { return original; } - using Base::TransformDependentTemplateSpecializationType; + using Base::TransformTemplateSpecializationType; - clang::QualType rebuild_dtst(clang::TypeLocBuilder& TLB, - clang::DependentTemplateSpecializationTypeLoc TL) { - auto* DTST = TL.getTypePtr(); - return TLB.push(clang::QualType(DTST, 0)) - .getType(); - } + clang::QualType TransformTemplateSpecializationType(clang::TypeLocBuilder& TLB, + clang::TemplateSpecializationTypeLoc TL) { + auto* TST = TL.getTypePtr(); + auto TN = TST->getTemplateName(); + auto* DTN = TN.getAsDependentTemplateName(); + + if(!DTN) { + // Non-dependent TST: handle alias templates. + if(TST->isTypeAlias()) { + clang::QualType type = substitute(TST->desugar()); + if(!type.isNull()) { + TLB.pushTrivial(context, type, {}); + return type; + } + } + return Base::TransformTemplateSpecializationType(TLB, TL); + } - clang::QualType TransformDependentTemplateSpecializationType( - clang::TypeLocBuilder& TLB, - clang::DependentTemplateSpecializationTypeLoc TL) { - auto* DTST = TL.getTypePtr(); - LOG_DEBUG("{}" "resolve DTST '{}'", pad(), clang::QualType(DTST, 0).getAsString()); + // Dependent template specialization handling. + LOG_DEBUG("{}" "resolve dep-TST '{}'", pad(), clang::QualType(TST, 0).getAsString()); ++indent; - if(auto iter = resolved.find(DTST); iter != resolved.end()) { + if(auto iter = resolved.find(TST); iter != resolved.end()) { --indent; TLB.pushTrivial(context, iter->second, {}); return iter->second; } auto NNSLoc = TransformNestedNameSpecifierLoc(TL.getQualifierLoc()); - if(!NNSLoc) { - LOG_DEBUG("{}→ ", pad()); - --indent; - return rebuild_dtst(TLB, TL); - } - auto* NNS = NNSLoc.getNestedNameSpecifier(); + clang::NestedNameSpecifier NNS = + NNSLoc ? NNSLoc.getNestedNameSpecifier() : DTN->getQualifier(); clang::TemplateArgumentListInfo info; - using iterator = clang::TemplateArgumentLocContainerIterator< - clang::DependentTemplateSpecializationTypeLoc>; - if(TransformTemplateArguments(iterator(TL, 0), iterator(TL, TL.getNumArgs()), info)) { - LOG_DEBUG("{}→ ", pad()); + using arg_iterator = + clang::TemplateArgumentLocContainerIterator; + if(TransformTemplateArguments(arg_iterator(TL, 0), + arg_iterator(TL, TL.getNumArgs()), + info)) { + auto original = clang::QualType(TST, 0); + TLB.pushTrivial(context, original, {}); --indent; - return rebuild_dtst(TLB, TL); + return original; } llvm::SmallVector arguments; @@ -1007,17 +983,19 @@ class PseudoInstantiator : public clang::TreeTransform { arguments.push_back(arg.getArgument()); } - auto* name = DTST->getDependentTemplateName().getName().getIdentifier(); + auto* name = DTN->getName().getIdentifier(); if(!name) { - LOG_DEBUG("{}→ ", pad()); + LOG_DEBUG("{}→ ", pad()); --indent; - return rebuild_dtst(TLB, TL); + auto original = clang::QualType(TST, 0); + TLB.pushTrivial(context, original, {}); + return original; } if(auto result = hole(NNS, name, arguments); !result.isNull()) { LOG_DEBUG("{}" "hole: '{}' → '{}'", pad(), name->getName().str(), result.getAsString()); --indent; - resolved.try_emplace(DTST, result); + resolved.try_emplace(TST, result); TLB.pushTrivial(context, result, {}); return result; } @@ -1027,7 +1005,6 @@ class PseudoInstantiator : public clang::TreeTransform { if(auto* TATD = llvm::dyn_cast(decl)) { if(deduce_template_arguments(TATD, arguments)) { auto type = substitute(TATD->getTemplatedDecl()->getUnderlyingType()); - // Pop lookup frames before further resolution. while(stack.data.size() > stack_size) { stack.pop(); } @@ -1037,26 +1014,25 @@ class PseudoInstantiator : public clang::TreeTransform { if(!type.isNull()) { LOG_DEBUG("{}" "→ '{}' (alias)", pad(), type.getAsString()); --indent; - resolved.try_emplace(DTST, type); + resolved.try_emplace(TST, type); TLB.pushTrivial(context, type, {}); return type; } } } else if(auto* CTD = llvm::dyn_cast(decl)) { - // Resolve DTST to a concrete TemplateSpecializationType. - // e.g. __alloc_traits>::rebind → rebind (a TST) - // This allows subsequent lookup of members (like "other") to work. - // Keep lookup frames on stack — the caller (e.g. TransformNestedNameSpecifierLoc - // processing A::B::C) needs them for parameter substitution. - clang::TemplateName TN(CTD); + clang::TemplateName CTN(CTD); llvm::SmallVector canonArgs; for(auto& arg: arguments) { canonArgs.push_back(context.getCanonicalTemplateArgument(arg)); } - auto result = context.getTemplateSpecializationType(TN, arguments, canonArgs); + auto result = + context.getTemplateSpecializationType(clang::ElaboratedTypeKeyword::None, + CTN, + arguments, + canonArgs); LOG_DEBUG("{}" "→ TST '{}' (class)", pad(), result.getAsString()); --indent; - resolved.try_emplace(DTST, result); + resolved.try_emplace(TST, result); TLB.pushTrivial(context, result, {}); return result; } @@ -1065,11 +1041,12 @@ class PseudoInstantiator : public clang::TreeTransform { stack.pop(); } - LOG_DEBUG("{}→ ", pad()); + LOG_DEBUG("{}→ ", pad()); --indent; - auto fallback = rebuild_dtst(TLB, TL); - resolved.try_emplace(DTST, fallback); - return fallback; + auto original = clang::QualType(TST, 0); + resolved.try_emplace(TST, original); + TLB.pushTrivial(context, original, {}); + return original; } /// Desugar dependent typedefs by delegating to SubstituteOnly. @@ -1077,14 +1054,11 @@ class PseudoInstantiator : public clang::TreeTransform { /// its own TransformTypedefType). Using substitute() here ensures that typedef /// expansion does NOT trigger heuristic lookup, preventing the typedef ↔ lookup cycle. clang::QualType TransformTypedefType(clang::TypeLocBuilder& TLB, clang::TypedefTypeLoc TL) { - if(auto* TND = TL.getTypedefNameDecl()) { + if(auto* TND = TL.getDecl()) { auto underlying = TND->getUnderlyingType(); if(underlying->isDependentType()) { auto type = substitute(underlying); if(!type.isNull()) { - if(auto ET = llvm::dyn_cast(type)) { - type = ET->getNamedType(); - } TLB.pushTrivial(context, type, {}); return type; } @@ -1138,7 +1112,7 @@ clang::QualType TemplateResolver::resugar(clang::QualType type, clang::Decl* dec return resugar.TransformType(type); } -TemplateResolver::lookup_result TemplateResolver::lookup(const clang::NestedNameSpecifier* NNS, +TemplateResolver::lookup_result TemplateResolver::lookup(clang::NestedNameSpecifier NNS, clang::DeclarationName name) { PseudoInstantiator instantiator(sema, resolved); return instantiator.lookup(NNS, name); diff --git a/src/semantic/resolver.h b/src/semantic/resolver.h index 15557eabe..8d752ada1 100644 --- a/src/semantic/resolver.h +++ b/src/semantic/resolver.h @@ -41,23 +41,12 @@ class TemplateResolver { using lookup_result = clang::DeclContext::lookup_result; /// Look up the name in the given nested name specifier. - lookup_result lookup(const clang::NestedNameSpecifier* NNS, clang::DeclarationName name); + lookup_result lookup(clang::NestedNameSpecifier NNS, clang::DeclarationName name); lookup_result lookup(const clang::DependentNameType* type) { return lookup(type->getQualifier(), type->getIdentifier()); } - lookup_result lookup(const clang::DependentTemplateSpecializationType* type) { - auto& template_name = type->getDependentTemplateName(); - auto identifier = template_name.getName().getIdentifier(); - if(identifier) { - return lookup(template_name.getQualifier(), identifier); - } else { - /// TODO: Operators don't have an IdentifierInfo; need DeclarationName-based lookup. - return {}; - } - } - lookup_result lookup(const clang::DependentScopeDeclRefExpr* expr) { return lookup(expr->getQualifier(), expr->getNameInfo().getName()); } diff --git a/src/semantic/selection.cpp b/src/semantic/selection.cpp index f30970afd..c51ad25ca 100644 --- a/src/semantic/selection.cpp +++ b/src/semantic/selection.cpp @@ -756,8 +756,8 @@ class SelectionVisitor : public clang::RecursiveASTVisitor { return traverse_node(X, [&] { return Base::TraverseDecl(X); }); } - bool TraverseTypeLoc(clang::TypeLoc X) { - return traverse_node(&X, [&] { return Base::TraverseTypeLoc(X); }); + bool TraverseTypeLoc(clang::TypeLoc X, bool TraverseQualifier = true) { + return traverse_node(&X, [&] { return Base::TraverseTypeLoc(X, TraverseQualifier); }); } bool TraverseTemplateArgumentLoc(const clang::TemplateArgumentLoc& X) { @@ -814,7 +814,8 @@ class SelectionVisitor : public clang::RecursiveASTVisitor { // This means we'd never see 'int' in 'const int'! Work around that here. // (The reason for the behavior is to avoid traversing the nested Type twice, // but we ignore TraverseType anyway). - bool TraverseQualifiedTypeLoc(clang::QualifiedTypeLoc QX) { + bool TraverseQualifiedTypeLoc(clang::QualifiedTypeLoc QX, + [[maybe_unused]] bool TraverseQualifier = true) { return traverse_node(&QX, [&] { return TraverseTypeLoc(QX.getUnqualifiedLoc()); }); @@ -825,7 +826,7 @@ class SelectionVisitor : public clang::RecursiveASTVisitor { } // Uninteresting parts of the AST that don't have locations within them. - bool TraverseNestedNameSpecifier(clang::NestedNameSpecifier*) { + bool TraverseNestedNameSpecifier(clang::NestedNameSpecifier) { return true; } diff --git a/src/semantic/semantic_visitor.h b/src/semantic/semantic_visitor.h index 747291958..cf0e65fdf 100644 --- a/src/semantic/semantic_visitor.h +++ b/src/semantic/semantic_visitor.h @@ -515,7 +515,7 @@ class SemanticVisitor : public FilteredASTVisitor> { /// using Foo = int; Foo foo; /// ^~~~ reference VISIT_TYPELOC(TypedefTypeLoc) { - auto decl = loc.getTypedefNameDecl(); + auto decl = loc.getDecl(); auto location = loc.getNameLoc(); handleDeclOccurrence(decl, RelationKind::Reference, location); handleRelation(decl, RelationKind::Reference, decl, location); @@ -559,49 +559,25 @@ class SemanticVisitor : public FilteredASTVisitor> { return true; } - /// std::allocator::rebind - /// ^~~~ reference - VISIT_TYPELOC(DependentTemplateSpecializationTypeLoc) { - auto location = loc.getTemplateNameLoc(); - // for(auto decl: resolver.lookup(loc.getTypePtr())) { - // handleDeclOccurrence(decl, RelationKind::WeakReference, location); - // handleRelation(decl, RelationKind::WeakReference, decl, location); - // } - return true; - } - /// ============================================================================ /// Specifier /// ============================================================================ bool VisitNestedNameSpecifierLoc(clang::NestedNameSpecifierLoc loc) { auto NNS = loc.getNestedNameSpecifier(); - switch(NNS->getKind()) { - case clang::NestedNameSpecifier::Namespace: { - auto decl = NNS->getAsNamespace(); - auto location = loc.getLocalBeginLoc(); - handleDeclOccurrence(decl, RelationKind::Reference, location); - handleRelation(decl, RelationKind::Reference, decl, location); - break; - } - - case clang::NestedNameSpecifier::NamespaceAlias: { - auto decl = NNS->getAsNamespaceAlias(); + switch(NNS.getKind()) { + case clang::NestedNameSpecifier::Kind::Namespace: { + auto [NS, Prefix] = loc.castAsNamespaceAndPrefix(); auto location = loc.getLocalBeginLoc(); - handleDeclOccurrence(decl, RelationKind::Reference, location); - handleRelation(decl, RelationKind::Reference, decl, location); - break; - } - - case clang::NestedNameSpecifier::Identifier: { - assert(NNS->isDependent() && "Identifier NNS should be dependent"); - // FIXME: use TemplateResolver here. + handleDeclOccurrence(NS, RelationKind::Reference, location); + handleRelation(NS, RelationKind::Reference, NS, location); break; } - case clang::NestedNameSpecifier::TypeSpec: - case clang::NestedNameSpecifier::Global: - case clang::NestedNameSpecifier::Super: { + case clang::NestedNameSpecifier::Kind::Null: + case clang::NestedNameSpecifier::Kind::Type: + case clang::NestedNameSpecifier::Kind::Global: + case clang::NestedNameSpecifier::Kind::MicrosoftSuper: { break; }; } diff --git a/src/syntax/scan.cpp b/src/syntax/scan.cpp index 34603b82d..434386038 100644 --- a/src/syntax/scan.cpp +++ b/src/syntax/scan.cpp @@ -10,6 +10,7 @@ #include "clang/Basic/FileEntry.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/SourceManager.h" +#include "clang/Driver/CreateInvocationFromArgs.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Lex/PPCallbacks.h" @@ -319,9 +320,10 @@ std::unique_ptr } auto instance = std::make_unique(std::move(invocation)); - instance->createDiagnostics(*vfs, new clang::IgnoringDiagConsumer(), true); + instance->createDiagnostics(new clang::IgnoringDiagConsumer(), true); instance->getDiagnostics().setSuppressAllDiagnostics(true); - instance->createFileManager(vfs); + instance->setVirtualFileSystem(vfs); + instance->createFileManager(); return instance; } diff --git a/tests/unit/command/argument_parser_tests.cpp b/tests/unit/command/argument_parser_tests.cpp index ee9ae8c04..d8398cc15 100644 --- a/tests/unit/command/argument_parser_tests.cpp +++ b/tests/unit/command/argument_parser_tests.cpp @@ -1,7 +1,7 @@ #include "test/test.h" #include "command/argument_parser.h" -#include "clang/Driver/Options.h" +#include "clang/Options/Options.h" namespace clice::testing { @@ -9,7 +9,7 @@ namespace { TEST_SUITE(ArgumentParser) { -using option = clang::driver::options::ID; +using option = clang::options::ID; void EXPECT_ID(llvm::StringRef command, option opt) { auto id = get_option_id(command); diff --git a/tests/unit/feature/inlay_hint_tests.cpp b/tests/unit/feature/inlay_hint_tests.cpp index f6eb89df6..cb1d534d4 100644 --- a/tests/unit/feature/inlay_hint_tests.cpp +++ b/tests/unit/feature/inlay_hint_tests.cpp @@ -624,7 +624,7 @@ TEST_CASE(Types) { )c"); EXPECT_SIZE(2); EXPECT_HINT("0", ": S1"); - EXPECT_HINT("1", ": S2::Inner"); + EXPECT_HINT("1", ": Inner"); // Lambda run(R"c( diff --git a/tests/unit/semantic/selection_tests.cpp b/tests/unit/semantic/selection_tests.cpp index 7ab4f2ea4..fb72c7ec8 100644 --- a/tests/unit/semantic/selection_tests.cpp +++ b/tests/unit/semantic/selection_tests.cpp @@ -442,9 +442,9 @@ TEST_CASE(Types) { TEST_CASE(CXXFeatures) { EXPECT_SELECT(R"( template - int x = @[T::$U::]ccc(); + int x = T::@[$U]::ccc(); )", - "NestedNameSpecifierLoc"); + "DependentNameTypeLoc"); EXPECT_SELECT(R"( struct Foo {}; struct Bar : @[v$ir$tual private Foo] {}; @@ -486,9 +486,9 @@ TEST_CASE(UsingEnum) { "TypedefTypeLoc"); EXPECT_SELECT(R"( namespace ns { enum class A {}; }; - using enum @[$ns::]A; + @[using enum $ns::A]; )", - "NestedNameSpecifierLoc"); + "UsingEnumDecl"); EXPECT_SELECT(R"( namespace ns { enum class A {}; }; @[using $enum ns::A];