diff --git a/.github/workflows/build-larch-macos-netam.yml b/.github/workflows/build-larch-macos-netam.yml new file mode 100644 index 00000000..a2e7412a --- /dev/null +++ b/.github/workflows/build-larch-macos-netam.yml @@ -0,0 +1,45 @@ +name: Build larch on macOS (netam) + +on: + workflow_dispatch: + push: + branches: + - fix-macos-build + +jobs: + build-larch-macos-netam: + runs-on: macos-14 + + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + submodules: true + + - name: Install Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + miniconda-version: latest + activate-environment: larch-dev + environment-file: environment-dev-macos-netam.yml + channels: pytorch,conda-forge,bioconda + auto-activate-base: false + + - name: Build (cmake step) + shell: bash -el {0} + run: | + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_USHER=yes -DUSE_SYSTEM_TBB=yes -DUSE_NETAM=yes -DProtobuf_ROOT=$CONDA_PREFIX -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. + + - name: Build (make step) + shell: bash -el {0} + run: | + cd build + make VERBOSE=1 -j$(sysctl -n hw.ncpu) + + - name: Run tests + shell: bash -el {0} + run: | + cd build + ln -s ../data + ./bin/larch-test -tag slow diff --git a/.github/workflows/build-larch-macos.yml b/.github/workflows/build-larch-macos.yml new file mode 100644 index 00000000..09b841b1 --- /dev/null +++ b/.github/workflows/build-larch-macos.yml @@ -0,0 +1,45 @@ +name: Build larch on macOS + +on: + workflow_dispatch: + push: + branches: + - fix-macos-build + +jobs: + build-larch-macos: + runs-on: macos-14 + + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + submodules: true + + - name: Install Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + miniconda-version: latest + activate-environment: larch-dev + environment-file: environment-dev-macos.yml + channels: conda-forge,bioconda + auto-activate-base: false + + - name: Build (cmake step) + shell: bash -el {0} + run: | + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_USHER=yes -DUSE_SYSTEM_TBB=yes -DProtobuf_ROOT=$CONDA_PREFIX .. + + - name: Build (make step) + shell: bash -el {0} + run: | + cd build + make VERBOSE=1 -j$(sysctl -n hw.ncpu) + + - name: Run tests + shell: bash -el {0} + run: | + cd build + ln -s ../data + ./bin/larch-test -tag slow diff --git a/.github/workflows/build-larch.yml b/.github/workflows/build-larch.yml index 8adf1583..5f881e23 100644 --- a/.github/workflows/build-larch.yml +++ b/.github/workflows/build-larch.yml @@ -1,64 +1,45 @@ -name: Build larch on linux and run tests +name: Build larch on linux on: workflow_dispatch: + push: + branches: + - fix-macos-build jobs: build-larch: runs-on: ubuntu-latest - - strategy: - matrix: - environment: [build-larch-debug, build-larch-release] - environment: ${{ matrix.environment }} steps: - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: true - - name: Prepare build environment - run: | - sudo apt-get update - sudo apt-get install build-essential zlib1g-dev python3-pip python3-pytest git git-lfs - - name: Install conda - run: | - curl -L https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh > Miniconda3-latest-Linux-x86_64.sh - bash Miniconda3-latest-Linux-x86_64.sh -b -p ~/miniconda - ~/miniconda/bin/conda update -n base -c defaults conda - mkdir build - - name: Create conda env - run: | - source ~/miniconda/etc/profile.d/conda.sh - conda config --prepend channels conda-forge - conda config --prepend channels bioconda - conda config --show channels - conda env create -f environment-dev.yml + + - name: Install Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + miniconda-version: latest + activate-environment: larch-dev + environment-file: environment-ci-linux.yml + channels: conda-forge,bioconda + auto-activate-base: false + - name: Build (cmake step) - env: - CMAKE_BUILD_TYPE: ${{ vars.CMAKE_BUILD_TYPE }} - CMAKE_NUM_THREADS: ${{ vars.CMAKE_NUM_THREADS }} - USE_USHER: ${{ vars.USE_USHER }} + shell: bash -el {0} run: | - source ~/miniconda/etc/profile.d/conda.sh - conda activate larch-dev - cd build - export CMAKE_NUM_THREADS=${CMAKE_NUM_THREADS} - cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DUSE_USHER=${USE_USHER} .. + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Debug -DUSE_USHER=yes .. + - name: Build (make step) - env: - MAKE_NUM_THREADS: ${{ vars.MAKE_NUM_THREADS }} + shell: bash -el {0} run: | - source ~/miniconda/etc/profile.d/conda.sh - conda activate larch-dev cd build - make VERBOSE=1 -j${MAKE_NUM_THREADS} + make VERBOSE=1 -j$(nproc) + - name: Run tests - env: - LARCH_TEST_FLAGS: ${{ vars.LARCH_TEST_FLAGS }} + shell: bash -el {0} run: | - source ~/miniconda/etc/profile.d/conda.sh - conda activate larch-dev cd build ln -s ../data - ./larch-test ${LARCH_TEST_FLAGS} + ./bin/larch-test -tag slow diff --git a/CMakeLists.txt b/CMakeLists.txt index c5e20fc1..1b27d13c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,10 +58,10 @@ include(ExternalProject) # Find Protobuf early, before any dependencies that might bundle their own protobuf # On Linux: clear CMAKE_PREFIX_PATH to prevent finding libtorch's vendored protobuf -# On FreeBSD: use CONFIG mode explicitly to create all protobuf targets (including upb) +# On FreeBSD/macOS: use CONFIG mode explicitly to create all protobuf targets (including upb) # so that when Torch's Caffe2 calls find_package(Protobuf CONFIG), protobuf-targets.cmake # sees all targets already defined and returns early instead of erroring -if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") +if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD" OR APPLE) find_package(Protobuf CONFIG REQUIRED) find_package(absl CONFIG REQUIRED) else() @@ -87,8 +87,8 @@ function(lib_install PRODUCT) ) endfunction() -# FreeBSD linker options (not supported on macOS) -if(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") +# GNU ld linker option (not supported on macOS) +if(NOT APPLE) add_link_options("-Wl,--undefined-version") endif() @@ -311,7 +311,7 @@ endif() # set larch compiler options function(larch_compile_opts PRODUCT) if(USE_NETAM) - target_compile_options(${PRODUCT} PUBLIC -march=native -std=c++23 -pipe) + target_compile_options(${PRODUCT} PUBLIC -march=native -std=c++2b -pipe) else() target_compile_options(${PRODUCT} PUBLIC -march=native -std=c++20 -pipe) endif() @@ -472,7 +472,7 @@ if(USE_NETAM) target_include_directories(netam PUBLIC include) target_include_directories(netam PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/deps/range-v3/install/include) add_dependencies(netam range-v3) - target_compile_options(netam PUBLIC -std=c++23 -Wno-unknown-warning-option) + target_compile_options(netam PUBLIC -std=c++2b -Wno-unknown-warning-option) target_compile_options(netam PRIVATE -Wall -Wextra -pedantic) target_link_libraries(netam PUBLIC fmt::fmt) target_link_libraries(netam PUBLIC ${TORCH_LIBRARIES}) @@ -489,7 +489,7 @@ if(USE_USHER) src/usher_globals.cpp ) if(USE_NETAM) - target_compile_options(usher PRIVATE -march=native -std=c++23 -w -fno-omit-frame-pointer -DUSE_USHER) + target_compile_options(usher PRIVATE -march=native -std=c++2b -w -fno-omit-frame-pointer -DUSE_USHER) else() target_compile_options(usher PRIVATE -march=native -std=c++20 -w -fno-omit-frame-pointer -DUSE_USHER) endif() diff --git a/environment-ci-linux-netam.yml b/environment-ci-linux-netam.yml new file mode 100644 index 00000000..a7abd8e0 --- /dev/null +++ b/environment-ci-linux-netam.yml @@ -0,0 +1,32 @@ +name: larch-dev +channels: + - pytorch + - conda-forge + - bioconda +dependencies: + # larch: build time + - python + - gcc >=12.4.0,<14.0.0 + - gxx >=12.4.0,<14.0.0 + - cmake>=3.19.6 + - make + - openmpi-mpicc >=4.1.0,<5.0 + - openmpi-mpicxx >=4.1.0,<5.0 + - automake + - autoconf + - libtool + - yasm + - openssh + - libprotobuf + - protobuf + # larch: run time + - openmpi >=4.1.0,<5.0 + - boost-cpp >=1.82 + - ucx >=1.16.0 + - zlib >=1.2 + - libhwloc + # netam + - pytorch-cpu + # development + - git + - git-lfs diff --git a/environment-ci-linux.yml b/environment-ci-linux.yml new file mode 100644 index 00000000..33b787c9 --- /dev/null +++ b/environment-ci-linux.yml @@ -0,0 +1,28 @@ +name: larch-dev +channels: + - conda-forge + - bioconda +dependencies: + # larch: build time + - python + - gcc >=12.4.0,<14.0.0 + - gxx >=12.4.0,<14.0.0 + - cmake>=3.19.6 + - make + - openmpi-mpicc >=4.1.0,<5.0 + - openmpi-mpicxx >=4.1.0,<5.0 + - automake + - autoconf + - libtool + - yasm + - openssh + - protobuf =3.12.3 + # larch: run time + - openmpi >=4.1.0,<5.0 + - boost-cpp =1.82 + - ucx >=1.16.0 + - zlib >=1.2 + - libhwloc + # development + - git + - git-lfs diff --git a/environment-dev-macos-netam.yml b/environment-dev-macos-netam.yml new file mode 100644 index 00000000..67af8597 --- /dev/null +++ b/environment-dev-macos-netam.yml @@ -0,0 +1,31 @@ +name: larch-dev +channels: + - pytorch + - conda-forge + - bioconda +dependencies: + # larch: build time (compiler comes from Xcode on macOS) + - cmake>=3.19.6 + - make + - openmpi-mpicc >=4.1.0,<5.0 + - openmpi-mpicxx >=4.1.0,<5.0 + - automake + - autoconf + - libtool + - openssh + - libprotobuf + - protobuf + # larch: run time + - openmpi >=4.1.0,<5.0 + - boost-cpp >=1.82 + - tbb-devel + - zlib >=1.2 + # netam + - pytorch-cpu + # development + - git + - git-lfs + - python + - rsync + - wget + - bzip2 diff --git a/environment-dev-macos.yml b/environment-dev-macos.yml new file mode 100644 index 00000000..64231f7d --- /dev/null +++ b/environment-dev-macos.yml @@ -0,0 +1,28 @@ +name: larch-dev +channels: + - conda-forge + - bioconda +dependencies: + # larch: build time (compiler comes from Xcode on macOS) + - cmake>=3.19.6 + - make + - openmpi-mpicc >=4.1.0,<5.0 + - openmpi-mpicxx >=4.1.0,<5.0 + - automake + - autoconf + - libtool + - openssh + - libprotobuf + - protobuf + # larch: run time + - openmpi >=4.1.0,<5.0 + - boost-cpp >=1.82 + - tbb-devel + - zlib >=1.2 + # development + - git + - git-lfs + - python + - rsync + - wget + - bzip2 diff --git a/include/larch/common.hpp b/include/larch/common.hpp index ec1721f0..a6a31b6e 100644 --- a/include/larch/common.hpp +++ b/include/larch/common.hpp @@ -17,6 +17,7 @@ #include #include #include +#include ////////////////////////////////////////////////////////////////////////////////////// @@ -302,8 +303,7 @@ inline constexpr const auto HashCombine = [](size_t lhs, size_t rhs) noexcept { #endif #else #define Assert(x) \ - { \ - } + {} #endif [[noreturn]] inline void Fail(const char* msg) { diff --git a/include/larch/dag/dag_storage.hpp b/include/larch/dag/dag_storage.hpp index 429f9944..fde34c5f 100644 --- a/include/larch/dag/dag_storage.hpp +++ b/include/larch/dag/dag_storage.hpp @@ -12,7 +12,7 @@ struct ExtraStorage { using FeatureTypes = std::tuple; - template