Arrow 16 upgrade

.github/workflows/c-cpp.yml

@@ -13,7 +13,7 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/conda-actions.yml

@@ -22,7 +22,7 @@ jobs:
       matrix:
         include:
           # 20.04 supports CUDA 11.0+
-          - os: ubuntu-20.04
+          - os: ubuntu-22.04
             cuda: "11.5.2"
             gcc: 9
 
@@ -51,6 +51,8 @@ jobs:
 
     - uses: conda-incubator/setup-miniconda@v2
       with:
+        miniconda-version: "latest"  # You can specify a version or leave it as "latest"
+        auto-update-conda: true
         activate-environment: gcylon_dev
         environment-file: conda/environments/gcylon.yml
 

.github/workflows/conda-cpp-redis.yml

@@ -22,7 +22,7 @@ jobs:
       matrix:
         include:
           # 20.04 supports CUDA 11.0+
-          - os: ubuntu-20.04
+          - os: ubuntu-22.04
             gcc: 9
             ucc: "master"
 
@@ -38,6 +38,8 @@ jobs:
 
       - uses: conda-incubator/setup-miniconda@v2
         with:
+          miniconda-version: "latest"  # You can specify a version or leave it as "latest"
+          auto-update-conda: true
           activate-environment: cylon_dev
           environment-file: conda/environments/cylon.yml
 

.github/workflows/conda-cpp.yml

@@ -22,9 +22,9 @@ jobs:
       matrix:
         include:
           # 20.04 supports CUDA 11.0+
-          - os: ubuntu-20.04
+          - os: ubuntu-22.04
             gcc: 9
-            ucc: "master"
+            ucc: "ucc-v1.2.0"
             ucx: "override-remote-address3"
 
     steps:
@@ -33,12 +33,14 @@ jobs:
       # Specify the correct host compilers
       - name: Install/Select gcc and g++
         run: |
-          sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} git
+          sudo apt-get install -y wget bzip2 ca-certificates curl git build-essential
           echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV
           echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV
 
       - uses: conda-incubator/setup-miniconda@v2
         with:
+          miniconda-version: "latest"  # You can specify a version or leave it as "latest"
+          auto-update-conda: true
           activate-environment: cylon_dev
           environment-file: conda/environments/cylon_NoUCX.yml
 
@@ -49,7 +51,7 @@ jobs:
         run: |
           git clone --single-branch https://github.com/nirandaperera/gloo.git $HOME/gloo
           mkdir -p $HOME/gloo/build && cd $HOME/gloo/build
-          cmake .. -DBUILD_SHARED_LIBS=1 -DUSE_MPI=1 -DCMAKE_INSTALL_PREFIX=$HOME/gloo/install
+          cmake .. -DBUILD_SHARED_LIBS=1 -DUSE_MPI=1 -DCMAKE_INSTALL_PREFIX=$HOME/gloo/install -DCMAKE_POLICY_VERSION_MINIMUM=3.5
           make install
 
       - name: Install UCX
@@ -62,7 +64,7 @@ jobs:
 
       - name: Install UCC
         run: |
-          git clone --single-branch -b ${{ matrix.ucc }} https://github.com/openucx/ucc.git $HOME/ucc
+          git clone --single-branch -b ${{ matrix.ucc }} https://github.com/mstaylor/ucc.git $HOME/ucc
           cd $HOME/ucc
           ./autogen.sh
           ./configure --prefix=$HOME/ucc/install --with-ucx=$HOME/ucx/install

.gitignore

@@ -38,6 +38,7 @@
 
 # ignore these folders
 build/
+debug/
 **/bin/
 
 

conda/environments/cylon.yml

@@ -3,19 +3,22 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python>=3.8,<3.10
-  - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=9
-  - pyarrow=9.0.0
+  - python>=3.9,<3.12
+  - cmake
+  - pyarrow=16.1.0
+  - libarrow-acero==16.1.0.*
+  - libarrow-dataset==16.1.0.*
+  - libarrow==16.1.0.*
   - glog
-  - openmpi=4.1.3=ha1ae619_105
-  - ucx>=1.12.1
-  - cython>=0.29.31,<3
-  - numpy<1.24.4
-  - pandas>=1.0,<2.0.0
+  - openmpi
+  - ucx
+  - cython>=0.29.31
+  - numpy>=1.23,<2.0a0
+  - pandas>=2.0,<2.2.3dev
   - fsspec>=0.6.0
   - setuptools
   # they are not needed for using pygcylon or compiling it
   - pytest
   - pytest-mpi
   - mpi4py
+

conda/environments/cylon_NoUCX.yml

@@ -3,15 +3,17 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python>=3.8,<3.10
-  - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=9
-  - pyarrow=9.0.0
+  - python>=3.9,<3.12
+  - cmake
+  - pyarrow=16.1.0
+  - libarrow-acero==16.1.0.*
+  - libarrow-dataset==16.1.0.*
+  - libarrow==16.1.0.*
   - glog
-  - openmpi=4.1.3=ha1ae619_105
-  - cython>=0.29.31,<3
-  - numpy<1.24.4
-  - pandas>=1.0,<2.0.0
+  - openmpi
+  - cython>=0.29.31
+  - numpy>=1.23,<2.0a0
+  - pandas>=2.0,<2.2.3dev
   - fsspec>=0.6.0
   - setuptools
   # they are not needed for using pygcylon or compiling it

cpp/CMake/Modules/SetupCxxFlags.cmake

@@ -117,9 +117,9 @@ if(CYLON_CPU_FLAG STREQUAL "armv8")
     if(NOT CXX_SUPPORTS_ARMV8_ARCH)
         message(FATAL_ERROR "Unsupported arch flag: ${CYLON_ARMV8_ARCH_FLAG}.")
     endif()
-    if(CYLON_ARMV8_ARCH_FLAG MATCHES "native")
-        message(FATAL_ERROR "native arch not allowed, please specify arch explicitly.")
-    endif()
+#    if(CYLON_ARMV8_ARCH_FLAG MATCHES "native")
+#        message(FATAL_ERROR "native arch not allowed, please specify arch explicitly.")
+    #endif()
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} ${CYLON_ARMV8_ARCH_FLAG}")
 
     add_definitions(-DCYLON_HAVE_NEON)

cpp/CMakeLists.txt

@@ -24,17 +24,18 @@
 cmake_minimum_required(VERSION 3.17 FATAL_ERROR)
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-project(CYLON VERSION 0.6.0)
+set(CYLON_VERSION 0.7.0)
+
+project(CYLON VERSION ${CYLON_VERSION})
 
-set(CYLON_VERSION 0.6.0)
 
 ## defaults to release build
 if (NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release)
 endif ()
 
 # cmake modules directories
-set(CYLON_ARROW_VERSION 9.0.0)
+set(CYLON_ARROW_VERSION 16.1.0)
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake/Modules/" ${CMAKE_MODULE_PATH})
 list(APPEND CMAKE_MODULE_PATH ${CYLON_SOURCE_DIR}/CMake)
 
@@ -74,7 +75,7 @@ else ()
 endif ()
 
 # C++ standard
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(GCC_ABI_COMPILE_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0")
@@ -398,7 +399,19 @@ if (${ARROW_BUILD_TYPE} STREQUAL "SYSTEM")
     message(STATUS "Arrow dataset lib: ${ARROW_DATASET_LIB}")
 
     if (PYCYLON_BUILD)
-        find_library(ARROW_PY_LIB arrow_python ${CYLON_ARROW_VERSION} REQUIRED)
+        find_package (Python3 COMPONENTS Interpreter)
+        write_file("find_site_packages.py" [=[import sysconfig; print(sysconfig.get_paths()["purelib"])]=])
+        execute_process(COMMAND ${Python3_EXECUTABLE} "find_site_packages.py"   
+            RESULT_VARIABLE PROC_RES 
+            OUTPUT_VARIABLE PROC_OUT
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            ERROR_STRIP_TRAILING_WHITESPACE)
+        if (PROC_RES)
+            message(FATAL_ERROR "Unable to find python site packages dir: ${PROC_OUT}")
+        else()
+            message(STATUS "python site packages dir: ${PROC_OUT}")
+        endif()
+        find_library(ARROW_PY_LIB arrow_python PATHS "${PROC_OUT}" PATH_SUFFIXES "pyarrow" REQUIRED NO_DEFAULT_PATH)
         message(STATUS "Arrow py lib: ${ARROW_PY_LIB}")
     endif (PYCYLON_BUILD)
 

cpp/src/cylon/arrow/arrow_comparator.cpp

@@ -150,7 +150,7 @@ struct CompareFunc<ArrowT, Asc, arrow::enable_if_floating_point<ArrowT>> {
 template<typename ArrowT, bool Asc>
 struct CompareFunc<ArrowT, Asc, arrow::enable_if_has_string_view<ArrowT>> {
 
-  static int compare(const arrow::util::string_view &v1, const arrow::util::string_view &v2) {
+  static int compare(const std::string_view &v1, const std::string_view &v2) {
     if (Asc) {
       return v1.compare(v2);
     } else {