Adding a windows build only CI job

.github/workflows/build.yml

@@ -68,6 +68,61 @@ jobs:
 ##    - run: | 
 ##        dir -r D:\a\RAJA\RAJA\build\test
 ## ====================================
+## Run tests action
+      - uses: threeal/ctest-action@v1.1.0
+        with:
+          build-config: Debug
+
+  build_windows_cuda:
+    strategy:
+      matrix:
+        shared:
+          - args:
+              BUILD_SHARED_LIBS=Off
+          - args:
+              BUILD_SHARED_LIBS=On
+              CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=On
+
+  # Running on Visual Studio 2022 for better CUDA 12 compatibility
+  # windows-2025 would be locked to Visual Studio 2026
+  # windows-latest can be changed by GitHub
+    runs-on: windows-2022
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - uses: Jimver/cuda-toolkit@v0.2.35
+        with:
+          cuda: '12.9.1'
+          sub-packages: '["nvcc", "visual_studio_integration", "cudart", "nvrtc", "thrust"]'
+
+      - name: Check CUDA
+        shell: pwsh
+        run: |
+          nvcc --version
+          echo "CUDA_PATH=$env:CUDA_PATH"
+
+      - uses: threeal/cmake-action@v1.3.0
+        with:
+          build-dir: build
+          options:
+            ENABLE_WARNINGS_AS_ERRORS=Off
+            BLT_CXX_STD="c++20"
+            CMAKE_CUDA_FLAGS="-Xcompiler=/TP"
+            CMAKE_CXX_STANDARD=20
+            CMAKE_BUILD_TYPE=Release
+            CMAKE_CUDA_ARCHITECTURES=75
+            ENABLE_CUDA=On
+            ${{ matrix.shared.args }}
+          run-build: true
+#          build-args: '--parallel 16'
+## ====================================
+## Print the contents of the test directory in the build space (debugging)
+##    - run: |
+##        dir -r D:\a\RAJA\RAJA\build\test
+## ====================================
 ## Run tests action
       - uses: threeal/ctest-action@v1.1.0
         with:

.gitlab/jobs/matrix.yml

@@ -36,3 +36,8 @@ clang_18_1_8_cuda_12_6_0:
   variables:
     SPEC: " ~shared +cuda cuda_arch=90 +tests %llvm@=18.1.8 ^cuda@12.6.0+allow-unsupported-compilers"
   extends: .job_on_matrix
+
+clang_18_1_8_cuda_13_1_1:
+  variables:
+    SPEC: " ~shared +cuda cuda_arch=90 +tests %llvm@=18.1.8 ^cuda@13.1.1+allow-unsupported-compilers"
+  extends: .job_on_matrix

CMakeLists.txt

@@ -343,6 +343,12 @@ endif()
 
 if(RAJA_ENABLE_TESTS)
   add_subdirectory(test)
+else()
+  # For Spack stand-alone tests need host config information for building simple
+  # examples even if full tests are not done.
+  configure_file(
+    test/install/using-with-cmake/host-config.cmake.in
+    ${PROJECT_BINARY_DIR}/examples/using-with-cmake/host-config.cmake)
 endif()
 
 if(RAJA_ENABLE_REPRODUCERS)

cmake/thirdparty/FindCUB.cmake

@@ -14,7 +14,8 @@ find_path(CUB_INCLUDE_DIRS
   HINTS
     ${CUB_DIR}/
     ${CUB_DIR}/include
-    ${CUDA_TOOLKIT_ROOT_DIR}/include)
+    ${CUDA_TOOLKIT_ROOT_DIR}/include
+    ${CUDA_TOOLKIT_ROOT_DIR}/include/cccl)
 
 find_package_handle_standard_args(
   CUB

include/RAJA/policy/cuda/MemUtils_CUDA.hpp

@@ -146,10 +146,26 @@ struct DevicePinnedAllocator
     void* ptr;
     CAMP_CUDA_API_INVOKE_AND_CHECK(cudaMallocManaged, &ptr, nbytes,
                                    cudaMemAttachGlobal);
+#if CUDART_VERSION >= 13000
+    cudaMemLocation devLoc {};
+    devLoc.type = cudaMemLocationTypeDevice;
+    devLoc.id   = device;
+
+    CAMP_CUDA_API_INVOKE_AND_CHECK(cudaMemAdvise, ptr, nbytes,
+                                   cudaMemAdviseSetPreferredLocation, devLoc);
+
+    cudaMemLocation cpuLoc {};
+    cpuLoc.type = cudaMemLocationTypeHost;
+    cpuLoc.id   = 0;
+
+    CAMP_CUDA_API_INVOKE_AND_CHECK(cudaMemAdvise, ptr, nbytes,
+                                   cudaMemAdviseSetAccessedBy, cpuLoc);
+#else
     CAMP_CUDA_API_INVOKE_AND_CHECK(cudaMemAdvise, ptr, nbytes,
                                    cudaMemAdviseSetPreferredLocation, device);
     CAMP_CUDA_API_INVOKE_AND_CHECK(cudaMemAdvise, ptr, nbytes,
                                    cudaMemAdviseSetAccessedBy, cudaCpuDeviceId);
+#endif
 
     return ptr;
   }

test/unit/resource/tests/test-resource-AsyncTime.hpp

@@ -32,22 +32,31 @@ gpu_time_wait_for(float time, float clockrate) {
 int get_clockrate()
 {
   int cuda_device = 0;
-  cudaDeviceProp deviceProp;
   CAMP_CUDA_API_INVOKE_AND_CHECK(cudaGetDevice, &cuda_device);
-  CAMP_CUDA_API_INVOKE_AND_CHECK(cudaGetDeviceProperties, &deviceProp, cuda_device);
-  if ((deviceProp.concurrentKernels == 0))
+
+  int concurrentKernels = 0;
+  CAMP_CUDA_API_INVOKE_AND_CHECK(cudaDeviceGetAttribute,
+                                 &concurrentKernels,
+                                 cudaDevAttrConcurrentKernels,
+                                 cuda_device);
+
+  if (concurrentKernels == 0)
   {
     printf("> GPU does not support concurrent kernel execution\n");
     printf("  CUDA kernel runs will be serialized\n");
     return -1;
   }
-  //printf("> Detected Compute SM %d.%d hardware with %d multi-processors\n",
-  //    deviceProp.major, deviceProp.minor, deviceProp.multiProcessorCount);
+
+  int clockRate = 0;
+  CAMP_CUDA_API_INVOKE_AND_CHECK(cudaDeviceGetAttribute,
+                                 &clockRate,
+                                 cudaDevAttrClockRate,
+                                 cuda_device);
 
 #if defined(__arm__) || defined(__aarch64__)
-  return deviceProp.clockRate/1000;
+  return clockRate / 1000;
 #else
-  return deviceProp.clockRate;
+  return clockRate;
 #endif
 }
 

tmp-windows-cuda-notes/README.md

@@ -0,0 +1,192 @@
+# RAJA Build with CUDA on Windows
+
+## Window CUDA CI build
+
+A Windows CUDA CI job was added.   Not working since the RAJA compile is not working.
+
+See ../.github/workflows/build.yml
+
+Should adjust CUDA and MSVCC versions when a working compile.
+
+Do we want to test multiple CUDA MSVCC versions?
+
+## MS toolchain
+
+### Install Visual Studio
+
+2026   https://visualstudio.microsoft.com/downloads/
+2022   https://aka.ms/vs/17/release/vs\_community.exe
+
+Steve : tried both initially to see if one would show fewer errors.   The errors seem to be mostly the same for both so used 2026.
+
+NOTE: MS seems to remove references to old VS community versions, found above 2022 link on reddit not sure how long links remain for the old versions.
+Paid subscribers can get access to the old VS versions.
+
+### Install Cmake
+
+https://cmake.org/download/
+
+### Install Git
+
+https://git-scm.com/install/windows
+
+### Install CUDA
+
+https://developer.nvidia.com/cuda-downloads
+
+Steve: Tried both CUDA 12 and CUDA 13.   Both had issues so went with CUDA 13 since we have added support for it.
+
+##  Git stuff
+
+All generic stuff.   Added a key for GitHub.
+
+```
+ssh-keygen -t ed25519 -C "smith84@llnl.gov"
+type %userprofile%\\.ssh\\id\_ed25519.pub | clip
+
+git clone --recursive git@github.com:llnl/RAJA.git
+```
+
+## Failed attempts to build.
+
+Steve : Tried combinations of MSVCC 2022 and 2026 with Cuda 12 and 13 to see if some combination might work....needed to start deeper patching.
+
+## Steve patching
+
+### BLT
+
+Apply the blt.patch to fix argument passing to MSVCC.   nvcc does not pass "/arg" to MSVCC by default which causes nvcc to interpret the arguments as directories/files.
+
+Kenny has similar patch for blt in AXOM branch.
+
+```
+
+cd blt
+
+git apply ..\\tmp-windows-cuda-notes\\blt.patch
+
+```
+
+### Camp 
+
+MSVCC has issue parsing variadic macros.   Uses non-standard CPP parser.   Did come up with patch for CAMP but Nvidia CUDA 13 recommends using the "-Xcompiler=/Zc:preprocessor" flag which uses more standard conforming CPP.   Seemed more reasonable approach.
+
+Camp patch is in camp-variadic.patch.
+
+## RAJA config/build
+
+CUDA 13 recommends the preprocessor flag and fix a CAMP issue.   MSVCC uses some non-standard conforming parsing if this is not done.
+
+"/TP" treats .C files as C++.   Was fixing some issues with CUDA 12 toolkit during initial attempts but should double check this requirement.
+
+```
+cmake -B cudabuild RAJA -DENABLE\_CUDA=ON -DCMAKE\_CUDA\_FLAGS=" --keep  -Xcompiler=/TP -Xcompiler=/Zc:preprocessor" -DCMAKE\_CUDA\_ARCHITECTURES=75
+
+```
+
+```
+cmake --build cudabuild --target atomic-histogram\_solution > zz.build.txt 2>\&1
+```
+
+Steve: reached a point with a template issue, which above build exposes.
+
+```
+ atomic-histogram\_solution.cpp
+  atomic-histogram\_solution.cudafe1.cpp
+C:\\Users\\smith84\\projects\\raja\\clean\\RAJA\\include\\RAJA/index/IndexSet.hpp(754): warning C4244: 'return': conversion from '\_\_int64' to 'int', possible
+ loss of data \[C:\\Users\\smith84\\projects\\raja\\clean\\cudabuild\\exercises\\atomic-histogram\_solution.vcxproj]
+C:\\Users\\smith84\\projects\\raja\\clean\\RAJA\\include\\RAJA/index/IndexSet.hpp(759): warning C4244: 'return': conversion from 'const \_\_int64' to 'int', po
+ssible loss of data \[C:\\Users\\smith84\\projects\\raja\\clean\\cudabuild\\exercises\\atomic-histogram\_solution.vcxproj]
+C:\\Users\\smith84\\projects\\raja\\clean\\cudabuild\\exercises\\atomic-h.9E0645A7\\x64\\Debug\\atomic-histogram\_solution.cudafe1.stub.c(37): error C2912: expli
+cit specialization 'void RAJA::policy::cuda::impl::\_\_wrapper\_\_device\_stub\_forallp\_cuda\_kernel<RAJA::policy::cuda::cuda\_exec\_explicit<RAJA::iteration\_
+mapping::Direct,RAJA::cuda::IndexGlobal<RAJA::named\_dim::x,256,0>,RAJA::CudaDefaultConcretizer,1,false>,1,RAJA::Iterators::numeric\_iterator<StorageT,
+DiffT,Type \*>,\_\_nv\_dl\_wrapper\_t<\_\_nv\_dl\_tag<int (\_\_cdecl \*)(int,char \*\*),\&int main(int,char \*\*),1>,int \*,int \*>,int,RAJA::expt::ForallParamPack<>,RAJ
+A::iteration\_mapping::Direct,RAJA::cuda::IndexGlobal<RAJA::named\_dim::x,256,0>,256>(const \_ZN4camp5decayIZ4mainEUliE0\_EE \&,const \_ZN4camp5decayIN4RAJ
+A9Iterators16numeric\_iteratorIiiPiEEEE \&,const \_ZN4camp5decayIiEE \&,\_ZN4camp5decayIN4RAJA4expt15ForallParamPackIJEEEEE \&)' is not a specialization of
+&#x20;a function template \[C:\\Users\\smith84\\projects\\raja\\clean\\cudabuild\\exercises\\atomic-histogram\_solution.vcxproj]
+&#x20;         with
+&#x20;         \[
+&#x20;             StorageT=int,
+&#x20;             DiffT=int,
+&#x20;             Type=int
+&#x20;         ]
+C:\\Program Files\\Microsoft Visual Studio\\18\\Community\\MSBuild\\Microsoft\\VC\\v180\\BuildCustomizations\\CUDA 13.3.targets(804,9): error MSB3721: The comm
+and ""C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v13.3\\bin\\nvcc.exe"  --use-local-env -ccbin "C:\\Program Files\\Microsoft Visual Studio\\18\\Com
+munity\\VC\\Tools\\MSVC\\14.51.36231\\bin\\HostX64\\x64" -x cu   -IC:\\Users\\smith84\\projects\\raja\\clean\\RAJA\\include -IC:\\Users\\smith84\\projects\\raja\\clean\\
+cudabuild\\include -IC:\\Users\\smith84\\projects\\raja\\clean\\RAJA\\tpl\\camp\\include -IC:\\Users\\smith84\\projects\\raja\\clean\\cudabuild\\tpl\\camp\\include -I"C
+:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v13.3\\include" -I"C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v13.3\\include\\cccl" -I"C:\\Prog
+ram Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v13.3\\include"  -G  --keep --keep-dir atomic-h.9E0645A7\\x64\\Debug  -maxrregcount=0     --machine 64 --com
+pile -cudart static -forward-unknown-to-host-compiler -restrict --expt-extended-lambda --expt-relaxed-constexpr -Xcudafe --display\_error\_number -O0 -
+std=c++20 --generate-code=arch=compute\_75,code=\[compute\_75,sm\_75] -Xcompiler="/TP /Zc:preprocessor" -g  -DRAJA\_WIN\_STATIC\_BUILD -DCAMP\_WIN\_STATIC\_BUI
+LD -D"CMAKE\_INTDIR=\\"Debug\\"" -D\_MBCS -DWIN32 -D\_WINDOWS -DRAJA\_WIN\_STATIC\_BUILD -DCAMP\_WIN\_STATIC\_BUILD -D"CMAKE\_INTDIR=\\"Debug\\"" -Xcompiler "/EHsc
+&#x20;/W3 /nologo  /FS /Zi /RTC1 /MTd /GR" -Xcompiler "/Fdatomic-histogram\_solution.dir\\Debug\\vc145.pdb" -o atomic-histogram\_solution.dir\\Debug\\atomic-his
+togram\_solution.obj "C:\\Users\\smith84\\projects\\raja\\clean\\RAJA\\exercises\\atomic-histogram\_solution.cpp"" exited with code 2. \[C:\\Users\\smith84\\projec
+ts\\raja\\clean\\cudabuild\\exercises\\atomic-histogram\_solution.vcxproj]
+```
+
+## Kenny Weiss work on AXOM
+
+Steve : Learned of Kenny branch of AXOM.    Kenny took Steve's CUDA 13 branch and made patches with extra fixes for Windows.   Steve was starting to parse through Kenny patching.   The template issue above might be have been similar to what is patched in "kweiss-windows-cuda-msvc.patch" pulled from Kenny's repository.   Current RAJA has modified the templates so this needs some work.
+
+Steve goes on vacation :(
+
+Kenny/AI found some other changes, note Kenny was using VS 2022 and Cuda 13.
+
+## Kenny Notes:
+
+Hi Steve,
+
+I do!
+
+(CC-ing Rich and Brian Han since they’re also likely interested in the details)
+
+I’m still working through some of the details, so I haven’t created a PR yet, but the branch is here: https://github.com/llnl/axom/compare/develop...feature/kweiss/windows-gpu
+
+I used codex w/ gpt@5.5 to work through a bunch of the issues. The compilation times on my laptop (6 core i7; I forget the GPU, but it was CUDA\_ARCHITECTURE 75) can be really long, so each round took a few hours, but it eventually got there.
+
+Axom uses vcpkg (https://github.com/microsoft/vcpkg)  through uberenv ( https://github.com/llnl/uberenv/  |  https://uberenv.readthedocs.io/en/latest/)
+
+to manage our TPLs on windows. We manage some local ports of raja, umpire, mfem, conduit, …  inside of axom’s   ./scripts/vcpkg\_ports  directory.
+
+(ports are equivalent to spack package recipes)
+
+To get this working w/ uberenv, I had to add some slight edits: https://github.com/llnl/uberenv/pull/160 -- which should be merged soon
+
+I also have a few changes to BLT, but I haven’t yet looked closely enough to determine which are strictly necessary, and haven’t posted a PR/branch to BLT’s github yet
+
+My branch has a bunch of updates to our vcpkg ports  to get raja, umpire and camp to build on msvc+cuda.
+
+Since my build uses cuda@13, I needed some patches from your raja and camp PRs related to cuda@13.
+
+To set up my build environment on Windows, I had to install:
+
+the cuda toolkit ( https://developer.nvidia.com/cuda-downloads --  I am using cuda@13)
+
+MS visual studio community ( https://visualstudio.microsoft.com/vs/community/  --
+
+I am using visual studio community 2022. Presumably it’d work w/ visual studio community 2026 ).
+
+If you need MPI, you also need to install MS MPI -- https://learn.microsoft.com/en-us/message-passing-interface/microsoft-mpi
+
+Some gotchas in my msvc+cuda porting:
+
+cuda@13 has some changes (but you’re clearly aware of those)
+
+MSVC does not support extended lambdas inside of if constexpr --
+
+See item 11 in Section 5.3.8.4  -- https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/cpp-language-support.html#extended-lambda-restrictions
+
+Microsoft defines some regular words as macros, e.g.  min, max, small, large
+
+I hope this helps and would be happy to discuss this further/answer questions,
+
+Kenny
+
+\--
+
+Kenneth Weiss
+
+Numerics, Modular Applications and Performance Group Leader
+
+ASQ Division, Computing Directorate, LLNL
+

tmp-windows-cuda-notes/blt.patch

@@ -0,0 +1,28 @@
+diff --git a/thirdparty_builtin/CMakeLists.txt b/thirdparty_builtin/CMakeLists.txt
+index 74a4f8d..56c59a3 100644
+--- a/thirdparty_builtin/CMakeLists.txt
++++ b/thirdparty_builtin/CMakeLists.txt
+@@ -111,10 +111,19 @@ if(ENABLE_TESTS)
+         #
+         # For more info see: https://github.com/LLNL/blt/issues/79
+         # 
+-        
+-        blt_append_custom_compiler_flag( FLAGS_VAR gtest_extra_flags
+-                                         DEFAULT " " 
+-                                         MSVC  "/wd4251" "/wd4275" )
++        if(MSVC AND ENABLE_CUDA)
++           # For CUDA nvcc compiler pass arg to MSVCC with -Xcompiler
++           set(gtest_extra_flags
++              "$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/wd4251>"
++              "$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/wd4275>"
++              "$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/wd4251>"
++              "$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/wd4275>"
++              )
++         else()
++           blt_append_custom_compiler_flag( FLAGS_VAR gtest_extra_flags
++                                            DEFAULT " "
++                                            MSVC  "/wd4251" "/wd4275" )
++         endif()
+
+         set (INSTALL_GMOCK OFF CACHE BOOL "")
+         set (INSTALL_GTEST OFF CACHE BOOL "")