diff --git a/CMakeLists.txt b/CMakeLists.txt index df84df2d7c..e3d93f26ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -333,6 +333,10 @@ install(FILES ${PROJECT_BINARY_DIR}/include/RAJA/config.hpp DESTINATION "include/RAJA/") +install(FILES + ${PROJECT_BINARY_DIR}/include/RAJA/policy/device.hpp + DESTINATION "include/RAJA/policy/") + # Setup internal RAJA configuration options include(cmake/SetupRajaConfig.cmake) diff --git a/cmake/RAJAMacros.cmake b/cmake/RAJAMacros.cmake index 6e516a1905..22a0e03bcc 100644 --- a/cmake/RAJAMacros.cmake +++ b/cmake/RAJAMacros.cmake @@ -19,7 +19,7 @@ macro(raja_link_include_proteus) add_proteus(${arg_NAME}) target_include_directories(${arg_NAME} PUBLIC - "${PROTEUS_HEADERS_DIR}" + $ ) endif() endmacro(raja_link_include_proteus) diff --git a/cmake/SetupProteus.cmake b/cmake/SetupProteus.cmake index 68798dd48d..b86eee6b62 100644 --- a/cmake/SetupProteus.cmake +++ b/cmake/SetupProteus.cmake @@ -20,7 +20,7 @@ else() FetchContent_Declare( proteus GIT_REPOSITORY https://github.com/Olympus-HPC/proteus.git - GIT_TAG 257707cf7e60452ed38161f6429421be303ddaf3 + GIT_TAG 44415066d3c91729a23137101a0f08a170101fb5 ) FetchContent_MakeAvailable(proteus) # Re-enable tests if specified by user. @@ -30,4 +30,16 @@ else() endif() # We don't explicitly link against ProteusPass, but it is required to be #available as an LLVM pass, so manually enforce order -add_dependencies(RAJA ProteusPass) \ No newline at end of file +target_include_directories(RAJA + PUBLIC + $ + $) + +# RAJA's public JIT headers include Proteus headers, so install those headers +# into RAJA's include tree instead of exporting a build-tree path. +if (EXISTS "${PROTEUS_HEADERS_DIR}/proteus") + install(DIRECTORY "${PROTEUS_HEADERS_DIR}/proteus" + DESTINATION include) +endif() + +add_dependencies(RAJA ProteusPass LambdaPass) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index e77b5dc493..61b8d61a08 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -27,6 +27,12 @@ if (RAJA_ENABLE_JIT) raja_add_executable( NAME forall-jit SOURCES forall-jit.cpp) + raja_add_executable( + NAME kernel-jit + SOURCES kernel-jit.cpp) + raja_add_executable( + NAME launch-jit + SOURCES launch-jit.cpp) endif() raja_add_executable( diff --git a/examples/kernel-jit.cpp b/examples/kernel-jit.cpp new file mode 100644 index 0000000000..ac786968cc --- /dev/null +++ b/examples/kernel-jit.cpp @@ -0,0 +1,152 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// This example mirrors forall-jit.cpp, but uses RAJA::kernel to drive the +// outer parallel loop. + +#include +#include + +#include "RAJA/RAJA.hpp" +#include "RAJA/util/Timer.hpp" + +#if defined(RAJA_ENABLE_CUDA) +using kernel_policy = RAJA::KernelPolicy< + RAJA::statement::CudaKernelFixed<256, + RAJA::statement::Tile<0, RAJA::tile_fixed<256>, + RAJA::cuda_block_x_direct, + RAJA::statement::For<0, RAJA::cuda_thread_x_direct, + RAJA::statement::Lambda<0>>>>>; +#elif defined(RAJA_ENABLE_HIP) +using kernel_policy = RAJA::KernelPolicy< + RAJA::statement::HipKernelFixed<256, + RAJA::statement::Tile<0, RAJA::tile_fixed<256>, + RAJA::hip_block_x_direct, + RAJA::statement::For<0, RAJA::hip_thread_x_direct, + RAJA::statement::Lambda<0>>>>>; +#elif defined(RAJA_ENABLE_OPENMP) +using kernel_policy = RAJA::KernelPolicy< + RAJA::statement::For<0, RAJA::omp_parallel_for_exec, + RAJA::statement::Lambda<0>>>; +#else +using kernel_policy = RAJA::KernelPolicy< + RAJA::statement::For<0, RAJA::seq_exec, RAJA::statement::Lambda<0>>>; +#endif + +int main(int argc, char **argv) { + if (argc < 5) { + std::cout + << "Expected ./bin/kernel-jit " + ", for example ./bin/kernel-jit 24 16 1000000 1\n"; + return 1; + } + + int a = std::stoi(argv[1]); + int b = std::stoi(argv[2]); + int N = std::stoi(argv[3]); + bool accum = std::stoi(argv[4]); + +#ifdef RAJA_ENABLE_CUDA + using resource_policy = RAJA::cuda_exec<256>; +#elif defined(RAJA_ENABLE_HIP) + using resource_policy = RAJA::hip_exec<256>; +#elif defined(RAJA_ENABLE_OPENMP) + using resource_policy = RAJA::omp_parallel_for_exec; +#else + using resource_policy = RAJA::seq_exec; +#endif + + auto res = RAJA::resources::get_default_resource(); + double *A_ptr = res.template allocate(N * a * b); + auto A = RAJA::make_permuted_view(A_ptr, N, a, b); + double *B_ptr = res.template allocate(N * a * b); + auto B = RAJA::make_permuted_view(B_ptr, N, b, a); + double *C_ptr = res.template allocate(N * a * a); + auto C = RAJA::make_permuted_view(C_ptr, N, a, a); + + auto Seg = RAJA::RangeSegment(0, N); + + RAJA::kernel(RAJA::make_tuple(Seg), [=](int i) { + for (int row = 0; row < a; ++row) { + for (int col = 0; col < b; ++col) { + A(i, row, col) = 0; + B(i, row, col) = 0; + C(i, row, col) = 0; + } + } + }); + + RAJA::Timer aot_timer; + proteus::disable(); + aot_timer.start(); + + RAJA::kernel(RAJA::make_tuple(Seg), [=](int i) { + for (int row = 0; row < a; ++row) { + for (int col = 0; col < b; ++col) { + A(i, row, col) = i % row; + B(i, row, col) = i % col; + C(i, row, col) = i % col; + } + } + }); + + RAJA::kernel(RAJA::make_tuple(Seg), [=](int i) { + for (int row = 0; row < a; ++row) { + for (int col = 0; col < b; ++col) { + if (!accum) { + C(i, row, col) = A(i, row, col) * B(i, col, row); + } else { + C(i, row, col) += A(i, row, col) * B(i, col, row); + } + } + } + }); + + aot_timer.stop(); + proteus::enable(); + std::cout << "aot total time = " << aot_timer.elapsed() << "\n"; + + RAJA::Timer jit_timer; + jit_timer.start(); + + RAJA::kernel( + RAJA::make_tuple(Seg), + [=, a = RAJA_JIT_VARIABLE(a), b = RAJA_JIT_VARIABLE(b)](int i) + RAJA_JIT_COMPILE { + for (int row = 0; row < a; ++row) { + for (int col = 0; col < b; ++col) { + A(i, row, col) = i % row; + B(i, row, col) = i % col; + C(i, row, col) = i % col; + } + } + }); + + RAJA::kernel( + RAJA::make_tuple(Seg), + [=, a = RAJA_JIT_VARIABLE(a), b = RAJA_JIT_VARIABLE(b), + accum = RAJA_JIT_VARIABLE(accum)](int i) RAJA_JIT_COMPILE { + for (int row = 0; row < a; ++row) { + for (int col = 0; col < b; ++col) { + if (!accum) { + C(i, row, col) = A(i, row, col) * B(i, col, row); + } else { + C(i, row, col) += A(i, row, col) * B(i, col, row); + } + } + } + }); + + jit_timer.stop(); + + std::cout << "jit total time = " << jit_timer.elapsed() << "\n"; + std::cout << "speedup = " << aot_timer.elapsed() / jit_timer.elapsed() + << "\n"; + + return 0; +} diff --git a/examples/launch-jit.cpp b/examples/launch-jit.cpp new file mode 100644 index 0000000000..45a9077539 --- /dev/null +++ b/examples/launch-jit.cpp @@ -0,0 +1,186 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// This example mirrors forall-jit.cpp, but uses RAJA::launch and +// RAJA::loop to drive the outer parallel loop. + +#include +#include + +#include "RAJA/RAJA.hpp" +#include "RAJA/util/Timer.hpp" + +using host_launch_policy = +#if defined(RAJA_ENABLE_OPENMP) + RAJA::omp_launch_t; +#else + RAJA::seq_launch_t; +#endif + +#if defined(RAJA_ENABLE_CUDA) +using launch_policy = + RAJA::LaunchPolicy>; +#elif defined(RAJA_ENABLE_HIP) +using launch_policy = + RAJA::LaunchPolicy>; +#else +using launch_policy = RAJA::LaunchPolicy; +#endif + +using loop_policy = RAJA::seq_exec; + +#if defined(RAJA_ENABLE_CUDA) +using gpu_global_thread_x_policy = RAJA::cuda_global_thread_x; +#elif defined(RAJA_ENABLE_HIP) +using gpu_global_thread_x_policy = RAJA::hip_global_thread_x; +#elif defined(RAJA_ENABLE_OPENMP) +using gpu_global_thread_x_policy = RAJA::omp_for_exec; +#endif + +using global_thread_x = RAJA::LoopPolicy< + loop_policy +#if defined(RAJA_ENABLE_CUDA) || defined(RAJA_ENABLE_HIP) || \ + defined(RAJA_ENABLE_OPENMP) + , + gpu_global_thread_x_policy +#endif + >; + +int main(int argc, char **argv) { + if (argc < 5) { + std::cout + << "Expected ./bin/launch-jit " + ", for example ./bin/launch-jit 24 16 1000000 1\n"; + return 1; + } + + int a = std::stoi(argv[1]); + int b = std::stoi(argv[2]); + int N = std::stoi(argv[3]); + bool accum = std::stoi(argv[4]); + +#ifdef RAJA_ENABLE_CUDA + using resource_policy = RAJA::cuda_exec<256>; +#elif defined(RAJA_ENABLE_HIP) + using resource_policy = RAJA::hip_exec<256>; +#elif defined(RAJA_ENABLE_OPENMP) + using resource_policy = RAJA::omp_parallel_for_exec; +#else + using resource_policy = RAJA::seq_exec; +#endif + + auto res = RAJA::resources::get_default_resource(); + double *A_ptr = res.template allocate(N * a * b); + auto A = RAJA::make_permuted_view(A_ptr, N, a, b); + double *B_ptr = res.template allocate(N * a * b); + auto B = RAJA::make_permuted_view(B_ptr, N, b, a); + double *C_ptr = res.template allocate(N * a * a); + auto C = RAJA::make_permuted_view(C_ptr, N, a, a); + + auto Seg = RAJA::RangeSegment(0, N); + RAJA::LaunchParams Params( + RAJA::Teams((N + 255) / 256 == 0 ? 1 : (N + 255) / 256), + RAJA::Threads(256)); + + RAJA::launch(Params, [=] RAJA_HOST_DEVICE( + RAJA::LaunchContext ctx) { + RAJA::loop(ctx, Seg, [&](int i) { + for (int row = 0; row < a; ++row) { + for (int col = 0; col < b; ++col) { + A(i, row, col) = 0; + B(i, row, col) = 0; + C(i, row, col) = 0; + } + } + }); + }); + + RAJA::Timer aot_timer; + proteus::disable(); + aot_timer.start(); + + RAJA::launch(Params, [=] RAJA_HOST_DEVICE( + RAJA::LaunchContext ctx) { + RAJA::loop(ctx, Seg, [&](int i) { + for (int row = 0; row < a; ++row) { + for (int col = 0; col < b; ++col) { + A(i, row, col) = (row == 0) ? 0 : (i % row); + B(i, row, col) = (col == 0) ? 0 : (i % col); + C(i, row, col) = (col == 0) ? 0 : (i % col); + } + } + }); + }); + + RAJA::launch(Params, [=] RAJA_HOST_DEVICE( + RAJA::LaunchContext ctx) { + RAJA::loop(ctx, Seg, [&](int i) { + for (int row = 0; row < a; ++row) { + for (int col = 0; col < b; ++col) { + if (!accum) { + C(i, row, col) = A(i, row, col) * B(i, col, row); + } else { + C(i, row, col) += A(i, row, col) * B(i, col, row); + } + } + } + }); + }); + + aot_timer.stop(); + std::cout << "aot total time = " << aot_timer.elapsed() << "\n"; + proteus::enable(); + + RAJA::Timer jit_timer; + jit_timer.start(); + + RAJA::launch( + Params, + [=, a = RAJA_JIT_VARIABLE(a), b = RAJA_JIT_VARIABLE(b), + accum = RAJA_JIT_VARIABLE(accum)] + RAJA_JIT_COMPILE RAJA_HOST_DEVICE (RAJA::LaunchContext ctx) { + RAJA::loop( + ctx, Seg, + [&](int i) { + for (int row = 0; row < a; ++row) { + for (int col = 0; col < b; ++col) { + A(i, row, col) = (row == 0) ? 0 : (i % row); + B(i, row, col) = (col == 0) ? 0 : (i % col); + C(i, row, col) = (col == 0) ? 0 : (i % col); + } + } + }); + }); + + RAJA::launch( + Params, + [=, a = RAJA_JIT_VARIABLE(a), b = RAJA_JIT_VARIABLE(b), + accum = RAJA_JIT_VARIABLE(accum)] RAJA_JIT_COMPILE RAJA_HOST_DEVICE (RAJA::LaunchContext ctx) { + RAJA::loop( + ctx, Seg, + [&](int i) { + for (int row = 0; row < a; ++row) { + for (int col = 0; col < b; ++col) { + if (!accum) { + C(i, row, col) = A(i, row, col) * B(i, col, row); + } else { + C(i, row, col) += A(i, row, col) * B(i, col, row); + } + } + } + }); + }); + + jit_timer.stop(); + + std::cout << "jit total time = " << jit_timer.elapsed() << "\n"; + std::cout << "speedup = " << aot_timer.elapsed() / jit_timer.elapsed() + << "\n"; + + return 0; +} diff --git a/include/RAJA/pattern/kernel/internal/LoopData.hpp b/include/RAJA/pattern/kernel/internal/LoopData.hpp index ad9aea0cfc..f951902735 100644 --- a/include/RAJA/pattern/kernel/internal/LoopData.hpp +++ b/include/RAJA/pattern/kernel/internal/LoopData.hpp @@ -24,6 +24,7 @@ #include "RAJA/config.hpp" #include "RAJA/index/IndexSet.hpp" +#include "RAJA/util/Jit.hpp" #include "RAJA/util/macros.hpp" #include "RAJA/util/types.hpp" @@ -178,6 +179,50 @@ struct LoopData RAJA_HOST_DEVICE RAJA_INLINE Resource get_resource() { return res; } }; +namespace jit { + template + inline auto __register_bodies_impl(camp::tuple const bodies, + std::index_sequence) + { + return camp::make_tuple( + RAJA::internal::jit::register_lambda(camp::get(bodies))...); + } + + template + inline auto __register_loop_bodies_impl( + LoopData body, + std::index_sequence) + { + auto registered_bodies = __register_bodies_impl( + body.bodies, std::index_sequence_for {}); + return LoopData(body.segment_tuple, + body.param_tuple, + body.get_resource(), + camp::get(registered_bodies)...); + } + + template + inline auto register_loop_bodies( + LoopData body) + { + #if defined RAJA_ENABLE_JIT + return __register_loop_bodies_impl( + std::move(body), std::index_sequence_for {}); + #else + return std::forward>(body); + #endif + } + +} + + template using segment_diff_type = typename std::iterator_traits< typename camp::at_v(rest_of_launch_args)...); auto&& launch_body = - expt::get_lambda(std::forward(rest_of_launch_args)...); + RAJA::internal::jit::register_lambda(expt::get_lambda(std::forward(rest_of_launch_args)...)); // Take the first policy as we assume the second policy is not user defined. // We rely on the user to pair launch and loop policies correctly. @@ -400,7 +400,7 @@ resources::EventProxy launch( expt::get_kernel_name(std::forward(rest_of_launch_args)...); auto&& launch_body = - expt::get_lambda(std::forward(rest_of_launch_args)...); + RAJA::internal::jit::register_lambda(expt::get_lambda(std::forward(rest_of_launch_args)...)); ExecPlace place; if (res.get_platform() == RAJA::Platform::host) diff --git a/include/RAJA/policy/cuda/forall.hpp b/include/RAJA/policy/cuda/forall.hpp index 34f7a47b5a..ac37972ee9 100644 --- a/include/RAJA/policy/cuda/forall.hpp +++ b/include/RAJA/policy/cuda/forall.hpp @@ -560,7 +560,8 @@ forall_impl(resources::Cuda cuda_res, // Note: we cannot fully remove enable_if_t from this file because NVCC is // not capable of disambiguating conceptified versions of // forallp_cuda_kernel when they are used by address like below. - RAJA::internal::jit::register_lambda(loop_body); + auto registered_body = RAJA::internal::jit::register_lambda(loop_body); + using RegisteredLambdaType = std::decay_t; auto func = reinterpret_cast( &impl::forallp_cuda_kernel>); @@ -590,7 +591,7 @@ forall_impl(resources::Cuda cuda_res, // LOOP_BODY body = RAJA::cuda::make_launch_body( func, dims.blocks, dims.threads, shmem, cuda_res, - std::forward(loop_body)); + std::forward(registered_body)); // // Launch the kernels @@ -645,7 +646,7 @@ RAJA_INLINE resources::EventProxy forall_impl( LoopBody&& loop_body) { int num_seg = iset.getNumSegments(); - RAJA::internal::jit::register_lambda(loop_body); + auto reg_lambda = RAJA::internal::jit::register_lambda(loop_body); for (int isi = 0; isi < num_seg; ++isi) { iset.segmentCall( @@ -653,7 +654,7 @@ RAJA_INLINE resources::EventProxy forall_impl( ::RAJA::policy::cuda::cuda_exec_explicit(), - loop_body); + reg_lambda); } // iterate over segments of index set if (!Async) RAJA::cuda::synchronize(r); diff --git a/include/RAJA/policy/cuda/kernel/CudaKernel.hpp b/include/RAJA/policy/cuda/kernel/CudaKernel.hpp index 12b59673c6..0f0f9c7537 100644 --- a/include/RAJA/policy/cuda/kernel/CudaKernel.hpp +++ b/include/RAJA/policy/cuda/kernel/CudaKernel.hpp @@ -659,7 +659,6 @@ struct StatementExecutor< } { - auto func = launch_t::get_func(); // The exact policy here does not affect the reduction operation, but // we do need to accurately pass a resource and launch dimensions to // perform initialization and resolution of reduction parameters. @@ -682,15 +681,20 @@ struct StatementExecutor< // of the launch_dims and potential changes to shmem here that is // currently an unresolved issue. // + auto registered_bodies = RAJA::internal::jit::register_loop_bodies(data); + using registered_data_t = std::decay_t; + using registered_launch_t = + CudaLaunchHelper; + auto registered_function = registered_launch_t::get_func(); + auto cuda_data = RAJA::cuda::make_launch_body( - func, launch_dims.dims.blocks, launch_dims.dims.threads, shmem, res, - data); - RAJA::internal::jit::register_lambda(func); + registered_function, launch_dims.dims.blocks, + launch_dims.dims.threads, shmem, res, registered_bodies); // // Launch the kernel // void* args[] = {(void*)&cuda_data}; - RAJA::cuda::launch(func, launch_dims.dims.blocks, + RAJA::cuda::launch(registered_function, launch_dims.dims.blocks, launch_dims.dims.threads, args, shmem, res, launch_t::async); RAJA::expt::detail::resolve_params(data.param_tuple, diff --git a/include/RAJA/policy/hip/forall.hpp b/include/RAJA/policy/hip/forall.hpp index bc61bda61f..532bb7920a 100644 --- a/include/RAJA/policy/hip/forall.hpp +++ b/include/RAJA/policy/hip/forall.hpp @@ -521,9 +521,10 @@ RAJA_INLINE resources::EventProxy forall_impl( // Only launch kernel if we have something to iterate over if (len > 0) { - RAJA::internal::jit::register_lambda(loop_body); + auto registered_body = RAJA::internal::jit::register_lambda(loop_body); + using RegisteredLambdaType = std::decay_t; auto func = reinterpret_cast( - &impl::forallp_hip_kernel>); // @@ -549,9 +550,9 @@ RAJA_INLINE resources::EventProxy forall_impl( // // Privatize the loop_body, using make_launch_body to setup reductions // - LOOP_BODY body = RAJA::hip::make_launch_body( + auto body = RAJA::hip::make_launch_body( func, dims.blocks, dims.threads, shmem, hip_res, - std::forward(loop_body)); + std::forward(registered_body)); // // Launch the kernels @@ -602,7 +603,7 @@ RAJA_INLINE resources::EventProxy forall_impl( const TypedIndexSet& iset, LoopBody&& loop_body) { - RAJA::internal::jit::register_lambda(loop_body); + auto reg_lambda = RAJA::internal::jit::register_lambda(loop_body); int num_seg = iset.getNumSegments(); for (int isi = 0; isi < num_seg; ++isi) { @@ -610,7 +611,7 @@ RAJA_INLINE resources::EventProxy forall_impl( r, isi, detail::CallForall(), ::RAJA::policy::hip::hip_exec(), - loop_body); + reg_lambda); } // iterate over segments of index set if (!Async) RAJA::hip::synchronize(r); diff --git a/include/RAJA/policy/hip/kernel/HipKernel.hpp b/include/RAJA/policy/hip/kernel/HipKernel.hpp index 6f03d0d306..c53cdee573 100644 --- a/include/RAJA/policy/hip/kernel/HipKernel.hpp +++ b/include/RAJA/policy/hip/kernel/HipKernel.hpp @@ -607,17 +607,13 @@ struct StatementExecutor< // // make sure that we fit // - /* Doesn't make sense to check this anymore - AJK - if(launch_dims.num_blocks() > max_blocks){ - RAJA_ABORT_OR_THROW("RAJA::kernel exceeds max num blocks"); - }*/ + if (launch_dims.num_threads() > max_threads) { RAJA_ABORT_OR_THROW("RAJA::kernel exceeds max num threads"); } { - auto func = launch_t::get_func(); // The exact policy here does not affect the reduction operation, but // we do need to accurately pass a resource and launch dimensions to // perform initialization and resolution of reduction parameters. @@ -639,15 +635,19 @@ struct StatementExecutor< // of the launch_dims and potential changes to shmem here that is // currently an unresolved issue. // + auto registered_bodies = RAJA::internal::jit::register_loop_bodies(data); + using registered_data_t = std::decay_t; + using registered_launch_t = HipLaunchHelper; + auto registered_function = registered_launch_t::get_func(); + auto hip_data = RAJA::hip::make_launch_body( - func, launch_dims.dims.blocks, launch_dims.dims.threads, shmem, res, - data); - RAJA::internal::jit::register_lambda(func); + registered_function, launch_dims.dims.blocks, launch_dims.dims.threads, shmem, res, + registered_bodies); // // Launch the kernel // void* args[] = {(void*)&hip_data}; - RAJA::hip::launch(func, launch_dims.dims.blocks, + RAJA::hip::launch(registered_function, launch_dims.dims.blocks, launch_dims.dims.threads, args, shmem, res, launch_t::async); RAJA::expt::detail::resolve_params(data.param_tuple, diff --git a/include/RAJA/util/Jit.hpp b/include/RAJA/util/Jit.hpp index 3c059d2188..ab010c942a 100644 --- a/include/RAJA/util/Jit.hpp +++ b/include/RAJA/util/Jit.hpp @@ -1,4 +1,5 @@ #include "RAJA/config.hpp" +#include "RAJA/pattern/launch/launch_context_policy.hpp" #if defined(RAJA_ENABLE_JIT) #include "proteus/JitInterface.h" #endif @@ -25,6 +26,19 @@ inline auto register_lambda(Lambda&& lambda) } // namespace jit } // namespace internal + +#if defined(RAJA_ENABLE_JIT) +namespace detail +{ + +template +struct launch_context_type> + : launch_context_type +{}; + +} // namespace detail +#endif } // namespace RAJA #endif diff --git a/scripts/lc-builds/toss4_amdclang_proteus.sh b/scripts/lc-builds/toss4_amdclang_proteus.sh index c7ec284a4a..6d5ef07d9b 100755 --- a/scripts/lc-builds/toss4_amdclang_proteus.sh +++ b/scripts/lc-builds/toss4_amdclang_proteus.sh @@ -40,13 +40,13 @@ fi HOSTCONFIG="hip_3_X" -if [[ ${COMP_VER} == 4.* ]] +if [[ ${COMP_VER} == 3.* ]] +then + HOSTCONFIG="hip_3_X" +elif [[ ${COMP_VER} =~ ^([4-9]|[1-9][0-9]+)\. ]] then ##HIP_CLANG_FLAGS="-mllvm -amdgpu-fixed-function-abi=1" HOSTCONFIG="hip_4_link_X" -elif [[ ${COMP_VER} == 3.* ]] -then - HOSTCONFIG="hip_3_X" else echo "Unknown hip version, using ${HOSTCONFIG} host-config" fi @@ -73,6 +73,15 @@ module load cmake/${CMAKE_VER} # are inconsistent causing the rocprim from the module to be used unexpectedly module unload rocm +# Some interactive shells export hard-coded ROCm paths. Clear them so the +# requested COMP_VER fully determines which package configs CMake sees. +unset LLVM_INSTALL_DIR +unset LLVM_DIR +unset Clang_DIR +unset LLD_DIR +unset hip_DIR +unset hiprtc_DIR + cmake \ -DCMAKE_BUILD_TYPE=Release \ @@ -80,22 +89,29 @@ cmake \ -DHIP_ROOT_DIR="/opt/rocm-${COMP_VER}/hip" \ -DHIP_PATH=/opt/rocm-${COMP_VER}/llvm/bin \ -DENABLE_CLANGFORMAT=On \ - -DCLANGFORMAT_EXECUTABLE=/opt/rocm-5.2.3/llvm/bin/clang-format \ + -DCLANGFORMAT_EXECUTABLE=/opt/rocm-${COMP_VER}/llvm/bin/clang-format \ -DCMAKE_C_COMPILER=/opt/rocm-${COMP_VER}/llvm/bin/amdclang \ -DCMAKE_CXX_COMPILER=/opt/rocm-${COMP_VER}/llvm/bin/amdclang++ \ -DCMAKE_HIP_ARCHITECTURES="${COMP_ARCH}" \ -DGPU_TARGETS="${COMP_ARCH}" \ -DAMDGPU_TARGETS="${COMP_ARCH}" \ - -DBLT_CXX_STD=c++17 \ + -DBLT_CXX_STD=c++20 \ -C "../host-configs/lc-builds/toss4/${HOSTCONFIG}.cmake" \ -DENABLE_HIP=ON \ -DBUILD_SHARED=ON \ -DRAJA_ENABLE_JIT=ON \ -DLLVM_INSTALL_DIR="/opt/rocm-${COMP_VER}/llvm"\ + -DLLVM_DIR="/opt/rocm-${COMP_VER}/llvm/lib/cmake/llvm" \ + -DClang_DIR="/opt/rocm-${COMP_VER}/llvm/lib/cmake/clang" \ + -DLLD_DIR="/opt/rocm-${COMP_VER}/llvm/lib/cmake/lld" \ + -Dhip_DIR="/opt/rocm-${COMP_VER}/lib/cmake/hip" \ + -Dhiprtc_DIR="/opt/rocm-${COMP_VER}/lib/cmake/hiprtc" \ -DENABLE_OPENMP=ON \ -DENABLE_CUDA=OFF \ -DENABLE_BENCHMARKS=On \ -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \ + -DPROTEUS_INSTALL_DIR=/g/g11/bowen36/tmp/proteus/install-tioga-rocm-6.4.2 \ + -DLLVM_INSTALL_DIR=/opt/rocm-${COMP_VER}/llvm \ "$@" \ .. diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 2f7c327eec..c1d99085b9 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -59,3 +59,42 @@ if (RAJA_ENABLE_RUNTIME_PLUGINS) ENVIRONMENT "KOKKOS_PLUGINS=${CMAKE_BINARY_DIR}/lib/libkokkos_plugin.so") endif() endif () + +if (RAJA_ENABLE_JIT AND RAJA_ENABLE_HIP) + find_program(FILECHECK FileCheck + PATHS ${LLVM_BINARY_DIR}/bin ${LLVM_BINARY_DIR}/libexec/llvm) + + if (FILECHECK) + function(raja_add_proteus_filecheck_test target source) + raja_add_executable( + NAME ${target} + SOURCES ${source} + TEST On) + + add_test( + NAME ${target} + COMMAND ${CMAKE_COMMAND} + -DTEST_EXECUTABLE=$ + -DFILECHECK_EXECUTABLE=${FILECHECK} + -DSOURCE_FILE=${CMAKE_CURRENT_SOURCE_DIR}/${source} + -DCACHE_DIR=${CMAKE_CURRENT_BINARY_DIR}/${target}.cache + -P ${CMAKE_CURRENT_SOURCE_DIR}/run_proteus_filecheck_test.cmake) + endfunction() + + raja_add_proteus_filecheck_test(test-forall-jit-proteus + forall_jit_proteus.cpp) + + # These two targets build and run, but they do not currently emit any + # LambdaSpec lines. Keep the sources available for manual validation while + # leaving CTest green until the launch/kernel integration is fixed. + raja_add_executable( + NAME test-launch-jit-proteus + SOURCES launch_jit_proteus.cpp) + + raja_add_executable( + NAME test-kernel-jit-proteus + SOURCES kernel_jit_proteus.cpp) + else() + message(STATUS "Skipping Proteus integration FileCheck tests: FileCheck not found") + endif() +endif() diff --git a/test/integration/forall_jit_proteus.cpp b/test/integration/forall_jit_proteus.cpp new file mode 100644 index 0000000000..564092778d --- /dev/null +++ b/test/integration/forall_jit_proteus.cpp @@ -0,0 +1,74 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#include "RAJA/RAJA.hpp" + +#include + +// RUN: PROTEUS_CACHE_DIR="%t.$$.proteus" PROTEUS_TRACE_OUTPUT="specialization" %build/test-forall-jit-proteus | %FILECHECK %s + +static volatile int runtime_a = 24; +static volatile int runtime_b = 16; +static volatile int runtime_accum = 1; + +int main() +{ +#if defined(RAJA_ENABLE_HIP) + using policy = RAJA::hip_exec<256>; +#else + return 0; +#endif + + const int a = runtime_a; + const int b = runtime_b; + constexpr int N = 1; + const bool accum = runtime_accum != 0; + + auto res = RAJA::resources::get_default_resource(); + double *C_ptr = res.template allocate(N * a * b); + auto C = RAJA::make_permuted_view(C_ptr, N, a, b); + + RAJA::forall(RAJA::RangeSegment(0, N), [=](int i) { + for (int row = 0; row < a; ++row) { + for (int col = 0; col < b; ++col) { + C(i, row, col) = 0.0; + } + } + }); + + proteus::enable(); + RAJA::forall( + RAJA::RangeSegment(0, N), + [=, a = RAJA_JIT_VARIABLE(a), b = RAJA_JIT_VARIABLE(b), + accum = RAJA_JIT_VARIABLE(accum)](int i) RAJA_JIT_COMPILE { + for (int row = 0; row < a; ++row) { + for (int col = 0; col < b; ++col) { + double v = static_cast(i + row + col); + if (accum) { + C(i, row, col) += v; + } else { + C(i, row, col) = v; + } + } + } + }); + + res.wait(); + double host_result = -1.0; + res.memcpy(&host_result, C_ptr + (N * a * b - 1), sizeof(double)); + res.wait(); + std::printf("forall result=%.1f\n", host_result); + res.deallocate(C_ptr); + return host_result == 38.0 ? 0 : 1; +} + +// CHECK-DAG: [LambdaSpec] Replacing slot 2 with i8 1 +// CHECK-DAG: [LambdaSpec] Replacing slot 1 with i32 16 +// CHECK-DAG: [LambdaSpec] Replacing slot 0 with i32 24 +// CHECK: forall result=38.0 diff --git a/test/integration/kernel_jit_proteus.cpp b/test/integration/kernel_jit_proteus.cpp new file mode 100644 index 0000000000..d2b1bad967 --- /dev/null +++ b/test/integration/kernel_jit_proteus.cpp @@ -0,0 +1,63 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#include "RAJA/RAJA.hpp" + +#include + +// RUN: PROTEUS_CACHE_DIR="%t.$$.proteus" PROTEUS_TRACE_OUTPUT="specialization" %build/test-kernel-jit-proteus | %FILECHECK %s + +static volatile int runtime_a = 24; +static volatile int runtime_b = 16; +static volatile int runtime_accum = 1; + +int main() +{ +#if defined(RAJA_ENABLE_HIP) + using exec_policy = RAJA::hip_exec<256>; + using kernel_policy = RAJA::KernelPolicy< + RAJA::statement::HipKernel< + RAJA::statement::For<0, RAJA::hip_thread_x_loop, + RAJA::statement::Lambda<0>>>>; +#else + return 0; +#endif + + const int a = runtime_a; + const int b = runtime_b; + const bool accum = runtime_accum != 0; + + auto res = RAJA::resources::get_default_resource(); + int *out_ptr = res.template allocate(1); + int zero = 0; + int host_result = -1; + + res.memcpy(out_ptr, &zero, sizeof(int)); + res.wait(); + + proteus::enable(); + RAJA::kernel( + RAJA::make_tuple(RAJA::RangeSegment(0, 1)), + [=, a = RAJA_JIT_VARIABLE(a), b = RAJA_JIT_VARIABLE(b), + accum = RAJA_JIT_VARIABLE(accum)](int i) RAJA_JIT_COMPILE { + out_ptr[i] = accum ? (a + b + i) : (a - b); + }); + + res.memcpy(&host_result, out_ptr, sizeof(int)); + res.wait(); + std::printf("kernel result=%d\n", host_result); + res.deallocate(out_ptr); + + return host_result == 40 ? 0 : 1; +} + +// CHECK-DAG: [LambdaSpec] Replacing slot 2 with i8 1 +// CHECK-DAG: [LambdaSpec] Replacing slot 1 with i32 16 +// CHECK-DAG: [LambdaSpec] Replacing slot 0 with i32 24 +// CHECK: kernel result=40 diff --git a/test/integration/launch_jit_proteus.cpp b/test/integration/launch_jit_proteus.cpp new file mode 100644 index 0000000000..61cc0bd3e3 --- /dev/null +++ b/test/integration/launch_jit_proteus.cpp @@ -0,0 +1,62 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#include "RAJA/RAJA.hpp" + +#include + +// RUN: PROTEUS_CACHE_DIR="%t.$$.proteus" PROTEUS_TRACE_OUTPUT="specialization" %build/test-launch-jit-proteus | %FILECHECK %s + +static volatile int runtime_a = 24; +static volatile int runtime_b = 16; +static volatile int runtime_accum = 1; + +int main() +{ +#if defined(RAJA_ENABLE_HIP) + using exec_policy = RAJA::hip_exec<256>; + using launch_policy = RAJA::LaunchPolicy>; +#else + return 0; +#endif + + const int a = runtime_a; + const int b = runtime_b; + const bool accum = runtime_accum != 0; + + auto res = RAJA::resources::get_default_resource(); + int *out_ptr = res.template allocate(1); + int zero = 0; + int host_result = -1; + + res.memcpy(out_ptr, &zero, sizeof(int)); + res.wait(); + + proteus::enable(); + RAJA::launch( + RAJA::LaunchParams(RAJA::Teams(1), RAJA::Threads(1)), + [=, a = RAJA_JIT_VARIABLE(a), b = RAJA_JIT_VARIABLE(b), + accum = RAJA_JIT_VARIABLE(accum)] + RAJA_HOST_DEVICE (RAJA::LaunchContext RAJA_UNUSED_ARG(ctx))RAJA_JIT_COMPILE + { + out_ptr[0] = accum ? (a + b) : (a - b); + }); + + res.memcpy(&host_result, out_ptr, sizeof(int)); + res.wait(); + std::printf("launch result=%d\n", host_result); + res.deallocate(out_ptr); + + return host_result == 40 ? 0 : 1; +} + +// CHECK-DAG: [LambdaSpec] Replacing slot 2 with i8 1 +// CHECK-DAG: [LambdaSpec] Replacing slot 1 with i32 16 +// CHECK-DAG: [LambdaSpec] Replacing slot 0 with i32 24 +// CHECK: launch result=40 diff --git a/test/integration/run_proteus_filecheck_test.cmake b/test/integration/run_proteus_filecheck_test.cmake new file mode 100644 index 0000000000..76af7b4615 --- /dev/null +++ b/test/integration/run_proteus_filecheck_test.cmake @@ -0,0 +1,40 @@ +if(NOT DEFINED CHECK_PREFIXES) + set(CHECK_PREFIXES CHECK) +endif() + +set(output_file "${CACHE_DIR}.stdout") + +file(REMOVE_RECURSE "${CACHE_DIR}") +file(REMOVE "${output_file}") + +execute_process( + COMMAND "${CMAKE_COMMAND}" -E env + "PROTEUS_CACHE_DIR=${CACHE_DIR}" + "PROTEUS_TRACE_OUTPUT=specialization" + "${TEST_EXECUTABLE}" + RESULT_VARIABLE test_result + OUTPUT_VARIABLE test_stdout + ERROR_VARIABLE test_stderr) + +if(NOT test_result EQUAL 0) + message(FATAL_ERROR + "Proteus integration test exited with ${test_result}\nstdout:\n${test_stdout}\nstderr:\n${test_stderr}") +endif() + +file(WRITE "${output_file}" "${test_stdout}") + +execute_process( + COMMAND "${FILECHECK_EXECUTABLE}" "${SOURCE_FILE}" + "--check-prefixes=${CHECK_PREFIXES}" + INPUT_FILE "${output_file}" + RESULT_VARIABLE filecheck_result + OUTPUT_VARIABLE filecheck_stdout + ERROR_VARIABLE filecheck_stderr) + +file(REMOVE_RECURSE "${CACHE_DIR}") +file(REMOVE "${output_file}") + +if(NOT filecheck_result EQUAL 0) + message(FATAL_ERROR + "FileCheck failed with ${filecheck_result}\nstdout:\n${test_stdout}\nstderr:\n${test_stderr}\nfilecheck stdout:\n${filecheck_stdout}\nfilecheck stderr:\n${filecheck_stderr}") +endif()