diff --git a/docs/sphinx/user_guide/feature/messages.rst b/docs/sphinx/user_guide/feature/messages.rst new file mode 100644 index 0000000000..c36a064d60 --- /dev/null +++ b/docs/sphinx/user_guide/feature/messages.rst @@ -0,0 +1,195 @@ +.. ## +.. ## Copyright (c) Lawrence Livermore National Security, LLC and other +.. ## RAJA Project Developers. See top-level LICENSE and COPYRIGHT +.. ## files for dates and other details. No copyright assignment is required +.. ## to contribute to RAJA. +.. ## +.. ## SPDX-License-Identifier: (BSD-3-Clause) +.. ## + +.. _feat-message-label: + +=============================== +Messages +=============================== + +RAJA provides a portable interface and type-safe way to store function arguments that are +passed to a function at a later time. For example, from a GPU, arguments can be stored +and passed to function that prints to a file on the CPU. + +.. warning:: This capability is new. We would like users to try it out and give feedback to improve it. + + +-------------------------- +How to manage messages? +-------------------------- +All messages are handled via the ``message_manager``, which is responsible for +storing callbacks and a list of messages. For the purposes of ``RAJA::messages``, +a single message can be thought of: + +* ``MsgHeader``: helper data internal to ``RAJA`` +* ``MsgArgs``: a tuple of arguments needed to pass to the function + +To create the ``message_manager``: + +.. literalinclude:: ../../../../examples/messages-forall.cpp + :start-after: _raja_msg_manager_start + :end-before: _raja_msg_manager_end + :language: C++ + +``buf_sz`` is the size of the buffer that stores messages. ``res_host`` is the resource of the +execution policy, which determines the memory space in which messages will be stored. For GPU resources, for example, +this is ``PINNED`` memory. + +Subscribing callbacks +^^^^^^^^^^^^^^^^^^^^^ +To create a specific message type, callbacks must subscribe first. This can be done in two ways: + +* ``subscribe(Callable)``: Subscribing with just a callable will create a new type of message with the + type depending on the parameters. +* ``subscribe(msg_queue_id, Callable)``: Subscribing with ``msg_queue_id`` and a callable will append the new + callback to the already existing callback list. + +As an example for subscribing with both methods: + +.. literalinclude:: ../../../../examples/messages-forall.cpp + :start-after: _raja_msg_subscribe_start + :end-before: _raja_msg_subscribe_end + :language: C++ + +Unsubscribing callbacks +^^^^^^^^^^^^^^^^^^^^^^^ +If a particular callback no longer needs to be subscribed to a message type, then the callback can be +unsubcribed. This can be achieved in three ways: + +* ``unsubscribe(msg_queue_id, Callable)``: Looks for a specific callback that is subscribed to a particular message. If + the callback is subscribed, remove from callback list. Otherwise, throws an exception. +* ``unsubscribe_all(msg_queue_id)``: Removes all callbacks subscribed to a particular message +* ``unsubscribe_all()``: Removes all callbacks and messages. + +An example for unsubscribing a callback: + +.. literalinclude:: ../../../../examples/messages-forall.cpp + :start-after: _raja_msg_unsubscribe_start + :end-before: _raja_msg_unsubscribe_end + :language: C++ + + +Publishing messages +^^^^^^^^^^^^^^^^^^^ +Messages can be published/stored in a ``MessageQueue``. These are non-owning adapters to the ``MessageBus``, which is +responsible for storing all messages. The ``MessageQueue`` will contain additional type information as well as +the ``msg_queue_id``. A queue is created once a callback is subscribed to a new message type. Since ``MessageQueue`` is +non-owning, these can be copied. + +Here is how the ``MessageQueue`` can be used to publish messages: + +.. literalinclude:: ../../../../examples/messages-forall.cpp + :start-after: _raja_msg_k2_start + :end-before: _raja_msg_k2_end + :language: C++ + +Handling messages +^^^^^^^^^^^^^^^^^ +Lastly, there needs to be a way to direct messages to the corresponding callback(s). +This is handled with the ``message_manager``, which forces a synchronize on the resource provided. + +.. literalinclude:: ../../../../examples/messages-forall.cpp + :start-after: _raja_msg_wait_start + :end-before: _raja_msg_wait_end + :language: C++ + +Handling messages with a GPU +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Here is a complete example using the ``RAJA::messages`` with a GPU kernel. + +.. literalinclude:: ../../../../examples/messages-forall.cpp + :start-after: _raja_msg_gpu1_start + :end-before: _raja_msg_gpu1_end + :language: C++ + +.. note:: + In this example, ``gpu_policy`` depends on the build (i.e., CUDA, HIP), ``res`` is the default resource for + ``gpu_policy``, and ``d_*`` arrays are allocated for the device. These are removed above from the example to just + show the ``RAJA::messages`` interface. + +Handling messages across multiple streams +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Here is a complete example using the ``RAJA::messages`` with multiple resources. + +.. literalinclude:: ../../../../examples/messages-forall.cpp + :start-after: _raja_msg_gpu2_start + :end-before: _raja_msg_gpu2_end + :language: C++ + +.. note:: + In this example, ``res_gpu1`` and ``res_gpu2`` depend on the build (i.e., CUDA, HIP), ``EXEC_POLICY`` the + exeuction policy for the loops (also depends on the build), and ``d_*`` arrays are allocated for the device + while ``h_*`` are allocated for the host. These are removed from the example above to just show the + ``RAJA::messages`` interface. + +Message queue policies +^^^^^^^^^^^^^^^^^^^^^^ +Message queues can support various policies depending on the requirements of that queue, such as +the number of producers/consumers or the type of atomic operations. + + ======================= ============================ + Message queue Policies Brief description + ======================= ============================ + spsc Supports a single producer, + single consumer; i.e., no + atomic operations + mpsc Supports multiple producers, + a single consumer; i.e., + requires atomic operations. + Automatically determines + which atomic operations to + use. + ======================= ============================ + +.. note:: Producers and consumers can not operate at the same time + +Building and running the example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The example ``examples/messages-forall.cpp`` is built from the RAJA source tree when: + +* ``ENABLE_EXAMPLES=On`` + +To run the example: + +.. code-block:: bash + + ./bin/messages-forall + +This example will show how callbacks can be subscribed to various types of messages as +well as how to publish messages on multiple different platforms. For the purposes of +this example, output for messages will be printed using ``std::cout``. + +-------------------------- +Application considerations +-------------------------- + +There are several things to consider when using ``RAJA::messages`` in an application. + +* The ``MessageQueue`` with the correct argument types is created when a callback subscribes. Certain + patterns will cause this storage to slowly grow overtime. For example, creating a new + ``MessageQueue`` every function call within a loop. Therefore, applications that use this pattern + will want to unsubscribe at some point to avoid running out of memory. +* Upon creation of the ``MessageManager``, the ``MessageBus`` will be allocated with some size. This + can be resized; however, resizing will force a synchronize and will loss any messages currently stored. + Also, by default, the allocation is done through the resource, which can be less performant depending on the resource. +* Since the ``MessageQueue`` is a fixed size, there is a chance of lossing messages. The ``try_post_message`` function + will return a ``boolean``. This will be ``true`` if the message is successfully added to the queue; otherwise, this + is ``false``. + +Custom Allocators +^^^^^^^^^^^^^^^^^ +Allocation of the message bus is done through the ``RAJA::ResourceAllocator`` +by default. This allocator uses the resource provided to allocate/deallocate, +which can be slow. During the creation of the ``MessageManager``, a custom +allocator can be provided. This allocator should follow the C++ standard +requirements for an allocator. + +.. note:: When using the move assignment operator, the ``MessageManager`` + assumes that there are no currently pending messages. diff --git a/docs/sphinx/user_guide/features.rst b/docs/sphinx/user_guide/features.rst index 47f2107042..ec21875c30 100644 --- a/docs/sphinx/user_guide/features.rst +++ b/docs/sphinx/user_guide/features.rst @@ -36,4 +36,5 @@ materials that provide detailed examples of usage. feature/workgroup feature/vectorization feature/jit + feature/messages feature/plugins diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index e77b5dc493..5cfa297ea6 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -19,6 +19,10 @@ raja_add_executable( NAME resource-forall SOURCES resource-forall.cpp) +raja_add_executable( + NAME messages-forall + SOURCES messages-forall.cpp) + raja_add_executable( NAME dynamic-forall SOURCES dynamic-forall.cpp) diff --git a/examples/messages-forall.cpp b/examples/messages-forall.cpp new file mode 100644 index 0000000000..c74d932e65 --- /dev/null +++ b/examples/messages-forall.cpp @@ -0,0 +1,390 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2016-25, Lawrence Livermore National Security, LLC +// and RAJA project contributors. See the RAJA/LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#include +#include +#include + +#include "memoryManager.hpp" + +#include "RAJA/RAJA.hpp" +#include "RAJA/util/resource.hpp" + +/* + * RAJA::messages example + * + * The purpose of this example to show how RAJA::messages can be used + * with RAJA::forall. This includes the following: + * - storing messages on various execution policies (serial, OpenMP, GPU) + * - printing to a file on the GPU using the RAJA::messages + * - interacting with multiple GPU streams + * - creating custom types that can be stored (note: these types should be + * trivially destructible and trivially copyable) + * + */ + +// This is a simplified example fixed length string to show how +// custom types can be used with the message queue. +template +class my_string +{ +public: + char m_data[N]; + + my_string(const char* str) + { + if (str == NULL) { return; } + std::size_t i = 0; + for (i = 0; *str != '\0' && i < N; str++, i++) { + m_data[i] = *str; + } + + std::size_t len = (i < N) ? i : N-1; + m_data[len] = '\0'; + } + + const char* c_str() const + { + return m_data; + } +}; + +// +// Functions for checking and printing results +// +void checkResult(int* res, int len); +void printResult(int* res, int len); + + +int main(int RAJA_UNUSED_ARG(argc), char **RAJA_UNUSED_ARG(argv[])) +{ + + std::cout << "\n\nRAJA RAJA::messages with vector addition example...\n"; + + RAJA::resources::Host res_host{}; + +// +// Define number of messages that can be stored +// + const std::size_t num_messages = 1; + const std::size_t message_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs, int*, int, int>)); + + const std::size_t buf_sz = num_messages*message_sz; + +// +// Allocate and initialize message handler and queue +// +// _raja_msg_manager_start + auto msg_manager = RAJA::make_message_manager(buf_sz, res_host); +// _raja_msg_manager_end + +// _raja_msg_subscribe_start + auto cpu_msg_queue = msg_manager.subscribe( + [](const my_string<128>& str, int* ptr, int idx, int value) { + std::cout << "\n " << str.c_str() << " " << ptr << " a[" << idx << "] = " << value << "\n"; + } + ); + + auto err_callback = [] (const my_string<128>& str, int* ptr, int idx, int value) { + std::cerr << "\n " << str.c_str() << " " << ptr << " a[" << idx << "] = " << value << "\n"; + }; + msg_manager.subscribe(cpu_msg_queue.get_id(), err_callback); +// _raja_msg_subscribe_end + + +// _raja_msg_unsubscribe_start + msg_manager.unsubscribe(cpu_msg_queue.get_id(), err_callback); +// _raja_msg_unsubscribe_end + + constexpr int N = 100000; + + int *a = res_host.allocate(N); + int *b = res_host.allocate(N); + int *c = res_host.allocate(N); + + int *a_ = res_host.allocate(N); + int *b_ = res_host.allocate(N); + int *c_ = res_host.allocate(N); + + + for (int i = 0; i < N; ++i) { + a[i] = -i; + b[i] = 2 * i; + a_[i] = -i; + b_[i] = 2 * i; + + } + + +//----------------------------------------------------------------------------// + + std::cout << "\n Running C-style vector addition...\n"; + +// _raja_msg_k1_start + for (int i = 0; i < N; ++i) { + if (a[i] < 0) { + cpu_msg_queue.try_post_message("message from C-style loop", a, i, a[i]); + } + c[i] = a[i] + b[i]; + } +// _raja_msg_k1_end + + checkResult(c, N); +// _raja_msg_wait_start + msg_manager.wait_all(); +// _raja_msg_wait_end + + +//----------------------------------------------------------------------------// +// RAJA::seq_exec policy enforces sequential execution.... +//----------------------------------------------------------------------------// + + std::cout << "\n Running RAJA sequential vector addition...\n"; + + +// _raja_msg_k2_start + RAJA::forall(res_host, RAJA::RangeSegment(0, N), [=] (int i) { + if (a[i] < 0) { + cpu_msg_queue.try_post_message("message from RAJA seq_exec loop", a, i, a[i]); + } + c[i] = a[i] + b[i]; + }); +// _raja_msg_k2_end + + checkResult(c, N); + msg_manager.wait_all(); + +#if defined(RAJA_ENABLE_OPENMP) +//----------------------------------------------------------------------------// +// RAJA::omp_for_parallel_exec policy execution.... +//----------------------------------------------------------------------------// + + std::cout << "\n Running RAJA omp_parallel_for_exec vector addition...\n"; + + RAJA::forall(res_host, RAJA::RangeSegment(0, N), + [=] (int i) { + if (a[i] < 0) { + cpu_msg_queue.try_post_message("message from RAJA omp_parallel_for_exec loop", a, i, a[i]); + } + c[i] = a[i] + b[i]; + }); + + checkResult(c, N); + msg_manager.wait_all(); +#endif + + + +#if defined(RAJA_ENABLE_CUDA) || defined(RAJA_ENABLE_HIP) || defined(RAJA_ENABLE_SYCL) + +/* + GPU_BLOCK_SIZE - specifies the number of threads in a CUDA/HIP thread block +*/ +const int GPU_BLOCK_SIZE = 256; + +//----------------------------------------------------------------------------// +// RAJA::cuda/hip_exec policy execution.... +//----------------------------------------------------------------------------// +{ + std::cout << "\n Running RAJA GPU vector addition on RAJA's default streams...\n"; +#if defined(RAJA_ENABLE_CUDA) + using gpu_policy = RAJA::cuda_exec_async; +#elif defined(RAJA_ENABLE_HIP) + using gpu_policy = RAJA::hip_exec_async; +#elif defined(RAJA_ENABLE_SYCL) + using gpu_policy = RAJA::sycl_exec; +#endif + auto res = RAJA::resources::get_default_resource(); + + int* d_a1 = res.allocate(N); + int* d_b1 = res.allocate(N); + int* d_c1 = res.allocate(N); + + res.memcpy(d_a1, a, sizeof(int)* N); + res.memcpy(d_b1, b, sizeof(int)* N); + + // _raja_msg_gpu1_start + auto msg_manager = RAJA::make_message_manager(message_sz*10); + + auto log = [](const my_string<32>& str, int idx, int value) { + std::cout << "[INFO]: " << str.c_str() << "[" << idx << "] = " << value << "\n"; + }; + + // Create two types of messages: + // queue1 stores one message type and prints with one callback + // queue2 stores the other types of messages and forwards the message to multiple callbacks + auto msg_queue1 = msg_manager.subscribe(log); + msg_manager.subscribe(msg_queue1.get_id(), + [](const my_string<32>& str, int idx, int value) { + std::cout << "echo msg: " << str.c_str() << "[" << idx << "] = " << value << "\n"; + } + ); + auto msg_queue2 = msg_manager.subscribe(log); + + RAJA::forall(RAJA::RangeSegment(0, N), + [=] RAJA_DEVICE (int i) { + if (d_a1[i] < 0 && i == 1) { + msg_queue1.try_post_message("d_a1", i, d_a1[i]); + } + if (d_b1[i] > 0 && i == 1) { + msg_queue2.try_post_message("d_b1", i, d_b1[i]); + } + d_c1[i] = d_a1[i] + d_b1[i]; + }); + msg_manager.wait_all(); + // _raja_msg_gpu1_end + + res.memcpy(c, d_c1, sizeof(int)*N ); + res.wait(); + + res.deallocate(d_a1); + res.deallocate(d_b1); + res.deallocate(d_c1); + + checkResult(c, N); +} + + +//----------------------------------------------------------------------------// +// RAJA::cuda/hip_exec policy with waiting event.... +//----------------------------------------------------------------------------// +{ + std::cout << "\n Running RAJA GPU vector with dependency between two seperate streams...\n"; +#if defined(RAJA_ENABLE_CUDA) + RAJA::resources::Cuda res_gpu1; + RAJA::resources::Cuda res_gpu2; + + using EXEC_POLICY = RAJA::cuda_exec_async; +#elif defined(RAJA_ENABLE_HIP) + RAJA::resources::Hip res_gpu1; + RAJA::resources::Hip res_gpu2; + + using EXEC_POLICY = RAJA::hip_exec_async; +#elif defined(RAJA_ENABLE_SYCL) + RAJA::resources::Sycl res_gpu1; + RAJA::resources::Sycl res_gpu2; + + using EXEC_POLICY = RAJA::sycl_exec; +#endif + + int* d_array1 = res_gpu1.allocate(N); + int* d_array2 = res_gpu2.allocate(N); + int* h_array = res_host.allocate(N); + + // _raja_msg_gpu2_start + auto allocator1 = RAJA::ResourceAllocator{res_gpu1}; + auto gpu_logger1 = RAJA::make_message_manager(buf_sz, res_gpu1, allocator1); + auto gpu_msg_queue1 = gpu_logger1.subscribe( + [](int* ptr, int idx, int value) { + std::cout << "\n gpu stream 1: pointer (" << ptr << ") d_array1[" << idx << "] = " << value << "\n"; + } + ); + + auto allocator2 = RAJA::ResourceAllocator{res_gpu2}; + auto gpu_logger2 = RAJA::make_message_manager(buf_sz, res_gpu2, allocator2); + auto gpu_msg_queue2 = gpu_logger2.subscribe( + [](int* ptr, int idx, int value) { + std::cout << "\n gpu stream 2: pointer (" << ptr << ") d_array2[" << idx << "] = " << value << "\n"; + } + ); + + RAJA::forall(res_gpu1, RAJA::RangeSegment(0,N), + [=] RAJA_HOST_DEVICE (int i) { + d_array1[i] = i; + gpu_msg_queue1.try_post_message(d_array1, i, d_array1[i]); + } + ); + + // Log message for stream 1 + gpu_logger1.wait_all(); + + RAJA::forall(res_gpu2, RAJA::RangeSegment(0,N), + [=] RAJA_HOST_DEVICE (int i) { + d_array2[i] = -1; + gpu_msg_queue2.try_post_message(d_array2, i, d_array2[i]); + } + ); + + RAJA::forall(res_gpu1, RAJA::RangeSegment(0,N), + [=] RAJA_HOST_DEVICE (int i) { + d_array1[i] *= -1; + gpu_msg_queue1.try_post_message(d_array1, i, d_array1[i]); + } + ); + + // Log message for stream 2 + gpu_logger2.wait_all(); + + // Log message for stream 1 + gpu_logger1.wait_all(); + // _raja_msg_gpu2_end + + res_gpu1.memcpy(h_array, d_array1, sizeof(int) * N); + res_gpu1.wait(); + + res_gpu1.deallocate(d_array1); + res_gpu2.deallocate(d_array2); + + bool check = true; + RAJA::forall(res_host, RAJA::RangeSegment(0,N), + [&check, h_array] (int i) { + if(h_array[i] != -i) {check = false;} + } + ); + res_host.deallocate(h_array); + + std::cout << "\n result -- "; + if (check) std::cout << "PASS\n"; + else std::cout << "FAIL\n"; +} + +#endif +// +// +// Clean up. +// + res_host.deallocate(a); + res_host.deallocate(b); + res_host.deallocate(c); + + res_host.deallocate(a_); + res_host.deallocate(b_); + res_host.deallocate(c_); + + std::cout << "\n DONE!...\n"; + + return 0; +} + +// +// Function to check result and report P/F. +// +void checkResult(int* res, int len) +{ + bool correct = true; + for (int i = 0; i < len; i++) { + if ( res[i] != i ) { correct = false; } + } + if ( correct ) { + std::cout << "\n\t result -- PASS\n"; + } else { + std::cout << "\n\t result -- FAIL\n"; + } +} + +// +// Function to print result. +// +void printResult(int* res, int len) +{ + std::cout << std::endl; + for (int i = 0; i < len; i++) { + std::cout << "result[" << i << "] = " << res[i] << std::endl; + } + std::cout << std::endl; +} diff --git a/include/RAJA/RAJA.hpp b/include/RAJA/RAJA.hpp index 343d552f76..03348e57bd 100644 --- a/include/RAJA/RAJA.hpp +++ b/include/RAJA/RAJA.hpp @@ -183,6 +183,11 @@ // #include "RAJA/pattern/synchronize.hpp" +// +// Message handler to pass messages between host and device +// +#include "RAJA/pattern/messages.hpp" + // ////////////////////////////////////////////////////////////////////// // diff --git a/include/RAJA/pattern/launch/launch_context_policy.hpp b/include/RAJA/pattern/launch/launch_context_policy.hpp index 31e38faf61..2c07ad09b4 100644 --- a/include/RAJA/pattern/launch/launch_context_policy.hpp +++ b/include/RAJA/pattern/launch/launch_context_policy.hpp @@ -23,6 +23,8 @@ #include +#include "RAJA/util/FunctionTypeTraits.hpp" + namespace RAJA { @@ -34,47 +36,6 @@ class LaunchContextHostPolicy; namespace detail { -template -struct first_argument; - -template -struct first_argument -{ - using type = Arg0; -}; - -template -struct first_argument - : first_argument -{}; - -template -struct first_argument - : first_argument -{}; - -template -struct first_argument - : first_argument -{}; - -template -struct first_argument - : first_argument -{}; - -template -struct callable_signature -{ - using type = camp::decay; -}; - -template -struct callable_signature::operator())>> -{ - using type = decltype(&camp::decay::operator()); -}; - template struct launch_context_type { @@ -82,12 +43,12 @@ struct launch_context_type }; template -struct launch_context_type::type>>::type>> +struct launch_context_type< + T, + std::void_t::operator())>>> { using type = camp::decay< - typename first_argument::type>::type>; + internal::func_arg_t<0, decltype(&camp::decay::operator())>>; }; diff --git a/include/RAJA/pattern/messages.hpp b/include/RAJA/pattern/messages.hpp new file mode 100644 index 0000000000..4c53b22f40 --- /dev/null +++ b/include/RAJA/pattern/messages.hpp @@ -0,0 +1,27 @@ +/*! + ****************************************************************************** + * + * \file + * + * \brief RAJA header file containing headers for RAJA::messages functionality + * + ****************************************************************************** + */ + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#ifndef RAJA_pattern_messages_HPP +#define RAJA_pattern_messages_HPP + +#include "RAJA/pattern/messages/msg_header.hpp" +#include "RAJA/pattern/messages/msg_callback.hpp" +#include "RAJA/pattern/messages/msg_manager.hpp" + +#endif /* RAJA_pattern_messages_HPP */ diff --git a/include/RAJA/pattern/messages/msg_callback.hpp b/include/RAJA/pattern/messages/msg_callback.hpp new file mode 100644 index 0000000000..673a1d291a --- /dev/null +++ b/include/RAJA/pattern/messages/msg_callback.hpp @@ -0,0 +1,83 @@ +/*! + ****************************************************************************** + * + * \file + * + * \brief RAJA header file defining callback helpers for RAJA::messages. + * + ****************************************************************************** + */ + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#ifndef RAJA_MSG_CALLBACK_HPP +#define RAJA_MSG_CALLBACK_HPP + +#include +#include + +#include "RAJA/pattern/messages/msg_header.hpp" +#include "RAJA/util/FunctionTypeTraits.hpp" + +namespace RAJA +{ +class IMsgCallback +{ +public: + virtual ~IMsgCallback() = default; + + virtual std::type_index get_type() const { return typeid(void); } + + virtual void operator()(char*) const = 0; +}; + +template +class MsgCallback; + +template +class MsgCallback : public IMsgCallback +{ +public: + using return_t = Ret; + + explicit MsgCallback(const Callable& callable) : m_callable {callable} {} + + explicit MsgCallback(Callable&& callable) : m_callable {std::move(callable)} + {} + + std::type_index get_type() const final { return typeid(Callable); } + + void operator()(char* args_buf) const final override + { + auto& msg = *std::launder( + reinterpret_cast...>*>(args_buf)); + camp::apply(m_callable, msg.args); + msg.~MsgArgs...>(); + } + +private: + Callable m_callable; +}; + +template +MsgCallback(R (*)(Args...)) -> MsgCallback; + +template< + typename Callable, + typename Signature = internal::signature_t> +MsgCallback(const Callable&) -> MsgCallback; + +template< + typename Callable, + typename Signature = internal::signature_t> +MsgCallback(Callable&&) -> MsgCallback; +} // namespace RAJA + +#endif // RAJA_MSG_CALLBACK_HPP diff --git a/include/RAJA/pattern/messages/msg_header.hpp b/include/RAJA/pattern/messages/msg_header.hpp new file mode 100644 index 0000000000..2bc7afbd78 --- /dev/null +++ b/include/RAJA/pattern/messages/msg_header.hpp @@ -0,0 +1,44 @@ +/*! + ****************************************************************************** + * + * \file + * + * \brief RAJA header file defining additional helpers for RAJA::messages + * to be properly aligned. + * + ****************************************************************************** + */ + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#ifndef RAJA_MSG_HEADER_HPP +#define RAJA_MSG_HEADER_HPP + +#include +#include "camp/tuple.hpp" + +namespace RAJA +{ +struct MsgHeader +{ + std::size_t sz; + std::size_t type; + std::size_t hash; + char* args; +}; + +template +struct MsgArgs +{ + camp::tuple args; +}; +} // namespace RAJA + +#endif // RAJA_MSG_HEADER_HPP diff --git a/include/RAJA/pattern/messages/msg_manager.hpp b/include/RAJA/pattern/messages/msg_manager.hpp new file mode 100644 index 0000000000..f305cc18af --- /dev/null +++ b/include/RAJA/pattern/messages/msg_manager.hpp @@ -0,0 +1,478 @@ +/*! + ****************************************************************************** + * + * \file + * + * \brief RAJA header file defining a GPU to CPU message handler class. + * + ****************************************************************************** + */ + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#ifndef RAJA_MSG_MANAGER_HPP +#define RAJA_MSG_MANAGER_HPP + +#include +#include +#include +#include +#include + +#include "RAJA/util/align.hpp" +#include "RAJA/util/HashCombiner.hpp" +#include "RAJA/util/ResourceAllocator.hpp" + +#include "RAJA/pattern/messages/msg_header.hpp" +#include "RAJA/pattern/messages/msg_callback.hpp" +#include "RAJA/policy/msg_queue.hpp" + +#include "camp/resource.hpp" + +namespace RAJA +{ +/// +/// Owning wrapper for a message queue. This is used for ownership +/// of the message queue and is a move-only class. For getting a view-like +/// class, use the `get_queue` member function, which allows copying. +/// +template +class MessageBus; + +/// +/// Specialized case from message bus. +/// This will store a MsgHeader and arguments in a char* buffer. These +/// are later reinterpretted to the correct message arguments. +/// +template +class MessageBus +{ + // Internal classes +public: + // Queue is public due to limitation with extended lambdas + // in nvcc + struct Queue + { + using value_type = char; + using size_type = unsigned long long; + using pointer = value_type*; + using const_pointer = const value_type*; + using iterator = value_type*; + using const_iterator = const value_type*; + + size_type m_begin {0}; + size_type m_end {0}; + size_type m_capacity {0}; + pointer m_data {nullptr}; + }; + +private: + struct MsgIterator + { + using value_type = char; + using pointer = value_type*; + using reference = value_type&; + using difference_type = std::ptrdiff_t; + using iterator_categroy = std::forward_iterator_tag; + + MsgIterator(pointer ptr) : cur_ptr(ptr) {} + + MsgHeader& operator*() const + { + return *std::launder(reinterpret_cast(cur_ptr)); + } + + MsgHeader* operator->() const + { + return std::launder(reinterpret_cast(cur_ptr)); + } + + MsgIterator& operator++() + { + MsgHeader& msg = *std::launder(reinterpret_cast(cur_ptr)); + cur_ptr += msg.sz + align_sz(sizeof(MsgHeader)); + + return (*this); + } + + MsgIterator operator++(int) + { + MsgIterator temp = *this; + ++(*this); + return temp; + } + + bool operator==(const MsgIterator& other) const + { + return (cur_ptr == other.cur_ptr); + } + + private: + pointer cur_ptr; + }; + + struct ResourceDeleter + { + public: + using resource_type = camp::resources::Resource; + using allocator_type = + typename std::allocator_traits::template rebind_alloc; + + template + ResourceDeleter(Resource res, allocator_type alloc) + : m_res {res}, + m_alloc {alloc} + {} + + void operator()(Queue* ptr) + { + m_res.wait(); + ptr->~Queue(); + m_alloc.deallocate(ptr, 1); + } + + allocator_type& get_allocator() noexcept { return m_alloc; } + + resource_type& get_resource() noexcept { return m_res; } + + private: + resource_type m_res; + [[no_unique_address]] allocator_type m_alloc; + }; + +public: + using value_type = char; + using size_type = unsigned long long; + using pointer = value_type*; + using const_pointer = const value_type*; + using iterator = MsgIterator; + using const_iterator = const iterator; + using resource_type = typename ResourceDeleter::resource_type; + // Allocator for queue buffer + using allocator_type = Allocator; + // Allocator for queue struct + using queue_allocator = typename ResourceDeleter::allocator_type; + + MessageBus() : MessageBus(camp::resources::Host()) {} + + template + MessageBus(Resource res, Allocator alloc = Allocator {}) + : m_bus {allocate_bus(res, alloc)} + {} + + template + MessageBus(const size_type bus_sz, + Resource res, + Allocator alloc = Allocator {}) + : MessageBus {res, alloc} + { + reserve(bus_sz); + } + + ~MessageBus() { reset(); } + + // Copy ctor/operator + MessageBus(const MessageBus&) = delete; + MessageBus& operator=(const MessageBus&) = delete; + + // Move ctor/operator + MessageBus(MessageBus&&) = default; + + MessageBus& operator=(MessageBus&& other) + { + using alloc_traits = std::allocator_traits; + + if constexpr (alloc_traits::propagate_on_container_move_assignment) + { + m_bus = std::move(other.m_bus); + } + else + { + if (get_allocator() == other.get_allocator()) + { + m_bus->m_begin = other.m_bus->m_begin; + m_bus->m_end = other.m_bus->m_end; + m_bus->m_capacity = other.m_bus->m_capacity; + m_bus->m_data = other.m_bus->m_data; + + other.m_bus->m_begin = 0; + other.m_bus->m_end = 0; + other.m_bus->m_capacity = 0; + other.m_bus->m_data = nullptr; + } + else + { + // This assumes that the bus is empty (i.e., no messages stored) + reserve(other.m_bus->m_capacity); + other.reset(); + } + } + } + + void reserve(size_type bus_sz) + { + reset(); + m_bus->m_data = get_allocator().allocate(bus_sz); + m_bus->m_capacity = bus_sz; + } + + void reset() + { + // Verify that queue is not in use + if (m_bus->m_data != nullptr) + { + get_resource().wait(); + get_allocator().deallocate(m_bus->m_data, m_bus->m_capacity); + m_bus->m_data = nullptr; + } + m_bus->m_capacity = 0; + m_bus->m_end = 0; + m_bus->m_begin = 0; + } + + bool has_pending_messages() + { + get_resource().wait(); + return (m_bus->m_end != 0); + } + + void clear_messages() + { + get_resource().wait(); + m_bus->m_end = 0; + m_bus->m_begin = 0; + } + + template + auto get_queue(std::pair id) noexcept + { + return RAJA::messages::Queue> { + id, m_bus.get()}; + } + + template + auto get_queue(std::pair id) const noexcept + { + return RAJA::messages::Queue> { + id, m_bus.get()}; + } + + auto get_allocator() noexcept + { + return allocator_type {m_bus.get_deleter().get_allocator()}; + } + + resource_type& get_resource() noexcept + { + return m_bus.get_deleter().get_resource(); + } + + iterator begin() noexcept { return iterator {m_bus->m_data}; } + + iterator begin() const noexcept { return iterator {m_bus->m_data}; } + + iterator end() noexcept { return iterator {m_bus->m_data + m_bus->m_end}; } + + iterator end() const noexcept + { + return iterator {m_bus->m_data + m_bus->m_end}; + } + +private: + template + auto allocate_bus(Resource res, Allocator alloc) + { + ResourceDeleter deleter {res, alloc}; + Queue* queue {new (deleter.get_allocator().allocate(1)) Queue {}}; + return std::unique_ptr {queue, std::move(deleter)}; + } + + std::unique_ptr m_bus; +}; + +/// +/// Provides a way to handle messages from a GPU. This currently +/// stores messages from the GPU and then calls a callback +/// function from the host. +/// +/// Note: +/// Currently, this forces a synchronize prior to calling +/// the callback function or testing if there are any messages. +/// +template +class MessageManager +{ +public: + using msg_fn_list_t = std::vector>; + using msg_id = std::pair; + using msg_bus = MessageBus; + +public: + template + MessageManager(const std::size_t bus_sz, Resource res, Allocator alloc) + : m_bus {bus_sz, res, alloc} + {} + + ~MessageManager() = default; + + // Doesn't support copying + MessageManager(const MessageManager&) = delete; + MessageManager& operator=(const MessageManager&) = delete; + + // Move ctor/operator + MessageManager(MessageManager&&) = default; + MessageManager& operator=(MessageManager&&) = default; + + template + auto subscribe(Callable&& c) + { + msg_id id = + std::make_pair(m_callback_map.size(), typeid(Callable).hash_code()); + + return get_queue_impl( + id, RAJA::MsgCallback {std::forward(c)}); + } + + template + void subscribe(msg_id id, Callable&& c) + { + RAJA::MsgCallback callback {std::forward(c)}; + auto& fn_list = m_callback_map.at(id); + auto it = std::ranges::find_if(fn_list, [](const auto& fn) { + return std::type_index {typeid(Callable)} == fn->get_type(); + }); + + using msg_callback_t = decltype(callback); + if (it != fn_list.end()) + { + *it = std::make_unique(std::move(callback)); + } + else + { + fn_list.emplace_back( + std::make_unique(std::move(callback))); + } + } + + template + void unsubscribe(msg_id id, Callable&&) + { + auto& fn_list = m_callback_map.at(id); + auto it = std::find_if(fn_list.begin(), fn_list.end(), [](const auto& fn) { + return std::type_index {typeid(Callable)} == fn->get_type(); + }); + + if (it != fn_list.end()) + { + fn_list.erase(it); + } + else + { + throw std::runtime_error("Callable is not subscribed"); + } + } + + void unsubscribe_all(msg_id id) { m_callback_map.erase(id); } + + void unsubscribe_all() { m_callback_map.clear(); } + + void clear() { m_bus.clear_messages(); } + + bool test_any() { return m_bus.has_pending_messages(); } + + auto get_messages() + { + std::vector messages; + + if (test_any()) + { + for (const auto& msg : m_bus) + { + messages.emplace_back(&msg); + } + } + + return messages; + } + + /** + * This takes in a container of messages and applies them to the + * callbacks. Once messages are handled, then container is cleared. + */ + template + void handle_all(Container& messages) + { + if (!m_callback_map.empty()) + { + for (const auto& msg : messages) + { + msg_id id = std::make_pair(msg->type, msg->hash); + for (auto& callback : m_callback_map[id]) + { + (*callback)(msg->args); + } + msg->~MsgHeader(); + } + messages.clear(); + } + clear(); + } + + void wait_all() + { + // This checks to verify there are callbacks before getting messages + // since `get_messages` will force a sync. + if (!m_callback_map.empty()) + { + auto messages = get_messages(); + handle_all(messages); + } + clear(); + } + +private: + template + auto get_queue_impl(msg_id id, MsgCallback&& c) + { + using msg_callback_t = MsgCallback; + + m_callback_map[id].emplace_back( + std::make_unique(std::move(c))); + + return m_bus.template get_queue...>(id); + } + + msg_bus m_bus; + std::unordered_map + m_callback_map; +}; + +template> +auto make_message_manager(std::size_t bus_sz, + Resource r, + Allocator alloc = Allocator {}) +{ + return RAJA::MessageManager(bus_sz, r, alloc); +} + +template())>> +auto make_message_manager(std::size_t bus_sz, Allocator alloc = Allocator {}) +{ + auto r = RAJA::resources::get_default_resource(); + return RAJA::MessageManager(bus_sz, r, alloc); +} + +} // namespace RAJA + +#endif /* RAJA_MSG_MANAGER_HPP */ diff --git a/include/RAJA/policy/msg_queue.hpp b/include/RAJA/policy/msg_queue.hpp new file mode 100644 index 0000000000..894de8cf5d --- /dev/null +++ b/include/RAJA/policy/msg_queue.hpp @@ -0,0 +1,24 @@ +/*! + ****************************************************************************** + * + * \file + * + * \brief Header file containing RAJA headers for message queue policies. + * + ****************************************************************************** + */ + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2016-25, Lawrence Livermore National Security, LLC +// and RAJA project contributors. See the RAJA/LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#ifndef RAJA_msg_queue_HPP +#define RAJA_msg_queue_HPP + +#include "RAJA/policy/msg_queue/mpsc_queue.hpp" +#include "RAJA/policy/msg_queue/spsc_queue.hpp" + +#endif // closing endif for header file include guard diff --git a/include/RAJA/policy/msg_queue/mpsc_queue.hpp b/include/RAJA/policy/msg_queue/mpsc_queue.hpp new file mode 100644 index 0000000000..ca51189534 --- /dev/null +++ b/include/RAJA/policy/msg_queue/mpsc_queue.hpp @@ -0,0 +1,108 @@ +/*! + ****************************************************************************** + * + * \file + * + * \brief Header file containing implementation for a MPSC + * message queue policy. By SPSC, means multi-producer + * single-consumer. In other words, messages produced + * could be from multiple thread may require atomics. + * + ****************************************************************************** + */ + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2016-25, Lawrence Livermore National Security, LLC +// and RAJA project contributors. See the RAJA/LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#ifndef RAJA_mpsc_queue_HPP +#define RAJA_mpsc_queue_HPP + +#include + +#include "RAJA/util/align.hpp" +#include "RAJA/util/concepts.hpp" +#include "RAJA/pattern/atomic.hpp" + +#include "RAJA/pattern/messages/msg_header.hpp" +#include "RAJA/policy/msg_queue/policy.hpp" + +namespace RAJA +{ +namespace messages +{ + +template +class Queue> +{ +public: + using policy = RAJA::mpsc_queue; + + using args_type = camp::tuple; + using size_type = typename MsgBusType::size_type; + + explicit Queue(std::pair id, MsgBusType& bus) + : m_type {id.first}, + m_hash {id.second}, + m_bus {&bus} + {} + + explicit Queue(std::pair id, MsgBusType* bus) + : m_type {id.first}, + m_hash {id.second}, + m_bus {bus} + {} + + auto get_id() const noexcept { return std::make_pair(m_type, m_hash); } + + /// Posts message to queue. This is marked `const` to pass to lambda by + /// copy. This throws away messages that are over the capacity of the + /// container. + template + RAJA_HOST_DEVICE bool try_post_message(Ts&&... args) const + { + if (m_bus != nullptr) + { + constexpr size_type header_sz = align_sz(sizeof(MsgHeader)); + constexpr size_type args_sz = align_sz(sizeof(MsgArgs)); + constexpr size_type msg_sz = header_sz + args_sz; + + const size_type capacity = m_bus->m_capacity; + + // Checks if message can fit in queue. If so, adds msg_sz to end of queue + // to reserve space. Otherwise, message doesn't fit and no space is + // reserved. In other words, the CAS-loop below performs the follwing + // operation: + // (*address + msg_sz <= capacity) ? (*address + msg_sz) : *address; + size_type local_sz = RAJA::atomicGeneric( + &(m_bus->m_end), [=](size_type old_sz) { + return (old_sz + msg_sz <= capacity) ? (old_sz + msg_sz) : old_sz; + }); + + if (m_bus->m_data != nullptr && local_sz + msg_sz <= capacity) + { + char* buf = m_bus->m_data + local_sz; + new (buf) MsgHeader {args_sz, m_type, m_hash, buf + header_sz}; + new (buf + header_sz) + MsgArgs {args_type(std::forward(args)...)}; + + return true; + } + } + + return false; + } + +private: + std::size_t m_type; + std::size_t m_hash; + MsgBusType* m_bus; +}; + +} // namespace messages +} // namespace RAJA + +#endif // closing endif for header file include guard diff --git a/include/RAJA/policy/msg_queue/policy.hpp b/include/RAJA/policy/msg_queue/policy.hpp new file mode 100644 index 0000000000..0448336092 --- /dev/null +++ b/include/RAJA/policy/msg_queue/policy.hpp @@ -0,0 +1,69 @@ +/*! + ****************************************************************************** + * + * \file + * + * \brief Header file containing RAJA message queue policy definitions. + * + ****************************************************************************** + */ + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2016-25, Lawrence Livermore National Security, LLC +// and RAJA project contributors. See the RAJA/LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#ifndef policy_msg_queue_HPP +#define policy_msg_queue_HPP + +#include "RAJA/policy/PolicyBase.hpp" + +namespace RAJA +{ +namespace messages +{ + +/// +/// This is a view-like queue so that message queues can be copied to kernels. +/// +template +class Queue; + +} // namespace messages + +namespace policy +{ +namespace messages +{ + +// +////////////////////////////////////////////////////////////////////// +// +// Queue policies +// +////////////////////////////////////////////////////////////////////// +// + +template +struct mpsc_queue +{ + static constexpr bool should_overwrite = Overwrite; +}; + +template +struct spsc_queue +{ + static constexpr bool should_overwrite = Overwrite; +}; + +} // namespace messages +} // namespace policy + +using spsc_queue = policy::messages::spsc_queue; +using mpsc_queue = policy::messages::mpsc_queue; + +} // namespace RAJA + +#endif diff --git a/include/RAJA/policy/msg_queue/spsc_queue.hpp b/include/RAJA/policy/msg_queue/spsc_queue.hpp new file mode 100644 index 0000000000..2381d2eb1c --- /dev/null +++ b/include/RAJA/policy/msg_queue/spsc_queue.hpp @@ -0,0 +1,97 @@ +/*! + ****************************************************************************** + * + * \file + * + * \brief Header file containing implementation for a SPSC + * message queue policy. By SPSC, means single-producer + * single-consumer. In other words, messages will be + * produced from one thread and no atomics needed. + * + ****************************************************************************** + */ + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2016-25, Lawrence Livermore National Security, LLC +// and RAJA project contributors. See the RAJA/LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#ifndef RAJA_spsc_queue_HPP +#define RAJA_spsc_queue_HPP + +#include + +#include "RAJA/util/concepts.hpp" +#include "RAJA/pattern/atomic.hpp" + +#include "RAJA/pattern/messages/msg_header.hpp" +#include "RAJA/policy/msg_queue/policy.hpp" + +namespace RAJA +{ +namespace messages +{ + +template +class Queue> +{ +public: + using policy = RAJA::spsc_queue; + + using args_type = camp::tuple; + using size_type = typename MsgBusType::size_type; + + explicit Queue(std::pair id, MsgBusType& bus) + : m_type {id.first}, + m_hash {id.second}, + m_bus {&bus} + {} + + explicit Queue(std::pair id, MsgBusType* bus) + : m_type {id.first}, + m_hash {id.second}, + m_bus {bus} + {} + + auto get_id() const noexcept { return std::make_pair(m_type, m_hash); } + + /// Posts message to queue. This is marked `const` to pass to lambda by + /// copy. This throws away messages that are over the capacity of the + /// bus. + template + bool try_post_message(Ts&&... args) const + { + if (m_bus != nullptr) + { + constexpr size_type header_sz = align_sz(sizeof(MsgHeader)); + constexpr size_type args_sz = align_sz(sizeof(MsgArgs)); + constexpr size_type msg_sz = header_sz + args_sz; + + auto local_size = m_bus->m_end; + if (m_bus->m_data != nullptr && local_size + msg_sz <= m_bus->m_capacity) + { + m_bus->m_end += msg_sz; + char* buf = m_bus->m_data + local_size; + new (buf) MsgHeader {args_sz, m_type, m_hash, buf + header_sz}; + new (buf + header_sz) + MsgArgs {args_type(std::forward(args)...)}; + + return true; + } + } + + return false; + } + +private: + std::size_t m_type; + std::size_t m_hash; + MsgBusType* m_bus; +}; + +} // namespace messages +} // namespace RAJA + +#endif // closing endif for header file include guard diff --git a/include/RAJA/util/FunctionTypeTraits.hpp b/include/RAJA/util/FunctionTypeTraits.hpp new file mode 100644 index 0000000000..068c15aa37 --- /dev/null +++ b/include/RAJA/util/FunctionTypeTraits.hpp @@ -0,0 +1,64 @@ +/*! + ****************************************************************************** + * + * \file + * + * \brief RAJA header file for utility metaprogramming related to functions. + * + ****************************************************************************** + */ + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#ifndef RAJA_FUNCTION_SIGNATURE_UTILS_HPP +#define RAJA_FUNCTION_SIGNATURE_UTILS_HPP + +#include "camp/tuple.hpp" + +namespace RAJA +{ +namespace internal +{ + +template +struct signature_impl; + +template +struct signature_impl +{ + using type = R(Args...); + using args_t = camp::tuple; +}; + +template +struct signature_impl + : public signature_impl +{}; + +template +struct signature_impl + : public signature_impl +{}; + +template +struct signature_impl + : public signature_impl +{}; + +template +using signature_t = typename signature_impl::type; + +template +using func_arg_t = + camp::tuple_element_t::args_t>; + +} // namespace internal +} // namespace RAJA +#endif // RAJA_FUNCTION_SIGNATURE_UTILS_HPP diff --git a/include/RAJA/util/HashCombiner.hpp b/include/RAJA/util/HashCombiner.hpp new file mode 100644 index 0000000000..60d74fdd72 --- /dev/null +++ b/include/RAJA/util/HashCombiner.hpp @@ -0,0 +1,70 @@ +/*! + ****************************************************************************** + * + * \file + * + * \brief Header file for RAJA HashCombiner. + * + ****************************************************************************** + */ + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#ifndef RAJA_HashCombiner_HPP +#define RAJA_HashCombiner_HPP + +#include +#include + +namespace RAJA +{ +namespace detail +{ +/*! + * @brief Combines hashes of all types together. + * @param values Objects to combine hashes for + * @return Returns hash combined of objects + * + * This will combine the hash values of various objects. The + * hash value of each type uses `std::hash`. + * + * Note: this is a simple hash combiner that can generate + * collisons. + * + */ +template +constexpr std::size_t hash_combine(Ts&&... values) +{ + std::size_t hash = 0; + + ((hash ^= (std::hash> {}(values) << 1)), ...); + + return hash; +} + +/*! + * @brief Combines hashes of pair types together. + * + * This will combine the hash values of the two stored types using + * `RAJA::hash_combine`. + * + */ +struct PairHash +{ + template + constexpr std::size_t operator()(const std::pair& p) const + { + return RAJA::detail::hash_combine(p.first, p.second); + } +}; +} // end namespace detail +} // end namespace RAJA + +#endif /* RAJA_HashComber_HPP */ diff --git a/include/RAJA/util/ResourceAllocator.hpp b/include/RAJA/util/ResourceAllocator.hpp new file mode 100644 index 0000000000..7d3f9fd110 --- /dev/null +++ b/include/RAJA/util/ResourceAllocator.hpp @@ -0,0 +1,116 @@ +/*! + ****************************************************************************** + * + * \file + * + * \brief RAJA header file for a C++ style allocator with a resource. + * + ****************************************************************************** + */ + +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#ifndef RAJA_RESOURCE_ALLOCATOR_HPP +#define RAJA_RESOURCE_ALLOCATOR_HPP + +#include + +#include "RAJA/config.hpp" +#include "RAJA/util/resource.hpp" +#include "RAJA/util/macros.hpp" + +namespace RAJA +{ + +namespace detail +{ +template +struct ResourceAllocator +{ + template + struct allocator + { + using value_type = T; + + allocator() + : allocator(Resource::get_default(), + RAJA::resources::MemoryAccess::Pinned) + {} + + allocator(Resource res, + RAJA::resources::MemoryAccess mem_type = + RAJA::resources::MemoryAccess::Pinned) + : m_res {res}, + m_mem_type {mem_type} + {} + + allocator(allocator const&) = default; + allocator(allocator&&) = default; + + allocator& operator=(allocator const&) = default; + allocator& operator=(allocator&&) = default; + + template + allocator(allocator const& other) noexcept + : m_res(other.get_resource()), + m_mem_type {other.get_mem_access()} + {} + + /*[[nodiscard]]*/ + value_type* allocate(std::size_t num) + { + if (num > std::numeric_limits::max() / sizeof(value_type)) + { + throw std::bad_alloc(); + } + + value_type* ptr = m_res.template allocate(num, m_mem_type); + + if (!ptr) + { + throw std::bad_alloc(); + } + + return ptr; + } + + void deallocate(value_type* ptr, std::size_t) noexcept + { + m_res.deallocate(ptr, m_mem_type); + } + + Resource const& get_resource() const { return m_res; } + + Resource get_resource() { return m_res; } + + RAJA::resources::MemoryAccess get_mem_access() const { return m_mem_type; } + + RAJA::resources::MemoryAccess get_mem_access() { return m_mem_type; } + + template + friend inline bool operator==(allocator const& lhs, allocator const& rhs) + { + return lhs.get_resource() == rhs.get_resource(); + } + + private: + Resource m_res; + RAJA::resources::MemoryAccess m_mem_type; + }; +}; +} // namespace detail + +template +using ResourceAllocator = + typename detail::ResourceAllocator::template allocator; + +} // namespace RAJA + +#endif // RAJA_RESOURCE_ALLOCATOR_HPP diff --git a/include/RAJA/util/align.hpp b/include/RAJA/util/align.hpp index 5333273a42..be65c395ef 100644 --- a/include/RAJA/util/align.hpp +++ b/include/RAJA/util/align.hpp @@ -20,10 +20,24 @@ #ifndef RAJA_ALIGN_HPP #define RAJA_ALIGN_HPP +#include + #include "RAJA/config.hpp" +#include "RAJA/util/macros.hpp" namespace RAJA { +RAJA_HOST_DEVICE +constexpr std::size_t align_sz(std::size_t size, std::size_t alignment = 16) +/** Returns the aligned size. This would use `alignof(std::max_align_t)`; + * however, the device side can get a different value compared to the + * host. + * + * @return The size with proper alignment + */ +{ + return ((size + alignment - 1) / alignment) * alignment; +} //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// // Taken from libc++ diff --git a/test/functional/forall/CMakeLists.txt b/test/functional/forall/CMakeLists.txt index d2cffa9346..3c1f1f0697 100644 --- a/test/functional/forall/CMakeLists.txt +++ b/test/functional/forall/CMakeLists.txt @@ -41,6 +41,8 @@ add_subdirectory(reduce-multiple-indexset) add_subdirectory(multi-reduce-basic) +add_subdirectory(messages) + add_subdirectory(resource-indexset) add_subdirectory(resource-segment) diff --git a/test/functional/forall/messages/CMakeLists.txt b/test/functional/forall/messages/CMakeLists.txt new file mode 100644 index 0000000000..9ec191e3cb --- /dev/null +++ b/test/functional/forall/messages/CMakeLists.txt @@ -0,0 +1,30 @@ +############################################################################### +# Copyright (c) Lawrence Livermore National Security, LLC and other +# RAJA Project Developers. See top-level LICENSE and COPYRIGHT +# files for dates and other details. No copyright assignment is required +# to contribute to RAJA. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +# +# List of experimental reduction types for generating test files. +# +set(DATATYPES CoreMsgDataTypeList) + + +# +# Generate core reduction tests for each enabled RAJA back-end +# +# Note: FORALL_BACKENDS is defined in ../CMakeLists.txt +# + +foreach( BACKEND ${FORALL_BACKENDS} ) + configure_file( test-forall-basic-msg.cpp.in + test-forall-basic-msg-${BACKEND}.cpp ) + raja_add_test( NAME test-forall-basic-msg-${BACKEND} + SOURCES ${CMAKE_CURRENT_BINARY_DIR}/test-forall-basic-msg-${BACKEND}.cpp ) + + target_include_directories(test-forall-basic-msg-${BACKEND}.exe + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests) +endforeach() diff --git a/test/functional/forall/messages/test-forall-basic-msg.cpp.in b/test/functional/forall/messages/test-forall-basic-msg.cpp.in new file mode 100644 index 0000000000..4febb3c27c --- /dev/null +++ b/test/functional/forall/messages/test-forall-basic-msg.cpp.in @@ -0,0 +1,55 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +// +// test/include headers +// +#include "RAJA_test-base.hpp" +#include "RAJA_test-camp.hpp" +#include "RAJA_test-index-types.hpp" + +#include "RAJA_test-forall-data.hpp" +#include "RAJA_test-forall-execpol.hpp" + + +// +// Header for tests in ./tests directory +// +// Note: CMake adds ./tests as an include dir for these tests. +// +#include "test-forall-basic-msg.hpp" + +// +// Data types for core reduction basic tests +// +using CoreMsgDataTypeList = camp::list< int, + float, + double >; + +// +// These tests exercise only one index type. We parameterize here to +// make it easier to expand types in the future if needed. +// +using TestIdxTypeList = camp::list< RAJA::Index_type >; + +// +// Cartesian product of types used in parameterized tests +// +using @BACKEND@ForallMsgBasicTypes = + Test< camp::cartesian_product>::Types; + +// +// Instantiate parameterized test +// +INSTANTIATE_TYPED_TEST_SUITE_P(@BACKEND@, + ForallMsgBasicTest, + @BACKEND@ForallMsgBasicTypes); diff --git a/test/functional/forall/messages/tests/test-forall-basic-msg.hpp b/test/functional/forall/messages/tests/test-forall-basic-msg.hpp new file mode 100644 index 0000000000..0415f370e0 --- /dev/null +++ b/test/functional/forall/messages/tests/test-forall-basic-msg.hpp @@ -0,0 +1,157 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) Lawrence Livermore National Security, LLC and other +// RAJA Project Developers. See top-level LICENSE and COPYRIGHT +// files for dates and other details. No copyright assignment is required +// to contribute to RAJA. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#ifndef __TEST_FORALL_BASIC_MSG_HPP__ +#define __TEST_FORALL_BASIC_MSG_HPP__ + +#include +#include +#include +#include + +template +void ForallMsgBasicTestImpl(const SEG_TYPE& seg, + const std::vector& seg_idx, + WORKING_RES working_res) +{ + IDX_TYPE data_len = seg_idx[seg_idx.size() - 1] + 1; + IDX_TYPE idx_len = static_cast( seg_idx.size() ); + + DATA_TYPE* working_array; + DATA_TYPE* check_array; + DATA_TYPE* test_array; + + allocateForallTestData(data_len, + working_res, + &working_array, + &check_array, + &test_array); + + const int modval = 100; + + for (IDX_TYPE i = 0; i < data_len; ++i) { + test_array[i] = static_cast( rand() % modval ); + } + + DATA_TYPE ref_max = 0; + for (IDX_TYPE i = 0; i < idx_len; ++i) { + ref_max = std::max(ref_max, test_array[ seg_idx[i] ]); + } + + working_res.memcpy(working_array, test_array, sizeof(DATA_TYPE) * data_len); + + const std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(msg_sz*idx_len, working_res); + + DATA_TYPE msg_max = 0; + auto msg_queue = msg_manager.template subscribe([&] (DATA_TYPE data) { + msg_max = std::max(msg_max, data); + }); + + RAJA::forall(seg, RAJA::Name("Test Messages"), [=] RAJA_HOST_DEVICE(IDX_TYPE idx) { + msg_queue.try_post_message(working_array[idx]); + }); + + auto messages = msg_manager.get_messages(); + ASSERT_EQ(static_cast(messages.size()), idx_len); + + msg_manager.handle_all(messages); + ASSERT_EQ(msg_max, ref_max); + + deallocateForallTestData(working_res, + working_array, + check_array, + test_array); +} + + +TYPED_TEST_SUITE_P(ForallMsgBasicTest); +template +class ForallMsgBasicTest : public ::testing::Test +{ +}; + +TYPED_TEST_P(ForallMsgBasicTest, MsgBasicForall) +{ + using IDX_TYPE = typename camp::at>::type; + using DATA_TYPE = typename camp::at>::type; + using WORKING_RES = typename camp::at>::type; + using EXEC_POLICY = typename camp::at>::type; + + auto working_res = WORKING_RES::get_default(); + + std::vector seg_idx; + +// Range segment tests + RAJA::TypedRangeSegment r1( 0, 28 ); + RAJA::getIndices(seg_idx, r1); + ForallMsgBasicTestImpl, + EXEC_POLICY>( + r1, seg_idx, working_res); + + seg_idx.clear(); + RAJA::TypedRangeSegment r2( 3, 642 ); + RAJA::getIndices(seg_idx, r2); + ForallMsgBasicTestImpl, + EXEC_POLICY>( + r2, seg_idx, working_res); + + seg_idx.clear(); + RAJA::TypedRangeSegment r3( 0, 2057 ); + RAJA::getIndices(seg_idx, r3); + ForallMsgBasicTestImpl, + EXEC_POLICY>( + r3, seg_idx, working_res); + +// Range-stride segment tests + seg_idx.clear(); + RAJA::TypedRangeStrideSegment r4( 0, 188, 2 ); + RAJA::getIndices(seg_idx, r4); + ForallMsgBasicTestImpl, + EXEC_POLICY>( + r4, seg_idx, working_res); + + seg_idx.clear(); + RAJA::TypedRangeStrideSegment r5( 3, 1029, 3 ); + RAJA::getIndices(seg_idx, r5); + ForallMsgBasicTestImpl, + EXEC_POLICY>( + r5, seg_idx, working_res); + +// List segment tests + seg_idx.clear(); + IDX_TYPE last = 10567; + srand( time(NULL) ); + for (IDX_TYPE i = 0; i < last; ++i) { + IDX_TYPE randval = IDX_TYPE( rand() % RAJA::stripIndexType(last) ); + if ( i < randval ) { + seg_idx.push_back(i); + } + } + RAJA::TypedListSegment l1( &seg_idx[0], seg_idx.size(), + working_res ); + ForallMsgBasicTestImpl, + EXEC_POLICY>( + l1, seg_idx, working_res); +} + +REGISTER_TYPED_TEST_SUITE_P(ForallMsgBasicTest, + MsgBasicForall); + +#endif // __TEST_FORALL_BASIC_MSG_HPP__ diff --git a/test/include/RAJA_test-forall-execpol.hpp b/test/include/RAJA_test-forall-execpol.hpp index 1fad039c8a..cb7432c414 100644 --- a/test/include/RAJA_test-forall-execpol.hpp +++ b/test/include/RAJA_test-forall-execpol.hpp @@ -30,6 +30,8 @@ using SequentialForallReduceExecPols = camp::list< RAJA::seq_exec >; using SequentialForallAtomicExecPols = camp::list< RAJA::seq_exec >; +using SequentialForallMsgExecPols = camp::list< RAJA::seq_exec >; + #if defined(RAJA_ENABLE_OPENMP) using OpenMPForallExecPols = camp::list< RAJA::omp_parallel_for_exec @@ -93,6 +95,8 @@ using OpenMPForallAtomicExecPols = #endif >; +using OpenMPForallMsgExecPols = OpenMPForallExecPols; + #endif // RAJA_ENABLE_OPENMP #if defined(RAJA_ENABLE_TARGET_OPENMP) @@ -104,6 +108,8 @@ using OpenMPTargetForallReduceExecPols = OpenMPTargetForallExecPols; using OpenMPTargetForallAtomicExecPols = OpenMPTargetForallExecPols; +using OpenMPTargetForallMsgExecPols = OpenMPTargetForallExecPols; + #endif #if defined(RAJA_ENABLE_CUDA) @@ -118,6 +124,8 @@ using CudaForallReduceExecPols = CudaForallExecPols; using CudaForallAtomicExecPols = CudaForallExecPols; +using CudaForallMsgExecPols = CudaForallExecPols; + #endif #if defined(RAJA_ENABLE_HIP) @@ -131,6 +139,8 @@ using HipForallReduceExecPols = HipForallExecPols; using HipForallAtomicExecPols = HipForallExecPols; +using HipForallMsgExecPols = HipForallExecPols; + #endif #if defined(RAJA_ENABLE_SYCL) @@ -141,6 +151,8 @@ using SyclForallReduceExecPols = SyclForallExecPols; using SyclForallAtomicExecPols = SyclForallExecPols; +using SyclForallMsgExecPols = SyclForallExecPols; + #endif #endif // __RAJA_test_forall_execpol_HPP__ diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 8c5cb53271..7beedb84b1 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -19,3 +19,4 @@ add_subdirectory(view-layout) add_subdirectory(algorithm) add_subdirectory(workgroup) add_subdirectory(indexing) +add_subdirectory(messages) diff --git a/test/unit/messages/CMakeLists.txt b/test/unit/messages/CMakeLists.txt new file mode 100644 index 0000000000..a16e68735a --- /dev/null +++ b/test/unit/messages/CMakeLists.txt @@ -0,0 +1,11 @@ +############################################################################### +# Copyright (c) 2016-25, Lawrence Livermore National Security, LLC +# and RAJA project contributors. See the RAJA/LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) +############################################################################### + +raja_add_test( + NAME test-messages + SOURCES test-messages.cpp) + diff --git a/test/unit/messages/test-messages.cpp b/test/unit/messages/test-messages.cpp new file mode 100644 index 0000000000..5ec4985387 --- /dev/null +++ b/test/unit/messages/test-messages.cpp @@ -0,0 +1,314 @@ +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// +// Copyright (c) 2018-25, Lawrence Livermore National Security, LLC +// and Camp project contributors. See the camp/LICENSE file for details. +// +// SPDX-License-Identifier: (BSD-3-Clause) +//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~// + +#include "camp/array.hpp" +#include "RAJA_test-base.hpp" + +#include "gtest/gtest.h" + +TEST(message_handler, initialize) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(msg_sz); + + int test = 0; + msg_manager.subscribe([&](int val) { + test = val; + }); + + ASSERT_EQ(msg_manager.test_any(), false); + ASSERT_EQ(test, 0); +} + +TEST(message_handler, initialize_with_resource) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(msg_sz, camp::resources::Host()); + + int test = 0; + msg_manager.subscribe([&](int val) { + test = val; + }); + + ASSERT_EQ(msg_manager.test_any(), false); + ASSERT_EQ(test, 0); +} + +TEST(message_handler, clear) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(msg_sz); + + int test = 0; + auto q = msg_manager.subscribe([&](int val) { + test = val; + }); + + ASSERT_EQ(q.try_post_message(5), true); + + msg_manager.clear(); + msg_manager.wait_all(); + + ASSERT_EQ(test, 0); +} + +TEST(message_handler, try_post_message) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(msg_sz); + + int test = 0; + msg_manager.subscribe([&](int val) { + test = val; + }); + + ASSERT_EQ(test, 0); +} + +TEST(message_handler, try_post_message_overflow) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(msg_sz); + + int test = 0; + auto q = msg_manager.subscribe([&](int val) { + test = val; + }); + + ASSERT_EQ(q.try_post_message(5), true); + ASSERT_EQ(q.try_post_message(7), false); + + ASSERT_EQ(test, 0); +} + +TEST(message_handler, wait_all) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(msg_sz); + + int test = 0; + auto q = msg_manager.subscribe([&](int val) { + test = val; + }); + + ASSERT_EQ(q.try_post_message(1), true); + + msg_manager.wait_all(); + + ASSERT_EQ(test, 1); +} + +TEST(message_handler, wait_all_overalloc) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(2*msg_sz); + + int test = 0; + auto q = msg_manager.subscribe([&](int val) { + test = val; + }); + + ASSERT_EQ(q.try_post_message(1), true); + + msg_manager.wait_all(); + + ASSERT_EQ(test, 1); +} + +TEST(message_handler, wait_all_array) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs>)); + + auto msg_manager = RAJA::make_message_manager(msg_sz); + + camp::array test = {0, 0, 0}; + auto q = msg_manager.subscribe( + [&](camp::array val) { + test[0] = val[0]; + test[1] = val[1]; + test[2] = val[2]; + } + ); + + camp::array a{1,2,3}; + ASSERT_EQ(q.try_post_message(a), true); + + msg_manager.wait_all(); + + ASSERT_EQ(test[0], 1); + ASSERT_EQ(test[1], 2); + ASSERT_EQ(test[2], 3); +} + +TEST(message_handler, wait_all_overflow) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(msg_sz); + + int test = 0; + auto q = msg_manager.subscribe([&](int val) { + test = val; + }); + + ASSERT_EQ(q.try_post_message(5), true); + ASSERT_EQ(q.try_post_message(7), false); + + msg_manager.wait_all(); + + ASSERT_EQ(test, 5); +} + +TEST(message_handler, subscribe) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(msg_sz); + + int test = 0; + auto q = msg_manager.subscribe([&] (int val) { + test += val; + }); + msg_manager.subscribe(q.get_id(), [&] (int val) { + test *= val; + }); + + ASSERT_EQ(q.try_post_message(5), true); + + msg_manager.wait_all(); + + ASSERT_EQ(test, 25); +} + +TEST(message_handler, unsubscribe) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(msg_sz); + + int test = 0; + auto update = [&](int val) { test = val; }; + auto q = msg_manager.subscribe(update); + msg_manager.subscribe(q.get_id(), [&] (int val) { + test += val; + }); + + ASSERT_EQ(q.try_post_message(1), true); + + msg_manager.unsubscribe(q.get_id(), update); + msg_manager.wait_all(); + + ASSERT_EQ(test, 1); +} + +TEST(message_handler, unsubscribe_all_id) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(2*msg_sz); + + int test1 = 0; + int test2 = 0; + auto q1 = msg_manager.subscribe([&]() { + test1 = 1; + }); + auto q2 = msg_manager.subscribe([&]() { + test2 = 2; + }); + + ASSERT_EQ(q1.try_post_message(), true); + ASSERT_EQ(q2.try_post_message(), true); + + msg_manager.unsubscribe_all(q1.get_id()); + msg_manager.wait_all(); + + ASSERT_EQ(test1, 0); + ASSERT_EQ(test2, 2); +} + +TEST(message_handler, unsubscribe_all) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(2*msg_sz); + + int test1 = 0; + int test2 = 0; + auto q1 = msg_manager.subscribe([&]() { + test1 = 1; + }); + auto q2 = msg_manager.subscribe([&]() { + test2 = 2; + }); + + ASSERT_EQ(q1.try_post_message(), true); + ASSERT_EQ(q2.try_post_message(), true); + + msg_manager.unsubscribe_all(); + msg_manager.wait_all(); + + ASSERT_EQ(test1, 0); + ASSERT_EQ(test2, 0); +} + +TEST(message_handler, get_messages) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs)); + + auto msg_manager = RAJA::make_message_manager(20*msg_sz); + + int test1 = 0; + auto q1 = msg_manager.subscribe([&](int val) { + test1 = val; + }); + + ASSERT_EQ(q1.try_post_message(1), true); + ASSERT_EQ(q1.try_post_message(2), true); + ASSERT_EQ(q1.try_post_message(3), true); + + + auto msg_list = msg_manager.get_messages(); + ASSERT_EQ(msg_list.size(), 3); + ASSERT_EQ(test1, 0); +} + +TEST(message_handler, handle_all_sort) { + constexpr std::size_t msg_sz = RAJA::align_sz(sizeof(RAJA::MsgHeader)) + + RAJA::align_sz(sizeof(RAJA::MsgArgs<>)); + + auto msg_manager = RAJA::make_message_manager(20*msg_sz); + + int test1 = 0; + auto q1 = msg_manager.subscribe([&]() { + test1 = 1; + }); + auto q2 = msg_manager.subscribe([&]() { + test1 = 2; + }); + + ASSERT_EQ(q1.try_post_message(), true); + ASSERT_EQ(q2.try_post_message(), true); + ASSERT_EQ(q1.try_post_message(), true); + + auto msg_list = msg_manager.get_messages(); + ASSERT_EQ(msg_list.size(), 3); + + // Forces all q2 messages to the end + std::sort(msg_list.begin(), msg_list.end(), [] (auto msg1, auto msg2) { + return msg1->type < msg2->type; + }); + msg_manager.handle_all(msg_list); + + ASSERT_EQ(test1, 2); +}