diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e17446854e..f341af91ee7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,6 +66,7 @@ option(GINKGO_FORCE_GPU_AWARE_MPI "Assert that the MPI library is GPU aware. Thi catastrophically in case the MPI implementation is not GPU Aware, and GPU aware functionality has been forced" OFF) set(GINKGO_CI_TEST_OMP_PARALLELISM "4" CACHE STRING "The number of OpenMP threads to use for a test binary during CTest resource file-constrained test.") +option(GINKGO_EXTENSION_KOKKOS_CHECK_TYPE_ALIGNMENT "Enables mapping to Kokkos types to check the alignment of the source and target type." ON) gko_rename_cache(GINKGO_COMPILER_FLAGS CMAKE_CXX_FLAGS BOOL "Flags used by the CXX compiler during all build types.") gko_rename_cache(GINKGO_CUDA_COMPILER_FLAGS CMAKE_CUDA_FLAGS BOOL "Flags used by the CUDA compiler during all build types.") @@ -280,6 +281,10 @@ else() endif() configure_file(${Ginkgo_SOURCE_DIR}/include/ginkgo/config.hpp.in ${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp @ONLY) +configure_file(${Ginkgo_SOURCE_DIR}/include/ginkgo/extensions/kokkos/config.hpp.in + ${Ginkgo_BINARY_DIR}/include/ginkgo/extensions/kokkos/config.hpp + @ONLY +) # Ginkgo core libraries # Needs to be first in order for `CMAKE_CUDA_DEVICE_LINK_EXECUTABLE` to be @@ -308,6 +313,8 @@ if(GINKGO_BUILD_TESTS) endif() # Non core directories and targets +add_subdirectory(extensions) + if(GINKGO_BUILD_EXAMPLES) add_subdirectory(examples) endif() diff --git a/cmake/build_helpers.cmake b/cmake/build_helpers.cmake index a1a1735f84e..e8691b77587 100644 --- a/cmake/build_helpers.cmake +++ b/cmake/build_helpers.cmake @@ -56,6 +56,7 @@ function(ginkgo_check_headers target defines) list(FILTER CXX_HEADERS EXCLUDE REGEX ".*\.hip\.hpp$") list(FILTER CXX_HEADERS EXCLUDE REGEX "^test.*") list(FILTER CXX_HEADERS EXCLUDE REGEX "^base/kernel_launch.*") + list(FILTER CXX_HEADERS EXCLUDE REGEX "^ginkgo/extensions/.*") list(FILTER CUDA_HEADERS EXCLUDE REGEX "^test.*") list(FILTER CUDA_HEADERS EXCLUDE REGEX "^base/kernel_launch.*") list(FILTER HIP_HEADERS EXCLUDE REGEX "^test.*") diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake index ecb75d5da39..7fdbe7d4e53 100644 --- a/cmake/create_test.cmake +++ b/cmake/create_test.cmake @@ -1,7 +1,7 @@ set(gko_test_resource_args "RESOURCE_LOCAL_CORES;RESOURCE_TYPE") set(gko_test_single_args "MPI_SIZE;EXECUTABLE_NAME;${gko_test_resource_args}") set(gko_test_multi_args "DISABLE_EXECUTORS;ADDITIONAL_LIBRARIES;ADDITIONAL_INCLUDES") -set(gko_test_option_args "NO_RESOURCES") +set(gko_test_option_args "NO_RESOURCES;NO_GTEST_MAIN") ## Replaces / by _ to create valid target names from relative paths function(ginkgo_build_test_name test_name target_name) @@ -14,7 +14,7 @@ endfunction() ## Set up shared target properties and handle ADDITIONAL_LIBRARIES/ADDITIONAL_INCLUDES ## `MPI_SIZE size` causes the tests to be run with `size` MPI processes. function(ginkgo_set_test_target_properties test_target_name test_library_suffix) - cmake_parse_arguments(PARSE_ARGV 1 set_properties "" "${gko_test_single_args}" "${gko_test_multi_args}") + cmake_parse_arguments(PARSE_ARGV 1 set_properties "${gko_test_option_args}" "${gko_test_single_args}" "${gko_test_multi_args}") if (GINKGO_FAST_TESTS) target_compile_definitions(${test_target_name} PRIVATE GINKGO_FAST_TESTS) endif() @@ -27,10 +27,12 @@ function(ginkgo_set_test_target_properties test_target_name test_library_suffix) if(GINKGO_CHECK_CIRCULAR_DEPS) target_link_libraries(${test_target_name} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}") endif() - if(set_properties_MPI_SIZE) - target_link_libraries(${test_target_name} PRIVATE ginkgo_gtest_main_mpi${test_library_suffix}) - else() - target_link_libraries(${test_target_name} PRIVATE ginkgo_gtest_main${test_library_suffix}) + if(NOT set_properties_NO_GTEST_MAIN) + if(set_properties_MPI_SIZE) + target_link_libraries(${test_target_name} PRIVATE ginkgo_gtest_main_mpi${test_library_suffix}) + else() + target_link_libraries(${test_target_name} PRIVATE ginkgo_gtest_main${test_library_suffix}) + endif() endif() target_compile_features(${test_target_name} PUBLIC cxx_std_14) # we set these properties regardless of the enabled backends, diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake index 6b904189151..63f43c645f0 100644 --- a/cmake/get_info.cmake +++ b/cmake/get_info.cmake @@ -54,7 +54,7 @@ FUNCTION(ginkgo_print_flags log_type var_name) set(str_value "${${var_string}}") endif() string(SUBSTRING " --- ${var_string}: " 0 55 upd_string) +-- ${var_string}: " 0 60 upd_string) string(APPEND upd_string "${str_value}") FILE(APPEND ${log_type} ${upd_string}) ENDFUNCTION() @@ -62,7 +62,7 @@ ENDFUNCTION() function(ginkgo_print_variable log_type var_name) string(SUBSTRING " --- ${var_name}: " 0 55 upd_string) +-- ${var_name}: " 0 60 upd_string) if(${var_name} STREQUAL "") set(str_value "") else() @@ -76,7 +76,7 @@ endfunction() function(ginkgo_print_env_variable log_type var_name) string(SUBSTRING " --- ${var_name}: " 0 55 upd_string) +-- ${var_name}: " 0 60 upd_string) if(DEFINED ENV{${var_name}}) set(str_value "$ENV{${var_name}}") else() @@ -114,13 +114,13 @@ foreach(log_type ${log_types}) "PROJECT_SOURCE_DIR;PROJECT_BINARY_DIR") string(SUBSTRING " --- CMAKE_CXX_COMPILER: " 0 55 print_string) +-- CMAKE_CXX_COMPILER: " 0 60 print_string) set(str2 "${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} on platform ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR}") string(APPEND print_string "${str2}") FILE(APPEND ${${log_type}} "${print_string}") string(SUBSTRING " --- " 0 55 print_string) +-- " 0 60 print_string) set(str2 "${CMAKE_CXX_COMPILER}") string(APPEND print_string "${str2}") FILE(APPEND ${${log_type}} "${print_string}") @@ -197,7 +197,6 @@ if(TARGET PAPI::PAPI) ginkgo_print_variable(${detailed_log} "PAPI_INCLUDE_DIR") ginkgo_print_flags(${detailed_log} "PAPI_LIBRARY") endif() - ginkgo_print_variable(${minimal_log} "GINKGO_BUILD_HWLOC") ginkgo_print_variable(${detailed_log} "GINKGO_BUILD_HWLOC") if(TARGET hwloc) @@ -205,6 +204,10 @@ if(TARGET hwloc) ginkgo_print_variable(${detailed_log} "HWLOC_LIBRARIES") ginkgo_print_variable(${detailed_log} "HWLOC_INCLUDE_DIRS") endif() +ginkgo_print_module_footer(${detailed_log} "") + +ginkgo_print_generic_header(${detailed_log} " Extensions:") +ginkgo_print_variable(${detailed_log} "GINKGO_EXTENSION_KOKKOS_CHECK_TYPE_ALIGNMENT") _minimal( " diff --git a/cmake/install_helpers.cmake b/cmake/install_helpers.cmake index 898988142c9..7165cad0c2b 100644 --- a/cmake/install_helpers.cmake +++ b/cmake/install_helpers.cmake @@ -79,14 +79,15 @@ function(ginkgo_install) # install the public header files install(DIRECTORY "${Ginkgo_SOURCE_DIR}/include/" - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" - COMPONENT Ginkgo_Development - FILES_MATCHING PATTERN "*.hpp" - ) - install(FILES "${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp" - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/ginkgo" - COMPONENT Ginkgo_Development - ) + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT Ginkgo_Development + FILES_MATCHING PATTERN "*.hpp" + ) + install(DIRECTORY "${Ginkgo_BINARY_DIR}/include/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT Ginkgo_Development + FILES_MATCHING PATTERN "*.hpp" + ) if (GINKGO_HAVE_HWLOC AND NOT HWLOC_FOUND) get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY) diff --git a/dev_tools/scripts/regroup b/dev_tools/scripts/regroup index e35bd37efee..b10570f4982 100644 --- a/dev_tools/scripts/regroup +++ b/dev_tools/scripts/regroup @@ -2,7 +2,7 @@ IncludeBlocks: Regroup IncludeCategories: - Regex: '^<(nlohmann|gflags|gtest|papi).*' Priority: 3 - - Regex: '^<(omp|cu|hip|thrust|CL/|cooperative|oneapi|mpi|nvToolsExt).*' + - Regex: '^<(omp|cu|hip|thrust|CL/|cooperative|oneapi|mpi|nvToolsExt|Kokkos_Core).*' Priority: 2 - Regex: '^ "${HEADER_LIST}"; then echo "${WARNING_PREFIX} "'The `find` command returned with an error!' 1>&2 diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 69ff244fce0..653d52a1e88 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -56,15 +56,9 @@ if(GINKGO_BUILD_MPI) list(APPEND EXAMPLES_LIST distributed-solver) endif() -find_package(Kokkos QUIET) +find_package(Kokkos 4.1.00 QUIET) if(Kokkos_FOUND) - if(GINKGO_WITH_CCACHE) - message(WARNING "The CMAKE_CXX_COMPILER_LAUNCHER is set due to " - "GINKGO_WITH_CCACHE=ON which is known to cause issues with CUDA enabled " - "Kokkos (https://github.com/kokkos/kokkos/issues/4821) including compilation " - "failures. This can be prevented by setting GINKGO_WITH_CCACHE=OFF.") - endif() - list(APPEND EXAMPLES_LIST kokkos_assembly) + list(APPEND EXAMPLES_LIST kokkos-assembly) else() message(STATUS "No Kokkos found, disabling examples with Kokkos assembly.") endif() diff --git a/examples/kokkos_assembly/CMakeLists.txt b/examples/kokkos-assembly/CMakeLists.txt similarity index 59% rename from examples/kokkos_assembly/CMakeLists.txt rename to examples/kokkos-assembly/CMakeLists.txt index e60c1cf3731..39c4cabd57b 100644 --- a/examples/kokkos_assembly/CMakeLists.txt +++ b/examples/kokkos-assembly/CMakeLists.txt @@ -5,8 +5,10 @@ project(kokkos-assembly CXX) if(NOT GINKGO_BUILD_EXAMPLES) find_package(Ginkgo 1.8.0 REQUIRED) endif() +find_package(Kokkos 4.1.00 REQUIRED) -find_package(Kokkos 3.7 REQUIRED) +# Kokkos doesn't handle any compiler launcher well, so it's disable it +unset(CMAKE_CXX_COMPILER_LAUNCHER) -add_executable(kokkos-assembly kokkos_assembly.cpp) +add_executable(kokkos-assembly kokkos-assembly.cpp) target_link_libraries(kokkos-assembly Ginkgo::ginkgo Kokkos::kokkos) diff --git a/examples/kokkos_assembly/build.sh b/examples/kokkos-assembly/build.sh similarity index 100% rename from examples/kokkos_assembly/build.sh rename to examples/kokkos-assembly/build.sh diff --git a/examples/kokkos_assembly/doc/builds-on b/examples/kokkos-assembly/doc/builds-on similarity index 100% rename from examples/kokkos_assembly/doc/builds-on rename to examples/kokkos-assembly/doc/builds-on diff --git a/examples/kokkos_assembly/doc/intro.dox b/examples/kokkos-assembly/doc/intro.dox similarity index 100% rename from examples/kokkos_assembly/doc/intro.dox rename to examples/kokkos-assembly/doc/intro.dox diff --git a/examples/kokkos_assembly/doc/kind b/examples/kokkos-assembly/doc/kind similarity index 100% rename from examples/kokkos_assembly/doc/kind rename to examples/kokkos-assembly/doc/kind diff --git a/examples/kokkos_assembly/doc/results.dox b/examples/kokkos-assembly/doc/results.dox similarity index 100% rename from examples/kokkos_assembly/doc/results.dox rename to examples/kokkos-assembly/doc/results.dox diff --git a/examples/kokkos_assembly/doc/short-intro b/examples/kokkos-assembly/doc/short-intro similarity index 100% rename from examples/kokkos_assembly/doc/short-intro rename to examples/kokkos-assembly/doc/short-intro diff --git a/examples/kokkos_assembly/doc/tooltip b/examples/kokkos-assembly/doc/tooltip similarity index 100% rename from examples/kokkos_assembly/doc/tooltip rename to examples/kokkos-assembly/doc/tooltip diff --git a/examples/kokkos_assembly/kokkos_assembly.cpp b/examples/kokkos-assembly/kokkos-assembly.cpp similarity index 59% rename from examples/kokkos_assembly/kokkos_assembly.cpp rename to examples/kokkos-assembly/kokkos-assembly.cpp index 7cbfdf87189..d1c19d1b3e7 100644 --- a/examples/kokkos_assembly/kokkos_assembly.cpp +++ b/examples/kokkos-assembly/kokkos-assembly.cpp @@ -3,27 +3,90 @@ // SPDX-License-Identifier: BSD-3-Clause #include -#include -#include #include -#include +#include +#include #include -// enable compatibility with both kokkos 3.7 and 4.x -#if KOKKOS_VERSION / 10000 < 4 -namespace Kokkos { +namespace gko::ext::kokkos::detail { + + +/** + * Specialization of type mapper for gko::device_matrix_data. + * + * @tparam ValueType The value type of the matrix elements + * @tparam IndexType The index type of the matrix elements + * @tparam MemorySpace The Kokkos memory space to use. + */ +template +struct mapper, MemorySpace> { + using index_mapper = mapper, MemorySpace>; + using value_mapper = mapper, MemorySpace>; + + /** + * This struct defines the layout of the device_matrix_data type in terms + * of arrays. + * + * @tparam ValueType_c The value type of the matrix elements, might have + * other cv qualifiers than ValueType + * @tparam IndexType_c The index type of the matrix elements, might have + * other cv qualifiers than IndexType + */ + template + struct type { + using index_array = typename index_mapper::template type; + using value_array = typename value_mapper::template type; + + /** + * Constructor based on size and raw pointers + * + * @param size The number of stored elements + * @param row_idxs Pointer to the row indices + * @param col_idxs Pointer to the column indices + * @param values Pointer to the values + * + * @return An object which has each gko::array of the + * device_matrix_data mapped to a Kokkos view + */ + static type map(size_type size, IndexType_c* row_idxs, + IndexType_c* col_idxs, ValueType_c* values) + { + return {index_mapper::map(row_idxs, size), + index_mapper::map(col_idxs, size), + value_mapper::map(values, size)}; + } + index_array row_idxs; + index_array col_idxs; + value_array values; + }; + + static type map( + device_matrix_data& md) + { + assert_compatibility(md); + return type::map( + md.get_num_stored_elements(), md.get_row_idxs(), md.get_col_idxs(), + md.get_values()); + } -using Experimental::abs; + static type map( + const device_matrix_data& md) + { + assert_compatibility(md); + return type::map( + md.get_num_stored_elements(), md.get_const_row_idxs(), + md.get_const_col_idxs(), md.get_const_values()); + } +}; -} -#endif +} // namespace gko::ext::kokkos::detail // Creates a stencil matrix in CSR format for the given number of discretization @@ -41,12 +104,7 @@ void generate_stencil_matrix(gko::matrix::Csr* matrix) discretization_points * 3); // Create Kokkos views on Ginkgo data. - Kokkos::View v_row_idxs(md.get_row_idxs(), - md.get_num_stored_elements()); - Kokkos::View v_col_idxs(md.get_col_idxs(), - md.get_num_stored_elements()); - Kokkos::View v_values(md.get_values(), - md.get_num_stored_elements()); + auto k_md = gko::ext::kokkos::map_data(md); // Create the matrix entries. This also creates zero entries for the // first and second row to handle all rows uniformly. @@ -63,9 +121,9 @@ void generate_stencil_matrix(gko::matrix::Csr* matrix) auto mask = static_cast(0 <= col && col < discretization_points); - v_row_idxs[i] = mask * row; - v_col_idxs[i] = mask * col; - v_values[i] = mask * coefs[ofs + 1]; + k_md.row_idxs[i] = mask * row; + k_md.col_idxs[i] = mask * col; + k_md.values[i] = mask * coefs[ofs + 1]; }); // Add up duplicate (zero) entries. @@ -82,18 +140,17 @@ void generate_rhs(Closure&& f, ValueType u0, ValueType u1, gko::matrix::Dense* rhs) { const auto discretization_points = rhs->get_size()[0]; - auto values = rhs->get_values(); - Kokkos::View values_view(values, discretization_points); + auto k_rhs = gko::ext::kokkos::map_data(rhs); Kokkos::parallel_for( "generate_rhs", discretization_points, KOKKOS_LAMBDA(int i) { const ValueType h = 1.0 / (discretization_points + 1); const ValueType xi = ValueType(i + 1) * h; - values_view[i] = -f(xi) * h * h; + k_rhs(i, 0) = -f(xi) * h * h; if (i == 0) { - values_view[i] += u0; + k_rhs(i, 0) += u0; } if (i == discretization_points - 1) { - values_view[i] += u1; + k_rhs(i, 0) += u1; } }); } @@ -106,15 +163,14 @@ double calculate_error(int discretization_points, const gko::matrix::Dense* u, Closure&& correct_u) { - Kokkos::View v_u(u->get_const_values(), - discretization_points); + auto k_u = gko::ext::kokkos::map_data(u); auto error = 0.0; Kokkos::parallel_reduce( "calculate_error", discretization_points, KOKKOS_LAMBDA(int i, double& lsum) { const auto h = 1.0 / (discretization_points + 1); const auto xi = (i + 1) * h; - lsum += Kokkos::abs((v_u(i) - correct_u(xi)) / + lsum += Kokkos::abs((k_u(i, 0) - correct_u(xi)) / Kokkos::abs(correct_u(xi))); }, error); @@ -124,8 +180,6 @@ double calculate_error(int discretization_points, int main(int argc, char* argv[]) { - Kokkos::ScopeGuard kokkos(argc, argv); - // Some shortcuts using ValueType = double; using RealValueType = gko::remove_complex; @@ -141,39 +195,17 @@ int main(int argc, char* argv[]) if (argc == 2 && (std::string(argv[1]) == "--help")) { std::cerr << "Usage: " << argv[0] << " [discretization_points] [kokkos-options]" << std::endl; - std::exit(-1); + Kokkos::ScopeGuard kokkos(argc, argv); // print Kokkos help + std::exit(1); } - const unsigned int discretization_points = - argc >= 2 ? std::atoi(argv[1]) : 100u; + Kokkos::ScopeGuard kokkos(argc, argv); + + const auto discretization_points = + static_cast(argc >= 2 ? std::atoi(argv[1]) : 100u); // chooses the executor that corresponds to the Kokkos DefaultExecutionSpace - auto exec = []() -> std::shared_ptr { -#ifdef KOKKOS_ENABLE_SERIAL - if (std::is_same::value) { - return gko::ReferenceExecutor::create(); - } -#endif -#ifdef KOKKOS_ENABLE_OPENMP - if (std::is_same::value) { - return gko::OmpExecutor::create(); - } -#endif -#ifdef KOKKOS_ENABLE_CUDA - if (std::is_same::value) { - return gko::CudaExecutor::create(0, - gko::ReferenceExecutor::create()); - } -#endif -#ifdef KOKKOS_ENABLE_HIP - if (std::is_same::value) { - return gko::HipExecutor::create(0, - gko::ReferenceExecutor::create()); - } -#endif - }(); + auto exec = gko::ext::kokkos::create_default_executor(); // problem: auto correct_u = [] KOKKOS_FUNCTION(ValueType x) { return x * x * x; }; @@ -185,9 +217,7 @@ int main(int argc, char* argv[]) auto rhs = vec::create(exec, gko::dim<2>(discretization_points, 1)); generate_rhs(f, u0, u1, rhs.get()); auto u = vec::create(exec, gko::dim<2>(discretization_points, 1)); - for (int i = 0; i < u->get_size()[0]; ++i) { - u->get_values()[i] = 0.0; - } + u->fill(0.0); // initialize the stencil matrix auto A = share(mtx::create( diff --git a/extensions/CMakeLists.txt b/extensions/CMakeLists.txt new file mode 100644 index 00000000000..9c935a74270 --- /dev/null +++ b/extensions/CMakeLists.txt @@ -0,0 +1,3 @@ +if (GINKGO_BUILD_TESTS) + add_subdirectory(test) +endif () diff --git a/extensions/test/CMakeLists.txt b/extensions/test/CMakeLists.txt new file mode 100644 index 00000000000..a729661d909 --- /dev/null +++ b/extensions/test/CMakeLists.txt @@ -0,0 +1,6 @@ +include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake) + +find_package(Kokkos 4.1.00 QUIET) +if (Kokkos_FOUND) + add_subdirectory(kokkos) +endif () diff --git a/extensions/test/kokkos/CMakeLists.txt b/extensions/test/kokkos/CMakeLists.txt new file mode 100644 index 00000000000..47b5e24ca0c --- /dev/null +++ b/extensions/test/kokkos/CMakeLists.txt @@ -0,0 +1,49 @@ +# Kokkos doesn't handle any compiler launcher well, so it's disabled for these tests +set(_CMAKE_CXX_COMPILER_LAUNCHER ${CMAKE_CXX_COMPILER_LAUNCHER}) +unset(CMAKE_CXX_COMPILER_LAUNCHER) + +function(_ginkgo_check_build_config _build) + string(TOUPPER ${_build} _build) + if(NOT GINKGO_BUILD_${_build}) + message(FATAL_ERROR "Building test with ${_build} enabled for Kokkos, but not for Ginkgo.") + endif() + unset(_build) +endfunction() + +if(Kokkos_ENABLE_CUDA) + _ginkgo_check_build_config(cuda) + set(resource_type "cudagpu") + set(definitions "GKO_COMPILING_CUDA") +elseif(Kokkos_ENABLE_HIP) + _ginkgo_check_build_config(hip) + set(resource_type "hipgpu") + set(definitions "GKO_COMPILING_HIP") +elseif(Kokkos_ENABLE_SYCL) + _ginkgo_check_build_config(sycl) + set(resource_type "sycl") + set(definitions "GKO_COMPILING_DPCPP") +else() + set(resource_type "cpu") + if(Kokkos_ENABLE_OPENMP) + _ginkgo_check_build_config(omp) + set(definitions "GKO_COMPILING_OMP") + endif() +endif() + +function(create_gtest_main_kokkos) + add_library(ginkgo_gtest_main_kokkos STATIC kokkos_main.cpp ${PROJECT_SOURCE_DIR}/core/test/gtest/resources.cpp) + target_link_libraries(ginkgo_gtest_main_kokkos PUBLIC Ginkgo::ginkgo GTest::GTest Kokkos::kokkos) + target_compile_definitions(ginkgo_gtest_main_kokkos PRIVATE ${definitions}) + ginkgo_compile_features(ginkgo_gtest_main_kokkos) +endfunction() +create_gtest_main_kokkos() + +function(ginkgo_create_test_kokkos test_name) + ginkgo_create_test(${test_name} NO_GTEST_MAIN RESOURCE_TYPE ${resource_type} ADDITIONAL_LIBRARIES Kokkos::kokkos ginkgo_gtest_main_kokkos ${ARGN}) +endfunction() + +ginkgo_create_test_kokkos(types) +ginkgo_create_test_kokkos(spaces) + +# restore the compiler launcher variable +set(CMAKE_CXX_COMPILER_LAUNCHER ${_CMAKE_CXX_COMPILER_LAUNCHER} PARENT_SCOPE) diff --git a/extensions/test/kokkos/kokkos_main.cpp b/extensions/test/kokkos/kokkos_main.cpp new file mode 100644 index 00000000000..e541d362244 --- /dev/null +++ b/extensions/test/kokkos/kokkos_main.cpp @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + + +#include + + +#include "core/test/gtest/environments.hpp" + + +int get_device_id() +{ +#if defined(KOKKOS_ENABLE_CUDA) + return ResourceEnvironment::cuda_device_id; +#elif defined(KOKKOS_ENABLE_HIP) + return ResourceEnvironment::hip_device_id; +#elif defined(KOKKOS_ENABLE_SYCL) + return ResourceEnvironment::sycl_device_id; +#else + return 0; +#endif +} + + +class KokkosEnvironment : public ::testing::Environment { +public: + void SetUp() override + { + Kokkos::initialize( + Kokkos::InitializationSettings().set_device_id(get_device_id())); + } + + void TearDown() override { Kokkos::finalize(); } +}; + + +int ResourceEnvironment::omp_threads = 0; +int ResourceEnvironment::cuda_device_id = 0; +int ResourceEnvironment::hip_device_id = 0; +int ResourceEnvironment::sycl_device_id = 0; + + +int main(int argc, char** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + + ::testing::AddGlobalTestEnvironment(new ResourceEnvironment); + ::testing::AddGlobalTestEnvironment(new DeviceEnvironment(0)); + ::testing::AddGlobalTestEnvironment(new KokkosEnvironment); + int result = RUN_ALL_TESTS(); + return result; +} diff --git a/extensions/test/kokkos/spaces.cpp b/extensions/test/kokkos/spaces.cpp new file mode 100644 index 00000000000..47e24aac93e --- /dev/null +++ b/extensions/test/kokkos/spaces.cpp @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + + +#include + + +#include + + +#include "core/test/gtest/environments.hpp" +#include "core/test/utils.hpp" + + +TEST(Executor, CanCreateDefaultExecutor) +{ + auto exec = gko::ext::kokkos::create_default_executor(); + +#if defined(KOKKOS_ENABLE_CUDA) + ASSERT_TRUE(std::dynamic_pointer_cast(exec)); +#elif defined(KOKKOS_ENABLE_HIP) + ASSERT_TRUE(std::dynamic_pointer_cast(exec)); +#elif defined(KOKKOS_ENABLE_SYCL) + ASSERT_TRUE(std::dynamic_pointer_cast(exec)); +#elif defined(KOKKOS_ENABLE_OPENMP) + // necessary because of our executor hierarchy... + ASSERT_FALSE(std::dynamic_pointer_cast(exec)); + ASSERT_TRUE(std::dynamic_pointer_cast(exec)); +#elif defined(KOKKOS_ENABLE_SERIAL) + ASSERT_TRUE(std::dynamic_pointer_cast(exec)); +#else + ASSERT_TRUE(false); +#endif +} + + +TEST(Executor, CanCreateExecutorWithExecutorSpace) +{ +#ifdef KOKKOS_ENABLE_SERIAL + auto exec = gko::ext::kokkos::create_executor(Kokkos::Serial{}); + + ASSERT_TRUE(std::dynamic_pointer_cast(exec)); +#endif +} + + +TEST(Executor, ThrowsIfIncompatibleSpaces) +{ +#ifdef KOKKOS_ENABLE_SERIAL + auto exec = gko::ext::kokkos::create_executor(Kokkos::Serial{}); + auto obj = gko::array(exec); + +#ifdef KOKKOS_ENABLE_CUDA + ASSERT_THROW( + gko::ext::kokkos::detail::assert_compatibility(obj), + gko::InvalidStateError); +#elif KOKKOS_ENABLE_HIP + ASSERT_THROW( + gko::ext::kokkos::detail::assert_compatibility(obj), + gko::InvalidStateError); +#elif KOKKOS_ENABLE_SYCL + ASSERT_THROW(gko::ext::kokkos::detail::assert_compatibility< + Kokkos::Experimental::SYCL>(obj), + gko::InvalidStateError); +#endif +#endif +} + + +TEST(Executor, DoesntThrowIfCompatibleSpaces) +{ +#ifdef KOKKOS_ENABLE_SERIAL + auto exec = gko::ext::kokkos::create_executor(Kokkos::Serial{}); + auto obj = gko::array(exec); + + ASSERT_NO_THROW( + gko::ext::kokkos::detail::assert_compatibility(obj)); +#endif +} + + +void shared_memory_access() +{ + auto exec = gko::ext::kokkos::create_executor( + Kokkos::DefaultExecutionSpace{}, Kokkos::SharedSpace{}); + auto data = exec->alloc(1); + + // If `create_executor` would not use the UVM allocators, than this + // would crash the program. Otherwise, the device memory is accessible + // on the CPU without issues. + *data = 10; + + exec->free(data); + std::exit(EXIT_SUCCESS); +} + + +TEST(Executor, CanCreateExecutorWithMemorySpace) +{ + GTEST_FLAG_SET(death_test_style, "threadsafe"); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + EXPECT_EXIT(shared_memory_access(), testing::ExitedWithCode(EXIT_SUCCESS), + ""); +#endif +} diff --git a/extensions/test/kokkos/types.cpp b/extensions/test/kokkos/types.cpp new file mode 100644 index 00000000000..4bff41499e9 --- /dev/null +++ b/extensions/test/kokkos/types.cpp @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include + + +#include + + +#include + + +#include +#include +#include + + +#include "core/test/utils.hpp" + + +using DefaultMemorySpace = Kokkos::DefaultExecutionSpace::memory_space; + + +template +class ArrayMapper : public ::testing::Test { +protected: + using value_type = ValueType; + + std::shared_ptr exec = + gko::ext::kokkos::create_default_executor(); + gko::array array = {exec, I{1, 2, 3, 4}}; +}; + +TYPED_TEST_SUITE(ArrayMapper, gko::test::ValueTypes, TypenameNameGenerator); + + +TYPED_TEST(ArrayMapper, CanMapDefault) +{ + using value_type = typename TestFixture::value_type; + using kokkos_value_type = + typename gko::ext::kokkos::detail::value_type::type; + + auto mapped_array = gko::ext::kokkos::map_data(this->array); + + using mapped_type = + std::remove_cv_t>; + static_assert( + std::is_same_v>>); + ASSERT_EQ(reinterpret_cast(mapped_array.data()), + reinterpret_cast(this->array.get_data())); + ASSERT_EQ(mapped_array.extent(0), this->array.get_size()); + ASSERT_EQ(mapped_array.size(), this->array.get_size()); + ASSERT_EQ(mapped_array.stride(0), 1); +} + + +TYPED_TEST(ArrayMapper, CanMapConst) +{ + using value_type = typename TestFixture::value_type; + using kokkos_value_type = + typename gko::ext::kokkos::detail::value_type::type; + + auto mapped_array = gko::ext::kokkos::map_data( + const_cast&>(this->array)); + + using mapped_type = + std::remove_cv_t>; + static_assert(std::is_same_v< + mapped_type, + Kokkos::View>>); + ASSERT_EQ(reinterpret_cast(mapped_array.data()), + reinterpret_cast(this->array.get_data())); + ASSERT_EQ(mapped_array.extent(0), this->array.get_size()); + ASSERT_EQ(mapped_array.size(), this->array.get_size()); + ASSERT_EQ(mapped_array.stride(0), 1); +} + + +template +class DenseMapper : public ::testing::Test { +protected: + using value_type = ValueType; + using mtx_type = gko::matrix::Dense; + + std::shared_ptr exec = + gko::ext::kokkos::create_default_executor(); + std::unique_ptr mtx = + gko::initialize({1, 2, 3, 4}, exec); +}; + +TYPED_TEST_SUITE(DenseMapper, gko::test::ValueTypes, TypenameNameGenerator); + + +TYPED_TEST(DenseMapper, CanMapDefault) +{ + using value_type = typename TestFixture::value_type; + using kokkos_value_type = + typename gko::ext::kokkos::detail::value_type::type; + + auto mapped_mtx = gko::ext::kokkos::map_data(this->mtx); + + using mapped_type = + std::remove_cv_t>; + static_assert( + std::is_same_v>>); + ASSERT_EQ(reinterpret_cast(mapped_mtx.data()), + reinterpret_cast(this->mtx->get_values())); + ASSERT_EQ(mapped_mtx.extent(0), this->mtx->get_size()[0]); + ASSERT_EQ(mapped_mtx.extent(1), this->mtx->get_size()[1]); + ASSERT_EQ(mapped_mtx.size(), + this->mtx->get_size()[0] * this->mtx->get_size()[1]); + ASSERT_EQ(mapped_mtx.span(), this->mtx->get_num_stored_elements()); + ASSERT_EQ(mapped_mtx.stride(0), this->mtx->get_stride()); + ASSERT_EQ(mapped_mtx.stride(1), 1); +} + + +TYPED_TEST(DenseMapper, CanMapConst) +{ + using value_type = typename TestFixture::value_type; + using kokkos_value_type = + typename gko::ext::kokkos::detail::value_type::type; + + auto mapped_mtx = gko::ext::kokkos::map_data( + const_cast*>(this->mtx.get())); + + using mapped_type = + std::remove_cv_t>; + static_assert( + std::is_same_v>>); + ASSERT_EQ(reinterpret_cast(mapped_mtx.data()), + reinterpret_cast(this->mtx->get_values())); + ASSERT_EQ(mapped_mtx.extent(0), this->mtx->get_size()[0]); + ASSERT_EQ(mapped_mtx.extent(1), this->mtx->get_size()[1]); + ASSERT_EQ(mapped_mtx.size(), + this->mtx->get_size()[0] * this->mtx->get_size()[1]); + ASSERT_EQ(mapped_mtx.span(), this->mtx->get_num_stored_elements()); + ASSERT_EQ(mapped_mtx.stride(0), this->mtx->get_stride()); + ASSERT_EQ(mapped_mtx.stride(1), 1); +} + + +TYPED_TEST(DenseMapper, CanMapStrided) +{ + using mtx_type = typename TestFixture::mtx_type; + using value_type = typename TestFixture::value_type; + using kokkos_value_type = + typename gko::ext::kokkos::detail::value_type::type; + std::unique_ptr mtx = mtx_type ::create( + this->exec, gko::dim<2>{2, 2}, + gko::array{this->exec, {1, 2, -1, 3, 4, -10}}, 3); + + auto mapped_mtx = gko::ext::kokkos::map_data(mtx); + + using mapped_type = + std::remove_cv_t>; + static_assert( + std::is_same_v>>); + ASSERT_EQ(reinterpret_cast(mapped_mtx.data()), + reinterpret_cast(mtx->get_values())); + ASSERT_EQ(mapped_mtx.extent(0), mtx->get_size()[0]); + ASSERT_EQ(mapped_mtx.extent(1), mtx->get_size()[1]); + ASSERT_EQ(mapped_mtx.size(), mtx->get_size()[0] * mtx->get_size()[1]); + ASSERT_EQ(mapped_mtx.span(), mtx->get_num_stored_elements()); + ASSERT_EQ(mapped_mtx.stride(0), mtx->get_stride()); + ASSERT_EQ(mapped_mtx.stride(1), 1); +} diff --git a/include/ginkgo/extensions/kokkos.hpp b/include/ginkgo/extensions/kokkos.hpp new file mode 100644 index 00000000000..d3f1c46dcf2 --- /dev/null +++ b/include/ginkgo/extensions/kokkos.hpp @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#ifndef GKO_PUBLIC_EXTENSIONS_KOKKOS_HPP_ +#define GKO_PUBLIC_EXTENSIONS_KOKKOS_HPP_ + + +#include +#include +#include + + +#endif // GKO_PUBLIC_EXTENSIONS_KOKKOS_HPP_ diff --git a/include/ginkgo/extensions/kokkos/config.hpp.in b/include/ginkgo/extensions/kokkos/config.hpp.in new file mode 100644 index 00000000000..9e635ef76bd --- /dev/null +++ b/include/ginkgo/extensions/kokkos/config.hpp.in @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2024 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#ifndef GINKGO_INCLUDE_GINKGO_EXTENSIONS_KOKKOS_CONFIG_HPP_IN +#define GINKGO_INCLUDE_GINKGO_EXTENSIONS_KOKKOS_CONFIG_HPP_IN + + +/* Do we need to check the type alignment in the Kokkos type maps? */ +// clang-format off +#cmakedefine01 GINKGO_EXTENSION_KOKKOS_CHECK_TYPE_ALIGNMENT +// clang-format on + + +#endif // GINKGO_INCLUDE_GINKGO_EXTENSIONS_KOKKOS_CONFIG_HPP_IN diff --git a/include/ginkgo/extensions/kokkos/spaces.hpp b/include/ginkgo/extensions/kokkos/spaces.hpp new file mode 100644 index 00000000000..6875f931152 --- /dev/null +++ b/include/ginkgo/extensions/kokkos/spaces.hpp @@ -0,0 +1,280 @@ +// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#ifndef GINKGO_EXTENSIONS_KOKKOS_SPACES_HPP +#define GINKGO_EXTENSIONS_KOKKOS_SPACES_HPP + +#include + + +#include +#include +#include +#include + + +namespace gko { +namespace ext { +namespace kokkos { +namespace detail { + + +/** + * Helper to check if an executor type can access the memory of an memory space + * + * @tparam MemorySpace Type fulfilling the Kokkos MemorySpace concept. + * @tparam ExecType One of the Ginkgo executor types. + */ +template +struct compatible_space + : std::integral_constant {}; + +template <> +struct compatible_space : std::true_type { +}; + +template +struct compatible_space { + // need manual implementation of std::integral_constant because, + // while compiling for cuda, somehow bool is replaced by __nv_bool + static constexpr bool value = + Kokkos::SpaceAccessibility::accessible; +}; + +#ifdef KOKKOS_ENABLE_OPENMP +template +struct compatible_space + : compatible_space {}; +#endif + +#ifdef KOKKOS_ENABLE_CUDA +template +struct compatible_space { + static constexpr bool value = + Kokkos::SpaceAccessibility::accessible; +}; +#endif + +#ifdef KOKKOS_ENABLE_HIP +template +struct compatible_space { + static constexpr bool value = + Kokkos::SpaceAccessibility::accessible; +}; +#endif + +#ifdef KOKKOS_ENABLE_SYCL +template +struct compatible_space { + static constexpr bool value = + Kokkos::SpaceAccessibility::accessible; +}; +#endif + + +/** + * Checks if the memory space is accessible by the executor + * + * @tparam MemorySpace A Kokkos memory space type + * @tparam ExecType A Ginkgo executor type + * + * @return true if the memory space is accessible by the executor + */ +template +inline bool check_compatibility(std::shared_ptr) +{ + return compatible_space::value; +} + + +/** + * Checks if the memory space is accessible by the executor + * + * @tparam MemorySpace A Kokkos memory space + * @param exec The executor which should access the memory space + * + * @return true if the memory space is accessible by the executor + */ + +template +inline bool check_compatibility(std::shared_ptr exec) +{ + if (auto p = std::dynamic_pointer_cast(exec)) { + return check_compatibility(p); + } + if (auto p = std::dynamic_pointer_cast(exec)) { + return check_compatibility(p); + } + if (auto p = std::dynamic_pointer_cast(exec)) { + return check_compatibility(p); + } + if (auto p = std::dynamic_pointer_cast(exec)) { + return check_compatibility(p); + } + if (auto p = std::dynamic_pointer_cast(exec)) { + return check_compatibility(p); + } + GKO_NOT_IMPLEMENTED; +} + + +/** + * Throws if the memory space is ~not~ accessible by the executor associated + * with the passed in Ginkgo object. + * + * @tparam MemorySpace A Kokkos memory space type. + * @tparam T A Ginkgo type that has the member function `get_executor`. + * + * @param obj Object which executor is used to check the access to the memory + * space. + */ +template +inline void assert_compatibility(T&& obj) +{ + GKO_THROW_IF_INVALID(check_compatibility(obj.get_executor()), + "Executor type and memory space are incompatible"); +} + + +} // namespace detail + + +/** + * Creates an Executor matching the Kokkos::DefaultHostExecutionSpace. + * + * If no kokkos host execution space is enabled, this throws an exception. + * + * @return An executor of type either ReferenceExecutor or OmpExecutor. + */ +inline std::shared_ptr create_default_host_executor() +{ +#ifdef KOKKOS_ENABLE_SERIAL + if constexpr (std::is_same_v) { + return ReferenceExecutor::create(); + } +#endif +#ifdef KOKKOS_ENABLE_OPENMP + if constexpr (std::is_same_v) { + return OmpExecutor::create(); + } +#endif + GKO_NOT_IMPLEMENTED; +} + + +/** + * Creates an Executor for a specific Kokkos ExecutionSpace. + * + * This function supports the following Kokkos ExecutionSpaces: + * - Serial + * - OpenMP + * - Cuda + * - HIP + * - Experimental::SYCL + * If none of these spaces are enabled, then this function throws an exception. + * For Cuda, HIP, SYCL, the device-id used by Kokkos is passed to the Executor + * constructor. + * + * @tparam ExecSpace A supported Kokkos ExecutionSpace. + * @tparam MemorySpace A supported Kokkos MemorySpace. Defaults to the one + * defined in the ExecSpace. + * + * @param ex the execution space + * + * @return An executor matching the type of the ExecSpace. + */ +template +inline std::shared_ptr create_executor(ExecSpace ex, MemorySpace = {}) +{ + static_assert( + Kokkos::SpaceAccessibility::accessible); +#ifdef KOKKOS_ENABLE_SERIAL + if constexpr (std::is_same_v) { + return ReferenceExecutor::create(); + } +#endif +#ifdef KOKKOS_ENABLE_OPENMP + if constexpr (std::is_same_v) { + return OmpExecutor::create(); + } +#endif +#ifdef KOKKOS_ENABLE_CUDA + if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { + return CudaExecutor::create( + Kokkos::device_id(), create_default_host_executor(), + std::make_shared(), ex.cuda_stream()); + } + if constexpr (std::is_same_v) { + return CudaExecutor::create( + Kokkos::device_id(), create_default_host_executor(), + std::make_shared(Kokkos::device_id()), + ex.cuda_stream()); + } + if constexpr (std::is_same_v) { + return CudaExecutor::create( + Kokkos::device_id(), create_default_host_executor(), + std::make_shared(Kokkos::device_id()), + ex.cuda_stream()); + } + } +#endif +#ifdef KOKKOS_ENABLE_HIP + if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { + return HipExecutor::create( + Kokkos::device_id(), create_default_host_executor(), + std::make_shared(), ex.hip_stream()); + } + if constexpr (std::is_same_v) { + return HipExecutor::create( + Kokkos::device_id(), create_default_host_executor(), + std::make_shared(Kokkos::device_id()), + ex.hip_stream()); + } + if constexpr (std::is_same_v) { + return HipExecutor::create( + Kokkos::device_id(), create_default_host_executor(), + std::make_shared(Kokkos::device_id()), + ex.hip_stream()); + } + } +#endif +#ifdef KOKKOS_ENABLE_SYCL + if constexpr (std::is_same_v) { + static_assert( + std::is_same_v, + "Ginkgo doesn't support shared memory space allocation for SYCL"); + return DpcppExecutor::create(Kokkos::device_id(), + create_default_host_executor()); + } +#endif + GKO_NOT_IMPLEMENTED; +} + + +/** + * Creates an Executor matching the Kokkos::DefaultExecutionSpace. + * + * @return An executor matching the type of Kokkos::DefaultExecutionSpace. + */ +inline std::shared_ptr create_default_executor( + Kokkos::DefaultExecutionSpace ex = {}) +{ + return create_executor(std::move(ex)); +} + + +} // namespace kokkos +} // namespace ext +} // namespace gko + + +#endif // GINKGO_EXTENSIONS_KOKKOS_SPACES_HPP diff --git a/include/ginkgo/extensions/kokkos/types.hpp b/include/ginkgo/extensions/kokkos/types.hpp new file mode 100644 index 00000000000..88362f317b1 --- /dev/null +++ b/include/ginkgo/extensions/kokkos/types.hpp @@ -0,0 +1,286 @@ +// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#ifndef GINKGO_EXTENSIONS_KOKKOS_TYPES_HPP +#define GINKGO_EXTENSIONS_KOKKOS_TYPES_HPP + +#include + + +#include + + +#include +#include +#include +#include + + +namespace gko { +namespace ext { +namespace kokkos { +namespace detail { + + +/** + * Maps arithmetic types to corresponding Kokkos types. + * + * @tparam T An arithmetic type. + */ +template +struct value_type_impl { + using type = T; +}; + +template +struct value_type_impl { + using type = const typename value_type_impl::type; +}; + +template +struct value_type_impl> { + using type = Kokkos::complex; +}; + + +template +struct value_type { + using type = typename value_type_impl::type; + + static_assert(sizeof(std::decay_t) == sizeof(std::decay_t), + "Can't handle C++ data type and corresponding Kokkos " + "type with mismatching type sizes."); +#if GINKGO_EXTENSION_KOKKOS_CHECK_TYPE_ALIGNMENT + static_assert( + alignof(std::decay_t) == alignof(std::decay_t), + "Can't handle C++ data type and corresponding Kokkos type with " + "mismatching alignments. If std::complex is used, please make sure " + "to configure Kokkos with `KOKKOS_ENABLE_COMPLEX_ALIGN=ON`.\n" + "Alternatively, disable this check by setting the CMake option " + "-DGINKGO_EXTENSION_KOKKOS_CHECK_TYPE_ALIGNMENT=OFF."); +#endif +}; + +template +using value_type_t = typename value_type::type; + + +/** + * Maps ginkgo types to the corresponding kokkos type + * + * @tparam T A ginkgo type to map + * @tparam MemorySpace A kokkos MemorySpace which the resulting type will use + * + * @warning The MemorySpace has to match the internally used executor of T. + * + * @note This only describes the interface of the mapper class. The actual + * implementation is done via specialization of this class. + */ +template +struct mapper { + static auto map(T&); + + static auto map(const T&); +}; + + +/** + * Type that maps a Ginkgo array to an unmanaged 1D Kokkos::View. + * + * @warning Using std::complex as data type might lead to issues, since the + * alignment of Kokkos::complex is not necessarily the same. + * + * @tparam MemorySpace The memory space type the mapped object should use. + */ +template +struct mapper, MemorySpace> { + template + using type = + Kokkos::View::type*, MemorySpace, + Kokkos::MemoryTraits>; + + /** + * Constructor based on size and raw pointer + * + * @tparam ValueType_c The same type as ValueType, but possible + * const-qualified + * @param data The pointer to the data + * @param size The size of the array + * @return A Kokkos 1D-view to the data + */ + template + static type map(ValueType_c* data, size_type size); + + /** + * Maps a mutable array. + */ + static type map(array& arr); + + /** + * Maps a const array. + */ + static type map(const array& arr); + + /** + * Maps a const_array_view (same as mapping a const array). + */ + static type map( + const ::gko::detail::const_array_view& arr); +}; + +template +template +typename mapper, MemorySpace>::template type +mapper, MemorySpace>::map(ValueType_c* data, size_type size) +{ + return type{reinterpret_cast*>(data), + size}; +} + + +template +typename mapper, MemorySpace>::template type +mapper, MemorySpace>::map(array& arr) +{ + assert_compatibility(arr); + + return map(arr.get_data(), arr.get_size()); +} + + +template +typename mapper, MemorySpace>::template type +mapper, MemorySpace>::map(const array& arr) +{ + assert_compatibility(arr); + + return map(arr.get_const_data(), arr.get_size()); +} + + +template +typename mapper, MemorySpace>::template type +mapper, MemorySpace>::map( + const ::gko::detail::const_array_view& arr) +{ + assert_compatibility(arr); + + return map(arr.get_const_data(), arr.get_size()); +} + + +/** + * Type that maps a Ginkgo matrix::Dense to an unmanaged 2D Kokkos::View. + * + * @warning Using std::complex as data type might lead to issues, since the + * alignment of Kokkos::complex is not necessarily the same. + * + * @tparam MemorySpace The memory space type the mapped object should use. + */ +template +struct mapper, MemorySpace> { + template + using type = Kokkos::View::type**, + Kokkos::LayoutStride, MemorySpace, + Kokkos::MemoryTraits>; + + /** + * Maps a mutable dense matrix. + */ + static type map(matrix::Dense& m); + + /** + * Maps a const dense matrix. + */ + static type map(const matrix::Dense& m); +}; + +template +typename mapper, MemorySpace>::template type +mapper, MemorySpace>::map(matrix::Dense& m) +{ + assert_compatibility(m); + + auto size = m.get_size(); + + return type{ + reinterpret_cast*>(m.get_values()), + Kokkos::LayoutStride{size[0], m.get_stride(), size[1], 1}}; +} + + +template +typename mapper, + MemorySpace>::template type +mapper, MemorySpace>::map( + const matrix::Dense& m) +{ + assert_compatibility(m); + + auto size = m.get_size(); + + return type{ + reinterpret_cast*>(m.get_const_values()), + Kokkos::LayoutStride{size[0], m.get_stride(), size[1], 1}}; +} + + +} // namespace detail + + +/** + * Maps Ginkgo object to a type compatible with Kokkos. + * + * @tparam T The Ginkgo type. + * @tparam MemorySpace The Kokkos memory space that will be used + * + * @param data The Ginkgo object. + * + * @return A wrapper for the Ginkgo object that is compatible with Kokkos + */ +template +inline auto map_data(T&& data) +{ + return detail::mapper, MemorySpace>::map( + std::forward(data)); +} + +/** + * @copydoc map_data(T&&, MemorySpace) + */ +template +inline auto map_data(T* data) +{ + return map_data(*data); +} + +/** + * @copydoc map_data(T&&, MemorySpace) + */ +template +inline auto map_data(std::unique_ptr& data) +{ + return map_data(*data); +} + +/** + * @copydoc map_data(T&&, MemorySpace) + */ +template +inline auto map_data(std::shared_ptr& data) +{ + return map_data(*data); +} + + +} // namespace kokkos +} // namespace ext +} // namespace gko + + +#endif // GINKGO_EXTENSIONS_KOKKOS_TYPES_HPP