Merge Kokkos Mapping

This merge adds a mapping to Kokkos types and functions to choose the Ginkgo executor based on the Kokkos execution space.

Currently, the mapping only supports `array` `matrix::Dense`.
Other mappings can easily be defined based on those. An example of that is `device_matrix_data` which is handled in the kokkos-assembly example.

CMakeLists.txt

@@ -66,6 +66,7 @@ option(GINKGO_FORCE_GPU_AWARE_MPI "Assert that the MPI library is GPU aware. Thi
      catastrophically in case the MPI implementation is not GPU Aware, and GPU aware functionality has been forced" OFF)
 set(GINKGO_CI_TEST_OMP_PARALLELISM "4" CACHE STRING
     "The number of OpenMP threads to use for a test binary during CTest resource file-constrained test.")
+option(GINKGO_EXTENSION_KOKKOS_CHECK_TYPE_ALIGNMENT "Enables mapping to Kokkos types to check the alignment of the source and target type." ON)
 gko_rename_cache(GINKGO_COMPILER_FLAGS CMAKE_CXX_FLAGS BOOL "Flags used by the CXX compiler during all build types.")
 gko_rename_cache(GINKGO_CUDA_COMPILER_FLAGS CMAKE_CUDA_FLAGS BOOL "Flags used by the CUDA compiler during all build types.")
 
@@ -280,6 +281,10 @@ else()
 endif()
 configure_file(${Ginkgo_SOURCE_DIR}/include/ginkgo/config.hpp.in
     ${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp @ONLY)
+configure_file(${Ginkgo_SOURCE_DIR}/include/ginkgo/extensions/kokkos/config.hpp.in
+        ${Ginkgo_BINARY_DIR}/include/ginkgo/extensions/kokkos/config.hpp
+        @ONLY
+)
 
 # Ginkgo core libraries
 # Needs to be first in order for `CMAKE_CUDA_DEVICE_LINK_EXECUTABLE` to be
@@ -308,6 +313,8 @@ if(GINKGO_BUILD_TESTS)
 endif()
 
 # Non core directories and targets
+add_subdirectory(extensions)
+
 if(GINKGO_BUILD_EXAMPLES)
     add_subdirectory(examples)
 endif()

cmake/build_helpers.cmake

@@ -56,6 +56,7 @@ function(ginkgo_check_headers target defines)
     list(FILTER CXX_HEADERS EXCLUDE REGEX ".*\.hip\.hpp$")
     list(FILTER CXX_HEADERS EXCLUDE REGEX "^test.*")
     list(FILTER CXX_HEADERS EXCLUDE REGEX "^base/kernel_launch.*")
+    list(FILTER CXX_HEADERS EXCLUDE REGEX "^ginkgo/extensions/.*")
     list(FILTER CUDA_HEADERS EXCLUDE REGEX "^test.*")
     list(FILTER CUDA_HEADERS EXCLUDE REGEX "^base/kernel_launch.*")
     list(FILTER HIP_HEADERS EXCLUDE REGEX "^test.*")

cmake/create_test.cmake

@@ -1,7 +1,7 @@
 set(gko_test_resource_args "RESOURCE_LOCAL_CORES;RESOURCE_TYPE")
 set(gko_test_single_args "MPI_SIZE;EXECUTABLE_NAME;${gko_test_resource_args}")
 set(gko_test_multi_args "DISABLE_EXECUTORS;ADDITIONAL_LIBRARIES;ADDITIONAL_INCLUDES")
-set(gko_test_option_args "NO_RESOURCES")
+set(gko_test_option_args "NO_RESOURCES;NO_GTEST_MAIN")
 
 ## Replaces / by _ to create valid target names from relative paths
 function(ginkgo_build_test_name test_name target_name)
@@ -14,7 +14,7 @@ endfunction()
 ## Set up shared target properties and handle ADDITIONAL_LIBRARIES/ADDITIONAL_INCLUDES
 ## `MPI_SIZE size` causes the tests to be run with `size` MPI processes.
 function(ginkgo_set_test_target_properties test_target_name test_library_suffix)
-    cmake_parse_arguments(PARSE_ARGV 1 set_properties "" "${gko_test_single_args}" "${gko_test_multi_args}")
+    cmake_parse_arguments(PARSE_ARGV 1 set_properties "${gko_test_option_args}" "${gko_test_single_args}" "${gko_test_multi_args}")
     if (GINKGO_FAST_TESTS)
         target_compile_definitions(${test_target_name} PRIVATE GINKGO_FAST_TESTS)
     endif()
@@ -27,10 +27,12 @@ function(ginkgo_set_test_target_properties test_target_name test_library_suffix)
     if(GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${test_target_name} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
-    if(set_properties_MPI_SIZE)
-        target_link_libraries(${test_target_name} PRIVATE ginkgo_gtest_main_mpi${test_library_suffix})
-    else()
-        target_link_libraries(${test_target_name} PRIVATE ginkgo_gtest_main${test_library_suffix})
+    if(NOT set_properties_NO_GTEST_MAIN)
+        if(set_properties_MPI_SIZE)
+            target_link_libraries(${test_target_name} PRIVATE ginkgo_gtest_main_mpi${test_library_suffix})
+        else()
+            target_link_libraries(${test_target_name} PRIVATE ginkgo_gtest_main${test_library_suffix})
+        endif()
     endif()
     target_compile_features(${test_target_name} PUBLIC cxx_std_14)
     # we set these properties regardless of the enabled backends,

cmake/get_info.cmake

@@ -54,15 +54,15 @@ FUNCTION(ginkgo_print_flags log_type var_name)
         set(str_value "${${var_string}}")
     endif()
     string(SUBSTRING "
---        ${var_string}:                                                        " 0 55 upd_string)
+--        ${var_string}:                                                        " 0 60 upd_string)
     string(APPEND upd_string "${str_value}")
     FILE(APPEND ${log_type} ${upd_string})
 ENDFUNCTION()
 
 function(ginkgo_print_variable log_type var_name)
     string(SUBSTRING
         "
---        ${var_name}:                                                          " 0 55 upd_string)
+--        ${var_name}:                                                          " 0 60 upd_string)
     if(${var_name} STREQUAL "")
         set(str_value "<empty>")
     else()
@@ -76,7 +76,7 @@ endfunction()
 function(ginkgo_print_env_variable log_type var_name)
     string(SUBSTRING
         "
---        ${var_name}:                                                          " 0 55 upd_string)
+--        ${var_name}:                                                          " 0 60 upd_string)
     if(DEFINED ENV{${var_name}})
         set(str_value "$ENV{${var_name}}")
     else()
@@ -114,13 +114,13 @@ foreach(log_type ${log_types})
         "PROJECT_SOURCE_DIR;PROJECT_BINARY_DIR")
     string(SUBSTRING
         "
---        CMAKE_CXX_COMPILER:                                                   " 0 55 print_string)
+--        CMAKE_CXX_COMPILER:                                                   " 0 60 print_string)
     set(str2 "${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} on platform ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR}")
     string(APPEND print_string "${str2}")
     FILE(APPEND ${${log_type}} "${print_string}")
     string(SUBSTRING
         "
---                                                                              " 0 55 print_string)
+--                                                                              " 0 60 print_string)
     set(str2 "${CMAKE_CXX_COMPILER}")
     string(APPEND print_string "${str2}")
     FILE(APPEND ${${log_type}} "${print_string}")
@@ -197,14 +197,17 @@ if(TARGET PAPI::PAPI)
     ginkgo_print_variable(${detailed_log} "PAPI_INCLUDE_DIR")
     ginkgo_print_flags(${detailed_log} "PAPI_LIBRARY")
 endif()
-
 ginkgo_print_variable(${minimal_log} "GINKGO_BUILD_HWLOC")
 ginkgo_print_variable(${detailed_log} "GINKGO_BUILD_HWLOC")
 if(TARGET hwloc)
     ginkgo_print_variable(${detailed_log} "HWLOC_VERSION")
     ginkgo_print_variable(${detailed_log} "HWLOC_LIBRARIES")
     ginkgo_print_variable(${detailed_log} "HWLOC_INCLUDE_DIRS")
 endif()
+ginkgo_print_module_footer(${detailed_log} "")
+
+ginkgo_print_generic_header(${detailed_log} "  Extensions:")
+ginkgo_print_variable(${detailed_log} "GINKGO_EXTENSION_KOKKOS_CHECK_TYPE_ALIGNMENT")
 
 _minimal(
     "

cmake/install_helpers.cmake

@@ -79,14 +79,15 @@ function(ginkgo_install)
 
     # install the public header files
     install(DIRECTORY "${Ginkgo_SOURCE_DIR}/include/"
-        DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-        COMPONENT Ginkgo_Development
-        FILES_MATCHING PATTERN "*.hpp"
-        )
-    install(FILES "${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp"
-        DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/ginkgo"
-        COMPONENT Ginkgo_Development
-        )
+            DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
+            COMPONENT Ginkgo_Development
+            FILES_MATCHING PATTERN "*.hpp"
+    )
+    install(DIRECTORY "${Ginkgo_BINARY_DIR}/include/"
+            DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
+            COMPONENT Ginkgo_Development
+            FILES_MATCHING PATTERN "*.hpp"
+    )
 
     if  (GINKGO_HAVE_HWLOC AND NOT HWLOC_FOUND)
         get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)

dev_tools/scripts/regroup

@@ -2,7 +2,7 @@ IncludeBlocks: Regroup
 IncludeCategories:
   - Regex: '^<(nlohmann|gflags|gtest|papi).*'
     Priority: 3
-  - Regex: '^<(omp|cu|hip|thrust|CL/|cooperative|oneapi|mpi|nvToolsExt).*'
+  - Regex: '^<(omp|cu|hip|thrust|CL/|cooperative|oneapi|mpi|nvToolsExt|Kokkos_Core).*'
     Priority: 2
   - Regex: '^<ginkgo.*'
     Priority: 5

dev_tools/scripts/update_ginkgo_header.sh

@@ -42,6 +42,7 @@ fi
 # Put all header files as a list (separated by newlines) in the file ${HEADER_LIST}
 # Requires detected files (including the path) to not contain newlines
 if ! find "${TOP_HEADER_FOLDER}" -name '*.hpp' -type f -print | \
+        grep -v 'ginkgo/extensions/' | \
         grep -v 'residual_norm_reduction.hpp' | \
         grep -v 'solver/.*_trs.hpp' > "${HEADER_LIST}"; then
     echo "${WARNING_PREFIX} "'The `find` command returned with an error!' 1>&2

examples/CMakeLists.txt

@@ -56,15 +56,9 @@ if(GINKGO_BUILD_MPI)
     list(APPEND EXAMPLES_LIST distributed-solver)
 endif()
 
-find_package(Kokkos QUIET)
+find_package(Kokkos 4.1.00 QUIET)
 if(Kokkos_FOUND)
-    if(GINKGO_WITH_CCACHE)
-        message(WARNING "The CMAKE_CXX_COMPILER_LAUNCHER is set due to "
-            "GINKGO_WITH_CCACHE=ON which is known to cause issues with CUDA enabled "
-            "Kokkos (https://github.com/kokkos/kokkos/issues/4821) including compilation "
-            "failures. This can be prevented by setting GINKGO_WITH_CCACHE=OFF.")
-    endif()
-    list(APPEND EXAMPLES_LIST kokkos_assembly)
+    list(APPEND EXAMPLES_LIST kokkos-assembly)
 else()
     message(STATUS "No Kokkos found, disabling examples with Kokkos assembly.")
 endif()

examples/kokkos_assembly/CMakeLists.txt → examples/kokkos-assembly/CMakeLists.txt

@@ -5,8 +5,10 @@ project(kokkos-assembly CXX)
 if(NOT GINKGO_BUILD_EXAMPLES)
     find_package(Ginkgo 1.8.0 REQUIRED)
 endif()
+find_package(Kokkos 4.1.00 REQUIRED)
 
-find_package(Kokkos 3.7 REQUIRED)
+# Kokkos doesn't handle any compiler launcher well, so it's disable it
+unset(CMAKE_CXX_COMPILER_LAUNCHER)
 
-add_executable(kokkos-assembly kokkos_assembly.cpp)
+add_executable(kokkos-assembly kokkos-assembly.cpp)
 target_link_libraries(kokkos-assembly Ginkgo::ginkgo Kokkos::kokkos)