Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Load rocSOLVER with dlopen at runtime #903

Open
wants to merge 11 commits into
base: develop
Choose a base branch
from
+1,411 −64
Open

Load rocSOLVER with dlopen at runtime #903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
97d5a66
Using BUIILD_SOLVER_TESTS cmake variable in prep of runtime solver
daineAMD Aug 7, 2024
c0062ef
Partial changes.
daineAMD Aug 13, 2024
04b0448
Merge branch 'runtimesolver' of github.com:daineamd/hipblas into runt…
daineAMD Aug 13, 2024
b83fcbd
working rocsolver using dlopen.
daineAMD Aug 16, 2024
3aa4527
Merge branch 'develop' of github.com:rocmsoftwareplatform/hipblas int…
daineAMD Aug 16, 2024
88730fe
Fix rocsolver at build-time flag.
daineAMD Aug 17, 2024
23bd75b
Update documentation.
daineAMD Aug 19, 2024
89efef2
add solverBuildtime flag to jenkins.
daineAMD Aug 19, 2024
c252edc
Fix rocsolver geqrf symbols
daineAMD Aug 20, 2024
f4e88db
Fix static build; fix g++ build; fix lib filenames.
daineAMD Aug 20, 2024
0e28557
Find rocsolver path in rpath
daineAMD Aug 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .jenkins/precheckin.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def runCI =

def prj = new rocProject('hipBLAS', 'PreCheckin')

if (env.BRANCH_NAME ==~ /PR-\d+/ && pullRequest.labels.contains("noSolver"))
if (env.BRANCH_NAME ==~ /PR-\d+/ && pullRequest.labels.contains("noSolverClients"))
{
prj.libraryDependencies = ['rocBLAS', 'hipBLAS-common']
}
Expand All @@ -24,6 +24,11 @@ def runCI =
prj.libraryDependencies = ['rocBLAS', 'rocSPARSE', 'rocSOLVER', 'hipBLAS-common']
}

if (env.BRANCH_NAME ==~ /PR-\d+/ && pullRequest.labels.contains("solverBuildtime"))
{
buildCommand += ' --solver-buildtime'
}

if (env.BRANCH_NAME ==~ /PR-\d+/ && pullRequest.labels.contains('g++'))
{
buildCommand += ' --compiler=g++'
Expand Down
34 changes: 23 additions & 11 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,19 @@ rocm_setup_version( VERSION ${VERSION_STRING} )

option( BUILD_VERBOSE "Output additional build information" OFF )

option( BUILD_WITH_SOLVER "Add additional functions from rocSOLVER" ON )

if( BUILD_WITH_SOLVER )
add_definitions( -D__HIP_PLATFORM_SOLVER__ )
endif( )

# BUILD_SHARED_LIBS is a cmake built-in; we make it an explicit option such that it shows in cmake-gui
option( BUILD_SHARED_LIBS "Build hipBLAS as a shared library" ON )

if(BUILD_SHARED_LIBS)
set(BUILD_WITH_SOLVER_DEFAULT OFF)
else()
Comment on lines +77 to +79
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this just for testing or default to not link? Default to not link may may need to review effect on rpath or package consumption documentation.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Default will be to not link. I think it'll be ok but I'll test out consuming the hipblas package now to make sure it find rocsolver.

set(BUILD_WITH_SOLVER_DEFAULT ON)
endif()

option( BUILD_WITH_SOLVER "Build hipBLAS with LAPACK functionality as provided by rocSOLVER available at build time (requires installed rocSOLVER dependency; only with AMD backend)" "${BUILD_WITH_SOLVER_DEFAULT}" )

option(BUILD_SOLVER_TESTS "Build additional tests to cover LAPACK functionality provided by rocSOLVER with AMD backend" ON)

# Deprecated USE_CUDA option
if(DEFINED USE_CUDA)
if(USE_CUDA)
Expand Down Expand Up @@ -125,7 +129,6 @@ if(BUILD_ADDRESS_SANITIZER)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address -shared-libasan")
endif()


# FOR HANDLING ENABLE/DISABLE OPTIONAL BACKWARD COMPATIBILITY for FILE/FOLDER REORG
option(BUILD_FILE_REORG_BACKWARD_COMPATIBILITY "Build with file/folder reorg with backward compatibility enabled" OFF)
if(BUILD_FILE_REORG_BACKWARD_COMPATIBILITY AND NOT WIN32)
Expand Down Expand Up @@ -202,11 +205,20 @@ endif( )

# Package specific CPACK vars
if(HIP_PLATFORM STREQUAL amd)
if(BUILD_WITH_SOLVER)
set(rocsolver_minimum 3.27.0)
rocm_package_add_dependencies(SHARED_DEPENDS "rocsolver >= ${rocsolver_minimum}")
rocm_package_add_rpm_dependencies(STATIC_DEPENDS "rocsolver-static-devel >= ${rocsolver_minimum}")
rocm_package_add_deb_dependencies(STATIC_DEPENDS "rocsolver-static-dev >= ${rocsolver_minimum}")
else()
list(APPEND CPACK_DEBIAN_RUNTIME_PACKAGE_RECOMMENDS "rocsolver")
list(APPEND CPACK_RPM_RUNTIME_PACKAGE_SUGGESTS "rocsolver")
endif()

set(rocblas_minimum 4.3.0)
set(rocsolver_minimum 3.27.0)
rocm_package_add_dependencies(SHARED_DEPENDS "rocblas >= ${rocblas_minimum}" "rocsolver >= ${rocsolver_minimum}")
rocm_package_add_rpm_dependencies(STATIC_DEPENDS "rocblas-static-devel >= ${rocblas_minimum}" "rocsolver-static-devel >= ${rocsolver_minimum}")
rocm_package_add_deb_dependencies(STATIC_DEPENDS "rocblas-static-dev >= ${rocblas_minimum}" "rocsolver-static-dev >= ${rocsolver_minimum}")
rocm_package_add_dependencies(SHARED_DEPENDS "rocblas >= ${rocblas_minimum}")
rocm_package_add_rpm_dependencies(STATIC_DEPENDS "rocblas-static-devel >= ${rocblas_minimum}")
rocm_package_add_deb_dependencies(STATIC_DEPENDS "rocblas-static-dev >= ${rocblas_minimum}")
endif( )

set(hipblas_common_minimum 1.0.0)
Expand Down
3 changes: 2 additions & 1 deletion clients/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,9 @@ endif()

if( BUILD_CLIENTS_TESTS OR BUILD_CLIENTS_BENCHMARKS OR BUILD_CLIENTS_SAMPLES )
if( NOT WIN32 )
if( BUILD_WITH_SOLVER )
if( BUILD_SOLVER_TESTS )
add_library(hipblas_fortran_client STATIC ${hipblas_f90_source_clients_solver})

else()
add_library(hipblas_fortran_client STATIC ${hipblas_f90_source_clients_no_solver})
endif()
Expand Down
5 changes: 5 additions & 0 deletions clients/benchmarks/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,11 @@ target_compile_options(hipblas_v2-bench PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${COMM
target_compile_definitions( hipblas-bench PRIVATE HIPBLAS_BENCH ${COMMON_DEFINES} ${BLIS_DEFINES} )
target_compile_definitions( hipblas_v2-bench PRIVATE HIPBLAS_BENCH ${COMMON_DEFINES} ${BLIS_DEFINES} HIPBLAS_V2 )

if(BUILD_SOLVER_TESTS)
target_compile_definitions(hipblas-bench PRIVATE -DBUILD_SOLVER_TESTS)
target_compile_definitions(hipblas_v2-bench PRIVATE -DBUILD_SOLVER_TESTS)
endif()

target_link_libraries( hipblas-bench PRIVATE ${BLAS_LIBRARY} ${COMMON_LINK_LIBS} )
target_link_libraries( hipblas_v2-bench PRIVATE ${BLAS_LIBRARY} ${COMMON_LINK_LIBS} )
if (NOT WIN32)
Expand Down
2 changes: 1 addition & 1 deletion clients/common/cblas_interface.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3916,7 +3916,7 @@ void ref_trmm<hipblasDoubleComplex>(hipblasSideMode_t side,
* ===========================================================================
*/

#ifdef __HIP_PLATFORM_SOLVER__
#ifdef BUILD_SOLVER_TESTS

class ipiv_wrapper : public host_vector<int64_t>
{
Expand Down
10 changes: 5 additions & 5 deletions clients/common/clients_common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/* ************************************************************************
* Copyright (C) 2016-2023 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (C) 2016-2024 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
Expand Down Expand Up @@ -221,7 +221,7 @@
#include "blas_ex/testing_trsm_ex.hpp"
#include "blas_ex/testing_trsm_strided_batched_ex.hpp"
// solver functions
#ifdef __HIP_PLATFORM_SOLVER__
#ifdef BUILD_SOLVER_TESTS
#include "solver/testing_gels.hpp"
#include "solver/testing_gels_batched.hpp"
#include "solver/testing_gels_strided_batched.hpp"
Expand Down Expand Up @@ -459,7 +459,7 @@ void get_test_name(const Arguments& arg, std::string& name)
{"trtri_batched", testname_trtri_batched},
{"trtri_strided_batched", testname_trtri_strided_batched},

#ifdef __HIP_PLATFORM_SOLVER__
#ifdef BUILD_SOLVER_TESTS
{"geqrf", testname_geqrf},
{"geqrf_batched", testname_geqrf_batched},
{"geqrf_strided_batched", testname_geqrf_strided_batched},
Expand Down Expand Up @@ -677,7 +677,7 @@ struct perf_blas<T, U, std::enable_if_t<std::is_same<T, float>{} || std::is_same
{"trsm_strided_batched", testing_trsm_strided_batched<T>},
{"trsm_strided_batched_ex", testing_trsm_strided_batched_ex<T>},

#ifdef __HIP_PLATFORM_SOLVER__
#ifdef BUILD_SOLVER_TESTS
{"geqrf", testing_geqrf<T>},
{"geqrf_batched", testing_geqrf_batched<T>},
{"geqrf_strided_batched", testing_geqrf_strided_batched<T>},
Expand Down Expand Up @@ -899,7 +899,7 @@ struct perf_blas<
{"trmm_batched", testing_trmm_batched<T>},
{"trmm_strided_batched", testing_trmm_strided_batched<T>},

#ifdef __HIP_PLATFORM_SOLVER__
#ifdef BUILD_SOLVER_TESTS
{"geqrf", testing_geqrf<T>},
{"geqrf_batched", testing_geqrf_batched<T>},
{"geqrf_strided_batched", testing_geqrf_strided_batched<T>},
Expand Down
2 changes: 1 addition & 1 deletion clients/common/hipblas_template_specialization.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13798,7 +13798,7 @@ hipblasStatus_t hipblasZgeamStridedBatchedCast_64(hipblasHandle_t ha
batchCount);
}

#ifdef __HIP_PLATFORM_SOLVER__
#ifdef BUILD_SOLVER_TESTS

// getrf
hipblasStatus_t hipblasCgetrfCast(
Expand Down
9 changes: 7 additions & 2 deletions clients/gtest/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ set(hipblas_test_source
blas_ex/gemm_ex_gtest.cpp
)

if( BUILD_WITH_SOLVER )
if( BUILD_SOLVER_TESTS )
set( hipblas_solver_test_source
solver/getrf_gtest.cpp
solver/getrs_gtest.cpp
Expand Down Expand Up @@ -179,6 +179,11 @@ target_compile_options(hipblas_v2-test PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${COMMO
target_compile_definitions( hipblas-test PRIVATE ${COMMON_DEFINES} )
target_compile_definitions( hipblas_v2-test PRIVATE ${COMMON_DEFINES} HIPBLAS_V2 )

if(BUILD_SOLVER_TESTS)
target_compile_definitions(hipblas-test PRIVATE -DBUILD_SOLVER_TESTS)
target_compile_definitions(hipblas_v2-test PRIVATE -DBUILD_SOLVER_TESTS)
endif()

target_link_libraries( hipblas-test PRIVATE ${BLAS_LIBRARY} ${COMMON_LINK_LIBS} )
target_link_libraries( hipblas_v2-test PRIVATE ${BLAS_LIBRARY} ${COMMON_LINK_LIBS} )
if (NOT WIN32)
Expand Down Expand Up @@ -275,7 +280,7 @@ set( HIPBLAS_L3_YAML_DATA blas3/dgmm_gtest.yaml blas3/geam_gtest.yaml blas3/gemm
set( HIPBLAS_EX_YAML_DATA blas_ex/axpy_ex_gtest.yaml blas_ex/dot_ex_gtest.yaml blas_ex/nrm2_ex_gtest.yaml
blas_ex/rot_ex_gtest.yaml blas_ex/scal_ex_gtest.yaml blas_ex/gemm_ex_gtest.yaml blas_ex/trsm_ex_gtest.yaml )

if( BUILD_WITH_SOLVER )
if( BUILD_SOLVER_TESTS )
set( HIPBLAS_SOLVER_YAML_DATA solver/gels_gtest.yaml solver/geqrf_gtest.yaml solver/getrf_gtest.yaml solver/getri_gtest.yaml solver/getrs_gtest.yaml )
endif()

Expand Down
2 changes: 1 addition & 1 deletion clients/include/cblas_interface.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -798,7 +798,7 @@ void ref_trmm(hipblasSideMode_t side,
* ===========================================================================
*/

#ifdef __HIP_PLATFORM_SOLVER__
#ifdef BUILD_SOLVER_TESTS

// potrf
template <typename T>
Expand Down
2 changes: 1 addition & 1 deletion clients/include/hipblas.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11884,7 +11884,7 @@ namespace
MAP2CF_V2(hipblasTrtriStridedBatched, hipblasComplex, hipblasCtrtriStridedBatched);
MAP2CF_V2(hipblasTrtriStridedBatched, hipblasDoubleComplex, hipblasZtrtriStridedBatched);

#ifdef __HIP_PLATFORM_SOLVER__
#ifdef BUILD_SOLVER_TESTS

// getrf
template <typename T, bool FORTRAN = false>
Expand Down
6 changes: 6 additions & 0 deletions docs/functions.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,12 @@ Some functions in hipBLAS may use atomic operations to increase performance whic
By default, the rocBLAS backend allows the use of atomics while the cuBLAS backend disallows the use of atomics. To set the desired behavior, users should call
:any:`hipblasSetAtomicsMode`. Please see the rocBLAS or cuBLAS documentation for more information regarding specifics of atomic operations in the backend library.

LAPACK Functionality
====================

LAPACK functionality is provided by the rocSOLVER library when running with AMD backend. By default, hipBLAS tries to find the rocSOLVER dependency
at runtime using dlopen(). If rocSOLVER cannot be found, this functionality will return HIPBLAS_STATUS_NOT_SUPPORTED.

*************
hipBLAS Types
*************
Expand Down
17 changes: 7 additions & 10 deletions docs/install/Linux_Install_Guide.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,16 +79,6 @@ in the table below.
| | directory, you do not |
| | need the -i flag. |
+-------------------------------------------+--------------------------+
| ``python3 rmake.py -n`` | Build library without |
| | functionality provided |
| | by rocSOLVER. |
| | rocSOLVER, rocSPARSE, |
| | and rocPRIM dependencies |
| | will not be needed. |
| | This flag has no effect |
| | when building with cuda |
| | backend. |
+-------------------------------------------+--------------------------+


Build library dependencies + client dependencies + library + client
Expand Down Expand Up @@ -158,6 +148,13 @@ in the table below.
| | directory, you do not |
| | need the -i flag. |
+-------------------------------------------+--------------------------+
| ``python3 rmake.py -nc`` | Build library and client |
| | in your local directory. |
| | Functionality provided |
| | by rocSOLVER will not |
| | be available in the |
| | clients. |
+-------------------------------------------+--------------------------+

Dependencies for building library
==================================
Expand Down
17 changes: 7 additions & 10 deletions docs/install/Windows_Install_Guide.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,16 +114,6 @@ in the table below.
| | directory, you do not |
| | need the -i flag. |
+-------------------------------------------+--------------------------+
| ``python3 rmake.py -n`` | Build library without |
| | functionality provided |
| | by rocSOLVER. |
| | rocSOLVER, rocSPARSE, |
| | and rocPRIM dependencies |
| | will not be needed. |
| | This flag has no effect |
| | when building with cuda |
| | backend. |
+-------------------------------------------+--------------------------+


Build library dependencies + client dependencies + library + client
Expand Down Expand Up @@ -193,6 +183,13 @@ in the table below.
| | directory, you do not |
| | need the -i flag. |
+-------------------------------------------+--------------------------+
| ``python3 rmake.py -nc`` | Build library and client |
| | in your local directory. |
| | Functionality provided |
| | by rocSOLVER will not |
| | be available in the |
| | clients. |
+-------------------------------------------+--------------------------+

Dependencies for building library
==================================
Expand Down
14 changes: 10 additions & 4 deletions install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ install_packages( )

# Do not install rocsolver if --rocsolver_path flag is set,
if [[ -z ${rocsolver_path+foo} ]]; then
if [[ "${build_solver}" == true ]]; then
if [[ "${solver_buildtime}" == true ]]; then
library_dependencies_ubuntu+=( "rocsolver-dev" )
library_dependencies_centos_rhel+=( "rocsolver-devel" )
library_dependencies_centos_rhel_8+=( "rocsolver-devel" )
Expand Down Expand Up @@ -374,7 +374,9 @@ cat <<EOF

-k, --relwithdebinfo Build in release debug mode, equivalent to set CMAKE_BUILD_TYPE=RelWithDebInfo. (Default build type is Release)

-n, --no-solver Build hipLBAS library without rocSOLVER dependency
-n, --no-solver Build without rocSOLVER-based tests enabled. hipBLAS will still try to load rocSOLVER at runtime with AMD backend.

--solver-buildtime Build with rocSOLVER functionality enabled at built time with AMD backend.

--rocblas-path <blasdir> Specify path to an existing rocBLAS install directory (e.g. /src/rocBLAS/build/release/rocblas-install).

Expand All @@ -390,7 +392,8 @@ install_package=false
install_dependencies=false
install_prefix=hipblas-install
build_clients=false
build_solver=true
solver_buildtime=true
build_solver_tests=true
build_release=true
install_cuda=false
cuda_version_install=default
Expand Down Expand Up @@ -442,7 +445,10 @@ while true; do
build_clients=true
shift ;;
-n|--no-solver)
build_solver=false
build_solver_tests=false
shift ;;
--solver-buildtime)
solver_buildtime=true
shift ;;
-g|--debug)
build_release=false
Expand Down
9 changes: 8 additions & 1 deletion library/src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ endfunction( )
prepend_path( ".." hipblas_headers_public relative_hipblas_headers_public )

if(HIP_PLATFORM STREQUAL amd)
set( hipblas_source "${CMAKE_CURRENT_SOURCE_DIR}/amd_detail/hipblas.cpp" )
set( hipblas_source "${CMAKE_CURRENT_SOURCE_DIR}/amd_detail/hipblas.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/amd_detail/dlopen/load_rocsolver.cpp" )
else( )
set( hipblas_source "${CMAKE_CURRENT_SOURCE_DIR}/nvidia_detail/hipblas.cpp" )
endif( )
Expand Down Expand Up @@ -82,6 +82,11 @@ if(HIP_PLATFORM STREQUAL amd)
endif( )
endif( )

if(NOT BUILD_WITH_SOLVER)
target_link_libraries(hipblas PRIVATE $<$<PLATFORM_ID:Linux>:${CMAKE_DL_LIBS}>)
set_property(TARGET hipblas APPEND PROPERTY BUILD_RPATH ${ROCSOLVER_PATH}/lib)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What prompted this change?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe this is needed to find the rocsolver library if hipblas is built with the install script --rocsolver-path option

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Keep in mind that rocsolver might not actually even exist on the system at build-time. Personally, I wouldn't bother mucking around with the RPATH, as it can always be handled after the build by setting the LD_LIBRARY_PATH.

If you prefer to set the RPATH, it should be guarded to only be set when rocsolver exists and --rocsolver-path has been used. The hardcoded lib may be a problem, as the library directory can sometimes have other names in third-party builds, like lib64 or <target-triple>/lib. You could use ${CMAKE_INSTALL_LIBDIR} instead, but I think it would be more robust to extract the path from the target:

if(DEFINED ROCSOLVER_PATH)
  find_package(rocsolver QUIET PATHS "${ROCSOLVER_PATH}" NO_DEFAULT_PATH)
  if(rocsolver_FOUND)
    get_target_property(rocsolver_location roc::rocsolver LOCATION)
    get_filename_component(rocsolver_directory ${rocsolver_location} DIRECTORY)
    set_property(TARGET hipblas APPEND PROPERTY BUILD_RPATH ${rocsolver_directory})
  endif()
endif()

With all that said, I prefer less code to more code. IMO, it's fine to expect users to set LD_LIBRARY_PATH if rocSOLVER is not in a system directory or /opt/rocm.

endif()

list(APPEND static_depends PACKAGE rocblas)
target_link_libraries( hipblas PRIVATE roc::rocblas )
target_link_libraries( hipblas PUBLIC hip::host )
Expand All @@ -93,6 +98,8 @@ if(HIP_PLATFORM STREQUAL amd)
set ( ENV{rocsolver_DIR} ${CUSTOM_ROCSOLVER})
find_package( rocsolver REQUIRED CONFIG NO_CMAKE_PATH )

target_compile_definitions(hipblas PRIVATE -DBUILD_WITH_SOLVER)

# in case of using custom rocsolver and not custom rocblas, we need to have
# custom rocsolver include directories before rocblas/hip include directories
# in case there is a rocsolver installed on the system.
Expand Down
Loading