Manage sparse dependencies using dlopen (#302)

* dlopen for rocSPARSE

* dlopen for CHOLMOD

* Updated hipsolver_sparse.cpp

* Updated tests

* Updated documentation

* Default BUILD_WITH_SPARSE to ON for static library

* Added comment

* Add packages to recommends

* Rename --embed-sparse option to --sparse

* Addressed review comments

Author: tfalders

CHANGELOG.md

@@ -11,6 +11,10 @@ Full documentation for hipSOLVER is available at the [hipSOLVER Documentation](h
 ### Optimized
 ### Changed
 - Binaries in debug builds no longer have a "-d" suffix.
+- Changed rocSPARSE and SuiteSparse to be run-time dependencies by default. The `BUILD_WITH_SPARSE` CMake option can still be used
+  to convert them into build-time dependencies (now off by default).
+- The --no-sparse option of the install script now only affects the hipSOLVER clients and their dependency on hipSPARSE. Use the
+  `BUILD_HIPSPARSE_TESTS` CMake option to enable tests for the hipsolverSp API (on by default).
 
 ### Deprecated
 ### Removed

CMakeLists.txt

@@ -101,14 +101,21 @@ endif()
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${ROCM_PATH}/lib/cmake/hip ${HIP_PATH}/cmake)
 
 # NOTE:  workaround until hip cmake modules fixes symlink logic in their config files; remove when fixed
-list(APPEND CMAKE_PREFIX_PATH /opt/rocm /opt/rocm/llvm /opt/rocm/hip)
+list(APPEND CMAKE_PREFIX_PATH ${ROCM_PATH}/lib/cmake/hip /opt/rocm /opt/rocm/llvm /opt/rocm/hip)
+
+option(BUILD_SHARED_LIBS "Build hipSOLVER as a shared library" ON)
+if(BUILD_SHARED_LIBS)
+  set(BUILD_WITH_SPARSE_DEFAULT OFF)
+else()
+  set(BUILD_WITH_SPARSE_DEFAULT ON)
+endif()
 
 option(BUILD_ADDRESS_SANITIZER "Build with address sanitizer enabled" OFF)
 option(BUILD_CODE_COVERAGE "Build hipSOLVER with code coverage enabled" OFF)
 option(BUILD_HIPBLAS_TESTS "Build additional tests to ensure hipBLAS and hipSOLVER are compatible (requires installed hipBLAS)" OFF)
+option(BUILD_HIPSPARSE_TESTS "Build additional tests to cover sparse functionality (requires installed hipSPARSE)" ON)
 # BUILD_SHARED_LIBS is a cmake built-in; we make it an explicit option such that it shows in cmake-gui
-option(BUILD_SHARED_LIBS "Build hipSOLVER as a shared library" ON)
-option(BUILD_WITH_SPARSE "Build hipSOLVER with sparse functionality and tests enabled (requires additional dependencies)" "${UNIX}")
+option(BUILD_WITH_SPARSE "Build hipSOLVER with sparse functionality available at build time (requires installed dependencies)" "${BUILD_WITH_SPARSE_DEFAULT}")
 option(BUILD_VERBOSE "Output additional build information" OFF)
 option(USE_CUDA "Look for CUDA and use that as a backend if found" OFF)
 option(HIPSOLVER_FIND_PACKAGE_LAPACK_CONFIG "Skip module mode search for LAPACK" ON)
@@ -228,20 +235,24 @@ if(NOT USE_CUDA)
   rocm_package_add_rpm_dependencies(STATIC_DEPENDS "rocblas-static-devel >= ${rocblas_minimum}" "rocsolver-static-devel >= ${rocsolver_minimum}")
   rocm_package_add_deb_dependencies(STATIC_DEPENDS "rocblas-static-dev >= ${rocblas_minimum}" "rocsolver-static-dev >= ${rocsolver_minimum}")
 
+  if(SYSTEM_OS STREQUAL "centos" OR SYSTEM_OS STREQUAL "rhel" OR SYSTEM_OS STREQUAL "mariner")
+    list(APPEND hipsolver_pkgdeps "suitesparse")
+  elseif(SYSTEM_OS STREQUAL "ubuntu" AND SYSTEM_OS_VERSION VERSION_GREATER_EQUAL "24.04")
+    list(APPEND hipsolver_pkgdeps "libcholmod5" "libsuitesparseconfig7")
+  else()
+    list(APPEND hipsolver_pkgdeps "libcholmod3" "libsuitesparseconfig5")
+  endif()
+
   if(BUILD_WITH_SPARSE)
     set(rocsparse_minimum 2.3.0)
     rocm_package_add_dependencies(SHARED_DEPENDS "rocsparse >= ${rocsparse_minimum}")
     rocm_package_add_rpm_dependencies(STATIC_DEPENDS "rocsparse-static-devel >= ${rocsparse_minimum}")
     rocm_package_add_deb_dependencies(STATIC_DEPENDS "rocsparse-static-dev >= ${rocsparse_minimum}")
 
-    if(SYSTEM_OS STREQUAL "centos" OR SYSTEM_OS STREQUAL "rhel" OR SYSTEM_OS STREQUAL "mariner")
-      list(APPEND hipsolver_pkgdeps "suitesparse")
-    elseif(SYSTEM_OS STREQUAL "ubuntu" AND SYSTEM_OS_VERSION VERSION_GREATER_EQUAL "24.04")
-      list(APPEND hipsolver_pkgdeps "libcholmod5" "libsuitesparseconfig7")
-    else()
-      list(APPEND hipsolver_pkgdeps "libcholmod3" "libsuitesparseconfig5")
-    endif()
     rocm_package_add_dependencies(DEPENDS ${hipsolver_pkgdeps})
+  else()
+    list(APPEND CPACK_DEBIAN_RUNTIME_PACKAGE_RECOMMENDS "rocsparse" ${hipsolver_pkgdeps})
+    list(APPEND CPACK_RPM_RUNTIME_PACKAGE_SUGGESTS "rocsparse" ${hipsolver_pkgdeps})
   endif()
 endif()
 

clients/benchmarks/CMakeLists.txt

@@ -20,7 +20,7 @@
 #
 # ########################################################################
 
-if(BUILD_WITH_SPARSE)
+if(BUILD_HIPSPARSE_TESTS)
   if(NOT TARGET hipsparse)
     if(CUSTOM_HIPSPARSE)
       set(ENV{hipsparse_DIR} ${CUSTOM_HIPSPARSE})
@@ -52,7 +52,7 @@ target_link_libraries(hipsolver-bench PRIVATE
   clients-common
   roc::hipsolver
 )
-if(BUILD_WITH_SPARSE)
+if(BUILD_HIPSPARSE_TESTS)
   set_source_files_properties(client.cpp
     PROPERTIES
       COMPILE_DEFINITIONS HAVE_HIPSPARSE

clients/gtest/CMakeLists.txt

@@ -33,7 +33,7 @@ if(BUILD_HIPBLAS_TESTS)
   endif()
 endif()
 
-if(BUILD_WITH_SPARSE)
+if(BUILD_HIPSPARSE_TESTS)
   if(NOT TARGET hipsparse)
     if(CUSTOM_HIPSPARSE)
       set(ENV{hipsparse_DIR} ${CUSTOM_HIPSPARSE})
@@ -117,7 +117,7 @@ if(BUILD_HIPBLAS_TESTS)
   target_link_libraries(hipsolver-test PRIVATE roc::hipblas)
   target_sources(hipsolver-test PRIVATE hipblas_include1_gtest.cpp hipblas_include2_gtest.cpp)
 endif()
-if(BUILD_WITH_SPARSE)
+if(BUILD_HIPSPARSE_TESTS)
   target_link_libraries(hipsolver-test PRIVATE roc::hipsparse)
   target_sources(hipsolver-test PRIVATE ${hipsolverSp_test_source})
 endif()

clients/gtest/csrlsvchol_gtest.cpp

@@ -66,6 +66,11 @@ template <bool HOST>
 class CSRLSVCHOL_BASE : public ::TestWithParam<csrlsvchol_tuple>
 {
 protected:
+    void SetUp() override
+    {
+        if(hipsolverSpCreate(nullptr) == HIPSOLVER_STATUS_NOT_SUPPORTED)
+            GTEST_SKIP() << "Sparse dependencies could not be loaded";
+    }
     void TearDown() override
     {
         EXPECT_EQ(hipGetLastError(), hipSuccess);

docs/howto/usage.rst

@@ -246,7 +246,8 @@ user-provided and automatically allocated workspaces.
 
 Using rocSOLVER's in-place functions
 --------------------------------------
-The solvers `gesv` and `gels` in cuSOLVER are out-of-place in the sense that the solution vectors `X` do not overwrite the input matrix `B`. In rocSOLVER this is not the case; when `hipsolverXXgels` or `hipsolverXXgesv` call rocSOLVER, some data
+The solvers `gesv` and `gels` in cuSOLVER are out-of-place in the sense that the solution vectors `X` do not overwrite the input matrix `B`.
+In rocSOLVER this is not the case; when `hipsolverXXgels` or `hipsolverXXgesv` call rocSOLVER, some data
 movements must be done internally to restore `B` and copy the results back to `X`. These copies could introduce noticeable
 overhead depending on the size of the matrices. To avoid this potential problem, users can pass `X = B` to `hipsolverXXgels`
 or `hipsolverXXgesv` when using the rocSOLVER backend; in this case, no data movements will be required, and the solution

docs/installation/install.rst

@@ -15,9 +15,6 @@ Download pre-built packages from :doc:`ROCm's package servers <rocm-install-on-l
 
 * ``sudo apt update && sudo apt install hipsolver``
 
-.. note::
-    The pre-built packages depend on the third-party library SuiteSparse, which must be installed on the system prior to installing hipSOLVER. SuiteSparse can be installed using the package manager of most distros.
-
 
 Build & install library using script (Ubuntu only)
 ===================================================
@@ -28,7 +25,6 @@ so it may prompt you for a password.
 * ``./install.sh -id`` --- build library, build dependencies, and install (-d flag only needs to be passed once on a system).
 * ``./install.sh -ic`` --- build library, build clients (tests, benchmarks, and samples), and install.
 * ``./install.sh --cuda`` --- build library on a CUDA-enabled machine, with cuSOLVER as the backend.
-* ``./install.sh --no-sparse`` --- build library without hipsolverSp functionality, with rocSOLVER as the backend.
 
 To see more options, use the help option of the install script.
 
@@ -64,17 +60,18 @@ The rocSOLVER backend has the following dependencies:
 
 1. `rocSOLVER <https://github.com/ROCmSoftwarePlatform/rocSOLVER>`_
 2. `rocBLAS <https://github.com/ROCmSoftwarePlatform/rocBLAS>`_
-3. `rocSPARSE <https://github.com/ROCmSoftwarePlatform/rocSPARSE>`_ (optional, required by default)
-4. `SuiteSparse <https://github.com/DrTimothyAldenDavis/SuiteSparse>`_ modules CHOLMOD and SuiteSparse_config (optional, required by default)
+3. `rocSPARSE <https://github.com/ROCmSoftwarePlatform/rocSPARSE>`_ (optional)
+4. `SuiteSparse <https://github.com/DrTimothyAldenDavis/SuiteSparse>`_ modules CHOLMOD and SuiteSparse_config (optional)
 
 rocSOLVER itself depends on rocBLAS and rocSPARSE, therefore all three libraries should be present with a standard rocSOLVER installation. For more information
 about building and installing rocSOLVER, refer to the :doc:`rocSOLVER documentation <rocsolver:installation/installlinux>`.
 
 SuiteSparse is a third-party library, and can be installed using the package managers of most distros. Together with rocSPARSE, it is used to provide
-functionality for the hipsolverSp API. If only hipsolverDn and/or hipsolverRf are needed, these dependencies can be ignored by setting the ``BUILD_WITH_SPARSE``
-option to ``OFF``.
+functionality for the hipsolverSp API. By default, both libraries are run-time dependencies and will be dynamically loaded at run-time by dlopen if they are
+present on the system. If the ``BUILD_WITH_SPARSE`` option is set to ``ON``, these will instead become build-time dependencies and must be present on the
+system in order to build hipSOLVER.
 
-* ``DBUILD_WITH_SPARSE=OFF``
+* ``DBUILD_WITH_SPARSE=ON``
 
 
 Build library + tests + benchmarks + samples manually
@@ -104,11 +101,11 @@ the ``CMAKE_PREFIX_PATH`` definition. The following is a sequence of steps to bu
     make -j$(nproc) install
 
 hipBLAS is only required if the ``BUILD_HIPBLAS_TESTS`` option is set to ``ON``, and is used to ensure compatibility between the hipblas enums defined
-separately by hipBLAS and hipSOLVER. hipSPARSE is required by default but can be ignored if the ``BUILD_WITH_SPARSE`` option is set to ``OFF``, and is used
+separately by hipBLAS and hipSOLVER. hipSPARSE is required by default but can be ignored if the ``BUILD_HIPSPARSE_TESTS`` option is set to ``OFF``, and is used
 to create objects required by tests for the hipsolverSp API.
 
 * ``DBUILD_HIPBLAS_TESTS=ON``
-* ``DBUILD_WITH_SPARSE=OFF``
+* ``DBUILD_HIPSPARSE_TESTS=OFF``
 
 Both libraries can be installed similarly to hipSOLVER. For example, the install scripts for hipBLAS and hipSPARSE can each be invoked to build and
 install the respective library via:

docs/reference/refactor-api/index.rst

@@ -16,5 +16,5 @@ For ease of porting from existing cuSOLVER applications to hipSOLVER, functions
 method signatures that are consistent with the cusolverRf interface. At present, equivalent functions have not been added to hipSOLVER's regular API.
 
   * :ref:`refactor_types`
-  * :ref:`refactor_helpers` 
+  * :ref:`refactor_helpers`
   * :ref:`refactor_refactorfunc`

docs/reference/sparse-api/index.rst

@@ -18,6 +18,6 @@ regular API. Note that there are :ref:`some performance limitations <sparse_perf
 functionality required for optimal performance has been implemented yet.
 
   * :ref:`sparse_types`
-  * :ref:`sparse_helpers` 
+  * :ref:`sparse_helpers`
   * :ref:`sparse_sparsefunc`
 

install.sh

@@ -54,7 +54,8 @@ function display_help()
   echo "    [--static] Create static library instead of shared library"
   echo "    [--codecoverage] Build with code coverage profiling enabled, excluding release mode."
   echo "    [--address-sanitizer] Build with address sanitizer enabled. Uses hipcc to compile"
-  echo "    [--no-sparse] Build with sparse functionality and tests disabled."
+  echo "    [--sparse] Build with sparse functionality enabled at build time."
+  echo "    [--no-sparse] Build with sparse functionality tests disabled."
   echo "    [--docs] (experimental) Pass this flag to build the documentation from source."
   echo "    [--cmake-arg] Forward the given argument to CMake when configuring the build"
 }
@@ -363,7 +364,8 @@ build_cuda=false
 build_hip_clang=true
 build_release=true
 build_relocatable=false
-build_sparse=true
+build_sparse=false
+build_sparse_tests=true
 build_docs=false
 cmake_prefix_path=/opt/rocm
 cuda_path=/usr/local/cuda
@@ -383,7 +385,7 @@ declare -a cmake_client_options
 # check if we have a modern version of getopt that can handle whitespace and long parameters
 getopt -T
 if [[ $? -eq 4 ]]; then
-  GETOPT_PARSE=$(getopt --name "${0}" --longoptions help,install,codecoverage,clients,no-solver,dependencies,debug,relwithdebinfo,hip-clang,no-hip-clang,compiler:,cuda,use-cuda,cudapath:,static,cmakepp,relocatable:,rocm-dev:,rocblas:,rocblas-path:,hipblas-path:,rocsolver:,rocsolver-path:,rocsparse:,rocsparse-path:,hipsparse-path:,custom-target:,docs,address-sanitizer,no-sparse,cmake-arg: --options rhicndgkp:v:b:s: -- "$@")
+  GETOPT_PARSE=$(getopt --name "${0}" --longoptions help,install,codecoverage,clients,no-solver,dependencies,debug,relwithdebinfo,hip-clang,no-hip-clang,compiler:,cuda,use-cuda,cudapath:,static,cmakepp,relocatable:,rocm-dev:,rocblas:,rocblas-path:,hipblas-path:,rocsolver:,rocsolver-path:,rocsparse:,rocsparse-path:,hipsparse-path:,custom-target:,docs,address-sanitizer,sparse,no-sparse,cmake-arg: --options rhicndgkp:v:b:s: -- "$@")
 else
   echo "Need a new version of getopt"
   exit 1
@@ -450,8 +452,11 @@ while true; do
         build_address_sanitizer=true
         compiler=hipcc
         shift ;;
+    --sparse)
+        build_sparse=true
+        shift ;;
     --no-sparse)
-        build_sparse=false
+        build_sparse_tests=false
         shift ;;
     -p|--cmakepp)
         cmake_prefix_path=${2}
@@ -675,9 +680,12 @@ fi
     cmake_common_options+=("-DBUILD_ADDRESS_SANITIZER=ON")
   fi
 
-  # no sparse
-  if [[ "${build_sparse}" == false ]]; then
-    cmake_common_options+=("-DBUILD_WITH_SPARSE=OFF")
+  # build with sparse
+  if [[ "${build_sparse}" == true ]]; then
+    cmake_common_options+=("-DBUILD_WITH_SPARSE=ON")
+  fi
+  if [[ "${build_sparse_tests}" == false ]]; then
+    cmake_common_options+=("-DBUILD_HIPSPARSE_TESTS=OFF")
   fi
 
   # Build library

library/src/CMakeLists.txt

@@ -42,6 +42,8 @@ prepend_path(".." hipsolver_headers_public relative_hipsolver_headers_public)
 
 if(NOT USE_CUDA)
   set(hipsolver_source
+    "${CMAKE_CURRENT_SOURCE_DIR}/amd_detail/dlopen/cholmod.cpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/amd_detail/dlopen/rocsparse.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/amd_detail/hipsolver_conversions.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/amd_detail/hipsolver.cpp"
     "${CMAKE_CURRENT_SOURCE_DIR}/amd_detail/hipsolver_dense.cpp"
@@ -100,6 +102,10 @@ endif()
 
 set(static_depends)
 
+if(NOT BUILD_WITH_SPARSE)
+  target_link_libraries(hipsolver PRIVATE $<$<PLATFORM_ID:Linux>:${CMAKE_DL_LIBS}>)
+endif()
+
 # Build hipsolver from source on AMD platform
 if(NOT USE_CUDA)
   # Find rocBLAS

library/src/amd_detail/dlopen/cholmod.cpp

@@ -0,0 +1,117 @@
+/* ************************************************************************
+ * Copyright (C) 2024 Advanced Micro Devices, Inc. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop-
+ * ies of the Software, and to permit persons to whom the Software is furnished
+ * to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM-
+ * PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE-
+ * CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ *
+ * ************************************************************************ */
+
+#include "cholmod.hpp"
+#include "load_function.hpp"
+
+HIPSOLVER_BEGIN_NAMESPACE
+
+fp_cholmod_start            g_cholmod_start;
+fp_cholmod_finish           g_cholmod_finish;
+fp_cholmod_allocate_sparse  g_cholmod_allocate_sparse;
+fp_cholmod_free_sparse      g_cholmod_free_sparse;
+fp_cholmod_allocate_dense   g_cholmod_allocate_dense;
+fp_cholmod_free_dense       g_cholmod_free_dense;
+fp_cholmod_free_factor      g_cholmod_free_factor;
+fp_cholmod_drop             g_cholmod_drop;
+fp_cholmod_analyze          g_cholmod_analyze;
+fp_cholmod_analyze_ordering g_cholmod_analyze_ordering;
+fp_cholmod_factorize        g_cholmod_factorize;
+fp_cholmod_solve            g_cholmod_solve;
+
+static bool load_cholmod()
+{
+#ifdef _WIN32
+    // Library users will need to call SetErrorMode(SEM_FAILCRITICALERRORS) if
+    // they wish to avoid an error message box when this library is not found.
+    // The call is not done by hipSOLVER directly, as it is not thread-safe and
+    // will affect the global state of the program.
+    void* handle = LoadLibraryW(L"cholmod.dll");
+#else
+    void* handle = dlopen("libcholmod.so.5", RTLD_NOW | RTLD_LOCAL);
+    char* err    = dlerror(); // clear errors
+
+    if(!handle)
+    {
+        handle = dlopen("libcholmod.so.4", RTLD_NOW | RTLD_LOCAL);
+        err    = dlerror(); // clear errors
+    }
+
+    if(!handle)
+    {
+        handle = dlopen("libcholmod.so.3", RTLD_NOW | RTLD_LOCAL);
+        err    = dlerror(); // clear errors
+    }
+
+    if(!handle)
+    {
+        handle = dlopen("libcholmod.so", RTLD_NOW | RTLD_LOCAL);
+        err    = dlerror(); // clear errors
+    }
+#ifndef NDEBUG
+    if(!handle)
+        std::cerr << "hipsolver: error loading libcholmod.so: " << err << std::endl;
+#endif
+#endif /* _WIN32 */
+    if(!handle)
+        return false;
+
+    if(!load_function(handle, "cholmod_start", g_cholmod_start))
+        return false;
+    if(!load_function(handle, "cholmod_finish", g_cholmod_finish))
+        return false;
+
+    if(!load_function(handle, "cholmod_allocate_sparse", g_cholmod_allocate_sparse))
+        return false;
+    if(!load_function(handle, "cholmod_free_sparse", g_cholmod_free_sparse))
+        return false;
+    if(!load_function(handle, "cholmod_allocate_dense", g_cholmod_allocate_dense))
+        return false;
+    if(!load_function(handle, "cholmod_free_dense", g_cholmod_free_dense))
+        return false;
+    if(!load_function(handle, "cholmod_free_factor", g_cholmod_free_factor))
+        return false;
+
+    if(!load_function(handle, "cholmod_drop", g_cholmod_drop))
+        return false;
+    if(!load_function(handle, "cholmod_analyze", g_cholmod_analyze))
+        return false;
+    if(!load_function(handle, "cholmod_analyze_ordering", g_cholmod_analyze_ordering))
+        return false;
+    if(!load_function(handle, "cholmod_factorize", g_cholmod_factorize))
+        return false;
+    if(!load_function(handle, "cholmod_solve", g_cholmod_solve))
+        return false;
+
+    return true;
+}
+
+bool try_load_cholmod()
+{
+    // Function-scope static initialization has been thread-safe since C++11.
+    // There is an implicit mutex guarding the initialization.
+    static bool result = load_cholmod();
+    return result;
+}
+
+HIPSOLVER_END_NAMESPACE