Use CudaRdcUtils module to build/link/use AdePT

Due to VecGeom's legacy CUDA design and the exposure of device functions in public APIs of libraries, ensuring correct singular device links in final executables is awkward. The Celeritas project has evolved the build/link pattern used in VecGeom into the "CudaRdcUtils" CMake module that enables projects using VecGeom (or building libraries that expose device APIs) to be correctly integrated and used by downstream clients. This is acheived through the use of custom CMake properties and wrappers around the fundamental target_XXX interfaces CMake provides, making use as transparent as it can be given the problem faced. Use CudaRdcUtils to build, link, install, and use AdePT - Provide FindVecGeom wrapper module to add RDC properties to VecGeom targets - Replace target_XXX calls with cuda_rdc_XXX equivalents - Remove obsolete "AdePT_cuda" and "AdePT_standalone" libraries that CudaRdcUtils build and handles automatically - Install and use FindVecGeom and CudaRdcUtils for use by downstream clients Tests and executables build and run successfully on a local Rocky 9, GCC11, CUDA 12 environment.
apt-sim · Feb 29, 2024 · c699300 · c699300
1 parent f0a1c30
commit c699300
Show file tree

Hide file tree

Showing 8 changed files with 79 additions and 133 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -138,13 +138,6 @@ endif()
 #----------------------------------------------------------------------------#
 # Build Targets
 #----------------------------------------------------------------------------#
-set(ADEPT_G4_INTEGRATION_SRCS
-  src/AdePTTrackingManager.cc
-  src/AdePTPhysics.cc
-  src/HepEMPhysics.cc
-  src/AdePTGeant4Integration.cpp
-)
-
 set(ADEPT_SRCS
   src/AdePTTransport.cpp
   src/AdePTConfigurationMessenger.cc
@@ -155,88 +148,62 @@ set(ADEPT_CUDA_SRCS
   src/HostScoring.cu
 )
 
-add_library(CopCore INTERFACE)
-target_include_directories(CopCore
+set(ADEPT_G4_INTEGRATION_SRCS
+  src/AdePTTrackingManager.cc
+  src/AdePTPhysics.cc
+  src/HepEMPhysics.cc
+  src/AdePTGeant4Integration.cpp
+)
+
+# - Build RDC/Non-RDC libraries
+# NB: The ALIAS call after the main library MUST be here to make 
+# RDC-enabled libraries to work when installed with a NAMESPACE.
+
+# - CopCore
+cuda_rdc_add_library(CopCore INTERFACE)
+cuda_rdc_add_library(AdePT::CopCore ALIAS CopCore)
+cuda_rdc_target_include_directories(CopCore
   INTERFACE
     $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include/AdePT/copcore/>
     $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}>
 )
 
-add_library(AdePT_G4_integration SHARED ${ADEPT_G4_INTEGRATION_SRCS})
-target_include_directories(AdePT_G4_integration 
-  PUBLIC 
+# - AdePT
+cuda_rdc_add_library(AdePT SHARED ${ADEPT_SRCS} ${ADEPT_CUDA_SRCS})
+cuda_rdc_add_library(AdePT::AdePT ALIAS AdePT)
+cuda_rdc_target_include_directories(AdePT 
+  PUBLIC
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-    $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}>
+    $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
 )
-target_link_libraries(AdePT_G4_integration 
+cuda_rdc_target_link_libraries(AdePT 
   PUBLIC 
     CopCore
     VecGeom::vecgeom
     VecGeom::vgdml
-    ${Geant4_LIBRARIES}
     G4HepEm::g4HepEm
     G4HepEm::g4HepEmData
 )
 
-add_library(AdePT SHARED ${ADEPT_SRCS})
-target_include_directories(AdePT 
-  PUBLIC
+# - AdePT_G4_integration
+cuda_rdc_add_library(AdePT_G4_integration SHARED ${ADEPT_G4_INTEGRATION_SRCS})
+cuda_rdc_add_library(AdePT::AdePT_G4_integration ALIAS AdePT_G4_integration)
+cuda_rdc_target_include_directories(AdePT_G4_integration 
+  PUBLIC 
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-    $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}>
+    $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
 )
-target_link_libraries(AdePT 
+cuda_rdc_target_link_libraries(AdePT_G4_integration 
   PUBLIC 
+    AdePT
     CopCore
     VecGeom::vecgeom
     VecGeom::vgdml
+    ${Geant4_LIBRARIES}
     G4HepEm::g4HepEm
     G4HepEm::g4HepEmData
 )
 
-add_library(AdePT_cuda SHARED ${ADEPT_CUDA_SRCS})
-target_include_directories(AdePT_cuda 
-  PUBLIC 
-    ${Geant4_INCLUDE_DIRS}
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-    $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}>
-)
-target_link_libraries(AdePT_cuda 
-  PUBLIC
-    VecGeom::vecgeomcuda_static
-    CopCore
-    G4HepEm::g4HepEmInit
-    G4HepEm::g4HepEmRun
-)
-set_target_properties(AdePT_cuda
-  PROPERTIES 
-    CUDA_SEPARABLE_COMPILATION ON 
-    CUDA_RESOLVE_DEVICE_SYMBOLS ON
-)
-
-# AdePT_cuda links against vecgeomcuda_static. This is fine as long as the final executable doesn't link with 
-# vecgeomcuda itself, as is the case with Geant4 integration examples. However for standalone applications not 
-# using geant4, that may need features from vecgeomcuda, this doesn't work. In that case AdePT_cuda has to link 
-# to the shared version of vecgeomcuda, and the final executable to vecgeomcuda_static
-add_library(AdePT_cuda_standalone SHARED ${ADEPT_CUDA_SRCS})
-target_include_directories(AdePT_cuda_standalone 
-  PUBLIC 
-    ${Geant4_INCLUDE_DIRS}
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-    $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}>
-)
-target_link_libraries(AdePT_cuda_standalone
-  PUBLIC
-    VecGeom::vecgeomcuda
-    CopCore
-    G4HepEm::g4HepEmInit
-    G4HepEm::g4HepEmRun
-)
-set_target_properties(AdePT_cuda_standalone
-  PROPERTIES 
-    CUDA_SEPARABLE_COMPILATION ON
-    CUDA_RESOLVE_DEVICE_SYMBOLS ON
-)
-
 # Optional library to activate NVTX annotations for profiling:
 option(NVTX OFF "Add NVTX instrumentation for profiling (only for annotated examples)")
 add_library(NVTX INTERFACE)
@@ -262,7 +229,7 @@ configure_package_config_file(cmake/${PROJECT_NAME}Config.cmake.in
 )
 
 #Install the libraries
-install(TARGETS CopCore AdePT AdePT_cuda AdePT_G4_integration AdePT_cuda_standalone 
+cuda_rdc_install(TARGETS CopCore AdePT AdePT_G4_integration
   EXPORT ${PROJECT_NAME}Targets
   ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
   LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
@@ -274,16 +241,13 @@ install(DIRECTORY include/AdePT
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
 )
 
-#Install the configuration file
-install(FILES "${PROJECT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/AdePTConfig.cmake" 
+#Install the configuration files and needed modules for downstream
+install(FILES "${PROJECT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/AdePTConfig.cmake"
+  "${PROJECT_SOURCE_DIR}/cmake/CudaRdcUtils.cmake"
+  "${PROJECT_SOURCE_DIR}/cmake/FindVecGeom.cmake"
   DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}
 )
 
-#Export the targets file
-export(TARGETS CopCore AdePT_G4_integration AdePT AdePT_cuda AdePT_cuda_standalone
-  FILE "${PROJECT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${PROJECT_NAME}Targets.cmake"
-)
-
 #Install the targets file
 install(EXPORT ${PROJECT_NAME}Targets
   NAMESPACE ${PROJECT_NAME}::

diff --git a/README.md b/README.md
@@ -109,35 +109,33 @@ $ CUDA_VISIBLE_DEVICES=0 BuildProducts/bin/<executable>   ### use the device num
 
 In order to include AdePT in a separate project we need to run:
 
-```
+```cmake
 find_package(AdePT)
 ```
 
 Which has the same dependencies as before (VecGeom, VecCore and G4HepEM).
 
-Then, for the targets using AdePT:
+Due to the design of the CUDA interface in the VecGeom dependency, linking to
+AdePT whether or not your project uses CUDA code requires use of CMake wrapper
+functions to ensure the correct device linking. For an executable target:
 
-```
-target_include_directories(example_target <SCOPE> 
-                          <TARGET INCLUDE DIRECTORIES>
-                          ${AdePT_INCLUDE_DIRS})
+```cmake
+include(CudaRdcUtils)
 
-target_link_libraries(example_target <SCOPE>
-                      <TARGET LINK LIBRARIES>
-                      ${AdePT_LIBRARIES})
+add_executable(MyExe MyExe.cc)
+cuda_rdc_target_link_libraries(MyExe AdePT::AdePT)
 ```
 
-### Targets dependent on vecgeomcuda
+and for a library plus executable:
 
-When using AdePT_LIBRARIES we are linking against the library AdePT::AdePT_cuda, which in turn links to the static library VecGeom::vecgeomcuda_static. For most purposes this will be fine, however, in case the final executable uses some feature from vecgeomcuda and needs to link against it, this aproach will not work.
+```cmake
+include(CudaRdcUtils)
 
-In order to address this we provide AdePT::AdePT_cuda_standalone, which links with the shared VecGeom::vecgeomcuda. This allows the final executable to correctly link with VecGeom::vecgeomcuda_static. This library is provided as part of AdePT_LIBRARIES_EXTRA. 
+cuda_rdc_add_library(MyLib ...)
+cuda_rdc_target_link_libraries(MyLib AdePT::AdePT)
 
-```
-target_link_libraries(example_target <SCOPE>
-                      <TARGET LINK LIBRARIES>
-                      VecGeom::vecgeomcuda_static
-                      ${AdePT_LIBRARIES_EXTRA})
+add_executable(MyExe MyExe.cc)
+cuda_rdc_target_link_libraries(MyExe MyLib)
 ```
 
 ## Copyright

diff --git a/cmake/AdePTConfig.cmake.in b/cmake/AdePTConfig.cmake.in
@@ -7,21 +7,17 @@
 set(AdePT_INCLUDE_DIRS "@PACKAGE_CMAKE_INSTALL_INCLUDEDIR@")
 set_and_check(AdePT_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}")
 
-set(AdePT_LIBRARIES
-    AdePT::CopCore
-    AdePT::AdePT_G4_integration
-    AdePT::AdePT
-    AdePT::AdePT_cuda)
+# So that we find VecGeom and set appropriate RDC flags, we need
+# to use our own wrapper module, but only here.
+list(PREPEND CMAKE_MODULE_PATH "${AdePT_CMAKE_DIR}")
 
-set(AdePT_LIBRARIES_EXTRA
+set(AdePT_LIBRARIES
     AdePT::CopCore
     AdePT::AdePT_G4_integration
-    AdePT::AdePT
-    AdePT::AdePT_cuda_standalone)
+    AdePT::AdePT)
 
 # Find required dependencies
-
-SET(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags")
+set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags")
 
 include(CMakeFindDependencyMacro)
 
@@ -41,4 +37,8 @@ if(G4HepEm_FOUND)
 endif()
 
 # Include the targets file
-include("${AdePT_CMAKE_DIR}/AdePTTargets.cmake")
+include("${AdePT_CMAKE_DIR}/AdePTTargets.cmake")
+
+# Restore CMAKE_MODULE_PATH, with ours at the back
+list(REMOVE_ITEM CMAKE_MODULE_PATH "${AdePT_CMAKE_DIR}")
+list(APPEND CMAKE_MODULE_PATH "${AdePT_CMAKE_DIR}")
diff --git a/examples/Example1/CMakeLists.txt b/examples/Example1/CMakeLists.txt
@@ -55,11 +55,10 @@ target_include_directories(example1
     ${PROJECT_SOURCE_DIR}/examples/common/include
     ${HEPMC3_INCLUDE_DIR}
 )
-target_link_libraries(example1
+cuda_rdc_target_link_libraries(example1
   PRIVATE
     AdePT_G4_integration
     AdePT
-    AdePT_cuda
     ${HEPMC3_LIBRARIES} 
     ${HEPMC3_FIO_LIBRARIES}
 )

diff --git a/examples/IntegrationBenchmark/CMakeLists.txt b/examples/IntegrationBenchmark/CMakeLists.txt
@@ -56,11 +56,10 @@ target_include_directories(integrationBenchmark
     ${PROJECT_SOURCE_DIR}/examples/common/include
     ${HEPMC3_INCLUDE_DIR}
 )
-target_link_libraries(integrationBenchmark
+cuda_rdc_target_link_libraries(integrationBenchmark
   PRIVATE
     AdePT_G4_integration
     AdePT
-    AdePT_cuda
     ${HEPMC3_LIBRARIES} 
     ${HEPMC3_FIO_LIBRARIES}
 )

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -8,8 +8,8 @@ macro(build_tests TESTS)
   foreach(TEST ${TESTS})
     get_filename_component(TARGET_NAME ${TEST} NAME_WE)
     add_executable(${TARGET_NAME} ${TEST})
-    set_target_properties(${TARGET_NAME} PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
-    target_link_libraries(${TARGET_NAME} AdePT_cuda AdePT_G4_integration AdePT)
+    # NB: We do not use PRIVATE for executables due to an apparent limitation in CudaRDCUtils
+    cuda_rdc_target_link_libraries(${TARGET_NAME} AdePT_G4_integration AdePT)
    endforeach()
 endmacro()
 

diff --git a/test/TestEm3/CMakeLists.txt b/test/TestEm3/CMakeLists.txt
@@ -8,19 +8,12 @@ endif()
 
 # TestEm3 for validation of particle transportation with GPUs.
 add_executable(TestEm3 TestEm3.cu electrons.cu gammas.cu TestEm3.cpp)
-target_include_directories(TestEm3 PRIVATE ${PROJECT_SOURCE_DIR}/include)
-target_link_libraries(TestEm3 
-  PRIVATE 
-    CopCore 
-    AdePT_G4_integration
-    AdePT 
-    AdePT_cuda_standalone 
-    VecGeom::vecgeomcuda_static
-)
-set_target_properties(TestEm3 
-  PROPERTIES 
-    CUDA_SEPARABLE_COMPILATION ON 
-    CUDA_RESOLVE_DEVICE_SYMBOLS ON
+cuda_rdc_target_include_directories(TestEm3 PRIVATE ${PROJECT_SOURCE_DIR}/include)
+    # NB: We do not use PRIVATE for executables due to an apparent limitation in CudaRDCUtils
+cuda_rdc_target_link_libraries(TestEm3 
+  CopCore 
+  G4HepEm::g4HepEm
+  VecGeom::vecgeom
 )
 set(CMAKE_EXE_LINKER_FLAGS "-Wl,--disable-new-dtags")
 

diff --git a/test/testField/CMakeLists.txt b/test/testField/CMakeLists.txt
@@ -29,20 +29,13 @@ target_include_directories(testField
     $<BUILD_INTERFACE:${Geant4_INCLUDE_DIRS}>
     $<BUILD_INTERFACE:${G4HepEm_INCLUDE_DIR}>
 )
-target_link_libraries(testField
-  PRIVATE
-    CopCore
-    VecGeom::vecgeom
-    VecGeom::vecgeomcuda_static
-    VecGeom::vgdml
-    ${Geant4_LIBRARIES}
-    ${G4HepEm_LIBRARIES}
-    CUDA::cudart
-)
-set_target_properties(testField 
-  PROPERTIES 
-    CUDA_SEPARABLE_COMPILATION ON 
-    CUDA_RESOLVE_DEVICE_SYMBOLS ON
+    # NB: We do not use PRIVATE for executables due to an apparent limitation in CudaRDCUtils
+cuda_rdc_target_link_libraries(testField
+  CopCore
+  VecGeom::vecgeom
+  VecGeom::vgdml
+  ${Geant4_LIBRARIES}
+  ${G4HepEm_LIBRARIES}
 )
 
 # Tests