From 1c00ac89bd83a762f48ed004020fcef6e7c3c750 Mon Sep 17 00:00:00 2001 From: Caspar Jungbacker Date: Tue, 26 Nov 2024 17:09:19 +0100 Subject: [PATCH 1/6] CMake: remove presets Not really compatible with ecbuild --- .github/workflows/compile.yml | 2 +- .github/workflows/precision.yml | 2 +- .github/workflows/test.yml | 2 +- CMakeLists.txt | 4 +++ CMakePresets.json | 60 ------------------------------- cmake/dales_compile_options.cmake | 17 +++++++++ 6 files changed, 24 insertions(+), 63 deletions(-) delete mode 100644 CMakePresets.json create mode 100644 cmake/dales_compile_options.cmake diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 2c42a554..7d6dc2ae 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -85,7 +85,7 @@ jobs: fi mkdir build cd build - cmake --preset=${{ matrix.toolchain }} -DCMAKE_BUILD_TYPE=DEBUG -DENABLE_HYPRE=${{ matrix.hypre }} ../ + cmake -DCMAKE_BUILD_TYPE=DEBUG -DENABLE_HYPRE=${{ matrix.hypre }} ../ - name : Build run: | diff --git a/.github/workflows/precision.yml b/.github/workflows/precision.yml index e655a70a..d188c62d 100644 --- a/.github/workflows/precision.yml +++ b/.github/workflows/precision.yml @@ -39,7 +39,7 @@ jobs: run: | mkdir build cd build - cmake --preset=gnu \ + cmake \ -DENABLE_HYPRE=${{ matrix.hypre }} \ -DENABLE_FP32_FIELDS=${{ matrix.sp_fields }} \ -DENABLE_FP32_POIS=${{ matrix.sp_pois }} \ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e4864b63..9bc54f72 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,7 @@ jobs: run: | mkdir build cd build - cmake --preset=gnu -DCMAKE_BUILD_TYPE=Debug ../ + cmake -DCMAKE_BUILD_TYPE=Debug ../ - name: Build run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index d0da8efd..005a9f59 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,10 @@ ecbuild_add_option( FEATURE DALESLIB # Check for required dependencies ecbuild_find_package( NetCDF REQUIRED COMPONENTS Fortran ) + ecbuild_find_mpi( COMPONENTS Fortran REQUIRED ) +ecbuild_info( "MPI compiler: ${MPI_Fortran_COMPILER}") +set ( CMAKE_Fortran_COMPILER ${MPI_Fortran_COMPILER} ) # Set precision @@ -74,4 +77,5 @@ if( ENABLE_NVTX ) add_compile_definitions( USE_NVTX ) endif() +include( dales_compile_options ) add_subdirectory( src ) \ No newline at end of file diff --git a/CMakePresets.json b/CMakePresets.json deleted file mode 100644 index f58e3865..00000000 --- a/CMakePresets.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "version": 9, - "cmakeMinimumRequired": { - "major": 3, - "minor": 17, - "patch": 0 - }, - "configurePresets": [ - { - "name": "gnu", - "displayName": "GNU toolchain", - "cacheVariables": { - "CMAKE_Fortran_COMPILER": "mpif90", - "CMAKE_Fortran_FLAGS": "-cpp -W -Wall -Wno-tabs -Wno-compare-reals -fdefault-real-8 -fdefault-double-8 -march=native -ffree-line-length-none -std=gnu -Werror=implicit-interface", - "CMAKE_Fortran_FLAGS_RELEASE": "-funroll-all-loops -fno-f2c -Ofast -g -fbacktrace", - "CMAKE_Fortran_FLAGS_DEBUG": "-finit-real=nan -fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" - } - }, - { - "name": "intel", - "displayName": "Intel toolchain", - "cacheVariables": { - "CMAKE_Fortran_COMPILER": "mpiifx", - "CMAKE_Fortran_FLAGS": "-cpp -r8 -ftz", - "CMAKE_Fortran_FLAGS_RELEASE": "-O3", - "CMAKE_Fortran_FLAGS_DEBUG": "-traceback -fpe1 -O0 -g -check all" - } - }, - { - "name": "nvidia-cpu", - "displayName": "NVIDIA HPC SDK (CPU build)", - "cacheVariables": { - "CMAKE_Fortran_COMPILER": "mpif90", - "CMAKE_Fortran_FLAGS": "-Mpreprocess -Mr8 -Mfree", - "CMAKE_Fortran_FLAGS_RELEASE": "-Ofast", - "CMAKE_Fortran_FLAGS_DEBUG": "-traceback -fpe1 -O0 -g -check all" - } - }, - { - "name": "nvidia-gpu", - "displayName": "NVIDIA HPC SDK (GPU build)", - "cacheVariables": { - "CMAKE_Fortran_COMPILER": "mpif90", - "CMAKE_Fortran_FLAGS": "-Mpreprocess -Mr8 -Mfree -acc=gpu,host -cuda -cudalib=cufft -gpu=cc60,cc70,cc80,cc90,fastmath -Minfo=accel", - "CMAKE_Fortran_FLAGS_RELEASE": "-Ofast", - "CMAKE_Fortran_FLAGS_DEBUG": "-traceback -fpe1 -O0 -g -check all" - } - }, - { - "name": "fujitsu", - "displayName": "Fujitsu FX compiler", - "cacheVariables": { - "CMAKE_Fortran_COMPILER": "mpifrtpx", - "CMAKE_Fortran_FLAGS": "-Cpp -CcdRR8 -g -Koptmsg=2", - "CMAKE_Fortran_FLAGS_RELEASE": "-Kfast -x 128", - "CMAKE_Fortran_FLAGS_DEBUG": "-Haefosux -O0" - } - } - ] -} \ No newline at end of file diff --git a/cmake/dales_compile_options.cmake b/cmake/dales_compile_options.cmake new file mode 100644 index 00000000..750eb7e0 --- /dev/null +++ b/cmake/dales_compile_options.cmake @@ -0,0 +1,17 @@ +if( CMAKE_Fortran_COMPILER_ID MATCHES "GNU" ) + ecbuild_add_fortran_flags( "-cpp -W -Wall -Wno-tabs -Wno-compare-reals -fdefault-real-8 -fdefault-double-8 -march=native -ffree-line-length-none -std=gnu -Werror=implicit-interface" ) + ecbuild_add_fortran_flags( "-funroll-all-loops -fno-f2c -Ofast -g -fbacktrace" BUILD RELEASE ) + ecbuild_add_fortran_flags( "-finit-real=nan -fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" BUILD DEBUG ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Intel" ) + ecbuild_add_fortran_flags( "-cpp -r8 -ftz" ) + ecbuild_add_fortran_flags( "-O3" BUILD RELEASE ) + ecbuild_add_fortran_flags( "-traceback -fpe1 -O0 -g -check all" BUILD DEBUG ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "NVHPC" ) + ecbuild_add_fortran_flags( "-Mpreprocess -Mr8 -Mfree" ) + ecbuild_add_fortran_flags( "-Ofast" BUILD RELEASE ) + ecbuild_add_fortran_flags( "-traceback -fpe1 -O0 -g -check all" BUILD DEBUG ) # These look suspiciously similar to Intel (LLVM?) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Fujitsu" ) + ecbuild_add_fortran_flags( "-Cpp -CcdRR8 -g -Koptmsg=2" ) + ecbuild_add_fortran_flags( "-Kfast -x 128" BUILD RELEASE ) + ecbuild_add_fortran_flags( "-Haefosux -O0" BUILD DEBUG ) +endif() \ No newline at end of file From b5260d74eecd23fac63b66c8df195bc3343622db Mon Sep 17 00:00:00 2001 From: Caspar Jungbacker Date: Tue, 26 Nov 2024 17:39:35 +0100 Subject: [PATCH 2/6] CMake: fix Intel compiler ID --- cmake/dales_compile_options.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/dales_compile_options.cmake b/cmake/dales_compile_options.cmake index 750eb7e0..80a2d905 100644 --- a/cmake/dales_compile_options.cmake +++ b/cmake/dales_compile_options.cmake @@ -14,4 +14,4 @@ elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Fujitsu" ) ecbuild_add_fortran_flags( "-Cpp -CcdRR8 -g -Koptmsg=2" ) ecbuild_add_fortran_flags( "-Kfast -x 128" BUILD RELEASE ) ecbuild_add_fortran_flags( "-Haefosux -O0" BUILD DEBUG ) -endif() \ No newline at end of file +endif() From c54775254bb26a5cd70bd38c3562162a3fa3dd3e Mon Sep 17 00:00:00 2001 From: Caspar Jungbacker Date: Tue, 26 Nov 2024 17:47:05 +0100 Subject: [PATCH 3/6] CI: explicitly point to mpiifx for Intel builds --- .github/workflows/compile.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 7d6dc2ae..5d4eec0f 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -82,6 +82,7 @@ jobs: run: | if [[ -f /opt/intel/oneapi/setvars.sh ]]; then source /opt/intel/oneapi/setvars.sh + export FC=mpiifx fi mkdir build cd build From 655c013a480ddedf74af6681803fa668b5d9e8a6 Mon Sep 17 00:00:00 2001 From: Caspar Jungbacker Date: Tue, 26 Nov 2024 19:00:35 +0100 Subject: [PATCH 4/6] CMake: fix NVHPC debug flags --- cmake/dales_compile_options.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/dales_compile_options.cmake b/cmake/dales_compile_options.cmake index 80a2d905..a7c7459f 100644 --- a/cmake/dales_compile_options.cmake +++ b/cmake/dales_compile_options.cmake @@ -9,7 +9,7 @@ elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Intel" ) elseif( CMAKE_Fortran_COMPILER_ID MATCHES "NVHPC" ) ecbuild_add_fortran_flags( "-Mpreprocess -Mr8 -Mfree" ) ecbuild_add_fortran_flags( "-Ofast" BUILD RELEASE ) - ecbuild_add_fortran_flags( "-traceback -fpe1 -O0 -g -check all" BUILD DEBUG ) # These look suspiciously similar to Intel (LLVM?) + ecbuild_add_fortran_flags( "-Minit-real=snan -traceback -O0 -g -ffpe-trap=invalid,zero,overflow" BUILD DEBUG ) elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Fujitsu" ) ecbuild_add_fortran_flags( "-Cpp -CcdRR8 -g -Koptmsg=2" ) ecbuild_add_fortran_flags( "-Kfast -x 128" BUILD RELEASE ) From 3a93c9e5e8230c98a89fc33a639996f5907e0c3f Mon Sep 17 00:00:00 2001 From: Caspar Jungbacker Date: Wed, 27 Nov 2024 14:46:51 +0100 Subject: [PATCH 5/6] CMake: re-add support for GPU builds --- CMakeLists.txt | 33 ++++++++++++-- cmake/dales_compile_options.cmake | 9 ++-- cmake/dales_find_cuda.cmake | 21 +++++++++ src/CMakeLists.txt | 75 ++++++++++++++++++------------- 4 files changed, 100 insertions(+), 38 deletions(-) create mode 100644 cmake/dales_find_cuda.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 005a9f59..294049ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,9 +33,13 @@ ecbuild_add_option( FEATURE DALESLIB DESCRIPTION "Build DALES libraries for use with OMUSE" DEFAULT OFF ) +ecbuild_add_option( FEATURE ACC + DESCRIPTION "Build with GPU support through OpenACC" + DEFAULT OFF ) + # Check for required dependencies -ecbuild_find_package( NetCDF REQUIRED COMPONENTS Fortran ) +ecbuild_find_package( NetCDF REQUIRED COMPONENTS Fortran C ) ecbuild_find_mpi( COMPONENTS Fortran REQUIRED ) ecbuild_info( "MPI compiler: ${MPI_Fortran_COMPILER}") @@ -73,9 +77,32 @@ if( ENABLE_HYPRE ) endif() if( ENABLE_NVTX ) - ecbuild_add_fortran_flags( -lnvToolsExt ) add_compile_definitions( USE_NVTX ) endif() +# GPU stuff + +include( dales_find_cuda ) + +if( ENABLE_ACC ) + find_package( OpenACC COMPONENTS Fortran QUIET ) + if( OpenACC_Fortran_FOUND ) + set( OpenACC ON ) + ecbuild_info( "Found OpenACC" ) + endif() + + # For debug builds, enable accelerator info + if( ${CMAKE_Fortran_COMPILER_ID} MATCHES PGI|NVHPC ) + ecbuild_add_fortran_flags( "-Minfo=accel" BUILD DEBUG ) + endif() + + add_compile_definitions( DALES_GPU ) +endif() + +# For GPU builds and/or NVTX profiling, we need the CUDA toolkit +if( ENABLE_ACC OR ENABLE_NVTX ) + dales_find_cuda() +endif() + include( dales_compile_options ) -add_subdirectory( src ) \ No newline at end of file +add_subdirectory( src ) diff --git a/cmake/dales_compile_options.cmake b/cmake/dales_compile_options.cmake index a7c7459f..b04429df 100644 --- a/cmake/dales_compile_options.cmake +++ b/cmake/dales_compile_options.cmake @@ -1,16 +1,17 @@ -if( CMAKE_Fortran_COMPILER_ID MATCHES "GNU" ) +# Compiler flag presets for some frequently used compilers +if( CMAKE_Fortran_COMPILER_ID MATCHES GNU ) ecbuild_add_fortran_flags( "-cpp -W -Wall -Wno-tabs -Wno-compare-reals -fdefault-real-8 -fdefault-double-8 -march=native -ffree-line-length-none -std=gnu -Werror=implicit-interface" ) ecbuild_add_fortran_flags( "-funroll-all-loops -fno-f2c -Ofast -g -fbacktrace" BUILD RELEASE ) ecbuild_add_fortran_flags( "-finit-real=nan -fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" BUILD DEBUG ) -elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Intel" ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES Intel ) ecbuild_add_fortran_flags( "-cpp -r8 -ftz" ) ecbuild_add_fortran_flags( "-O3" BUILD RELEASE ) ecbuild_add_fortran_flags( "-traceback -fpe1 -O0 -g -check all" BUILD DEBUG ) -elseif( CMAKE_Fortran_COMPILER_ID MATCHES "NVHPC" ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC ) ecbuild_add_fortran_flags( "-Mpreprocess -Mr8 -Mfree" ) ecbuild_add_fortran_flags( "-Ofast" BUILD RELEASE ) ecbuild_add_fortran_flags( "-Minit-real=snan -traceback -O0 -g -ffpe-trap=invalid,zero,overflow" BUILD DEBUG ) -elseif( CMAKE_Fortran_COMPILER_ID MATCHES "Fujitsu" ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES Fujitsu ) ecbuild_add_fortran_flags( "-Cpp -CcdRR8 -g -Koptmsg=2" ) ecbuild_add_fortran_flags( "-Kfast -x 128" BUILD RELEASE ) ecbuild_add_fortran_flags( "-Haefosux -O0" BUILD DEBUG ) diff --git a/cmake/dales_find_cuda.cmake b/cmake/dales_find_cuda.cmake new file mode 100644 index 00000000..2a2327c1 --- /dev/null +++ b/cmake/dales_find_cuda.cmake @@ -0,0 +1,21 @@ +macro( dales_find_cuda ) + # Check if a target architecture has been set. If not, default to 80 (A100 & RTX3090) + # For H100, use 90 + if( NOT DEFINED CMAKE_CUDA_ARCHITECTURES ) + set( CMAKE_CUDA_ARCHITECTURES 80 ) + endif() + + # Look for the CUDA Toolkit, which has cuFFT and NVTX + find_package( CUDAToolkit ) + if( NOT TARGET CUDA::cufft AND ENABLE_ACC ) + ecbuild_error( "Could not find cuFFT, which is required for GPU builds!" ) + else() + ecbuild_info( "Found cuFFT: ${CUDA_cufft_LIBRARY}" ) + endif() + if( NOT TARGET CUDA::nvToolsExt AND ENABLE_NVTX ) + ecbuild_error( "Could not find NVTX library for profiling" ) + else() + ecbuild_info( "Found NVTX: ${CUDA_nvToolsExt_LIBRARY}" ) + endif() +endmacro() + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4a75e252..0d8081cb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -219,39 +219,52 @@ add_custom_target( tag_git_version ALL BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90 ) +# Temporary, because we use CUDA-Fortran in these places +if( ENABLE_ACC ) + set_source_files_properties( modmpi.f90 modgpumpiinterface.f90 PROPERTIES COMPILE_OPTIONS -cuda ) +endif() + # Stand-alone DALES program ecbuild_add_executable( TARGET dales - SOURCES - ${dales_srcs} - ${rrtmg_lw_srcs} - ${rrtmg_sw_srcs} - ${rrtmgp_srcs} - ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90 - DEPENDS tag_git_version - INCLUDES - ${NETCDF_INCLUDE_DIR} - $<$:${FFTW_INCLUDE_DIRS}> - LIBS - NetCDF::NetCDF_Fortran - $<$:FFTW::fftw3> - $<$:FFTW::fftw3f> - $<$:${HYPRE_LIB}> ) + SOURCES + ${dales_srcs} + ${rrtmg_lw_srcs} + ${rrtmg_sw_srcs} + ${rrtmgp_srcs} + ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90 + DEPENDS tag_git_version + INCLUDES + ${NETCDF_INCLUDE_DIR} + $<$:${FFTW_INCLUDE_DIRS}> + LIBS + NetCDF::NetCDF_Fortran + NetCDF::NetCDF_C + $<$:OpenACC::OpenACC_Fortran> + $<$:FFTW::fftw3> + $<$:FFTW::fftw3f> + $<$:${HYPRE_LIB}> + $<$:CUDA::cufft> + $<$:CUDA::nvToolsExt> +) +# Temporary +target_link_options( dales PUBLIC -cuda ) # DALES library, e.g. for use with OMUSE. Only built if -DENABLE_DALESLIB is turned on. ecbuild_add_library( TARGET libdales - SOURCES - ${dales_srcs} - ${rrtmg_lw_srcs} - ${rrtmg_sw_srcs} - ${rrtmgp_srcs} - ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90 - DEPENDS tag_git_version - PUBLIC_INCLUDES - ${NETCDF_INCLUDE_DIR} - $<$:${FFTW_INCLUDE_DIRS}> - PUBLIC_LIBS - NetCDF::NetCDF_Fortran - $<$:FFTW::fftw3> - $<$:FFTW::fftw3f> - $<$:${HYPRE_LIB}> - CONDITION ENABLE_DALESLIB ) \ No newline at end of file + SOURCES + ${dales_srcs} + ${rrtmg_lw_srcs} + ${rrtmg_sw_srcs} + ${rrtmgp_srcs} + ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90 + DEPENDS tag_git_version + PUBLIC_INCLUDES + ${NETCDF_INCLUDE_DIR} + $<$:${FFTW_INCLUDE_DIRS}> + PUBLIC_LIBS + NetCDF::NetCDF_Fortran + $<$:FFTW::fftw3> + $<$:FFTW::fftw3f> + $<$:${HYPRE_LIB}> + CONDITION ENABLE_DALESLIB +) From 5ebe7498e81c16df8d2256c40d893161b893d342 Mon Sep 17 00:00:00 2001 From: Caspar Jungbacker Date: Thu, 28 Nov 2024 10:39:05 +0100 Subject: [PATCH 6/6] CMake: only enable link option -cuda when using ACC --- src/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0d8081cb..ad783f46 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -247,7 +247,9 @@ ecbuild_add_executable( TARGET dales $<$:CUDA::nvToolsExt> ) # Temporary -target_link_options( dales PUBLIC -cuda ) +if( ENABLE_ACC ) + target_link_options( dales PUBLIC -cuda ) +endif() # DALES library, e.g. for use with OMUSE. Only built if -DENABLE_DALESLIB is turned on. ecbuild_add_library( TARGET libdales