diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 2c42a554..5d4eec0f 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -82,10 +82,11 @@ jobs: run: | if [[ -f /opt/intel/oneapi/setvars.sh ]]; then source /opt/intel/oneapi/setvars.sh + export FC=mpiifx fi mkdir build cd build - cmake --preset=${{ matrix.toolchain }} -DCMAKE_BUILD_TYPE=DEBUG -DENABLE_HYPRE=${{ matrix.hypre }} ../ + cmake -DCMAKE_BUILD_TYPE=DEBUG -DENABLE_HYPRE=${{ matrix.hypre }} ../ - name : Build run: | diff --git a/.github/workflows/precision.yml b/.github/workflows/precision.yml index e655a70a..d188c62d 100644 --- a/.github/workflows/precision.yml +++ b/.github/workflows/precision.yml @@ -39,7 +39,7 @@ jobs: run: | mkdir build cd build - cmake --preset=gnu \ + cmake \ -DENABLE_HYPRE=${{ matrix.hypre }} \ -DENABLE_FP32_FIELDS=${{ matrix.sp_fields }} \ -DENABLE_FP32_POIS=${{ matrix.sp_pois }} \ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e4864b63..9bc54f72 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,7 @@ jobs: run: | mkdir build cd build - cmake --preset=gnu -DCMAKE_BUILD_TYPE=Debug ../ + cmake -DCMAKE_BUILD_TYPE=Debug ../ - name: Build run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index d0da8efd..294049ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,10 +33,17 @@ ecbuild_add_option( FEATURE DALESLIB DESCRIPTION "Build DALES libraries for use with OMUSE" DEFAULT OFF ) +ecbuild_add_option( FEATURE ACC + DESCRIPTION "Build with GPU support through OpenACC" + DEFAULT OFF ) + # Check for required dependencies -ecbuild_find_package( NetCDF REQUIRED COMPONENTS Fortran ) +ecbuild_find_package( NetCDF REQUIRED COMPONENTS Fortran C ) + ecbuild_find_mpi( COMPONENTS Fortran REQUIRED ) +ecbuild_info( "MPI compiler: ${MPI_Fortran_COMPILER}") +set ( CMAKE_Fortran_COMPILER ${MPI_Fortran_COMPILER} ) # Set precision @@ -70,8 +77,32 @@ if( ENABLE_HYPRE ) endif() if( ENABLE_NVTX ) - ecbuild_add_fortran_flags( -lnvToolsExt ) add_compile_definitions( USE_NVTX ) endif() -add_subdirectory( src ) \ No newline at end of file +# GPU stuff + +include( dales_find_cuda ) + +if( ENABLE_ACC ) + find_package( OpenACC COMPONENTS Fortran QUIET ) + if( OpenACC_Fortran_FOUND ) + set( OpenACC ON ) + ecbuild_info( "Found OpenACC" ) + endif() + + # For debug builds, enable accelerator info + if( ${CMAKE_Fortran_COMPILER_ID} MATCHES PGI|NVHPC ) + ecbuild_add_fortran_flags( "-Minfo=accel" BUILD DEBUG ) + endif() + + add_compile_definitions( DALES_GPU ) +endif() + +# For GPU builds and/or NVTX profiling, we need the CUDA toolkit +if( ENABLE_ACC OR ENABLE_NVTX ) + dales_find_cuda() +endif() + +include( dales_compile_options ) +add_subdirectory( src ) diff --git a/CMakePresets.json b/CMakePresets.json deleted file mode 100644 index f58e3865..00000000 --- a/CMakePresets.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "version": 9, - "cmakeMinimumRequired": { - "major": 3, - "minor": 17, - "patch": 0 - }, - "configurePresets": [ - { - "name": "gnu", - "displayName": "GNU toolchain", - "cacheVariables": { - "CMAKE_Fortran_COMPILER": "mpif90", - "CMAKE_Fortran_FLAGS": "-cpp -W -Wall -Wno-tabs -Wno-compare-reals -fdefault-real-8 -fdefault-double-8 -march=native -ffree-line-length-none -std=gnu -Werror=implicit-interface", - "CMAKE_Fortran_FLAGS_RELEASE": "-funroll-all-loops -fno-f2c -Ofast -g -fbacktrace", - "CMAKE_Fortran_FLAGS_DEBUG": "-finit-real=nan -fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" - } - }, - { - "name": "intel", - "displayName": "Intel toolchain", - "cacheVariables": { - "CMAKE_Fortran_COMPILER": "mpiifx", - "CMAKE_Fortran_FLAGS": "-cpp -r8 -ftz", - "CMAKE_Fortran_FLAGS_RELEASE": "-O3", - "CMAKE_Fortran_FLAGS_DEBUG": "-traceback -fpe1 -O0 -g -check all" - } - }, - { - "name": "nvidia-cpu", - "displayName": "NVIDIA HPC SDK (CPU build)", - "cacheVariables": { - "CMAKE_Fortran_COMPILER": "mpif90", - "CMAKE_Fortran_FLAGS": "-Mpreprocess -Mr8 -Mfree", - "CMAKE_Fortran_FLAGS_RELEASE": "-Ofast", - "CMAKE_Fortran_FLAGS_DEBUG": "-traceback -fpe1 -O0 -g -check all" - } - }, - { - "name": "nvidia-gpu", - "displayName": "NVIDIA HPC SDK (GPU build)", - "cacheVariables": { - "CMAKE_Fortran_COMPILER": "mpif90", - "CMAKE_Fortran_FLAGS": "-Mpreprocess -Mr8 -Mfree -acc=gpu,host -cuda -cudalib=cufft -gpu=cc60,cc70,cc80,cc90,fastmath -Minfo=accel", - "CMAKE_Fortran_FLAGS_RELEASE": "-Ofast", - "CMAKE_Fortran_FLAGS_DEBUG": "-traceback -fpe1 -O0 -g -check all" - } - }, - { - "name": "fujitsu", - "displayName": "Fujitsu FX compiler", - "cacheVariables": { - "CMAKE_Fortran_COMPILER": "mpifrtpx", - "CMAKE_Fortran_FLAGS": "-Cpp -CcdRR8 -g -Koptmsg=2", - "CMAKE_Fortran_FLAGS_RELEASE": "-Kfast -x 128", - "CMAKE_Fortran_FLAGS_DEBUG": "-Haefosux -O0" - } - } - ] -} \ No newline at end of file diff --git a/cmake/dales_compile_options.cmake b/cmake/dales_compile_options.cmake new file mode 100644 index 00000000..b04429df --- /dev/null +++ b/cmake/dales_compile_options.cmake @@ -0,0 +1,18 @@ +# Compiler flag presets for some frequently used compilers +if( CMAKE_Fortran_COMPILER_ID MATCHES GNU ) + ecbuild_add_fortran_flags( "-cpp -W -Wall -Wno-tabs -Wno-compare-reals -fdefault-real-8 -fdefault-double-8 -march=native -ffree-line-length-none -std=gnu -Werror=implicit-interface" ) + ecbuild_add_fortran_flags( "-funroll-all-loops -fno-f2c -Ofast -g -fbacktrace" BUILD RELEASE ) + ecbuild_add_fortran_flags( "-finit-real=nan -fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" BUILD DEBUG ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES Intel ) + ecbuild_add_fortran_flags( "-cpp -r8 -ftz" ) + ecbuild_add_fortran_flags( "-O3" BUILD RELEASE ) + ecbuild_add_fortran_flags( "-traceback -fpe1 -O0 -g -check all" BUILD DEBUG ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC ) + ecbuild_add_fortran_flags( "-Mpreprocess -Mr8 -Mfree" ) + ecbuild_add_fortran_flags( "-Ofast" BUILD RELEASE ) + ecbuild_add_fortran_flags( "-Minit-real=snan -traceback -O0 -g -ffpe-trap=invalid,zero,overflow" BUILD DEBUG ) +elseif( CMAKE_Fortran_COMPILER_ID MATCHES Fujitsu ) + ecbuild_add_fortran_flags( "-Cpp -CcdRR8 -g -Koptmsg=2" ) + ecbuild_add_fortran_flags( "-Kfast -x 128" BUILD RELEASE ) + ecbuild_add_fortran_flags( "-Haefosux -O0" BUILD DEBUG ) +endif() diff --git a/cmake/dales_find_cuda.cmake b/cmake/dales_find_cuda.cmake new file mode 100644 index 00000000..2a2327c1 --- /dev/null +++ b/cmake/dales_find_cuda.cmake @@ -0,0 +1,21 @@ +macro( dales_find_cuda ) + # Check if a target architecture has been set. If not, default to 80 (A100 & RTX3090) + # For H100, use 90 + if( NOT DEFINED CMAKE_CUDA_ARCHITECTURES ) + set( CMAKE_CUDA_ARCHITECTURES 80 ) + endif() + + # Look for the CUDA Toolkit, which has cuFFT and NVTX + find_package( CUDAToolkit ) + if( NOT TARGET CUDA::cufft AND ENABLE_ACC ) + ecbuild_error( "Could not find cuFFT, which is required for GPU builds!" ) + else() + ecbuild_info( "Found cuFFT: ${CUDA_cufft_LIBRARY}" ) + endif() + if( NOT TARGET CUDA::nvToolsExt AND ENABLE_NVTX ) + ecbuild_error( "Could not find NVTX library for profiling" ) + else() + ecbuild_info( "Found NVTX: ${CUDA_nvToolsExt_LIBRARY}" ) + endif() +endmacro() + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4a75e252..ad783f46 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -219,39 +219,54 @@ add_custom_target( tag_git_version ALL BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90 ) +# Temporary, because we use CUDA-Fortran in these places +if( ENABLE_ACC ) + set_source_files_properties( modmpi.f90 modgpumpiinterface.f90 PROPERTIES COMPILE_OPTIONS -cuda ) +endif() + # Stand-alone DALES program ecbuild_add_executable( TARGET dales - SOURCES - ${dales_srcs} - ${rrtmg_lw_srcs} - ${rrtmg_sw_srcs} - ${rrtmgp_srcs} - ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90 - DEPENDS tag_git_version - INCLUDES - ${NETCDF_INCLUDE_DIR} - $<$:${FFTW_INCLUDE_DIRS}> - LIBS - NetCDF::NetCDF_Fortran - $<$:FFTW::fftw3> - $<$:FFTW::fftw3f> - $<$:${HYPRE_LIB}> ) + SOURCES + ${dales_srcs} + ${rrtmg_lw_srcs} + ${rrtmg_sw_srcs} + ${rrtmgp_srcs} + ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90 + DEPENDS tag_git_version + INCLUDES + ${NETCDF_INCLUDE_DIR} + $<$:${FFTW_INCLUDE_DIRS}> + LIBS + NetCDF::NetCDF_Fortran + NetCDF::NetCDF_C + $<$:OpenACC::OpenACC_Fortran> + $<$:FFTW::fftw3> + $<$:FFTW::fftw3f> + $<$:${HYPRE_LIB}> + $<$:CUDA::cufft> + $<$:CUDA::nvToolsExt> +) +# Temporary +if( ENABLE_ACC ) + target_link_options( dales PUBLIC -cuda ) +endif() # DALES library, e.g. for use with OMUSE. Only built if -DENABLE_DALESLIB is turned on. ecbuild_add_library( TARGET libdales - SOURCES - ${dales_srcs} - ${rrtmg_lw_srcs} - ${rrtmg_sw_srcs} - ${rrtmgp_srcs} - ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90 - DEPENDS tag_git_version - PUBLIC_INCLUDES - ${NETCDF_INCLUDE_DIR} - $<$:${FFTW_INCLUDE_DIRS}> - PUBLIC_LIBS - NetCDF::NetCDF_Fortran - $<$:FFTW::fftw3> - $<$:FFTW::fftw3f> - $<$:${HYPRE_LIB}> - CONDITION ENABLE_DALESLIB ) \ No newline at end of file + SOURCES + ${dales_srcs} + ${rrtmg_lw_srcs} + ${rrtmg_sw_srcs} + ${rrtmgp_srcs} + ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90 + DEPENDS tag_git_version + PUBLIC_INCLUDES + ${NETCDF_INCLUDE_DIR} + $<$:${FFTW_INCLUDE_DIRS}> + PUBLIC_LIBS + NetCDF::NetCDF_Fortran + $<$:FFTW::fftw3> + $<$:FFTW::fftw3f> + $<$:${HYPRE_LIB}> + CONDITION ENABLE_DALESLIB +)