Merge branch 'bartgol/e3sm/kokkos-4.2' into master (PR #6101)

This PR will take time to integrate, I'm opening it so I can keep track of what I check.

e3sm_integration:
  chrysalis (intel): all PASS
  pm-cpu (intel) 127 PASS and 1 DIFF: SMS_Ld2.ne30pg2_r05_IcoswISC30E3r5.BGCEXP_CNTL_CNPECACNT_1850.pm-cpu_intel.elm-bgcexp (which is currently consistently failing in the  e3sm_integration_next_intel nightly build)
e3sm_developer:
  pm-cpu (gnu): 75 PASS and 1 DIFF: ERP_Ld3.ne4pg2_oQU480.F2010.pm-cpu_gnu
homme_integration:
 chrysalis (intel): PASS
 pm-cpu (gnu): builds PASS, stuck in Q for run, so I cancelled it. pm-cpu is not tested in nightlies anyways
eamxx testing (from eamxx repo, with a few additional commits for eamxx)
  v1 (CIME)
    chrysalis (intel): 10 PASS (all scream v1) 5 DIFF (all scream v0)
    pm-cpu (gnu): 3 PASS, 1 DIFF
    frontier PEND
    ascent: no longer part of eamxx nightlies
    pm-gpu (gnugpu): 7 PASS, 5 DIFF. All DIFF are in debug mode, while all non-debug builds pass. I'm trying to understand what's the catch.
standalone
  mappy (gnu): all PASS
  weaver (gnu+cuda): all PASS

[BFB]

Author: jgfouca

cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake

@@ -34,5 +34,5 @@ if (COMP_NAME STREQUAL gptl)
 endif()
 set(PIO_FILESYSTEM_HINTS "lustre")
 
-string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_VEGA90A=On -DCMAKE_CXX_FLAGS='-std=gnu++14'")
+string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_VEGA90A=On -DCMAKE_CXX_FLAGS='-std=gnu++14' -DKokkos_ENABLE_OPENMP=OFF")
 set(USE_HIP "TRUE")

components/eamxx/src/share/util/scream_utils.hpp

@@ -11,6 +11,7 @@
 #include <list>
 #include <algorithm>
 #include <map>
+#include <iostream>
 
 namespace scream {
 

components/homme/CMakeLists.txt

@@ -447,18 +447,22 @@ if(HOMME_BUILD_EXECS AND NOT BUILD_HOMME_WITHOUT_PIOLIBRARY)
   ENDIF ()
 ENDIF ()
 
-# If we don't need kokkos we don't need EKAT, and if
-# Homme is built in EAMxx EKAT is already built
-IF (HOMME_USE_KOKKOS AND HOMME_STANDALONE)
-  # Add ekat's cmake/pkg_build folder to cmake path
+IF (HOMME_USE_KOKKOS)
+  # Add ekat's cmake scripts folders to cmake path
   set (EKAT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../externals/ekat)
   set (EKAT_CMAKE_PATH ${EKAT_SOURCE_DIR}/cmake)
   list(APPEND CMAKE_MODULE_PATH
-       ${EKAT_CMAKE_PATH}
-       ${EKAT_CMAKE_PATH}/pkg_build
-       ${EKAT_CMAKE_PATH}/tpls
+    ${EKAT_CMAKE_PATH}
+    ${EKAT_CMAKE_PATH}/tpls
   )
-  include (EkatBuildKokkos)
+
+  # We first try to use find_package. If that doesn't work, build from EKAT's submodule
+  include (EkatFindKokkos)
+  if (NOT Kokkos_FOUND)
+    # The following script checks if Kokkos is already available as a target, and if so does nothing.
+    # For instance, if HOMME is built inside EAMxx, Kokkos will already be available
+    include (EkatBuildKokkos)
+  endif()
 ENDIF ()
 
 # This folder contains the CMake macro used to build cxx unit tests

components/homme/cmake/SetCompilerFlags.cmake

@@ -68,12 +68,6 @@ IF (${HOMME_USE_CXX})
   SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
 
   INCLUDE(CheckCXXCompilerFlag)
-  CHECK_CXX_COMPILER_FLAG("-std=c++14" CXX14_SUPPORTED)
-  IF (CXX14_SUPPORTED)
-    SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
-  ELSEIF (${HOMME_USE_KOKKOS})
-    MESSAGE (FATAL_ERROR "Kokkos needs C++14, but the C++ compiler does not support it.")
-  ENDIF ()
   CHECK_CXX_COMPILER_FLAG("-cxxlib" CXXLIB_SUPPORTED)
   IF (CXXLIB_SUPPORTED)
     SET(CXXLIB_SUPPORTED_CACHE TRUE CACHE BOOL "")

components/homme/src/preqx_kokkos/CMakeLists.txt

@@ -168,6 +168,8 @@ MACRO(PREQX_KOKKOS_SETUP)
     ${SRC_SHARE_DIR}/cxx/EulerStepFunctor.cpp
     ${SRC_SHARE_DIR}/cxx/ExecSpaceDefs.cpp
     ${SRC_SHARE_DIR}/cxx/FunctorsBuffersManager.cpp
+    ${SRC_SHARE_DIR}/cxx/GllFvRemap.cpp
+    ${SRC_SHARE_DIR}/cxx/GllFvRemapImpl.cpp
     ${SRC_SHARE_DIR}/cxx/Hommexx_Session.cpp
     ${SRC_SHARE_DIR}/cxx/HybridVCoord.cpp
     ${SRC_SHARE_DIR}/cxx/HyperviscosityFunctor.cpp

components/homme/src/preqx_kokkos/config.h.cmake.in

@@ -58,9 +58,8 @@
 /* ZOLTAN2 SUBPACKAGE OF TRILINOS  library */
 #cmakedefine01 TRILINOS_HAVE_ZOLTAN2
 
-/* When doing BFB testing, we occasionally must use modified code. */
-/* Use this flag to protect such code. */
-#cmakedefine HOMMEXX_BFB_TESTING
+/* Detect whether this is a kokkos target */
+#cmakedefine01 KOKKOS_TARGET
 
 /* Whether to use OpenMP4 */
 #cmakedefine OMP4

components/homme/src/share/compose/cedr_util.hpp

@@ -4,11 +4,12 @@
 #ifndef INCLUDE_CEDR_UTIL_HPP
 #define INCLUDE_CEDR_UTIL_HPP
 
-#include <sstream>
-
 #include "cedr_kokkos.hpp"
 #include "cedr_mpi.hpp"
 
+#include <sstream>  // For std::stringstream
+#include <iostream> // For std::cerr
+
 namespace cedr {
 namespace util {
 

components/homme/src/share/compose/compose_slmm.cpp

@@ -276,8 +276,6 @@ extern "C" {
 // Interface for Homme, through compose_mod.F90.
 void kokkos_init () {
   amb::dev_init_threads();
-  Kokkos::InitArguments args;
-  args.disable_warnings = true;
   initialize_kokkos();
   // Test these initialize correctly.
   Kokkos::View<int> v("hi");

components/homme/src/share/compose/compose_slmm_siqk.cpp

@@ -75,7 +75,7 @@ class TestSphereToRefKernel {
   }
 
   KOKKOS_INLINE_FUNCTION
-  void join (volatile value_type& dst, volatile value_type const& src) const {
+  void join (value_type& dst, value_type const& src) const {
     dst.max_nits = max(dst.max_nits, src.max_nits);
     dst.sum_nits += src.sum_nits;
     dst.nfails += src.nfails;

components/homme/src/share/cxx/ExecSpaceDefs.cpp

@@ -187,7 +187,7 @@ team_num_threads_vectors (const int num_parallel_iterations,
 #elif defined(KOKKOS_ENABLE_HIP)
   // Use 64 wavefronts per CU and 120 CUs.
   const int num_warps_device = 120*64; // no such thing Kokkos::Impl::hip_internal_maximum_warp_count();
-  const int num_threads_warp = Kokkos::Experimental::Impl::HIPTraits::WarpSize;
+  const int num_threads_warp = Kokkos::Impl::HIPTraits::WarpSize;
 #else
   // I want thread-distribution rules to be unit-testable even when GPU spaces
   // are off. Thus, make up a GPU-like machine:

components/homme/src/share/cxx/ExecSpaceDefs.hpp

@@ -143,11 +143,7 @@ struct DefaultThreadsDistribution {
   team_num_threads_vectors(const int num_parallel_iterations,
                            const ThreadPreferences tp = ThreadPreferences()) {
     return Parallel::team_num_threads_vectors_from_pool(
-#ifdef KOKKOS_ENABLE_DEPRECATED_CODE
-      ExecSpaceType::thread_pool_size()
-#else
-      ExecSpaceType::impl_thread_pool_size()
-#endif
+      ExecSpaceType().impl_thread_pool_size()
       , num_parallel_iterations, tp);
   }
 };

components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp

@@ -10,6 +10,7 @@
 #include "utilities/TestUtils.hpp"
 
 #include <random>
+#include <iostream>
 
 TEST_CASE("remap_interface", "vertical remap") {
 

components/homme/test_execs/share_kokkos_ut/boundary_exchange_ut.cpp

@@ -11,6 +11,7 @@
 
 #include <random>
 #include <iomanip>
+#include <iostream>
 
 using namespace Homme;
 

components/homme/test_execs/share_kokkos_ut/col_ops_ut.cpp

@@ -1,15 +1,17 @@
 #include <catch2/catch.hpp>
 
-#include <random>
-
 #include "ColumnOps.hpp"
-#include "Types.hpp"
 
 #include "utilities/TestUtils.hpp"
 #include "utilities/SubviewUtils.hpp"
 #include "utilities/SyncUtils.hpp"
 #include "utilities/ViewUtils.hpp"
 
+#include "Types.hpp"
+
+#include <random>
+#include <iostream>
+
 using namespace Homme;
 
 

components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp

@@ -8,6 +8,7 @@
 #include "utilities/TestUtils.hpp"
 
 #include <random>
+#include <iostream>
 
 using namespace Homme;
 

components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp

@@ -13,6 +13,7 @@
 #include <assert.h>
 #include <stdio.h>
 #include <random>
+#include <iostream>
 
 using namespace Homme;
 

components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp

@@ -13,6 +13,7 @@
 #include <assert.h>
 #include <stdio.h>
 #include <random>
+#include <iostream>
 
 using namespace Homme;
 

components/homme/test_execs/thetal_kokkos_ut/elem_ops_ut.cpp

@@ -1,14 +1,15 @@
 #include <catch2/catch.hpp>
 
-#include <random>
-
 #include "ElementOps.hpp"
-#include "Types.hpp"
 
 #include "utilities/TestUtils.hpp"
 #include "utilities/SubviewUtils.hpp"
 #include "utilities/SyncUtils.hpp"
 #include "utilities/ViewUtils.hpp"
+#include "Types.hpp"
+
+#include <iostream>
+#include <random>
 
 using namespace Homme;