Unify FM over all grids

components/eamxx/src/control/atmosphere_driver.hpp

@@ -161,7 +161,7 @@ class AtmosphereDriver
   // NOTE: if already finalized, this is a no-op
   void finalize ();
 
-  field_mgr_ptr get_field_mgr (const std::string& grid_name) const;
+  field_mgr_ptr get_field_mgr () const { return m_field_mgr; }
 
   // Get atmosphere time stamp
   const util::TimeStamp& get_atm_time_stamp () const { return m_current_ts; }
@@ -191,7 +191,7 @@ class AtmosphereDriver
                               const std::string& file_name);
   void register_groups ();
 
-  std::map<std::string,field_mgr_ptr>       m_field_mgrs;
+  field_mgr_ptr                             m_field_mgr;
 
   std::shared_ptr<AtmosphereProcessGroup>   m_atm_process_group;
 

components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp

@@ -196,7 +196,7 @@ void HommeDynamics::set_grids (const std::shared_ptr<const GridsManager> grids_m
     add_field<Required>("T_mid",         rg_scalar3d_mid,K,     rgn,N);
     add_field<Required>("ps",            rg_scalar2d    ,Pa,    rgn);
     add_field<Required>("phis",          rg_scalar2d    ,m2/s2, rgn);
-    add_group<Required>("tracers",rgn,N, Bundling::Required, DerivationType::Import, "tracers", pgn);
+    add_group<Required>("tracers",rgn,N, Bundling::Required);
     fv_phys_rrtmgp_active_gases_init(grids_manager);
     // This is needed for the dp_ref init in initialize_homme_state.
     add_field<Computed>("pseudo_density",rg_scalar3d_mid,Pa,    rgn,N);

components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp

@@ -86,40 +86,34 @@ TEST_CASE("dyn_grid_io")
   FieldIdentifier fid_phys_2 ("field_2",phys_vector3d_mid,nondim,phys_grid->name());
   FieldIdentifier fid_phys_3 ("field_3",phys_scalar2d    ,nondim,phys_grid->name());
 
-  // The starting FM
-  auto fm_dyn = std::make_shared<FieldManager> (dyn_grid);
+  // FM with dyn and phys where we read into
+  auto fm = std::make_shared<FieldManager> (gm);
 
   // The FM we will manually remap onto
   auto fm_ctrl= std::make_shared<FieldManager> (phys_grid);
 
-  // The FM we will read into, and compare against the previous
-  auto fm_phys= std::make_shared<FieldManager> (phys_grid);
-
-  fm_dyn->registration_begins();
-  fm_phys->registration_begins();
+  fm->registration_begins();
   fm_ctrl->registration_begins();
 
   const int ps = HOMMEXX_PACK_SIZE;
   util::TimeStamp t0({2000,1,1},{0,0,0});
 
-  fm_dyn->register_field(FieldRequest(fid_dyn_1,ps));
-  fm_dyn->register_field(FieldRequest(fid_dyn_2,ps));
-  fm_dyn->register_field(FieldRequest(fid_dyn_3));
-
-  fm_phys->register_field(FieldRequest(fid_phys_1,ps));
-  fm_phys->register_field(FieldRequest(fid_phys_2,ps));
-  fm_phys->register_field(FieldRequest(fid_phys_3));
+  fm->register_field(FieldRequest(fid_dyn_1,ps));
+  fm->register_field(FieldRequest(fid_dyn_2,ps));
+  fm->register_field(FieldRequest(fid_dyn_3));
+  fm->register_field(FieldRequest(fid_phys_1,ps));
+  fm->register_field(FieldRequest(fid_phys_2,ps));
+  fm->register_field(FieldRequest(fid_phys_3));
 
   fm_ctrl->register_field(FieldRequest(fid_phys_1,ps));
   fm_ctrl->register_field(FieldRequest(fid_phys_2,ps));
   fm_ctrl->register_field(FieldRequest(fid_phys_3));
 
-  fm_dyn->registration_ends();
-  fm_phys->registration_ends();
+  fm->registration_ends();
   fm_ctrl->registration_ends();
-  fm_dyn->init_fields_time_stamp(t0);
-  fm_phys->init_fields_time_stamp(t0);
-  fm_ctrl->init_fields_time_stamp(t0);
+  fm->init_fields_time_stamp(t0,dyn_grid->name());
+  fm->init_fields_time_stamp(t0, phys_grid->name());
+  fm_ctrl->init_fields_time_stamp(t0, phys_grid->name());
 
   std::vector<std::string> fnames = {"field_1", "field_2", "field_3"};
 
@@ -129,13 +123,13 @@ TEST_CASE("dyn_grid_io")
   auto dyn2ctrl = gm->create_remapper(dyn_grid,phys_grid);
   dyn2ctrl->registration_begins();
   for (const auto& fn : fnames) {
-    auto fd = fm_dyn->get_field(fn);
-    auto fc = fm_ctrl->get_field(fn);
+    auto fd = fm->get_field(fn,dyn_grid->name());
+    auto fc = fm_ctrl->get_field(fn,phys_grid->name());
     dyn2ctrl->register_field(fd,fc);
     randomize(fd,engine,pdf);
 
     // Init phys field to something obviously wrong
-    auto fp = fm_phys->get_field(fn);
+    auto fp = fm->get_field(fn,phys_grid->name());
     fp.deep_copy(-1.0);
   }
   dyn2ctrl->registration_ends();
@@ -155,7 +149,7 @@ TEST_CASE("dyn_grid_io")
 
   OutputManager output;
   output.initialize(comm, out_params, t0, false);
-  output.setup (fm_dyn, gm);
+  output.setup (fm, {dyn_grid->name()});
   output.run(t0);
   output.finalize();
 
@@ -166,14 +160,21 @@ TEST_CASE("dyn_grid_io")
   ekat::ParameterList in_params;
   in_params.set<std::string>("Filename",filename);
   in_params.set<std::vector<std::string>>("Field Names",fnames);
+
+  // AtmosphereInput expects a FM on a single grid, create
+  // a phys FM and add fields.
+  auto fm_phys = std::make_shared<FieldManager>(phys_grid);
+  for (auto& f_it : fm->get_repo(phys_grid->name())) {
+    fm_phys->add_field(*f_it.second);
+  }
   AtmosphereInput input (in_params,fm_phys);
   input.read_variables();
   input.finalize();
 
   // Compare against ctrl fields
   for (const auto& fn : fnames) {
-    auto fp = fm_phys->get_field(fn);
-    auto fc = fm_ctrl->get_field(fn);
+    auto fp = fm->get_field(fn,phys_grid->name());
+    auto fc = fm_ctrl->get_field(fn,phys_grid->name());
     REQUIRE(views_are_equal(fp,fc));
   }
 

components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp

@@ -160,7 +160,7 @@ create_om (const std::string& filename_prefix,
 
   auto om = std::make_shared<OutputManager>();
   om->initialize(comm,params,t0,false);
-  om->setup(fm,gm);
+  om->setup(fm,gm->get_grid_names());
   return om;
 }
 

components/eamxx/src/share/atm_process/IOPDataManager.cpp

@@ -613,27 +613,27 @@ read_iop_file_data (const util::TimeStamp& current_ts)
 }
 
 void IOPDataManager::
-set_fields_from_iop_data(const field_mgr_ptr field_mgr)
+set_fields_from_iop_data(const field_mgr_ptr field_mgr, const std::string& grid_name)
 {
-  if (m_params.get<bool>("zero_non_iop_tracers") && field_mgr->has_group("tracers")) {
+  if (m_params.get<bool>("zero_non_iop_tracers") && field_mgr->has_group("tracers", grid_name)) {
     // Zero out all tracers before setting iop tracers (if requested)
-    field_mgr->get_field_group("tracers").m_bundle->deep_copy(0);
+    field_mgr->get_field_group("tracers", grid_name).m_bundle->deep_copy(0);
   }
 
-  EKAT_REQUIRE_MSG(field_mgr->get_grid()->name() == "Physics GLL",
+  EKAT_REQUIRE_MSG(grid_name == "Physics GLL",
                    "Error! Attempting to set non-GLL fields using "
                    "data from the IOP file.\n");
 
   // Find which fields need to be written
-  const bool set_ps            = field_mgr->has_field("ps") && has_iop_field("Ps");
-  const bool set_T_mid         = field_mgr->has_field("T_mid") && has_iop_field("T");
-  const bool set_horiz_winds_u = field_mgr->has_field("horiz_winds") && has_iop_field("u");
-  const bool set_horiz_winds_v = field_mgr->has_field("horiz_winds") && has_iop_field("v");
-  const bool set_qv            = field_mgr->has_field("qv") && has_iop_field("q");
-  const bool set_nc            = field_mgr->has_field("nc") && has_iop_field("NUMLIQ");
-  const bool set_qc            = field_mgr->has_field("qc") && has_iop_field("CLDLIQ");
-  const bool set_qi            = field_mgr->has_field("qi") && has_iop_field("CLDICE");
-  const bool set_ni            = field_mgr->has_field("ni") && has_iop_field("NUMICE");
+  const bool set_ps            = field_mgr->has_field("ps", grid_name) && has_iop_field("Ps");
+  const bool set_T_mid         = field_mgr->has_field("T_mid", grid_name) && has_iop_field("T");
+  const bool set_horiz_winds_u = field_mgr->has_field("horiz_winds", grid_name) && has_iop_field("u");
+  const bool set_horiz_winds_v = field_mgr->has_field("horiz_winds", grid_name) && has_iop_field("v");
+  const bool set_qv            = field_mgr->has_field("qv", grid_name) && has_iop_field("q");
+  const bool set_nc            = field_mgr->has_field("nc", grid_name) && has_iop_field("NUMLIQ");
+  const bool set_qc            = field_mgr->has_field("qc", grid_name) && has_iop_field("CLDLIQ");
+  const bool set_qi            = field_mgr->has_field("qi", grid_name) && has_iop_field("CLDICE");
+  const bool set_ni            = field_mgr->has_field("ni", grid_name) && has_iop_field("NUMICE");
 
   // Create views/scalars for these field's data
   view_1d<Real> ps;
@@ -644,47 +644,47 @@ set_fields_from_iop_data(const field_mgr_ptr field_mgr)
   view_1d<Real> t_iop, u_iop, v_iop, qv_iop, nc_iop, qc_iop, qi_iop, ni_iop;
 
   if (set_ps) {
-    ps = field_mgr->get_field("ps").get_view<Real*>();
+    ps = field_mgr->get_field("ps", grid_name).get_view<Real*>();
     get_iop_field("Ps").sync_to_host();
     ps_iop = get_iop_field("Ps").get_view<Real, Host>()();
   }
   if (set_T_mid) {
-    T_mid = field_mgr->get_field("T_mid").get_view<Real**>();
+    T_mid = field_mgr->get_field("T_mid", grid_name).get_view<Real**>();
     t_iop = get_iop_field("T").get_view<Real*>();
   }
   if (set_horiz_winds_u || set_horiz_winds_v) {
-    horiz_winds = field_mgr->get_field("horiz_winds").get_view<Real***>();
+    horiz_winds = field_mgr->get_field("horiz_winds", grid_name).get_view<Real***>();
     if (set_horiz_winds_u) u_iop = get_iop_field("u").get_view<Real*>();
     if (set_horiz_winds_v) v_iop = get_iop_field("v").get_view<Real*>();
   }
   if (set_qv) {
-    qv = field_mgr->get_field("qv").get_view<Real**>();
+    qv = field_mgr->get_field("qv", grid_name).get_view<Real**>();
     qv_iop = get_iop_field("q").get_view<Real*>();
   }
   if (set_nc) {
-    nc = field_mgr->get_field("nc").get_view<Real**>();
+    nc = field_mgr->get_field("nc", grid_name).get_view<Real**>();
     nc_iop = get_iop_field("NUMLIQ").get_view<Real*>();
   }
   if (set_qc) {
-    qc = field_mgr->get_field("qc").get_view<Real**>();
+    qc = field_mgr->get_field("qc", grid_name).get_view<Real**>();
     qc_iop = get_iop_field("CLDLIQ").get_view<Real*>();
   }
   if (set_qi) {
-    qi = field_mgr->get_field("qi").get_view<Real**>();
+    qi = field_mgr->get_field("qi", grid_name).get_view<Real**>();
     qi_iop = get_iop_field("CLDICE").get_view<Real*>();
   }
   if (set_ni) {
-    ni = field_mgr->get_field("ni").get_view<Real**>();
+    ni = field_mgr->get_field("ni", grid_name).get_view<Real**>();
     ni_iop = get_iop_field("NUMICE").get_view<Real*>();
   }
 
   // Check if t_iop has any 0 entires near the top of the model
   // and correct t_iop and q_iop accordingly.
-  correct_temperature_and_water_vapor(field_mgr);
+  correct_temperature_and_water_vapor(field_mgr, grid_name);
 
   // Loop over all columns and copy IOP field values to FM views
-  const auto ncols = field_mgr->get_grid()->get_num_local_dofs();
-  const auto nlevs = field_mgr->get_grid()->get_num_vertical_levels();
+  const auto ncols = field_mgr->get_grids_manager()->get_grid(grid_name)->get_num_local_dofs();
+  const auto nlevs = field_mgr->get_grids_manager()->get_grid(grid_name)->get_num_vertical_levels();
   const auto policy = ESU::get_default_team_policy(ncols, nlevs);
   Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const KT::MemberType& team) {
     const auto icol = team.league_rank();
@@ -722,11 +722,11 @@ set_fields_from_iop_data(const field_mgr_ptr field_mgr)
 }
 
 void IOPDataManager::
-correct_temperature_and_water_vapor(const field_mgr_ptr field_mgr)
+correct_temperature_and_water_vapor(const field_mgr_ptr field_mgr, const std::string& grid_name)
 {
   // Find the first valid level index for t_iop, i.e., first non-zero entry
   int first_valid_idx;
-  const auto nlevs = field_mgr->get_grid()->get_num_vertical_levels();
+  const auto nlevs = field_mgr->get_grids_manager()->get_grid(grid_name)->get_num_vertical_levels();
   auto t_iop = get_iop_field("T").get_view<Real*>();
   Kokkos::parallel_reduce(nlevs, KOKKOS_LAMBDA (const int ilev, int& lmin) {
     if (t_iop(ilev) > 0 && ilev < lmin) lmin = ilev;
@@ -736,12 +736,12 @@ correct_temperature_and_water_vapor(const field_mgr_ptr field_mgr)
   // levels 0,...,first_valid_idx-1
   if (first_valid_idx > 0) {
     // If we have values of T and q to correct, we must have both T_mid and qv as FM fields
-    EKAT_REQUIRE_MSG(field_mgr->has_field("T_mid"), "Error! IOP requires FM to define T_mid.\n");
-    EKAT_REQUIRE_MSG(field_mgr->has_field("qv"),    "Error! IOP requires FM to define qv.\n");
+    EKAT_REQUIRE_MSG(field_mgr->has_field("T_mid", grid_name), "Error! IOP requires FM to define T_mid.\n");
+    EKAT_REQUIRE_MSG(field_mgr->has_field("qv", grid_name),    "Error! IOP requires FM to define qv.\n");
 
     // Replace values of T and q where t_iop contains zeros
-    auto T_mid = field_mgr->get_field("T_mid").get_view<const Real**>();
-    auto qv   = field_mgr->get_field("qv").get_view<const Real**>();
+    auto T_mid = field_mgr->get_field("T_mid", grid_name).get_view<const Real**>();
+    auto qv   = field_mgr->get_field("qv", grid_name).get_view<const Real**>();
     auto q_iop = get_iop_field("q").get_view<Real*>();
     Kokkos::parallel_for(Kokkos::RangePolicy<>(0, first_valid_idx), KOKKOS_LAMBDA (const int ilev) {
       t_iop(ilev) = T_mid(0, ilev);

components/eamxx/src/share/atm_process/IOPDataManager.hpp

@@ -74,7 +74,7 @@ class IOPDataManager
                                      const std::shared_ptr<const AbstractGrid>& tgt_grid);
 
   // Set fields using data loaded from the iop file
-  void set_fields_from_iop_data(const field_mgr_ptr field_mgr);
+  void set_fields_from_iop_data(const field_mgr_ptr field_mgr, const std::string& grid_name);
 
   // The IOP file may contain temperature values that are
   // 0 at or above the surface. Correct these values using
@@ -85,7 +85,7 @@ class IOPDataManager
   // Note: We only need to use the first column because during
   //       the loading of ICs, every columns will have the same
   //       data.
-  void correct_temperature_and_water_vapor(const field_mgr_ptr field_mgr);
+  void correct_temperature_and_water_vapor(const field_mgr_ptr field_mgr, const std::string& grid_name);
 
   ekat::ParameterList& get_params() { return m_params; }
 

components/eamxx/src/share/atm_process/atmosphere_process.hpp

@@ -358,9 +358,8 @@ class AtmosphereProcess : public ekat::enable_shared_from_this<AtmosphereProcess
   // Group requests
   template<RequestType RT>
   void add_group (const std::string& name, const std::string& grid, const int ps, const Bundling b,
-                  const DerivationType t, const std::string& src_name, const std::string& src_grid,
-                  const std::list<std::string>& excl = {})
-  { add_group<RT>(GroupRequest(name,grid,ps,b,t,src_name,src_grid,excl)); }
+                  const bool imported, const std::string& src_name, const std::string& src_grid)
+  { add_group<RT>(GroupRequest(name,grid,ps,b,imported,src_name,src_grid)); }
 
   template<RequestType RT>
   void add_group (const std::string& name, const std::string& grid_name,

components/eamxx/src/share/field/field.cpp

@@ -34,10 +34,15 @@ Field::alias (const std::string& name) const {
 
 Field
 Field::clone(const std::string& name) const {
+  return clone(name, get_header().get_identifier().get_grid_name());
+}
+
+Field
+Field::clone(const std::string& name, const std::string& grid_name) const {
   // Create new field
   const auto& my_fid = get_header().get_identifier();
   FieldIdentifier fid(name,my_fid.get_layout(),my_fid.get_units(),
-                      my_fid.get_grid_name(),my_fid.data_type());
+                      grid_name,my_fid.data_type());
   Field f(fid);
 
   // Ensure alloc props match
@@ -186,6 +191,7 @@ Field Field::subfield(const std::string& sf_name,
 
   auto sf_layout = lt.clone();
   sf_layout.reset_dim(idim, index_end - index_beg);
+
   // Create identifier for subfield
   FieldIdentifier sf_id(sf_name, sf_layout, sf_units, id.get_grid_name(), id.data_type());
 

components/eamxx/src/share/field/field.hpp

@@ -118,6 +118,8 @@ class Field {
   // It is created with a pristine header (no providers/customers)
   Field clone () const;
   Field clone (const std::string& name) const;
+  Field clone (const std::string& name, const std::string& grid_name) const;
+
   Field alias (const std::string& name) const;
 
   // Allows to get the underlying view, reshaped for a different data type.

components/eamxx/src/share/field/field_alloc_prop.cpp

@@ -88,7 +88,7 @@ FieldAllocProp FieldAllocProp::subview(const int idim,
   EKAT_REQUIRE_MSG(index_beg < index_end,
                    "Error! Slice indices are invalid (non-increasing).\n");
   EKAT_REQUIRE_MSG(
-      index_beg >= 0 && index_end < m_layout.dim(idim),
+      index_beg >= 0 && index_end-1 < m_layout.dim(idim),
       "Error! Slice index range along the idim dimension is out of bounds.\n");
 
   // Set new layout basic stuff