Take XIOS file frequency configuration from iodef.xml where possible

.gitignore

@@ -71,3 +71,5 @@ infrastructure/**/bin
 infrastructure/**/working
 infrastructure/**/test
 infrastructure/**/documents
+science/**/bin
+science/**/working

components/lfric-xios/build/testframework/xiostest.py

@@ -8,6 +8,7 @@
 import subprocess
 from pathlib import Path
 import sys
+import shutil
 from typing import List
 
 from testframework import MpiTest
@@ -20,40 +21,85 @@ class LFRicXiosTest(MpiTest):
     Base for LFRic-XIOS integration tests.
     """
 
-    def __init__(self, command=sys.argv[1], processes=1):
+    def __init__(self, command=sys.argv[1], processes=1, iodef_file=None):
+        if iodef_file is None:
+            self.iodef_file = "iodef.xml"
+        else:
+            self.iodef_file = iodef_file
+
         super().__init__(command, processes)
+
         self.xios_out: List[XiosOutput] = []
         self.xios_err: List[XiosOutput] = []
 
-    def gen_data(self, source: Path, dest: Path):
+        # Setup test working directory
+        self.test_top_level: Path = Path(os.getcwd())
+        self.resources_dir: Path = self.test_top_level / "resources"
+        self.test_working_dir: Path = self.test_top_level / "working" / type(self).__name__
+        if not os.path.exists(self.test_working_dir):
+            os.makedirs(self.test_working_dir)
+
+        # Create symlink to test executable in working directory
+        if not os.path.exists(Path(self.test_working_dir) / command[0].split("/")[-1]):
+            os.symlink(Path(command[0]), Path(self.test_working_dir) / command[0].split("/")[-1])
+
+        # Change to test working directory
+        os.chdir(self.test_working_dir)
+
+
+    def gen_data(self, source: str, dest: str):
         """
-        Create input data files from CDL formatted text.
+        Create input data files from CDL formatted text. Looks for source file
+        in resources/data directory and generates dest file in test working directory.
         """
+        dest_path: Path = Path(self.test_working_dir) / Path(dest)
+        source_path: Path = Path(self.resources_dir, 'data') / Path(source)
+        dest_path.unlink(missing_ok=True)
+
         proc = subprocess.Popen(
-            ['ncgen', '-k', 'nc4', '-o', f'{dest}', f'{source}'],
+            ['ncgen', '-k', 'nc4', '-o', f'{dest_path}', f'{source_path }'],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             )
         _, err = proc.communicate()
         if proc.returncode != 0:
             raise Exception("Test data generation failed:\n" + f"{err}")
-
-    def gen_config(self, config_source: Path, config_out: Path, new_config: dict):
+
+
+    def gen_config(self, config_source: str, config_out: str, new_config: dict):
         """
-        Create an LFRic configuration namelist.
+        Create an LFRic configuration namelist. Looks for source file
+        in resources/configs directory and generates dest file in test working directory.
         """
-        config_in = open(config_source, 'r')
+        config_in = open(Path(self.resources_dir, 'configs', config_source), 'r')
         config = config_in.readlines()
         for key in new_config.keys():
             for i in range(len(config)):
                 if key in config[i]:
-                    config[i] = f"  {key}={new_config[key]}\n"
+                    if type(new_config[key]) == str:
+                        config[i] = f"  {key}='{new_config[key]}'\n"
+                    else:
+                        config[i] = f"  {key}={new_config[key]}\n"
         config_in.close()
 
-        f = open(config_out, "w")
+        f = open(Path(self.test_working_dir, config_out), "w")
         for line in config:
             f.write(line)
-        f.close()            
+        f.close()
+
+
+    def performTest(self):
+        """
+        Removes any old log files and runs the executable.
+        """
+
+        # Handle iodef file
+        if os.path.exists(self.iodef_file):
+            os.remove(self.iodef_file)
+        shutil.copy(self.resources_dir / self.iodef_file, self.test_working_dir / "iodef.xml")
+
+        return super().performTest()
+
 
     def nc_kgo_check(self, output: Path, kgo: Path):
         """
@@ -92,8 +138,11 @@ def post_execution(self, return_code):
         """
 
         for proc in range(self._processes):
-            self.xios_out.append(XiosOutput(f"xios_client_{proc}.out"))
-            self.xios_err.append(XiosOutput(f"xios_client_{proc}.err"))
+            self.xios_out.append(XiosOutput(self.test_working_dir / f"xios_client_{proc}.out"))
+            self.xios_err.append(XiosOutput(self.test_working_dir / f"xios_client_{proc}.err"))
+
+        # Return to top level directory
+        os.chdir(self.test_top_level)
 
 
 class XiosOutput:
@@ -102,7 +151,7 @@ class XiosOutput:
     """
 
     def __init__(self, filename):
-        self.path: Path = Path(os.getcwd()) / Path(filename)
+        self.path: Path = Path(filename)
 
         with open(self.path, "rt") as handle:
             self.contents = handle.read()

components/lfric-xios/integration-test/lfric_xios_context_test.py

@@ -11,6 +11,7 @@
 
 from testframework import TestEngine, TestFailed
 from xiostest import LFRicXiosTest
+from pathlib import Path
 import sys
 
 ###############################################################################
@@ -20,7 +21,8 @@ class LfricXiosContextTest(LFRicXiosTest):
     """
 
     def __init__(self):
-        super().__init__(command=[sys.argv[1], "resources/configs/context.nml"], processes=1)
+        super().__init__(command=[sys.argv[1], "context.nml"], processes=1)
+        self.gen_config( "context.nml", "context.nml", {} )
 
     def test(self, returncode: int, out: str, err: str):
         """

components/lfric-xios/integration-test/lfric_xios_temporal_iodef_test.f90

@@ -0,0 +1,86 @@
+!-----------------------------------------------------------------------------
+! (C) Crown copyright 2025 Met Office. All rights reserved.
+! The file LICENCE, distributed with this code, contains details of the terms
+! under which the code may be used.
+!-----------------------------------------------------------------------------
+
+! Tests the LFRic-XIOS temporal reading functionality using iodef file configuration.
+! Correct behaviour is to read only the minimal required time-entries from
+! input file at the correct times. The validity of the data written from this
+! test is checked against the input data in the python part of the test.
+program lfric_xios_temporal_iodef_test
+
+  use constants_mod,          only: r_def
+  use event_mod,              only: event_action
+  use event_actor_mod,        only: event_actor_type
+  use field_mod,              only: field_type, field_proxy_type
+  use file_mod,               only: FILE_MODE_READ, FILE_MODE_WRITE
+  use io_context_mod,         only: callback_clock_arg
+  use lfric_xios_action_mod,  only: advance
+  use lfric_xios_context_mod, only: lfric_xios_context_type
+  use lfric_xios_driver_mod,  only: lfric_xios_initialise, lfric_xios_finalise
+  use lfric_xios_file_mod,    only: lfric_xios_file_type, OPERATION_TIMESERIES
+  use linked_list_mod,        only: linked_list_type
+  use log_mod,                only: log_event, log_level_info
+  use test_db_mod,            only: test_db_type
+
+  implicit none
+
+  type(test_db_type)                                 :: test_db
+  type(lfric_xios_context_type), target, allocatable :: io_context
+
+  procedure(callback_clock_arg), pointer :: before_close
+  type(linked_list_type),        pointer :: file_list
+  class(event_actor_type),       pointer :: context_actor
+  procedure(event_action),       pointer :: context_advance
+  type(field_type),              pointer :: rfield
+  type(field_proxy_type)                 :: rproxy
+
+  call test_db%initialise()
+  call lfric_xios_initialise( "test", test_db%comm, .false. )
+
+  ! =============================== Start test ================================
+
+  allocate(io_context)
+  call io_context%initialise( "test_io_context", 1, 10 )
+
+  file_list => io_context%get_filelist()
+  call file_list%insert_item( lfric_xios_file_type( "lfric_xios_temporal_input",         &
+                                                    xios_id="lfric_xios_temporal_input", &
+                                                    io_mode=FILE_MODE_READ,              &
+                                                    operation=OPERATION_TIMESERIES,      &
+                                                    fields_in_file=test_db%temporal_fields ) )
+  call file_list%insert_item( lfric_xios_file_type( "lfric_xios_temporal_output",         &
+                                                    xios_id="lfric_xios_temporal_output", &
+                                                    io_mode=FILE_MODE_WRITE,              &
+                                                    operation=OPERATION_TIMESERIES,       &
+                                                    freq=1,                               &
+                                                    fields_in_file=test_db%temporal_fields ) )
+
+  before_close => null()
+  call io_context%initialise_xios_context( test_db%comm,                    &
+                                           test_db%chi,  test_db%panel_id,  &
+                                           test_db%clock, test_db%calendar, &
+                                           before_close )
+
+
+  context_advance => advance
+  context_actor => io_context
+  call test_db%clock%add_event( context_advance, context_actor )
+  call io_context%set_active(.true.)
+
+  do while (test_db%clock%tick())
+    call test_db%temporal_fields%get_field("temporal_field", rfield)
+    rproxy = rfield%get_proxy()
+    call log_event("Valid data for this TS:", log_level_info)
+    print*,rproxy%data(1)
+  end do
+
+  deallocate(io_context)
+
+  ! ============================== Finish test =================================
+
+  call lfric_xios_finalise()
+  call test_db%finalise()
+
+end program lfric_xios_temporal_iodef_test

components/lfric-xios/integration-test/lfric_xios_temporal_iodef_test.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+##############################################################################
+# (C) Crown copyright 2025 Met Office. All rights reserved.
+# The file LICENCE, distributed with this code, contains details of the terms
+# under which the code may be used.
+##############################################################################
+"""
+A set of tests which exercise the temporal reading functionality provided by
+the LFRic-XIOS component. For these tests the file is configured mainly via
+the iodef.xml file, rather than the fortran API.
+The tests cover the reading of a piece of non-cyclic temporal data with data
+points ranging from 15:01 to 15:10 in 10 1-minute intervals. The model start
+time is changed to change how the model interacts with the data.
+"""
+from testframework import TestEngine, TestFailed
+from xiostest import LFRicXiosTest
+from pathlib import Path
+import sys
+
+###############################################################################
+class LfricXiosFullNonCyclicIodefTest(LFRicXiosTest):  # pylint: disable=too-few-public-methods
+    """
+    Tests the LFRic-XIOS temporal reading functionality for a full set of non-cyclic data
+    """
+
+    def __init__(self):
+        super().__init__(command=[sys.argv[1], "non_cyclic_full.nml"], processes=1, iodef_file="iodef_temporal.xml")
+        self.gen_data('temporal_data.cdl', 'lfric_xios_temporal_input.nc')
+        self.gen_config( "non_cyclic_base.nml", "non_cyclic_full.nml", {} )
+
+    def test(self, returncode: int, out: str, err: str):
+        """
+        Test the output of the context test
+        """
+
+        if returncode != 0:
+            print(out)
+            raise TestFailed(f"Unexpected failure of test executable: {returncode}\n" +
+                             f"stderr:\n" +
+                             f"{err}")
+        if not self.nc_data_match(Path(self.test_working_dir, 'lfric_xios_temporal_input.nc'),
+                                  Path(self.test_working_dir, 'lfric_xios_temporal_output.nc'),
+                                  'temporal_field'):
+            raise TestFailed("Output data does not match input data for same time values")
+
+        return "Reading full set of non-cylic data okay..."
+
+
+class LfricXiosFullNonCyclicIodefHighFreqTest(LFRicXiosTest):  # pylint: disable=too-few-public-methods
+    """
+    Tests the LFRic-XIOS temporal reading functionality for a full set of
+    non-cyclic data at hieher model frequency
+    """
+
+    def __init__(self):
+        super().__init__(command=[sys.argv[1], "non_cyclic_full.nml"], processes=1, iodef_file="iodef_temporal.xml")
+        self.gen_data('temporal_data.cdl', 'lfric_xios_temporal_input.nc')
+        self.gen_config( "non_cyclic_base.nml", "non_cyclic_full.nml", {"dt": 10.0,
+                          "timestep_end": '60'} )
+
+    def test(self, returncode: int, out: str, err: str):
+        """
+        Test the output of the context test
+        """
+
+        if returncode != 0:
+            print(out)
+            raise TestFailed(f"Unexpected failure of test executable: {returncode}\n" +
+                             f"stderr:\n" +
+                             f"{err}")
+        if not self.nc_data_match(Path(self.test_working_dir, 'lfric_xios_temporal_input.nc'),
+                                  Path(self.test_working_dir, 'lfric_xios_temporal_output.nc'),
+                                  'temporal_field'):
+            raise TestFailed("Output data does not match input data for same time values")
+
+        return "Reading full set of non-cylic data okay..."
+
+
+class LfricXiosFullNonCyclicIodefNoFreqTest(LFRicXiosTest):  # pylint: disable=too-few-public-methods
+    """
+    Tests the error handling for the case where there is no frequency set in either
+    the iodef or the fortran configuration.
+    """
+
+    def __init__(self):
+        super().__init__(command=[sys.argv[1], "non_cyclic_full.nml"], processes=1)
+        self.gen_data('temporal_data.cdl', 'lfric_xios_temporal_input.nc')
+        self.gen_config( "non_cyclic_base.nml", "non_cyclic_full.nml", {} )
+
+    def test(self, returncode: int, out: str, err: str):
+        """
+        Test the output of the context test
+        """
+
+        expected_xios_errs = ['In file "type_impl.hpp", function "void xios::CType<T>::_checkEmpty() const [with T = xios::CDuration]",  line 210 -> Data is not initialized',
+                              'In file "type_impl.hpp", function "void xios::CType<xios::CDuration>::_checkEmpty() const [T = xios::CDuration]",  line 210 -> Data is not initialized']
+
+        if returncode == 134:
+            if self.xios_err[0].contents.strip() in expected_xios_errs:
+                return "Expected failure of test executable due to missing frequency setting."
+            else:
+                raise TestFailed("Test executable failed, but with unexpected error message.")
+        elif returncode == 0:
+            raise TestFailed("Test executable succeeded unexpectedly despite missing frequency setting.")
+        else:
+            raise TestFailed("Test executable failed with unexpected return code.")
+
+
+##############################################################################
+if __name__ == "__main__":
+    TestEngine.run(LfricXiosFullNonCyclicIodefTest())
+    TestEngine.run(LfricXiosFullNonCyclicIodefHighFreqTest())
+    TestEngine.run(LfricXiosFullNonCyclicIodefNoFreqTest())