Merge main into release branch

documentation/Sphinx/user_guide/setting-up/setting-up.rst

@@ -114,11 +114,23 @@ Environment Variables
 
    ``VERNIER_IO_MODE``
 
-     Determines the output mode to use. Currently only supports being set to 
-     **multi** but single-file-output may be added in the future.
+     Determines the output mode to use. Currently supported values are:
+
+     * **multi**: Every task writes to its own output file
+
+     * **single**: All tasks write to a single global output file
+
+     If the environment variable is not set, **multi** mode is used.
 
    ``VERNIER_OUTPUT_FILENAME``
 
-     Sets the output filename, which is "vernier-output" by default. Vernier
-     will append the MPI rank onto the end of this name by default, resulting
-     in a file called `vernier-output-0` for the first MPI rank, for example.
+     Sets the output filename, which is ``vernier-output`` by
+     default.
+
+     Vernier will append the MPI rank onto the end of this
+     name when running in **multi** mode, resulting in a file called
+     `vernier-output-0` for the first MPI rank, for example.
+
+     Vernier will append ``-global`` to the name when running in
+     **single** mode and the file will contain formatted entries for
+     each task ordered by MPI rank.

src/c++/CMakeLists.txt

@@ -19,6 +19,7 @@ add_library(${CMAKE_PROJECT_NAME}
         hashvec_handler.cpp
         writer/writer.cpp
         writer/multi.cpp
+        writer/single.cpp
         formatter.cpp
         hashvec.cpp
         vernier_gettime.cpp

src/c++/formatter.cpp

@@ -45,7 +45,13 @@ meto::Formatter::Formatter() {
  * @param[in] hashvec  Vector of data that the format method will operate on
  */
 
-void meto::Formatter::execute_format(std::ofstream &os, hashvec_t hashvec) {
+void meto::Formatter::execute_format(std::ostream &os, hashvec_t hashvec,
+                                     MPIContext &mpi_context) {
+  // Add an MPI task identifier to each output file
+  os << "\n"
+     << "Task " << (mpi_context.get_rank() + 1) << " of "
+     << mpi_context.get_size() << " : MPI rank ID " << mpi_context.get_rank()
+     << "\n";
   (this->*format_)(os, hashvec);
 }
 
@@ -56,7 +62,7 @@ void meto::Formatter::execute_format(std::ofstream &os, hashvec_t hashvec) {
  * @param[in] hashvec  Vector containing all the necessary data
  */
 
-void meto::Formatter::threads(std::ofstream &os, hashvec_t hashvec) {
+void meto::Formatter::threads(std::ostream &os, hashvec_t hashvec) {
 
   // Write key
   os << "\n";
@@ -98,7 +104,7 @@ void meto::Formatter::threads(std::ofstream &os, hashvec_t hashvec) {
  * @param[in] hashvec  Vector containing all the necessary data
  */
 
-void meto::Formatter::drhook(std::ofstream &os, hashvec_t hashvec) {
+void meto::Formatter::drhook(std::ostream &os, hashvec_t hashvec) {
 
   int num_threads = 1;
 #ifdef _OPENMP

src/c++/formatter.h

@@ -16,12 +16,14 @@
 #define FORMATTER_H
 
 #include <fstream>
+#include <ostream>
 
 #ifdef _OPENMP
 #include <omp.h>
 #endif
 
 #include "hashvec.h"
+#include "mpi_context.h"
 
 namespace meto {
 
@@ -36,18 +38,18 @@ class Formatter {
 
 private:
   // Format method
-  void (Formatter::*format_)(std::ofstream &, hashvec_t);
+  void (Formatter::*format_)(std::ostream &, hashvec_t);
 
   // Individual formatter functions
-  void threads(std::ofstream &os, hashvec_t);
-  void drhook(std::ofstream &os, hashvec_t);
+  void threads(std::ostream &os, hashvec_t);
+  void drhook(std::ostream &os, hashvec_t);
 
 public:
   // Constructor
   explicit Formatter();
 
   // Execute the format method
-  void execute_format(std::ofstream &os, hashvec_t);
+  void execute_format(std::ostream &os, hashvec_t, MPIContext &);
 };
 
 } // namespace meto

src/c++/hashvec_handler.cpp

@@ -31,6 +31,8 @@ meto::HashVecHandler::HashVecHandler(MPIContext const &mpi_context) {
   // Allocate writer to be of required type.
   if (io_mode == "multi") {
     writer_strategy_ = std::make_unique<Multi>(mpi_context);
+  } else if (io_mode == "single") {
+    writer_strategy_ = std::make_unique<SingleFile>(mpi_context);
   } else {
     error_handler("Invalid IO mode choice", EXIT_FAILURE);
   }
@@ -61,7 +63,4 @@ void meto::HashVecHandler::sort() {
  *
  */
 
-void meto::HashVecHandler::write() {
-  std::ofstream os;
-  writer_strategy_->write(os, hashvec_);
-}
+void meto::HashVecHandler::write() { writer_strategy_->write(hashvec_); }

src/c++/hashvec_handler.h

@@ -20,6 +20,7 @@
 #include "hashvec.h"
 #include "mpi_context.h"
 #include "writer/multi.h"
+#include "writer/single.h"
 #include "writer/writer.h"
 
 namespace meto {

src/c++/mpi_context.cpp

@@ -8,6 +8,9 @@
 #include <cassert>
 #include <stdexcept>
 
+#include <fstream>
+#include <sstream>
+
 #include "error_handler.h"
 #include "mpi_context.h"
 
@@ -115,6 +118,13 @@ int meto::MPIContext::get_rank() { return comm_rank_; }
 
 int meto::MPIContext::get_size() { return comm_size_; }
 
+/**
+ * @brief Gets the MPI communicator handle.
+ * @returns The MPI communicator handle.
+ */
+
+int meto::MPIContext::get_handle() { return comm_handle_; }
+
 /**
  * @brief Gets the identifying tag.
  * @returns The tag, as a string.

src/c++/mpi_context.h

@@ -55,6 +55,8 @@ class MPIContext {
   // Getters
   int get_size();
   int get_rank();
+  int get_handle();
+
   std::string get_tag() const;
 };
 

src/c++/writer/multi.cpp

@@ -17,11 +17,9 @@ meto::Multi::Multi(MPIContext const &mpi_context)
 
 /**
  * @brief  Opens a unique file per mpi rank
- *
- * @param[in] os  Output stream to write to
  */
 
-void meto::Multi::open_files(std::ofstream &os) {
+void meto::Multi::open_files() {
 
   // Append the MPI rank to the output filename.
   std::string mpi_filename_tail = "-" + std::to_string(mpi_context_.get_rank());
@@ -34,13 +32,12 @@ void meto::Multi::open_files(std::ofstream &os) {
  * @brief  The main write method. Includes filehandling and calls formatter
  *         strategy.
  *
- * @param[in] os       The output stream to write to
  * @param[in] hashvec  The vector containing all necessary data
  */
 
-void meto::Multi::write(std::ofstream &os, hashvec_t hashvec) {
-  open_files(os);
-  formatter_.execute_format(os, hashvec);
+void meto::Multi::write(hashvec_t hashvec) {
+  open_files();
+  formatter_.execute_format(os, hashvec, mpi_context_);
   os.flush();
   os.close();
 }

src/c++/writer/multi.h

@@ -28,15 +28,17 @@ namespace meto {
 class Multi : public Writer {
 
 private:
+  // Per-file output stream
+  std::ofstream os;
   // Method
-  void open_files(std::ofstream &os);
+  void open_files();
 
 public:
   // Constructor
   Multi(MPIContext const &);
 
   // Implementation of pure virtual function.
-  void write(std::ofstream &os, hashvec_t) override;
+  void write(hashvec_t) override;
 };
 
 } // namespace meto

src/c++/writer/single.cpp

@@ -0,0 +1,90 @@
+/* -----------------------------------------------------------------------------
+ *  (c) Crown copyright 2025 Met Office. All rights reserved.
+ *  The file LICENCE, distributed with this code, contains details of the terms
+ *  under which the code may be used.
+ * -----------------------------------------------------------------------------
+ */
+
+#include "single.h"
+
+/**
+ * @brief  Construct a new single file writer.
+ * @param[in] mpi_context  The MPI context the writer will use.
+ */
+
+meto::SingleFile::SingleFile(MPIContext const &mpi_context)
+    : meto::SingleFile::Writer(mpi_context) {}
+
+/**
+ * @brief  The main write method.
+ *
+ * @param[in] hashvec  The vector containing all necessary data
+ */
+void meto::SingleFile::write(hashvec_t hashvec) {
+  /* This is a complete cheat for now: ignore the ofstream and do
+   * everything through MPI IO.
+   */
+  std::ostringstream buffer; // Formatted output
+  std::string mpi_filename_tail = "-collated";
+
+  // Format the report on each task and buffer it on each task
+  formatter_.execute_format(buffer, hashvec, mpi_context_);
+
+  std::string filename = output_filename_ + mpi_filename_tail;
+
+#ifdef USE_VERNIER_MPI_STUB
+  /*
+   * Rather than dummy out all the MPI calls, replace with a simple
+   * file open and write when running without MPI.
+   */
+  std::ofstream os(filename);
+
+  os << buffer.str();
+  os.flush();
+  os.close();
+
+#else // USE_VERNIER_MPI_STUB
+
+  int length;              // Local string length
+  int max_length;          // Global maximum string length
+  MPI_Datatype mpi_buffer; // Buffer as a contiguous item
+  MPI_File file_handle;    // MPI File accessor
+  MPI_Status status;       // Result of write
+  MPI_Offset displacement; // Displacement in bytes on this rank.
+
+  // Global maximum string size is required on every task to set up
+  // the custom data type
+  length = static_cast<int>(buffer.str().length());
+  MPI_Allreduce(&length, &max_length, 1, MPI_INT, MPI_MAX,
+                mpi_context_.get_handle());
+
+  // Pad out with spaces
+  buffer << std::string(
+      static_cast<std::string::size_type>(max_length - length), ' ');
+
+  MPI_Type_contiguous(max_length, MPI_CHAR, &mpi_buffer);
+  MPI_Type_commit(&mpi_buffer);
+
+  // Open the global output file and create a view for each task which
+  // represents a unique, non-overlapping region
+  MPI_File_open(mpi_context_.get_handle(), filename.c_str(),
+                MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle);
+
+  displacement = static_cast<MPI_Offset>(mpi_context_.get_rank() * max_length *
+                                         static_cast<int>(sizeof(char)));
+  MPI_File_set_view(file_handle, displacement, MPI_CHAR, mpi_buffer, "native",
+                    MPI_INFO_NULL);
+
+  // Collective write operation
+  // Some evidence of memory leak risk with MPI_File_write_all.
+  MPI_File_write(file_handle, buffer.str().c_str(), max_length, MPI_CHAR,
+                 &status);
+
+  // Tidy up resources
+  MPI_File_close(&file_handle);
+  MPI_Type_free(&mpi_buffer);
+
+#endif // USE_VERNIER_MPI_STUB
+
+  return;
+}

src/c++/writer/single.h

@@ -0,0 +1,38 @@
+/* -----------------------------------------------------------------------------
+ *  (c) Crown copyright 2025 Met Office. All rights reserved.
+ *  The file LICENCE, distributed with this code, contains details of the terms
+ *  under which the code may be used.
+ * -----------------------------------------------------------------------------
+ */
+
+/**
+ * @file   single.h
+ * @brief  SingleFile class, derived from Writer.
+ *
+ */
+
+#ifndef VERNIER_SINGLE_H
+#define VERNIER_SINGLE_H
+
+#include <ostream>
+#include <sstream>
+
+#include "../mpi_context.h"
+#include "writer.h"
+
+namespace meto {
+
+/**
+ * @brief   Single parallel output file
+ * @details Creates a single file for all ranks using MPI IO
+ */
+
+class SingleFile : public Writer {
+public:
+  SingleFile(MPIContext const &);
+  void write(hashvec_t) override;
+};
+
+} // namespace meto
+
+#endif // VERNIER_SINGLE_H

src/c++/writer/writer.h

@@ -43,7 +43,7 @@ class Writer {
   virtual ~Writer() = default;
 
   // Pure virtual write method
-  virtual void write(std::ofstream &os, hashvec_t) = 0;
+  virtual void write(hashvec_t) = 0;
 };
 
 } // namespace meto

tests/system_tests/c++/CMakeLists.txt

@@ -34,8 +34,10 @@ if (BUILD_TESTS)
   endfunction()
 
   add_cxx_system_test(TestTags test-tags.cpp 1)
+  add_cxx_system_test(TestOutputs test-output-files.cpp 1)
 
   if (ENABLE_MPI)
     add_cxx_system_test(TestVernierMPIHeaders test-vernier-mpi-headers.cpp 2)
+    add_cxx_system_test(TestMPIOutputs test-output-files.cpp 2)
   endif()
 endif ()