diff --git a/documentation/Sphinx/user_guide/setting-up/setting-up.rst b/documentation/Sphinx/user_guide/setting-up/setting-up.rst index 44b35dfc..4461059c 100644 --- a/documentation/Sphinx/user_guide/setting-up/setting-up.rst +++ b/documentation/Sphinx/user_guide/setting-up/setting-up.rst @@ -114,11 +114,23 @@ Environment Variables ``VERNIER_IO_MODE`` - Determines the output mode to use. Currently only supports being set to - **multi** but single-file-output may be added in the future. + Determines the output mode to use. Currently supported values are: + + * **multi**: Every task writes to its own output file + + * **single**: All tasks write to a single global output file + + If the environment variable is not set, **multi** mode is used. ``VERNIER_OUTPUT_FILENAME`` - Sets the output filename, which is "vernier-output" by default. Vernier - will append the MPI rank onto the end of this name by default, resulting - in a file called `vernier-output-0` for the first MPI rank, for example. + Sets the output filename, which is ``vernier-output`` by + default. + + Vernier will append the MPI rank onto the end of this + name when running in **multi** mode, resulting in a file called + `vernier-output-0` for the first MPI rank, for example. + + Vernier will append ``-global`` to the name when running in + **single** mode and the file will contain formatted entries for + each task ordered by MPI rank. diff --git a/src/c++/CMakeLists.txt b/src/c++/CMakeLists.txt index 40ea7e50..c4029e9b 100644 --- a/src/c++/CMakeLists.txt +++ b/src/c++/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(${CMAKE_PROJECT_NAME} hashvec_handler.cpp writer/writer.cpp writer/multi.cpp + writer/single.cpp formatter.cpp hashvec.cpp vernier_gettime.cpp diff --git a/src/c++/formatter.cpp b/src/c++/formatter.cpp index 4830258d..32f234ec 100644 --- a/src/c++/formatter.cpp +++ b/src/c++/formatter.cpp @@ -45,7 +45,13 @@ meto::Formatter::Formatter() { * @param[in] hashvec Vector of data that the format method will operate on */ -void meto::Formatter::execute_format(std::ofstream &os, hashvec_t hashvec) { +void meto::Formatter::execute_format(std::ostream &os, hashvec_t hashvec, + MPIContext &mpi_context) { + // Add an MPI task identifier to each output file + os << "\n" + << "Task " << (mpi_context.get_rank() + 1) << " of " + << mpi_context.get_size() << " : MPI rank ID " << mpi_context.get_rank() + << "\n"; (this->*format_)(os, hashvec); } @@ -56,7 +62,7 @@ void meto::Formatter::execute_format(std::ofstream &os, hashvec_t hashvec) { * @param[in] hashvec Vector containing all the necessary data */ -void meto::Formatter::threads(std::ofstream &os, hashvec_t hashvec) { +void meto::Formatter::threads(std::ostream &os, hashvec_t hashvec) { // Write key os << "\n"; @@ -98,7 +104,7 @@ void meto::Formatter::threads(std::ofstream &os, hashvec_t hashvec) { * @param[in] hashvec Vector containing all the necessary data */ -void meto::Formatter::drhook(std::ofstream &os, hashvec_t hashvec) { +void meto::Formatter::drhook(std::ostream &os, hashvec_t hashvec) { int num_threads = 1; #ifdef _OPENMP diff --git a/src/c++/formatter.h b/src/c++/formatter.h index 41e203fe..deb10dba 100644 --- a/src/c++/formatter.h +++ b/src/c++/formatter.h @@ -16,12 +16,14 @@ #define FORMATTER_H #include +#include #ifdef _OPENMP #include #endif #include "hashvec.h" +#include "mpi_context.h" namespace meto { @@ -36,18 +38,18 @@ class Formatter { private: // Format method - void (Formatter::*format_)(std::ofstream &, hashvec_t); + void (Formatter::*format_)(std::ostream &, hashvec_t); // Individual formatter functions - void threads(std::ofstream &os, hashvec_t); - void drhook(std::ofstream &os, hashvec_t); + void threads(std::ostream &os, hashvec_t); + void drhook(std::ostream &os, hashvec_t); public: // Constructor explicit Formatter(); // Execute the format method - void execute_format(std::ofstream &os, hashvec_t); + void execute_format(std::ostream &os, hashvec_t, MPIContext &); }; } // namespace meto diff --git a/src/c++/hashvec_handler.cpp b/src/c++/hashvec_handler.cpp index 46ecf45c..5f063022 100644 --- a/src/c++/hashvec_handler.cpp +++ b/src/c++/hashvec_handler.cpp @@ -31,6 +31,8 @@ meto::HashVecHandler::HashVecHandler(MPIContext const &mpi_context) { // Allocate writer to be of required type. if (io_mode == "multi") { writer_strategy_ = std::make_unique(mpi_context); + } else if (io_mode == "single") { + writer_strategy_ = std::make_unique(mpi_context); } else { error_handler("Invalid IO mode choice", EXIT_FAILURE); } @@ -61,7 +63,4 @@ void meto::HashVecHandler::sort() { * */ -void meto::HashVecHandler::write() { - std::ofstream os; - writer_strategy_->write(os, hashvec_); -} +void meto::HashVecHandler::write() { writer_strategy_->write(hashvec_); } diff --git a/src/c++/hashvec_handler.h b/src/c++/hashvec_handler.h index 1c5d3a1d..04ba1f23 100644 --- a/src/c++/hashvec_handler.h +++ b/src/c++/hashvec_handler.h @@ -20,6 +20,7 @@ #include "hashvec.h" #include "mpi_context.h" #include "writer/multi.h" +#include "writer/single.h" #include "writer/writer.h" namespace meto { diff --git a/src/c++/mpi_context.cpp b/src/c++/mpi_context.cpp index e255928e..551273af 100644 --- a/src/c++/mpi_context.cpp +++ b/src/c++/mpi_context.cpp @@ -8,6 +8,9 @@ #include #include +#include +#include + #include "error_handler.h" #include "mpi_context.h" @@ -115,6 +118,13 @@ int meto::MPIContext::get_rank() { return comm_rank_; } int meto::MPIContext::get_size() { return comm_size_; } +/** + * @brief Gets the MPI communicator handle. + * @returns The MPI communicator handle. + */ + +int meto::MPIContext::get_handle() { return comm_handle_; } + /** * @brief Gets the identifying tag. * @returns The tag, as a string. diff --git a/src/c++/mpi_context.h b/src/c++/mpi_context.h index ef7c7c78..cb6668a7 100644 --- a/src/c++/mpi_context.h +++ b/src/c++/mpi_context.h @@ -55,6 +55,8 @@ class MPIContext { // Getters int get_size(); int get_rank(); + int get_handle(); + std::string get_tag() const; }; diff --git a/src/c++/writer/multi.cpp b/src/c++/writer/multi.cpp index a59e766e..2f72acbe 100644 --- a/src/c++/writer/multi.cpp +++ b/src/c++/writer/multi.cpp @@ -17,11 +17,9 @@ meto::Multi::Multi(MPIContext const &mpi_context) /** * @brief Opens a unique file per mpi rank - * - * @param[in] os Output stream to write to */ -void meto::Multi::open_files(std::ofstream &os) { +void meto::Multi::open_files() { // Append the MPI rank to the output filename. std::string mpi_filename_tail = "-" + std::to_string(mpi_context_.get_rank()); @@ -34,13 +32,12 @@ void meto::Multi::open_files(std::ofstream &os) { * @brief The main write method. Includes filehandling and calls formatter * strategy. * - * @param[in] os The output stream to write to * @param[in] hashvec The vector containing all necessary data */ -void meto::Multi::write(std::ofstream &os, hashvec_t hashvec) { - open_files(os); - formatter_.execute_format(os, hashvec); +void meto::Multi::write(hashvec_t hashvec) { + open_files(); + formatter_.execute_format(os, hashvec, mpi_context_); os.flush(); os.close(); } diff --git a/src/c++/writer/multi.h b/src/c++/writer/multi.h index e7eec06f..6c50d98a 100644 --- a/src/c++/writer/multi.h +++ b/src/c++/writer/multi.h @@ -28,15 +28,17 @@ namespace meto { class Multi : public Writer { private: + // Per-file output stream + std::ofstream os; // Method - void open_files(std::ofstream &os); + void open_files(); public: // Constructor Multi(MPIContext const &); // Implementation of pure virtual function. - void write(std::ofstream &os, hashvec_t) override; + void write(hashvec_t) override; }; } // namespace meto diff --git a/src/c++/writer/single.cpp b/src/c++/writer/single.cpp new file mode 100644 index 00000000..6e5c3feb --- /dev/null +++ b/src/c++/writer/single.cpp @@ -0,0 +1,90 @@ +/* ----------------------------------------------------------------------------- + * (c) Crown copyright 2025 Met Office. All rights reserved. + * The file LICENCE, distributed with this code, contains details of the terms + * under which the code may be used. + * ----------------------------------------------------------------------------- + */ + +#include "single.h" + +/** + * @brief Construct a new single file writer. + * @param[in] mpi_context The MPI context the writer will use. + */ + +meto::SingleFile::SingleFile(MPIContext const &mpi_context) + : meto::SingleFile::Writer(mpi_context) {} + +/** + * @brief The main write method. + * + * @param[in] hashvec The vector containing all necessary data + */ +void meto::SingleFile::write(hashvec_t hashvec) { + /* This is a complete cheat for now: ignore the ofstream and do + * everything through MPI IO. + */ + std::ostringstream buffer; // Formatted output + std::string mpi_filename_tail = "-collated"; + + // Format the report on each task and buffer it on each task + formatter_.execute_format(buffer, hashvec, mpi_context_); + + std::string filename = output_filename_ + mpi_filename_tail; + +#ifdef USE_VERNIER_MPI_STUB + /* + * Rather than dummy out all the MPI calls, replace with a simple + * file open and write when running without MPI. + */ + std::ofstream os(filename); + + os << buffer.str(); + os.flush(); + os.close(); + +#else // USE_VERNIER_MPI_STUB + + int length; // Local string length + int max_length; // Global maximum string length + MPI_Datatype mpi_buffer; // Buffer as a contiguous item + MPI_File file_handle; // MPI File accessor + MPI_Status status; // Result of write + MPI_Offset displacement; // Displacement in bytes on this rank. + + // Global maximum string size is required on every task to set up + // the custom data type + length = static_cast(buffer.str().length()); + MPI_Allreduce(&length, &max_length, 1, MPI_INT, MPI_MAX, + mpi_context_.get_handle()); + + // Pad out with spaces + buffer << std::string( + static_cast(max_length - length), ' '); + + MPI_Type_contiguous(max_length, MPI_CHAR, &mpi_buffer); + MPI_Type_commit(&mpi_buffer); + + // Open the global output file and create a view for each task which + // represents a unique, non-overlapping region + MPI_File_open(mpi_context_.get_handle(), filename.c_str(), + MPI_MODE_CREATE | MPI_MODE_WRONLY, MPI_INFO_NULL, &file_handle); + + displacement = static_cast(mpi_context_.get_rank() * max_length * + static_cast(sizeof(char))); + MPI_File_set_view(file_handle, displacement, MPI_CHAR, mpi_buffer, "native", + MPI_INFO_NULL); + + // Collective write operation + // Some evidence of memory leak risk with MPI_File_write_all. + MPI_File_write(file_handle, buffer.str().c_str(), max_length, MPI_CHAR, + &status); + + // Tidy up resources + MPI_File_close(&file_handle); + MPI_Type_free(&mpi_buffer); + +#endif // USE_VERNIER_MPI_STUB + + return; +} diff --git a/src/c++/writer/single.h b/src/c++/writer/single.h new file mode 100644 index 00000000..85daa626 --- /dev/null +++ b/src/c++/writer/single.h @@ -0,0 +1,38 @@ +/* ----------------------------------------------------------------------------- + * (c) Crown copyright 2025 Met Office. All rights reserved. + * The file LICENCE, distributed with this code, contains details of the terms + * under which the code may be used. + * ----------------------------------------------------------------------------- + */ + +/** + * @file single.h + * @brief SingleFile class, derived from Writer. + * + */ + +#ifndef VERNIER_SINGLE_H +#define VERNIER_SINGLE_H + +#include +#include + +#include "../mpi_context.h" +#include "writer.h" + +namespace meto { + +/** + * @brief Single parallel output file + * @details Creates a single file for all ranks using MPI IO + */ + +class SingleFile : public Writer { +public: + SingleFile(MPIContext const &); + void write(hashvec_t) override; +}; + +} // namespace meto + +#endif // VERNIER_SINGLE_H diff --git a/src/c++/writer/writer.h b/src/c++/writer/writer.h index 046ccbf0..d033b8a7 100644 --- a/src/c++/writer/writer.h +++ b/src/c++/writer/writer.h @@ -43,7 +43,7 @@ class Writer { virtual ~Writer() = default; // Pure virtual write method - virtual void write(std::ofstream &os, hashvec_t) = 0; + virtual void write(hashvec_t) = 0; }; } // namespace meto diff --git a/tests/system_tests/c++/CMakeLists.txt b/tests/system_tests/c++/CMakeLists.txt index 57ac07e6..2400c0d4 100644 --- a/tests/system_tests/c++/CMakeLists.txt +++ b/tests/system_tests/c++/CMakeLists.txt @@ -34,8 +34,10 @@ if (BUILD_TESTS) endfunction() add_cxx_system_test(TestTags test-tags.cpp 1) + add_cxx_system_test(TestOutputs test-output-files.cpp 1) if (ENABLE_MPI) add_cxx_system_test(TestVernierMPIHeaders test-vernier-mpi-headers.cpp 2) + add_cxx_system_test(TestMPIOutputs test-output-files.cpp 2) endif() endif () diff --git a/tests/system_tests/c++/test-output-files.cpp b/tests/system_tests/c++/test-output-files.cpp new file mode 100644 index 00000000..865d82c1 --- /dev/null +++ b/tests/system_tests/c++/test-output-files.cpp @@ -0,0 +1,136 @@ +/*----------------------------------------------------------------------------*\ + (c) Crown copyright 2025 Met Office. All rights reserved. + The file LICENCE, distributed with this code, contains details of the terms + under which the code may be used. +\*----------------------------------------------------------------------------*/ + +// System test to check that the file outputs are correct + +#include "vernier.h" +#include "vernier_mpi.h" + +#include +#include +#include +#include + +/* + * Check the first few lines of the output file + */ +bool check_output_file(std::string path, std::string format) { + std::filebuf fb; + std::string buffer; + std::string expected; + + if (!fb.open(path, std::ios::in)) { + std::cerr << "failed to open " << path << std::endl; + return false; + } + + std::istream input(&fb); + + // Check the header lines + std::getline(input, buffer); + if (buffer.compare("") != 0) { + std::cerr << "Invalid line: " << buffer << std::endl; + return false; + } + + std::getline(input, buffer); + if (buffer.compare(0, 9, "Task 1 of") != 0) { + std::cerr << "Invalid line: " << buffer << std::endl; + return false; + } + + // Match the next line to the start of an output format + if (format.compare("drhook") == 0) { + expected = "Profiling on "; + } else if (format.compare("threads") == 0) { + // Skip extra blank line in threads format + std::getline(input, buffer); + expected = "region_name@thread_id"; + } else { + std::cerr << "unknown format" << std::endl; + return false; + } + + std::getline(input, buffer); + if (buffer.compare(0, expected.length(), expected) != 0) { + std::cerr << "Invalid next line" << std::endl; + std::cerr << "Got: " << buffer << std::endl; + std::cerr << "Expected: " << expected << std::endl; + return false; + } + + return true; +} + +/* + * Generate some profiler output and check the contents + */ +bool create_output(std::string mode, std::string format, int rank) { + std::string path = format + "-vernier-output"; + meto::vernier.init(MPI_COMM_WORLD); + + // Create some data + auto vnr_handle = meto::vernier.start("main"); + + // Create some asymmetry in the number of calls made by different ranks. + if (rank % 2 == 0) { + auto vnr_handle_even_rank = meto::vernier.start("even_rank"); + meto::vernier.stop(vnr_handle_even_rank); + } + + meto::vernier.stop(vnr_handle); + + setenv("VERNIER_OUTPUT_MODE", mode.c_str(), 1); + setenv("VERNIER_OUTPUT_FORMAT", format.c_str(), 1); + setenv("VERNIER_OUTPUT_FILENAME", path.c_str(), 1); + + meto::vernier.write(); + + unsetenv("VERNIER_OUTPUT_FILENAME"); + unsetenv("VERNIER_OUTPUT_FORMAT"); + unsetenv("VERNIER_OUTPUT_MODE"); + + meto::vernier.finalize(); + + // Append the expected extension + if (mode.compare("multi") == 0) { + path.append("-0"); + } else if (mode.compare("single") == 0) { + path.append("-collated"); + } else { + std::cerr << "unknown mode" << std::endl; + return false; + } + + if (rank == 0 && !check_output_file(path, format)) { + std::cerr << mode << " file " << format << " test failed" << std::endl; + return false; + } + + return true; +} + +/* + * Main function + */ +int main() { + int rank; + std::string modes[] = {"multi", "single"}; + std::string formats[] = {"drhook", "threads"}; + + MPI_Init(NULL, NULL); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + for (auto mode : modes) { + for (auto format : formats) { + if (!create_output(mode, format, rank)) { + MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); + } + } + } + + MPI_Finalize(); +} diff --git a/tests/unit_tests/c++/test_proftests.cpp b/tests/unit_tests/c++/test_proftests.cpp index 9ff3bb79..47bd67de 100644 --- a/tests/unit_tests/c++/test_proftests.cpp +++ b/tests/unit_tests/c++/test_proftests.cpp @@ -115,7 +115,7 @@ TEST(DeathTest, InvalidIOModeTest) { { meto::MPIContext mpi_context; - const char *invalidIOMode = "single"; + const char *invalidIOMode = "invalid-mode"; setenv("VERNIER_OUTPUT_MODE", invalidIOMode, 1); meto::HashVecHandler object(mpi_context);