diff --git a/docs/configuration.rst b/docs/configuration.rst index aa253be1..fef3dd0b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -497,7 +497,11 @@ mesh The entire mesh may be written to Paraview’s vtu format for visualization in Paraview and for analysis. This is denoted by a ``"mesh":{ ... }`` key. -For more details, please see the :ref:`output` section. +For more details on the output files, please see the :ref:`output` section. + +.. note:: + The default behaviour of ``mesh`` output is not to output any vtu files. The user must specify the output frequency + or timing. .. confval:: base_name @@ -509,7 +513,7 @@ For more details, please see the :ref:`output` section. :type: ``[ "variable", ... ]`` - The default behaviour to is write every variable at each timestep. This may produce an undesirable amount of output. This takes a list of variables to output. + The default behaviour to is to write every variable at each timestep. This may produce an undesirable amount of output. This takes a list of variables to output. .. code:: json @@ -525,7 +529,10 @@ For more details, please see the :ref:`output` section. :type: int :default: 1 - Frequency can be set to write ever *N* timesteps. + Frequency can be set to write ever *N* timesteps. If the automatic checkpoint system is used, this might not + produce a consistent output frequency and time. For example, suppose daily midnight output was chosen + (``frequency:24``) as the model simulation starts at 00:00. However, the auto-checkpoint suspends at the 8am + timestep, the next output will be at 8am instead of midnight. .. confval write_parameters:: @@ -541,8 +548,26 @@ For more details, please see the :ref:`output` section. Write each MPI rank's ghost face data to vtu output +.. confval specific_datetime:: + + :type: string + :default: "" + + Output at a specific date-time, given in the iso format, e.g., ``"specific_datetime": "20191227T160000"`` + + +.. confval specific_time:: + + :type: string + :default: "" + + Output at a specific time every day, given in a "HH:MM" 24hr-format, e.g., ``"specific_time": "14:00"`` + + Example: +All of the frequency options can mixed together, allowing more complex output frequency selections + .. code:: json "output": @@ -556,6 +581,7 @@ Example: "iswr" ], "frequency": "24", + "specific_datetime": "20191227T160000", "write_parameters": false, "write_ghost_neighbors": false } diff --git a/src/core.cpp b/src/core.cpp index 4a22af07..72ca8ca1 100644 --- a/src/core.cpp +++ b/src/core.cpp @@ -1041,21 +1041,37 @@ void core::config_output(pt::ptree &value) } catch (pt::ptree_bad_path &e) { - SPDLOG_DEBUG("Writing all variables to output mesh"); + SPDLOG_WARN("Writing all variables to output mesh"); } - out.frequency = itr.second.get("frequency",1); //defaults to every timestep - SPDLOG_DEBUG("Output every {} timesteps.", out.frequency); + out.frequency = itr.second.get_optional("frequency"); //defaults to every timestep - out.only_last_n = itr.second.get("only_last_n",-1); //defaults to keep everything - SPDLOG_DEBUG("Output only on last n = {} timesteps", out.only_last_n); + out.only_last_n = itr.second.get_optional("only_last_n"); - if(out.frequency > 1 && out.only_last_n != -1) + if(out.frequency && out.only_last_n) { SPDLOG_WARN("Only only_last_n output option will be used"); + out.frequency.reset(); } + auto specific_datetime = itr.second.get_optional("specific_datetime"); + if(specific_datetime) + { + out.specific_datetime = boost::posix_time::ptime(boost::posix_time::from_iso_string(*specific_datetime)); + } + + auto specific_time = itr.second.get_optional("specific_time"); + if(specific_time) + { + // ptime needs a real date, but we will only ever be checking the minute and hour + auto time = "3000-01-01 " + *specific_time + ":00"; + out.specific_time = boost::posix_time::ptime(boost::posix_time::time_from_string(time)); + } + + out.mesh_output_formats.push_back(output_info::mesh_outputs::vtu); + out.name = "vtu output"; + out.list_outputs(); } else { @@ -2125,263 +2141,247 @@ void core::run() size_t max_ts = _metdata->n_timestep(); bool done = false; + while (!done) + { + boost::posix_time::ptime t; + _global->_current_date = _metdata->current_time(); - while (!done) - { - boost::posix_time::ptime t; - - _global->_current_date = _metdata->current_time(); - - SPDLOG_DEBUG("Timestep: {}\tstep#{}", boost::posix_time::to_simple_string(_global->posix_time()), current_ts); + SPDLOG_DEBUG("Timestep: {}\tstep#{}", boost::posix_time::to_simple_string(_global->posix_time()), current_ts); - std::stringstream ss; - ss << _global->posix_time(); + std::stringstream ss; + ss << _global->posix_time(); - c.tic(); - size_t chunks = 0; - try + c.tic(); + size_t chunks = 0; + try + { + for (auto &itr : _chunked_modules) { - for (auto &itr : _chunked_modules) - { - if (itr.at(0)->parallel_type() == module_base::parallel::data) - { + if (itr.at(0)->parallel_type() == module_base::parallel::data) + { #ifdef OMP_SAFE_EXCEPTION - ompException e; + ompException e; #endif - #pragma omp parallel for - for (size_t i = 0; i < _mesh->size_faces(); i++) - { - auto face = _mesh->face(i); - if (point_mode.enable && face->_debug_name != _outputs[0].name) - continue; + #pragma omp parallel for + for (size_t i = 0; i < _mesh->size_faces(); i++) + { + auto face = _mesh->face(i); + if (point_mode.enable && face->_debug_name != _outputs[0].name) + continue; - //module calls - for (auto &jtr : itr) - { + //module calls + for (auto &jtr : itr) + { #ifdef OMP_SAFE_EXCEPTION - e.Run( - [&] - { + e.Run( + [&] + { #endif - jtr->run(face); + jtr->run(face); #ifdef OMP_SAFE_EXCEPTION - }); + }); #endif - } - } + } + } #ifdef OMP_SAFE_EXCEPTION - e.Rethrow(); + e.Rethrow(); #endif - } else + } else + { + //module calls for domain parallel + for (auto &jtr : itr) { - //module calls for domain parallel - for (auto &jtr : itr) - { - jtr->run(_mesh); - } + jtr->run(_mesh); } + } - chunks++; + chunks++; - } } - catch (exception_base &e) - { - SPDLOG_ERROR("Exception at timestep: {}", boost::posix_time::to_simple_string(_global->posix_time())); - //if we die in a module, try to dump our time series out so we can figure out wtf went wrong - SPDLOG_ERROR("Exception has occured. Timeseries and meshes WILL BE INCOMPLETE!"); - *_end_ts = _global->posix_time(); - done = true; - SPDLOG_ERROR(boost::diagnostic_information(e)); + } + catch (exception_base &e) + { + SPDLOG_ERROR("Exception at timestep: {}", boost::posix_time::to_simple_string(_global->posix_time())); + //if we die in a module, try to dump our time series out so we can figure out wtf went wrong + SPDLOG_ERROR("Exception has occured. Timeseries and meshes WILL BE INCOMPLETE!"); + *_end_ts = _global->posix_time(); + done = true; + SPDLOG_ERROR(boost::diagnostic_information(e)); - } - catch(std::exception& e) - { - SPDLOG_ERROR("Exception at timestep: {}", boost::posix_time::to_simple_string(_global->posix_time())); - SPDLOG_ERROR(e.what()); - *_end_ts = _global->posix_time(); - done = true; - SPDLOG_ERROR(e.what()); - } + } + catch(std::exception& e) + { + SPDLOG_ERROR("Exception at timestep: {}", boost::posix_time::to_simple_string(_global->posix_time())); + SPDLOG_ERROR(e.what()); + *_end_ts = _global->posix_time(); + done = true; + SPDLOG_ERROR(e.what()); + } - //check that we actually need a mesh output. - for (auto &itr : _outputs) + // check that we actually need a mesh output this timestep + for (auto &itr : _outputs) + { + if(itr.type == output_info::output_type::mesh && + itr.should_output(max_ts, current_ts, _global->_current_date)) { - if(itr.type == output_info::output_type::mesh) - { - std::vector output; - output.assign(itr.variables.begin(),itr.variables.end()); //convert to list to match internal lists + std::vector output; + output.assign(itr.variables.begin(),itr.variables.end()); //convert to list to match internal lists - _mesh->update_vtk_data(output); //update the internal vtk mesh - break; // we're done as soon as we've called update once. No need to do it multiple times. - } + _mesh->update_vtk_data(output); //update the internal vtk mesh + break; // we're done as soon as we've called update once. No need to do it multiple times. } + } - // save the current state - if(_checkpoint_opts.should_checkpoint(current_ts, - (max_ts-1) == current_ts, - _hpc_scheduler_info - )) // -1 because current_ts is 0 indexed - { - SPDLOG_DEBUG("Checkpointing..."); + // save the current state + if(_checkpoint_opts.should_checkpoint(current_ts, + (max_ts-1) == current_ts, + _hpc_scheduler_info + )) // -1 because current_ts is 0 indexed + { + SPDLOG_DEBUG("Checkpointing..."); - netcdf savestate; //file to save to when checkpointing. + netcdf savestate; //file to save to when checkpointing. - auto timestamp = _global->posix_time() + boost::posix_time::seconds(_global->_dt); - //also write it out in seconds because netcdf is struggling with the string - unsigned long long int ts_sec = _global->posix_time_int()+_global->_dt; + auto timestamp = _global->posix_time() + boost::posix_time::seconds(_global->_dt); + //also write it out in seconds because netcdf is struggling with the string + unsigned long long int ts_sec = _global->posix_time_int()+_global->_dt; - auto timestr = boost::posix_time::to_iso_string(timestamp); // start from current TS + dt + auto timestr = boost::posix_time::to_iso_string(timestamp); // start from current TS + dt - size_t rank = 0; + size_t rank = 0; #ifdef USE_MPI - rank = _comm_world.rank(); + rank = _comm_world.rank(); #endif - auto dirpath = _checkpoint_opts.ckpt_path / timestr; - boost::filesystem::create_directories(dirpath); + auto dirpath = _checkpoint_opts.ckpt_path / timestr; + boost::filesystem::create_directories(dirpath); - //this parses both the input and the output paths for the checkpoint. - auto fname = ("chkp"+timestr + "_" + std::to_string(rank) + ".nc"); - auto f = dirpath / fname; - savestate.create( f.string()); + //this parses both the input and the output paths for the checkpoint. + auto fname = ("chkp"+timestr + "_" + std::to_string(rank) + ".nc"); + auto f = dirpath / fname; + savestate.create( f.string()); - c.tic(); - for (auto &itr : _chunked_modules) + c.tic(); + for (auto &itr : _chunked_modules) + { + //module calls + for (auto &jtr : itr) { - //module calls - for (auto &jtr : itr) - { - jtr->checkpoint(_mesh, savestate); - } + jtr->checkpoint(_mesh, savestate); } + } - auto& ids = _mesh->get_global_IDs(); - savestate.create_variable1D("global_id",ids.size()); + auto& ids = _mesh->get_global_IDs(); + savestate.create_variable1D("global_id",ids.size()); - for (size_t i = 0; i < ids.size(); i++) - { - savestate.put_var1D("global_id", i, ids[i]); - } + for (size_t i = 0; i < ids.size(); i++) + { + savestate.put_var1D("global_id", i, ids[i]); + } - savestate.get_ncfile().putAtt("restart_time",boost::posix_time::to_simple_string(timestamp)); - savestate.get_ncfile().putAtt("restart_time_sec", netCDF::ncUint64,ts_sec); + savestate.get_ncfile().putAtt("restart_time",boost::posix_time::to_simple_string(timestamp)); + savestate.get_ncfile().putAtt("restart_time_sec", netCDF::ncUint64,ts_sec); - pt::ptree tree; + pt::ptree tree; - int nranks = 1; + int nranks = 1; #ifdef USE_MPI - nranks = _comm_world.size(); + nranks = _comm_world.size(); #endif - tree.put("ranks", nranks); - tree.put("restart_time_sec", ts_sec); - tree.put("startdate", timestr); + tree.put("ranks", nranks); + tree.put("restart_time_sec", ts_sec); + tree.put("startdate", timestr); - pt::ptree files; + pt::ptree files; - pt::ptree tmp_files; - for (size_t i = 0; i < nranks; ++i) - { - pt::ptree s; + pt::ptree tmp_files; + for (size_t i = 0; i < nranks; ++i) + { + pt::ptree s; - s.put("", timestr +"/" + "chkp"+timestr + "_" + std::to_string(i) + ".nc"); - tmp_files.push_back(std::make_pair("", s)); - } - tree.add_child("files", tmp_files); + s.put("", timestr +"/" + "chkp"+timestr + "_" + std::to_string(i) + ".nc"); + tmp_files.push_back(std::make_pair("", s)); + } + tree.add_child("files", tmp_files); - if(rank == 0) - { - pt::write_json( - (_checkpoint_opts.ckpt_path / ("checkpoint_" + timestr + ".np" + std::to_string(nranks) + ".json")).string(), - tree); - } + if(rank == 0) + { + pt::write_json( + (_checkpoint_opts.ckpt_path / ("checkpoint_" + timestr + ".np" + std::to_string(nranks) + ".json")).string(), + tree); + } - SPDLOG_DEBUG("Done checkpoint [ {} s]", c.toc()); + SPDLOG_DEBUG("Done checkpoint [ {} s]", c.toc()); - // if we checkpointed because we are out of time, we need to stop the simulation - if(_checkpoint_opts.checkpoint_request_terminate) - { - done = true; + // if we checkpointed because we are out of time, we need to stop the simulation + if(_checkpoint_opts.checkpoint_request_terminate) + { + done = true; - // we bailed early because of wall clock, so this is not a clean exit - clean_exit = false; - } + // we bailed early because of wall clock, so this is not a clean exit + clean_exit = false; } + } - for (auto &itr : _outputs) + for (auto &itr : _outputs) + { + if (itr.type == output_info::output_type::mesh) { - if (itr.type == output_info::output_type::mesh) - { - // check if we should output or not - bool should_output = false; - - if(itr.only_last_n != -1) - { - auto ts_left = max_ts - current_ts; - if( ts_left <= itr.only_last_n) // if we are within the last n timesteps, output - should_output = true; - } - else - { - if(current_ts % itr.frequency == 0) - should_output = true; - } - + // check if we should output or not + bool do_output = itr.should_output(max_ts, current_ts, _global->_current_date); + if(do_output) + { - if(should_output) + #pragma omp parallel { - - #pragma omp parallel + #pragma omp single { - #pragma omp single + for (auto jtr : itr.mesh_output_formats) { - for (auto jtr : itr.mesh_output_formats) + #pragma omp task { - #pragma omp task - { - std::string base_name = itr.fname + std::to_string(_global->posix_time_int()); - boost::filesystem::path p(base_name); + std::string base_name = itr.fname + std::to_string(_global->posix_time_int()); + boost::filesystem::path p(base_name); - if (jtr == output_info::mesh_outputs::vtu ) - { + if (jtr == output_info::mesh_outputs::vtu ) + { - // this really only works if we let rank0 handle the io. - // If we let each process do it, they walk all over each other's output + // this really only works if we let rank0 handle the io. + // If we let each process do it, they walk all over each other's output #ifdef USE_MPI - if(_comm_world.rank() == 0) + if(_comm_world.rank() == 0) + { + for(int rank = 0; rank < _comm_world.size(); rank++) { - for(int rank = 0; rank < _comm_world.size(); rank++) - { #else - int rank = 0; + int rank = 0; #endif - pt::ptree &dataset = pvd.add("VTKFile.Collection.DataSet", ""); - dataset.add(".timestep", _global->posix_time_int()); - dataset.add(".group", ""); - dataset.add(".part", rank); - dataset.add(".file", p.filename().string()+"_"+std::to_string(rank) + ".vtu"); + pt::ptree &dataset = pvd.add("VTKFile.Collection.DataSet", ""); + dataset.add(".timestep", _global->posix_time_int()); + dataset.add(".group", ""); + dataset.add(".part", rank); + dataset.add(".file", p.filename().string()+"_"+std::to_string(rank) + ".vtu"); #ifdef USE_MPI - } } + } #endif - //because a full path can be provided for the base_name, we need to strip this off - //to make it a relative path in the xml file. + //because a full path can be provided for the base_name, we need to strip this off + //to make it a relative path in the xml file. #ifdef USE_MPI - _mesh->write_vtu(base_name + "_"+std::to_string(_comm_world.rank() )+ ".vtu"); + _mesh->write_vtu(base_name + "_"+std::to_string(_comm_world.rank() )+ ".vtu"); #else - _mesh->write_vtu(base_name + "_"+std::to_string(rank)+ ".vtu"); + _mesh->write_vtu(base_name + "_"+std::to_string(rank)+ ".vtu"); #endif - } } } } @@ -2389,6 +2389,7 @@ void core::run() } } } + } //If we are output a timeseries at specific triangles, we do that here //Each output knows what face it corresponds to diff --git a/src/core.hpp b/src/core.hpp index 544813d1..5b344a0a 100644 --- a/src/core.hpp +++ b/src/core.hpp @@ -333,7 +333,7 @@ class core { public: output_info(): - frequency{1}, fname{""}, + fname{""}, latitude{0}, longitude{0}, name{""}, x{0}, y{0}, @@ -341,6 +341,7 @@ class core { face = nullptr; } + enum output_type { time_series, @@ -353,6 +354,59 @@ class core ascii }; + // Should we output? + bool should_output(const size_t& max_ts, + const size_t& current_ts, + const boost::posix_time::ptime& _current_date + ) + { + bool should_output = false; + + if(only_last_n) + { + auto ts_left = max_ts - current_ts; + if( ts_left <= *only_last_n) // if we are within the last n timesteps, output + should_output = true; + } + + if(frequency) + { + if(current_ts % *frequency == 0) + should_output = true; + } + + if(specific_time) + { + if( (_current_date.time_of_day().hours() == specific_time->time_of_day().hours()) && + (_current_date.time_of_day().minutes() == specific_time->time_of_day().minutes()) ) + should_output = true; + } + + if(specific_datetime) + { + if(_current_date == *specific_datetime) + should_output = true; + } + + return should_output; + + } + + // print to stdout DEBUG all the valid outputs selected + void list_outputs() + { + SPDLOG_DEBUG("Output frequency options for {}", name); + + if(only_last_n) + SPDLOG_DEBUG("\tonly_last_n = {}", *only_last_n); + if(frequency) + SPDLOG_DEBUG("\tfrequency = {}", *frequency); + if(specific_time) + SPDLOG_DEBUG("\tspecific_time = {}", std::to_string(specific_time->time_of_day().hours()) + ":" + std::to_string(specific_time->time_of_day().minutes())); + if(specific_datetime) + SPDLOG_DEBUG("\tspecific_datetime = {}", boost::posix_time::to_simple_string(*specific_datetime)); + } + output_type type; // the type of output std::string name; std::vector mesh_output_formats; @@ -366,14 +420,24 @@ class core double x; double y; - std::set variables; mesh_elem face; timeseries ts; - size_t frequency; + + // Output options + + // every n timesteps + boost::optional frequency; + + // at a specific date-time + boost::optional specific_datetime; + + // at a specific time + boost::optional specific_time; + //Only output the last n timesteps. -1 = all - size_t only_last_n; + boost::optional only_last_n; };