diff --git a/Cargo.toml b/Cargo.toml index 9ade134..dff812a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "gourd" -version = "1.2.0" +version = "1.2.1" edition = "2021" default-run = "gourd" authors = [ @@ -64,6 +64,9 @@ elf = "0.7.4" # To execute threads locally using a thread-pool executor. tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +tokio-stream = { version = "0.1.17", default-features = false } +futures = { version = "0.3.31", default-features = false, features = ["std"] } +num_cpus = "1.16.0" # To encode/decode data in gourd.toml and other Gourd files. toml = "0.8.12" diff --git a/README.md b/README.md index 921b79f..36b722b 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ The distributed installers are: (One of these should be put in place of `[your s Customizing installation folders is explained [here](https://gourd.chla.cz/manpages/maintainer.pdf). +See [the releases page on Github](https://github.com/ConSol-Lab/gourd/releases) for details on installing or updating to the latest version of `gourd`. + ### Usage Verify that `gourd` is installed by running: @@ -52,9 +54,9 @@ There are extensive tutorials and documentation which can be accessed in many di With a web browser: -- [**gourd.1 in your browser!**](https://gourd.chla.cz/manpages/gourd.1.html), supports both dark and light mode! -- [**gourd.toml.5 in your browser!**](https://gourd.chla.cz/manpages/gourd.toml.5.html), supports both dark and light mode! -- [**gourd-tutorial.7 in your browser!**](https://gourd.chla.cz/manpages/gourd-tutorial.7.html), supports both dark and light mode! +- [**gourd.1 in your browser!**](https://andreats.com/gourd/gourd.1.html), supports both dark and light mode! +- [**gourd.toml.5 in your browser!**](https://andreats.com/gourd/gourd.toml.5.html), supports both dark and light mode! +- [**gourd-tutorial.7 in your browser!**](https://andreats.com/gourd/gourd-tutorial.7.html), supports both dark and light mode! As a manpage, with the `man` command: ``` @@ -65,9 +67,9 @@ $ man gourd-tutorial As a PDF file: -- [**gourd.1 in your PDF reader!**](https://gourd.chla.cz/manpages/gourd.1.pdf) -- [**gourd.toml.5 in your PDF reader!**](https://gourd.chla.cz/manpages/gourd.toml.5.pdf) -- [**gourd-tutorial.7 in your PDF reader!**](https://gourd.chla.cz/manpages/gourd-tutorial.7.pdf) +- [**gourd.1 in your PDF reader!**](https://andreats.com/gourd/gourd.1.pdf) +- [**gourd.toml.5 in your PDF reader!**](https://andreats.com/gourd/gourd.toml.5.pdf) +- [**gourd-tutorial.7 in your PDF reader!**](https://andreats.com/gourd/gourd-tutorial.7.pdf) Please refer to these if you want to familiarize yourself with the software. @@ -96,8 +98,12 @@ An artifact is available from the GitLab pipeline `documentation` job, or ### Authors -- Mikołaj Gazeel, m.j.gazeel@student.tudelft.nl +- Mikołaj Gazeel, m.j.gazeel@tudelft.nl - Lukáš Chládek, l@chla.cz -- Ανδρέας Τσατσάνης, a.tsatsanis@student.tudelft.nl -- Rūta Giedrytė, r.giedryte@student.tudelft.nl +- Ανδρέας Τσατσάνης, a.tsatsanis@tudelft.nl +- Rūta Giedrytė, r.giedryte@tudelft.nl - Jan Piotrowski, me@jan.wf + +### Maintainers +- Ανδρέας Τσατσάνης, a.tsatsanis@tudelft.nl + diff --git a/docs/maintainer/building.tex b/docs/maintainer/building.tex index 70e2204..3cec3d6 100644 --- a/docs/maintainer/building.tex +++ b/docs/maintainer/building.tex @@ -64,6 +64,10 @@ \subsection{Documentation} All of the resulting files are placed in: \texttt{target/release/manpages}. +To build the library documentation as well, run: +\begin{verbatim} + cargo doc --no-deps --color=always --all-features --release +\end{verbatim} \subsection{Distribution} \textcolor{red!30!black}{\textbf{ diff --git a/docs/maintainer/title.tex b/docs/maintainer/title.tex index 8272ce6..ce70317 100644 --- a/docs/maintainer/title.tex +++ b/docs/maintainer/title.tex @@ -10,13 +10,9 @@ { Lukáš Chládek - Rūta Giedrytė - Mikołaj Gazeel Ανδρέας Τσατσάνης - - Jan Piotrowski } \vspace{0.5cm} diff --git a/docs/maintainer/version-history/section.tex b/docs/maintainer/version-history/section.tex index 35aba80..7434b5f 100644 --- a/docs/maintainer/version-history/section.tex +++ b/docs/maintainer/version-history/section.tex @@ -5,6 +5,11 @@ \section{Version History} \input{version-history/definitions} % \version{x} for start of version x section +\version{1.2.1}{Sponge Gourd}{} +Minor patch, adds the \texttt{local.num\_threads} option in +the config, as well automatically detecting the number of CPU +cores on the system. Added the \texttt{gourd --version} command. + \version{1.2.0}{Sponge Gourd}{} Major internal reworkings, redesigned \texttt{gourd analyse}. diff --git a/docs/user/gourd-tutorial.7.tex b/docs/user/gourd-tutorial.7.tex index 4c00c09..713a634 100644 --- a/docs/user/gourd-tutorial.7.tex +++ b/docs/user/gourd-tutorial.7.tex @@ -363,12 +363,11 @@ \Prog{gourd.toml(5)} - \section{AUTHORS} - Ανδρέας Τσατσάνης <\Email{a.tsatsanis@student.tudelft.nl}>\\[0.1cm]\MANbr - Rūta Giedrytė <\Email{r.giedryte@student.tudelft.nl}>\\[0.1cm]\MANbr - Mikołaj Gazeel <\Email{m.j.gazeel@student.tudelft.nl}>\\[0.1cm]\MANbr - Jan Piotrowski <\Email{me@jan.wf}>\\[0.1cm]\MANbr + \section{CONTACT} + Ανδρέας Τσατσάνης <\Email{a.tsatsanis@tudelft.nl}>\\[0.1cm]\MANbr Lukáš Chládek <\Email{l@chla.cz}>\\[0.1cm]\MANbr + Mikołaj Gazeel <\Email{m.j.gazeel@tudelft.nl}>\\[0.1cm]\MANbr + %@% IF LATEX %@% \end{adjustwidth} %@% END-IF %@% diff --git a/docs/user/gourd.1.tex b/docs/user/gourd.1.tex index 058161c..5486d72 100644 --- a/docs/user/gourd.1.tex +++ b/docs/user/gourd.1.tex @@ -354,9 +354,9 @@ \subsubsection{Summary} The \Prog{gourd} \Arg{continue} command schedules runs that are part of an existing - experiment, but have not yet been scheduled. + experiment, but have not yet been scheduled. This includes runs created by \Prog{gourd} \Arg{rerun}, as well as runs that were not scheduled due to a run limit. For example, an experiment with 30,000 distinct runs can be scheduled in three batches - of 10,000 each if that is the maximum number of queued supercomputer jobs. + of 10,000 each if that is the maximum number of queued supercomputer jobs. \subsubsection{Synopsis} \Prog{gourd} \Arg{continue} @@ -535,7 +535,8 @@ according to the newest available data. The option \texttt{--format} can be used to specify whether the plot output - should be in PNG or SVG format. + should be in PNG or SVG format, for example: + \Prog{gourd} \Arg{analyse} \Arg{plot} \Arg{--format="png"} (png is also the default output) \subsection{GOURD VERSION} @@ -572,12 +573,10 @@ \Prog{gourd.toml(5)} - \section{AUTHORS} - Rūta Giedrytė <\Email{r.giedryte@student.tudelft.nl}>\\[0.1cm]\MANbr + \section{CONTACT} + Ανδρέας Τσατσάνης <\Email{a.tsatsanis@tudelft.nl}>\\[0.1cm]\MANbr Lukáš Chládek <\Email{l@chla.cz}>\\[0.1cm]\MANbr - Jan Piotrowski <\Email{me@jan.wf}>\\[0.1cm]\MANbr - Mikołaj Gazeel <\Email{m.j.gazeel@student.tudelft.nl}>\\[0.1cm]\MANbr - Ανδρέας Τσατσάνης <\Email{a.tsatsanis@student.tudelft.nl}>\\[0.1cm]\MANbr + Mikołaj Gazeel <\Email{m.j.gazeel@tudelft.nl}>\\[0.1cm]\MANbr %@% IF LATEX %@% \end{adjustwidth} diff --git a/docs/user/gourd.toml.5.tex b/docs/user/gourd.toml.5.tex index 01b4278..f8165f4 100644 --- a/docs/user/gourd.toml.5.tex +++ b/docs/user/gourd.toml.5.tex @@ -193,6 +193,23 @@ mem_per_cpu = 512 \end{verbatim} + \section{LOCAL} + \begin{Description}[Options]\setlength{\itemsep}{0cm} + \item[\Opt{num\_threads?} = number] + How many threads should \Prog{gourd} \Arg{run} \Arg{local} use. + \end{Description} + + \subsection{NUM\_THREADS} + For the parallel execution of \Prog{gourd} \Arg{run} \Arg{local} you can + limit the number of threads that will be used by specifying: + \begin{verbatim} +[local] +num_threads = 8 + \end{verbatim} + If this option is not specified, the program will try to detect the number + of CPUs present on the system, and use that number of threads. Setting a + value of 0 will result in a number of threads equal to the number of runs in + the program (and the OS will limit the resource use thereafter). \section{PROGRAMS} Multiple programs can be specified. @@ -665,12 +682,11 @@ \section{SEE ALSO} \Prog{gourd(1)} \Prog{gourd-tutorial(7)} - \section{AUTHORS} + \section{CONTACT} + Ανδρέας Τσατσάνης <\Email{a.tsatsanis@tudelft.nl}>\\[0.1cm]\MANbr Lukáš Chládek <\Email{l@chla.cz}>\\[0.1cm]\MANbr - Rūta Giedrytė <\Email{r.giedryte@student.tudelft.nl}>\\[0.1cm]\MANbr - Ανδρέας Τσατσάνης <\Email{a.tsatsanis@student.tudelft.nl}>\\[0.1cm]\MANbr - Mikołaj Gazeel <\Email{m.j.gazeel@student.tudelft.nl}>\\[0.1cm]\MANbr - Jan Piotrowski <\Email{me@jan.wf}> + Mikołaj Gazeel <\Email{m.j.gazeel@tudelft.nl}>\\[0.1cm]\MANbr + %@% IF LATEX %@% \end{adjustwidth} %@% END-IF %@% diff --git a/docs/user/html/preamble.html b/docs/user/html/preamble.html index fc321df..0423596 100644 --- a/docs/user/html/preamble.html +++ b/docs/user/html/preamble.html @@ -36,18 +36,18 @@ The top bar contains links to the three documentation pages that gourd ships with.

You may also access the documentation in manpage format and PDF format.

- For more information go to the repostory: on the TuDelft GitLab! + For more information go to the repository on GitHub!

-Technical documentation: You can view the rustdoc generated documentation here +Technical documentation: You can view the rustdoc generated documentation here
diff --git a/src/gourd/analyse/tests/plotting.rs b/src/gourd/analyse/tests/plotting.rs index 7855d0e..ef5b577 100644 --- a/src/gourd/analyse/tests/plotting.rs +++ b/src/gourd/analyse/tests/plotting.rs @@ -138,6 +138,7 @@ fn test_analysis_png_plot_success() { env: Environment::Local, labels: Default::default(), slurm: None, + num_threads: 0, chunks: vec![], groups: vec![], }; diff --git a/src/gourd/cli/def.rs b/src/gourd/cli/def.rs index b41487e..a349ea7 100644 --- a/src/gourd/cli/def.rs +++ b/src/gourd/cli/def.rs @@ -11,7 +11,8 @@ use clap::ValueEnum; #[derive(Parser, Debug)] #[command( about = "Gourd, an empirical evaluator", - disable_help_subcommand = true + disable_help_subcommand = true, + version )] pub struct Cli { /// The main command issued. diff --git a/src/gourd/experiments/mod.rs b/src/gourd/experiments/mod.rs index ae88df9..ae388a2 100644 --- a/src/gourd/experiments/mod.rs +++ b/src/gourd/experiments/mod.rs @@ -13,6 +13,7 @@ use gourd_lib::experiment::programs::expand_programs; use gourd_lib::experiment::Environment; use gourd_lib::experiment::Experiment; use gourd_lib::file_system::FileOperations; +use log::debug; use crate::experiments::dfs::dfs; @@ -101,6 +102,14 @@ impl ExperimentExt for Experiment { metrics_folder: fs.truncate_and_canonicalize_folder(&conf.metrics_path)?, env, + num_threads: conf.local.map_or_else( + || { + let cpus = num_cpus::get(); + debug!("detected {cpus} cpus, using {cpus} threads for local runs"); + cpus + }, + |l| l.num_threads, + ), resource_limits: conf.resource_limits, labels: conf.labels.clone().unwrap_or_default(), diff --git a/src/gourd/init/interactive.rs b/src/gourd/init/interactive.rs index 03f8c24..a3a3a34 100644 --- a/src/gourd/init/interactive.rs +++ b/src/gourd/init/interactive.rs @@ -57,6 +57,7 @@ pub fn init_interactive( resource_limits: None, wrapper: WRAPPER_DEFAULT(), labels: None, + local: None, input_schema: None, }; diff --git a/src/gourd/local/mod.rs b/src/gourd/local/mod.rs index 911f63c..c6ed40d 100644 --- a/src/gourd/local/mod.rs +++ b/src/gourd/local/mod.rs @@ -29,7 +29,7 @@ pub async fn run_local( experiment.save(fs)?; let len = cmds.len(); - run_locally(cmds, force, sequential).await?; + run_locally(cmds, force, sequential, experiment.num_threads).await?; Ok(len + pre_fin) } diff --git a/src/gourd/local/runner.rs b/src/gourd/local/runner.rs index 9d1a7d1..b83295d 100644 --- a/src/gourd/local/runner.rs +++ b/src/gourd/local/runner.rs @@ -5,16 +5,21 @@ use std::process::Output; use anyhow::Context; use anyhow::Result; +use futures::StreamExt; use gourd_lib::bailc; use gourd_lib::constants::NAME_STYLE; use gourd_lib::constants::PRIMARY_STYLE; use gourd_lib::constants::TASK_LIMIT; use log::error; use log::trace; -use tokio::task::JoinSet; /// Run a list of tasks locally in a multithreaded way. -pub async fn run_locally(tasks: Vec, force: bool, sequential: bool) -> Result<()> { +pub async fn run_locally( + tasks: Vec, + force: bool, + sequential: bool, + mut num_threads: usize, +) -> Result<()> { if tasks.len() > TASK_LIMIT && !force && !sequential { bailc!( "task limit exceeded", ; @@ -52,19 +57,30 @@ pub async fn run_locally(tasks: Vec, force: bool, sequential: bool) -> handle_output(task.output()); } } else { - let mut set = JoinSet::new(); - - for mut task in tasks { - trace!("Queueing task: {task:?}"); - set.spawn_blocking(move || task.output()); + // Buffering 0 tasks will prevent anything from happening. + // We use 0 to indicate no upper limit. See documentation + if num_threads == 0 { + num_threads = usize::MAX; } - while let Some(result) = set.join_next().await { - if let Ok(join) = result { - handle_output(join); - } else { - error!("Could not join the child in the multithreaded runtime"); - process::exit(1); + let handles = tokio_stream::iter(tasks) + .map(|mut task| { + trace!("Queueing task: {task:?}"); + tokio::task::spawn_blocking(move || task.output()) + }) + // only poll up to `num_threads` of tasks at once: + .buffer_unordered(num_threads); + + tokio::pin!(handles); + while let Some(join_result) = handles.next().await { + match join_result { + Ok(output) => handle_output(output), + Err(join_error) => { + error!( + "Could not join the child in the multithreaded runtime: {join_error}" + ); + process::exit(1); + } } } } diff --git a/src/gourd/local/tests/runner.rs b/src/gourd/local/tests/runner.rs index 48f2f94..249a15c 100644 --- a/src/gourd/local/tests/runner.rs +++ b/src/gourd/local/tests/runner.rs @@ -22,7 +22,7 @@ async fn runner_fibonacci_test() { commands.push(cmd); } - let results = run_locally(commands, false, false).await; + let results = run_locally(commands, false, false, 0).await; assert!(results.is_ok(), "Executing children processes failed"); } @@ -37,7 +37,7 @@ async fn runner_sleep_test() { commands.push(cmd); } - let results = run_locally(commands, false, false).await; + let results = run_locally(commands, false, false, 0).await; assert!(results.is_ok(), "Executing children processes failed"); } @@ -51,7 +51,7 @@ async fn test_limit() { commands.push(cmd); } - let results = run_locally(commands, false, false).await; + let results = run_locally(commands, false, false, 0).await; assert!(results.is_err(), "Executing children processes failed"); } diff --git a/src/gourd/test_utils.rs b/src/gourd/test_utils.rs index f4b5729..56d245d 100644 --- a/src/gourd/test_utils.rs +++ b/src/gourd/test_utils.rs @@ -58,6 +58,7 @@ pub fn create_sample_experiment( parameters: None, slurm: None, resource_limits: None, + local: None, labels: Some(BTreeMap::new()), }; diff --git a/src/gourd_lib/config/mod.rs b/src/gourd_lib/config/mod.rs index b87a7b6..625f05f 100644 --- a/src/gourd_lib/config/mod.rs +++ b/src/gourd_lib/config/mod.rs @@ -232,6 +232,15 @@ pub struct SubParameter { pub values: Vec, } +/// Options for configuring the execution of +/// `gourd run local` +#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize, Hash, Eq)] +#[serde(deny_unknown_fields)] +pub struct LocalOptions { + /// The number of threads to use for parallel execution of jobs locally. + pub num_threads: usize, +} + /// A label that can be assigned to a job based on the afterscript output. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Hash, Eq)] #[serde(deny_unknown_fields)] @@ -304,6 +313,10 @@ pub struct Config { )] pub wrapper: String, + /// Options for configuring the execution of + /// `gourd run local` + pub local: Option, + /// Allow custom labels to be assigned based on the afterscript output. /// /// syntax is: @@ -334,6 +347,7 @@ impl Default for Config { parameters: None, slurm: None, resource_limits: None, + local: None, labels: Some(BTreeMap::new()), } } diff --git a/src/gourd_lib/config/tests/mod.rs b/src/gourd_lib/config/tests/mod.rs index 1148668..b1b771d 100644 --- a/src/gourd_lib/config/tests/mod.rs +++ b/src/gourd_lib/config/tests/mod.rs @@ -35,6 +35,7 @@ fn breaking_changes_config_struct() { input_schema: None, slurm: None, resource_limits: None, + local: None, labels: Some(BTreeMap::new()), }; } @@ -73,6 +74,7 @@ fn breaking_changes_config_file_all_values() { input_schema: None, slurm: None, resource_limits: None, + local: None, labels: None, }, Config::from_file(file_pathbuf.as_path(), &REAL_FS).expect("Unexpected config read error.") @@ -109,6 +111,7 @@ fn breaking_changes_config_file_required_values() { input_schema: None, slurm: None, resource_limits: None, + local: None, labels: None, }, Config::from_file(file_pb.as_path(), &REAL_FS).expect("Unexpected config read error.") @@ -279,6 +282,7 @@ fn parse_valid_escape_hatch_file() { metrics_path: dir.path().join("43"), experiments_folder: dir.path().join("44"), parameters: None, + local: None, programs: vec![( "x".to_string(), UserProgram { diff --git a/src/gourd_lib/experiment/mod.rs b/src/gourd_lib/experiment/mod.rs index 6cc5202..3967f9c 100644 --- a/src/gourd_lib/experiment/mod.rs +++ b/src/gourd_lib/experiment/mod.rs @@ -186,6 +186,9 @@ pub struct Experiment { /// Environment of the experiment pub env: Environment, + /// How many threads to use for local execution + pub num_threads: usize, + /// Labels used in this experiment. pub labels: BTreeMap,