Configure Tokio threads

Cargo.toml

@@ -2,7 +2,7 @@
 
 [package]
 name = "gourd"
-version = "1.2.0"
+version = "1.2.1"
 edition = "2021"
 default-run = "gourd"
 authors = [
@@ -64,6 +64,9 @@ elf = "0.7.4"
 
 # To execute threads locally using a thread-pool executor.
 tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
+tokio-stream = { version = "0.1.17", default-features = false }
+futures = { version = "0.3.31", default-features = false, features = ["std"] }
+num_cpus = "1.16.0"
 
 # To encode/decode data in gourd.toml and other Gourd files.
 toml = "0.8.12"

README.md

@@ -39,6 +39,8 @@ The distributed installers are: (One of these should be put in place of `[your s
 
 Customizing installation folders is explained [here](https://gourd.chla.cz/manpages/maintainer.pdf).
 
+See [the releases page on Github](https://github.com/ConSol-Lab/gourd/releases) for details on installing or updating to the latest version of `gourd`.
+
 ### Usage
 
 Verify that `gourd` is installed by running:
@@ -52,9 +54,9 @@ There are extensive tutorials and documentation which can be accessed in many di
 
 With a web browser:
 
-- [**gourd.1 in your browser!**](https://gourd.chla.cz/manpages/gourd.1.html), supports both dark and light mode!
-- [**gourd.toml.5 in your browser!**](https://gourd.chla.cz/manpages/gourd.toml.5.html), supports both dark and light mode!
-- [**gourd-tutorial.7 in your browser!**](https://gourd.chla.cz/manpages/gourd-tutorial.7.html), supports both dark and light mode!
+- [**gourd.1 in your browser!**](https://andreats.com/gourd/gourd.1.html), supports both dark and light mode!
+- [**gourd.toml.5 in your browser!**](https://andreats.com/gourd/gourd.toml.5.html), supports both dark and light mode!
+- [**gourd-tutorial.7 in your browser!**](https://andreats.com/gourd/gourd-tutorial.7.html), supports both dark and light mode!
 
 As a manpage, with the `man` command:
 ```
@@ -65,9 +67,9 @@ $ man gourd-tutorial
 
 As a PDF file:
 
-- [**gourd.1 in your PDF reader!**](https://gourd.chla.cz/manpages/gourd.1.pdf)
-- [**gourd.toml.5 in your PDF reader!**](https://gourd.chla.cz/manpages/gourd.toml.5.pdf)
-- [**gourd-tutorial.7 in your PDF reader!**](https://gourd.chla.cz/manpages/gourd-tutorial.7.pdf)
+- [**gourd.1 in your PDF reader!**](https://andreats.com/gourd/gourd.1.pdf)
+- [**gourd.toml.5 in your PDF reader!**](https://andreats.com/gourd/gourd.toml.5.pdf)
+- [**gourd-tutorial.7 in your PDF reader!**](https://andreats.com/gourd/gourd-tutorial.7.pdf)
 
 Please refer to these if you want to familiarize yourself with the software.
 
@@ -96,8 +98,12 @@ An artifact is available from the GitLab pipeline `documentation` job, or
 
 ### Authors
 
-- Mikołaj Gazeel, m.j.gazeel@student.tudelft.nl
+- Mikołaj Gazeel, m.j.gazeel@tudelft.nl
 - Lukáš Chládek, l@chla.cz
-- Ανδρέας Τσατσάνης, a.tsatsanis@student.tudelft.nl
-- Rūta Giedrytė, r.giedryte@student.tudelft.nl
+- Ανδρέας Τσατσάνης, a.tsatsanis@tudelft.nl
+- Rūta Giedrytė, r.giedryte@tudelft.nl
 - Jan Piotrowski, me@jan.wf
+
+### Maintainers
+- Ανδρέας Τσατσάνης, a.tsatsanis@tudelft.nl
+

docs/maintainer/building.tex

@@ -64,6 +64,10 @@ \subsection{Documentation}
 
 All of the resulting files are placed in: \texttt{target/release/manpages}.
 
+To build the library documentation as well, run:
+\begin{verbatim}
+    cargo doc --no-deps --color=always --all-features --release
+\end{verbatim}
 
 \subsection{Distribution}
 \textcolor{red!30!black}{\textbf{

docs/maintainer/title.tex

@@ -10,13 +10,9 @@
   {
     Lukáš Chládek
 
-    Rūta Giedrytė
-
     Mikołaj Gazeel
 
     Ανδρέας Τσατσάνης
-
-    Jan Piotrowski
   }
 
   \vspace{0.5cm}

docs/maintainer/version-history/section.tex

@@ -5,6 +5,11 @@ \section{Version History}
 \input{version-history/definitions}
 % \version{x} for start of version x section
 
+\version{1.2.1}{Sponge Gourd}{}
+Minor patch, adds the \texttt{local.num\_threads} option in 
+the config, as well automatically detecting the number of CPU
+cores on the system. Added the \texttt{gourd --version} command.
+
 \version{1.2.0}{Sponge Gourd}{}
 
 Major internal reworkings, redesigned \texttt{gourd analyse}.

docs/user/gourd-tutorial.7.tex

@@ -363,12 +363,11 @@
 
     \Prog{gourd.toml(5)}
 
-    \section{AUTHORS}
-    Ανδρέας Τσατσάνης <\Email{a.tsatsanis@student.tudelft.nl}>\\[0.1cm]\MANbr
-    Rūta Giedrytė <\Email{r.giedryte@student.tudelft.nl}>\\[0.1cm]\MANbr
-    Mikołaj Gazeel <\Email{m.j.gazeel@student.tudelft.nl}>\\[0.1cm]\MANbr
-    Jan Piotrowski <\Email{me@jan.wf}>\\[0.1cm]\MANbr
+    \section{CONTACT}
+    Ανδρέας Τσατσάνης <\Email{a.tsatsanis@tudelft.nl}>\\[0.1cm]\MANbr
     Lukáš Chládek <\Email{l@chla.cz}>\\[0.1cm]\MANbr
+    Mikołaj Gazeel <\Email{m.j.gazeel@tudelft.nl}>\\[0.1cm]\MANbr
+
 %@% IF LATEX %@%
 \end{adjustwidth}
 %@% END-IF %@%

docs/user/gourd.1.tex

@@ -354,9 +354,9 @@
 
             \subsubsection{Summary}
                 The \Prog{gourd} \Arg{continue} command schedules runs that are part of an existing
-                experiment, but have not yet been scheduled.
+                experiment, but have not yet been scheduled. This includes runs created by \Prog{gourd} \Arg{rerun}, as well as runs that were not scheduled due to a run limit.
                 For example, an experiment with 30,000 distinct runs can be scheduled in three batches
-                of 10,000 each if that is the maximum number of queued supercomputer jobs.
+                of 10,000 each if that is the maximum number of queued supercomputer jobs. 
 
             \subsubsection{Synopsis}
                 \Prog{gourd} \Arg{continue}
@@ -535,7 +535,8 @@
                 according to the newest available data.
 
                 The option \texttt{--format} can be used to specify whether the plot output
-                should be in PNG or SVG format.
+                should be in PNG or SVG format, for example: 
+                \Prog{gourd} \Arg{analyse} \Arg{plot} \Arg{--format="png"} (png is also the default output)
 
         \subsection{GOURD VERSION}
 
@@ -572,12 +573,10 @@
 
         \Prog{gourd.toml(5)}
 
-    \section{AUTHORS}
-    Rūta Giedrytė <\Email{r.giedryte@student.tudelft.nl}>\\[0.1cm]\MANbr
+    \section{CONTACT}
+    Ανδρέας Τσατσάνης <\Email{a.tsatsanis@tudelft.nl}>\\[0.1cm]\MANbr
     Lukáš Chládek <\Email{l@chla.cz}>\\[0.1cm]\MANbr
-    Jan Piotrowski <\Email{me@jan.wf}>\\[0.1cm]\MANbr
-    Mikołaj Gazeel <\Email{m.j.gazeel@student.tudelft.nl}>\\[0.1cm]\MANbr
-    Ανδρέας Τσατσάνης <\Email{a.tsatsanis@student.tudelft.nl}>\\[0.1cm]\MANbr
+    Mikołaj Gazeel <\Email{m.j.gazeel@tudelft.nl}>\\[0.1cm]\MANbr
 
 %@% IF LATEX %@%
 \end{adjustwidth}

docs/user/gourd.toml.5.tex

@@ -193,6 +193,23 @@
 mem_per_cpu = 512
               \end{verbatim}
 
+    \section{LOCAL}
+        \begin{Description}[Options]\setlength{\itemsep}{0cm}
+            \item[\Opt{num\_threads?} = number]
+                How many threads should \Prog{gourd} \Arg{run} \Arg{local} use.
+        \end{Description}
+
+        \subsection{NUM\_THREADS}
+            For the parallel execution of \Prog{gourd} \Arg{run} \Arg{local} you can
+            limit the number of threads that will be used by specifying:
+            \begin{verbatim}
+[local]
+num_threads = 8
+            \end{verbatim}
+            If this option is not specified, the program will try to detect the number 
+            of CPUs present on the system, and use that number of threads. Setting a 
+            value of 0 will result in a number of threads equal to the number of runs in
+            the program (and the OS will limit the resource use thereafter).
     \section{PROGRAMS}
 
         Multiple programs can be specified.
@@ -665,12 +682,11 @@
   \section{SEE ALSO}
       \Prog{gourd(1)} \Prog{gourd-tutorial(7)}
 
-  \section{AUTHORS}
+    \section{CONTACT}
+    Ανδρέας Τσατσάνης <\Email{a.tsatsanis@tudelft.nl}>\\[0.1cm]\MANbr
     Lukáš Chládek <\Email{l@chla.cz}>\\[0.1cm]\MANbr
-    Rūta Giedrytė <\Email{r.giedryte@student.tudelft.nl}>\\[0.1cm]\MANbr
-    Ανδρέας Τσατσάνης <\Email{a.tsatsanis@student.tudelft.nl}>\\[0.1cm]\MANbr
-    Mikołaj Gazeel <\Email{m.j.gazeel@student.tudelft.nl}>\\[0.1cm]\MANbr
-    Jan Piotrowski <\Email{me@jan.wf}>
+    Mikołaj Gazeel <\Email{m.j.gazeel@tudelft.nl}>\\[0.1cm]\MANbr
+
 %@% IF LATEX %@%
 \end{adjustwidth}
 %@% END-IF %@%

docs/user/html/preamble.html

@@ -36,18 +36,18 @@
     The top bar contains links to the three documentation pages that gourd ships with. <br/> <br/>
 
     You may also access the documentation in manpage format and PDF format. <br/> <br/>
-    For more information go to the repostory: <a class="stovrd " href="https://gitlab.ewi.tudelft.nl/cse2000-software-project/2023-2024/cluster-w/18c/gourd" >on the TuDelft GitLab</a>!
+    For more information go to <a class="stovrd " href="https://github.com/ConSol-Lab/gourd" >the repository on GitHub!</a>
 </nav>
 <article>
 <h1></h1>
-<span class="notice-info">Technical documentation: You can view the rustdoc generated documentation <a href="../../doc/gourd/index.html">here</a></span>
+<span class="notice-info">Technical documentation: You can view the rustdoc generated documentation <a href="./doc/gourd/index.html">here</a></span>
 <nav class="nav-doc">
     <p><span style="border-bottom: 1px solid black">Hello!</span></p>
     This is the gourd user documentation in HTML form, you may nagivate it here.
     The top bar contains links to the three documentation pages that gourd ships with. <br/> <br/>
 
     You may also access the documentation in manpage format and PDF format. <br/> <br/>
-    For more information go to the repostory: <a href="https://gitlab.ewi.tudelft.nl/cse2000-software-project/2023-2024/cluster-w/18c/gourd" >on the TuDelft GitLab</a>!
+    For more information go to <a class="stovrd " href="https://github.com/ConSol-Lab/gourd" >the repository on GitHub!</a>
 </nav>
 </article>
 

src/gourd/analyse/tests/plotting.rs

@@ -138,6 +138,7 @@ fn test_analysis_png_plot_success() {
         env: Environment::Local,
         labels: Default::default(),
         slurm: None,
+        num_threads: 0,
         chunks: vec![],
         groups: vec![],
     };

src/gourd/cli/def.rs

@@ -11,7 +11,8 @@ use clap::ValueEnum;
 #[derive(Parser, Debug)]
 #[command(
     about = "Gourd, an empirical evaluator",
-    disable_help_subcommand = true
+    disable_help_subcommand = true,
+    version
 )]
 pub struct Cli {
     /// The main command issued.

src/gourd/experiments/mod.rs

@@ -13,6 +13,7 @@ use gourd_lib::experiment::programs::expand_programs;
 use gourd_lib::experiment::Environment;
 use gourd_lib::experiment::Experiment;
 use gourd_lib::file_system::FileOperations;
+use log::debug;
 
 use crate::experiments::dfs::dfs;
 
@@ -101,6 +102,14 @@ impl ExperimentExt for Experiment {
             metrics_folder: fs.truncate_and_canonicalize_folder(&conf.metrics_path)?,
 
             env,
+            num_threads: conf.local.map_or_else(
+                || {
+                    let cpus = num_cpus::get();
+                    debug!("detected {cpus} cpus, using {cpus} threads for local runs");
+                    cpus
+                },
+                |l| l.num_threads,
+            ),
             resource_limits: conf.resource_limits,
             labels: conf.labels.clone().unwrap_or_default(),
 

src/gourd/init/interactive.rs

@@ -57,6 +57,7 @@ pub fn init_interactive(
         resource_limits: None,
         wrapper: WRAPPER_DEFAULT(),
         labels: None,
+        local: None,
         input_schema: None,
     };
 

src/gourd/local/mod.rs

@@ -29,7 +29,7 @@ pub async fn run_local(
     experiment.save(fs)?;
 
     let len = cmds.len();
-    run_locally(cmds, force, sequential).await?;
+    run_locally(cmds, force, sequential, experiment.num_threads).await?;
 
     Ok(len + pre_fin)
 }

src/gourd/local/runner.rs

@@ -5,16 +5,21 @@ use std::process::Output;
 
 use anyhow::Context;
 use anyhow::Result;
+use futures::StreamExt;
 use gourd_lib::bailc;
 use gourd_lib::constants::NAME_STYLE;
 use gourd_lib::constants::PRIMARY_STYLE;
 use gourd_lib::constants::TASK_LIMIT;
 use log::error;
 use log::trace;
-use tokio::task::JoinSet;
 
 /// Run a list of tasks locally in a multithreaded way.
-pub async fn run_locally(tasks: Vec<Command>, force: bool, sequential: bool) -> Result<()> {
+pub async fn run_locally(
+    tasks: Vec<Command>,
+    force: bool,
+    sequential: bool,
+    mut num_threads: usize,
+) -> Result<()> {
     if tasks.len() > TASK_LIMIT && !force && !sequential {
         bailc!(
           "task limit exceeded", ;
@@ -52,19 +57,30 @@ pub async fn run_locally(tasks: Vec<Command>, force: bool, sequential: bool) ->
                 handle_output(task.output());
             }
         } else {
-            let mut set = JoinSet::new();
-
-            for mut task in tasks {
-                trace!("Queueing task: {task:?}");
-                set.spawn_blocking(move || task.output());
+            // Buffering 0 tasks will prevent anything from happening.
+            // We use 0 to indicate no upper limit. See documentation
+            if num_threads == 0 {
+                num_threads = usize::MAX;
             }
 
-            while let Some(result) = set.join_next().await {
-                if let Ok(join) = result {
-                    handle_output(join);
-                } else {
-                    error!("Could not join the child in the multithreaded runtime");
-                    process::exit(1);
+            let handles = tokio_stream::iter(tasks)
+                .map(|mut task| {
+                    trace!("Queueing task: {task:?}");
+                    tokio::task::spawn_blocking(move || task.output())
+                })
+                // only poll up to `num_threads` of tasks at once:
+                .buffer_unordered(num_threads);
+
+            tokio::pin!(handles);
+            while let Some(join_result) = handles.next().await {
+                match join_result {
+                    Ok(output) => handle_output(output),
+                    Err(join_error) => {
+                        error!(
+                            "Could not join the child in the multithreaded runtime: {join_error}"
+                        );
+                        process::exit(1);
+                    }
                 }
             }
         }

src/gourd/local/tests/runner.rs

@@ -22,7 +22,7 @@ async fn runner_fibonacci_test() {
         commands.push(cmd);
     }
 
-    let results = run_locally(commands, false, false).await;
+    let results = run_locally(commands, false, false, 0).await;
 
     assert!(results.is_ok(), "Executing children processes failed");
 }
@@ -37,7 +37,7 @@ async fn runner_sleep_test() {
         commands.push(cmd);
     }
 
-    let results = run_locally(commands, false, false).await;
+    let results = run_locally(commands, false, false, 0).await;
 
     assert!(results.is_ok(), "Executing children processes failed");
 }
@@ -51,7 +51,7 @@ async fn test_limit() {
         commands.push(cmd);
     }
 
-    let results = run_locally(commands, false, false).await;
+    let results = run_locally(commands, false, false, 0).await;
 
     assert!(results.is_err(), "Executing children processes failed");
 }

src/gourd/test_utils.rs

@@ -58,6 +58,7 @@ pub fn create_sample_experiment(
         parameters: None,
         slurm: None,
         resource_limits: None,
+        local: None,
         labels: Some(BTreeMap::new()),
     };