From 3b68adec9dd7fc42d8b9b1ba7505270085ac0c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Chl=C3=A1dek?= Date: Tue, 13 May 2025 14:31:24 +0200 Subject: [PATCH 1/2] bump version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index dff812a..76af081 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ [package] name = "gourd" -version = "1.2.1" +version = "1.2.2" edition = "2021" default-run = "gourd" authors = [ From a0e4429bd95156439bf79aa1b012097e42cdc126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Chl=C3=A1dek?= Date: Mon, 12 May 2025 16:39:29 +0200 Subject: [PATCH 2/2] basic functionality for modules --- docs/user/gourd.toml.5.tex | 5 ++++ src/gourd/init/interactive.rs | 2 ++ src/gourd/slurm/handler.rs | 11 ++++++++ src/gourd/slurm/interactor.rs | 4 +++ src/gourd/slurm/tests/handler.rs | 47 ++++++++++++++++++++++++++++++++ src/gourd_lib/config/slurm.rs | 6 ++++ src/gourd_lib/constants.rs | 3 ++ 7 files changed, 78 insertions(+) diff --git a/docs/user/gourd.toml.5.tex b/docs/user/gourd.toml.5.tex index f8165f4..17fcc95 100644 --- a/docs/user/gourd.toml.5.tex +++ b/docs/user/gourd.toml.5.tex @@ -139,6 +139,11 @@ For example one account available on DelftBlue is "Education-EEMCS-MSc-CS". To get a list of available accounts on your cluster, you can use Slurm's \Prog{sacctmgr}~\Arg{show}~\Arg{account} command + \item[\Opt{modules} = list of strings] + Modules to load when running through Slurm using the "module" utility. + These are loaded in sequential order. + For example, to use R in the DelftBlue cluster, \texttt{["2024r1", "r/3.4.0"]}. + By default, no modules are loaded. \item[\Opt{mail\_type} = string] Choose one of Slurm's options for sending emails when a run's status changes. Default is "NONE". Valid options are: diff --git a/src/gourd/init/interactive.rs b/src/gourd/init/interactive.rs index a3a3a34..369e5ea 100644 --- a/src/gourd/init/interactive.rs +++ b/src/gourd/init/interactive.rs @@ -7,6 +7,7 @@ use gourd_lib::bailc; use gourd_lib::config::slurm::SlurmConfig; use gourd_lib::config::Config; use gourd_lib::constants::CMD_STYLE; +use gourd_lib::constants::EMPTY_MODULES; use gourd_lib::constants::WRAPPER_DEFAULT; use gourd_lib::ctx; use gourd_lib::file_system::FileOperations; @@ -103,6 +104,7 @@ pub fn init_interactive( experiment_name: "my-experiment".to_string(), output_folder: PathBuf::from("./slurmout/"), partition: "".to_string(), + modules: EMPTY_MODULES(), array_size_limit: None, max_submit: None, account: "".to_string(), diff --git a/src/gourd/slurm/handler.rs b/src/gourd/slurm/handler.rs index 88eaa93..73ba462 100644 --- a/src/gourd/slurm/handler.rs +++ b/src/gourd/slurm/handler.rs @@ -143,6 +143,17 @@ pub fn parse_optional_args(slurm_config: &SlurmConfig) -> String { result } +/// Helper function to create string with modules for slurm +pub fn parse_modules(slurm_config: &SlurmConfig) -> String { + let mut result = "".to_string(); + + for module in &slurm_config.modules { + result.push_str(&format!("module load {module}\n")) + } + + result +} + #[cfg(test)] #[path = "tests/handler.rs"] mod tests; diff --git a/src/gourd/slurm/interactor.rs b/src/gourd/slurm/interactor.rs index a7eeba1..6268b58 100644 --- a/src/gourd/slurm/interactor.rs +++ b/src/gourd/slurm/interactor.rs @@ -21,6 +21,7 @@ use log::debug; use log::info; use log::trace; +use super::handler::parse_modules; use super::handler::parse_optional_args; use super::SacctOutput; use crate::chunks::Chunk; @@ -215,6 +216,7 @@ impl SlurmInteractor for SlurmCli { let chunk_index = experiment.register_runs(&chunk.runs); let optional_args = parse_optional_args(slurm_config); + let modules = parse_modules(slurm_config); // `%A` gets replaced with array *job* id, `%a` with the array *task* id // this is read in `src/gourd/status/slurm_files.rs` to get the output. @@ -239,6 +241,7 @@ impl SlurmInteractor for SlurmCli { #SBATCH --error={:?} {} set -x +{} {} {} {} $SLURM_ARRAY_TASK_ID ", @@ -253,6 +256,7 @@ set -x slurm_out, slurm_err, optional_args, + modules, experiment.wrapper, exp_path.display(), chunk_index diff --git a/src/gourd/slurm/tests/handler.rs b/src/gourd/slurm/tests/handler.rs index 72b492f..3194cac 100644 --- a/src/gourd/slurm/tests/handler.rs +++ b/src/gourd/slurm/tests/handler.rs @@ -11,6 +11,7 @@ fn parse_optional_args_test_all() { experiment_name: "test experiment".to_string(), output_folder: Default::default(), partition: "memory".to_string(), + modules: Default::default(), array_size_limit: None, max_submit: None, account: "test-account".to_string(), @@ -34,6 +35,7 @@ fn parse_optional_args_test_only_begin() { experiment_name: "test experiment".to_string(), output_folder: Default::default(), partition: "memory".to_string(), + modules: Default::default(), array_size_limit: None, max_submit: None, account: "test-account".to_string(), @@ -69,6 +71,7 @@ fn parse_optional_args_test_custom_args() { experiment_name: "test experiment".to_string(), output_folder: Default::default(), partition: "memory".to_string(), + modules: Default::default(), array_size_limit: None, max_submit: None, account: "test-account".to_string(), @@ -86,3 +89,47 @@ fn parse_optional_args_test_custom_args() { assert_eq!(output, desired_output) } + +#[test] +fn parse_modules_test_empty() { + let config = SlurmConfig { + experiment_name: "test experiment".to_string(), + output_folder: Default::default(), + partition: "memory".to_string(), + modules: Vec::new(), + array_size_limit: None, + max_submit: None, + account: "test-account".to_string(), + begin: None, + mail_type: Some("ALL".to_string()), + mail_user: Some("testUSER".to_string()), + additional_args: None, + }; + let output = parse_modules(&config); + let desired_output = ""; + + assert_eq!(output, desired_output) +} + +#[test] +fn parse_modules_test_not_empty() { + let config = SlurmConfig { + experiment_name: "test experiment".to_string(), + output_folder: Default::default(), + partition: "memory".to_string(), + modules: vec!["2024r1".to_string(), "r/3.4.0".to_string()], + array_size_limit: None, + max_submit: None, + account: "test-account".to_string(), + begin: None, + mail_type: Some("ALL".to_string()), + mail_user: Some("testUSER".to_string()), + additional_args: None, + }; + let output = parse_modules(&config); + let desired_output = "module load 2024r1 +module load r/3.4.0 +"; + + assert_eq!(output, desired_output) +} diff --git a/src/gourd_lib/config/slurm.rs b/src/gourd_lib/config/slurm.rs index 6ba0227..778cdc7 100644 --- a/src/gourd_lib/config/slurm.rs +++ b/src/gourd_lib/config/slurm.rs @@ -6,6 +6,8 @@ use std::time::Duration; use serde::Deserialize; use serde::Serialize; +use crate::constants::EMPTY_MODULES; + /// The config options when running through Slurm #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -27,6 +29,10 @@ pub struct SlurmConfig { /// - "visual" pub partition: String, + /// Which modules to load using the 'modules' command + #[serde(default = "EMPTY_MODULES")] + pub modules: Vec, + /// Override the maximum number of jobs to schedule in a Slurm array. /// /// If left `None`, a value fetched directly from slurm will be used. diff --git a/src/gourd_lib/constants.rs b/src/gourd_lib/constants.rs index 6dad626..0786c1f 100644 --- a/src/gourd_lib/constants.rs +++ b/src/gourd_lib/constants.rs @@ -36,6 +36,9 @@ pub const LABEL_OVERLAP_DEFAULT: fn() -> bool = || false; /// The default arguments for an input. pub const EMPTY_ARGS: fn() -> Vec = Vec::new; +/// The default modules for a Slurm job. +pub const EMPTY_MODULES: fn() -> Vec = Vec::new; + /// The prefix which will cause an argument to be interpreted as a glob. /// Ensure matches: /// - docs/user/gourd.toml.5