From e28f0182905d29947060d4fd3b8577735e8af500 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 12:57:50 +0100 Subject: [PATCH 01/21] Create ucd_sonic.config --- conf/ucd_sonic.config | 1 + 1 file changed, 1 insertion(+) create mode 100644 conf/ucd_sonic.config diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/conf/ucd_sonic.config @@ -0,0 +1 @@ + From 2e2f008428e455d96e0519dcf9a515994b76f1f2 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 12:58:38 +0100 Subject: [PATCH 02/21] Create ucd_sonic.md --- docs/ucd_sonic.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/ucd_sonic.md diff --git a/docs/ucd_sonic.md b/docs/ucd_sonic.md new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/docs/ucd_sonic.md @@ -0,0 +1 @@ + From 6d88c33a2396804f16e88c013b3642750bb10064 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 12:59:44 +0100 Subject: [PATCH 03/21] Update nfcore_custom.config --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 642ddab90..452ee94e9 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -82,6 +82,7 @@ profiles { seawulf { includeConfig "${params.custom_config_base}/conf/seawulf.config"} seg_globe { includeConfig "${params.custom_config_base}/conf/seg_globe.config"} tigem { includeConfig "${params.custom_config_base}/conf/tigem.config"} + ucd_sonic { includeConfig "${params.custom_config_base}/conf/ucd_sonic.config"} ucl_myriad { includeConfig "${params.custom_config_base}/conf/ucl_myriad.config"} uct_hpc { includeConfig "${params.custom_config_base}/conf/uct_hpc.config" } uge { includeConfig "${params.custom_config_base}/conf/uge.config" } From 7d8f9abe250d52fdcdd3e1bb542775363d336210 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 13:00:41 +0100 Subject: [PATCH 04/21] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f96a910de..18c6ed58a 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,7 @@ Currently documentation is available for the following systems: - [SEG_GLOBE](docs/seg_globe.md) - [Super Computing Wales](docs/scw.md) - [TIGEM](docs/tigem.md) +- [UCD_SONIC](docs/ucd_sonic.md) - [UCL_MYRIAD](docs/ucl_myriad.md) - [UCT_HPC](docs/uct_hpc.md) - [UGE](docs/uge.md) From 028e5c0def7b69f93decad55b07be1b9b30feeef Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 13:01:46 +0100 Subject: [PATCH 05/21] Update main.yml --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 86e9a1627..08341563f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -100,6 +100,7 @@ jobs: - "seawulf" - "seg_globe" - "tigem" + - "ucd_sonic" - "ucl_myriad" - "uct_hpc" - "uge" From b2ab61eb734cf1677a0d626c51f3b8b30b299235 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 14:38:46 +0100 Subject: [PATCH 06/21] Update ucd_sonic.config --- conf/ucd_sonic.config | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config index 8b1378917..fcd4705fa 100644 --- a/conf/ucd_sonic.config +++ b/conf/ucd_sonic.config @@ -1 +1,38 @@ +//UCD_SONIC config +params { + config_profile_name = 'UCD_SONIC' + config_profile_description = 'University College Dublin Sonic HPC profile provided by nf-core/configs.' + config_profile_contact = 'Bruce Moran (@brucemoran)' + config_profile_url = 'https://www.ucd.ie/itservices/ourservices/researchit/researchcomputing/sonichpc/' + max_memory = 1.MB //yes really, allocation made based on other factors and cannot be done via params.max_memory... + max_cpus = 40 + max_time = 12.h +} +process { + executor = 'slurm' + queue = 'shared' + maxRetries = 2 + beforeScript = 'module load nextflow/22.04.5.5708 singularity/3.5.2' + +} + +executor { + queueSize = 50 + submitRateLimit = '10 sec' +} + +env { + NXF_OPTS="-Xms2G -Xmx40G" +} + +profiles { + moranb { + singularity { + enabled = true + autoMounts = true + cacheDir = '/home/people/moranb/singularity/nextflow' + runOptions = '--bind /home/people/moranb,/scratch/moranb' + } + } +} From 671e6167d1654f45a100c7d5d31335e94a81c0c7 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 14:43:46 +0100 Subject: [PATCH 07/21] Update ucd_sonic.config max_memory removed --- conf/ucd_sonic.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config index fcd4705fa..5c9f22f8e 100644 --- a/conf/ucd_sonic.config +++ b/conf/ucd_sonic.config @@ -4,7 +4,7 @@ params { config_profile_description = 'University College Dublin Sonic HPC profile provided by nf-core/configs.' config_profile_contact = 'Bruce Moran (@brucemoran)' config_profile_url = 'https://www.ucd.ie/itservices/ourservices/researchit/researchcomputing/sonichpc/' - max_memory = 1.MB //yes really, allocation made based on other factors and cannot be done via params.max_memory... + //max_memory = 1.MB //yes really, allocation made based on other factors and cannot be done via params.max_memory... max_cpus = 40 max_time = 12.h } From 08ea9f60817c657934cd9ffbbf1de4ee66aaa65d Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 14:47:26 +0100 Subject: [PATCH 08/21] Update ucd_sonic.config clusterOptions --- conf/ucd_sonic.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config index 5c9f22f8e..7f215be5f 100644 --- a/conf/ucd_sonic.config +++ b/conf/ucd_sonic.config @@ -14,6 +14,8 @@ process { queue = 'shared' maxRetries = 2 beforeScript = 'module load nextflow/22.04.5.5708 singularity/3.5.2' + clusterOptions = { "--cpus-per-task=1" } + clusterOptions = { "--tasks-per-node=1" } } From d25044dec3be772f672b9f3b229458dea2c079d8 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 14:51:06 +0100 Subject: [PATCH 09/21] Update ucd_sonic.config single clusterOptions line --- conf/ucd_sonic.config | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config index 7f215be5f..6a73a90c1 100644 --- a/conf/ucd_sonic.config +++ b/conf/ucd_sonic.config @@ -14,9 +14,7 @@ process { queue = 'shared' maxRetries = 2 beforeScript = 'module load nextflow/22.04.5.5708 singularity/3.5.2' - clusterOptions = { "--cpus-per-task=1" } - clusterOptions = { "--tasks-per-node=1" } - + clusterOptions = { "--cpus-per-task=1 --tasks-per-node=1" } } executor { From bcd301084aabc31ca39fb163c7972d02c73dd1da Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 15:07:00 +0100 Subject: [PATCH 10/21] Update ucd_sonic.config cache --- conf/ucd_sonic.config | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config index 6a73a90c1..8f9c5aef3 100644 --- a/conf/ucd_sonic.config +++ b/conf/ucd_sonic.config @@ -12,14 +12,11 @@ params { process { executor = 'slurm' queue = 'shared' - maxRetries = 2 - beforeScript = 'module load nextflow/22.04.5.5708 singularity/3.5.2' - clusterOptions = { "--cpus-per-task=1 --tasks-per-node=1" } -} - -executor { queueSize = 50 submitRateLimit = '10 sec' + maxRetries = 2 + beforeScript = 'module load nextflow/22.04.5.5708 singularity/3.5.2' + cache = 'lenient' } env { From dac9f380220aac1706e0b621c75d834cd37d2857 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 15:26:07 +0100 Subject: [PATCH 11/21] Update ucd_sonic.config another attempt --- conf/ucd_sonic.config | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config index 8f9c5aef3..3adbcdabb 100644 --- a/conf/ucd_sonic.config +++ b/conf/ucd_sonic.config @@ -15,14 +15,11 @@ process { queueSize = 50 submitRateLimit = '10 sec' maxRetries = 2 - beforeScript = 'module load nextflow/22.04.5.5708 singularity/3.5.2' + beforeScript = 'export NXF_OPTS="-Xms2G -Xmx40G"; module load nextflow/22.04.5.5708 singularity/3.5.2' + clusterOptions = '--ntasks=1 --cpus-per-task=1 --tasks-per-node=1 --mem-per-cpu=1mb' cache = 'lenient' } -env { - NXF_OPTS="-Xms2G -Xmx40G" -} - profiles { moranb { singularity { @@ -31,5 +28,9 @@ profiles { cacheDir = '/home/people/moranb/singularity/nextflow' runOptions = '--bind /home/people/moranb,/scratch/moranb' } + + env { + + } } } From 3ad8cd1e51cb9ec89ffc3de0b536a33686990dd8 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 15:32:37 +0100 Subject: [PATCH 12/21] Update ucd_sonic.config another approach --- conf/ucd_sonic.config | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config index 3adbcdabb..d7c0bbb52 100644 --- a/conf/ucd_sonic.config +++ b/conf/ucd_sonic.config @@ -16,8 +16,8 @@ process { submitRateLimit = '10 sec' maxRetries = 2 beforeScript = 'export NXF_OPTS="-Xms2G -Xmx40G"; module load nextflow/22.04.5.5708 singularity/3.5.2' - clusterOptions = '--ntasks=1 --cpus-per-task=1 --tasks-per-node=1 --mem-per-cpu=1mb' cache = 'lenient' + memory = 1.MB } profiles { @@ -28,9 +28,8 @@ profiles { cacheDir = '/home/people/moranb/singularity/nextflow' runOptions = '--bind /home/people/moranb,/scratch/moranb' } - env { - + NXF_OPTS="-Xms2G -Xmx40G" } } } From 995aaa69d735d7531b29127a90dd69ad1ab6cb9b Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 16:04:05 +0100 Subject: [PATCH 13/21] Update ucd_sonic.config --mem 1M --- conf/ucd_sonic.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config index d7c0bbb52..aa5fb5e06 100644 --- a/conf/ucd_sonic.config +++ b/conf/ucd_sonic.config @@ -16,6 +16,7 @@ process { submitRateLimit = '10 sec' maxRetries = 2 beforeScript = 'export NXF_OPTS="-Xms2G -Xmx40G"; module load nextflow/22.04.5.5708 singularity/3.5.2' + clusterOptions = { "--mem 1M" } cache = 'lenient' memory = 1.MB } From 0fed7669d8fbc811f6cef209a1269b8b73e237b3 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 16:58:37 +0100 Subject: [PATCH 14/21] Update ucd_sonic.md --- docs/ucd_sonic.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/ucd_sonic.md b/docs/ucd_sonic.md index 8b1378917..85da3dd06 100644 --- a/docs/ucd_sonic.md +++ b/docs/ucd_sonic.md @@ -1 +1,18 @@ +# nf-core/configs: UCD Sonic Configuration +The nf-core pipeline [sarek](https://nf-co.re/sarek) has been successfully tested on [Sonic HPC] (https://www.ucd.ie/itservices/ourservices/researchit/researchcomputing/sonichpc) using this config. + +This Sonic config offers very minimal options to users but should allow the use of all nf-core pipelines. + +## sabtch_nxf_creator +To run a pipeline, you should use the [sbatch_nxf_creator](https://github.com/brucemoran/sbatch_nxf_creator) method. + +This requires you to write a small YAML file including all of your sample names that you want to process individually, as well as your basic Nextflow command and where the output should be written. + +There is some weirdness on Sonic via SLURM (maybe it's just me!) which required me to write this method, maybe you can bypass it. + +The `sbatch_nxf_creator.sh` script takes you YAML and makes an `sbatch` file that can be submitted to the scheduler via a simple `sbatch my_file.sbatch` command. + +Practically, it means appending `srun` to the command and appending `-with-mpi`, as well as a few other vaguaries like specifying output and loading `Nextflow` and `Singularity`. + +Even more practical is that as new samples arrive to you, you can add them to your YAML and then create new sbatch files, which I think is good. From 7f621294172573f6fed3747d9c53429d006a1377 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Mon, 17 Apr 2023 16:59:38 +0100 Subject: [PATCH 15/21] Update ucd_sonic.config cleanup --- conf/ucd_sonic.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config index aa5fb5e06..dd72c1641 100644 --- a/conf/ucd_sonic.config +++ b/conf/ucd_sonic.config @@ -21,6 +21,8 @@ process { memory = 1.MB } +cleanup = true + profiles { moranb { singularity { From b1d15d3ad6f105786a8a11372f7234bb2c2a12b9 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 3 Jul 2023 09:04:09 +0000 Subject: [PATCH 16/21] [automated] Fix linting with Prettier --- docs/ucd_sonic.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/ucd_sonic.md b/docs/ucd_sonic.md index 85da3dd06..a5d88b398 100644 --- a/docs/ucd_sonic.md +++ b/docs/ucd_sonic.md @@ -5,6 +5,7 @@ The nf-core pipeline [sarek](https://nf-co.re/sarek) has been successfully teste This Sonic config offers very minimal options to users but should allow the use of all nf-core pipelines. ## sabtch_nxf_creator + To run a pipeline, you should use the [sbatch_nxf_creator](https://github.com/brucemoran/sbatch_nxf_creator) method. This requires you to write a small YAML file including all of your sample names that you want to process individually, as well as your basic Nextflow command and where the output should be written. @@ -13,6 +14,6 @@ There is some weirdness on Sonic via SLURM (maybe it's just me!) which required The `sbatch_nxf_creator.sh` script takes you YAML and makes an `sbatch` file that can be submitted to the scheduler via a simple `sbatch my_file.sbatch` command. -Practically, it means appending `srun` to the command and appending `-with-mpi`, as well as a few other vaguaries like specifying output and loading `Nextflow` and `Singularity`. +Practically, it means appending `srun` to the command and appending `-with-mpi`, as well as a few other vaguaries like specifying output and loading `Nextflow` and `Singularity`. Even more practical is that as new samples arrive to you, you can add them to your YAML and then create new sbatch files, which I think is good. From 30f0c11b9818117dc15feefe45bae4d0369bbd6a Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Tue, 4 Jul 2023 13:58:50 +0100 Subject: [PATCH 17/21] Update ucd_sonic.md Offers sbatch_nxf_creator as an option. --- docs/ucd_sonic.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/ucd_sonic.md b/docs/ucd_sonic.md index a5d88b398..f269b05cf 100644 --- a/docs/ucd_sonic.md +++ b/docs/ucd_sonic.md @@ -4,16 +4,14 @@ The nf-core pipeline [sarek](https://nf-co.re/sarek) has been successfully teste This Sonic config offers very minimal options to users but should allow the use of all nf-core pipelines. -## sabtch_nxf_creator +## sbatch_nxf_creator -To run a pipeline, you should use the [sbatch_nxf_creator](https://github.com/brucemoran/sbatch_nxf_creator) method. +To run a pipeline, you can use the [sbatch_nxf_creator](https://github.com/brucemoran/sbatch_nxf_creator) method. -This requires you to write a small YAML file including all of your sample names that you want to process individually, as well as your basic Nextflow command and where the output should be written. +This allows you to write a YAML file including all of your sample names that you want to process individually, as well as your basic Nextflow command and where the output should be written. -There is some weirdness on Sonic via SLURM (maybe it's just me!) which required me to write this method, maybe you can bypass it. +There are some additional requirements on Sonic based on its use of SLURM which are resolved using this write this method. -The `sbatch_nxf_creator.sh` script takes you YAML and makes an `sbatch` file that can be submitted to the scheduler via a simple `sbatch my_file.sbatch` command. +The `sbatch_nxf_creator.sh` parses your YAML and makes an `sbatch` file that can be submitted to the scheduler via a simple `sbatch my_file.sbatch` command. Practically, it means appending `srun` to the command and appending `-with-mpi`, as well as a few other vaguaries like specifying output and loading `Nextflow` and `Singularity`. - -Even more practical is that as new samples arrive to you, you can add them to your YAML and then create new sbatch files, which I think is good. From 509b40726db7100abe8f8d2eeb99880dcea139af Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Tue, 4 Jul 2023 14:03:00 +0100 Subject: [PATCH 18/21] Update conf/ucd_sonic.config Co-authored-by: James A. Fellows Yates --- conf/ucd_sonic.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config index dd72c1641..5788121a6 100644 --- a/conf/ucd_sonic.config +++ b/conf/ucd_sonic.config @@ -1,4 +1,3 @@ -//UCD_SONIC config params { config_profile_name = 'UCD_SONIC' config_profile_description = 'University College Dublin Sonic HPC profile provided by nf-core/configs.' From 208515dd475cec6a6a9357c2091e1e85ed0e6d80 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Tue, 15 Aug 2023 15:50:45 +0100 Subject: [PATCH 19/21] Update conf/ucd_sonic.config Co-authored-by: James A. Fellows Yates --- conf/ucd_sonic.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config index 5788121a6..7365e5580 100644 --- a/conf/ucd_sonic.config +++ b/conf/ucd_sonic.config @@ -3,7 +3,6 @@ params { config_profile_description = 'University College Dublin Sonic HPC profile provided by nf-core/configs.' config_profile_contact = 'Bruce Moran (@brucemoran)' config_profile_url = 'https://www.ucd.ie/itservices/ourservices/researchit/researchcomputing/sonichpc/' - //max_memory = 1.MB //yes really, allocation made based on other factors and cannot be done via params.max_memory... max_cpus = 40 max_time = 12.h } From 01ae9c9a5f79d9f78be38e3b101ee09772517617 Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Tue, 15 Aug 2023 15:50:53 +0100 Subject: [PATCH 20/21] Update docs/ucd_sonic.md Co-authored-by: James A. Fellows Yates --- docs/ucd_sonic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ucd_sonic.md b/docs/ucd_sonic.md index f269b05cf..e9f6693ce 100644 --- a/docs/ucd_sonic.md +++ b/docs/ucd_sonic.md @@ -6,7 +6,7 @@ This Sonic config offers very minimal options to users but should allow the use ## sbatch_nxf_creator -To run a pipeline, you can use the [sbatch_nxf_creator](https://github.com/brucemoran/sbatch_nxf_creator) method. +To run a pipeline, you can optionally use the [sbatch_nxf_creator](https://github.com/brucemoran/sbatch_nxf_creator) method. This allows you to write a YAML file including all of your sample names that you want to process individually, as well as your basic Nextflow command and where the output should be written. From 7e113e2ffadc37d8e800bf74f573e09d7ef7bbab Mon Sep 17 00:00:00 2001 From: Bruce Moran Date: Tue, 15 Aug 2023 15:53:12 +0100 Subject: [PATCH 21/21] Remove bmoran profile --- conf/ucd_sonic.config | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/conf/ucd_sonic.config b/conf/ucd_sonic.config index 7365e5580..9eb519446 100644 --- a/conf/ucd_sonic.config +++ b/conf/ucd_sonic.config @@ -20,17 +20,3 @@ process { } cleanup = true - -profiles { - moranb { - singularity { - enabled = true - autoMounts = true - cacheDir = '/home/people/moranb/singularity/nextflow' - runOptions = '--bind /home/people/moranb,/scratch/moranb' - } - env { - NXF_OPTS="-Xms2G -Xmx40G" - } - } -}