Skip to content

Commit

Permalink
Merge pull request #719 from nvnieuwk/vsc_ugent/shinx
Browse files Browse the repository at this point in the history 
Add shinx cluster to VSC ugent + simplification in usage of config
  • Loading branch information
@nvnieuwk
nvnieuwk authored Jul 22, 2024
2 parents fe61ca4 + 2d45724 commit 48b0fdc
Show file tree
Hide file tree
Showing 2 changed files with 125 additions and 125 deletions.
197 changes: 92 additions & 105 deletions conf/vsc_ugent.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,61 +80,32 @@ aws {
maxErrorRetry = 3
}

// Define profiles for each cluster
profiles {
skitty {
params {
config_profile_description = 'HPC_SKITTY profile for use on the Skitty cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 177.GB
max_cpus = 36
max_time = "3day"
}

process {
executor = 'slurm'
queue = 'skitty'
}
}
def cluster = System.getenv("SLURM_CLUSTERS") ?: System.getenv("HPCUGENT_FAMILY_CLUSTER_VERSION") ?: ""

swalot {
params {
config_profile_description = 'HPC_SWALOT profile for use on the Swalot cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 116.GB
max_cpus = 20
max_time = "3day"
}
if( !cluster ) {
System.err.println("WARNING: Could not get the name of the currently used cluster, defaulting to doduo")
cluster = "doduo"
}

process {
executor = 'slurm'
queue = 'swalot'
}
}
params.config_profile_description = 'Configuration profile for execution of Nextflow pipelines on the VSC UGhent HPC.'
params.config_profile_contact = 'ict@cmgg.be'
params.config_profile_url = 'https://www.ugent.be/hpc/en'

victini {
switch(cluster) {
case "skitty":
params {
config_profile_description = 'HPC_VICTINI profile for use on the Victini cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 88.GB
max_memory = 177.GB
max_cpus = 36
max_time = "3day"
}

process {
executor = 'slurm'
queue = 'victini'
queue = 'skitty'
}
}

kirlia {
break
case "kirlia":
params {
config_profile_description = 'HPC_KIRLIA profile for use on the Kirlia cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 738.GB
max_cpus = 36
max_time = "3day"
Expand All @@ -144,13 +115,9 @@ profiles {
executor = 'slurm'
queue = 'kirlia'
}
}

doduo {
break
case "doduo":
params {
config_profile_description = 'HPC_DODUO profile for use on the Doduo cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 250.GB
max_cpus = 96
max_time = "3day"
Expand All @@ -160,13 +127,21 @@ profiles {
executor = 'slurm'
queue = 'doduo'
}
}
break
case "shinx":
params {
max_memory = 360.GB
max_cpus = 192
max_time = "3day"
}

cpu_rome {
process {
executor = 'slurm'
queue = 'shinx'
}
break
case "cpu_rome":
params {
config_profile_description = 'HPC_DODRIO_cpu_rome profile for use on the Dodrio/cpu_rome cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 256.GB
max_cpus = 128
max_time = "3day"
Expand All @@ -177,13 +152,9 @@ profiles {
queue = 'dodrio/cpu_rome'
clusterOptions = "-A ${tier1_project}"
}
}

cpu_rome_512 {
break
case "cpu_rome_512":
params {
config_profile_description = 'HPC_DODRIO_cpu_rome_512 profile for use on the Dodrio/cpu_rome_512 cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 512.GB
max_cpus = 128
max_time = "3day"
Expand All @@ -194,13 +165,9 @@ profiles {
queue = 'dodrio/cpu_rome_512'
clusterOptions = "-A ${tier1_project}"
}
}

cpu_milan {
break
case "cpu_milan":
params {
config_profile_description = 'HPC_DODRIO_cpu_milan profile for use on the Dodrio/cpu_milan cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 256.GB
max_cpus = 128
max_time = "3day"
Expand All @@ -211,13 +178,9 @@ profiles {
queue = 'dodrio/cpu_milan'
clusterOptions = "-A ${tier1_project}"
}
}

gpu_rome_a100_40 {
break
case "gpu_rome_a100_40":
params {
config_profile_description = 'HPC_DODRIO_gpu_rome_a100_40 profile for use on the Dodrio/gpu_rome_a100_40 cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 256.GB
max_cpus = 48
max_time = "3day"
Expand All @@ -228,13 +191,9 @@ profiles {
queue = 'dodrio/gpu_rome_a100_40'
clusterOptions = "-A ${tier1_project}"
}
}

gpu_rome_a100_80 {
break
case "gpu_rome_a100_80":
params {
config_profile_description = 'HPC_DODRIO_gpu_rome_a100_80 profile for use on the Dodrio/gpu_rome_a100_80 cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 512.GB
max_cpus = 48
max_time = "3day"
Expand All @@ -245,13 +204,9 @@ profiles {
queue = 'dodrio/gpu_rome_a100_80'
clusterOptions = "-A ${tier1_project}"
}
}

debug_rome {
break
case "debug_rome":
params {
config_profile_description = 'HPC_DODRIO_debug_rome profile for use on the Dodrio/debug_rome cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 256.GB
max_cpus = 48
max_time = "3day"
Expand All @@ -262,13 +217,9 @@ profiles {
queue = 'dodrio/debug_rome'
clusterOptions = "-A ${tier1_project}"
}
}

cpu_rome_all {
break
case "cpu_rome_all":
params {
config_profile_description = 'HPC_DODRIO_cpu_rome_all profile for use on the Dodrio/cpu_rome_all cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 250.GB
max_cpus = 128
max_time = "3day"
Expand All @@ -279,13 +230,9 @@ profiles {
queue = 'dodrio/cpu_rome_all'
clusterOptions = "-A ${tier1_project}"
}
}

gpu_rome_a100 {
break
case "gpu_rome_a100":
params {
config_profile_description = 'HPC_DODRIO_gpu_rome_a100 profile for use on the Dodrio/gpu_rome_a100 cluster of the VSC HPC.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 384.GB
max_cpus = 48
max_time = "3day"
Expand All @@ -296,16 +243,56 @@ profiles {
queue = 'dodrio/gpu_rome_a100'
clusterOptions = "-A ${tier1_project}"
}
break
}

// Define profiles for each cluster
profiles {
skitty {
System.err.println("Using the '-profile vsc_ugent,<cluster>' is deprecated in favor of '-profile vsc_ugent'. The config will now automatically determine the currently used cluster.")
}

stub {
params {
config_profile_description = 'Stub profile for the VSC HPC. Please also specify the `-stub` argument when using this profile.'
config_profile_contact = 'ict@cmgg.be'
config_profile_url = 'https://www.ugent.be/hpc/en'
max_memory = 2.GB
max_cpus = 1
max_time = 1.h
}
kirlia {
System.err.println("Using the '-profile vsc_ugent,<cluster>' is deprecated in favor of '-profile vsc_ugent'. The config will now automatically determine the currently used cluster.")
}

doduo {
System.err.println("Using the '-profile vsc_ugent,<cluster>' is deprecated in favor of '-profile vsc_ugent'. The config will now automatically determine the currently used cluster.")
}

shinx {
System.err.println("Using the '-profile vsc_ugent,<cluster>' is deprecated in favor of '-profile vsc_ugent'. The config will now automatically determine the currently used cluster.")
}

cpu_rome {
System.err.println("Using the '-profile vsc_ugent,<cluster>' is deprecated in favor of '-profile vsc_ugent'. The config will now automatically determine the currently used cluster.")
}

cpu_rome_512 {
System.err.println("Using the '-profile vsc_ugent,<cluster>' is deprecated in favor of '-profile vsc_ugent'. The config will now automatically determine the currently used cluster.")
}

cpu_milan {
System.err.println("Using the '-profile vsc_ugent,<cluster>' is deprecated in favor of '-profile vsc_ugent'. The config will now automatically determine the currently used cluster.")
}

gpu_rome_a100_40 {
System.err.println("Using the '-profile vsc_ugent,<cluster>' is deprecated in favor of '-profile vsc_ugent'. The config will now automatically determine the currently used cluster.")
}

gpu_rome_a100_80 {
System.err.println("Using the '-profile vsc_ugent,<cluster>' is deprecated in favor of '-profile vsc_ugent'. The config will now automatically determine the currently used cluster.")
}

debug_rome {
System.err.println("Using the '-profile vsc_ugent,<cluster>' is deprecated in favor of '-profile vsc_ugent'. The config will now automatically determine the currently used cluster.")
}

cpu_rome_all {
System.err.println("Using the '-profile vsc_ugent,<cluster>' is deprecated in favor of '-profile vsc_ugent'. The config will now automatically determine the currently used cluster.")
}

gpu_rome_a100 {
System.err.println("Using the '-profile vsc_ugent,<cluster>' is deprecated in favor of '-profile vsc_ugent'. The config will now automatically determine the currently used cluster.")
}
}
53 changes: 33 additions & 20 deletions docs/vsc_ugent.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
# nf-core/configs: University of Ghent High Performance Computing Infrastructure (VSC)

> **NB:** You will need an [account](https://www.ugent.be/hpc/en/access/faq/access) to use the HPC cluster to run the pipeline.
## Setup

> [!IMPORTANT]
> You will need an [account](https://www.ugent.be/hpc/en/access/faq/access) to use the HPC cluster to run the pipeline.
Regarding environment variables in `~/.bashrc`, make sure you have a setup similar to the one below. If you're already part of a VO, ask for one or use `VSC_DATA_USER` instead of `VSC_DATA_VO_USER`.

Expand Down Expand Up @@ -30,11 +33,11 @@ Before running the pipeline you will need to create a PBS script to submit as a

module load Nextflow

nextflow run <pipeline> -profile vsc_ugent,<CLUSTER> <Add your other parameters>
nextflow run <pipeline> -profile vsc_ugent <Add your other parameters>
```

All of the intermediate files required to run the pipeline will be stored in the `work/` directory. It is recommended to delete this directory after the pipeline has finished successfully because it can get quite large, and all of the main output files will be saved in the `results/` directory anyway.
The config contains a `cleanup` command that removes the `work/` directory automatically once the pipeline has completed successfully. If the run does not complete successfully then the `work/` dir should be removed manually to save storage space. The default work directory is set to `$VSC_SCRATCH_VO_USER/work` per this configuration
The config contains a `cleanup` command that removes the `work/` directory automatically once the pipeline has completed successfully. If the run does not complete successfully then the `work/` dir should be removed manually to save storage space.

You can also add several TORQUE options to the PBS script. More about this on this [link](http://hpcugent.github.io/vsc_user_docs/pdf/intro-HPC-linux-gent.pdf#appendix.B).

Expand All @@ -44,13 +47,21 @@ To submit your job to the cluster by using the following command:
qsub <script name>.pbs
```

> [!NOTE]
> The profile only works for the clusters `shinx`, `skitty`, `kirlia`, `doduo` and all tier1 clusters.
> [!NOTE]
> The default directory where the `work/` and `singularity/` (cache directory for images) is located in `$VSC_SCRATCH_VO_USER` (when you are part of a VO) or `$VSC_SCRATCH` (when you are not part of a VO) for tier2 clusters and `$VSC_SCRATCH_PROJECTS_BASE/<tier1_project_name>` for tier1 clusters.
## Use Apptainer containers

The VSC does [not support](https://docs.hpc.ugent.be/Linux/apptainer/) Apptainer containers provided via a URL (e.g., shub://... or docker://...).
One solution is to download all the containers beforehand, like in [this pipeline](https://github.com/saeyslab/spotless-benchmark).

First get the containers.json file from the pipeline you want to run:

```bash
nextflow inspect main.nf -profile vsc_ugent,<CLUSTER> > containers.json
nextflow inspect main.nf -profile vsc_ugent > containers.json
```

Then run a build script (script appended below) to build all the containers. This can take a long time and a lot of space, but it is a one-time cost. For many large images, consider running this as a job.
Expand All @@ -59,23 +70,9 @@ Then run a build script (script appended below) to build all the containers. Thi
bash build_all_containers.sh containers.json
```

Overwrite the container in your `nextflow.config`. If you need GPU support, also apply the label 'use_gpu':

```nextflow
process {
withName: DEEPCELL_MESMER {
label = 'use_gpu'
// container "docker.io/vanvalenlab/deepcell-applications:0.4.1"
container = "./DEEPCELL_MESMER_GPU.sif"
}
}
```

> **NB:** The profile only works for the clusters `skitty`, `swalot`, `victini`, `kirlia` and `doduo`.
<details>

> **NB:** The default directory where the `work/` and `singularity/` (cache directory for images) is located in `$VSC_SCRATCH_VO_USER`.
`build_all_containers.sh`:
<summary> <code>build_all_containers.sh</code> </summary>

```bash
#!/bin/env bash
Expand Down Expand Up @@ -118,3 +115,19 @@ paste <(echo "$NAMES") <(echo "$CONTAINERS") | while IFS=$'\t' read -r name cont
mv /tmp/$USER/$name.sif $name.sif
done
```

</details>

## Use GPUs for your pipelines

Overwrite the container in your `nextflow.config`. If you need GPU support, also apply the label 'use_gpu':

```nextflow
process {
withName: DEEPCELL_MESMER {
label = 'use_gpu'
// container "docker.io/vanvalenlab/deepcell-applications:0.4.1"
container = "./DEEPCELL_MESMER_GPU.sif"
}
}
```

0 comments on commit 48b0fdc

Please sign in to comment.