Template standardization

README.md

@@ -1,5 +1,7 @@
 # Exomiser
 
+# Exomiser
+
 ## Pipeline documentation
 
 Table of contents
@@ -62,7 +64,7 @@ This is a file needed by exomiser to run. It contains placeholders in the text t
 
 ### --exomiser_data
 
-This path refers to the reference data bundle needed by exomiser (~120 GB!). A copy of such files can be found [here](https://lifebit-featured-datasets.s3.eu-west-1.amazonaws.com/pipelines/exomiser-data-bundle/) . The reference dataset has been added as a parameter, allowing flexibility to pull the data from any resource (i.e. cloud, local storage, ftp, ...) and Nextlfow will automatically take care of fetching the data without having to add anything to the pipeline itself.
+This path refers to the reference data bundle needed by exomiser (~120 GB!). A copy of such files can be found [here](https://lifebit-featured-datasets.s3.eu-west-1.amazonaws.com/pipelines/exomiser-data-bundle/) . The reference dataset has been added as a parameter, allowing flexibility to pull the data from any resource (i.e. cloud, local storage, ftp, ...) and Nextflow will automatically take care of fetching the data without having to add anything to the pipeline itself.
 
 There are other parameters that can be tweaked to personalize the behaviour of the pipeline. These are referenced in `nextflow.config`
 
@@ -98,13 +100,19 @@ To run the pipeline with `docker` (used by default), type the following commands
 To test the pipeline on a multi-VCF:
 
 ```
-nextflow run main.nf -profile family_test
+nextflow run main.nf -profile test_full_family
+```
+
+or
+
+```
+nextflow run main.nf -profile test_full_multi_hpo
 ```
 
 To test the pipeline on a single-sample VCF:
 
 ```
-nextflow run main.nf -profile single_vcf_test
+nextflow run main.nf -profile test_full_single_vcf
 ```
 
 Be careful when running this, as the pipeline requires the staging of 120 GB of reference data, required by exomiser, so only that takes a while!
@@ -113,7 +121,8 @@ Be careful when running this, as the pipeline requires the staging of 120 GB of
 
 ### Profiles
 
-|  profile name   |                              Run locally                               | Run on CloudOS |                                   description                                   |
-| :-------------: | :--------------------------------------------------------------------: | :------------: | :-----------------------------------------------------------------------------: |
-|   test_family   | the data required is so big, it was tested on a c5.4xlarge EC2 machine |   Successful   | this test is designed to test the pipeline on a multi-VCF with trio information |
-| test_single_vcf | the data required is so big, it was tested on a c5.4xlarge EC2 machine |   Successful   |        this test is designed to test the pipeline on a single-sample-VCF        |
+|     profile name     |                              Run locally                               | Run on CloudOS |                                               description                                                |
+| :------------------: | :--------------------------------------------------------------------: | :------------: | :------------------------------------------------------------------------------------------------------: |
+|   test_full_family   | the data required is so big, it was tested on a c5.4xlarge EC2 machine |   Successful   |             this test is designed to test the pipeline on a multi-VCF with trio information              |
+| test_full_single_vcf | the data required is so big, it was tested on a c5.4xlarge EC2 machine |   Successful   |                    this test is designed to test the pipeline on a single-sample-VCF                     |
+| test_full_multi_hpo  | the data required is so big, it was tested on a c5.4xlarge EC2 machine |   Successful   | this test is designed to test the pipeline on a multi-VCF with trio information using multiple HPO terms |

conf/containers/dockerhub.config

@@ -0,0 +1,3 @@
+// params {
+//     main_container = 'dockerhub.io/lifebitaiorg/report:latest'
+// }

conf/containers/ecr.config

@@ -0,0 +1,5 @@
+// params {
+// main_container = 'https://${params.aws_account_id}.dkr.${params.aws_region}.amazonaws.com/lifebitaiorg/report:latest'
+// }
+// ECR pattern:
+// https://aws_account_id.dkr.ecr.region.amazonaws.com/lifebitaiorg/tool:version

conf/containers/quay.config

@@ -1,3 +1,3 @@
 params {
-    main_container = 'quay.io/lifebitai/exomiser:12.1.0'
-}
+    main_container = "quay.io/lifebitai/exomiser:${params.exomiser_container_tag}"
+    }

conf/customised_pipeline_resources.config

@@ -0,0 +1,37 @@
+params {
+
+    // process resources default
+    cpus = 1
+    memory = 2.GB
+    time = 8.h // do not change
+
+    // max resources limits defaults
+    max_cpus = 8
+    max_memory = 60.GB
+    max_time = 300.h // do not change
+
+    // process_micro defaults
+    micro_memory = 2.GB
+    micro_cpus = 1
+
+    // process_small defaults
+    small_memory = 4.GB
+    small_cpus = 2
+
+    // process_medium defaults
+    medium_memory = 6.GB
+    medium_cpus = 4
+
+    // process_large defaults
+    large_memory = 15.GB
+    large_cpus = 4
+
+    // other parameters
+    echo = false
+    errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'terminate' }
+    maxErrors = -1
+    maxRetries = 3
+    maxForks = 200
+    queueSize = 200
+
+}

conf/data/data.config

@@ -1 +1,7 @@
 // If there is any data that needs to be included in the config, it should be placed here using "${params.reference_data_bucket}/path/to/data"
+params {
+    exomiser_data   = "${params.reference_data_bucket}/pipelines/exomiser-data-bundle"
+    application_properties = "${params.reference_data_bucket}/pipelines/exomiser-nf/application.properties"
+    auto_config_yml = '${params.reference_data_bucket}/pipelines/exomiser-nf/auto_config_v2.yml'
+
+}

conf/executors/singularity.config

@@ -0,0 +1,20 @@
+/*
+ * -------------------------------------------------
+ *  Nextflow config file for running pipeline with Singularity locally
+ * -------------------------------------------------
+ * Base config needed for running with -profile singularity
+ */
+
+params {
+  singularity_cache = "local_singularity_cache"
+}
+
+singularity {
+  enabled = true
+  cacheDir = params.singularity_cache
+  autoMounts = true
+}
+
+docker {
+    enabled = false
+}

conf/tests/full/test_full_family.config

@@ -0,0 +1,7 @@
+params {
+    families_file     = 's3://lifebit-featured-datasets/pipelines/exomiser-nf/fam_file.tsv'
+    prioritisers    = 'hiPhivePrioritiser'
+    exomiser_data   = "s3://lifebit-featured-datasets/pipelines/exomiser-data-bundle"
+    application_properties = 's3://lifebit-featured-datasets/pipelines/exomiser-nf/application.properties'
+    auto_config_yml = 's3://lifebit-featured-datasets/pipelines/exomiser-nf/auto_config.yml'
+}

conf/tests/full/test_full_multi_hpo.config

@@ -0,0 +1,7 @@
+params {
+    families_file     = 's3://lifebit-featured-datasets/pipelines/exomiser-nf/fam_file_multi_hpo.tsv'
+    prioritisers    = 'hiPhivePrioritiser'
+    exomiser_data   = "s3://lifebit-featured-datasets/pipelines/exomiser-data-bundle"
+    application_properties = 's3://lifebit-featured-datasets/pipelines/exomiser-nf/application.properties'
+    auto_config_yml = 's3://lifebit-featured-datasets/pipelines/exomiser-nf/auto_config.yml'
+}

conf/tests/full/test_full_single_vcf.config

@@ -0,0 +1,8 @@
+params {
+    families_file     = 's3://lifebit-featured-datasets/pipelines/exomiser-nf/single_vcf.tsv'
+    hpo_terms_file  = 's3://lifebit-featured-datasets/pipelines/exomiser-nf/hpo_terms_file.txt'
+    prioritisers    = 'hiPhivePrioritiser'
+    exomiser_data   = "s3://lifebit-featured-datasets/pipelines/exomiser-data-bundle"
+    application_properties = 's3://lifebit-featured-datasets/pipelines/exomiser-nf/application.properties'
+    auto_config_yml = 's3://lifebit-featured-datasets/pipelines/exomiser-nf/auto_config.yml'
+}

nextflow.config

@@ -2,12 +2,16 @@ manifest {
     name        = 'lifebit-ai/exomiser-nf'
     description = 'A pipeline to perform variant prioritisation'
     mainScript  = 'main.nf'
-    version     = 'v1.0'
+    version     = 'v2.0'
 }
 
+includeConfig 'conf/customised_pipeline_resources.config'
+
 docker.enabled = true
 
 params {
+  raci_owner        = "Lifebit"
+
   // Exomiser specific parameters
   reference_data_bucket = "s3://lifebit-featured-datasets"
   bucket_pattern = "lifebit-featured-datasets"
@@ -23,8 +27,8 @@ params {
   exomiser_phenotype_data = 's3://lifebit-featured-datasets/pipelines/exomiser/very_fake/2102_phenotype'
   cadd_snvs = 's3://lifebit-featured-datasets/pipelines/exomiser/very_fake/cadd_snvs'
   phenix_data = 's3://lifebit-featured-datasets/pipelines/exomiser/very_fake/phenix'
-  application_properties = 's3://lifebit-featured-datasets/pipelines/exomiser-nf/application.properties'
-  auto_config_yml = 's3://lifebit-featured-datasets/pipelines/exomiser-nf/auto_config_v2.yml'
+  application_properties = "${params.reference_data_bucket}/pipelines/exomiser-nf/application.properties"
+  auto_config_yml = '${params.reference_data_bucket}/pipelines/exomiser-nf/auto_config_v2.yml'
   hpo_terms_file = false
   modes_of_inheritance = 'AUTOSOMAL_DOMINANT,AUTOSOMAL_RECESSIVE,X_RECESSIVE,UNDEFINED'
   prioritisers = 'hiPhivePrioritiser,phivePrioritiser,phenixPrioritiser'
@@ -48,11 +52,16 @@ params {
   exomiser_container_tag = '12.1.0'
   cloudos_cli_container_tag = '0.0.2'
 
-  // awsbatch specific
-  aws_batch_process_queue = null
-  aws_batch_cli_path = '/home/ec2-user/miniconda/bin/aws'
-  aws_batch_fetch_instance_type = true
-  aws_region = 'ap-east-1'
+     queueSize = 200
+    executor = false
+
+    // AWS batch
+    aws_region = 'eu-west-1'
+    aws_batch_default_queue = "optimal-instance-1tb-ami-on-demand-queue"
+    aws_batch_cli_path = '/home/ec2-user/miniconda/bin/aws' 
+    aws_batch_fetch_instance_type = true
+    aws_batch_max_parallel_transfers = 2
+    aws_batch_volumes = '/home/ec2-user/.aws:/root/.aws'
 
   //process resources
   memory = 6.GB  
@@ -63,25 +72,33 @@ params {
   maxRetries = 3
 }
 
+includeConfig 'conf/containers/quay.config'
+includeConfig 'conf/data/data.config' // Loads in data
+
 
 profiles {
    standard { includeConfig params.config }
-   test_family { includeConfig 'conf/family_test.config' }
-   test_single_vcf { includeConfig 'conf/single_vcf_test.config' }
-   test_multi_hpo { includeConfig 'conf/multi_hpo_test.config' }
    awsbatch { includeConfig 'conf/executors/awsbatch.config' }
    eu_west_1 { includeConfig 'conf/cloud-region/eu_west_1.config' }
    eu_west_2 { includeConfig 'conf/cloud-region/eu_west_2.config' }
    test_full { includeConfig "conf/tests/full/test_full.config" }
-   ci_test_data { includeConfig "conf/data/ci_test_data.config" }
+   test_full_family { includeConfig 'conf/tests/full/test_full_family.config' }
+   test_full_single_vcf { includeConfig 'conf/tests/full/test_full_single_vcf.config' }
+   test_full_multi_hpo { includeConfig 'conf/tests/full/test_full_multi_hpo.config' }
+   ci_test_data { includeConfig "conf/tests/ci/ci_test_data.config" }
+   singularity { includeConfig 'conf/executors/singularity.config' }
+   dockerhub { includeConfig 'conf/containers/dockerhub.config' }
    quay { includeConfig 'conf/containers/quay.config' }
+   ecr { includeConfig 'conf/containers/ecr.config' }
 }
 
+includeConfig 'conf/resources.config'
+
 process {
   echo = params.echo
   errorStrategy = params.errorStrategy
   withName: exomiser {
-    container = "quay.io/lifebitai/exomiser:${params.exomiser_container_tag}"
+    container = params.main_container
     containerOptions = "--volume ${params.exomiser_data_directory}:/data/"
     memory = params.memory
     cpus = params.cpus
@@ -90,4 +107,10 @@ process {
     errorStrategy = params.errorStrategy
     maxRetries = params.maxRetries
   }
+}
+
+
+executor {
+    name = params.executor
+    queueSize = params.queueSize
 }