Add ncbi egapx #6456
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| categories: | ||
| - Genome annotation | ||
| description: Eukaryotic Genome Annotation Pipeline - External (EGAPx) | ||
| homepage_url: https://github.com/ncbi/egapx | ||
| long_description: | | ||
| EGAPx is the publicly accessible version of the updated NCBI Eukaryotic | ||
| Genome Annotation Pipeline. EGAPx takes an assembly fasta file, a | ||
| taxid of the organism, and RNA-seq data. Based on the taxid, EGAPx will | ||
| pick protein sets and HMM models. The pipeline runs miniprot to align | ||
| protein sequences, and STAR to align RNA-seq to the assembly. Protein | ||
| alignments and RNA-seq read alignments are then passed to Gnomon for | ||
| gene prediction. In the first step of Gnomon, the short alignments | ||
| are chained together into putative gene models. In the second step, | ||
| these predictions are further supplemented by ab-initio predictions | ||
| based on HMM models. The final annotation for the input assembly is | ||
| produced as a gff file. | ||
| name: ncbi_egapx | ||
| owner: iuc | ||
| remote_repository_url: https://github.com/galaxyproject/tools-iuc/tree/main/tools/ncbi_egapx | ||
| type: unrestricted |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| <macros> | ||
| <xml name="requirements"> | ||
| <requirements> | ||
| <container type="docker">quay.io/richard-burhans/egapx:@TOOL_VERSION@</container> | ||
| </requirements> | ||
| </xml> | ||
| <token name="@TOOL_VERSION@">0.2-alpha</token> | ||
| <token name="@VERSION_SUFFIX@">0</token> | ||
| <token name="@PROFILE@">22.05</token> | ||
| <xml name="edam_ontology"> | ||
| <edam_operations> | ||
| <edam_operation>operation_0362</edam_operation> | ||
| </edam_operations> | ||
| </xml> | ||
| <xml name="citations"> | ||
| <citations> | ||
| <citation type="doi">10.1093/bioinformatics/bts573</citation> | ||
richard-burhans marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| </citations> | ||
| </xml> | ||
| </macros> | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,312 @@ | ||
| <tool id="ncbi_egapx" name="NCBI EGAPx" version="@TOOL_VERSION@+galaxy@VERSION_SUFFIX@" profile="@PROFILE@"> | ||
| <description>annotates eukaryotic genomes</description> | ||
| <macros> | ||
| <import>macros.xml</import> | ||
| </macros> | ||
| <expand macro="edam_ontology"/> | ||
| <expand macro="requirements"/> | ||
| <command detect_errors="aggressive"><![CDATA[ | ||
| #if str($cond_input_style.input_style) == "fillform" | ||
| #set yamlconfig = $egapx_config | ||
| #else | ||
| #set yamlconfig = $yamlin | ||
| #end if | ||
| source /galaxy/env.bash && | ||
|
There was a problem hiding this comment. can you add please a comment what this one is doing? |
||
| echo \${PATH} && | ||
| ln -s /galaxy/egapx/egapx_config && | ||
| python3 /galaxy/egapx/ui/egapx.py '$yamlconfig' -e galaxy -o 'egapx_out' | ||
| ]]></command> | ||
| <configfiles> | ||
| <configfile name="egapx_config"><![CDATA[ | ||
| #if str($cond_input_style.input_style) == "fillform" | ||
| # yaml generated by ncbi_egapx.xml | ||
| #if str($reference_genome.genome_type_select) == "history" | ||
| #set genome_value = $reference_genome.genome | ||
| #elif str($reference_genome.genome_type_select) == "indexed" | ||
| #set genome_value = $reference_genome.genome.fields.path | ||
| #else | ||
| #set genome_value = $reference_genome.uri | ||
| #end if | ||
| genome: $genome_value | ||
| taxid: $taxid | ||
| #if str($condrnaseq.rna_type_select) == "list" | ||
| #set $reads_values = $rnaseq.split() | ||
| #else | ||
| #set $reads_values = $rnaseq | ||
| #end if | ||
| reads: | ||
| #for r in [x.strip() for x in $reads_values] | ||
| - $r | ||
| #end for | ||
| #if str($proteins) != "None" | ||
| proteins: $proteins | ||
| #end if | ||
| #for row in $xtra.strip().split("\n") | ||
| $row | ||
| #end for | ||
| #end if | ||
| ]]></configfile> | ||
| </configfiles> | ||
| <inputs> | ||
| <conditional name="cond_input_style"> | ||
| <param name="input_style" type="select" label="Fill in a tool form or use an existing yaml configuration from the current history?" | ||
| help="Use the tool form to select inputs from the history, or use a pre-prepared yaml file."> | ||
| <option value="fillform" selected="True">Provide configuration details for conversion into a configuration yaml</option> | ||
| <option value="history">Use a pre-prepared yaml egapx configuration</option> | ||
| </param> | ||
| <when value="fillform"> | ||
| <conditional name="reference_genome"> | ||
| <param name="genome_type_select" type="select" label="Reference genome source for mapping supplied RNA-seq reads" | ||
| help="Select a built in, history or remote URI for the reference genome FASTA"> | ||
| <option value="history" selected="True">Use a genome FASTA file from the current history</option> | ||
| <option value="indexed">Use a Galaxy server built-in genome</option> | ||
| <option value="uri">Provide a remote web link URI ("https://...") pointing at the required genome reference FASTA file</option> | ||
| </param> | ||
| <when value="history"> | ||
| <param name="genome" type="data" format="fasta" label="Select the reference genome FASTA from the current history"/> | ||
| </when> | ||
| <when value="indexed"> | ||
| <param name="genome" type="select" label="Select a built in reference genome or custom genome" | ||
| help="If not listed, add a custom genome or use a reference genome from the history"> | ||
| <options from_data_table="all_fasta"> | ||
| <validator message="No genomes are available " type="no_options"/> | ||
| </options> | ||
| </param> | ||
| </when> | ||
| <when value="uri"> | ||
| <param name="uri" type="text" label="URI pointing to the reference genome FASTA file"> | ||
| </param> | ||
| </when> | ||
| </conditional> | ||
| <param name="taxid" type="integer" min="0" label="NCBI Taxon ID" help="Used to identify the HMM model files needed"/> | ||
| <conditional name="condrnaseq"> | ||
| <param name="rna_type_select" type="select" label="RNA sequence data source" | ||
| help="Select RNAseq input data from history or input a list of SRA identifiers or remote URI"> | ||
| <option value="list" selected="True">Type in a list of SRA identifiers and/or remote RNA-seq FASTA URI</option> | ||
| <option value="history">Select one or more RNA-seq fastq datasets from the current history</option> | ||
| </param> | ||
| <when value="list"> | ||
| <param name="rnaseq" type="text" area="true" label="List all required individual RNA-seq URI or SRA identifiers, separated by spaces or newlines" | ||
| help="Either a working URI for a RNA-seq FASTA, or a bare SRA identifier will work - can be mixed"> | ||
| <validator type="empty_field"/> | ||
| </param> | ||
| </when> | ||
| <when value="history"> | ||
| <param name="rnaseq" type="data" format="fastqsanger,fastqsanger.gz" multiple="true" label="Select multiple RNA-seq fastqsanger inputs from the current history" | ||
|
There was a problem hiding this comment. paired end is irrelevant? |
||
| help="All selected rna-seq fastqsanger will be added to the yaml for egapx configuration"/> | ||
| </when> | ||
| </conditional> | ||
| <param name="proteins" type="data" format="fasta,fasta.gz" optional="true" label="Select a protein set"/> | ||
| <param name="xtra" type="text" area="true" label="Additional yaml to append to the egapx.yaml configuration" | ||
|
There was a problem hiding this comment. We probably need a validator for free text entry (and disabling the sanitizer as definitely a bad idea). Maybe better allow yaml upload? |
||
| help="Not normally needed but useful for testing additional configuration elements"> | ||
| <sanitizer invalid_char=""> | ||
| <valid initial="string.printable"/> | ||
| </sanitizer> | ||
| </param> | ||
| </when> | ||
| <when value="history"> | ||
| <param name="yamlin" type="data" format="yaml" label="egapx configuration yaml file to pass to Nextflow"/> | ||
| </when> | ||
| </conditional> | ||
| </inputs> | ||
| <outputs> | ||
| <data name="output" format="gff" label="EGAPx annotation for ${on_string}" from_work_dir="egapx_out/accept.gff"/> | ||
| <collection name="nextflow_stats" type="list" label="EGAPx nextflow stats for ${on_string}"> | ||
| <data name="nf_log" format="txt" label="Nextflow execution log" from_work_dir="egapx_out/nextflow.log"/> | ||
| <data name="nf_report" format="html" label="Nextflow execution report" from_work_dir="egapx_out/run.report.html"/> | ||
| <data name="nf_trace" format="tabular" label="Nextflow trace file" from_work_dir="egapx_out/run.trace.txt"/> | ||
| <data name="nf_timeline" format="html" label="Nextflow execution timeline" from_work_dir="egapx_out/run.timeline.html"/> | ||
| <data name="nf_params" format="yaml" label="Nextflow run parameters" from_work_dir="egapx_out/run_params.yaml"/> | ||
| </collection> | ||
| </outputs> | ||
| <tests> | ||
| <test expect_test_failure="false"> | ||
| <param name="input_style" value="fillform"/> | ||
| <param name="taxid" value="6954"/> | ||
| <param name="genome_type_select" value="uri"/> | ||
| <param name="uri" value="https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/020/809/275/GCF_020809275.1_ASM2080927v1/GCF_020809275.1_ASM2080927v1_genomic.fna.gz"/> | ||
| <param name="rna_type_select" value="list"/> | ||
| <param name="rnaseq" value="https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/EGAP/sample_data/Dermatophagoides_farinae_small/SRR8506572.1 https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/EGAP/sample_data/Dermatophagoides_farinae_small/SRR8506572.2 https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/EGAP/sample_data/Dermatophagoides_farinae_small/SRR9005248.1 https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/EGAP/sample_data/Dermatophagoides_farinae_small/SRR9005248.2"/> | ||
| <param name="xtra" value="hmm: https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/EGAP/gnomon/hmm_parameters/6956.params tasks: star_wnode: star_wnode: -cpus-per-worker 4"/> | ||
| <output name="output"><assert_contents><has_size min="1"/></assert_contents></output> | ||
| <output_collection name="nextflow_stats" type="list"> | ||
| <element name="nf_log"><assert_contents><has_size min="1"/></assert_contents></element> | ||
| <element name="nf_report"><assert_contents><has_size min="1"/></assert_contents></element> | ||
| <element name="nf_trace"><assert_contents><has_size min="1"/></assert_contents></element> | ||
| <element name="nf_timeline"><assert_contents><has_size min="1"/></assert_contents></element> | ||
| <element name="nf_params"><assert_contents><has_size min="1"/></assert_contents></element> | ||
| </output_collection> | ||
| </test> | ||
| <test expect_test_failure="false"> | ||
| <param name="input_style" value="history"/> | ||
| <param name="yamlin" value="input.yaml"/> | ||
| <output name="output"><assert_contents><has_size min="1"/></assert_contents></output> | ||
| <output_collection name="nextflow_stats" type="list"> | ||
| <element name="nf_log"><assert_contents><has_size min="1"/></assert_contents></element> | ||
| <element name="nf_report"><assert_contents><has_size min="1"/></assert_contents></element> | ||
| <element name="nf_trace"><assert_contents><has_size min="1"/></assert_contents></element> | ||
| <element name="nf_timeline"><assert_contents><has_size min="1"/></assert_contents></element> | ||
| <element name="nf_params"><assert_contents><has_size min="1"/></assert_contents></element> | ||
| </output_collection> | ||
| </test> | ||
| </tests> | ||
| <help><![CDATA[ | ||
| Galaxy tool wrapping the Eukaryotic Genome Annotation Pipeline (EGAPx) | ||
| ================================================================================================= | ||
|
|
||
| .. class:: warningmark | ||
|
|
||
| **Proof of concept: a hack to run a NF workflow inside a specialised Galaxy tool wrapper** | ||
|
There was a problem hiding this comment. Is it really surprising that this works? We can execute anything that runs on a command line, or? |
||
|
|
||
| EGAPx is a big, complicated Nextflow workflow, challenging and costly to re-implement **properly**, requiring dozens of new tools and replicating a lot of | ||
|
There was a problem hiding this comment. My second thought was that, while it is certainly challenging and a lot of work to integrate the tools it would pay of for the Galaxy project as a whole.
Is it systematically tested (not as far as I see, at least not on github) |
||
| complicated *groovy* workflow logic. | ||
|
|
||
| It is also very new and in rapid development. Investing developer effort and keeping updated as EGAPx changes rapidly may be *inefficient of developer resources*. | ||
|
|
||
| This wrapper is designed to allow measuring how *inefficient* it is in terms of computing resource utilisation, in comparison to the developer effort | ||
|
There was a problem hiding this comment.
How will this be measured?
This was my 1st thought, having one resource allocation for the whole pipeline will not work on many systems. On our system jobs that are not utilizing the allocated CPUs/memory efficiently are automatically killed. We already have quite a few tools wrapping (non-NF) pipelines which all have this problem. On my system I can only assign 1 CPU to them, because they are killed otherwise. |
||
| required to convert Nextflow DDL into tools and WF logic. Balancing these competing requirements is a fundamental Galaxy challenge. | ||
|
|
||
|
|
||
| EGAPx requires very substantial resources to run with real data. *132GB and 32 cores* are the minimum requirement; *256GB and 64 cores* are recommended. | ||
|
|
||
| A special minimal example that can be run in 6GB with 4 cores is provided as a yaml configuration and is used for the tool test. | ||
|
|
||
| In this implementation, the user must supply a yaml configuration file as initial proof of concept. | ||
| History inputs and even a yaml editor might be provided in future. | ||
|
|
||
| The NF workflow to tool model tested here may be applicable to other NF workflows that take a single configuration yaml. | ||
|
|
||
| .. class:: warningmark | ||
|
|
||
| The computational resource cost of typing the wrong SRA identifiers into a tool form is potentially enormous with this tool! | ||
|
|
||
|
|
||
| Sample yaml configurations | ||
| =========================== | ||
|
|
||
| YAML sample configurations can be uploaded into your Galaxy history from the `EGAPx github repository <https://github.com/ncbi/egapx/tree/main/examples/>`_. | ||
| The simplest possible example is shown below - can be cut/paste into a history dataset in the upload tool. | ||
|
|
||
|
|
||
| *./examples/input_D_farinae_small.yaml* is shown below and can be cut and pasted into the upload form to create a yaml file. | ||
| RNA-seq data is provided as URI to the reads FASTA files. | ||
|
|
||
| input_D_farinae_small.yaml | ||
|
|
||
| :: | ||
|
|
||
| genome: https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/020/809/275/GCF_020809275.1_ASM2080927v1/GCF_020809275.1_ASM2080927v1_genomic.fna.gz | ||
| taxid: 6954 | ||
| reads: | ||
| - https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/EGAP/sample_data/Dermatophagoides_farinae_small/SRR8506572.1 | ||
| - https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/EGAP/sample_data/Dermatophagoides_farinae_small/SRR8506572.2 | ||
| - https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/EGAP/sample_data/Dermatophagoides_farinae_small/SRR9005248.1 | ||
| - https://ftp.ncbi.nlm.nih.gov/genomes/TOOLS/EGAP/sample_data/Dermatophagoides_farinae_small/SRR9005248.2 | ||
|
|
||
|
|
||
| input_Gavia_stellata.yaml | ||
|
|
||
| :: | ||
|
|
||
| genome: https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/030/936/135/GCF_030936135.1_bGavSte3.hap2/GCF_030936135.1_bGavSte3.hap2_genomic.fna.gz | ||
| reads: txid37040[Organism] AND biomol_transcript[properties] NOT SRS024887[Accession] | ||
| taxid: 37040 | ||
|
|
||
| input_C_longicornis.yaml | ||
|
|
||
| :: | ||
|
|
||
| genome: https://ftp.ncbi.nlm.nih.gov/genomes/all/GCF/029//603/195/GCF_029603195.1_ASM2960319v2/GCF_029603195.1_ASM2960319v2_genomic.fna.gz | ||
| reads: txid2530218[Organism] AND biomol_transcript[properties] NOT SRS024887[Accession] | ||
| taxid: 2530218 | ||
|
|
||
| Purpose | ||
| ======== | ||
|
|
||
| **This is not intended for production** | ||
|
|
||
| Just a proof of concept. | ||
| It is possibly too inefficient to be useful although it may turn out not to be a problem if run on a dedicated workstation. | ||
| At least the efficiency can now be more easily estimated. | ||
|
|
||
| This tool is not recommended for public deployment because of the resource demands. | ||
|
|
||
| EGAPx Overview | ||
| =============== | ||
|
|
||
| .. image:: $PATH_TO_IMAGES/Pipeline_sm_ncRNA_CAGE_80pct.png | ||
|
|
||
| **Warning:** | ||
| The current version is an alpha release with limited features and organism scope to collect initial feedback on execution. Outputs are not yet complete and not intended for production use. Please open a GitHub [Issue](https://github.com/ncbi/egapx/issues) if you encounter any problems with EGAPx. You can also write to cgr@nlm.nih.gov to give us your feedback or if you have any questions. | ||
|
|
||
| EGAPx is the publicly accessible version of the updated NCBI [Eukaryotic Genome Annotation Pipeline](https://www.ncbi.nlm.nih.gov/genome/annotation_euk/process/). | ||
|
|
||
| EGAPx takes an assembly FASTA file, a taxid of the organism, and RNA-seq data. Based on the taxid, EGAPx will pick protein sets and HMM models. The pipeline runs `miniprot` to align protein sequences, and `STAR` to align RNA-seq to the assembly. Protein alignments and RNA-seq read alignments are then passed to `Gnomon` for gene prediction. In the first step of `Gnomon`, the short alignments are chained together into putative gene models. | ||
| In the second step, these predictions are further supplemented by *ab-initio* predictions based on HMM models. The final annotation for the input assembly is produced as a `gff` file. | ||
|
|
||
| **Security Notice:** | ||
|
|
||
| EGAPx has dependencies in and outside of its execution path that include several thousand files from the [NCBI C++ toolkit](https://www.ncbi.nlm.nih.gov/toolkit), and more than a million total lines of code. Static Application Security Testing has shown a small number of verified buffer overrun security vulnerabilities. Users should consult with their organizational security team on risk and if there is concern, consider mitigating options like running via VM or cloud instance. | ||
|
|
||
|
|
||
| *To specify an array of NCBI SRA datasets in yaml* | ||
|
|
||
| :: | ||
|
|
||
| reads: | ||
| - SRR8506572 | ||
| - SRR9005248 | ||
|
|
||
|
|
||
| *To specify an SRA entrez query* | ||
|
|
||
| :: | ||
|
|
||
| reads: 'txid6954[Organism] AND biomol_transcript[properties] NOT SRS024887[Accession] AND (SRR8506572[Accession] OR SRR9005248[Accession] )' | ||
|
|
||
|
|
||
| **Note:** Both the above examples will have more RNA-seq data than the `input_D_farinae_small.yaml` example. To make sure the entrez query does not produce a large number of SRA runs, please run it first at the [NCBI SRA page](https://www.ncbi.nlm.nih.gov/sra). If there are too many SRA runs, then select a few of them and list it in the input yaml. | ||
|
|
||
| Output | ||
| ======= | ||
|
|
||
| EGAPx output will appear as a collection in the user history. The main annotation file is called *accept.gff*. | ||
|
|
||
| :: | ||
|
|
||
| accept.gff | ||
| annot_builder_output | ||
| nextflow.log | ||
| run.report.html | ||
| run.timeline.html | ||
| run.trace.txt | ||
| run_params.yaml | ||
|
|
||
|
|
||
| The *nextflow.log* is the log file that captures all the process information and their work directories. ``run_params.yaml`` has all the parameters that were used in the EGAPx run. More information about the process time and resources can be found in the other run* files. | ||
|
|
||
| ## Intermediate files | ||
|
|
||
| In the log, each line denotes the process that completed in the workflow. The first column (_e.g._ `[96/621c4b]`) is the subdirectory where the intermediate output files and logs are found for the process in the same line, _i.e._, `egapx:miniprot:run_miniprot`. To see the intermediate files for that process, you can go to the work directory path that you had supplied and traverse to the subdirectory `96/621c4b`: | ||
|
|
||
| :: | ||
|
|
||
| $ aws s3 ls s3://temp_datapath/D_farinae/96/ | ||
| PRE 06834b76c8d7ceb8c97d2ccf75cda4/ | ||
| PRE 621c4ba4e6e87a4d869c696fe50034/ | ||
| $ aws s3 ls s3://temp_datapath/D_farinae/96/621c4ba4e6e87a4d869c696fe50034/ | ||
| PRE output/ | ||
| 2024-03-27 11:19:18 0 | ||
| 2024-03-27 11:19:28 6 .command.begin | ||
| 2024-03-27 11:20:24 762 .command.err | ||
| 2024-03-27 11:20:26 762 .command.log | ||
| 2024-03-27 11:20:23 0 .command.out | ||
| 2024-03-27 11:19:18 13103 .command.run | ||
| 2024-03-27 11:19:18 129 .command.sh | ||
| 2024-03-27 11:20:24 276 .command.trace | ||
| 2024-03-27 11:20:25 1 .exitcode | ||
| $ aws s3 ls s3://temp_datapath/D_farinae/96/621c4ba4e6e87a4d869c696fe50034/output/ | ||
| 2024-03-27 11:20:24 17127134 aligns.paf | ||
| ]]></help> | ||
| <expand macro="citations"/> | ||
| </tool> | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
let me know if you need access to the Galaxy repo on quay.io - using this space would enable also automatically singulairy builds.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@bgruening Yes, access to the Galaxy repo on quay.io would be great!
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
invitation sent
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.