Merge pull request #103 from shandley/dev

Dev

Author: beardymcjohnface

hecatomb/hecatomb.VERSION

@@ -1 +1 @@
-v1.3.0
+v1.3.1

hecatomb/snakemake/workflow/AddHost.smk

@@ -44,7 +44,7 @@ rule minimap_viral_refseq:
         ref = config["args"]["hostFa"],
         vir = config["virRefSeq"]
     output:
-        temp(os.path.join(dir["out"]["temp"], f'{config["args"]["hostName"]}.bed'))
+        temp(os.path.join(dir["out"]["temp"], config["args"]["hostName"] + ".bed"))
     params:
         config["addHost"]["minViralAlnLen"]
     conda:
@@ -73,9 +73,9 @@ rule mask_fasta:
     """Mask the host genome using bedtools"""
     input:
         fa = config["args"]["hostFa"],
-        aln = os.path.join(dir["out"]["temp"], f'{config["args"]["hostName"]}.bed')
+        aln = os.path.join(dir["out"]["temp"], config["args"]["hostName"] + ".bed")
     output:
-        mask = temp(os.path.join(dir["out"]["temp"], f'{config["args"]["hostName"]}.mask.bed')),
+        mask = temp(os.path.join(dir["out"]["temp"], config["args"]["hostName"] + ".mask.bed")),
         fa = config["outFasta"]
     params:
         fa = os.path.join(dir["dbs"]["hostBase"],config["args"]["hostName"],"masked_ref.fa"),

hecatomb/snakemake/workflow/combineOutputs.smk

@@ -29,7 +29,7 @@ for f in config["args"]["combineRuns"]:
     ]:
         if not is_non_zero_file(assemblyFile):
             sys.stderr.write(
-                f"No/missing assembly files for {f}, skipping assembly files.\n"
+                "No/missing assembly files for " + f + ", skipping assembly files.\n"
             )
             assemblyFiles = False
     with open(os.path.join(f, "results", "sampleSeqCounts.tsv"), "r") as t:
@@ -43,7 +43,7 @@ for f in config["args"]["combineRuns"]:
                     allDirSmplLen[f] = {}
                     allDirSmplLen[f][l[0]] = l[1]
             else:
-                sys.stderr.write(f"Ignoring duplicated sample {l[0]} in {f}\n")
+                sys.stderr.write("Ignoring duplicated sample " + l[0] + " in " + f + "\n")
 
 
 # hijack contig_mapping.smk for remaking the contigSeqTable
@@ -97,10 +97,10 @@ rule combineSampleSeqCounts:
     output:
         os.path.join(dir["out"]["results"], 'sampleSeqCounts.tsv')
     run:
-        with open(output[0],'w') as o:
+        with open(output[0], 'w') as o:
             for oDir in allDirSmplLen.keys():
                 for smpl in allDirSmplLen[oDir].keys():
-                    o.write(f'{smpl}\t{allDirSmplLen[oDir][smpl]}\n')
+                    o.write(smpl + "\t" + allDirSmplLen[oDir][smpl] + "\n")
 
 
 rule combineBigtables:

hecatomb/snakemake/workflow/envs/koverage.yaml

@@ -4,4 +4,4 @@ channels:
   - bioconda
   - defaults
 dependencies:
-    - koverage>=0.1.9
+    - koverage>=0.1.10

hecatomb/snakemake/workflow/envs/trimnami.yaml

@@ -4,4 +4,4 @@ channels:
   - bioconda
   - defaults
 dependencies:
-    - trimnami>=0.1.0
+    - trimnami>=0.1.2

hecatomb/snakemake/workflow/rules/annotation/read_annotation.smk

@@ -34,8 +34,8 @@ rule primary_aa_search:
         os.path.join(dir["env"], "mmseqs2.yaml")
     shell:
         "mmseqs easy-search {input.seqs} {input.db} {output} {params.alnRes} "
-            " {params.filtaa} {params.sensaa} "
-            " --threads {threads} --split-memory-limit {params.memsplit} &> {log} "
+            "{params.filtaa} {params.sensaa} "
+            "--threads {threads} --split-memory-limit {params.memsplit} &> {log}; "
 
 
 
@@ -105,7 +105,8 @@ rule secondary_aa_tophit_lineage:
         os.path.join(dir["env"], "seqkit.yaml")
     resources:
         time = resources["sml"]["time"],
-        mem = resources["ram"]["mem"]
+        mem_mb=resources["ram"]["mem"],
+        mem=str(resources["ram"]["mem"]) + "MB",
     params:
         taxonFormat = lambda wildcards: config["immutable"]["taxonkitReformat"]
     benchmark:
@@ -134,7 +135,8 @@ rule secondary_aa_refactor_finalize:
         os.path.join(dir["env"], "seqkit.yaml")
     resources:
         time=resources["sml"]["time"],
-        mem=resources["ram"]["mem"]
+        mem_mb=resources["ram"]["mem"],
+        mem=str(resources["ram"]["mem"]) + "MB",
     params:
         taxonFormat = lambda wildcards: config["immutable"]["taxonkitReformat"]
     benchmark:
@@ -164,7 +166,8 @@ rule secondary_aa_output_table:
         nonvir = os.path.join(dir["out"]["secondaryAA"], "AA_bigtable.nonviral.tsv")
     resources:
         time=resources["sml"]["time"],
-        mem=resources["ram"]["mem"]
+        mem_mb=resources["ram"]["mem"],
+        mem=str(resources["ram"]["mem"]) + "MB",
     benchmark:
         os.path.join(dir["out"]["bench"], "secondary_aa_generate_output_table.txt")
     log:
@@ -187,7 +190,8 @@ rule secondary_aa_parsing:
         unclass_seqs = os.path.join(dir["out"]["primaryAA"], "primary.aa.unclassified.fasta")
     resources:
         time=resources["sml"]["time"],
-        mem=resources["ram"]["mem"]
+        mem_mb=resources["ram"]["mem"],
+        mem=str(resources["ram"]["mem"]) + "MB",
     benchmark:
         os.path.join(dir["out"]["bench"], "secondary_aa_parsing.txt")
     log:
@@ -301,7 +305,8 @@ rule secondary_nt_lca_table:
         os.path.join(dir["out"]["stderr"], "secondary_nt_lca_table.log")
     resources:
         time = resources["sml"]["time"],
-        mem = resources["ram"]["mem"]
+        mem_mb=resources["ram"]["mem"],
+        mem=str(resources["ram"]["mem"]) + "MB",
     group:
         "secondary_nt_parsing"
     script:
@@ -319,7 +324,8 @@ rule secondary_nt_calc_lca:
         top_lineage = os.path.join(dir["out"]["secondaryNT"], "top_lineage.tsv"),
     resources:
         time=resources["sml"]["time"],
-        mem=resources["ram"]["mem"]
+        mem_mb=resources["ram"]["mem"],
+        mem=str(resources["ram"]["mem"]) + "MB",
     params:
         taxonFormat = lambda wildcards: config["immutable"]["taxonkitReformat"],
     conda:
@@ -357,7 +363,8 @@ rule secondary_nt_generate_output_table:
         nonvir = os.path.join(dir["out"]["secondaryNT"], "NT_bigtable.nonviral.tsv")
     resources:
         time=resources["sml"]["time"],
-        mem=resources["ram"]["mem"]
+        mem_mb=resources["ram"]["mem"],
+        mem=str(resources["ram"]["mem"]) + "MB",
     params:
         taxIdIgnore = config["mmseqs"]["taxIdIgnore"].split(),
         bigtableHeader = config["immutable"]["bigtableHeader"]
@@ -384,7 +391,8 @@ rule combine_aa_nt:
         os.path.join(dir["out"]["stderr"], "combine_AA_NT.log")
     resources:
         time=resources["sml"]["time"],
-        mem=resources["ram"]["mem"]
+        mem_mb=resources["ram"]["mem"],
+        mem=str(resources["ram"]["mem"]) + "MB",
     group:
         "secondary_nt_parsing"
     shell:

hecatomb/snakemake/workflow/rules/assembly/shortreads.smk

@@ -34,7 +34,7 @@ rule cross_assembly:
         os.path.join(dir["env"],"megahit.yaml")
     shell:
         "if [[ -d {params.mh_dir} ]]; "
-        "then; "
+        "then "
             "rm -rf {params.mh_dir}; "
         "fi; "
         "megahit "
@@ -90,7 +90,7 @@ rule megahit_sample_paired:
         "assembly"
     shell:
         "if [[ -d {params.mh_dir} ]]; "
-        "then; "
+        "then "
             "rm -rf {params.mh_dir}; "
         "fi; "
         "megahit "
@@ -132,7 +132,7 @@ rule megahit_sample_unpaired:
         "assembly"
     shell:
         "if [[ -d {params.mh_dir} ]]; "
-        "then; "
+        "then "
             "rm -rf {params.mh_dir}; "
         "fi; "
         "megahit "

hecatomb/snakemake/workflow/rules/preflight/targets.smk

@@ -105,7 +105,10 @@ targets["summary"] = [
 
 # BUILD ENV TARGETS
 targets["envs"] = []
-
+# Envs in the envs/ directory
 for filename in os.listdir(dir["env"]):
     if filename.endswith(".yaml") or filename.endswith(".yml"):
-        targets["envs"].append(os.path.join(dir["out"]["temp"], filename + ".done"))
+        targets["envs"].append(os.path.join(dir["out"]["temp"], filename + ".done"))
+# Envs build by sub-snaketools
+targets["envs"].append(os.path.join(dir["out"]["temp"], "subenvs.trimnami"))
+targets["envs"].append(os.path.join(dir["out"]["temp"], "subenvs.koverage"))

hecatomb/snakemake/workflow/rules/preflight/validate.smk

@@ -26,7 +26,7 @@ for f in config["dbs"]["files"] + config["dbtax"]["files"]:
     dbFile = os.path.join(dir["dbs"]["base"], f)
     if not os.path.isfile(dbFile):
         dbFail = True
-        sys.stderr.write(f"    ERROR: missing database file {dbFile}\n")
+        sys.stderr.write("    ERROR: missing database file " + dbFile + "\n")
 if dbFail:
     sys.stderr.write("\n"
         "    FATAL: One or more database files is missing.\n"
@@ -39,7 +39,7 @@ onstart:
     if os.path.isdir(dir["out"]["stderr"]):
         oldLogs = filter(re.compile(r'^(?!old_).*').match, os.listdir(dir["out"]["stderr"]))
         for logfile in oldLogs:
-            os.rename(os.path.join(dir["out"]["stderr"], logfile), os.path.join(dir["out"]["stderr"], f'old_{logfile}'))
+            os.rename(os.path.join(dir["out"]["stderr"], logfile), os.path.join(dir["out"]["stderr"], "old_" + logfile))
 
 # Success message
 onsuccess:
@@ -50,7 +50,7 @@ onsuccess:
 onerror:
     copy_log()
     sys.stderr.write('\n\n    FATAL: Hecatomb encountered an error.\n\n')
-    sys.stderr.write(f'Check the Hecatomb logs directory for command-related errors:\n\n{dir["out"]["stderr"]}\n\n')
+    sys.stderr.write("Check the Hecatomb logs directory for command-related errors:\n\n" + dir["out"]["stderr"] + "\n\n")
     if config["args"]["profile"]:
         sys.stderr.write(
             'Also check your scheduler logs for sheduler-related errors. Your profile determins where these are saved'

hecatomb/snakemake/workflow/rules/preprocessing/preprocessing.smk

@@ -7,6 +7,15 @@ rule buildEnv:
         "touch {output}"
 
 
+rule subsnake_build_envs:
+    output:
+        touch(os.path.join(dir["out"]["temp"], "subenvs.{env}"))
+    conda:
+        lambda wildcards: os.path.join(dir["env"], wildcards.env + ".yaml")
+    shell:
+        "{wildcards.env} test build_envs"
+
+
 rule trimnami_config:
     output:
         os.path.join(dir["out"]["temp"], "trimnami.config.yaml")
@@ -63,7 +72,7 @@ rule cluster_sequences:
         temp(os.path.join(dir["out"]["temp"],"{sample}_R1_all_seqs.fasta"))
     params:
         respath=lambda wildcards, output: os.path.split(output[0])[0],
-        tmppath=lambda wildcards, output: os.path.join(os.path.split(output[0])[0],f"{wildcards.sample}_TMP"),
+        tmppath=lambda wildcards, output: os.path.join(os.path.split(output[0])[0], wildcards.sample + "_TMP"),
         prefix="{sample}_R1",
         config=config["mmseqs"]["linclustParams"]
     benchmark:
@@ -78,9 +87,12 @@ rule cluster_sequences:
     conda:
         os.path.join(dir["env"],"mmseqs2.yaml")
     shell:
-        "mmseqs easy-linclust {input.fq} {params.respath}/{params.prefix} {params.tmppath} "
+        "mmseqs easy-linclust {input.fq} "
+            "{params.respath}/{params.prefix} "
+            "{params.tmppath} "
             "{params.config} "
-            "--threads {threads} &> {log}; "
+            "--threads {threads} "
+            "&> {log}; "
 
 
 rule create_individual_seqtables:

hecatomb/snakemake/workflow/rules/reports/summaries_optional.smk

@@ -34,9 +34,9 @@ rule trimmedHostRemovedCounts:
     run:
         out_counts = dict()
         for sample in params.samples["names"]:
-            out_counts[sample]["s2_host_removed_R1"] = file_len(os.path.join(dir["out"]["temp"], "p04", f"{sample}_R1.all.fastq"))
+            out_counts[sample]["s2_host_removed_R1"] = file_len(os.path.join(dir["out"]["temp"], "p04", sample + "_R1.all.fastq"))
             try:
-                out_counts[sample]["s2_host_removed_R2"] = file_len(os.path.join(dir["out"]["temp"],"p04", f"{sample}_R2.all.fastq"))
+                out_counts[sample]["s2_host_removed_R2"] = file_len(os.path.join(dir["out"]["temp"],"p04", sample + "_R2.all.fastq"))
             except FileNotFoundError:
                 pass
         with open(output[0],"w") as stream:
@@ -134,9 +134,9 @@ rule unclassifiedSeqs:
                         try:
                             classSeq[id]
                         except KeyError:
-                            out_fh.write(f">{id}\n{seq}\n")
+                            out_fh.write(">" + id + "\n" + seq + "\n")
                     else:
-                        sys.stderr.write(f"malformed {input.fa} file? expecting {line} to be fasta header, complain to Mike")
+                        sys.stderr.write("malformed " + input.fa + " file? expecting " + line + " to be fasta header, complain to Mike")
                         exit(1)
 
 

hecatomb/snakemake/workflow/scripts/aaBigtable.py

@@ -15,7 +15,7 @@
     for line in baltfh:
         l = line.strip().split("\t")
         if len(l) == 3:
-            balt[l[0]] = f"{l[1]}\t{l[2]}"
+            balt[l[0]] = l[1] + "\t" + l[2]
 
 logging.debug("Reading in Taxon info for LCA seqs")
 lcaLin = {}

hecatomb/snakemake/workflow/scripts/aaPrimaryParse.py

@@ -22,10 +22,10 @@
                 id = line.strip().replace(">", "")
                 seq = inFa.readline().strip()
                 if id in topHit:
-                    outClass.write(f">{id}\n{seq}\n")
+                    outClass.write(">" + id + "\n" + seq + "\n")
             else:
                 sys.stderr.write(
-                    f"malformed {snakemake.input.seqs} file, or something, complain to Mike."
+                    "malformed " + snakemake.input.seqs + " file, or something, complain to Mike."
                 )
                 exit(1)
 

hecatomb/snakemake/workflow/scripts/aaSecondaryParse.py

@@ -22,10 +22,10 @@
                 id = line.strip().replace(">", "")
                 seq = inFa.readline().strip()
                 if not id in virSeqs:
-                    outFa.write(f">{id}\n{seq}\n")
+                    outFa.write(">" + id + "\n" + seq + "\n")
             else:
                 logging.error(
-                    f"malformed {snakemake.input.seqs} file, or something, complain to Mike"
+                    "malformed " + snakemake.input.seqs + " file, or something, complain to Mike"
                 )
                 exit(1)
 

hecatomb/snakemake/workflow/scripts/contigCountTable.py

@@ -8,7 +8,7 @@
 logging.basicConfig(filename=snakemake.log[0], filemode="w", level=logging.DEBUG)
 
 covStat = {}
-logging.debug(f"Reading {snakemake.input.covstats}")
+logging.debug("Reading " + snakemake.input.covstats)
 with open(snakemake.input.covstats, "r") as f:
     for line in f:
         if not line.startswith("#"):
@@ -18,15 +18,15 @@
             covStat[l[0]] = [l[1], l[3], l[4], l[5], l[9]]
 
 total = 0
-logging.debug(f"Reading {snakemake.input.rpkm}")
+logging.debug("Reading " + snakemake.input.rpkm)
 with open(snakemake.input.rpkm, "r") as f:
     for line in f:
         if not line.startswith("#"):
             l = line.split("\t")
             rpk = int(l[2]) / (int(l[1]) / 1000)  # RPK
             total += rpk / 1000000  # size factor per million
 
-logging.debug(f"Writing to {snakemake.output.count_tbl}")
+logging.debug("Writing to " + snakemake.output.count_tbl)
 with open(snakemake.output.count_tbl, "w") as o:
     o.write(
         "\t".join(

hecatomb/snakemake/workflow/scripts/contigKronaText.py

@@ -23,5 +23,5 @@
 logging.debug("Sorting and writing contig taxon info")
 outFH = open(snakemake.output[0], "w")
 for k in sorted(counts.keys()):
-    outFH.write(f"{counts[k]}\t{k}\n")
+    outFH.write(str(counts[k]) + "\t" + k + "\n")
 outFH.close()

hecatomb/snakemake/workflow/scripts/contigReadTaxon.py

@@ -46,7 +46,7 @@
             taxOut = "\t".join(
                 ([c[1], str((int(c[1]) / smplCounts[c[0]]) * 1000000)] + ["NA"] * 11)
             )
-        outFH.write(f"{infOut}\t{taxOut}\n")
+        outFH.write(infOut + "\t" + taxOut + "\n")
     bam.close()
 
 logging.debug("Done")

hecatomb/snakemake/workflow/scripts/kronaText.py

@@ -22,5 +22,5 @@
 logging.debug("Sorting, counting, and writing tax assignments")
 outFH = open(snakemake.output[0], "w")
 for k in sorted(counts.keys()):
-    outFH.write(f"{counts[k]}\t{k}\n")
+    outFH.write(str(counts[k]) + "\t" + k + "\n")
 outFH.close()