From c7444ee72d66b5d7b9b75f955731d1a32ac36429 Mon Sep 17 00:00:00 2001 From: jrobinso <933148+jrobinso@users.noreply.github.com> Date: Sat, 21 Oct 2023 21:44:46 -0700 Subject: [PATCH] fix unit tests --- js/bam/bamReader.js | 2 +- js/bam/bamReaderNonIndexed.js | 85 +++++++++++++++++++----------- js/bigwig/bwReader.js | 8 +-- js/genome/genome.js | 4 +- test/{ => old}/testHtsgetReader.js | 8 +-- test/testAlignmentUtils.js | 63 ++++++++++++---------- test/testBAM.js | 28 ++++++---- test/testBED.js | 2 +- test/testChromAlias.js | 61 +++++++++------------ test/testGenome.js | 39 +++++++------- test/testGenomeUtils.js | 18 ------- test/testSearch.js | 15 +----- test/testSeg.js | 2 +- test/utils/Genome.js | 44 ++++------------ 14 files changed, 178 insertions(+), 201 deletions(-) rename test/{ => old}/testHtsgetReader.js (94%) diff --git a/js/bam/bamReader.js b/js/bam/bamReader.js index dd74f5937..a7be50b96 100644 --- a/js/bam/bamReader.js +++ b/js/bam/bamReader.js @@ -44,7 +44,7 @@ class BamReader { for (let c of chunks) { const ba = await this._blockLoader.getData(c.minv, c.maxv) - const done = BamUtils.decodeBamRecords(ba, c.minv.offset, alignmentContainer, this.indexToChr, chrId, bpStart, bpEnd, this.filter) + const done = BamUtils.decodeBamRecords(ba, c.minv.offset, alignmentContainer, this.header.chrNames, chrId, bpStart, bpEnd, this.filter) if (done) { break } diff --git a/js/bam/bamReaderNonIndexed.js b/js/bam/bamReaderNonIndexed.js index f210659e9..1819af123 100644 --- a/js/bam/bamReaderNonIndexed.js +++ b/js/bam/bamReaderNonIndexed.js @@ -37,6 +37,8 @@ import {buildOptions, isDataURL} from "../util/igvUtils.js" */ class BamReaderNonIndexed { + chrAliasTable = new Map() + constructor(config, genome) { this.config = config this.genome = genome @@ -45,52 +47,75 @@ class BamReaderNonIndexed { BamUtils.setReaderDefaults(this, config) } - // Return an alignment container + /** + * + * @param chr + * @param bpStart + * @param bpEnd + * @returns {Promise} + */ async readAlignments(chr, bpStart, bpEnd) { - if (this.alignmentCache) { - const header = this.header - const queryChr = header.chrAliasTable.hasOwnProperty(chr) ? header.chrAliasTable[chr] : chr - const qAlignments = this.alignmentCache.queryFeatures(queryChr, bpStart, bpEnd) - const alignmentContainer = new AlignmentContainer(chr, bpStart, bpEnd, this.config) - for (let a of qAlignments) { - alignmentContainer.push(a) - } - alignmentContainer.finish() - return alignmentContainer - - } else { + if (!this.alignmentCache) { + // For a non-indexed BAM file all alignments are read at once and cached. + let unc if (this.isDataUri) { const data = decodeDataURI(this.bamPath) - const unc = BGZip.unbgzf(data.buffer) - this.parseAlignments(unc) - return this.fetchAlignments(chr, bpStart, bpEnd) + unc = BGZip.unbgzf(data.buffer) } else { const arrayBuffer = await igvxhr.loadArrayBuffer(this.bamPath, buildOptions(this.config)) - const unc = BGZip.unbgzf(arrayBuffer) - this.parseAlignments(unc) - return this.fetchAlignments(chr, bpStart, bpEnd) + unc = BGZip.unbgzf(arrayBuffer) } + this.alignmentCache = this.#parseAlignments(unc) } + const queryChr = await this.#getQueryChr(chr) + const qAlignments = this.alignmentCache.queryFeatures(queryChr, bpStart, bpEnd) + const alignmentContainer = new AlignmentContainer(chr, bpStart, bpEnd, this.config) + for (let a of qAlignments) { + alignmentContainer.push(a) + } + alignmentContainer.finish() + return alignmentContainer } - parseAlignments(data) { + #parseAlignments(data) { const alignments = [] this.header = BamUtils.decodeBamHeader(data) - BamUtils. decodeBamRecords(data, this.header.size, alignments, this.header.chrNames) - this.alignmentCache = new FeatureCache(alignments, this.genome) + BamUtils.decodeBamRecords(data, this.header.size, alignments, this.header.chrNames) + return new FeatureCache(alignments, this.genome) } - fetchAlignments(chr, bpStart, bpEnd) { - const queryChr = this.header.chrAliasTable.hasOwnProperty(chr) ? this.header.chrAliasTable[chr] : chr - const features = this.alignmentCache.queryFeatures(queryChr, bpStart, bpEnd) - const alignmentContainer = new AlignmentContainer(chr, bpStart, bpEnd, this.config) - for (let feature of features) { - alignmentContainer.push(feature) + async #getQueryChr(chr) { + + const ownNames = new Set(this.header.chrNames) + if (ownNames.has(chr)) { + return chr } - alignmentContainer.finish() - return alignmentContainer + + if (this.chrAliasTable.has(chr)) { + return this.chrAliasTable.get(chr) + } + + // Try alias + + if (this.genome) { + const aliasRecord = await this.genome.getAliasRecord(chr) + let alias + if (aliasRecord) { + const aliases = Object.keys(aliasRecord) + .filter(k => k !== "start" && k !== "end") + .map(k => aliasRecord[k]) + .filter(a => ownNames.has(a)) + if (aliases.length > 0) { + alias = aliases[0] + } + } + this.chrAliasTable.set(chr, alias) // alias may be undefined => no alias exists. Setting prevents repeated attempts + return alias + } + + return chr } } diff --git a/js/bigwig/bwReader.js b/js/bigwig/bwReader.js index 128881eb0..baa3b01b0 100644 --- a/js/bigwig/bwReader.js +++ b/js/bigwig/bwReader.js @@ -251,10 +251,10 @@ class BWReader { // Use a trix index if we have one to map entered term to indexed value in bb file if (this._trix) { - term = term.toLowerCase() - const trixResults = await this._trix.search(term) - if (trixResults && trixResults.has(term)) { // <= exact matches only for now - term = trixResults.get(term)[0] + const termLower = term.toLowerCase() + const trixResults = await this._trix.search(termLower) + if (trixResults && trixResults.has(termLower)) { // <= exact matches only for now + term = trixResults.get(termLower)[0] } } diff --git a/js/genome/genome.js b/js/genome/genome.js index 8c3d83896..e5a2622b1 100644 --- a/js/genome/genome.js +++ b/js/genome/genome.js @@ -64,7 +64,7 @@ class Genome { this.cytobandSource = new CytobandFile(config.cytobandURL, Object.assign({}, config)) } - if (false !== config.wholeGenomeView) { + if (false !== config.wholeGenomeView && this.chromosomes.size > 0) { // Set chromosome order for WG view and chromosome pulldown. If chromosome order is not specified sort if (config.chromosomeOrder) { if (Array.isArray(config.chromosomeOrder)) { @@ -73,7 +73,7 @@ class Genome { this.wgChromosomeNames = config.chromosomeOrder.split(',').map(nm => nm.trim()) } } else { - this.wgChromosomeNames = trimSmallChromosomes(chromosomes) + this.wgChromosomeNames = trimSmallChromosomes(this.chromosomes) } } diff --git a/test/testHtsgetReader.js b/test/old/testHtsgetReader.js similarity index 94% rename from test/testHtsgetReader.js rename to test/old/testHtsgetReader.js index 48ae94422..428fdd70e 100644 --- a/test/testHtsgetReader.js +++ b/test/old/testHtsgetReader.js @@ -1,8 +1,8 @@ -import "./utils/mockObjects.js" +import "../utils/mockObjects.js" import {assert} from 'chai' -import HtsgetBamReader from "../js/htsget/htsgetBamReader.js" -import HtsgetVariantReader from "../js/htsget/htsgetVariantReader.js" -import Browser from "../js/browser.js" +import HtsgetBamReader from "../../js/htsget/htsgetBamReader.js" +import HtsgetVariantReader from "../../js/htsget/htsgetVariantReader.js" +import Browser from "../../js/browser.js" // Mock genome with "1,2,3..." name convention diff --git a/test/testAlignmentUtils.js b/test/testAlignmentUtils.js index aef5a04df..0015d7405 100755 --- a/test/testAlignmentUtils.js +++ b/test/testAlignmentUtils.js @@ -8,46 +8,53 @@ suite("testAlignmentUtils", function () { test("Alignment packing", async function () { this.timeout(10000) - const chr = 'chr22' - const start = 24375132 - const end = 24385311 - const bamReader = new BamReaderNonIndexed({ - type: 'bam', - url: 'test/data/bam/gstt1_sample.bam', - indexed: false - }) + await _testPacking('chr22') + await _testPacking('22') // Test aliasing - const alignmentContainer = await bamReader.readAlignments(chr, start, end) - const rows = packAlignmentRows(alignmentContainer.alignments, start, end, false) + async function _testPacking(chr) { + const start = 24375132 + const end = 24385311 - let count = 0 - for (let r of rows) count += r.alignments.length + const bamReader = new BamReaderNonIndexed({ + type: 'bam', + url: 'test/data/bam/gstt1_sample.bam', + indexed: false + }) - // All alignments are packed - assert.equal(alignmentContainer.alignments.length, count) + const alignmentContainer = await bamReader.readAlignments(chr, start, end) - // No duplicates - const seen = new Set() - for (let r of rows) { - for (let a of r.alignments) { - if (seen.has(a)) { - assert.fail(a, undefined, 'Alignment seen twice') + const rows = packAlignmentRows(alignmentContainer.alignments, start, end, false) + + let count = 0 + for (let r of rows) count += r.alignments.length + + // All alignments are packed + assert.equal(alignmentContainer.alignments.length, count) + + // No duplicates + const seen = new Set() + for (let r of rows) { + for (let a of r.alignments) { + if (seen.has(a)) { + assert.fail(a, undefined, 'Alignment seen twice') + } + seen.add(a) } - seen.add(a) } - } - // No alignment overlaps - for (let r of rows) { - let lastEnd = -1 - for (let a of r.alignments) { - assert.isAtLeast(a.start, lastEnd, 'Alignment start is < last end') - lastEnd = a.end + // No alignment overlaps + for (let r of rows) { + let lastEnd = -1 + for (let a of r.alignments) { + assert.isAtLeast(a.start, lastEnd, 'Alignment start is < last end') + lastEnd = a.end + } } } + }) diff --git a/test/testBAM.js b/test/testBAM.js index f0e440e0c..4895e4f07 100755 --- a/test/testBAM.js +++ b/test/testBAM.js @@ -2,6 +2,10 @@ import "./utils/mockObjects.js" import BamReader from "../js/bam/bamReader.js" import {assert} from 'chai' import BamReaderNonIndexed from "../js/bam/bamReaderNonIndexed.js" +import {createGenome} from "./utils/Genome.js" + + +const genome = createGenome() suite("testBAM", function () { @@ -12,10 +16,11 @@ suite("testBAM", function () { const end = 155160000 const bamReader = new BamReader({ - type: 'bam', - url: 'test/data/bam/na12889.bam', - indexURL: 'test/data/bam/na12889.bam.csi' - }) + type: 'bam', + url: 'test/data/bam/na12889.bam', + indexURL: 'test/data/bam/na12889.bam.csi' + }, + genome) const alignmentContainer = await bamReader.readAlignments(chr, start, end) validate(assert, alignmentContainer) @@ -23,17 +28,20 @@ suite("testBAM", function () { test("BAM alignments - non indexed", async function () { - const chr = 'chr1' const start = 155140000 const end = 155160000 const bamReader = new BamReaderNonIndexed({ - type: 'bam', - url: 'test/data/bam/na12889.bam', - indexed: false - }) + type: 'bam', + url: 'test/data/bam/na12889.bam', + indexed: false + }, + genome) - const alignmentContainer = await bamReader.readAlignments(chr, start, end) + let alignmentContainer = await bamReader.readAlignments("chr1", start, end) + validate(assert, alignmentContainer) + + alignmentContainer = await bamReader.readAlignments("1", start, end) validate(assert, alignmentContainer) }) diff --git a/test/testBED.js b/test/testBED.js index 073d9a0bb..500307b28 100644 --- a/test/testBED.js +++ b/test/testBED.js @@ -5,7 +5,7 @@ import {assert} from 'chai' import {createGenome} from "./utils/Genome.js" const genome = createGenome() -import GenomeUtils from "../js/genome/genomeUtils.js" +import Genome from "../js/genome/genome.js" suite("testBed", function () { diff --git a/test/testChromAlias.js b/test/testChromAlias.js index d63f09556..d14bde29b 100644 --- a/test/testChromAlias.js +++ b/test/testChromAlias.js @@ -6,21 +6,35 @@ import {assert} from "chai" suite("chromAlias", function () { + const genome = { + chromosomes: new Map([ + ["NC_007194.1", {name: "NC_007194.1", bpLength: 1}], + ]) + } + + /** + * Test a UCSC style chrom alias flat file + * + * # refseq assembly genbank ncbi ucsc + * NC_007194.1 1 CM000169.1 1 chr1 + * NC_007195.1 2 CM000170.1 2 chr2 + * NC_007196.1 3 CM000171.1 3 chr3 + * NC_007197.1 4 CM000172.1 4 chr4 + * NC_007198.1 5 CM000173.1 5 chr5 + * NC_007199.1 6 CM000174.1 6 chr6 + * NC_007200.1 7 CM000175.1 7 chr7 + * NC_007201.1 8 CM000176.1 8 chr8 + */ test("test chromAlias.txt", async function () { - const url = "test/data/genomes/t2t.chromAlias.txt" - - const chromosomeNames = ["CP068254.1", "CP068255.2", "CP068256.2", "CP068257.2", "CP068258.2", "CP068259.2", - "CP068260.2", "CP068261.2", "CP068262.2", "CP068263.2", "CP068264.2", "CP068265.2", "CP068266.2", - "CP068267.2", "CP068268.2", "CP068269.2", "CP068270.2", "CP068271.2", "CP068272.2", "CP068273.2", - "CP068274.2", "CP068275.2", "CP068276.2", "CP068277.2", "CP086569.2" - ] - - const chromAlias = new ChromAliasFile(url, {}) - - await chromAlias.init(chromosomeNames) - + const url = "test/data/genomes/GCF_000002655.1.chromAlias.txt" + const chromAlias = new ChromAliasFile(url, {}, genome) + const chromAliasRecord = await chromAlias.search("1") + assert.equal(chromAliasRecord.chr, "NC_007194.1") + assert.equal(chromAliasRecord.genbank, "CM000169.1") + assert.equal(chromAliasRecord.ncbi, "1") + assert.equal(chromAliasRecord.ucsc, "chr1") }) test("test chromalias bb extra index search", async function () { @@ -32,28 +46,6 @@ suite("chromAlias", function () { const bbReader = new BWReader(config) - - // There are 5 extra indexes, 1 for each alias - const ncbiName = "3" - const f1 = await bbReader.search(ncbiName) - assert.equal(ncbiName, f1.ncbi) - - const ucscName = "chr2" - const f2 = await bbReader.search(ucscName) - assert.equal(ucscName, f2.ucsc) - }) - - test("test chromalias bb remote", async function () { - this.timeout(200000) - const config = { - url: "https://hgdownload.soe.ucsc.edu/hubs/GCA/009/914/755/GCA_009914755.4/GCA_009914755.4.chromAlias.bb", - format: "bigbed" - } - - const bbReader = new BWReader(config) - - const features = await bbReader.readFeatures("chr1") - // There are 5 extra indexes, 1 for each alias const ncbiName = "3" const f1 = await bbReader.search(ncbiName) @@ -63,5 +55,4 @@ suite("chromAlias", function () { const f2 = await bbReader.search(ucscName) assert.equal(ucscName, f2.ucsc) }) - // chromAliasBbURL: "https://hgdownload.soe.ucsc.edu/hubs/GCA/009/914/755/GCA_009914755.4/GCA_009914755.4.chromAlias.bb", }) diff --git a/test/testGenome.js b/test/testGenome.js index bd7ce9ab8..70b1970fd 100644 --- a/test/testGenome.js +++ b/test/testGenome.js @@ -1,5 +1,5 @@ import "./utils/mockObjects.js" -import GenomeUtils from "../js/genome/genomeUtils.js" +import Genome from "../js/genome/genome.js" import {assert} from 'chai' import CytobandFileBB from "../js/genome/cytobandFileBB.js" @@ -14,7 +14,7 @@ suite("testGenome", function () { id: "hg19", fastaURL: "https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq/1kg_v37/human_g1k_v37_decoy.fasta", cytobandURL: "https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq/b37/b37_cytoband.txt", - wholeGenomeView: false + wholeGenomeView: true } const genome = await Genome.loadGenome(reference) @@ -24,25 +24,22 @@ suite("testGenome", function () { }) - /** - * test parsing a cytoband BB file from the T2T hub - * Example feature - * { - * "chr": "chr1", - * "start": 0, - * "end": 1735965, - * "name": "p36.33", - * "gieStain": "gneg" - * } - */ - test("test cytoband bigbed", async function () { - - const url = "test/data/bb/cytoBandMapped.bb" - const src = new CytobandFileBB(url) - - const cytobands = await src.getCytobands("chr1") - const last = cytobands[cytobands.length-1]; - assert.equal(248387328, last.end) + test("2bit genome with chromSizes", async function() { + + this.timeout(400000) + + const reference = { + id: "GCF_016699485.2", + format: "2bit", + twoBitURL: "https://hgdownload.gi.ucsc.edu/hubs//GCA/011/100/615/GCA_011100615.1/GCA_011100615.1.2bit", + chromSizes: "https://hgdownload.gi.ucsc.edu/hubs//GCA/011/100/615/GCA_011100615.1/GCA_011100615.1.chrom.sizes.txt" + } + + const genome = await Genome.loadGenome(reference) + assert.ok(genome.chromosomes.size > 0) + assert.ok(genome.chromosomeNames.length > 0) }) + + }) diff --git a/test/testGenomeUtils.js b/test/testGenomeUtils.js index ed61ef15f..9e3e1884f 100644 --- a/test/testGenomeUtils.js +++ b/test/testGenomeUtils.js @@ -118,22 +118,4 @@ suite("testGenomeUtils", function () { }) - test("2bit genome", async function() { - - this.timeout(400000) - - const reference = { - id: "GCF_016699485.2", - format: "2bit", - twoBitURL: "https://hgdownload.gi.ucsc.edu/hubs//GCA/011/100/615/GCA_011100615.1/GCA_011100615.1.2bit", - chromSizes: "https://hgdownload.gi.ucsc.edu/hubs//GCA/011/100/615/GCA_011100615.1/GCA_011100615.1.chrom.sizes.txt" - } - - - const genome = await Genome.loadGenome(reference) - - assert.ok(genome) - - }) - }) \ No newline at end of file diff --git a/test/testSearch.js b/test/testSearch.js index 2c5e8d176..227ca728f 100644 --- a/test/testSearch.js +++ b/test/testSearch.js @@ -39,6 +39,8 @@ suite("testSearch", function () { snpField: "snp" }, + isSoftclipped: () => false + } test("locus strings", function () { @@ -48,12 +50,6 @@ suite("testSearch", function () { assert.equal(locus1.start, 99) assert.equal(locus1.end, 200) - // Chr name alias - const s2 = "1:100-200" - const locus2 = parseLocusString(s2) - assert.equal(locus2.chr, "1") - assert.equal(locus2.start, 99) - assert.equal(locus2.end, 200) // Single base const s3 = "1:100" @@ -62,9 +58,6 @@ suite("testSearch", function () { assert.equal(locus3.start, 79) assert.equal(locus3.end, 120) - const s4 = "egfr" - const locus4 = parseLocusString(s4) - assert.equal(locus4, undefined) }) test("webservice", async function () { @@ -77,7 +70,6 @@ suite("testSearch", function () { assert.equal(locus.chr, "chr8") assert.equal(locus.start, 127735432) assert.equal(locus.end, 127742951) - assert.equal(locus.locusSearchString, gene) }) test("search (main function)", async function () { @@ -94,19 +86,16 @@ suite("testSearch", function () { assert.equal(locus1.chr, "chr1") assert.equal(locus1.start, 99) assert.equal(locus1.end, 200) - assert.equal(locus1.locusSearchString, s1) const locus2 = results[1] assert.equal(locus2.chr, "chr8") assert.equal(locus2.start, 127735432) assert.equal(locus2.end, 127742951) - assert.equal(locus2.locusSearchString, s2) const locus3 = results[2] assert.equal(locus3.chr, "chr1") assert.equal(locus3.start, 155185822) assert.equal(locus3.end, 155192915) - assert.equal(locus3.locusSearchString, s3) }) test("search name with spaces from bed file", async function () { diff --git a/test/testSeg.js b/test/testSeg.js index a347ac48b..f4f3f88d1 100644 --- a/test/testSeg.js +++ b/test/testSeg.js @@ -1,6 +1,6 @@ import "./utils/mockObjects.js" import FeatureSource from "../js/feature/featureSource.js" -import GenomeUtils from "../js/genome/genomeUtils.js" +import Genome from "../js/genome/genome.js" import {assert} from 'chai' import {createGenome} from "./utils/Genome.js" diff --git a/test/utils/Genome.js b/test/utils/Genome.js index feb5decf2..4996f5927 100644 --- a/test/utils/Genome.js +++ b/test/utils/Genome.js @@ -40,7 +40,7 @@ function createGenome() { }, loadChromosome: async function (chr) { - return this.getChromosomechr + return this.getChromosome(chr) }, getChromosome: function (chr) { @@ -50,43 +50,21 @@ function createGenome() { return bpLength ? {name, bpLength} : undefined }, - wgChromosomeNames: Object.keys(sizes), + getAliasRecord : async function(chr) { + const chromosome = this.getChromosome(chr) - featureDB: new Map(), + return { + chr: chromosome.name, + start: 0, + end: chromosome.bpLength, + ncbi: chromosome.name.substring(3) + } - addFeaturesToDB: function (featureList, config) { + }, - const insertFeature = (name, feature) => { - const current = this.featureDB.get(name) - if (current) { - feature = (feature.end - feature.start) > (current.end - current.start) ? feature : current - } - this.featureDB.set(name, feature) - } + wgChromosomeNames: Object.keys(sizes), - for (let feature of featureList) { - if (feature.name) { - insertFeature(feature.name.toUpperCase(), feature) - } - if (feature.gene && feature.gene.name) { - insertFeature(feature.gene.name.toUpperCase(), feature) - } - - if (config.searchableFields) { - for (let f of config.searchableFields) { - const value = feature.getAttributeValue(f) - if (value) { - if (value.indexOf(" ") > 0) { - insertFeature(value.replaceAll(" ", "+").toUpperCase(), feature) - } else { - insertFeature(value.toUpperCase(), feature) - } - } - } - } - } - } } }