Merge pull request #74 from GMOD/add_abortable_promise_cache

Add abortable promise cache for recovering from index loading error

package.json

@@ -44,6 +44,7 @@
   "dependencies": {
     "@babel/runtime-corejs3": "^7.4.5",
     "@gmod/binary-parser": "^1.3.5",
+    "abortable-promise-cache": "^1.2.0",
     "buffer-crc32": "^0.2.13",
     "cross-fetch": "^3.0.0",
     "es6-promisify": "^6.0.1",

src/craiIndex.js

@@ -1,3 +1,6 @@
+const AbortablePromiseCache = require('abortable-promise-cache').default
+const QuickLRU = require('quick-lru')
+
 const { promisify } = require('es6-promisify')
 const zlib = require('zlib')
 
@@ -56,8 +59,11 @@ class CraiIndex {
    */
   constructor(args) {
     const filehandle = open(args.url, args.path, args.filehandle)
+    this._parseCache = new AbortablePromiseCache({
+      cache: new QuickLRU({ maxSize: 1 }),
+      fill: (data, signal) => this.parseIndex({ signal }),
+    })
     this.readFile = filehandle.readFile.bind(filehandle)
-    this.index = this.parseIndex()
   }
 
   parseIndex() {
@@ -122,8 +128,8 @@ class CraiIndex {
       })
   }
 
-  getIndex() {
-    return this.index
+  getIndex(opts = {}) {
+    return this._parseCache.get('index', null, opts.signal)
   }
 
   /**
@@ -132,7 +138,7 @@ class CraiIndex {
    * the given reference sequence ID, false otherwise
    */
   async hasDataForReferenceSequence(seqId) {
-    return !!(await this.index)[seqId]
+    return !!(await this.getIndex())[seqId]
   }
 
   /**
@@ -147,7 +153,7 @@ class CraiIndex {
    * `{start, span, containerStart, sliceStart, sliceBytes }`
    */
   async getEntriesForRange(seqId, queryStart, queryEnd) {
-    const seqEntries = (await this.index)[seqId]
+    const seqEntries = (await this.getIndex())[seqId]
     if (!seqEntries) return []
     const len = seqEntries.length
 

test/crai.test.js

@@ -8,7 +8,7 @@ describe('.crai reader', () => {
   it('can read xx#unsorted.tmp.cram.crai', async () => {
     const filehandle = testDataFile('xx#unsorted.tmp.cram.crai')
     const index = new CraiIndex({ filehandle })
-    let data = await index.getIndex()
+    const data = await index.getIndex()
     expect(data).to.deep.equal({
       '0': [
         {
@@ -51,27 +51,6 @@ describe('.crai reader', () => {
     expect(await index.getEntriesForRange(0, 0, 20)).to.deep.equal(data[0])
     expect(await index.getEntriesForRange(0, 1, 21)).to.deep.equal(data[0])
     expect(await index.getEntriesForRange(1, 0, 20)).to.deep.equal(data[1])
-
-    data = {
-      '0': [
-        { start: 1, span: 1 },
-        { start: 100, span: 1 },
-        { start: 101, span: 1 },
-        { start: 102, span: 1 },
-        { start: 300, span: 1 },
-        { start: 400, span: 10 },
-        { start: 404, span: 1 },
-        { start: 410, span: 1 },
-      ],
-    }
-    index.index = Promise.resolve(data)
-    expect(await index.getEntriesForRange(0, 1, 2)).to.deep.equal([data[0][0]])
-    expect(await index.getEntriesForRange(0, 1, 100)).to.deep.equal([
-      data[0][0],
-    ])
-    expect(await index.getEntriesForRange(0, 100, 101)).to.deep.equal([
-      data[0][1],
-    ])
   })
 
   it('throws an error if you try to read cramQueryWithCRAI.cram as a .crai', () => {

test/retry.test.js

@@ -10,6 +10,7 @@ describe('retry nonexist file', () => {
   it('file moves', async () => {
     let cram
     mock({})
+    let exception = 0
     try {
       cram = new IndexedCramFile({
         cramFilehandle: new LocalFile('./test/data/ce#tag_padded.tmp.cram'),
@@ -21,11 +22,35 @@ describe('retry nonexist file', () => {
       await cram.cram.getSamHeader()
     } catch (e) {
       /* console.error('initial error', e) */
+      exception = 1
     }
+    expect(exception).to.equal(1)
 
     mock.restore()
     const ret = await cram.cram.getSamHeader()
 
     expect(ret[0].tag).to.equal('HD')
   })
+  it('index moves', async () => {
+    let cram
+    mock({})
+    let exception = 0
+    try {
+      cram = new IndexedCramFile({
+        cramFilehandle: new LocalFile('./test/data/ce#tag_padded.tmp.cram'),
+        index: new CraiIndex({
+          filehandle: new LocalFile('./test/data/ce#tag_padded.tmp.cram.crai'),
+        }),
+      })
+
+      await cram.getRecordsForRange(0, 2, 200)
+    } catch (e) {
+      exception = 1
+    }
+    expect(exception).to.equal(1)
+
+    mock.restore()
+    const features = await cram.getRecordsForRange(0, 2, 200)
+    expect(features.length).to.equal(8)
+  })
 })

yarn.lock

@@ -939,6 +939,13 @@
     core-js-pure "^3.0.0"
     regenerator-runtime "^0.13.2"
 
+"@babel/runtime@^7.0.0":
+  version "7.10.4"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.10.4.tgz#a6724f1a6b8d2f6ea5236dbfe58c7d7ea9c5eb99"
+  integrity sha512-UpTN5yUJr9b4EX2CnGNWIvER7Ab83ibv0pcvvHc4UOdrBI5jb8bj+32cCwPX6xu0mt2daFNjYhoi+X7beH0RSw==
+  dependencies:
+    regenerator-runtime "^0.13.4"
+
 "@babel/template@^7.1.0", "@babel/template@^7.4.0", "@babel/template@^7.4.4", "@babel/template@^7.6.0":
   version "7.6.0"
   resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.6.0.tgz#7f0159c7f5012230dad64cca42ec9bdb5c9536e6"
@@ -1153,6 +1160,19 @@ abbrev@1:
   resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
   integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==
 
+abortable-promise-cache@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/abortable-promise-cache/-/abortable-promise-cache-1.2.0.tgz#cb48bc5583654383d23709cf5d0d5b56d9c8146f"
+  integrity sha512-kKEN5e569SV8br+gqDwMmpglTD2wlJshaBfU97o+5OCYMUQx5terlWTfgb0a0uq1bvt82SrwzEium5gR0n2frQ==
+  dependencies:
+    "@babel/runtime" "^7.0.0"
+    abortcontroller-polyfill "^1.2.9"
+
+abortcontroller-polyfill@^1.2.9:
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/abortcontroller-polyfill/-/abortcontroller-polyfill-1.4.0.tgz#0d5eb58e522a461774af8086414f68e1dda7a6c4"
+  integrity sha512-3ZFfCRfDzx3GFjO6RAkYx81lPGpUS20ISxux9gLxuKnqafNcFQo59+IoZqpO2WvQlyc287B62HDnDdNYRmlvWA==
+
 acorn-jsx@^5.0.0:
   version "5.0.2"
   resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.0.2.tgz#84b68ea44b373c4f8686023a551f61a21b7c4a4f"
@@ -5433,6 +5453,11 @@ regenerator-runtime@^0.13.2:
   resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.3.tgz#7cf6a77d8f5c6f60eb73c5fc1955b2ceb01e6bf5"
   integrity sha512-naKIZz2GQ8JWh///G7L3X6LaQUAMp2lvb1rvwwsURe/VXwD6VMfr+/1NuNw3ag8v2kY1aQ/go5SNn79O9JU7yw==
 
+regenerator-runtime@^0.13.4:
+  version "0.13.5"
+  resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.5.tgz#d878a1d094b4306d10b9096484b33ebd55e26697"
+  integrity sha512-ZS5w8CpKFinUzOwW3c83oPeVXoNsrLsaCoLtJvAClH135j/R77RuymhiSErhm2lKcwSCIpmvIWSbDkIfAqKQlA==
+
 regenerator-transform@^0.14.0:
   version "0.14.1"
   resolved "https://registry.yarnpkg.com/regenerator-transform/-/regenerator-transform-0.14.1.tgz#3b2fce4e1ab7732c08f665dfdb314749c7ddd2fb"