Export cachedAsyncBuffer

demo/workers/parquetWorkerClient.ts

@@ -1,6 +1,6 @@
+import { cachedAsyncBuffer } from '../../src/asyncBuffer.js'
 import type { AsyncBuffer, FileMetaData } from '../../src/hyparquet.js'
 import { asyncBufferFromUrl } from '../../src/utils.js'
-import { cachedAsyncBuffer } from '../asyncBuffer.js'
 
 // Serializable constructors for AsyncBuffers
 interface AsyncBufferFromFile {

src/asyncBuffer.js

@@ -0,0 +1,54 @@
+
+/**
+ * Returns a cached layer on top of an AsyncBuffer. For caching slices of a file
+ * that are read multiple times, possibly over a network.
+ *
+ * @typedef {import('./types.js').AsyncBuffer} AsyncBuffer
+ * @param {AsyncBuffer} file file-like object to cache
+ * @returns {AsyncBuffer} cached file-like object
+ */
+export function cachedAsyncBuffer({ byteLength, slice }) {
+  const cache = new Map()
+  return {
+    byteLength,
+    /**
+     * @param {number} start
+     * @param {number} [end]
+     * @returns {import('./types.js').Awaitable<ArrayBuffer>}
+     */
+    slice(start, end) {
+      const key = cacheKey(start, end, byteLength)
+      const cached = cache.get(key)
+      if (cached) return cached
+      // cache miss, read from file
+      const promise = slice(start, end)
+      cache.set(key, promise)
+      return promise
+    },
+  }
+}
+
+
+/**
+ * Returns canonical cache key for a byte range 'start,end'.
+ * Normalize int-range and suffix-range requests to the same key.
+ *
+ * @param {number} start start byte of range
+ * @param {number} [end] end byte of range, or undefined for suffix range
+ * @param {number} [size] size of file, or undefined for suffix range
+ * @returns {string}
+ */
+function cacheKey(start, end, size) {
+  if (start < 0) {
+    if (end !== undefined) throw new Error(`invalid suffix range [${start}, ${end}]`)
+    if (size === undefined) return `${start},`
+    return `${size + start},${size}`
+  } else if (end !== undefined) {
+    if (start > end) throw new Error(`invalid empty range [${start}, ${end}]`)
+    return `${start},${end}`
+  } else if (size === undefined) {
+    return `${start},`
+  } else {
+    return `${start},${size}`
+  }
+}

src/hyparquet.d.ts

@@ -130,6 +130,11 @@ export function asyncBufferFromFile(filename: string): Promise<AsyncBuffer>
  */
 export function byteLengthFromUrl(url: string): Promise<number>
 
+/**
+ * Returns a cached layer on top of an AsyncBuffer.
+ */
+export function cachedAsyncBuffer(asyncBuffer: AsyncBuffer): AsyncBuffer
+
 /**
  * Parquet query options for reading data
  */

src/hyparquet.js

@@ -9,6 +9,8 @@ export { snappyUncompress } from './snappy.js'
 
 export { asyncBufferFromFile, asyncBufferFromUrl, byteLengthFromUrl, toJson } from './utils.js'
 
+export { cachedAsyncBuffer } from './asyncBuffer.js'
+
 /**
  * @param {import('./hyparquet.js').ParquetReadOptions} options
  * @returns {Promise<Array<Record<string, any>>>}