N0taN3rd · ikreymer · Jun 1, 2019 · Jun 1, 2019 · Jun 2, 2019
diff --git a/dist/nodewarc.js b/dist/nodewarc.js
diff --git a/index.html b/index.html
@@ -0,0 +1,54 @@
+<html>
+<head>
+<script src="dist/nodewarc.js"></script>
+<script>
+function parseWARC(files) {
+    let reader = new FileReader();
+
+    reader.onloadend = function() {
+        var inflator, strm, _in, len, pos = 0;
+
+        var rstream = new stream.Readable();
+        rstream._read = () => {};
+
+        const data = reader.result;
+
+        do {
+            len = data.byteLength - pos;
+            _in = new Uint8Array(data, pos, len);
+
+            inflator = new pako.Inflate();
+            strm = inflator.strm;
+            inflator.push(_in, true);
+
+            rstream.push(inflator.result);
+
+            pos += strm.next_in;
+        } while (strm.avail_in);
+
+        rstream.push(null);
+
+        // attempt to set large window to avoid error
+        //var gunzip = rstream.pipe(zlib.createGunzip({chunkSize: 1024*1024*16, windowBits: 15, flush: zlib.Z_NO_FLUSH, finishFlush: zlib.Z_SYNC_FLUSH}));
+
+        const parser = new WARCParser(rstream);
+        parser.on('record', record => { console.log(record) })
+        parser.on('done', () => { console.log('finished') })
+        parser.on('error', error => { console.error(error) })
+        parser.start()
+    }
+
+    reader.readAsArrayBuffer(files[0]);
+}
+</script>
+</head>
+<body>
+
+        <span>
+                Test WARC:
+                <input id="file-input" type="file" name="file-input" onchange="parseWARC(this.files)" style="display: none;" />
+                <input id="coll-name" type="text" value="replay" name="coll-name"/>
+                <button onclick="document.getElementById('file-input').click();">Open WARC File</button>
+        </span>
+</body>
+</html>
diff --git a/index.js b/index.js
@@ -115,3 +115,5 @@ if (require('./lib/parsers/_canUseRecordIterator')) {
    */
   exports.recordIterator = require('./lib/parsers/recordterator')
 }
+
+
diff --git a/lib/parsers/autoWARCParser.js b/lib/parsers/autoWARCParser.js
@@ -1,7 +1,6 @@
-const fs = require('fs-extra')
+const getStream = require('../utils/getStream').getStream
 const zlib = require('zlib')
 const EventEmitter = require('eventemitter3')
-const untildify = require('untildify')
 const WARCStreamTransform = require('./warcStreamTransform')
 const GzipDetector = require('./gzipDetector')
 const canUseRecordIterator = require('./_canUseRecordIterator')
@@ -138,7 +137,7 @@ class AutoWARCParser extends EventEmitter {
    */
   _getStream () {
     const isGz = GzipDetector.isGzippedSync(this._wp)
-    const stream = fs.createReadStream(untildify(this._wp))
+    const stream = getStream(this._wp, true)
     if (isGz) return stream.pipe(zlib.createGunzip())
     return stream
   }

diff --git a/lib/parsers/gzipDetector.js b/lib/parsers/gzipDetector.js
@@ -1,5 +1,7 @@
-const fs = require('fs-extra')
-const untildify = require('untildify')
+//import { fs, untildify } from '../utils/getStream'
+const gs = require('../utils/getStream');
+const fs = gs.fs;
+const untildify = gs.untildify;
 
 /**
  * @see https://en.wikipedia.org/wiki/Gzip

diff --git a/lib/parsers/warcGzParser.js b/lib/parsers/warcGzParser.js
@@ -1,5 +1,4 @@
-const untildify = require('untildify')
-const fs = require('fs-extra')
+const getStream = require('../utils/getStream').getStream
 const zlib = require('zlib')
 const EventEmitter = require('eventemitter3')
 const WARCStreamTransform = require('./warcStreamTransform')
@@ -54,7 +53,7 @@ class WARCGzParser extends EventEmitter {
        */
       this[Symbol.asyncIterator] = () => {
         return recordIterator(
-          fs.createReadStream(this._wp).pipe(zlib.createGunzip())
+          getStream(this._wp).pipe(zlib.createGunzip())
         )
       }
     }
@@ -78,8 +77,7 @@ class WARCGzParser extends EventEmitter {
       }
       this._parsing = true
       start = true
-      fs
-        .createReadStream(untildify(this._wp))
+      getStream(this._wp, true)
         .pipe(zlib.createGunzip())
         .pipe(new WARCStreamTransform())
         .on('data', this._onRecord)

diff --git a/lib/parsers/warcParser.js b/lib/parsers/warcParser.js
@@ -1,5 +1,4 @@
-const untildify = require('untildify')
-const fs = require('fs-extra')
+const getStream = require('../utils/getStream').getStream
 const EventEmitter = require('eventemitter3')
 const WARCStreamTransform = require('./warcStreamTransform')
 const canUseRecordIterator = require('./_canUseRecordIterator')
@@ -52,7 +51,7 @@ class WARCParser extends EventEmitter {
        * @returns {AsyncIterator<WARCRecord>}
        */
       this[Symbol.asyncIterator] = () => {
-        return recordIterator(fs.createReadStream(this._wp))
+        return recordIterator(getStream(this._wp));
       }
     }
   }
@@ -75,8 +74,7 @@ class WARCParser extends EventEmitter {
       }
       this._parsing = true
       start = true
-      fs
-        .createReadStream(untildify(this._wp))
+      getStream(this._wp)
         .pipe(new WARCStreamTransform())
         .on('data', this._onRecord)
         .on('error', this._onError)

diff --git a/lib/utils/getStream.js b/lib/utils/getStream.js
@@ -0,0 +1,28 @@
+
+let fs = null;
+let untildify = null;
+let hasFS = null;
+
+try {
+    untildify = require('untildify')
+    fs = require('fs-extra')
+    hasFS = true;
+} catch (e) {
+    hasFS = false;
+}
+
+function getStream(wp, useUntildify) {
+    if (hasFS) {
+        if (useUntildify) {
+            wp = untildify(wp);
+        }
+
+        return fs.createReadStream(wp);
+    }
+
+    return wp;
+}
+
+//module.exports = getStream;
+export { getStream, fs, untildify };
+
diff --git a/lib/writers/warcWriterBase.js b/lib/writers/warcWriterBase.js
@@ -1,4 +1,4 @@
-const fs = require('fs-extra')
+const fs = require('../utils/getStream').fs;
 const zlib = require('zlib')
 const Path = require('path')
 const uuid = require('uuid/v1')
@@ -142,7 +142,10 @@ class WARCWriterBase extends EventEmitter {
   initWARC (warcPath, options) {
     this.opts = Object.assign({}, this.defaultOpts, options || {})
     const wfp = ensureWARCFileName(warcPath, this.opts.gzip)
-    if (this.opts.appending) {
+
+    if (!fs) {
+      this._warcOutStream = warcPath
+    } else if (this.opts.appending) {
       this._warcOutStream = fs.createWriteStream(wfp, {
         flags: 'a',
         encoding: 'utf8'

diff --git a/package.json b/package.json
@@ -29,6 +29,7 @@
     "node": ">=8.0.0"
   },
   "scripts": {
+    "build": "webpack --mode production",
     "lint": "standard | snazzy",
     "fix-lint": "standard --fix",
     "pretty": "prettier-standard lib/**/*.js",
@@ -48,7 +49,8 @@
     "fs-extra": "^7.0.1",
     "lodash": "^4.17.11",
     "untildify": "^3.0.3",
-    "uuid": "^3.3.2"
+    "uuid": "^3.3.2",
+    "webpack": "^4.32.2"
   },
   "devDependencies": {
     "@types/fs-extra": "^5.0.5",
@@ -71,7 +73,8 @@
     "puppeteer": "^1.12.2",
     "rxjs": "^6.4.0",
     "snazzy": "^8.0.0",
-    "standard": "^12.0.1"
+    "standard": "^12.0.1",
+    "webpack-cli": "^3.3.2"
   },
   "ava": {
     "files": [

diff --git a/webpack.config.js b/webpack.config.js
@@ -0,0 +1,21 @@
+const path = require('path');
+const webpack = require('webpack');
+
+module.exports = {
+  mode: 'production',
+  entry: {
+    'nodewarc': './index.js',
+  },
+  devtool: 'inline-source-map',
+  output: {
+    path: path.resolve(__dirname, 'dist'),
+    filename: '[name].js',
+    libraryTarget: 'global',
+    globalObject: 'self'
+  },
+
+  plugins: [
+    new webpack.IgnorePlugin(/fs/),
+    new webpack.IgnorePlugin(/untildify/),
+  ],
+};