-
Notifications
You must be signed in to change notification settings - Fork 6
/
index.js
77 lines (68 loc) · 2.14 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
const fs = require('fs');
const Promise = require('bluebird');
module.exports = function(knex, path, tableName, options) {
return new Promise(function(resolve, reject) {
options = Object.assign({
columnSeparator: '\t',
rowSeparator: '\n',
encoding: 'utf8',
ignoreFirstLine: false,
useFirstLineForColumns: true,
handleInsert: function(inserts, tableName) {
return knex(tableName).insert(inserts);
},
}, options);
let mapTo = options.mapTo;
if (mapTo) {
options.useFirstLineForColumns = false;
}
if (
typeof path === 'undefined' ||
typeof tableName === 'undefined' ||
(typeof options.mapTo === 'undefined' && !options.useFirstLineForColumns)
) {
reject(Error('path, tableName and mapTo needs to be defined'));
}
const stream = fs.createReadStream(path).setEncoding(options.encoding);
let splitEnd = '';
const inserts = [];
stream.on('data', function(chunk) {
const tempRows = chunk.split(options.rowSeparator);
if (splitEnd.length > 0) {
tempRows[0] = splitEnd + tempRows[0];
splitEnd = '';
}
if (!tempRows[tempRows.length-1].endsWith(options.rowSeparator)) {
splitEnd = tempRows.pop();
}
if (options.ignoreFirstLine || options.useFirstLineForColumns) {
if (options.useFirstLineForColumns) {
mapTo = tempRows.shift().split(options.columnSeparator);
} else {
tempRows.shift();
}
options.ignoreFirstLine = false;
options.useFirstLineForColumns = false;
}
tempRows.map(function(row) {
const cols = row.split(options.columnSeparator);
const knexRow = {};
mapTo.map(function(key, index) {
if (key === null) {
return;
}
try {
knexRow[key] = cols[index] ? JSON.parse(cols[index]) : null;
} catch (e) {
knexRow[key] = cols[index];
}
});
inserts.push(knexRow);
});
});
stream.on('end', function() {
options.handleInsert(inserts, tableName)
.then(resolve);
});
});
};