Skip to content

Commit ea07f0e

Browse files
rapravraprav
authored
fix: querystream into filestream on file export (#15)
* build: 3.0.5
1 parent 44d7c86 commit ea07f0e

File tree

3 files changed

+189
-2421
lines changed

3 files changed

+189
-2421
lines changed

‎index.js

Lines changed: 70 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const Excel = require('exceljs');
55
const csv = require('fast-csv');
66
const fs = require('fs');
77
const fsp = require('fs').promises;
8+
const JsonStreamStringify = require('json-stream-stringify');
89
const path = require('path');
910

1011
const Executor = require('@runnerty/module-core').Executor;
@@ -63,8 +64,9 @@ class mysqlExecutor extends Executor {
6364
// ****************
6465
if (params.xlsxFileExport) {
6566
await fsp.access(path.dirname(params.xlsxFileExport));
67+
const fileStreamWriter = fs.createWriteStream(params.xlsxFileExport);
6668
const options = {
67-
filename: params.xlsxFileExport,
69+
stream: fileStreamWriter,
6870
useStyles: true,
6971
useSharedStrings: true
7072
};
@@ -90,103 +92,88 @@ class mysqlExecutor extends Executor {
9092
queryStream.on('end', async _ => {
9193
await workbook.commit();
9294
this.prepareEndOptions(firstRow, rowCounter, resultSetHeader, results);
93-
this._end(this.endOptions);
95+
this._end();
9496
connection.end();
9597
});
9698
queryStream.on('error', err => {
97-
this.endOptions.end = 'error';
98-
this.endOptions.messageLog = `executeMysql: ${err}`;
99-
this.endOptions.err_output = `executeMysql: ${err}`;
100-
this._end(this.endOptions);
10199
connection.end();
100+
this._error(`executeMysql: ${err}`);
102101
});
103102
}
104103
// CSV FILE EXPORT
105104
// ***************
106105
else if (params.csvFileExport) {
107106
await fsp.access(path.dirname(params.csvFileExport));
108107
const fileStreamWriter = fs.createWriteStream(params.csvFileExport);
108+
109109
const paramsCSV = params.csvOptions || {};
110110
if (!paramsCSV.hasOwnProperty('headers')) paramsCSV.headers = true;
111111
const csvStream = csv
112112
.format(paramsCSV)
113113
.on('error', err => {
114-
this.endOptions.end = 'error';
115-
this.endOptions.messageLog = `executeMysql: ${err}`;
116-
this.endOptions.err_output = `executeMysql: ${err}`;
117-
this._end(this.endOptions);
118114
connection.end();
115+
this._error(`executeMysql - csvStream: ${err}`);
119116
})
120117
.on('end', () => {
121118
fileStreamWriter.end();
122119
this.prepareEndOptions(firstRow, rowCounter, resultSetHeader, results);
123-
this._end(this.endOptions);
120+
this._end();
124121
connection.end();
125122
});
126123

127-
csvStream.pipe(fileStreamWriter);
128-
129-
queryStream.on('result', row => {
130-
if (isFirstRow) {
131-
firstRow = row;
132-
if (row.constructor.name === 'ResultSetHeader') {
133-
resultSetHeader = row;
134-
}
135-
isFirstRow = false;
136-
}
137-
138-
rowCounter++;
139-
csvStream.write(row);
140-
});
141-
queryStream.on('end', _ => {
142-
csvStream.end();
143-
connection.end();
144-
});
145-
queryStream.on('error', err => {
146-
this.endOptions.end = 'error';
147-
this.endOptions.messageLog = `executeMysql: ${err}`;
148-
this.endOptions.err_output = `executeMysql: ${err}`;
149-
this._end(this.endOptions);
150-
connection.end();
151-
});
124+
// STREAMED
125+
queryStream
126+
.on('result', row => {
127+
if (isFirstRow) {
128+
firstRow = row;
129+
if (row.constructor.name === 'ResultSetHeader') {
130+
resultSetHeader = row;
131+
}
132+
isFirstRow = false;
133+
} else rowCounter++;
134+
csvStream.write(row);
135+
})
136+
.on('error', err => {
137+
connection.end();
138+
this._error(`executeMysql: ${err}`);
139+
})
140+
.stream()
141+
.pipe(csvStream)
142+
.pipe(fileStreamWriter);
152143
}
153144
// TEXT FILE EXPORT JSON
154145
// *********************
155146
else if (params.fileExport) {
156147
await fsp.access(path.dirname(params.fileExport));
157148
const fileStreamWriter = fs.createWriteStream(params.fileExport);
158149

159-
queryStream.on('result', row => {
160-
if (isFirstRow) {
161-
firstRow = row;
162-
if (row.constructor.name === 'ResultSetHeader') {
163-
resultSetHeader = row;
164-
}
165-
isFirstRow = false;
166-
fileStreamWriter.write('[\n');
167-
fileStreamWriter.write(JSON.stringify(row));
168-
} else {
169-
fileStreamWriter.write(',\n' + JSON.stringify(row));
170-
}
171-
rowCounter++;
172-
});
173-
174-
queryStream.on('end', () => {
175-
if (rowCounter) fileStreamWriter.write('\n]');
176-
177-
fileStreamWriter.end();
178-
this.prepareEndOptions(firstRow, rowCounter, resultSetHeader, results);
179-
this._end(this.endOptions);
180-
connection.end();
181-
});
150+
fileStreamWriter
151+
.on('finish', () => {
152+
this._end();
153+
})
154+
.on('error', err => {
155+
this._error(`ERROR executeMysql - fileStreamWriter ${err}`);
156+
});
182157

183-
queryStream.on('error', err => {
184-
this.endOptions.end = 'error';
185-
this.endOptions.messageLog = `executeMysql: ${err}`;
186-
this.endOptions.err_output = `executeMysql: ${err}`;
187-
this._end(this.endOptions);
188-
connection.end();
189-
});
158+
// STREAMED
159+
new JsonStreamStringify(
160+
queryStream
161+
.on('result', row => {
162+
if (isFirstRow) {
163+
firstRow = row;
164+
if (row.constructor.name === 'ResultSetHeader') {
165+
resultSetHeader = row;
166+
}
167+
isFirstRow = false;
168+
}
169+
rowCounter++;
170+
})
171+
.on('error', err => {
172+
connection.end();
173+
this._error(`ERROR executeMysql - queryStream ${err}`);
174+
})
175+
.stream()
176+
).pipe(fileStreamWriter);
190177
}
191178
// NO FILE EXPORT - DATA_OUTPUT
192179
// ****************************
@@ -203,24 +190,18 @@ class mysqlExecutor extends Executor {
203190
results.push(row);
204191
});
205192
queryStream.on('end', _ => {
206-
this.prepareEndOptions(firstRow, rowCounter, resultSetHeader, results);
207-
this._end(this.endOptions);
208193
connection.end();
194+
this.prepareEndOptions(firstRow, rowCounter, resultSetHeader, results);
195+
this._end();
209196
});
210197
queryStream.on('error', err => {
211-
this.endOptions.end = 'error';
212-
this.endOptions.messageLog = `executeMysql: ${err}`;
213-
this.endOptions.err_output = `executeMysql: ${err}`;
214-
this._end(this.endOptions);
215198
connection.end();
199+
this._error(`executeMysql: ${err}`);
216200
});
217201
}
218202
} catch (err) {
219-
this.endOptions.end = 'error';
220-
this.endOptions.messageLog = `executeMysql: ${err}`;
221-
this.endOptions.err_output = `executeMysql: ${err}`;
222-
this._end(this.endOptions);
223203
connection.end();
204+
this._error(`executeMysql: ${err}`);
224205
}
225206
}
226207

@@ -280,7 +261,7 @@ class mysqlExecutor extends Executor {
280261
this.endOptions.end = 'error';
281262
this.endOptions.messageLog = 'executeMysql dont have command or command_file';
282263
this.endOptions.err_output = 'executeMysql dont have command or command_file';
283-
this._end(this.endOptions);
264+
this._end();
284265
}
285266
}
286267
const query = await this.prepareQuery(params);
@@ -296,7 +277,7 @@ class mysqlExecutor extends Executor {
296277
this.endOptions.end = 'error';
297278
this.endOptions.messageLog = `executeMysql reading ssl file/s: ${error}`;
298279
this.endOptions.err_output = `executeMysql reading ssl file/s: ${error}`;
299-
this._end(this.endOptions);
280+
this._end();
300281
}
301282
}
302283

@@ -320,7 +301,7 @@ class mysqlExecutor extends Executor {
320301
this.endOptions.end = 'error';
321302
this.endOptions.messageLog = `executeMysql: ${err}`;
322303
this.endOptions.err_output = `executeMysql: ${err}`;
323-
this._end(this.endOptions);
304+
this._end();
324305
});
325306

326307
const result = await this.executeQuery(connection, query, params);
@@ -329,12 +310,19 @@ class mysqlExecutor extends Executor {
329310
this.endOptions.end = 'error';
330311
this.endOptions.messageLog = `executeMysql: ${err}`;
331312
this.endOptions.err_output = `executeMysql: ${err}`;
332-
this._end(this.endOptions);
313+
this._end();
333314
}
334315
}
335316

336-
_end(endOptions) {
337-
if (!this.ended) this.end(endOptions);
317+
_error(errMsg) {
318+
this.endOptions.end = 'error';
319+
this.endOptions.messageLog = errMsg;
320+
this.endOptions.err_output = errMsg;
321+
this._end();
322+
}
323+
324+
_end() {
325+
if (!this.ended) this.end(this.endOptions);
338326
this.ended = true;
339327
}
340328
}

0 commit comments

Comments
 (0)