From 5e92fca582a45780ca6f3de68b599a1f88f35860 Mon Sep 17 00:00:00 2001 From: Jane Kelly Date: Tue, 20 Apr 2021 01:25:02 -0500 Subject: [PATCH 01/61] Add Retool to PapaParse lovers list (#867) --- docs/resources/js/lovers.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/resources/js/lovers.js b/docs/resources/js/lovers.js index 530a1644..456c3d7f 100644 --- a/docs/resources/js/lovers.js +++ b/docs/resources/js/lovers.js @@ -126,5 +126,11 @@ var peopleLovePapa = [ name: "Visa SOP Sample", description: "Providing free guide to international students.", quote: "Use Papa Parse for many of side projects. Super fast and works all the time. Love it!" + }, + { + link: "https://retool.com/", + name: "Retool", + description: "A remarkably fast way to build internal tools.", + quote: "Papa makes it easy for our users to customize CSV parsing to match their business logic." } ]; From 113d561c4c69f66181a9f09ac17c39e9a8f879c3 Mon Sep 17 00:00:00 2001 From: Christopher Eady Date: Fri, 28 May 2021 01:39:04 -0500 Subject: [PATCH 02/61] Update ISO_DATE regex to match full string (#872) --- papaparse.js | 2 +- tests/test-cases.js | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index 26db15cf..83ee4d01 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1008,7 +1008,7 @@ License: MIT var MAX_FLOAT = Math.pow(2, 53); var MIN_FLOAT = -MAX_FLOAT; var FLOAT = /^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/; - var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/; + var ISO_DATE = /^(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))$/; var self = this; var _stepCounter = 0; // Number of times step was called (number of rows parsed) var _rowCounter = 0; // Number of rows that have been parsed so far diff --git a/tests/test-cases.js b/tests/test-cases.js index 8d15b291..d727bfe6 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -997,6 +997,15 @@ var PARSE_TESTS = [ errors: [] } }, + { + description: "Dynamic typing skips ISO date strings ocurring in other strings", + input: 'ISO date,String with ISO date\r\n2018-05-04T21:08:03.269Z,The date is 2018-05-04T21:08:03.269Z\r\n2018-05-08T15:20:22.642Z,The date is 2018-05-08T15:20:22.642Z', + config: { dynamicTyping: true }, + expected: { + data: [["ISO date", "String with ISO date"], [new Date("2018-05-04T21:08:03.269Z"), "The date is 2018-05-04T21:08:03.269Z"], [new Date("2018-05-08T15:20:22.642Z"), "The date is 2018-05-08T15:20:22.642Z"]], + errors: [] + } + }, { description: "Blank line at beginning", input: '\r\na,b,c\r\nd,e,f', From eaeb01a1ea54efdfc6213733ee26f97387cb02a1 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Fri, 28 May 2021 08:49:12 +0200 Subject: [PATCH 03/61] Minor version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 6404a35e..af0afbfe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.3.0", + "version": "5.3.1", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index 83ee4d01..090c6634 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.3.0 +v5.3.1 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index a92afc17..96aced42 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v5.3.0 +v5.3.1 https://github.com/mholt/PapaParse License: MIT */ -!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function s(){"use strict";var f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{};var n=!f.document&&!!f.postMessage,o=n&&/blob:/i.test((f.location||{}).protocol),a={},h=0,b={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;U(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!U(t.transform)&&t.transform,t.worker&&b.WORKERS_SUPPORTED){var r=function(){if(!b.WORKERS_SUPPORTED)return!1;var e=(i=f.URL||f.webkitURL||null,r=s.toString(),b.BLOB_URL||(b.BLOB_URL=i.createObjectURL(new Blob(["(",r,")();"],{type:"text/javascript"})))),t=new f.Worker(e);var i,r;return t.onmessage=m,t.id=h++,a[t.id]=t}();return r.userStep=t.step,r.userChunk=t.chunk,r.userComplete=t.complete,r.userError=t.error,t.step=U(t.step),t.chunk=U(t.chunk),t.complete=U(t.complete),t.error=U(t.error),delete t.worker,void r.postMessage({input:e,config:t,workerId:r.id})}var n=null;b.NODE_STREAM_INPUT,"string"==typeof e?n=t.download?new l(t):new p(t):!0===e.readable&&U(e.read)&&U(e.on)?n=new g(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new c(t));return n.stream(e)},unparse:function(e,t){var n=!1,m=!0,_=",",v="\r\n",s='"',a=s+s,i=!1,r=null,o=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||b.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(_=t.delimiter);("boolean"==typeof t.quotes||"function"==typeof t.quotes||Array.isArray(t.quotes))&&(n=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines);"string"==typeof t.newline&&(v=t.newline);"string"==typeof t.quoteChar&&(s=t.quoteChar);"boolean"==typeof t.header&&(m=t.header);if(Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s);"boolean"==typeof t.escapeFormulae&&(o=t.escapeFormulae)}();var h=new RegExp(q(s),"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return f(null,e,i);if("object"==typeof e[0])return f(r||u(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:u(e.data[0])),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),f(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e){if("object"!=typeof e)return[];var t=[];for(var i in e)t.push(i);return t}function f(e,t,i){var r="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(U(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!U(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){U(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var r;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i)}try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:r.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function c(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function p(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=i.substring(0,t),i=i.substring(t)):(e=i,i=""),this._finished=!i,this.parseChunk(e)}}}function g(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(_){var a,o,h,r=Math.pow(2,53),n=-r,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)(e[-+]?\d+)?\s*$/,u=/(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/,t=this,i=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(U(_.step)){var p=_.step;_.step=function(e){if(c=e,m())g();else{if(g(),0===c.data.length)return;i+=e.data.length,_.preview&&i>_.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function v(e){return"greedy"===_.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),_.skipEmptyLines)for(var e=0;e=l.length?"__parsed_extra":l[i]),_.transform&&(s=_.transform(s,n)),s=y(n,s),"__parsed_extra"===n?(r[n]=r[n]||[],r[n].push(s)):r[n]=s}return _.header&&(i>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+i,f+t):i=r.length/2?"\r\n":"\r"}(e,r)),h=!1,_.delimiter)U(_.delimiter)&&(_.delimiter=_.delimiter(e),c.meta.delimiter=_.delimiter);else{var n=function(e,t,i,r,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=L)return R(!0)}else for(_=M,M++;;){if(-1===(_=a.indexOf(O,_+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:M}),E();if(_===r-1)return E(a.substring(M,_).replace(m,O));if(O!==z||a[_+1]!==z){if(O===z||0===_||a[_-1]!==z){-1!==p&&p<_+1&&(p=a.indexOf(D,_+1)),-1!==g&&g<_+1&&(g=a.indexOf(I,_+1));var y=w(-1===g?p:Math.min(p,g));if(a[_+1+y]===D){f.push(a.substring(M,_).replace(m,O)),a[M=_+1+y+e]!==O&&(_=a.indexOf(O,M)),p=a.indexOf(D,M),g=a.indexOf(I,M);break}var k=w(g);if(a.substring(_+1+k,_+1+k+n)===I){if(f.push(a.substring(M,_).replace(m,O)),C(_+1+k+n),p=a.indexOf(D,M),_=a.indexOf(O,M),o&&(S(),j))return R();if(L&&h.length>=L)return R(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:M}),_++}}else _++}return E();function b(e){h.push(e),d=M}function w(e){var t=0;if(-1!==e){var i=a.substring(_+1,e);i&&""===i.trim()&&(t=i.length)}return t}function E(e){return i||(void 0===e&&(e=a.substring(M)),f.push(e),M=r,b(f),o&&S()),R()}function C(e){M=e,b(f),f=[],g=a.indexOf(I,M)}function R(e){return{data:h,errors:u,meta:{delimiter:D,linebreak:I,aborted:j,truncated:!!e,cursor:d+(t||0)}}}function S(){A(R()),h=[],u=[]}function x(e,t,i){var r={nextDelim:void 0,quoteSearch:void 0},n=a.indexOf(O,t+1);if(t=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var r;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=v(this._chunkLoaded,this),r.onerror=v(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i)}try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:r.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function c(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=v(this._chunkLoaded,this),r.onerror=v(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function p(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=i.substring(0,t),i=i.substring(t)):(e=i,i=""),this._finished=!i,this.parseChunk(e)}}}function g(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=v(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=v(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=v(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=v(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var a,o,h,r=Math.pow(2,53),n=-r,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,u=/^(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))$/,t=this,i=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(M(m.step)){var p=m.step;m.step=function(e){if(c=e,_())g();else{if(g(),0===c.data.length)return;i+=e.data.length,m.preview&&i>m.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=l.length?"__parsed_extra":l[i]),m.transform&&(s=m.transform(s,n)),s=v(n,s),"__parsed_extra"===n?(r[n]=r[n]||[],r[n].push(s)):r[n]=s}return m.header&&(i>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+i,f+t):i=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),c.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=D)return C(!0)}else for(m=F,F++;;){if(-1===(m=r.indexOf(S,m+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:F}),E();if(m===n-1)return E(r.substring(F,m).replace(_,S));if(S!==L||r[m+1]!==L){if(S===L||0===m||r[m-1]!==L){-1!==p&&p=D)return C(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:F}),m++}}else m++}return E();function k(e){h.push(e),d=F}function b(e){var t=0;if(-1!==e){var i=r.substring(m+1,e);i&&""===i.trim()&&(t=i.length)}return t}function E(e){return i||(void 0===e&&(e=r.substring(F)),f.push(e),F=n,k(f),o&&R()),C()}function w(e){F=e,k(f),f=[],g=r.indexOf(x,F)}function C(e){return{data:h,errors:u,meta:{delimiter:O,linebreak:x,aborted:z,truncated:!!e,cursor:d+(t||0)}}}function R(){T(C()),h=[],u=[]}},this.abort=function(){z=!0},this.getCharIndex=function(){return F}}function _(e){var t=e.data,i=a[t.workerId],r=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){r=!0,m(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:y,resume:y};if(M(i.userStep)){for(var s=0;s Date: Wed, 9 Jun 2021 08:25:55 +0200 Subject: [PATCH 04/61] Remove travis CI config file --- .travis.yml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f00e0fbb..00000000 --- a/.travis.yml +++ /dev/null @@ -1,6 +0,0 @@ -language: node_js -node_js: - - "8" - - "9" - - "10" - - "11" From a6fdfcb4a61d3fbce9694ba1f69b2921e67ce090 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Wed, 9 Jun 2021 08:29:45 +0200 Subject: [PATCH 05/61] Remove support for node10 (#876) --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 3072b48a..ac0723f7 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - node-version: [10.x, 12.x, 14.x, 15.x] + node-version: [12.x, 14.x, 15.x] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ steps: From 23e1b47f5c1ebcd4b06f696fc555563a18d6b74b Mon Sep 17 00:00:00 2001 From: janisdd Date: Fri, 18 Jun 2021 14:13:13 +0200 Subject: [PATCH 06/61] - fixes multi-character delimiter with quoted field issue (#879) --- docs/docs.html | 6 +++--- papaparse.js | 2 +- tests/test-cases.js | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index da3c8b62..c2d82bd6 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -303,7 +303,7 @@
Unparse Config Options
delimiter - The delimiting character. It must not be found in Papa.BAD_DELIMITERS. + The delimiting character. Multi-character delimiters are supported. It must not be found in Papa.BAD_DELIMITERS. @@ -470,7 +470,7 @@
Config Options
delimiter - The delimiting character. Leave blank to auto-detect from a list of most common delimiters, or any values passed in through delimitersToGuess. It can be a string or a function. If string, it must be one of length 1. If a function, it must accept the input as first parameter and it must return a string which will be used as delimiter. In both cases it cannot be found in Papa.BAD_DELIMITERS. + The delimiting character. Leave blank to auto-detect from a list of most common delimiters, or any values passed in through delimitersToGuess. It can be a string or a function. If a string, it can be of any length (so multi-character delimiters are supported). If a function, it must accept the input as first parameter and it must return a string which will be used as delimiter. In both cases it cannot be found in Papa.BAD_DELIMITERS. @@ -861,7 +861,7 @@
Read-Only
Papa.BAD_DELIMITERS - An array of characters that are not allowed as delimiters. + An array of characters that are not allowed as delimiters (\r, \n, ", \ufeff). diff --git a/papaparse.js b/papaparse.js index 090c6634..057a16cb 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1555,7 +1555,7 @@ License: MIT var spacesBetweenQuoteAndDelimiter = extraSpaces(checkUpTo); // Closing quote followed by delimiter or 'unnecessary spaces + delimiter' - if (input[quoteSearch + 1 + spacesBetweenQuoteAndDelimiter] === delim) + if (input.substr(quoteSearch + 1 + spacesBetweenQuoteAndDelimiter, delimLen) === delim) { row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar)); cursor = quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen; diff --git a/tests/test-cases.js b/tests/test-cases.js index d727bfe6..0c3a62df 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -847,6 +847,16 @@ var PARSE_TESTS = [ errors: [] } }, + { + description: "Multi-character delimiter (length 2) with quoted field", + input: 'a, b, "c, e", d', + config: { delimiter: ", " }, + notes: "The quotes must be immediately adjacent to the delimiter to indicate a quoted field", + expected: { + data: [['a', 'b', 'c, e', 'd']], + errors: [] + } + }, { description: "Callback delimiter", input: 'a$ b$ c', @@ -1713,6 +1723,12 @@ var UNPARSE_TESTS = [ config: { delimiter: ', ' }, expected: 'A, b, c\r\nd, e, f' }, + { + description: "Custom delimiter (Multi-character), field contains custom delimiter", + input: [['A', 'b', 'c'], ['d', 'e', 'f, g']], + config: { delimiter: ', ' }, + expected: 'A, b, c\r\nd, e, "f, g"' + }, { description: "Bad delimiter (\\n)", notes: "Should default to comma", From 26a86fdf9f2e97225b101dad18dacdbac517ce58 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 9 Dec 2021 17:00:00 +0100 Subject: [PATCH 07/61] Do not run tests on node15 --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index ac0723f7..ea743e3a 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - node-version: [12.x, 14.x, 15.x] + node-version: [12.x, 14.x] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ steps: From 1f2c7330d5f562630195c8c450e7ec9cf6233684 Mon Sep 17 00:00:00 2001 From: Cyril Auburtin Date: Fri, 10 Dec 2021 10:45:57 +0100 Subject: [PATCH 08/61] Add more cases to escapeFormulae and allow to pass RegExp (#904) --- docs/docs.html | 2 +- papaparse.js | 14 +++++++++----- tests/test-cases.js | 29 +++++++++++++++++++++++++++-- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index c2d82bd6..cf198a56 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -343,7 +343,7 @@
Unparse Config Options
escapeFormulae - If true, field values that begin with =, +, -, or @, will be prepended with a ' to defend against injection attacks, because Excel and LibreOffice will automatically parse such cells as formulae. + If true, field values that begin with =, +, -, @, \t, or \r, will be prepended with a ' to defend against injection attacks, because Excel and LibreOffice will automatically parse such cells as formulae. You can override those values by setting this option to a regular expression diff --git a/papaparse.js b/papaparse.js index 057a16cb..91de82fe 100755 --- a/papaparse.js +++ b/papaparse.js @@ -367,11 +367,11 @@ License: MIT _escapedQuote = _config.escapeChar + _quoteChar; } - if (typeof _config.escapeFormulae === 'boolean') - _escapeFormulae = _config.escapeFormulae; + if (typeof _config.escapeFormulae === 'boolean' || _config.escapeFormulae instanceof RegExp) { + _escapeFormulae = _config.escapeFormulae instanceof RegExp ? _config.escapeFormulae : /^[=+\-@\t\r].*$/; + } } - /** The double for loop that iterates the data and writes out a CSV string including header row */ function serialize(fields, data, skipEmptyLines) { @@ -444,13 +444,17 @@ License: MIT if (str.constructor === Date) return JSON.stringify(str).slice(1, 25); - if (_escapeFormulae === true && typeof str === "string" && (str.match(/^[=+\-@].*$/) !== null)) { + var needsQuotes = false; + + if (_escapeFormulae && typeof str === "string" && _escapeFormulae.test(str)) { str = "'" + str; + needsQuotes = true; } var escapedQuoteStr = str.toString().replace(quoteCharRegex, _escapedQuote); - var needsQuotes = (typeof _quotes === 'boolean' && _quotes) + needsQuotes = needsQuotes + || _quotes === true || (typeof _quotes === 'function' && _quotes(str, col)) || (Array.isArray(_quotes) && _quotes[col]) || hasAny(escapedQuoteStr, Papa.BAD_DELIMITERS) diff --git a/tests/test-cases.js b/tests/test-cases.js index 0c3a62df..ab806ba3 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1881,7 +1881,7 @@ var UNPARSE_TESTS = [ description: "Escape formulae", input: [{ "Col1": "=danger", "Col2": "@danger", "Col3": "safe" }, { "Col1": "safe=safe", "Col2": "+danger", "Col3": "-danger, danger" }, { "Col1": "'+safe", "Col2": "'@safe", "Col3": "safe, safe" }], config: { escapeFormulae: true }, - expected: 'Col1,Col2,Col3\r\n\'=danger,\'@danger,safe\r\nsafe=safe,\'+danger,"\'-danger, danger"\r\n\'+safe,\'@safe,"safe, safe"' + expected: 'Col1,Col2,Col3\r\n"\'=danger","\'@danger",safe\r\nsafe=safe,"\'+danger","\'-danger, danger"\r\n\'+safe,\'@safe,"safe, safe"' }, { description: "Don't escape formulae by default", @@ -1898,7 +1898,7 @@ var UNPARSE_TESTS = [ description: "Escape formulae with single-quote quoteChar and escapeChar", input: [{ "Col1": "=danger", "Col2": "@danger", "Col3": "safe" }, { "Col1": "safe=safe", "Col2": "+danger", "Col3": "-danger, danger" }, { "Col1": "'+safe", "Col2": "'@safe", "Col3": "safe, safe" }], config: { escapeFormulae: true, quoteChar: "'", escapeChar: "'" }, - expected: 'Col1,Col2,Col3\r\n\'\'=danger,\'\'@danger,safe\r\nsafe=safe,\'\'+danger,\'\'\'-danger, danger\'\r\n\'\'+safe,\'\'@safe,\'safe, safe\'' + expected: 'Col1,Col2,Col3\r\n\'\'\'=danger\',\'\'\'@danger\',safe\r\nsafe=safe,\'\'\'+danger\',\'\'\'-danger, danger\'\r\n\'\'+safe,\'\'@safe,\'safe, safe\'' }, { description: "Escape formulae with single-quote quoteChar and escapeChar and forced quotes", @@ -1906,6 +1906,31 @@ var UNPARSE_TESTS = [ config: { escapeFormulae: true, quotes: true, quoteChar: "'", escapeChar: "'" }, expected: '\'Col1\',\'Col2\',\'Col3\'\r\n\'\'\'=danger\',\'\'\'@danger\',\'safe\'\r\n\'safe=safe\',\'\'\'+danger\',\'\'\'-danger, danger\'\r\n\'\'\'+safe\',\'\'\'@safe\',\'safe, safe\'' }, + // new escapeFormulae values: + { + description: "Escape formulae with tab and carriage-return", + input: [{ "Col1": "\tdanger", "Col2": "\rdanger,", "Col3": "safe\t\r" }], + config: { escapeFormulae: true }, + expected: 'Col1,Col2,Col3\r\n"\'\tdanger","\'\rdanger,","safe\t\r"' + }, + { + description: "Escape formulae with tab and carriage-return, with forced quotes", + input: [{ "Col1": " danger", "Col2": "\rdanger,", "Col3": "safe\t\r" }], + config: { escapeFormulae: true, quotes: true }, + expected: '"Col1","Col2","Col3"\r\n"\'\tdanger","\'\rdanger,","safe\t\r"' + }, + { + description: "Escape formulae with tab and carriage-return, with single-quote quoteChar and escapeChar", + input: [{ "Col1": " danger", "Col2": "\rdanger,", "Col3": "safe, \t\r" }], + config: { escapeFormulae: true, quoteChar: "'", escapeChar: "'" }, + expected: 'Col1,Col2,Col3\r\n\'\'\'\tdanger\',\'\'\'\rdanger,\',\'safe, \t\r\'' + }, + { + description: "Escape formulae with tab and carriage-return, with single-quote quoteChar and escapeChar and forced quotes", + input: [{ "Col1": " danger", "Col2": "\rdanger,", "Col3": "safe, \t\r" }], + config: { escapeFormulae: true, quotes: true, quoteChar: "'", escapeChar: "'" }, + expected: '\'Col1\',\'Col2\',\'Col3\'\r\n\'\'\'\tdanger\',\'\'\'\rdanger,\',\'safe, \t\r\'' + }, ]; describe('Unparse Tests', function() { From 997c6923c8582fd1f978a001e077dfee87e6db72 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Fri, 4 Mar 2022 11:19:34 +0100 Subject: [PATCH 09/61] Remove broken links from lovers --- docs/resources/js/lovers.js | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/docs/resources/js/lovers.js b/docs/resources/js/lovers.js index 456c3d7f..27363ca2 100644 --- a/docs/resources/js/lovers.js +++ b/docs/resources/js/lovers.js @@ -55,24 +55,6 @@ var peopleLovePapa = [ name: "Wikipedia", description: "uses Papa Parse in VisualEditor to help article editors effortlessly build data tables from text files." }, - { - link: "https://www.webucator.com/webdesign/javascript.cfm", - name: "Webucator", - description: "created a video showing how to use Papa Parse and FileDrop.js to create a drag-and-drop CSV-JSON converter.", - quote: "It's often easy to convert data to CSV. With Papa, it's easy to turn that CSV into JSON." - }, - { - link: "http://www.yolpo.com/social/gist.github?1dbd4556e748bdb830b3&autoplay=1&interimresults=0&failfast=1", - name: "Yolpo", - description: "created a simple regression test for Papa Parse.", - quote: "Papa's API is so intuitive, it took me no time to get it to work." - }, - { - link: "https://www.appstax.com", - name: "Appstax", - description: "uses Papa Parse to import and export CSV data in their visual databrowser.", - quote: "Papa is a great for parsing CSV. And what a great tone of voice - love it!" - }, { link: "https://github.com/Nanofus/novel.js", name: "Novel.js", @@ -97,12 +79,6 @@ var peopleLovePapa = [ description: "is a brand-new messaging app made specifically for busy families. Automatically align all family members when sending text messages to parents in the kindergarten or school or when planning your kids birthday parties.", quote: "With Papa it was a joy to implement our tool for importing messages and places from external systems." }, - { - link: "https://explore.hua-gallery.com", - name: "Hua Explore", - description: "The premier destination for information on Contemporary Chinese Art.", - quote: "Papa makes processing data that galleries send us totally seamless." - }, { link: "https://monei.net", name: "MONEI", From ec36ab22d33633b82fd3440759df8d5c5b380786 Mon Sep 17 00:00:00 2001 From: Dan Onoshko Date: Mon, 14 Mar 2022 15:28:04 +0700 Subject: [PATCH 10/61] Upgrade mocha-headless-chrome --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index af0afbfe..d7140dcb 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "grunt": "^1.0.2", "grunt-contrib-uglify": "^3.3.0", "mocha": "^5.2.0", - "mocha-headless-chrome": "^2.0.1", + "mocha-headless-chrome": "^4.0.0", "open": "7.0.0", "serve-static": "^1.7.1" }, From e42059577dd3db756952348d5231fd3905e33be0 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Mon, 14 Mar 2022 10:18:07 +0100 Subject: [PATCH 11/61] Add support for node16 (#877) --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index ea743e3a..50a96731 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - node-version: [12.x, 14.x] + node-version: [12.x, 14.x, 16.x] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ steps: From 6bb7c335288b21df2dbd1fd67ff72f81815a68d1 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Mon, 14 Mar 2022 10:41:34 +0100 Subject: [PATCH 12/61] Add usage stats on lovers page Inspired by react-papaparse --- docs/index.html | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/index.html b/docs/index.html index ec841e27..93819a48 100644 --- a/docs/index.html +++ b/docs/index.html @@ -175,6 +175,24 @@

Features

People Papa

+
+
+

+ + PapaParse + +   + + PapaParse + +

+

From 8dba33e0c546f73ae8bdfb55d3dfd2695765a024 Mon Sep 17 00:00:00 2001 From: chafi9-code <63321019+chafi9-code@users.noreply.github.com> Date: Mon, 14 Mar 2022 12:24:49 +0100 Subject: [PATCH 13/61] Set empty string to config.quotechar when it's value is null (#925) Closes #914 --- papaparse.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/papaparse.js b/papaparse.js index 91de82fe..df7ba066 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1403,8 +1403,7 @@ License: MIT var preview = config.preview; var fastMode = config.fastMode; var quoteChar; - /** Allows for no quoteChar by setting quoteChar to undefined in config */ - if (config.quoteChar === undefined) { + if (config.quoteChar === undefined || config.quoteChar === null) { quoteChar = '"'; } else { quoteChar = config.quoteChar; From 4132d810abbc7e3699259f2314f627f32ba981a6 Mon Sep 17 00:00:00 2001 From: Simon <33730997+TheSlimvReal@users.noreply.github.com> Date: Tue, 15 Mar 2022 09:29:51 +0100 Subject: [PATCH 14/61] fixcolumns config works with input type object (#919) Closes #918 --- papaparse.js | 2 +- tests/test-cases.js | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index df7ba066..667a109f 100755 --- a/papaparse.js +++ b/papaparse.js @@ -307,7 +307,7 @@ License: MIT if (Array.isArray(_input.data)) { if (!_input.fields) - _input.fields = _input.meta && _input.meta.fields; + _input.fields = _input.meta && _input.meta.fields || _columns; if (!_input.fields) _input.fields = Array.isArray(_input.data[0]) diff --git a/tests/test-cases.js b/tests/test-cases.js index ab806ba3..469c74cc 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1865,6 +1865,13 @@ var UNPARSE_TESTS = [ config: {columns: ['a', 'b', 'c']}, expected: 'a,b,c\r\n1,2,\r\n\r\n3,,4' }, + { + description: "Column option used to manually specify keys with input type object", + notes: "Should not throw any error when attempting to serialize key not present in object. Columns are different than keys of the first object. When an object is missing a key then the serialized value should be an empty string.", + input: { data: [{a: 1, b: '2'}, {}, {a: 3, d: 'd', c: 4,}] }, + config: {columns: ['a', 'b', 'c']}, + expected: 'a,b,c\r\n1,2,\r\n\r\n3,,4' + }, { description: "Use different escapeChar", input: [{a: 'foo', b: '"quoted"'}], From a93c5c9806f15df120c563ff4fcdcc84c01784b3 Mon Sep 17 00:00:00 2001 From: bezrodnov Date: Tue, 15 Mar 2022 11:31:29 +0100 Subject: [PATCH 15/61] Improve row skipping performance (#911) (#912) Co-authored-by: Aleks Bezrodnov --- papaparse.js | 6 +++--- tests/test-cases.js | 29 +++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/papaparse.js b/papaparse.js index 667a109f..b7d3f183 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1163,9 +1163,9 @@ License: MIT if (_config.skipEmptyLines) { - for (var i = 0; i < _results.data.length; i++) - if (testEmptyLine(_results.data[i])) - _results.data.splice(i--, 1); + _results.data = _results.data.filter(function(d) { + return !testEmptyLine(d); + }); } if (needsHeaderRow()) diff --git a/tests/test-cases.js b/tests/test-cases.js index 469c74cc..1b0ee33a 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1608,6 +1608,31 @@ var PARSE_ASYNC_TESTS = [ data: [['A','B','C'],['X','Y','Z']], errors: [] } + }, + { + description: "File with a few regular and lots of empty lines", + disabled: !FILES_ENABLED, + input: FILES_ENABLED ? new File(["A,B,C\nX,Y,Z\n" + new Array(500000).fill(",,").join("\n")], "sample.csv") : false, + config: { + skipEmptyLines: "greedy" + }, + expected: { + data: [['A','B','C'],['X','Y','Z']], + errors: [] + } + }, + { + description: "File with a few regular and lots of empty lines + worker", + disabled: !FILES_ENABLED, + input: FILES_ENABLED ? new File(["A,B,C\nX,Y,Z\n" + new Array(500000).fill(",,").join("\n")], "sample.csv") : false, + config: { + worker: true, + skipEmptyLines: "greedy" + }, + expected: { + data: [['A','B','C'],['X','Y','Z']], + errors: [] + } } ]; @@ -1836,9 +1861,9 @@ var UNPARSE_TESTS = [ }, { description: "Returns without rows with no content when skipEmptyLines is 'greedy'", - input: [[null, ' '], [], ['1', '2']], + input: [[null, ' '], [], ['1', '2']].concat(new Array(500000).fill(['', ''])).concat([['3', '4']]), config: {skipEmptyLines: 'greedy'}, - expected: '1,2' + expected: '1,2\r\n3,4' }, { description: "Returns empty rows when empty rows are passed and skipEmptyLines is false with headers", From e11ee26581d9f497087b4c3ee00c2a77645b81d2 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Tue, 15 Mar 2022 11:34:12 +0100 Subject: [PATCH 16/61] Minor version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index d7140dcb..a0fdc770 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.3.1", + "version": "5.3.2", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index b7d3f183..1b9ea513 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.3.1 +v5.3.2 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index 96aced42..edfdb377 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v5.3.1 +v5.3.2 https://github.com/mholt/PapaParse License: MIT */ -!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function s(){"use strict";var f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{};var n=!f.document&&!!f.postMessage,o=n&&/blob:/i.test((f.location||{}).protocol),a={},h=0,b={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;M(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!M(t.transform)&&t.transform,t.worker&&b.WORKERS_SUPPORTED){var r=function(){if(!b.WORKERS_SUPPORTED)return!1;var e=(i=f.URL||f.webkitURL||null,r=s.toString(),b.BLOB_URL||(b.BLOB_URL=i.createObjectURL(new Blob(["(",r,")();"],{type:"text/javascript"})))),t=new f.Worker(e);var i,r;return t.onmessage=_,t.id=h++,a[t.id]=t}();return r.userStep=t.step,r.userChunk=t.chunk,r.userComplete=t.complete,r.userError=t.error,t.step=M(t.step),t.chunk=M(t.chunk),t.complete=M(t.complete),t.error=M(t.error),delete t.worker,void r.postMessage({input:e,config:t,workerId:r.id})}var n=null;b.NODE_STREAM_INPUT,"string"==typeof e?n=t.download?new l(t):new p(t):!0===e.readable&&M(e.read)&&M(e.on)?n=new g(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new c(t));return n.stream(e)},unparse:function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,i=!1,r=null,o=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||b.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter);("boolean"==typeof t.quotes||"function"==typeof t.quotes||Array.isArray(t.quotes))&&(n=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines);"string"==typeof t.newline&&(y=t.newline);"string"==typeof t.quoteChar&&(s=t.quoteChar);"boolean"==typeof t.header&&(_=t.header);if(Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s);"boolean"==typeof t.escapeFormulae&&(o=t.escapeFormulae)}();var h=new RegExp(j(s),"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||Object.keys(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e,t,i){var r="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var r;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=v(this._chunkLoaded,this),r.onerror=v(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i)}try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:r.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function c(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=v(this._chunkLoaded,this),r.onerror=v(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function p(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=i.substring(0,t),i=i.substring(t)):(e=i,i=""),this._finished=!i,this.parseChunk(e)}}}function g(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=v(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=v(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=v(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=v(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var a,o,h,r=Math.pow(2,53),n=-r,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,u=/^(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))$/,t=this,i=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(M(m.step)){var p=m.step;m.step=function(e){if(c=e,_())g();else{if(g(),0===c.data.length)return;i+=e.data.length,m.preview&&i>m.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines)for(var e=0;e=l.length?"__parsed_extra":l[i]),m.transform&&(s=m.transform(s,n)),s=v(n,s),"__parsed_extra"===n?(r[n]=r[n]||[],r[n].push(s)):r[n]=s}return m.header&&(i>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+i,f+t):i=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),c.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=D)return C(!0)}else for(m=F,F++;;){if(-1===(m=r.indexOf(S,m+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:F}),E();if(m===n-1)return E(r.substring(F,m).replace(_,S));if(S!==L||r[m+1]!==L){if(S===L||0===m||r[m-1]!==L){-1!==p&&p=D)return C(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:F}),m++}}else m++}return E();function k(e){h.push(e),d=F}function b(e){var t=0;if(-1!==e){var i=r.substring(m+1,e);i&&""===i.trim()&&(t=i.length)}return t}function E(e){return i||(void 0===e&&(e=r.substring(F)),f.push(e),F=n,k(f),o&&R()),C()}function w(e){F=e,k(f),f=[],g=r.indexOf(x,F)}function C(e){return{data:h,errors:u,meta:{delimiter:O,linebreak:x,aborted:z,truncated:!!e,cursor:d+(t||0)}}}function R(){T(C()),h=[],u=[]}},this.abort=function(){z=!0},this.getCharIndex=function(){return F}}function _(e){var t=e.data,i=a[t.workerId],r=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){r=!0,m(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:y,resume:y};if(M(i.userStep)){for(var s=0;s=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var r;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=v(this._chunkLoaded,this),r.onerror=v(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i)}try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:r.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function c(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=v(this._chunkLoaded,this),r.onerror=v(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function p(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=i.substring(0,t),i=i.substring(t)):(e=i,i=""),this._finished=!i,this.parseChunk(e)}}}function g(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=v(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=v(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=v(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=v(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var a,o,h,r=Math.pow(2,53),n=-r,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,u=/^(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))$/,t=this,i=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(M(m.step)){var p=m.step;m.step=function(e){if(c=e,_())g();else{if(g(),0===c.data.length)return;i+=e.data.length,m.preview&&i>m.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){return c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines&&(c.data=c.data.filter(function(e){return!y(e)})),_()&&function(){if(!c)return;function e(e,t){M(m.transformHeader)&&(e=m.transformHeader(e,t)),l.push(e)}if(Array.isArray(c.data[0])){for(var t=0;_()&&t=l.length?"__parsed_extra":l[i]),m.transform&&(s=m.transform(s,n)),s=v(n,s),"__parsed_extra"===n?(r[n]=r[n]||[],r[n].push(s)):r[n]=s}return m.header&&(i>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+i,f+t):i=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),c.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=D)return C(!0)}else for(m=F,F++;;){if(-1===(m=r.indexOf(S,m+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:F}),E();if(m===n-1)return E(r.substring(F,m).replace(_,S));if(S!==L||r[m+1]!==L){if(S===L||0===m||r[m-1]!==L){-1!==p&&p=D)return C(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:F}),m++}}else m++}return E();function k(e){h.push(e),d=F}function b(e){var t=0;if(-1!==e){var i=r.substring(m+1,e);i&&""===i.trim()&&(t=i.length)}return t}function E(e){return i||(void 0===e&&(e=r.substring(F)),f.push(e),F=n,k(f),o&&R()),C()}function w(e){F=e,k(f),f=[],g=r.indexOf(x,F)}function C(e){return{data:h,errors:u,meta:{delimiter:O,linebreak:x,aborted:z,truncated:!!e,cursor:d+(t||0)}}}function R(){T(C()),h=[],u=[]}},this.abort=function(){z=!0},this.getCharIndex=function(){return F}}function _(e){var t=e.data,i=a[t.workerId],r=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){r=!0,m(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:y,resume:y};if(M(i.userStep)){for(var s=0;s Date: Fri, 10 Jun 2022 09:25:46 +0200 Subject: [PATCH 17/61] Remove support for node12 --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 50a96731..efc9b2bf 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - node-version: [12.x, 14.x, 16.x] + node-version: [14.x, 16.x] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ steps: From c19cd2de475dd11147790e4083d19824decefc29 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Fri, 10 Jun 2022 09:34:34 +0200 Subject: [PATCH 18/61] Add support for node18 (#938) --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index efc9b2bf..0fccdf05 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - node-version: [14.x, 16.x] + node-version: [14.x, 16.x, 18.x] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ steps: From 07724570a111dc3439975a924dbc3ce6b3bd2235 Mon Sep 17 00:00:00 2001 From: Eddie Stanley Date: Tue, 14 Jun 2022 19:50:23 +1200 Subject: [PATCH 19/61] [GH-788] Made the check to determine whether the current web-worker is owned by PapaParse more reliable (#937) Co-authored-by: eddie.stanley --- papaparse.js | 5 +++-- tests/.eslintrc.js | 3 +++ tests/test-cases.js | 49 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/papaparse.js b/papaparse.js index 1b9ea513..60ee489b 100755 --- a/papaparse.js +++ b/papaparse.js @@ -49,11 +49,12 @@ License: MIT function getWorkerBlob() { var URL = global.URL || global.webkitURL || null; var code = moduleFactory.toString(); - return Papa.BLOB_URL || (Papa.BLOB_URL = URL.createObjectURL(new Blob(['(', code, ')();'], {type: 'text/javascript'}))); + return Papa.BLOB_URL || (Papa.BLOB_URL = URL.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ", '(', code, ')();'], {type: 'text/javascript'}))); } var IS_WORKER = !global.document && !!global.postMessage, - IS_PAPA_WORKER = IS_WORKER && /blob:/i.test((global.location || {}).protocol); + IS_PAPA_WORKER = global.IS_PAPA_WORKER || false; + var workers = {}, workerIdCounter = 0; var Papa = {}; diff --git a/tests/.eslintrc.js b/tests/.eslintrc.js index 8c8fc0c9..f75898e0 100644 --- a/tests/.eslintrc.js +++ b/tests/.eslintrc.js @@ -1,5 +1,8 @@ module.exports = { "extends": ["../.eslintrc.js"], + "parserOptions": { + "ecmaVersion": 8 + }, "env": { "mocha": true }, diff --git a/tests/test-cases.js b/tests/test-cases.js index 1b0ee33a..75a20db4 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -2649,3 +2649,52 @@ describe('Custom Tests', function() { generateTest(CUSTOM_TESTS[i]); } }); + +(typeof window !== "undefined" ? describe : describe.skip)("Browser Tests", () => { + it("When parsing synchronously inside a web-worker not owned by PapaParse we should not invoke postMessage", async() => { + // Arrange + const papaParseScriptPath = new URL("../papaparse.js", window.document.baseURI).href; + + // Define our custom web-worker that loads PapaParse and executes a synchronous parse + const blob = new Blob([ + ` + importScripts('${papaParseScriptPath}'); + + self.addEventListener("message", function(event) { + if (event.data === "ExecuteParse") { + // Perform our synchronous parse, as requested + const results = Papa.parse('x\\ny\\n'); + postMessage({type: "ParseExecutedSuccessfully", results}); + } else { + // Otherwise, send whatever we received back. We shouldn't be hitting this (!) If we're reached + // this it means PapaParse thinks it is running inside a web-worker that it owns + postMessage(event.data); + } + }); + ` + ], {type: 'text/javascript'}); + + const blobURL = window.URL.createObjectURL(blob); + const webWorker = new Worker(blobURL); + + const receiveMessagePromise = new Promise((resolve, reject) => { + webWorker.addEventListener("message", (event) => { + if (event.data.type === "ParseExecutedSuccessfully") { + resolve(event.data); + } else { + const error = new Error(`Received unexpected message: ${JSON.stringify(event.data, null, 2)}`); + error.data = event.data; + reject(error); + } + }); + }); + + // Act + webWorker.postMessage("ExecuteParse"); + const webWorkerMessage = await receiveMessagePromise; + + // Assert + assert.equal("ParseExecutedSuccessfully", webWorkerMessage.type); + assert.equal(3, webWorkerMessage.results.data.length); + }); +}); From bf19955b609464eec2bc5a167330d082d437ac43 Mon Sep 17 00:00:00 2001 From: Chris Roth Date: Wed, 9 Nov 2022 01:32:35 -0700 Subject: [PATCH 20/61] Add vertex.io to lovers (#958) --- docs/resources/js/lovers.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/resources/js/lovers.js b/docs/resources/js/lovers.js index 27363ca2..7d9073df 100644 --- a/docs/resources/js/lovers.js +++ b/docs/resources/js/lovers.js @@ -26,6 +26,12 @@ **/ var peopleLovePapa = [ + { + link: "https://www.vertex.io", + name: "The no-code database of the future.", + description: "No-code, Postgres-powered admin, internal tools, and backend suite.", + quote: "Vertex uses Papa to power all CSV related features!" + }, { link: "https://www.circlehd.com", name: "CircleHD Enterprise Video Platform", From bde83f5d3f6222eb5866cff1d1e35a79bad7227b Mon Sep 17 00:00:00 2001 From: Nikita Skovoroda Date: Mon, 14 Nov 2022 15:02:05 +0300 Subject: [PATCH 21/61] Fix iso-date regex (#959) --- papaparse.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index 60ee489b..f32e55ea 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1013,7 +1013,7 @@ License: MIT var MAX_FLOAT = Math.pow(2, 53); var MIN_FLOAT = -MAX_FLOAT; var FLOAT = /^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/; - var ISO_DATE = /^(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))$/; + var ISO_DATE = /^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/; var self = this; var _stepCounter = 0; // Number of times step was called (number of rows parsed) var _rowCounter = 0; // Number of rows that have been parsed so far From c1cbe16636be146f5f1d321cdd7cbddc9143b045 Mon Sep 17 00:00:00 2001 From: fortydegrees <38460957+fortydegrees@users.noreply.github.com> Date: Tue, 15 Nov 2022 06:26:14 +0000 Subject: [PATCH 22/61] Rename duplicate headers (#956) --- papaparse.js | 31 +++++++++++++++++++++++++++++++ tests/test-cases.js | 20 +++++++++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index f32e55ea..6c52906e 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1457,6 +1457,37 @@ License: MIT if (!input) return returnable(); + // Rename headers if there are duplicates + if (config.header) + { + var firstLine = input.split(newline)[0]; + var headers = firstLine.split(delim); + var separator = '_'; + var headerMap = []; + var headerCount = {}; + var duplicateHeaders = false; + + for (var j in headers) { + var header = headers[j]; + if (isFunction(config.transformHeader)) + header = config.transformHeader(header, j); + var headerName = header; + + var count = headerCount[header] || 0; + if (count > 0) { + duplicateHeaders = true; + headerName = header + separator + count; + } + headerCount[header] = count + 1; + + headerMap.push(headerName); + } + if (duplicateHeaders) { + var editedInput = input.split(newline); + editedInput[0] = headerMap.join(delim); + input = editedInput.join(newline); + } + } if (fastMode || (fastMode !== false && input.indexOf(quoteChar) === -1)) { var rows = input.split(newline); diff --git a/tests/test-cases.js b/tests/test-cases.js index 75a20db4..843ba33c 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -585,7 +585,25 @@ var CORE_PARSER_TESTS = [ data: [['a', 'b', 'c'], ['']], errors: [] } - } + }, + { + description: "Simple duplicate header names", + input: 'A,A,A,A\n1,2,3,4', + config: { header: true }, + expected: { + data: [['A', 'A_1', 'A_2', 'A_3'], ['1', '2', '3', '4']], + errors: [] + } + }, + { + description: "Duplicate header names with headerTransform", + input: 'A,A,A,A\n1,2,3,4', + config: { header: true, transformHeader: function(header) { return header.toLowerCase(); } }, + expected: { + data: [['a', 'a_1', 'a_2', 'a_3'], ['1', '2', '3', '4']], + errors: [] + } + }, ]; describe('Core Parser Tests', function() { From aa0046865f1b4e817ebba6966d6baf483e0652d7 Mon Sep 17 00:00:00 2001 From: Peter Uithoven Date: Fri, 25 Nov 2022 15:22:03 +0100 Subject: [PATCH 23/61] Handle parsing utf-8 bom encoded files (#961) Co-authored-by: peteruithoven --- papaparse.js | 9 +++++++++ tests/node-tests.js | 11 +++++++++++ tests/utf-8-bom-sample.csv | 2 ++ 3 files changed, 22 insertions(+) create mode 100644 tests/utf-8-bom-sample.csv diff --git a/papaparse.js b/papaparse.js index 6c52906e..f5a1f8ff 100755 --- a/papaparse.js +++ b/papaparse.js @@ -235,6 +235,7 @@ License: MIT } else if (typeof _input === 'string') { + _input = stripBom(_input); if (_config.download) streamer = new NetworkStreamer(_config); else @@ -248,6 +249,14 @@ License: MIT streamer = new FileStreamer(_config); return streamer.stream(_input); + + // Strip character from UTF-8 BOM encoded files that cause issue parsing the file + function stripBom(string) { + if (string.charCodeAt(0) === 0xfeff) { + return string.slice(1); + } + return string; + } } diff --git a/tests/node-tests.js b/tests/node-tests.js index 9c57a225..6d012ec8 100644 --- a/tests/node-tests.js +++ b/tests/node-tests.js @@ -5,6 +5,7 @@ var Papa = require("../papaparse.js"); var fs = require('fs'); var assert = require('assert'); var longSampleRawCsv = fs.readFileSync(__dirname + '/long-sample.csv', 'utf8'); +var utf8BomSampleRawCsv = fs.readFileSync(__dirname + '/utf-8-bom-sample.csv', 'utf8'); function assertLongSampleParsedCorrectly(parsedCsv) { assert.equal(8, parsedCsv.data.length); @@ -287,4 +288,14 @@ describe('PapaParse', function() { } }); }); + + it('handles utf-8 BOM encoded files', function(done) { + Papa.parse(utf8BomSampleRawCsv, { + header: true, + complete: function(parsedCsv) { + assert.deepEqual(parsedCsv.data[0], { A: 'X', B: 'Y', C: 'Z' }); + done(); + } + }); + }); }); diff --git a/tests/utf-8-bom-sample.csv b/tests/utf-8-bom-sample.csv new file mode 100644 index 00000000..4f85ff06 --- /dev/null +++ b/tests/utf-8-bom-sample.csv @@ -0,0 +1,2 @@ +A,B,C +X,Y,Z From db230fc9d996722e706bb1912504ec84ff042c50 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 2 Mar 2023 10:24:50 +0100 Subject: [PATCH 24/61] Minor version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index a0fdc770..b5b144d4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.3.2", + "version": "5.4.0", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index f5a1f8ff..422839a8 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.3.2 +v5.4.0 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index edfdb377..fc2ba580 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v5.3.2 +v5.4.0 https://github.com/mholt/PapaParse License: MIT */ -!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function s(){"use strict";var f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{};var n=!f.document&&!!f.postMessage,o=n&&/blob:/i.test((f.location||{}).protocol),a={},h=0,b={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;M(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!M(t.transform)&&t.transform,t.worker&&b.WORKERS_SUPPORTED){var r=function(){if(!b.WORKERS_SUPPORTED)return!1;var e=(i=f.URL||f.webkitURL||null,r=s.toString(),b.BLOB_URL||(b.BLOB_URL=i.createObjectURL(new Blob(["(",r,")();"],{type:"text/javascript"})))),t=new f.Worker(e);var i,r;return t.onmessage=_,t.id=h++,a[t.id]=t}();return r.userStep=t.step,r.userChunk=t.chunk,r.userComplete=t.complete,r.userError=t.error,t.step=M(t.step),t.chunk=M(t.chunk),t.complete=M(t.complete),t.error=M(t.error),delete t.worker,void r.postMessage({input:e,config:t,workerId:r.id})}var n=null;b.NODE_STREAM_INPUT,"string"==typeof e?n=t.download?new l(t):new p(t):!0===e.readable&&M(e.read)&&M(e.on)?n=new g(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new c(t));return n.stream(e)},unparse:function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,i=!1,r=null,o=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||b.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter);("boolean"==typeof t.quotes||"function"==typeof t.quotes||Array.isArray(t.quotes))&&(n=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines);"string"==typeof t.newline&&(y=t.newline);"string"==typeof t.quoteChar&&(s=t.quoteChar);"boolean"==typeof t.header&&(_=t.header);if(Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s);("boolean"==typeof t.escapeFormulae||t.escapeFormulae instanceof RegExp)&&(o=t.escapeFormulae instanceof RegExp?t.escapeFormulae:/^[=+\-@\t\r].*$/)}();var h=new RegExp(j(s),"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||Object.keys(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||r),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e,t,i){var r="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(M(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!M(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){M(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var r;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=v(this._chunkLoaded,this),r.onerror=v(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i)}try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:r.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function c(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),u.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=v(this._chunkLoaded,this),r.onerror=v(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function p(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=i.substring(0,t),i=i.substring(t)):(e=i,i=""),this._finished=!i,this.parseChunk(e)}}}function g(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=v(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=v(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=v(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=v(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var a,o,h,r=Math.pow(2,53),n=-r,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,u=/^(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))$/,t=this,i=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(M(m.step)){var p=m.step;m.step=function(e){if(c=e,_())g();else{if(g(),0===c.data.length)return;i+=e.data.length,m.preview&&i>m.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){return c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines&&(c.data=c.data.filter(function(e){return!y(e)})),_()&&function(){if(!c)return;function e(e,t){M(m.transformHeader)&&(e=m.transformHeader(e,t)),l.push(e)}if(Array.isArray(c.data[0])){for(var t=0;_()&&t=l.length?"__parsed_extra":l[i]),m.transform&&(s=m.transform(s,n)),s=v(n,s),"__parsed_extra"===n?(r[n]=r[n]||[],r[n].push(s)):r[n]=s}return m.header&&(i>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+i,f+t):i=r.length/2?"\r\n":"\r"}(e,r)),h=!1,m.delimiter)M(m.delimiter)&&(m.delimiter=m.delimiter(e),c.meta.delimiter=m.delimiter);else{var n=function(e,t,i,r,n){var s,a,o,h;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var u=0;u=D)return C(!0)}else for(m=F,F++;;){if(-1===(m=r.indexOf(S,m+1)))return i||u.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:h.length,index:F}),E();if(m===n-1)return E(r.substring(F,m).replace(_,S));if(S!==L||r[m+1]!==L){if(S===L||0===m||r[m-1]!==L){-1!==p&&p=D)return C(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:F}),m++}}else m++}return E();function k(e){h.push(e),d=F}function b(e){var t=0;if(-1!==e){var i=r.substring(m+1,e);i&&""===i.trim()&&(t=i.length)}return t}function E(e){return i||(void 0===e&&(e=r.substring(F)),f.push(e),F=n,k(f),o&&R()),C()}function w(e){F=e,k(f),f=[],g=r.indexOf(x,F)}function C(e){return{data:h,errors:u,meta:{delimiter:O,linebreak:x,aborted:z,truncated:!!e,cursor:d+(t||0)}}}function R(){T(C()),h=[],u=[]}},this.abort=function(){z=!0},this.getCharIndex=function(){return F}}function _(e){var t=e.data,i=a[t.workerId],r=!1;if(t.error)i.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){r=!0,m(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:y,resume:y};if(M(i.userStep)){for(var s=0;s=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(J(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!J(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){J(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var i;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),h.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=v(this._chunkLoaded,this),i.onerror=v(this._chunkError,this)),i.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r)}try{i.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===i.status&&this._chunkError()}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:i.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function c(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),h.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=v(this._chunkLoaded,this),i.onerror=v(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function p(e){var r;h.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=r.substring(0,t),r=r.substring(t)):(e=r,r=""),this._finished=!r,this.parseChunk(e)}}}function g(e){h.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){h.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){h.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=v(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=v(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=v(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=v(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(m){var a,o,u,i=Math.pow(2,53),n=-i,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,h=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,t=this,r=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(J(m.step)){var p=m.step;m.step=function(e){if(c=e,_())g();else{if(g(),0===c.data.length)return;r+=e.data.length,m.preview&&r>m.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){return c&&u&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),u=!1),m.skipEmptyLines&&(c.data=c.data.filter(function(e){return!y(e)})),_()&&function(){if(!c)return;function e(e,t){J(m.transformHeader)&&(e=m.transformHeader(e,t)),l.push(e)}if(Array.isArray(c.data[0])){for(var t=0;_()&&t=l.length?"__parsed_extra":l[r]),m.transform&&(s=m.transform(s,n)),s=v(n,s),"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return m.header&&(r>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+r,f+t):r=i.length/2?"\r\n":"\r"}(e,i)),u=!1,m.delimiter)J(m.delimiter)&&(m.delimiter=m.delimiter(e),c.meta.delimiter=m.delimiter);else{var n=function(e,t,r,i,n){var s,a,o,u;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var h=0;h=N)return L(!0)}else for(S=W,W++;;){if(-1===(S=i.indexOf(z,S+1)))return r||h.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:u.length,index:W}),T();if(S===n-1)return T(i.substring(W,S).replace(C,z));if(z!==K||i[S+1]!==K){if(z===K||0===S||i[S-1]!==K){-1!==w&&w=N)return L(!0);break}h.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:u.length,index:W}),S++}}else S++}return T();function I(e){u.push(e),d=W}function A(e){var t=0;if(-1!==e){var r=i.substring(S+1,e);r&&""===r.trim()&&(t=r.length)}return t}function T(e){return r||(void 0===e&&(e=i.substring(W)),f.push(e),W=n,I(f),o&&F()),L()}function D(e){W=e,I(f),f=[],R=i.indexOf(P,W)}function L(e){return{data:u,errors:h,meta:{delimiter:M,linebreak:P,aborted:H,truncated:!!e,cursor:d+(t||0)}}}function F(){q(L()),u=[],h=[]}},this.abort=function(){H=!0},this.getCharIndex=function(){return W}}function _(e){var t=e.data,r=a[t.workerId],i=!1;if(t.error)r.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){i=!0,m(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:y,resume:y};if(J(r.userStep)){for(var s=0;s Date: Tue, 14 Mar 2023 10:42:41 +0100 Subject: [PATCH 25/61] Rename duplicated headers Closes #982 --- papaparse.js | 5 ++++- tests/test-cases.js | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index 422839a8..8a3488ca 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1488,7 +1488,10 @@ License: MIT headerName = header + separator + count; } headerCount[header] = count + 1; - + // In case it already exists, we add more separtors + while (headerMap.includes(headerName)) { + headerName = headerName + separator + count; + } headerMap.push(headerName); } if (duplicateHeaders) { diff --git a/tests/test-cases.js b/tests/test-cases.js index 843ba33c..41f93d23 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -604,6 +604,15 @@ var CORE_PARSER_TESTS = [ errors: [] } }, + { + description: "Duplicate header names existing column", + input: 'c,c,c,c_1\n1,2,3,4', + config: { header: true }, + expected: { + data: [['c', 'c_1', 'c_2', 'c_1_0'], ['1', '2', '3', '4']], + errors: [] + } + }, ]; describe('Core Parser Tests', function() { From 824bbd9daf17168bddfc5485066771453cab423e Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Fri, 17 Mar 2023 09:43:15 +0100 Subject: [PATCH 26/61] Only test duplicate headers on first row Fixes #980 --- papaparse.js | 2 +- tests/test-cases.js | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/papaparse.js b/papaparse.js index 8a3488ca..1fd7dc4c 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1467,7 +1467,7 @@ License: MIT return returnable(); // Rename headers if there are duplicates - if (config.header) + if (config.header && !baseIndex) { var firstLine = input.split(newline)[0]; var headers = firstLine.split(delim); diff --git a/tests/test-cases.js b/tests/test-cases.js index 41f93d23..0243a2c5 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -2656,7 +2656,24 @@ var CUSTOM_TESTS = [ var results = Papa.parse('"A","B","C","D"'); callback(results.meta.delimiter); } - } + }, + { + description: "Data is correctly parsed with chunks and duplicated headers", + expected: [{h0: 'a', h1: 'a'}, {h0: 'b', h1: 'b'}], + run: function(callback) { + var data = []; + Papa.parse('h0,h1\na,a\nb,b', { + header: true, + chunkSize: 10, + chunk: function(results) { + data.push(results.data[0]); + }, + complete: function() { + callback(data); + } + }); + } + }, ]; describe('Custom Tests', function() { From e728bfe0f719a43ead95c33e941d514981cf24d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20=C5=A0im=C3=A1nek?= Date: Thu, 23 Mar 2023 13:22:18 +0100 Subject: [PATCH 27/61] Remove jsperf.com links from README.md. (#986) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 45313c3b..9af8ad80 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ Parse CSV with JavaScript ======================================== -Papa Parse is the [fastest](http://jsperf.com/javascript-csv-parsers/4) in-browser CSV (or delimited text) parser for JavaScript. It is reliable and correct according to [RFC 4180](https://tools.ietf.org/html/rfc4180), and it comes with these features: +Papa Parse is the fastest in-browser CSV (or delimited text) parser for JavaScript. It is reliable and correct according to [RFC 4180](https://tools.ietf.org/html/rfc4180), and it comes with these features: - Easy to use - Parse CSV files directly (local or over the network) -- Fast mode ([is really fast](http://jsperf.com/javascript-csv-parsers/3)) +- Fast mode - Stream large files (even via HTTP) - Reverse parsing (converts JSON to CSV) - Auto-detect delimiter From 3e7c4bc3e58187f0c4a8f80bd1f02a6722810b45 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 23 Mar 2023 13:48:23 +0100 Subject: [PATCH 28/61] Bugfix version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index b5b144d4..54f9952d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.4.0", + "version": "5.4.1", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index 1fd7dc4c..d94773bc 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.4.0 +v5.4.1 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index fc2ba580..eeaf9834 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v5.4.0 +v5.4.1 https://github.com/mholt/PapaParse License: MIT */ -!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function s(){"use strict";var f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{};var n=!f.document&&!!f.postMessage,o=f.IS_PAPA_WORKER||!1,a={},u=0,b={parse:function(e,t){var r=(t=t||{}).dynamicTyping||!1;J(r)&&(t.dynamicTypingFunction=r,r={});if(t.dynamicTyping=r,t.transform=!!J(t.transform)&&t.transform,t.worker&&b.WORKERS_SUPPORTED){var i=function(){if(!b.WORKERS_SUPPORTED)return!1;var e=(r=f.URL||f.webkitURL||null,i=s.toString(),b.BLOB_URL||(b.BLOB_URL=r.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ","(",i,")();"],{type:"text/javascript"})))),t=new f.Worker(e);var r,i;return t.onmessage=_,t.id=u++,a[t.id]=t}();return i.userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=J(t.step),t.chunk=J(t.chunk),t.complete=J(t.complete),t.error=J(t.error),delete t.worker,void i.postMessage({input:e,config:t,workerId:i.id})}var n=null;b.NODE_STREAM_INPUT,"string"==typeof e?(e=function(e){if(65279===e.charCodeAt(0))return e.slice(1);return e}(e),n=t.download?new l(t):new p(t)):!0===e.readable&&J(e.read)&&J(e.on)?n=new g(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new c(t));return n.stream(e)},unparse:function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,r=!1,i=null,o=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||b.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter);("boolean"==typeof t.quotes||"function"==typeof t.quotes||Array.isArray(t.quotes))&&(n=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(r=t.skipEmptyLines);"string"==typeof t.newline&&(y=t.newline);"string"==typeof t.quoteChar&&(s=t.quoteChar);"boolean"==typeof t.header&&(_=t.header);if(Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");i=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s);("boolean"==typeof t.escapeFormulae||t.escapeFormulae instanceof RegExp)&&(o=t.escapeFormulae instanceof RegExp?t.escapeFormulae:/^[=+\-@\t\r].*$/)}();var u=new RegExp(Q(s),"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return h(null,e,r);if("object"==typeof e[0])return h(i||Object.keys(e[0]),e,r)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||i),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),h(e.fields||[],e.data||[],r);throw new Error("Unable to serialize unrecognized input");function h(e,t,r){var i="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(J(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!J(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){J(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var i;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),h.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=v(this._chunkLoaded,this),i.onerror=v(this._chunkError,this)),i.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r)}try{i.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===i.status&&this._chunkError()}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:i.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function c(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),h.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=v(this._chunkLoaded,this),i.onerror=v(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function p(e){var r;h.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=r.substring(0,t),r=r.substring(t)):(e=r,r=""),this._finished=!r,this.parseChunk(e)}}}function g(e){h.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){h.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){h.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=v(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=v(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=v(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=v(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(m){var a,o,u,i=Math.pow(2,53),n=-i,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,h=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,t=this,r=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(J(m.step)){var p=m.step;m.step=function(e){if(c=e,_())g();else{if(g(),0===c.data.length)return;r+=e.data.length,m.preview&&r>m.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){return c&&u&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),u=!1),m.skipEmptyLines&&(c.data=c.data.filter(function(e){return!y(e)})),_()&&function(){if(!c)return;function e(e,t){J(m.transformHeader)&&(e=m.transformHeader(e,t)),l.push(e)}if(Array.isArray(c.data[0])){for(var t=0;_()&&t=l.length?"__parsed_extra":l[r]),m.transform&&(s=m.transform(s,n)),s=v(n,s),"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return m.header&&(r>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+r,f+t):r=i.length/2?"\r\n":"\r"}(e,i)),u=!1,m.delimiter)J(m.delimiter)&&(m.delimiter=m.delimiter(e),c.meta.delimiter=m.delimiter);else{var n=function(e,t,r,i,n){var s,a,o,u;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var h=0;h=N)return L(!0)}else for(S=W,W++;;){if(-1===(S=i.indexOf(z,S+1)))return r||h.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:u.length,index:W}),T();if(S===n-1)return T(i.substring(W,S).replace(C,z));if(z!==K||i[S+1]!==K){if(z===K||0===S||i[S-1]!==K){-1!==w&&w=N)return L(!0);break}h.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:u.length,index:W}),S++}}else S++}return T();function I(e){u.push(e),d=W}function A(e){var t=0;if(-1!==e){var r=i.substring(S+1,e);r&&""===r.trim()&&(t=r.length)}return t}function T(e){return r||(void 0===e&&(e=i.substring(W)),f.push(e),W=n,I(f),o&&F()),L()}function D(e){W=e,I(f),f=[],R=i.indexOf(P,W)}function L(e){return{data:u,errors:h,meta:{delimiter:M,linebreak:P,aborted:H,truncated:!!e,cursor:d+(t||0)}}}function F(){q(L()),u=[],h=[]}},this.abort=function(){H=!0},this.getCharIndex=function(){return W}}function _(e){var t=e.data,r=a[t.workerId],i=!1;if(t.error)r.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){i=!0,m(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:y,resume:y};if(J(r.userStep)){for(var s=0;s=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(J(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!J(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){J(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var i;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),h.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=v(this._chunkLoaded,this),i.onerror=v(this._chunkError,this)),i.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r)}try{i.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===i.status&&this._chunkError()}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:i.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function c(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),h.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=v(this._chunkLoaded,this),i.onerror=v(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function p(e){var r;h.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=r.substring(0,t),r=r.substring(t)):(e=r,r=""),this._finished=!r,this.parseChunk(e)}}}function g(e){h.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){h.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){h.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=v(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=v(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=v(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=v(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(m){var a,o,u,i=Math.pow(2,53),n=-i,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,h=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,t=this,r=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(J(m.step)){var p=m.step;m.step=function(e){if(c=e,_())g();else{if(g(),0===c.data.length)return;r+=e.data.length,m.preview&&r>m.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){return c&&u&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),u=!1),m.skipEmptyLines&&(c.data=c.data.filter(function(e){return!y(e)})),_()&&function(){if(!c)return;function e(e,t){J(m.transformHeader)&&(e=m.transformHeader(e,t)),l.push(e)}if(Array.isArray(c.data[0])){for(var t=0;_()&&t=l.length?"__parsed_extra":l[r]),m.transform&&(s=m.transform(s,n)),s=v(n,s),"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return m.header&&(r>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+r,f+t):r=i.length/2?"\r\n":"\r"}(e,i)),u=!1,m.delimiter)J(m.delimiter)&&(m.delimiter=m.delimiter(e),c.meta.delimiter=m.delimiter);else{var n=function(e,t,r,i,n){var s,a,o,u;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var h=0;h=N)return L(!0)}else for(S=W,W++;;){if(-1===(S=i.indexOf(z,S+1)))return r||h.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:u.length,index:W}),T();if(S===n-1)return T(i.substring(W,S).replace(C,z));if(z!==K||i[S+1]!==K){if(z===K||0===S||i[S-1]!==K){-1!==w&&w=N)return L(!0);break}h.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:u.length,index:W}),S++}}else S++}return T();function I(e){u.push(e),d=W}function A(e){var t=0;if(-1!==e){var r=i.substring(S+1,e);r&&""===r.trim()&&(t=r.length)}return t}function T(e){return r||(void 0===e&&(e=i.substring(W)),f.push(e),W=n,I(f),o&&F()),L()}function D(e){W=e,I(f),f=[],R=i.indexOf(P,W)}function L(e){return{data:u,errors:h,meta:{delimiter:M,linebreak:P,aborted:H,truncated:!!e,cursor:d+(t||0)}}}function F(){q(L()),u=[],h=[]}},this.abort=function(){H=!0},this.getCharIndex=function(){return W}}function _(e){var t=e.data,r=a[t.workerId],i=!1;if(t.error)r.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){i=!0,m(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:y,resume:y};if(J(r.userStep)){for(var s=0;s Date: Fri, 31 Mar 2023 12:34:02 +0200 Subject: [PATCH 29/61] Update minium es version to 6 --- .eslintrc.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.eslintrc.js b/.eslintrc.js index c3ed7c56..bdd8ea14 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,6 +1,6 @@ module.exports = { "parserOptions": { - "ecmaVersion": 5 + "ecmaVersion": 6 }, "env": { "browser": true, From 3a329014682666ad56e4fbdf726142fa1684106e Mon Sep 17 00:00:00 2001 From: Nicolas DUBIEN Date: Fri, 31 Mar 2023 14:49:08 +0200 Subject: [PATCH 30/61] Faster headerMap to find duplicated headers (#991) --- .eslintrc.js | 1 + papaparse.js | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index bdd8ea14..97a67265 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -3,6 +3,7 @@ module.exports = { "ecmaVersion": 6 }, "env": { + "es6": true, "browser": true, "worker": true, "node": true diff --git a/papaparse.js b/papaparse.js index d94773bc..6495e7d8 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1472,7 +1472,7 @@ License: MIT var firstLine = input.split(newline)[0]; var headers = firstLine.split(delim); var separator = '_'; - var headerMap = []; + var headerMap = new Set(); var headerCount = {}; var duplicateHeaders = false; @@ -1488,15 +1488,15 @@ License: MIT headerName = header + separator + count; } headerCount[header] = count + 1; - // In case it already exists, we add more separtors - while (headerMap.includes(headerName)) { + // In case it already exists, we add more separators + while (headerMap.has(headerName)) { headerName = headerName + separator + count; } - headerMap.push(headerName); + headerMap.add(headerName); } if (duplicateHeaders) { var editedInput = input.split(newline); - editedInput[0] = headerMap.join(delim); + editedInput[0] = Array.from(headerMap).join(delim); input = editedInput.join(newline); } } From e14a838b64c28a0734a03c0bf88129fdd0889f41 Mon Sep 17 00:00:00 2001 From: Junxiang Chen Date: Wed, 5 Apr 2023 21:05:45 +0100 Subject: [PATCH 31/61] feat: renamed headers meta (#990) * feat: add logic to store renamed headers metadata * fix: test failures * test: surface renamedHeader to meta and include test cases * doc: add documentation for the new property * fix: failing tests * fix: failing tests --- docs/docs.html | 10 +++++++--- papaparse.js | 12 +++++++++++- tests/node-tests.js | 1 + tests/test-cases.js | 43 ++++++++++++++++++++++++++++--------------- 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index cf198a56..49095b17 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -502,7 +502,8 @@

Config Options
header - If true, the first row of parsed data will be interpreted as field names. An array of field names will be returned in meta, and each row of data will be an object of values keyed by field name instead of a simple array. Rows with a different number of fields from the header row will produce an error. Warning: Duplicate field names will overwrite values in previous fields having the same name. + If true, the first row of parsed data will be interpreted as field names. An array of field names will be returned in meta, and each row of data will be an object of values keyed by field name instead of a simple array. Rows with a different number of fields from the header row will produce an error. + Warning: Duplicated field names will be automatically renamed to avoid values in previous fields having the same name to be overwritten. Renamed fields with original (or transformed by transformHeader) are stored in ParseResult.meta.renamedHeaders @@ -759,11 +760,13 @@
Data
[ { "Column 1": "foo", - "Column 2": "bar" + "Column 2": "bar", + "Column 1": "foo1", }, { "Column 1": "abc", - "Column 2": "def" + "Column 2": "def", + "Column 1": "abc1", } ]
@@ -811,6 +814,7 @@
Meta
aborted: // Whether process was aborted fields: // Array of field names truncated: // Whether preview consumed all input + renamedHeaders: // Headers that are automatically renamed by the library to avoid duplication. {Column 1_1: 'Column 1' // the later header 'Column 1' was renamed to 'Column 1_1'} }
diff --git a/papaparse.js b/papaparse.js index 6495e7d8..60997b05 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1413,6 +1413,8 @@ License: MIT var preview = config.preview; var fastMode = config.fastMode; var quoteChar; + var renamedHeaders = null; + if (config.quoteChar === undefined || config.quoteChar === null) { quoteChar = '"'; } else { @@ -1486,6 +1488,10 @@ License: MIT if (count > 0) { duplicateHeaders = true; headerName = header + separator + count; + // Initialise the variable if it hasn't been. + if (renamedHeaders === null) { + renamedHeaders = {}; + } } headerCount[header] = count + 1; // In case it already exists, we add more separators @@ -1493,6 +1499,9 @@ License: MIT headerName = headerName + separator + count; } headerMap.add(headerName); + if (count > 0) { + renamedHeaders[headerName] = header; + } } if (duplicateHeaders) { var editedInput = input.split(newline); @@ -1769,7 +1778,8 @@ License: MIT linebreak: newline, aborted: aborted, truncated: !!stopped, - cursor: lastCursor + (baseIndex || 0) + cursor: lastCursor + (baseIndex || 0), + renamedHeaders: renamedHeaders } }; } diff --git a/tests/node-tests.js b/tests/node-tests.js index 6d012ec8..cad8058e 100644 --- a/tests/node-tests.js +++ b/tests/node-tests.js @@ -32,6 +32,7 @@ function assertLongSampleParsedCorrectly(parsedCsv) { "linebreak": "\n", "aborted": false, "truncated": false, + renamedHeaders: null, "cursor": 1209 }); assert.equal(parsedCsv.errors.length, 0); diff --git a/tests/test-cases.js b/tests/test-cases.js index 0243a2c5..1bccea6b 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -29,7 +29,8 @@ var CORE_PARSER_TESTS = [ input: 'A,b,c', expected: { data: [['A', 'b', 'c']], - errors: [] + errors: [], + meta: {delimiter: ',', renamedHeaders: null} } }, { @@ -587,30 +588,33 @@ var CORE_PARSER_TESTS = [ } }, { - description: "Simple duplicate header names", + description: "Simple duplicated header names", input: 'A,A,A,A\n1,2,3,4', config: { header: true }, expected: { data: [['A', 'A_1', 'A_2', 'A_3'], ['1', '2', '3', '4']], - errors: [] + errors: [], + meta: {renamedHeaders: {A_1: 'A', A_2: 'A', A_3: 'A'}} } }, { - description: "Duplicate header names with headerTransform", + description: "Duplicated header names with headerTransform", input: 'A,A,A,A\n1,2,3,4', config: { header: true, transformHeader: function(header) { return header.toLowerCase(); } }, expected: { data: [['a', 'a_1', 'a_2', 'a_3'], ['1', '2', '3', '4']], - errors: [] + errors: [], + meta: {renamedHeaders: {a_1: 'a', a_2: 'a', a_3: 'a'}} } }, { - description: "Duplicate header names existing column", + description: "Duplicated header names existing column", input: 'c,c,c,c_1\n1,2,3,4', config: { header: true }, expected: { data: [['c', 'c_1', 'c_2', 'c_1_0'], ['1', '2', '3', '4']], - errors: [] + errors: [], + meta: {renamedHeaders: {c_1: 'c', c_2: 'c'}} } }, ]; @@ -621,6 +625,7 @@ describe('Core Parser Tests', function() { var actual = new Papa.Parser(test.config).parse(test.input); assert.deepEqual(actual.errors, test.expected.errors); assert.deepEqual(actual.data, test.expected.data); + assert.deepNestedInclude(actual.meta, test.expected.meta || {}); }); } @@ -1401,7 +1406,8 @@ var PARSE_TESTS = [ delimiter: ',', cursor: 23, aborted: false, - truncated: false + truncated: false, + renamedHeaders: null } } }, @@ -1417,7 +1423,8 @@ var PARSE_TESTS = [ delimiter: ',', cursor: 19, aborted: false, - truncated: false + truncated: false, + renamedHeaders: null } } }, @@ -1433,7 +1440,8 @@ var PARSE_TESTS = [ delimiter: ',', cursor: 28, aborted: false, - truncated: false + truncated: false, + renamedHeaders: null } } }, @@ -1449,7 +1457,8 @@ var PARSE_TESTS = [ delimiter: ',', cursor: 27, aborted: false, - truncated: false + truncated: false, + renamedHeaders: null } } }, @@ -1465,7 +1474,8 @@ var PARSE_TESTS = [ delimiter: ',', cursor: 29, aborted: false, - truncated: false + truncated: false, + renamedHeaders: null } } }, @@ -1481,7 +1491,8 @@ var PARSE_TESTS = [ delimiter: ',', cursor: 24, aborted: false, - truncated: false + truncated: false, + renamedHeaders: null } } }, @@ -1497,7 +1508,8 @@ var PARSE_TESTS = [ delimiter: ',', cursor: 27, aborted: false, - truncated: false + truncated: false, + renamedHeaders: null } } }, @@ -1513,7 +1525,8 @@ var PARSE_TESTS = [ delimiter: ',', cursor: 27, aborted: false, - truncated: false + truncated: false, + renamedHeaders: null } } }, From 30bab45f3dd885b038d814c99ae5cdcf14640f64 Mon Sep 17 00:00:00 2001 From: John Sherwood Date: Tue, 11 Apr 2023 16:45:16 +1000 Subject: [PATCH 32/61] Use for loop for header parsing to only interate over array elements (#987) --- papaparse.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index 60997b05..9071e6bc 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1478,7 +1478,8 @@ License: MIT var headerCount = {}; var duplicateHeaders = false; - for (var j in headers) { + // Using old-style 'for' loop to avoid prototype pollution that would be picked up with 'var j in headers' + for (var j = 0; j < headers.length; j++) { var header = headers[j]; if (isFunction(config.transformHeader)) header = config.transformHeader(header, j); From 713f7d81434c9f334f711936dccc70f4fd72e9d5 Mon Sep 17 00:00:00 2001 From: Jan Swist Date: Tue, 11 Apr 2023 10:08:01 +0200 Subject: [PATCH 33/61] Add DocTemple to lovers.js (#996) --- docs/resources/js/lovers.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/resources/js/lovers.js b/docs/resources/js/lovers.js index 7d9073df..503851b5 100644 --- a/docs/resources/js/lovers.js +++ b/docs/resources/js/lovers.js @@ -26,6 +26,12 @@ **/ var peopleLovePapa = [ + { + link: "http://doctempleapp.com", + name: "DocTemple", + description: "helps you effortlessly populate .docx documents with .csv data using scalable templating solution.", + quote: "Papa Parse made it extremely easy to load user CSV files in the browser. Implementing it was a breeze and it works perfect!" + }, { link: "https://www.vertex.io", name: "The no-code database of the future.", From b5365550968b46c5ec4a9fa37ea26365c92c9a15 Mon Sep 17 00:00:00 2001 From: jsakai <129206804+jsakaicribl@users.noreply.github.com> Date: Wed, 12 Apr 2023 21:44:32 -1000 Subject: [PATCH 34/61] Fixed cursor when duplicated header (#997) --- papaparse.js | 10 ++++++++-- tests/test-cases.js | 15 ++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/papaparse.js b/papaparse.js index 9071e6bc..3ce39621 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1469,9 +1469,10 @@ License: MIT return returnable(); // Rename headers if there are duplicates + var firstLine; if (config.header && !baseIndex) { - var firstLine = input.split(newline)[0]; + firstLine = input.split(newline)[0]; var headers = firstLine.split(delim); var separator = '_'; var headerMap = new Set(); @@ -1516,7 +1517,12 @@ License: MIT for (var i = 0; i < rows.length; i++) { row = rows[i]; - cursor += row.length; + // use firstline as row length may be changed due to duplicated headers + if (i === 0 && firstLine !== undefined) { + cursor += firstLine.length; + }else{ + cursor += row.length; + } if (i !== rows.length - 1) cursor += newline.length; else if (ignoreLastRow) diff --git a/tests/test-cases.js b/tests/test-cases.js index 1bccea6b..da489e8c 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -594,7 +594,10 @@ var CORE_PARSER_TESTS = [ expected: { data: [['A', 'A_1', 'A_2', 'A_3'], ['1', '2', '3', '4']], errors: [], - meta: {renamedHeaders: {A_1: 'A', A_2: 'A', A_3: 'A'}} + meta: { + renamedHeaders: {A_1: 'A', A_2: 'A', A_3: 'A'}, + cursor: 15 + } } }, { @@ -604,7 +607,10 @@ var CORE_PARSER_TESTS = [ expected: { data: [['a', 'a_1', 'a_2', 'a_3'], ['1', '2', '3', '4']], errors: [], - meta: {renamedHeaders: {a_1: 'a', a_2: 'a', a_3: 'a'}} + meta: { + renamedHeaders: {a_1: 'a', a_2: 'a', a_3: 'a'}, + cursor: 15 + } } }, { @@ -614,7 +620,10 @@ var CORE_PARSER_TESTS = [ expected: { data: [['c', 'c_1', 'c_2', 'c_1_0'], ['1', '2', '3', '4']], errors: [], - meta: {renamedHeaders: {c_1: 'c', c_2: 'c'}} + meta: { + renamedHeaders: {c_1: 'c', c_2: 'c'}, + cursor: 17 + } } }, ]; From 841e1d420dd2758cccb60a2bc1f2fcffd327c251 Mon Sep 17 00:00:00 2001 From: Cami Blanch Date: Fri, 2 Jun 2023 03:50:40 -0600 Subject: [PATCH 35/61] Add usage example (#1002) --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 9af8ad80..8484aac8 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,14 @@ can be installed with the following command: If you don't want to use npm, [papaparse.min.js](https://unpkg.com/papaparse@latest/papaparse.min.js) can be downloaded to your project source. +Usage +----- + + import Papa from 'papaparse'; + + Papa.parse(file, config); + + const csv = Papa.unparse(data[, config]); Homepage & Demo ---------------- From f1d10eea1cacecf1e40678b6552f97b46bc0219c Mon Sep 17 00:00:00 2001 From: Samuel J Palmer Date: Mon, 9 Oct 2023 15:09:55 +0100 Subject: [PATCH 36/61] Manage boolean values in escapeFormule setting (#1025) --- papaparse.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/papaparse.js b/papaparse.js index 3ce39621..b9e41d08 100755 --- a/papaparse.js +++ b/papaparse.js @@ -377,8 +377,10 @@ License: MIT _escapedQuote = _config.escapeChar + _quoteChar; } - if (typeof _config.escapeFormulae === 'boolean' || _config.escapeFormulae instanceof RegExp) { - _escapeFormulae = _config.escapeFormulae instanceof RegExp ? _config.escapeFormulae : /^[=+\-@\t\r].*$/; + if (_config.escapeFormulae instanceof RegExp) { + _escapeFormulae = _config.escapeFormulae; + } else if (typeof _config.escapeFormulae === 'boolean' && _config.escapeFormulae) { + _escapeFormulae = /^[=+\-@\t\r].*$/; } } From e2d570e77cc2720b2c256ec099f3e7e424c39e35 Mon Sep 17 00:00:00 2001 From: bhuvanesh <56244773+bhuvaneshwararaja@users.noreply.github.com> Date: Mon, 9 Oct 2023 19:44:28 +0530 Subject: [PATCH 37/61] Allow to skip first lines #738 (#1021) Co-authored-by: Bhuvaneshwara Raja --- docs/docs.html | 11 ++++++++++- papaparse.js | 19 +++++++++++++++++-- player/player.html | 1 + player/player.js | 1 + tests/test-cases.js | 36 ++++++++++++++++++++++++++++++++++++ 5 files changed, 65 insertions(+), 3 deletions(-) diff --git a/docs/docs.html b/docs/docs.html index 49095b17..b9292fad 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -450,7 +450,8 @@
Default Config With All Options
beforeFirstChunk: undefined, withCredentials: undefined, transform: undefined, - delimitersToGuess: [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP] + delimitersToGuess: [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP], + skipFirstNLines: 0 }
@@ -682,6 +683,14 @@
Config Options
An array of delimiters to guess from if the delimiter option is not set. + + + skipFirstNLines + + + To skip first N number of lines when converting a CSV file to JSON + + diff --git a/papaparse.js b/papaparse.js index b9e41d08..4bc3dc37 100755 --- a/papaparse.js +++ b/papaparse.js @@ -486,6 +486,7 @@ License: MIT } } + /** ChunkStreamer is the base prototype for various streamer implementations. */ function ChunkStreamer(config) { @@ -521,8 +522,23 @@ License: MIT // Rejoin the line we likely just split in two by chunking the file var aggregate = this._partialLine + chunk; + this._pendingSkip = parseInt(this._config.skipFirstNLines) || 0; + this._skipHeader = 0; + if (this._config.header) { + this._skipHeader++; + } + if (this._pendingSkip > 0) { + var splitChunk = aggregate.split('\n'); + var currentChunkLength = splitChunk.length; + if (currentChunkLength <= this._pendingSkip) { + aggregate = this._partialLine; + } + else{ + aggregate = this._partialLine + [...splitChunk.slice(0, this._skipHeader), ...splitChunk.slice(this._skipHeader + this._pendingSkip)].join('\n'); + } + this._pendingSkip -= currentChunkLength; + } this._partialLine = ''; - var results = this._handle.parse(aggregate, this._baseIndex, !this._finished); if (this._handle.paused() || this._handle.aborted()) { @@ -1931,7 +1947,6 @@ License: MIT { return function() { f.apply(self, arguments); }; } - function isFunction(func) { return typeof func === 'function'; diff --git a/player/player.html b/player/player.html index b90fe986..48015e35 100644 --- a/player/player.html +++ b/player/player.html @@ -24,6 +24,7 @@

Papa Parse Player

+ diff --git a/player/player.js b/player/player.js index 8150de64..f8b8e3ac 100644 --- a/player/player.js +++ b/player/player.js @@ -108,6 +108,7 @@ function buildConfig() skipEmptyLines: $('#skipEmptyLines').prop('checked'), chunk: $('#chunk').prop('checked') ? chunkFn : undefined, beforeFirstChunk: undefined, + skipFirstNLines: $('#skipFirstNLines').val() }; function getLineEnding() diff --git a/tests/test-cases.js b/tests/test-cases.js index da489e8c..68f367fc 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1574,6 +1574,42 @@ var PARSE_TESTS = [ data: [['a', 'b', 'c\n'], ['d', 'e', 'f']], errors: [] } + }, + { + description: "Skip First N number of lines , with header and 2 rows", + input: 'a,b,c,d\n1,2,3,4', + config: { header: true, skipFirstNLines: 1 }, + expected: { + data: [], + errors: [] + } + }, + { + description: "Skip First N number of lines , with header and 3 rows", + input: 'a,b,c,d\n1,2,3,4\n4,5,6,7', + config: { header: true, skipFirstNLines: 1 }, + expected: { + data: [{a: '4', b: '5', c: '6', d: '7'}], + errors: [] + } + }, + { + description: "Skip First N number of lines , with header false", + input: 'a,b,c,d\n1,2,3,4\n4,5,6,7', + config: { header: false, skipFirstNLines: 1 }, + expected: { + data: [['1','2','3','4'],['4','5','6','7']], + errors: [] + } + }, + { + description: "Skip First N number of lines , with header false and skipFirstNLines as negative value", + input: 'a,b,c,d\n1,2,3,4\n4,5,6,7', + config: { header: false, skipFirstNLines: -2 }, + expected: { + data: [['a','b','c','d'],['1','2','3','4'],['4','5','6','7']], + errors: [] + } } ]; From fbaeca2233e38a7e3528b241d5ce6978cd00e148 Mon Sep 17 00:00:00 2001 From: Kevin Hill <107003416+khill-fbmc@users.noreply.github.com> Date: Mon, 18 Dec 2023 07:06:21 -0800 Subject: [PATCH 38/61] Add language hints to the README examples (#1034) --- README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8484aac8..19319385 100644 --- a/README.md +++ b/README.md @@ -23,19 +23,21 @@ Install papaparse is available on [npm](https://www.npmjs.com/package/papaparse). It can be installed with the following command: - - npm install papaparse +```shell +npm install papaparse +``` If you don't want to use npm, [papaparse.min.js](https://unpkg.com/papaparse@latest/papaparse.min.js) can be downloaded to your project source. Usage ----- +```js +import Papa from 'papaparse'; - import Papa from 'papaparse'; - - Papa.parse(file, config); +Papa.parse(file, config); - const csv = Papa.unparse(data[, config]); +const csv = Papa.unparse(data[, config]); +``` Homepage & Demo ---------------- From a67ed3f7d23344e521aa9394c51bda539258fb2a Mon Sep 17 00:00:00 2001 From: Fabian Breitkreutz <78473803+maon-fp@users.noreply.github.com> Date: Mon, 19 Feb 2024 10:18:45 +0000 Subject: [PATCH 39/61] Improve readability on dynamicTyping option (#1041) Co-authored-by: Fabian Breitkreutz --- docs/docs.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs.html b/docs/docs.html index b9292fad..79eeb795 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -521,7 +521,7 @@
Config Options
dynamicTyping - If true, numeric and boolean data will be converted to their type instead of remaining strings. Numeric data must conform to the definition of a decimal literal. Numerical values greater than 2^53 or less than -2^53 will not be converted to numbers to preserve precision. European-formatted numbers must have commas and dots swapped. If also accepts an object or a function. If object it's values should be a boolean to indicate if dynamic typing should be applied for each column number (or header name if using headers). If it's a function, it should return a boolean value for each field number (or name if using headers) which will be passed as first argument. + If true, numeric and boolean data will be converted to their type instead of remaining strings. Numeric data must conform to the definition of a decimal literal. Numerical values greater than 2^53 or less than -2^53 will not be converted to numbers to preserve precision. European-formatted numbers must have commas and dots swapped. It also accepts an object or a function. If it's an object, its values should be a boolean to indicate if dynamic typing should be applied for each column number (or header name if using headers). If it's a function, it should return a boolean value for each field number (or name if using headers) which will be passed as first argument. From a39383e725237c962afe81f33c6b1206d1a436d6 Mon Sep 17 00:00:00 2001 From: Adam Jones Date: Tue, 5 Mar 2024 12:09:52 +0000 Subject: [PATCH 40/61] Update papaparse link in package.json to use HTTPS (#1044) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 54f9952d..a1298ab1 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "multi-threaded", "jquery-plugin" ], - "homepage": "http://papaparse.com", + "homepage": "https://www.papaparse.com/", "repository": { "type": "git", "url": "https://github.com/mholt/PapaParse.git" From 016effe152b218c2545ab70b818f75758cad2e3c Mon Sep 17 00:00:00 2001 From: Nicolas Lassaux Date: Wed, 27 Mar 2024 17:29:21 +0100 Subject: [PATCH 41/61] Add hellodata success story (#1047) --- docs/resources/js/lovers.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/resources/js/lovers.js b/docs/resources/js/lovers.js index 503851b5..1c939510 100644 --- a/docs/resources/js/lovers.js +++ b/docs/resources/js/lovers.js @@ -120,5 +120,11 @@ var peopleLovePapa = [ name: "Retool", description: "A remarkably fast way to build internal tools.", quote: "Papa makes it easy for our users to customize CSV parsing to match their business logic." + }, + { + link: "https://www.hellodata.ai/", + name: "HelloData", + description: "Automatic rent surveys with real-time data on over 25M multifamily units nationwide, direct from property websites.", + quote: "Papa Parse makes bulk data imports a breeze! It's helped us easily onboard our largest customers. It's robust and a true game-changer." } ]; From a116779198bc0b067a13aa859c32731bdee7a4a1 Mon Sep 17 00:00:00 2001 From: Nick Reilingh Date: Thu, 11 Jul 2024 03:27:44 -0400 Subject: [PATCH 42/61] Add missing space after readme link in faq (#1060) --- docs/faq.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.html b/docs/faq.html index b8ecdb29..403461c1 100644 --- a/docs/faq.html +++ b/docs/faq.html @@ -71,7 +71,7 @@
Why use Papa Parse?

Can I use Papa Parse server-side with Node.js?
-

Yes, Paparse supports Node. See our READMEfor further details. +

Yes, Paparse supports Node. See our README for further details.

Does Papa Parse have any dependencies?
From cf4bdedbdafbc84e7dd28e0ad71a8a30da2daba4 Mon Sep 17 00:00:00 2001 From: Jonas Krukenberg <115351184+jkruke@users.noreply.github.com> Date: Mon, 5 Aug 2024 18:49:01 +0200 Subject: [PATCH 43/61] Only skip first n lines in the first chunk and don't take the first line as header (#1045) (#1046) * Only skip first n lines in the first chunk and don't take the first line as header (#1045) * use newline from config (or guess if unset) --------- Co-authored-by: Jonas Krukenberg --- docs/resources/js/papaparse.js | 65 +++++++++++++++------------ papaparse.js | 80 ++++++++++++++++------------------ tests/test-cases.js | 15 +++++-- 3 files changed, 86 insertions(+), 74 deletions(-) diff --git a/docs/resources/js/papaparse.js b/docs/resources/js/papaparse.js index ac02f6d6..9dd7fa27 100644 --- a/docs/resources/js/papaparse.js +++ b/docs/resources/js/papaparse.js @@ -491,6 +491,16 @@ License: MIT this.parseChunk = function(chunk, isFakeChunk) { // First chunk pre-processing + const skipFirstNLines = parseInt(this._config.skipFirstNLines) || 0; + if (this.isFirstChunk && skipFirstNLines > 0) { + let _newline = this._config.newline; + if (!_newline) { + const quoteChar = this._config.quoteChar || '"'; + _newline = this._handle.guessLineEndings(chunk, quoteChar); + } + const splitChunk = chunk.split(_newline); + chunk = [...splitChunk.slice(skipFirstNLines)].join(_newline); + } if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) { var modifiedChunk = this._config.beforeFirstChunk(chunk); @@ -503,7 +513,6 @@ License: MIT // Rejoin the line we likely just split in two by chunking the file var aggregate = this._partialLine + chunk; this._partialLine = ''; - var results = this._handle.parse(aggregate, this._baseIndex, !this._finished); if (this._handle.paused() || this._handle.aborted()) { @@ -1048,7 +1057,7 @@ License: MIT { var quoteChar = _config.quoteChar || '"'; if (!_config.newline) - _config.newline = guessLineEndings(input, quoteChar); + _config.newline = this.guessLineEndings(input, quoteChar); _delimiterError = false; if (!_config.delimiter) @@ -1119,6 +1128,32 @@ License: MIT _input = ''; }; + this.guessLineEndings = function(input, quoteChar) + { + input = input.substr(0, 1024 * 1024); // max length 1 MB + // Replace all the text inside quotes + var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm'); + input = input.replace(re, ''); + + var r = input.split('\r'); + + var n = input.split('\n'); + + var nAppearsFirst = (n.length > 1 && n[0].length < r[0].length); + + if (r.length === 1 || nAppearsFirst) + return '\n'; + + var numWithN = 0; + for (var i = 0; i < r.length; i++) + { + if (r[i][0] === '\n') + numWithN++; + } + + return numWithN >= r.length / 2 ? '\r\n' : '\r'; + }; + function testEmptyLine(s) { return _config.skipEmptyLines === 'greedy' ? s.join('').trim() === '' : s.length === 1 && s[0].length === 0; } @@ -1321,32 +1356,6 @@ License: MIT }; } - function guessLineEndings(input, quoteChar) - { - input = input.substr(0, 1024 * 1024); // max length 1 MB - // Replace all the text inside quotes - var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm'); - input = input.replace(re, ''); - - var r = input.split('\r'); - - var n = input.split('\n'); - - var nAppearsFirst = (n.length > 1 && n[0].length < r[0].length); - - if (r.length === 1 || nAppearsFirst) - return '\n'; - - var numWithN = 0; - for (var i = 0; i < r.length; i++) - { - if (r[i][0] === '\n') - numWithN++; - } - - return numWithN >= r.length / 2 ? '\r\n' : '\r'; - } - function addError(type, code, msg, row) { _results.errors.push({ diff --git a/papaparse.js b/papaparse.js index 4bc3dc37..8a786649 100755 --- a/papaparse.js +++ b/papaparse.js @@ -511,6 +511,16 @@ License: MIT this.parseChunk = function(chunk, isFakeChunk) { // First chunk pre-processing + const skipFirstNLines = parseInt(this._config.skipFirstNLines) || 0; + if (this.isFirstChunk && skipFirstNLines > 0) { + let _newline = this._config.newline; + if (!_newline) { + const quoteChar = this._config.quoteChar || '"'; + _newline = this._handle.guessLineEndings(chunk, quoteChar); + } + const splitChunk = chunk.split(_newline); + chunk = [...splitChunk.slice(skipFirstNLines)].join(_newline); + } if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) { var modifiedChunk = this._config.beforeFirstChunk(chunk); @@ -522,22 +532,6 @@ License: MIT // Rejoin the line we likely just split in two by chunking the file var aggregate = this._partialLine + chunk; - this._pendingSkip = parseInt(this._config.skipFirstNLines) || 0; - this._skipHeader = 0; - if (this._config.header) { - this._skipHeader++; - } - if (this._pendingSkip > 0) { - var splitChunk = aggregate.split('\n'); - var currentChunkLength = splitChunk.length; - if (currentChunkLength <= this._pendingSkip) { - aggregate = this._partialLine; - } - else{ - aggregate = this._partialLine + [...splitChunk.slice(0, this._skipHeader), ...splitChunk.slice(this._skipHeader + this._pendingSkip)].join('\n'); - } - this._pendingSkip -= currentChunkLength; - } this._partialLine = ''; var results = this._handle.parse(aggregate, this._baseIndex, !this._finished); @@ -1093,7 +1087,7 @@ License: MIT { var quoteChar = _config.quoteChar || '"'; if (!_config.newline) - _config.newline = guessLineEndings(input, quoteChar); + _config.newline = this.guessLineEndings(input, quoteChar); _delimiterError = false; if (!_config.delimiter) @@ -1167,6 +1161,32 @@ License: MIT _input = ''; }; + this.guessLineEndings = function(input, quoteChar) + { + input = input.substring(0, 1024 * 1024); // max length 1 MB + // Replace all the text inside quotes + var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm'); + input = input.replace(re, ''); + + var r = input.split('\r'); + + var n = input.split('\n'); + + var nAppearsFirst = (n.length > 1 && n[0].length < r[0].length); + + if (r.length === 1 || nAppearsFirst) + return '\n'; + + var numWithN = 0; + for (var i = 0; i < r.length; i++) + { + if (r[i][0] === '\n') + numWithN++; + } + + return numWithN >= r.length / 2 ? '\r\n' : '\r'; + }; + function testEmptyLine(s) { return _config.skipEmptyLines === 'greedy' ? s.join('').trim() === '' : s.length === 1 && s[0].length === 0; } @@ -1373,32 +1393,6 @@ License: MIT }; } - function guessLineEndings(input, quoteChar) - { - input = input.substring(0, 1024 * 1024); // max length 1 MB - // Replace all the text inside quotes - var re = new RegExp(escapeRegExp(quoteChar) + '([^]*?)' + escapeRegExp(quoteChar), 'gm'); - input = input.replace(re, ''); - - var r = input.split('\r'); - - var n = input.split('\n'); - - var nAppearsFirst = (n.length > 1 && n[0].length < r[0].length); - - if (r.length === 1 || nAppearsFirst) - return '\n'; - - var numWithN = 0; - for (var i = 0; i < r.length; i++) - { - if (r[i][0] === '\n') - numWithN++; - } - - return numWithN >= r.length / 2 ? '\r\n' : '\r'; - } - function addError(type, code, msg, row) { var error = { diff --git a/tests/test-cases.js b/tests/test-cases.js index 68f367fc..661dcb94 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1585,11 +1585,11 @@ var PARSE_TESTS = [ } }, { - description: "Skip First N number of lines , with header and 3 rows", - input: 'a,b,c,d\n1,2,3,4\n4,5,6,7', + description: "Skip First N number of lines , with header and 2 rows", + input: 'to-be-ignored\na,b,c,d\n1,2,3,4', config: { header: true, skipFirstNLines: 1 }, expected: { - data: [{a: '4', b: '5', c: '6', d: '7'}], + data: [{a: '1', b: '2', c: '3', d: '4'}], errors: [] } }, @@ -1610,6 +1610,15 @@ var PARSE_TESTS = [ data: [['a','b','c','d'],['1','2','3','4'],['4','5','6','7']], errors: [] } + }, + { + description: "Skip first 2 lines , with custom newline character", + input: 'skip-this\rskip-this\r1,2,3,4', + config: { header: false, skipFirstNLines: 2, newline: '\r' }, + expected: { + data: [['1','2','3','4']], + errors: [] + } } ]; From 4af6882e0ea91b0baad30345f190b38d50e7535e Mon Sep 17 00:00:00 2001 From: August Marowski Date: Thu, 12 Sep 2024 21:43:03 +0200 Subject: [PATCH 44/61] Refactor header renaming logic to adress #1052, #1007 (#1058) And updated test to match the improved renaming logic. --- papaparse.js | 93 +++++++++++++++++++++------------------------ tests/test-cases.js | 4 +- 2 files changed, 45 insertions(+), 52 deletions(-) diff --git a/papaparse.js b/papaparse.js index 8a786649..26169868 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1480,61 +1480,14 @@ License: MIT if (!input) return returnable(); - // Rename headers if there are duplicates - var firstLine; - if (config.header && !baseIndex) - { - firstLine = input.split(newline)[0]; - var headers = firstLine.split(delim); - var separator = '_'; - var headerMap = new Set(); - var headerCount = {}; - var duplicateHeaders = false; - - // Using old-style 'for' loop to avoid prototype pollution that would be picked up with 'var j in headers' - for (var j = 0; j < headers.length; j++) { - var header = headers[j]; - if (isFunction(config.transformHeader)) - header = config.transformHeader(header, j); - var headerName = header; - - var count = headerCount[header] || 0; - if (count > 0) { - duplicateHeaders = true; - headerName = header + separator + count; - // Initialise the variable if it hasn't been. - if (renamedHeaders === null) { - renamedHeaders = {}; - } - } - headerCount[header] = count + 1; - // In case it already exists, we add more separators - while (headerMap.has(headerName)) { - headerName = headerName + separator + count; - } - headerMap.add(headerName); - if (count > 0) { - renamedHeaders[headerName] = header; - } - } - if (duplicateHeaders) { - var editedInput = input.split(newline); - editedInput[0] = Array.from(headerMap).join(delim); - input = editedInput.join(newline); - } - } if (fastMode || (fastMode !== false && input.indexOf(quoteChar) === -1)) { var rows = input.split(newline); for (var i = 0; i < rows.length; i++) { row = rows[i]; - // use firstline as row length may be changed due to duplicated headers - if (i === 0 && firstLine !== undefined) { - cursor += firstLine.length; - }else{ - cursor += row.length; - } + cursor += row.length; + if (i !== rows.length - 1) cursor += newline.length; else if (ignoreLastRow) @@ -1729,7 +1682,6 @@ License: MIT break; } - return finish(); @@ -1789,6 +1741,47 @@ License: MIT /** Returns an object with the results, errors, and meta. */ function returnable(stopped) { + if (config.header && !baseIndex && data.length) + { + const result = data[0]; + const headerCount = {}; // To track the count of each base header + const usedHeaders = new Set(result); // To track used headers and avoid duplicates + let duplicateHeaders = false; + + for (let i = 0; i < result.length; i++) { + let header = result[i]; + if (isFunction(config.transformHeader)) + header = config.transformHeader(header, i); + + if (!headerCount[header]) { + headerCount[header] = 1; + result[i] = header; + } else { + let newHeader; + let suffixCount = headerCount[header]; + + // Find a unique new header + do { + newHeader = `${header}_${suffixCount}`; + suffixCount++; + } while (usedHeaders.has(newHeader)); + + usedHeaders.add(newHeader); // Mark this new Header as used + result[i] = newHeader; + headerCount[header]++; + duplicateHeaders = true; + if (renamedHeaders === null) { + renamedHeaders = {}; + } + renamedHeaders[newHeader] = header; + } + + usedHeaders.add(header); // Ensure the original header is marked as used + } + if (duplicateHeaders) { + console.warn('Duplicate headers found and renamed.'); + } + } return { data: data, errors: errors, diff --git a/tests/test-cases.js b/tests/test-cases.js index 661dcb94..5eaa943e 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -618,10 +618,10 @@ var CORE_PARSER_TESTS = [ input: 'c,c,c,c_1\n1,2,3,4', config: { header: true }, expected: { - data: [['c', 'c_1', 'c_2', 'c_1_0'], ['1', '2', '3', '4']], + data: [['c', 'c_2', 'c_3', 'c_1'], ['1', '2', '3', '4']], errors: [], meta: { - renamedHeaders: {c_1: 'c', c_2: 'c'}, + renamedHeaders: {c_2: 'c', c_3: 'c'}, cursor: 17 } } From f490abb991216be6792757bc470540ef51f6015d Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 9 Jan 2025 10:07:19 +0100 Subject: [PATCH 45/61] Remove ES6 features to allow minifying paparse file --- papaparse.js | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/papaparse.js b/papaparse.js index 26169868..76bf50fa 100755 --- a/papaparse.js +++ b/papaparse.js @@ -511,15 +511,15 @@ License: MIT this.parseChunk = function(chunk, isFakeChunk) { // First chunk pre-processing - const skipFirstNLines = parseInt(this._config.skipFirstNLines) || 0; + var skipFirstNLines = parseInt(this._config.skipFirstNLines) || 0; if (this.isFirstChunk && skipFirstNLines > 0) { - let _newline = this._config.newline; + var _newline = this._config.newline; if (!_newline) { - const quoteChar = this._config.quoteChar || '"'; + var quoteChar = this._config.quoteChar || '"'; _newline = this._handle.guessLineEndings(chunk, quoteChar); } - const splitChunk = chunk.split(_newline); - chunk = [...splitChunk.slice(skipFirstNLines)].join(_newline); + var splitChunk = chunk.split(_newline); + chunk = splitChunk.slice(skipFirstNLines).join(_newline); } if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) { @@ -1743,13 +1743,13 @@ License: MIT { if (config.header && !baseIndex && data.length) { - const result = data[0]; - const headerCount = {}; // To track the count of each base header - const usedHeaders = new Set(result); // To track used headers and avoid duplicates - let duplicateHeaders = false; + var result = data[0]; + var headerCount = {}; // To track the count of each base header + var usedHeaders = new Set(result); // To track used headers and avoid duplicates + var duplicateHeaders = false; - for (let i = 0; i < result.length; i++) { - let header = result[i]; + for (var i = 0; i < result.length; i++) { + var header = result[i]; if (isFunction(config.transformHeader)) header = config.transformHeader(header, i); @@ -1757,12 +1757,12 @@ License: MIT headerCount[header] = 1; result[i] = header; } else { - let newHeader; - let suffixCount = headerCount[header]; + var newHeader; + var suffixCount = headerCount[header]; // Find a unique new header do { - newHeader = `${header}_${suffixCount}`; + newHeader = header + '_' + suffixCount; suffixCount++; } while (usedHeaders.has(newHeader)); From 338af86bde63d0d208ef22582f55fa129a5431d7 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Thu, 9 Jan 2025 10:08:52 +0100 Subject: [PATCH 46/61] Minor version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index a1298ab1..5ff48a1a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.4.1", + "version": "5.5.0", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index 76bf50fa..303491c6 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.4.1 +v5.5.0 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index eeaf9834..2b6af671 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v5.4.1 +v5.5.0 https://github.com/mholt/PapaParse License: MIT */ -!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function s(){"use strict";var f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{};var n=!f.document&&!!f.postMessage,o=f.IS_PAPA_WORKER||!1,a={},u=0,b={parse:function(e,t){var r=(t=t||{}).dynamicTyping||!1;J(r)&&(t.dynamicTypingFunction=r,r={});if(t.dynamicTyping=r,t.transform=!!J(t.transform)&&t.transform,t.worker&&b.WORKERS_SUPPORTED){var i=function(){if(!b.WORKERS_SUPPORTED)return!1;var e=(r=f.URL||f.webkitURL||null,i=s.toString(),b.BLOB_URL||(b.BLOB_URL=r.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ","(",i,")();"],{type:"text/javascript"})))),t=new f.Worker(e);var r,i;return t.onmessage=_,t.id=u++,a[t.id]=t}();return i.userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=J(t.step),t.chunk=J(t.chunk),t.complete=J(t.complete),t.error=J(t.error),delete t.worker,void i.postMessage({input:e,config:t,workerId:i.id})}var n=null;b.NODE_STREAM_INPUT,"string"==typeof e?(e=function(e){if(65279===e.charCodeAt(0))return e.slice(1);return e}(e),n=t.download?new l(t):new p(t)):!0===e.readable&&J(e.read)&&J(e.on)?n=new g(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new c(t));return n.stream(e)},unparse:function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,r=!1,i=null,o=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||b.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter);("boolean"==typeof t.quotes||"function"==typeof t.quotes||Array.isArray(t.quotes))&&(n=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(r=t.skipEmptyLines);"string"==typeof t.newline&&(y=t.newline);"string"==typeof t.quoteChar&&(s=t.quoteChar);"boolean"==typeof t.header&&(_=t.header);if(Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");i=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s);("boolean"==typeof t.escapeFormulae||t.escapeFormulae instanceof RegExp)&&(o=t.escapeFormulae instanceof RegExp?t.escapeFormulae:/^[=+\-@\t\r].*$/)}();var u=new RegExp(Q(s),"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return h(null,e,r);if("object"==typeof e[0])return h(i||Object.keys(e[0]),e,r)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||i),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),h(e.fields||[],e.data||[],r);throw new Error("Unable to serialize unrecognized input");function h(e,t,r){var i="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(o)f.postMessage({results:n,workerId:b.WORKER_ID,finished:a});else if(J(this._config.chunk)&&!t){if(this._config.chunk(n,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);n=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(n.data),this._completeResults.errors=this._completeResults.errors.concat(n.errors),this._completeResults.meta=n.meta),this._completed||!a||!J(this._config.complete)||n&&n.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),a||n&&n.meta.paused||this._nextChunk(),n}this._halted=!0},this._sendError=function(e){J(this._config.error)?this._config.error(e):o&&this._config.error&&f.postMessage({workerId:b.WORKER_ID,error:e,finished:!1})}}function l(e){var i;(e=e||{}).chunkSize||(e.chunkSize=b.RemoteChunkSize),h.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(i=new XMLHttpRequest,this._config.withCredentials&&(i.withCredentials=this._config.withCredentials),n||(i.onload=v(this._chunkLoaded,this),i.onerror=v(this._chunkError,this)),i.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)i.setRequestHeader(t,e[t])}if(this._config.chunkSize){var r=this._start+this._config.chunkSize-1;i.setRequestHeader("Range","bytes="+this._start+"-"+r)}try{i.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===i.status&&this._chunkError()}},this._chunkLoaded=function(){4===i.readyState&&(i.status<200||400<=i.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:i.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(i),this.parseChunk(i.responseText)))},this._chunkError=function(e){var t=i.statusText||e;this._sendError(new Error(t))}}function c(e){var i,n;(e=e||{}).chunkSize||(e.chunkSize=b.LocalChunkSize),h.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((i=new FileReader).onload=v(this._chunkLoaded,this),i.onerror=v(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function p(e){var r;h.call(this,e=e||{}),this.stream=function(e){return r=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=r.substring(0,t),r=r.substring(t)):(e=r,r=""),this._finished=!r,this.parseChunk(e)}}}function g(e){h.call(this,e=e||{});var t=[],r=!0,i=!1;this.pause=function(){h.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){h.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){i&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):r=!0},this._streamData=v(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),r&&(r=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=v(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=v(function(){this._streamCleanUp(),i=!0,this._streamData("")},this),this._streamCleanUp=v(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function r(m){var a,o,u,i=Math.pow(2,53),n=-i,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,h=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,t=this,r=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(J(m.step)){var p=m.step;m.step=function(e){if(c=e,_())g();else{if(g(),0===c.data.length)return;r+=e.data.length,m.preview&&r>m.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){return c&&u&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+b.DefaultDelimiter+"'"),u=!1),m.skipEmptyLines&&(c.data=c.data.filter(function(e){return!y(e)})),_()&&function(){if(!c)return;function e(e,t){J(m.transformHeader)&&(e=m.transformHeader(e,t)),l.push(e)}if(Array.isArray(c.data[0])){for(var t=0;_()&&t=l.length?"__parsed_extra":l[r]),m.transform&&(s=m.transform(s,n)),s=v(n,s),"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return m.header&&(r>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+r,f+t):r=i.length/2?"\r\n":"\r"}(e,i)),u=!1,m.delimiter)J(m.delimiter)&&(m.delimiter=m.delimiter(e),c.meta.delimiter=m.delimiter);else{var n=function(e,t,r,i,n){var s,a,o,u;n=n||[",","\t","|",";",b.RECORD_SEP,b.UNIT_SEP];for(var h=0;h=N)return L(!0)}else for(S=W,W++;;){if(-1===(S=i.indexOf(z,S+1)))return r||h.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:u.length,index:W}),T();if(S===n-1)return T(i.substring(W,S).replace(C,z));if(z!==K||i[S+1]!==K){if(z===K||0===S||i[S-1]!==K){-1!==w&&w=N)return L(!0);break}h.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:u.length,index:W}),S++}}else S++}return T();function I(e){u.push(e),d=W}function A(e){var t=0;if(-1!==e){var r=i.substring(S+1,e);r&&""===r.trim()&&(t=r.length)}return t}function T(e){return r||(void 0===e&&(e=i.substring(W)),f.push(e),W=n,I(f),o&&F()),L()}function D(e){W=e,I(f),f=[],R=i.indexOf(P,W)}function L(e){return{data:u,errors:h,meta:{delimiter:M,linebreak:P,aborted:H,truncated:!!e,cursor:d+(t||0)}}}function F(){q(L()),u=[],h=[]}},this.abort=function(){H=!0},this.getCharIndex=function(){return W}}function _(e){var t=e.data,r=a[t.workerId],i=!1;if(t.error)r.userError(t.error,t.file);else if(t.results&&t.results.data){var n={abort:function(){i=!0,m(t.workerId,{data:[],errors:[],meta:{aborted:!0}})},pause:y,resume:y};if(J(r.userStep)){for(var s=0;s=this._config.preview;if(d)f.postMessage({results:o,workerId:w.WORKER_ID,finished:u});else if(U(this._config.chunk)&&!t){if(this._config.chunk(o,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);o=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(o.data),this._completeResults.errors=this._completeResults.errors.concat(o.errors),this._completeResults.meta=o.meta),this._completed||!u||!U(this._config.complete)||o&&o.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),u||o&&o.meta.paused||this._nextChunk(),o}this._halted=!0},this._sendError=function(e){U(this._config.error)?this._config.error(e):d&&this._config.error&&f.postMessage({workerId:w.WORKER_ID,error:e,finished:!1})}}function u(e){var r;(e=e||{}).chunkSize||(e.chunkSize=w.RemoteChunkSize),h.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=v(this._chunkLoaded,this),r.onerror=v(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i)}try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:r.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function c(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=w.LocalChunkSize),h.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=v(this._chunkLoaded,this),r.onerror=v(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function p(e){var i;h.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=i.substring(0,t),i=i.substring(t)):(e=i,i=""),this._finished=!i,this.parseChunk(e)}}}function g(e){h.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){h.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){h.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=v(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=v(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=v(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=v(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var a,o,h,r=Math.pow(2,53),n=-r,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,u=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,t=this,i=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(U(m.step)){var p=m.step;m.step=function(e){if(c=e,_())g();else{if(g(),0===c.data.length)return;i+=e.data.length,m.preview&&i>m.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){return c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+w.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines&&(c.data=c.data.filter(function(e){return!y(e)})),_()&&function(){if(!c)return;function e(e,t){U(m.transformHeader)&&(e=m.transformHeader(e,t)),l.push(e)}if(Array.isArray(c.data[0])){for(var t=0;_()&&t=l.length?"__parsed_extra":l[i]),m.transform&&(s=m.transform(s,n)),s=v(n,s),"__parsed_extra"===n?(r[n]=r[n]||[],r[n].push(s)):r[n]=s}return m.header&&(i>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+i,f+t):i=r.length/2?"\r\n":"\r"}}function P(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function E(S){var O,x=(S=S||{}).delimiter,I=S.newline,A=S.comments,T=S.step,D=S.preview,L=S.fastMode,F=null,j=O=void 0===S.quoteChar||null===S.quoteChar?'"':S.quoteChar;if(void 0!==S.escapeChar&&(j=S.escapeChar),("string"!=typeof x||-1=D)return C(!0)}else for(m=z,z++;;){if(-1===(m=r.indexOf(O,m+1)))return t||d.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:f.length,index:z}),E();if(m===i-1)return E(r.substring(z,m).replace(_,O));if(O!==j||r[m+1]!==j){if(O===j||0===m||r[m-1]!==j){-1!==p&&p=D)return C(!0);break}d.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:f.length,index:z}),m++}}else m++}return E();function k(e){f.push(e),l=z}function w(e){var t=0;if(-1!==e){var i=r.substring(m+1,e);i&&""===i.trim()&&(t=i.length)}return t}function E(e){return t||(void 0===e&&(e=r.substring(z)),o.push(e),z=i,k(o),a&&R()),C()}function b(e){z=e,k(o),o=[],g=r.indexOf(I,z)}function C(e){if(S.header&&!u&&f.length){for(var t=f[0],i={},r=new Set(t),n=!1,s=0;s Date: Fri, 10 Jan 2025 09:16:53 +0100 Subject: [PATCH 47/61] Run build script in CI --- .github/workflows/node.js.yml | 2 +- package.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 0fccdf05..0d121858 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -26,4 +26,4 @@ jobs: with: node-version: ${{ matrix.node-version }} - run: npm install - - run: npm test + - run: npm test && npm run build diff --git a/package.json b/package.json index 5ff48a1a..1baab9e3 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ }, "scripts": { "lint": "eslint --no-ignore papaparse.js Gruntfile.js .eslintrc.js 'tests/**/*.js'", + "build": "grunt build", "test-browser": "node tests/test.js", "test-mocha-headless-chrome": "node tests/test.js --mocha-headless-chrome", "test-node": "mocha tests/node-tests.js tests/test-cases.js", From 9c42a1e1d10166c97959aebe16ccc9187c6cd606 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Fri, 10 Jan 2025 09:21:10 +0100 Subject: [PATCH 48/61] Update grunt-contrib-uglify version --- package.json | 4 ++-- papaparse.min.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 1baab9e3..990f5b43 100644 --- a/package.json +++ b/package.json @@ -40,8 +40,8 @@ "chai": "^4.2.0", "connect": "^3.3.3", "eslint": "^4.19.1", - "grunt": "^1.0.2", - "grunt-contrib-uglify": "^3.3.0", + "grunt": "^1.5.2", + "grunt-contrib-uglify": "^5.2.0", "mocha": "^5.2.0", "mocha-headless-chrome": "^4.0.0", "open": "7.0.0", diff --git a/papaparse.min.js b/papaparse.min.js index 2b6af671..08d0043f 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -4,4 +4,4 @@ v5.5.0 https://github.com/mholt/PapaParse License: MIT */ -!function(e,t){"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()}(this,function s(){"use strict";var f="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==f?f:{};var n=!f.document&&!!f.postMessage,d=f.IS_PAPA_WORKER||!1,a={},o=0,w={parse:function(e,t){var i=(t=t||{}).dynamicTyping||!1;U(i)&&(t.dynamicTypingFunction=i,i={});if(t.dynamicTyping=i,t.transform=!!U(t.transform)&&t.transform,t.worker&&w.WORKERS_SUPPORTED){var r=function(){if(!w.WORKERS_SUPPORTED)return!1;var e=(i=f.URL||f.webkitURL||null,r=s.toString(),w.BLOB_URL||(w.BLOB_URL=i.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ","(",r,")();"],{type:"text/javascript"})))),t=new f.Worker(e);var i,r;return t.onmessage=_,t.id=o++,a[t.id]=t}();return r.userStep=t.step,r.userChunk=t.chunk,r.userComplete=t.complete,r.userError=t.error,t.step=U(t.step),t.chunk=U(t.chunk),t.complete=U(t.complete),t.error=U(t.error),delete t.worker,void r.postMessage({input:e,config:t,workerId:r.id})}var n=null;w.NODE_STREAM_INPUT,"string"==typeof e?(e=function(e){if(65279===e.charCodeAt(0))return e.slice(1);return e}(e),n=t.download?new u(t):new p(t)):!0===e.readable&&U(e.read)&&U(e.on)?n=new g(t):(f.File&&e instanceof File||e instanceof Object)&&(n=new c(t));return n.stream(e)},unparse:function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,i=!1,r=null,o=!1;!function(){if("object"!=typeof t)return;"string"!=typeof t.delimiter||w.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter);("boolean"==typeof t.quotes||"function"==typeof t.quotes||Array.isArray(t.quotes))&&(n=t.quotes);"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines);"string"==typeof t.newline&&(y=t.newline);"string"==typeof t.quoteChar&&(s=t.quoteChar);"boolean"==typeof t.header&&(_=t.header);if(Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s);t.escapeFormulae instanceof RegExp?o=t.escapeFormulae:"boolean"==typeof t.escapeFormulae&&t.escapeFormulae&&(o=/^[=+\-@\t\r].*$/)}();var h=new RegExp(P(s),"g");"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||Object.keys(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||r),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e,t,i){var r="";"string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t));var n=Array.isArray(e)&&0=this._config.preview;if(d)f.postMessage({results:o,workerId:w.WORKER_ID,finished:u});else if(U(this._config.chunk)&&!t){if(this._config.chunk(o,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);o=void 0,this._completeResults=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(o.data),this._completeResults.errors=this._completeResults.errors.concat(o.errors),this._completeResults.meta=o.meta),this._completed||!u||!U(this._config.complete)||o&&o.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),u||o&&o.meta.paused||this._nextChunk(),o}this._halted=!0},this._sendError=function(e){U(this._config.error)?this._config.error(e):d&&this._config.error&&f.postMessage({workerId:w.WORKER_ID,error:e,finished:!1})}}function u(e){var r;(e=e||{}).chunkSize||(e.chunkSize=w.RemoteChunkSize),h.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=v(this._chunkLoaded,this),r.onerror=v(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e=this._config.downloadRequestHeaders;for(var t in e)r.setRequestHeader(t,e[t])}if(this._config.chunkSize){var i=this._start+this._config.chunkSize-1;r.setRequestHeader("Range","bytes="+this._start+"-"+i)}try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize?this._config.chunkSize:r.responseText.length,this._finished=!this._config.chunkSize||this._start>=function(e){var t=e.getResponseHeader("Content-Range");if(null===t)return-1;return parseInt(t.substring(t.lastIndexOf("/")+1))}(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){var t=r.statusText||e;this._sendError(new Error(t))}}function c(e){var r,n;(e=e||{}).chunkSize||(e.chunkSize=w.LocalChunkSize),h.call(this,e);var s="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,n=e.slice||e.webkitSlice||e.mozSlice,s?((r=new FileReader).onload=v(this._chunkLoaded,this),r.onerror=v(this._chunkError,this)):r=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(r.error)}}function p(e){var i;h.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){if(!this._finished){var e,t=this._config.chunkSize;return t?(e=i.substring(0,t),i=i.substring(t)):(e=i,i=""),this._finished=!i,this.parseChunk(e)}}}function g(e){h.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){h.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){h.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=v(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=v(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=v(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=v(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var a,o,h,r=Math.pow(2,53),n=-r,s=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,u=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,t=this,i=0,f=0,d=!1,e=!1,l=[],c={data:[],errors:[],meta:{}};if(U(m.step)){var p=m.step;m.step=function(e){if(c=e,_())g();else{if(g(),0===c.data.length)return;i+=e.data.length,m.preview&&i>m.preview?o.abort():(c.data=c.data[0],p(c,t))}}}function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){return c&&h&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+w.DefaultDelimiter+"'"),h=!1),m.skipEmptyLines&&(c.data=c.data.filter(function(e){return!y(e)})),_()&&function(){if(!c)return;function e(e,t){U(m.transformHeader)&&(e=m.transformHeader(e,t)),l.push(e)}if(Array.isArray(c.data[0])){for(var t=0;_()&&t=l.length?"__parsed_extra":l[i]),m.transform&&(s=m.transform(s,n)),s=v(n,s),"__parsed_extra"===n?(r[n]=r[n]||[],r[n].push(s)):r[n]=s}return m.header&&(i>l.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+l.length+" fields but parsed "+i,f+t):i=r.length/2?"\r\n":"\r"}}function P(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function E(S){var O,x=(S=S||{}).delimiter,I=S.newline,A=S.comments,T=S.step,D=S.preview,L=S.fastMode,F=null,j=O=void 0===S.quoteChar||null===S.quoteChar?'"':S.quoteChar;if(void 0!==S.escapeChar&&(j=S.escapeChar),("string"!=typeof x||-1=D)return C(!0)}else for(m=z,z++;;){if(-1===(m=r.indexOf(O,m+1)))return t||d.push({type:"Quotes",code:"MissingQuotes",message:"Quoted field unterminated",row:f.length,index:z}),E();if(m===i-1)return E(r.substring(z,m).replace(_,O));if(O!==j||r[m+1]!==j){if(O===j||0===m||r[m-1]!==j){-1!==p&&p=D)return C(!0);break}d.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:f.length,index:z}),m++}}else m++}return E();function k(e){f.push(e),l=z}function w(e){var t=0;if(-1!==e){var i=r.substring(m+1,e);i&&""===i.trim()&&(t=i.length)}return t}function E(e){return t||(void 0===e&&(e=r.substring(z)),o.push(e),z=i,k(o),a&&R()),C()}function b(e){z=e,k(o),o=[],g=r.indexOf(I,z)}function C(e){if(S.header&&!u&&f.length){for(var t=f[0],i={},r=new Set(t),n=!1,s=0;s{"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()})(this,function r(){var s="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==s?s:{};var d,n=!s.document&&!!s.postMessage,a=s.IS_PAPA_WORKER||!1,o={},h=0,v={};function u(e){this._handle=null,this._finished=!1,this._completed=!1,this._halted=!1,this._input=null,this._baseIndex=0,this._partialLine="",this._rowCount=0,this._start=0,this._nextChunk=null,this.isFirstChunk=!0,this._completeResults={data:[],errors:[],meta:{}},function(e){var t=w(e);t.chunkSize=parseInt(t.chunkSize),e.step||e.chunk||(t.chunkSize=null);this._handle=new i(t),(this._handle.streamer=this)._config=t}.call(this,e),this.parseChunk=function(e,t){var i=parseInt(this._config.skipFirstNLines)||0,i=(this.isFirstChunk&&0=this._config.preview);if(a)s.postMessage({results:r,workerId:v.WORKER_ID,finished:e});else if(P(this._config.chunk)&&!t){if(this._config.chunk(r,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);this._completeResults=r=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(r.data),this._completeResults.errors=this._completeResults.errors.concat(r.errors),this._completeResults.meta=r.meta),this._completed||!e||!P(this._config.complete)||r&&r.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),e||r&&r.meta.paused||this._nextChunk(),r}this._halted=!0},this._sendError=function(e){P(this._config.error)?this._config.error(e):a&&this._config.error&&s.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function f(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e,t=this._config.downloadRequestHeaders;for(e in t)r.setRequestHeader(e,t[e])}var i;this._config.chunkSize&&(i=this._start+this._config.chunkSize-1,r.setRequestHeader("Range","bytes="+this._start+"-"+i));try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize||r.responseText.length,this._finished=!this._config.chunkSize||this._start>=(e=>null!==(e=e.getResponseHeader("Content-Range"))?parseInt(e.substring(e.lastIndexOf("/")+1)):-1)(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){e=r.statusText||e;this._sendError(new Error(e))}}function l(e){(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),u.call(this,e);var i,r,n="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,r=e.slice||e.webkitSlice||e.mozSlice,n?((i=new FileReader).onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function c(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){var e,t;if(!this._finished)return e=this._config.chunkSize,i=e?(t=i.substring(0,e),i.substring(e)):(t=i,""),this._finished=!i,this.parseChunk(t)}}function p(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var n,s,a,t,o=Math.pow(2,53),h=-o,u=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,d=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,i=this,r=0,f=0,l=!1,e=!1,c=[],p={data:[],errors:[],meta:{}};function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(p&&a&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),a=!1),m.skipEmptyLines&&(p.data=p.data.filter(function(e){return!y(e)})),_()){if(p)if(Array.isArray(p.data[0])){for(var e=0;_()&&e(e=>(m.dynamicTypingFunction&&void 0===m.dynamicTyping[e]&&(m.dynamicTyping[e]=m.dynamicTypingFunction(e)),!0===(m.dynamicTyping[e]||m.dynamicTyping)))(e)?"true"===t||"TRUE"===t||"false"!==t&&"FALSE"!==t&&((e=>{if(u.test(e)){e=parseFloat(e);if(h=c.length?"__parsed_extra":c[r]:n,s=m.transform?m.transform(s,n):s);"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return m.header&&(r>c.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+c.length+" fields but parsed "+r,f+t):rm.preview?s.abort():(p.data=p.data[0],t(p,i))))}),this.parse=function(e,t,i){var r=m.quoteChar||'"',r=(m.newline||(m.newline=this.guessLineEndings(e,r)),a=!1,m.delimiter?P(m.delimiter)&&(m.delimiter=m.delimiter(e),p.meta.delimiter=m.delimiter):((r=((e,t,i,r,n)=>{var s,a,o,h;n=n||[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP];for(var u=0;u=i.length/2?"\r\n":"\r"}}function M(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function E(C){var S=(C=C||{}).delimiter,O=C.newline,x=C.comments,I=C.step,A=C.preview,T=C.fastMode,D=null,L=null==C.quoteChar?'"':C.quoteChar,F=L;if(void 0!==C.escapeChar&&(F=C.escapeChar),("string"!=typeof S||-1=A)return b(!0);break}f.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:d.length,index:j}),m++}}else if(x&&0===o.length&&i.substring(j,j+s)===x){if(-1===g)return b();j=g+n,g=i.indexOf(O,j),p=i.indexOf(S,j)}else if(-1!==p&&(p=A)return b(!0)}return E();function k(e){d.push(e),l=j}function v(e){var t=0;return t=-1!==e&&(e=i.substring(m+1,e))&&""===e.trim()?e.length:t}function E(e){return t||(void 0===e&&(e=i.substring(j)),o.push(e),j=r,k(o),a&&R()),b()}function w(e){j=e,k(o),o=[],g=i.indexOf(O,j)}function b(e){if(C.header&&!u&&d.length){for(var t=d[0],i={},r=new Set(t),n=!1,s=0;s65279!==e.charCodeAt(0)?e:e.slice(1))(e),i=new(t.download?f:c)(t)):!0===e.readable&&P(e.read)&&P(e.on)?i=new p(t):(s.File&&e instanceof File||e instanceof Object)&&(i=new l(t)),i.stream(e);(i=(()=>{var e;return!!v.WORKERS_SUPPORTED&&(e=(()=>{var e=s.URL||s.webkitURL||null,t=r.toString();return v.BLOB_URL||(v.BLOB_URL=e.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ","(",t,")();"],{type:"text/javascript"})))})(),(e=new s.Worker(e)).onmessage=g,e.id=h++,o[e.id]=e)})()).userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=P(t.step),t.chunk=P(t.chunk),t.complete=P(t.complete),t.error=P(t.error),delete t.worker,i.postMessage({input:e,config:t,workerId:i.id})},v.unparse=function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,i=!1,r=null,o=!1,h=((()=>{if("object"==typeof t){if("string"!=typeof t.delimiter||v.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter),"boolean"!=typeof t.quotes&&"function"!=typeof t.quotes&&!Array.isArray(t.quotes)||(n=t.quotes),"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines),"string"==typeof t.newline&&(y=t.newline),"string"==typeof t.quoteChar&&(s=t.quoteChar),"boolean"==typeof t.header&&(_=t.header),Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s),t.escapeFormulae instanceof RegExp?o=t.escapeFormulae:"boolean"==typeof t.escapeFormulae&&t.escapeFormulae&&(o=/^[=+\-@\t\r].*$/)}})(),new RegExp(M(s),"g"));"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||Object.keys(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||r),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e,t,i){var r="",n=("string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t)),Array.isArray(e)&&0{for(var i=0;i Date: Fri, 10 Jan 2025 09:21:17 +0100 Subject: [PATCH 49/61] Revert "Remove ES6 features to allow minifying paparse file" This reverts commit f490abb991216be6792757bc470540ef51f6015d. --- papaparse.js | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/papaparse.js b/papaparse.js index 303491c6..670bb906 100755 --- a/papaparse.js +++ b/papaparse.js @@ -511,15 +511,15 @@ License: MIT this.parseChunk = function(chunk, isFakeChunk) { // First chunk pre-processing - var skipFirstNLines = parseInt(this._config.skipFirstNLines) || 0; + const skipFirstNLines = parseInt(this._config.skipFirstNLines) || 0; if (this.isFirstChunk && skipFirstNLines > 0) { - var _newline = this._config.newline; + let _newline = this._config.newline; if (!_newline) { - var quoteChar = this._config.quoteChar || '"'; + const quoteChar = this._config.quoteChar || '"'; _newline = this._handle.guessLineEndings(chunk, quoteChar); } - var splitChunk = chunk.split(_newline); - chunk = splitChunk.slice(skipFirstNLines).join(_newline); + const splitChunk = chunk.split(_newline); + chunk = [...splitChunk.slice(skipFirstNLines)].join(_newline); } if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) { @@ -1743,13 +1743,13 @@ License: MIT { if (config.header && !baseIndex && data.length) { - var result = data[0]; - var headerCount = {}; // To track the count of each base header - var usedHeaders = new Set(result); // To track used headers and avoid duplicates - var duplicateHeaders = false; + const result = data[0]; + const headerCount = {}; // To track the count of each base header + const usedHeaders = new Set(result); // To track used headers and avoid duplicates + let duplicateHeaders = false; - for (var i = 0; i < result.length; i++) { - var header = result[i]; + for (let i = 0; i < result.length; i++) { + let header = result[i]; if (isFunction(config.transformHeader)) header = config.transformHeader(header, i); @@ -1757,12 +1757,12 @@ License: MIT headerCount[header] = 1; result[i] = header; } else { - var newHeader; - var suffixCount = headerCount[header]; + let newHeader; + let suffixCount = headerCount[header]; // Find a unique new header do { - newHeader = header + '_' + suffixCount; + newHeader = `${header}_${suffixCount}`; suffixCount++; } while (usedHeaders.has(newHeader)); From d0f58aa3c662b7460b2da093bae3f81ef2cb5e30 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Fri, 10 Jan 2025 09:28:53 +0100 Subject: [PATCH 50/61] Patch version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 990f5b43..5eece0aa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.5.0", + "version": "5.5.1", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index 670bb906..c1d63346 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.5.0 +v5.5.1 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index 08d0043f..bb9b4cce 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v5.5.0 +v5.5.1 https://github.com/mholt/PapaParse License: MIT */ -((e,t)=>{"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()})(this,function r(){var s="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==s?s:{};var d,n=!s.document&&!!s.postMessage,a=s.IS_PAPA_WORKER||!1,o={},h=0,v={};function u(e){this._handle=null,this._finished=!1,this._completed=!1,this._halted=!1,this._input=null,this._baseIndex=0,this._partialLine="",this._rowCount=0,this._start=0,this._nextChunk=null,this.isFirstChunk=!0,this._completeResults={data:[],errors:[],meta:{}},function(e){var t=w(e);t.chunkSize=parseInt(t.chunkSize),e.step||e.chunk||(t.chunkSize=null);this._handle=new i(t),(this._handle.streamer=this)._config=t}.call(this,e),this.parseChunk=function(e,t){var i=parseInt(this._config.skipFirstNLines)||0,i=(this.isFirstChunk&&0=this._config.preview);if(a)s.postMessage({results:r,workerId:v.WORKER_ID,finished:e});else if(P(this._config.chunk)&&!t){if(this._config.chunk(r,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);this._completeResults=r=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(r.data),this._completeResults.errors=this._completeResults.errors.concat(r.errors),this._completeResults.meta=r.meta),this._completed||!e||!P(this._config.complete)||r&&r.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),e||r&&r.meta.paused||this._nextChunk(),r}this._halted=!0},this._sendError=function(e){P(this._config.error)?this._config.error(e):a&&this._config.error&&s.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function f(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),u.call(this,e),this._nextChunk=n?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),n||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!n),this._config.downloadRequestHeaders){var e,t=this._config.downloadRequestHeaders;for(e in t)r.setRequestHeader(e,t[e])}var i;this._config.chunkSize&&(i=this._start+this._config.chunkSize-1,r.setRequestHeader("Range","bytes="+this._start+"-"+i));try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}n&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize||r.responseText.length,this._finished=!this._config.chunkSize||this._start>=(e=>null!==(e=e.getResponseHeader("Content-Range"))?parseInt(e.substring(e.lastIndexOf("/")+1)):-1)(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){e=r.statusText||e;this._sendError(new Error(e))}}function l(e){(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),u.call(this,e);var i,r,n="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,r=e.slice||e.webkitSlice||e.mozSlice,n?((i=new FileReader).onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function c(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){var e,t;if(!this._finished)return e=this._config.chunkSize,i=e?(t=i.substring(0,e),i.substring(e)):(t=i,""),this._finished=!i,this.parseChunk(t)}}function p(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var n,s,a,t,o=Math.pow(2,53),h=-o,u=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,d=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,i=this,r=0,f=0,l=!1,e=!1,c=[],p={data:[],errors:[],meta:{}};function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(p&&a&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),a=!1),m.skipEmptyLines&&(p.data=p.data.filter(function(e){return!y(e)})),_()){if(p)if(Array.isArray(p.data[0])){for(var e=0;_()&&e(e=>(m.dynamicTypingFunction&&void 0===m.dynamicTyping[e]&&(m.dynamicTyping[e]=m.dynamicTypingFunction(e)),!0===(m.dynamicTyping[e]||m.dynamicTyping)))(e)?"true"===t||"TRUE"===t||"false"!==t&&"FALSE"!==t&&((e=>{if(u.test(e)){e=parseFloat(e);if(h=c.length?"__parsed_extra":c[r]:n,s=m.transform?m.transform(s,n):s);"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return m.header&&(r>c.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+c.length+" fields but parsed "+r,f+t):rm.preview?s.abort():(p.data=p.data[0],t(p,i))))}),this.parse=function(e,t,i){var r=m.quoteChar||'"',r=(m.newline||(m.newline=this.guessLineEndings(e,r)),a=!1,m.delimiter?P(m.delimiter)&&(m.delimiter=m.delimiter(e),p.meta.delimiter=m.delimiter):((r=((e,t,i,r,n)=>{var s,a,o,h;n=n||[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP];for(var u=0;u=i.length/2?"\r\n":"\r"}}function M(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function E(C){var S=(C=C||{}).delimiter,O=C.newline,x=C.comments,I=C.step,A=C.preview,T=C.fastMode,D=null,L=null==C.quoteChar?'"':C.quoteChar,F=L;if(void 0!==C.escapeChar&&(F=C.escapeChar),("string"!=typeof S||-1=A)return b(!0);break}f.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:d.length,index:j}),m++}}else if(x&&0===o.length&&i.substring(j,j+s)===x){if(-1===g)return b();j=g+n,g=i.indexOf(O,j),p=i.indexOf(S,j)}else if(-1!==p&&(p=A)return b(!0)}return E();function k(e){d.push(e),l=j}function v(e){var t=0;return t=-1!==e&&(e=i.substring(m+1,e))&&""===e.trim()?e.length:t}function E(e){return t||(void 0===e&&(e=i.substring(j)),o.push(e),j=r,k(o),a&&R()),b()}function w(e){j=e,k(o),o=[],g=i.indexOf(O,j)}function b(e){if(C.header&&!u&&d.length){for(var t=d[0],i={},r=new Set(t),n=!1,s=0;s65279!==e.charCodeAt(0)?e:e.slice(1))(e),i=new(t.download?f:c)(t)):!0===e.readable&&P(e.read)&&P(e.on)?i=new p(t):(s.File&&e instanceof File||e instanceof Object)&&(i=new l(t)),i.stream(e);(i=(()=>{var e;return!!v.WORKERS_SUPPORTED&&(e=(()=>{var e=s.URL||s.webkitURL||null,t=r.toString();return v.BLOB_URL||(v.BLOB_URL=e.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ","(",t,")();"],{type:"text/javascript"})))})(),(e=new s.Worker(e)).onmessage=g,e.id=h++,o[e.id]=e)})()).userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=P(t.step),t.chunk=P(t.chunk),t.complete=P(t.complete),t.error=P(t.error),delete t.worker,i.postMessage({input:e,config:t,workerId:i.id})},v.unparse=function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,i=!1,r=null,o=!1,h=((()=>{if("object"==typeof t){if("string"!=typeof t.delimiter||v.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter),"boolean"!=typeof t.quotes&&"function"!=typeof t.quotes&&!Array.isArray(t.quotes)||(n=t.quotes),"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines),"string"==typeof t.newline&&(y=t.newline),"string"==typeof t.quoteChar&&(s=t.quoteChar),"boolean"==typeof t.header&&(_=t.header),Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s),t.escapeFormulae instanceof RegExp?o=t.escapeFormulae:"boolean"==typeof t.escapeFormulae&&t.escapeFormulae&&(o=/^[=+\-@\t\r].*$/)}})(),new RegExp(M(s),"g"));"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||Object.keys(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||r),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e,t,i){var r="",n=("string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t)),Array.isArray(e)&&0{for(var i=0;i{"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()})(this,function r(){var n="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==n?n:{};var d,s=!n.document&&!!n.postMessage,a=n.IS_PAPA_WORKER||!1,o={},h=0,v={};function u(e){this._handle=null,this._finished=!1,this._completed=!1,this._halted=!1,this._input=null,this._baseIndex=0,this._partialLine="",this._rowCount=0,this._start=0,this._nextChunk=null,this.isFirstChunk=!0,this._completeResults={data:[],errors:[],meta:{}},function(e){var t=w(e);t.chunkSize=parseInt(t.chunkSize),e.step||e.chunk||(t.chunkSize=null);this._handle=new i(t),(this._handle.streamer=this)._config=t}.call(this,e),this.parseChunk=function(t,e){var i=parseInt(this._config.skipFirstNLines)||0;if(this.isFirstChunk&&0=this._config.preview);if(a)n.postMessage({results:r,workerId:v.WORKER_ID,finished:i});else if(P(this._config.chunk)&&!e){if(this._config.chunk(r,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);this._completeResults=r=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(r.data),this._completeResults.errors=this._completeResults.errors.concat(r.errors),this._completeResults.meta=r.meta),this._completed||!i||!P(this._config.complete)||r&&r.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),i||r&&r.meta.paused||this._nextChunk(),r}this._halted=!0},this._sendError=function(e){P(this._config.error)?this._config.error(e):a&&this._config.error&&n.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function f(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),u.call(this,e),this._nextChunk=s?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),s||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!s),this._config.downloadRequestHeaders){var e,t=this._config.downloadRequestHeaders;for(e in t)r.setRequestHeader(e,t[e])}var i;this._config.chunkSize&&(i=this._start+this._config.chunkSize-1,r.setRequestHeader("Range","bytes="+this._start+"-"+i));try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}s&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize||r.responseText.length,this._finished=!this._config.chunkSize||this._start>=(e=>null!==(e=e.getResponseHeader("Content-Range"))?parseInt(e.substring(e.lastIndexOf("/")+1)):-1)(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){e=r.statusText||e;this._sendError(new Error(e))}}function l(e){(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),u.call(this,e);var i,r,n="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,r=e.slice||e.webkitSlice||e.mozSlice,n?((i=new FileReader).onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function c(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){var e,t;if(!this._finished)return e=this._config.chunkSize,i=e?(t=i.substring(0,e),i.substring(e)):(t=i,""),this._finished=!i,this.parseChunk(t)}}function p(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var n,s,a,t,o=Math.pow(2,53),h=-o,u=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,d=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,i=this,r=0,f=0,l=!1,e=!1,c=[],p={data:[],errors:[],meta:{}};function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(p&&a&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),a=!1),m.skipEmptyLines&&(p.data=p.data.filter(function(e){return!y(e)})),_()){if(p)if(Array.isArray(p.data[0])){for(var e=0;_()&&e(e=>(m.dynamicTypingFunction&&void 0===m.dynamicTyping[e]&&(m.dynamicTyping[e]=m.dynamicTypingFunction(e)),!0===(m.dynamicTyping[e]||m.dynamicTyping)))(e)?"true"===t||"TRUE"===t||"false"!==t&&"FALSE"!==t&&((e=>{if(u.test(e)){e=parseFloat(e);if(h=c.length?"__parsed_extra":c[r]:n,s=m.transform?m.transform(s,n):s);"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return m.header&&(r>c.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+c.length+" fields but parsed "+r,f+t):rm.preview?s.abort():(p.data=p.data[0],t(p,i))))}),this.parse=function(e,t,i){var r=m.quoteChar||'"',r=(m.newline||(m.newline=this.guessLineEndings(e,r)),a=!1,m.delimiter?P(m.delimiter)&&(m.delimiter=m.delimiter(e),p.meta.delimiter=m.delimiter):((r=((e,t,i,r,n)=>{var s,a,o,h;n=n||[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP];for(var u=0;u=i.length/2?"\r\n":"\r"}}function M(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function E(C){var S=(C=C||{}).delimiter,O=C.newline,x=C.comments,I=C.step,A=C.preview,T=C.fastMode,D=null,L=null==C.quoteChar?'"':C.quoteChar,F=L;if(void 0!==C.escapeChar&&(F=C.escapeChar),("string"!=typeof S||-1=A)return b(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:j}),m++}}else if(x&&0===d.length&&i.substring(j,j+a)===x){if(-1===g)return b();j=g+s,g=i.indexOf(O,j),p=i.indexOf(S,j)}else if(-1!==p&&(p=A)return b(!0)}return E();function k(e){h.push(e),f=j}function v(e){var t=0;return t=-1!==e&&(e=i.substring(m+1,e))&&""===e.trim()?e.length:t}function E(e){return r||(void 0===e&&(e=i.substring(j)),d.push(e),j=n,k(d),o&&R()),b()}function w(e){j=e,k(d),d=[],g=i.indexOf(O,j)}function b(e){if(C.header&&!t&&h.length){var s=h[0],a={},o=new Set(s);let n=!1;for(let r=0;r65279!==e.charCodeAt(0)?e:e.slice(1))(e),i=new(t.download?f:c)(t)):!0===e.readable&&P(e.read)&&P(e.on)?i=new p(t):(n.File&&e instanceof File||e instanceof Object)&&(i=new l(t)),i.stream(e);(i=(()=>{var e;return!!v.WORKERS_SUPPORTED&&(e=(()=>{var e=n.URL||n.webkitURL||null,t=r.toString();return v.BLOB_URL||(v.BLOB_URL=e.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ","(",t,")();"],{type:"text/javascript"})))})(),(e=new n.Worker(e)).onmessage=g,e.id=h++,o[e.id]=e)})()).userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=P(t.step),t.chunk=P(t.chunk),t.complete=P(t.complete),t.error=P(t.error),delete t.worker,i.postMessage({input:e,config:t,workerId:i.id})},v.unparse=function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,i=!1,r=null,o=!1,h=((()=>{if("object"==typeof t){if("string"!=typeof t.delimiter||v.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter),"boolean"!=typeof t.quotes&&"function"!=typeof t.quotes&&!Array.isArray(t.quotes)||(n=t.quotes),"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines),"string"==typeof t.newline&&(y=t.newline),"string"==typeof t.quoteChar&&(s=t.quoteChar),"boolean"==typeof t.header&&(_=t.header),Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s),t.escapeFormulae instanceof RegExp?o=t.escapeFormulae:"boolean"==typeof t.escapeFormulae&&t.escapeFormulae&&(o=/^[=+\-@\t\r].*$/)}})(),new RegExp(M(s),"g"));"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||Object.keys(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||r),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e,t,i){var r="",n=("string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t)),Array.isArray(e)&&0{for(var i=0;i Date: Fri, 10 Jan 2025 09:30:05 +0100 Subject: [PATCH 51/61] Fix package URL --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5eece0aa..8d6c3990 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "homepage": "https://www.papaparse.com/", "repository": { "type": "git", - "url": "https://github.com/mholt/PapaParse.git" + "url": "git+https://github.com/mholt/PapaParse.git" }, "author": { "name": "Matthew Holt", From 44c5686af9f39ebbbd76433c82549322023c7909 Mon Sep 17 00:00:00 2001 From: Gus Eggert Date: Tue, 28 Jan 2025 03:23:12 -0500 Subject: [PATCH 52/61] Only attempt to parse headers once (#1086) Closes #1085 --- papaparse.js | 4 +++- tests/test-cases.js | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index c1d63346..23513ff1 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1426,6 +1426,7 @@ License: MIT var fastMode = config.fastMode; var quoteChar; var renamedHeaders = null; + var headerParsed = false; if (config.quoteChar === undefined || config.quoteChar === null) { quoteChar = '"'; @@ -1741,7 +1742,7 @@ License: MIT /** Returns an object with the results, errors, and meta. */ function returnable(stopped) { - if (config.header && !baseIndex && data.length) + if (config.header && !baseIndex && data.length && !headerParsed) { const result = data[0]; const headerCount = {}; // To track the count of each base header @@ -1781,6 +1782,7 @@ License: MIT if (duplicateHeaders) { console.warn('Duplicate headers found and renamed.'); } + headerParsed = true; } return { data: data, diff --git a/tests/test-cases.js b/tests/test-cases.js index 5eaa943e..589a707c 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -2338,6 +2338,22 @@ var CUSTOM_TESTS = [ }); } }, + { + description: "Data is correctly parsed with steps when there are empty values", + expected: [{A: 'a', B: 'b', C: 'c', D: 'd'}, {A: 'a', B: '', C: '', D: ''}], + run: function(callback) { + var data = []; + Papa.parse('A,B,C,D\na,b,c,d\na,,,', { + header: true, + step: function(results) { + data.push(results.data); + }, + complete: function() { + callback(data); + } + }); + } + }, { description: "Step is called with the contents of the row", expected: ['A', 'b', 'c'], From c7e83216fb9c66f68f4bc6d23c2732e939b2e78b Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Tue, 28 Jan 2025 09:37:17 +0100 Subject: [PATCH 53/61] Do not run headless tests on CI (#1087) --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 0d121858..ca7a2b72 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -26,4 +26,4 @@ jobs: with: node-version: ${{ matrix.node-version }} - run: npm install - - run: npm test && npm run build + - run: npm run lint && npm run test-node && npm run build From e3c7b2628c68b868fd09862252eea312fbafdd84 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Tue, 28 Jan 2025 09:38:08 +0100 Subject: [PATCH 54/61] Patch version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 8d6c3990..97b256d7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.5.1", + "version": "5.5.2", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index 23513ff1..838c4216 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.5.1 +v5.5.2 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index bb9b4cce..42847529 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v5.5.1 +v5.5.2 https://github.com/mholt/PapaParse License: MIT */ -((e,t)=>{"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()})(this,function r(){var n="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==n?n:{};var d,s=!n.document&&!!n.postMessage,a=n.IS_PAPA_WORKER||!1,o={},h=0,v={};function u(e){this._handle=null,this._finished=!1,this._completed=!1,this._halted=!1,this._input=null,this._baseIndex=0,this._partialLine="",this._rowCount=0,this._start=0,this._nextChunk=null,this.isFirstChunk=!0,this._completeResults={data:[],errors:[],meta:{}},function(e){var t=w(e);t.chunkSize=parseInt(t.chunkSize),e.step||e.chunk||(t.chunkSize=null);this._handle=new i(t),(this._handle.streamer=this)._config=t}.call(this,e),this.parseChunk=function(t,e){var i=parseInt(this._config.skipFirstNLines)||0;if(this.isFirstChunk&&0=this._config.preview);if(a)n.postMessage({results:r,workerId:v.WORKER_ID,finished:i});else if(P(this._config.chunk)&&!e){if(this._config.chunk(r,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);this._completeResults=r=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(r.data),this._completeResults.errors=this._completeResults.errors.concat(r.errors),this._completeResults.meta=r.meta),this._completed||!i||!P(this._config.complete)||r&&r.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),i||r&&r.meta.paused||this._nextChunk(),r}this._halted=!0},this._sendError=function(e){P(this._config.error)?this._config.error(e):a&&this._config.error&&n.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function f(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),u.call(this,e),this._nextChunk=s?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),s||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!s),this._config.downloadRequestHeaders){var e,t=this._config.downloadRequestHeaders;for(e in t)r.setRequestHeader(e,t[e])}var i;this._config.chunkSize&&(i=this._start+this._config.chunkSize-1,r.setRequestHeader("Range","bytes="+this._start+"-"+i));try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}s&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize||r.responseText.length,this._finished=!this._config.chunkSize||this._start>=(e=>null!==(e=e.getResponseHeader("Content-Range"))?parseInt(e.substring(e.lastIndexOf("/")+1)):-1)(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){e=r.statusText||e;this._sendError(new Error(e))}}function l(e){(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),u.call(this,e);var i,r,n="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,r=e.slice||e.webkitSlice||e.mozSlice,n?((i=new FileReader).onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function c(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){var e,t;if(!this._finished)return e=this._config.chunkSize,i=e?(t=i.substring(0,e),i.substring(e)):(t=i,""),this._finished=!i,this.parseChunk(t)}}function p(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var n,s,a,t,o=Math.pow(2,53),h=-o,u=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,d=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,i=this,r=0,f=0,l=!1,e=!1,c=[],p={data:[],errors:[],meta:{}};function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(p&&a&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),a=!1),m.skipEmptyLines&&(p.data=p.data.filter(function(e){return!y(e)})),_()){if(p)if(Array.isArray(p.data[0])){for(var e=0;_()&&e(e=>(m.dynamicTypingFunction&&void 0===m.dynamicTyping[e]&&(m.dynamicTyping[e]=m.dynamicTypingFunction(e)),!0===(m.dynamicTyping[e]||m.dynamicTyping)))(e)?"true"===t||"TRUE"===t||"false"!==t&&"FALSE"!==t&&((e=>{if(u.test(e)){e=parseFloat(e);if(h=c.length?"__parsed_extra":c[r]:n,s=m.transform?m.transform(s,n):s);"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return m.header&&(r>c.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+c.length+" fields but parsed "+r,f+t):rm.preview?s.abort():(p.data=p.data[0],t(p,i))))}),this.parse=function(e,t,i){var r=m.quoteChar||'"',r=(m.newline||(m.newline=this.guessLineEndings(e,r)),a=!1,m.delimiter?P(m.delimiter)&&(m.delimiter=m.delimiter(e),p.meta.delimiter=m.delimiter):((r=((e,t,i,r,n)=>{var s,a,o,h;n=n||[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP];for(var u=0;u=i.length/2?"\r\n":"\r"}}function M(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function E(C){var S=(C=C||{}).delimiter,O=C.newline,x=C.comments,I=C.step,A=C.preview,T=C.fastMode,D=null,L=null==C.quoteChar?'"':C.quoteChar,F=L;if(void 0!==C.escapeChar&&(F=C.escapeChar),("string"!=typeof S||-1=A)return b(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:j}),m++}}else if(x&&0===d.length&&i.substring(j,j+a)===x){if(-1===g)return b();j=g+s,g=i.indexOf(O,j),p=i.indexOf(S,j)}else if(-1!==p&&(p=A)return b(!0)}return E();function k(e){h.push(e),f=j}function v(e){var t=0;return t=-1!==e&&(e=i.substring(m+1,e))&&""===e.trim()?e.length:t}function E(e){return r||(void 0===e&&(e=i.substring(j)),d.push(e),j=n,k(d),o&&R()),b()}function w(e){j=e,k(d),d=[],g=i.indexOf(O,j)}function b(e){if(C.header&&!t&&h.length){var s=h[0],a={},o=new Set(s);let n=!1;for(let r=0;r65279!==e.charCodeAt(0)?e:e.slice(1))(e),i=new(t.download?f:c)(t)):!0===e.readable&&P(e.read)&&P(e.on)?i=new p(t):(n.File&&e instanceof File||e instanceof Object)&&(i=new l(t)),i.stream(e);(i=(()=>{var e;return!!v.WORKERS_SUPPORTED&&(e=(()=>{var e=n.URL||n.webkitURL||null,t=r.toString();return v.BLOB_URL||(v.BLOB_URL=e.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ","(",t,")();"],{type:"text/javascript"})))})(),(e=new n.Worker(e)).onmessage=g,e.id=h++,o[e.id]=e)})()).userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=P(t.step),t.chunk=P(t.chunk),t.complete=P(t.complete),t.error=P(t.error),delete t.worker,i.postMessage({input:e,config:t,workerId:i.id})},v.unparse=function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,i=!1,r=null,o=!1,h=((()=>{if("object"==typeof t){if("string"!=typeof t.delimiter||v.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter),"boolean"!=typeof t.quotes&&"function"!=typeof t.quotes&&!Array.isArray(t.quotes)||(n=t.quotes),"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines),"string"==typeof t.newline&&(y=t.newline),"string"==typeof t.quoteChar&&(s=t.quoteChar),"boolean"==typeof t.header&&(_=t.header),Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s),t.escapeFormulae instanceof RegExp?o=t.escapeFormulae:"boolean"==typeof t.escapeFormulae&&t.escapeFormulae&&(o=/^[=+\-@\t\r].*$/)}})(),new RegExp(M(s),"g"));"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||Object.keys(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||r),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e,t,i){var r="",n=("string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t)),Array.isArray(e)&&0{for(var i=0;i{"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()})(this,function r(){var n="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==n?n:{};var d,s=!n.document&&!!n.postMessage,a=n.IS_PAPA_WORKER||!1,o={},h=0,v={};function u(e){this._handle=null,this._finished=!1,this._completed=!1,this._halted=!1,this._input=null,this._baseIndex=0,this._partialLine="",this._rowCount=0,this._start=0,this._nextChunk=null,this.isFirstChunk=!0,this._completeResults={data:[],errors:[],meta:{}},function(e){var t=w(e);t.chunkSize=parseInt(t.chunkSize),e.step||e.chunk||(t.chunkSize=null);this._handle=new i(t),(this._handle.streamer=this)._config=t}.call(this,e),this.parseChunk=function(t,e){var i=parseInt(this._config.skipFirstNLines)||0;if(this.isFirstChunk&&0=this._config.preview);if(a)n.postMessage({results:r,workerId:v.WORKER_ID,finished:i});else if(U(this._config.chunk)&&!e){if(this._config.chunk(r,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);this._completeResults=r=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(r.data),this._completeResults.errors=this._completeResults.errors.concat(r.errors),this._completeResults.meta=r.meta),this._completed||!i||!U(this._config.complete)||r&&r.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),i||r&&r.meta.paused||this._nextChunk(),r}this._halted=!0},this._sendError=function(e){U(this._config.error)?this._config.error(e):a&&this._config.error&&n.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function f(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),u.call(this,e),this._nextChunk=s?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),s||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!s),this._config.downloadRequestHeaders){var e,t=this._config.downloadRequestHeaders;for(e in t)r.setRequestHeader(e,t[e])}var i;this._config.chunkSize&&(i=this._start+this._config.chunkSize-1,r.setRequestHeader("Range","bytes="+this._start+"-"+i));try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}s&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize||r.responseText.length,this._finished=!this._config.chunkSize||this._start>=(e=>null!==(e=e.getResponseHeader("Content-Range"))?parseInt(e.substring(e.lastIndexOf("/")+1)):-1)(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){e=r.statusText||e;this._sendError(new Error(e))}}function l(e){(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),u.call(this,e);var i,r,n="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,r=e.slice||e.webkitSlice||e.mozSlice,n?((i=new FileReader).onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function c(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){var e,t;if(!this._finished)return e=this._config.chunkSize,i=e?(t=i.substring(0,e),i.substring(e)):(t=i,""),this._finished=!i,this.parseChunk(t)}}function p(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var n,s,a,t,o=Math.pow(2,53),h=-o,u=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,d=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,i=this,r=0,f=0,l=!1,e=!1,c=[],p={data:[],errors:[],meta:{}};function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(p&&a&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),a=!1),m.skipEmptyLines&&(p.data=p.data.filter(function(e){return!y(e)})),_()){if(p)if(Array.isArray(p.data[0])){for(var e=0;_()&&e(e=>(m.dynamicTypingFunction&&void 0===m.dynamicTyping[e]&&(m.dynamicTyping[e]=m.dynamicTypingFunction(e)),!0===(m.dynamicTyping[e]||m.dynamicTyping)))(e)?"true"===t||"TRUE"===t||"false"!==t&&"FALSE"!==t&&((e=>{if(u.test(e)){e=parseFloat(e);if(h=c.length?"__parsed_extra":c[r]:n,s=m.transform?m.transform(s,n):s);"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return m.header&&(r>c.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+c.length+" fields but parsed "+r,f+t):rm.preview?s.abort():(p.data=p.data[0],t(p,i))))}),this.parse=function(e,t,i){var r=m.quoteChar||'"',r=(m.newline||(m.newline=this.guessLineEndings(e,r)),a=!1,m.delimiter?U(m.delimiter)&&(m.delimiter=m.delimiter(e),p.meta.delimiter=m.delimiter):((r=((e,t,i,r,n)=>{var s,a,o,h;n=n||[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP];for(var u=0;u=i.length/2?"\r\n":"\r"}}function P(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function E(C){var S=(C=C||{}).delimiter,O=C.newline,x=C.comments,I=C.step,A=C.preview,T=C.fastMode,D=null,L=!1,F=null==C.quoteChar?'"':C.quoteChar,j=F;if(void 0!==C.escapeChar&&(j=C.escapeChar),("string"!=typeof S||-1=A)return b(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:z}),m++}}else if(x&&0===d.length&&i.substring(z,z+a)===x){if(-1===g)return b();z=g+s,g=i.indexOf(O,z),p=i.indexOf(S,z)}else if(-1!==p&&(p=A)return b(!0)}return E();function k(e){h.push(e),f=z}function v(e){var t=0;return t=-1!==e&&(e=i.substring(m+1,e))&&""===e.trim()?e.length:t}function E(e){return r||(void 0===e&&(e=i.substring(z)),d.push(e),z=n,k(d),o&&R()),b()}function w(e){z=e,k(d),d=[],g=i.indexOf(O,z)}function b(e){if(C.header&&!t&&h.length&&!L){var s=h[0],a={},o=new Set(s);let n=!1;for(let r=0;r65279!==e.charCodeAt(0)?e:e.slice(1))(e),i=new(t.download?f:c)(t)):!0===e.readable&&U(e.read)&&U(e.on)?i=new p(t):(n.File&&e instanceof File||e instanceof Object)&&(i=new l(t)),i.stream(e);(i=(()=>{var e;return!!v.WORKERS_SUPPORTED&&(e=(()=>{var e=n.URL||n.webkitURL||null,t=r.toString();return v.BLOB_URL||(v.BLOB_URL=e.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ","(",t,")();"],{type:"text/javascript"})))})(),(e=new n.Worker(e)).onmessage=g,e.id=h++,o[e.id]=e)})()).userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=U(t.step),t.chunk=U(t.chunk),t.complete=U(t.complete),t.error=U(t.error),delete t.worker,i.postMessage({input:e,config:t,workerId:i.id})},v.unparse=function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,i=!1,r=null,o=!1,h=((()=>{if("object"==typeof t){if("string"!=typeof t.delimiter||v.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter),"boolean"!=typeof t.quotes&&"function"!=typeof t.quotes&&!Array.isArray(t.quotes)||(n=t.quotes),"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines),"string"==typeof t.newline&&(y=t.newline),"string"==typeof t.quoteChar&&(s=t.quoteChar),"boolean"==typeof t.header&&(_=t.header),Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s),t.escapeFormulae instanceof RegExp?o=t.escapeFormulae:"boolean"==typeof t.escapeFormulae&&t.escapeFormulae&&(o=/^[=+\-@\t\r].*$/)}})(),new RegExp(P(s),"g"));"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||Object.keys(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||r),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e,t,i){var r="",n=("string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t)),Array.isArray(e)&&0{for(var i=0;i Date: Mon, 19 May 2025 23:47:57 +0900 Subject: [PATCH 55/61] Avoid infinite loop with duplicate counting (#1095) --- papaparse.js | 2 +- tests/test-cases.js | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/papaparse.js b/papaparse.js index 838c4216..b41439c0 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1745,7 +1745,7 @@ License: MIT if (config.header && !baseIndex && data.length && !headerParsed) { const result = data[0]; - const headerCount = {}; // To track the count of each base header + const headerCount = Object.create(null); // To track the count of each base header const usedHeaders = new Set(result); // To track used headers and avoid duplicates let duplicateHeaders = false; diff --git a/tests/test-cases.js b/tests/test-cases.js index 589a707c..2d984c85 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -626,6 +626,19 @@ var CORE_PARSER_TESTS = [ } } }, + { + description: "Duplicate header names with __proto__ field", + input: '__proto__,__proto__,__proto__\n1,2,3', + config: { header: true }, + expected: { + data: [['__proto__', '__proto___1', '__proto___2'], ['1', '2', '3']], + errors: [], + meta: { + renamedHeaders: {__proto___1: '__proto__', __proto___2: '__proto__'}, + cursor: 35 + } + } + }, ]; describe('Core Parser Tests', function() { From a4f8b0f1e30bf08e44da96ff5575ffdae7aa9b12 Mon Sep 17 00:00:00 2001 From: Sergi Almacellas Abellana Date: Mon, 19 May 2025 16:48:57 +0200 Subject: [PATCH 56/61] Patch version bump --- package.json | 2 +- papaparse.js | 2 +- papaparse.min.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 97b256d7..013a4b0c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "papaparse", - "version": "5.5.2", + "version": "5.5.3", "description": "Fast and powerful CSV parser for the browser that supports web workers and streaming large files. Converts CSV to JSON and JSON to CSV.", "keywords": [ "csv", diff --git a/papaparse.js b/papaparse.js index b41439c0..695df380 100755 --- a/papaparse.js +++ b/papaparse.js @@ -1,6 +1,6 @@ /* @license Papa Parse -v5.5.2 +v5.5.3 https://github.com/mholt/PapaParse License: MIT */ diff --git a/papaparse.min.js b/papaparse.min.js index 42847529..f3141104 100644 --- a/papaparse.min.js +++ b/papaparse.min.js @@ -1,7 +1,7 @@ /* @license Papa Parse -v5.5.2 +v5.5.3 https://github.com/mholt/PapaParse License: MIT */ -((e,t)=>{"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()})(this,function r(){var n="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==n?n:{};var d,s=!n.document&&!!n.postMessage,a=n.IS_PAPA_WORKER||!1,o={},h=0,v={};function u(e){this._handle=null,this._finished=!1,this._completed=!1,this._halted=!1,this._input=null,this._baseIndex=0,this._partialLine="",this._rowCount=0,this._start=0,this._nextChunk=null,this.isFirstChunk=!0,this._completeResults={data:[],errors:[],meta:{}},function(e){var t=w(e);t.chunkSize=parseInt(t.chunkSize),e.step||e.chunk||(t.chunkSize=null);this._handle=new i(t),(this._handle.streamer=this)._config=t}.call(this,e),this.parseChunk=function(t,e){var i=parseInt(this._config.skipFirstNLines)||0;if(this.isFirstChunk&&0=this._config.preview);if(a)n.postMessage({results:r,workerId:v.WORKER_ID,finished:i});else if(U(this._config.chunk)&&!e){if(this._config.chunk(r,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);this._completeResults=r=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(r.data),this._completeResults.errors=this._completeResults.errors.concat(r.errors),this._completeResults.meta=r.meta),this._completed||!i||!U(this._config.complete)||r&&r.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),i||r&&r.meta.paused||this._nextChunk(),r}this._halted=!0},this._sendError=function(e){U(this._config.error)?this._config.error(e):a&&this._config.error&&n.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function f(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),u.call(this,e),this._nextChunk=s?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),s||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!s),this._config.downloadRequestHeaders){var e,t=this._config.downloadRequestHeaders;for(e in t)r.setRequestHeader(e,t[e])}var i;this._config.chunkSize&&(i=this._start+this._config.chunkSize-1,r.setRequestHeader("Range","bytes="+this._start+"-"+i));try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}s&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize||r.responseText.length,this._finished=!this._config.chunkSize||this._start>=(e=>null!==(e=e.getResponseHeader("Content-Range"))?parseInt(e.substring(e.lastIndexOf("/")+1)):-1)(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){e=r.statusText||e;this._sendError(new Error(e))}}function l(e){(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),u.call(this,e);var i,r,n="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,r=e.slice||e.webkitSlice||e.mozSlice,n?((i=new FileReader).onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function c(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){var e,t;if(!this._finished)return e=this._config.chunkSize,i=e?(t=i.substring(0,e),i.substring(e)):(t=i,""),this._finished=!i,this.parseChunk(t)}}function p(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var n,s,a,t,o=Math.pow(2,53),h=-o,u=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,d=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,i=this,r=0,f=0,l=!1,e=!1,c=[],p={data:[],errors:[],meta:{}};function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(p&&a&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),a=!1),m.skipEmptyLines&&(p.data=p.data.filter(function(e){return!y(e)})),_()){if(p)if(Array.isArray(p.data[0])){for(var e=0;_()&&e(e=>(m.dynamicTypingFunction&&void 0===m.dynamicTyping[e]&&(m.dynamicTyping[e]=m.dynamicTypingFunction(e)),!0===(m.dynamicTyping[e]||m.dynamicTyping)))(e)?"true"===t||"TRUE"===t||"false"!==t&&"FALSE"!==t&&((e=>{if(u.test(e)){e=parseFloat(e);if(h=c.length?"__parsed_extra":c[r]:n,s=m.transform?m.transform(s,n):s);"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return m.header&&(r>c.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+c.length+" fields but parsed "+r,f+t):rm.preview?s.abort():(p.data=p.data[0],t(p,i))))}),this.parse=function(e,t,i){var r=m.quoteChar||'"',r=(m.newline||(m.newline=this.guessLineEndings(e,r)),a=!1,m.delimiter?U(m.delimiter)&&(m.delimiter=m.delimiter(e),p.meta.delimiter=m.delimiter):((r=((e,t,i,r,n)=>{var s,a,o,h;n=n||[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP];for(var u=0;u=i.length/2?"\r\n":"\r"}}function P(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function E(C){var S=(C=C||{}).delimiter,O=C.newline,x=C.comments,I=C.step,A=C.preview,T=C.fastMode,D=null,L=!1,F=null==C.quoteChar?'"':C.quoteChar,j=F;if(void 0!==C.escapeChar&&(j=C.escapeChar),("string"!=typeof S||-1=A)return b(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:z}),m++}}else if(x&&0===d.length&&i.substring(z,z+a)===x){if(-1===g)return b();z=g+s,g=i.indexOf(O,z),p=i.indexOf(S,z)}else if(-1!==p&&(p=A)return b(!0)}return E();function k(e){h.push(e),f=z}function v(e){var t=0;return t=-1!==e&&(e=i.substring(m+1,e))&&""===e.trim()?e.length:t}function E(e){return r||(void 0===e&&(e=i.substring(z)),d.push(e),z=n,k(d),o&&R()),b()}function w(e){z=e,k(d),d=[],g=i.indexOf(O,z)}function b(e){if(C.header&&!t&&h.length&&!L){var s=h[0],a={},o=new Set(s);let n=!1;for(let r=0;r65279!==e.charCodeAt(0)?e:e.slice(1))(e),i=new(t.download?f:c)(t)):!0===e.readable&&U(e.read)&&U(e.on)?i=new p(t):(n.File&&e instanceof File||e instanceof Object)&&(i=new l(t)),i.stream(e);(i=(()=>{var e;return!!v.WORKERS_SUPPORTED&&(e=(()=>{var e=n.URL||n.webkitURL||null,t=r.toString();return v.BLOB_URL||(v.BLOB_URL=e.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ","(",t,")();"],{type:"text/javascript"})))})(),(e=new n.Worker(e)).onmessage=g,e.id=h++,o[e.id]=e)})()).userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=U(t.step),t.chunk=U(t.chunk),t.complete=U(t.complete),t.error=U(t.error),delete t.worker,i.postMessage({input:e,config:t,workerId:i.id})},v.unparse=function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,i=!1,r=null,o=!1,h=((()=>{if("object"==typeof t){if("string"!=typeof t.delimiter||v.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter),"boolean"!=typeof t.quotes&&"function"!=typeof t.quotes&&!Array.isArray(t.quotes)||(n=t.quotes),"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines),"string"==typeof t.newline&&(y=t.newline),"string"==typeof t.quoteChar&&(s=t.quoteChar),"boolean"==typeof t.header&&(_=t.header),Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s),t.escapeFormulae instanceof RegExp?o=t.escapeFormulae:"boolean"==typeof t.escapeFormulae&&t.escapeFormulae&&(o=/^[=+\-@\t\r].*$/)}})(),new RegExp(P(s),"g"));"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||Object.keys(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||r),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e,t,i){var r="",n=("string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t)),Array.isArray(e)&&0{for(var i=0;i{"function"==typeof define&&define.amd?define([],t):"object"==typeof module&&"undefined"!=typeof exports?module.exports=t():e.Papa=t()})(this,function r(){var n="undefined"!=typeof self?self:"undefined"!=typeof window?window:void 0!==n?n:{};var d,s=!n.document&&!!n.postMessage,a=n.IS_PAPA_WORKER||!1,o={},h=0,v={};function u(e){this._handle=null,this._finished=!1,this._completed=!1,this._halted=!1,this._input=null,this._baseIndex=0,this._partialLine="",this._rowCount=0,this._start=0,this._nextChunk=null,this.isFirstChunk=!0,this._completeResults={data:[],errors:[],meta:{}},function(e){var t=b(e);t.chunkSize=parseInt(t.chunkSize),e.step||e.chunk||(t.chunkSize=null);this._handle=new i(t),(this._handle.streamer=this)._config=t}.call(this,e),this.parseChunk=function(t,e){var i=parseInt(this._config.skipFirstNLines)||0;if(this.isFirstChunk&&0=this._config.preview);if(a)n.postMessage({results:r,workerId:v.WORKER_ID,finished:i});else if(U(this._config.chunk)&&!e){if(this._config.chunk(r,this._handle),this._handle.paused()||this._handle.aborted())return void(this._halted=!0);this._completeResults=r=void 0}return this._config.step||this._config.chunk||(this._completeResults.data=this._completeResults.data.concat(r.data),this._completeResults.errors=this._completeResults.errors.concat(r.errors),this._completeResults.meta=r.meta),this._completed||!i||!U(this._config.complete)||r&&r.meta.aborted||(this._config.complete(this._completeResults,this._input),this._completed=!0),i||r&&r.meta.paused||this._nextChunk(),r}this._halted=!0},this._sendError=function(e){U(this._config.error)?this._config.error(e):a&&this._config.error&&n.postMessage({workerId:v.WORKER_ID,error:e,finished:!1})}}function f(e){var r;(e=e||{}).chunkSize||(e.chunkSize=v.RemoteChunkSize),u.call(this,e),this._nextChunk=s?function(){this._readChunk(),this._chunkLoaded()}:function(){this._readChunk()},this.stream=function(e){this._input=e,this._nextChunk()},this._readChunk=function(){if(this._finished)this._chunkLoaded();else{if(r=new XMLHttpRequest,this._config.withCredentials&&(r.withCredentials=this._config.withCredentials),s||(r.onload=y(this._chunkLoaded,this),r.onerror=y(this._chunkError,this)),r.open(this._config.downloadRequestBody?"POST":"GET",this._input,!s),this._config.downloadRequestHeaders){var e,t=this._config.downloadRequestHeaders;for(e in t)r.setRequestHeader(e,t[e])}var i;this._config.chunkSize&&(i=this._start+this._config.chunkSize-1,r.setRequestHeader("Range","bytes="+this._start+"-"+i));try{r.send(this._config.downloadRequestBody)}catch(e){this._chunkError(e.message)}s&&0===r.status&&this._chunkError()}},this._chunkLoaded=function(){4===r.readyState&&(r.status<200||400<=r.status?this._chunkError():(this._start+=this._config.chunkSize||r.responseText.length,this._finished=!this._config.chunkSize||this._start>=(e=>null!==(e=e.getResponseHeader("Content-Range"))?parseInt(e.substring(e.lastIndexOf("/")+1)):-1)(r),this.parseChunk(r.responseText)))},this._chunkError=function(e){e=r.statusText||e;this._sendError(new Error(e))}}function l(e){(e=e||{}).chunkSize||(e.chunkSize=v.LocalChunkSize),u.call(this,e);var i,r,n="undefined"!=typeof FileReader;this.stream=function(e){this._input=e,r=e.slice||e.webkitSlice||e.mozSlice,n?((i=new FileReader).onload=y(this._chunkLoaded,this),i.onerror=y(this._chunkError,this)):i=new FileReaderSync,this._nextChunk()},this._nextChunk=function(){this._finished||this._config.preview&&!(this._rowCount=this._input.size,this.parseChunk(e.target.result)},this._chunkError=function(){this._sendError(i.error)}}function c(e){var i;u.call(this,e=e||{}),this.stream=function(e){return i=e,this._nextChunk()},this._nextChunk=function(){var e,t;if(!this._finished)return e=this._config.chunkSize,i=e?(t=i.substring(0,e),i.substring(e)):(t=i,""),this._finished=!i,this.parseChunk(t)}}function p(e){u.call(this,e=e||{});var t=[],i=!0,r=!1;this.pause=function(){u.prototype.pause.apply(this,arguments),this._input.pause()},this.resume=function(){u.prototype.resume.apply(this,arguments),this._input.resume()},this.stream=function(e){this._input=e,this._input.on("data",this._streamData),this._input.on("end",this._streamEnd),this._input.on("error",this._streamError)},this._checkIsFinished=function(){r&&1===t.length&&(this._finished=!0)},this._nextChunk=function(){this._checkIsFinished(),t.length?this.parseChunk(t.shift()):i=!0},this._streamData=y(function(e){try{t.push("string"==typeof e?e:e.toString(this._config.encoding)),i&&(i=!1,this._checkIsFinished(),this.parseChunk(t.shift()))}catch(e){this._streamError(e)}},this),this._streamError=y(function(e){this._streamCleanUp(),this._sendError(e)},this),this._streamEnd=y(function(){this._streamCleanUp(),r=!0,this._streamData("")},this),this._streamCleanUp=y(function(){this._input.removeListener("data",this._streamData),this._input.removeListener("end",this._streamEnd),this._input.removeListener("error",this._streamError)},this)}function i(m){var n,s,a,t,o=Math.pow(2,53),h=-o,u=/^\s*-?(\d+\.?|\.\d+|\d+\.\d+)([eE][-+]?\d+)?\s*$/,d=/^((\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z)))$/,i=this,r=0,f=0,l=!1,e=!1,c=[],p={data:[],errors:[],meta:{}};function y(e){return"greedy"===m.skipEmptyLines?""===e.join("").trim():1===e.length&&0===e[0].length}function g(){if(p&&a&&(k("Delimiter","UndetectableDelimiter","Unable to auto-detect delimiting character; defaulted to '"+v.DefaultDelimiter+"'"),a=!1),m.skipEmptyLines&&(p.data=p.data.filter(function(e){return!y(e)})),_()){if(p)if(Array.isArray(p.data[0])){for(var e=0;_()&&e(e=>(m.dynamicTypingFunction&&void 0===m.dynamicTyping[e]&&(m.dynamicTyping[e]=m.dynamicTypingFunction(e)),!0===(m.dynamicTyping[e]||m.dynamicTyping)))(e)?"true"===t||"TRUE"===t||"false"!==t&&"FALSE"!==t&&((e=>{if(u.test(e)){e=parseFloat(e);if(h=c.length?"__parsed_extra":c[r]:n,s=m.transform?m.transform(s,n):s);"__parsed_extra"===n?(i[n]=i[n]||[],i[n].push(s)):i[n]=s}return m.header&&(r>c.length?k("FieldMismatch","TooManyFields","Too many fields: expected "+c.length+" fields but parsed "+r,f+t):rm.preview?s.abort():(p.data=p.data[0],t(p,i))))}),this.parse=function(e,t,i){var r=m.quoteChar||'"',r=(m.newline||(m.newline=this.guessLineEndings(e,r)),a=!1,m.delimiter?U(m.delimiter)&&(m.delimiter=m.delimiter(e),p.meta.delimiter=m.delimiter):((r=((e,t,i,r,n)=>{var s,a,o,h;n=n||[",","\t","|",";",v.RECORD_SEP,v.UNIT_SEP];for(var u=0;u=i.length/2?"\r\n":"\r"}}function P(e){return e.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function E(C){var S=(C=C||{}).delimiter,O=C.newline,x=C.comments,I=C.step,A=C.preview,T=C.fastMode,D=null,L=!1,F=null==C.quoteChar?'"':C.quoteChar,j=F;if(void 0!==C.escapeChar&&(j=C.escapeChar),("string"!=typeof S||-1=A)return w(!0);break}u.push({type:"Quotes",code:"InvalidQuotes",message:"Trailing quote on quoted field is malformed",row:h.length,index:z}),m++}}else if(x&&0===d.length&&i.substring(z,z+a)===x){if(-1===g)return w();z=g+s,g=i.indexOf(O,z),p=i.indexOf(S,z)}else if(-1!==p&&(p=A)return w(!0)}return E();function k(e){h.push(e),f=z}function v(e){var t=0;return t=-1!==e&&(e=i.substring(m+1,e))&&""===e.trim()?e.length:t}function E(e){return r||(void 0===e&&(e=i.substring(z)),d.push(e),z=n,k(d),o&&R()),w()}function b(e){z=e,k(d),d=[],g=i.indexOf(O,z)}function w(e){if(C.header&&!t&&h.length&&!L){var s=h[0],a=Object.create(null),o=new Set(s);let n=!1;for(let r=0;r65279!==e.charCodeAt(0)?e:e.slice(1))(e),i=new(t.download?f:c)(t)):!0===e.readable&&U(e.read)&&U(e.on)?i=new p(t):(n.File&&e instanceof File||e instanceof Object)&&(i=new l(t)),i.stream(e);(i=(()=>{var e;return!!v.WORKERS_SUPPORTED&&(e=(()=>{var e=n.URL||n.webkitURL||null,t=r.toString();return v.BLOB_URL||(v.BLOB_URL=e.createObjectURL(new Blob(["var global = (function() { if (typeof self !== 'undefined') { return self; } if (typeof window !== 'undefined') { return window; } if (typeof global !== 'undefined') { return global; } return {}; })(); global.IS_PAPA_WORKER=true; ","(",t,")();"],{type:"text/javascript"})))})(),(e=new n.Worker(e)).onmessage=g,e.id=h++,o[e.id]=e)})()).userStep=t.step,i.userChunk=t.chunk,i.userComplete=t.complete,i.userError=t.error,t.step=U(t.step),t.chunk=U(t.chunk),t.complete=U(t.complete),t.error=U(t.error),delete t.worker,i.postMessage({input:e,config:t,workerId:i.id})},v.unparse=function(e,t){var n=!1,_=!0,m=",",y="\r\n",s='"',a=s+s,i=!1,r=null,o=!1,h=((()=>{if("object"==typeof t){if("string"!=typeof t.delimiter||v.BAD_DELIMITERS.filter(function(e){return-1!==t.delimiter.indexOf(e)}).length||(m=t.delimiter),"boolean"!=typeof t.quotes&&"function"!=typeof t.quotes&&!Array.isArray(t.quotes)||(n=t.quotes),"boolean"!=typeof t.skipEmptyLines&&"string"!=typeof t.skipEmptyLines||(i=t.skipEmptyLines),"string"==typeof t.newline&&(y=t.newline),"string"==typeof t.quoteChar&&(s=t.quoteChar),"boolean"==typeof t.header&&(_=t.header),Array.isArray(t.columns)){if(0===t.columns.length)throw new Error("Option columns is empty");r=t.columns}void 0!==t.escapeChar&&(a=t.escapeChar+s),t.escapeFormulae instanceof RegExp?o=t.escapeFormulae:"boolean"==typeof t.escapeFormulae&&t.escapeFormulae&&(o=/^[=+\-@\t\r].*$/)}})(),new RegExp(P(s),"g"));"string"==typeof e&&(e=JSON.parse(e));if(Array.isArray(e)){if(!e.length||Array.isArray(e[0]))return u(null,e,i);if("object"==typeof e[0])return u(r||Object.keys(e[0]),e,i)}else if("object"==typeof e)return"string"==typeof e.data&&(e.data=JSON.parse(e.data)),Array.isArray(e.data)&&(e.fields||(e.fields=e.meta&&e.meta.fields||r),e.fields||(e.fields=Array.isArray(e.data[0])?e.fields:"object"==typeof e.data[0]?Object.keys(e.data[0]):[]),Array.isArray(e.data[0])||"object"==typeof e.data[0]||(e.data=[e.data])),u(e.fields||[],e.data||[],i);throw new Error("Unable to serialize unrecognized input");function u(e,t,i){var r="",n=("string"==typeof e&&(e=JSON.parse(e)),"string"==typeof t&&(t=JSON.parse(t)),Array.isArray(e)&&0{for(var i=0;i Date: Fri, 30 May 2025 09:16:33 +0200 Subject: [PATCH 57/61] Strip BOM character from column keys when parsing header Co-authored-by: Matthias Clausen --- papaparse.js | 20 +++++++++----------- tests/test-cases.js | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/papaparse.js b/papaparse.js index 695df380..2c7fd178 100755 --- a/papaparse.js +++ b/papaparse.js @@ -185,8 +185,13 @@ License: MIT global.onmessage = workerThreadReceivedMessage; } - - + // Strip character from UTF-8 BOM encoded files that cause issue parsing the file + function stripBom(string) { + if (string.charCodeAt(0) === 0xfeff) { + return string.slice(1); + } + return string; + } function CsvToJson(_input, _config) { @@ -249,14 +254,6 @@ License: MIT streamer = new FileStreamer(_config); return streamer.stream(_input); - - // Strip character from UTF-8 BOM encoded files that cause issue parsing the file - function stripBom(string) { - if (string.charCodeAt(0) === 0xfeff) { - return string.slice(1); - } - return string; - } } @@ -1234,6 +1231,7 @@ License: MIT function addHeader(header, i) { + header = stripBom(header); if (isFunction(_config.transformHeader)) header = _config.transformHeader(header, i); @@ -1750,7 +1748,7 @@ License: MIT let duplicateHeaders = false; for (let i = 0; i < result.length; i++) { - let header = result[i]; + let header = stripBom(result[i]); if (isFunction(config.transformHeader)) header = config.transformHeader(header, i); diff --git a/tests/test-cases.js b/tests/test-cases.js index 2d984c85..86439035 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -1552,6 +1552,24 @@ var PARSE_TESTS = [ } } }, + { + description: "UTF-8 BOM encoded input is stripped from invisible BOM character", + input: '\ufeffA,B\nX,Y', + config: {}, + expected: { + data: [['A', 'B'], ['X', 'Y']], + errors: [], + } + }, + { + description: "UTF-8 BOM encoded input with header produces column key stripped from invisible BOM character", + input: '\ufeffA,B\nX,Y', + config: { header: true }, + expected: { + data: [{A: 'X', B: 'Y'}], + errors: [], + } + }, { description: "Parsing with skipEmptyLines set to 'greedy'", notes: "Must parse correctly without lines with no content", From 530357fd8ab0ebce8d10d9e53c904e1c249b1301 Mon Sep 17 00:00:00 2001 From: Gabe Gorelick Date: Wed, 18 Jun 2025 02:26:23 -0400 Subject: [PATCH 58/61] Update Node versions (#1097) Drop versions that are EOL and add newer versions. Node 18 has been end of life since March 27, 2025 --- .github/workflows/node.js.yml | 2 +- tests/test-cases.js | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index ca7a2b72..ed0b6d40 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - node-version: [14.x, 16.x, 18.x] + node-version: [20.x, 22.x, 24.x] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ steps: diff --git a/tests/test-cases.js b/tests/test-cases.js index 86439035..f20368fc 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -13,6 +13,8 @@ var UNIT_SEP = String.fromCharCode(31); var FILES_ENABLED = false; try { new File([""], ""); // eslint-disable-line no-new + // Required since Node20 as it ads a FileGlobal but not a FileReaderSync + new FileReaderSync(); // eslint-disable-line no-new FILES_ENABLED = true; } catch (e) {} // safari, ie @@ -2817,7 +2819,7 @@ describe('Custom Tests', function() { const blob = new Blob([ ` importScripts('${papaParseScriptPath}'); - + self.addEventListener("message", function(event) { if (event.data === "ExecuteParse") { // Perform our synchronous parse, as requested From 58bcb73d5cb978d1e115a4f40bf38565ce66249e Mon Sep 17 00:00:00 2001 From: Emily Marigold Klassen <760204+forivall@users.noreply.github.com> Date: Mon, 28 Jul 2025 23:26:10 -0700 Subject: [PATCH 59/61] Mention BYTE_ORDER_MARK in documentation (#1096) --- docs/docs.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/docs.html b/docs/docs.html index 79eeb795..ac39363f 100644 --- a/docs/docs.html +++ b/docs/docs.html @@ -877,6 +877,12 @@
Read-Only
An array of characters that are not allowed as delimiters (\r, \n, ", \ufeff). + + Papa.BYTE_ORDER_MARK + + The unicode Byte Order Mark (\ufeff). + + Papa.RECORD_SEP From 8ecc42d3488ef22a6b6c6ae6aab3c8d708abc59b Mon Sep 17 00:00:00 2001 From: David Boskovic Date: Mon, 11 Aug 2025 14:42:25 -0400 Subject: [PATCH 60/61] confirms issue 1024 is resolveD --- tests/test-cases.js | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/test-cases.js b/tests/test-cases.js index f20368fc..3a090e9f 100644 --- a/tests/test-cases.js +++ b/tests/test-cases.js @@ -641,6 +641,7 @@ var CORE_PARSER_TESTS = [ } } }, + ]; describe('Core Parser Tests', function() { @@ -1652,7 +1653,8 @@ var PARSE_TESTS = [ data: [['1','2','3','4']], errors: [] } - } + }, + ]; describe('Parse Tests', function() { @@ -1671,6 +1673,26 @@ describe('Parse Tests', function() { for (var i = 0; i < PARSE_TESTS.length; i++) { generateTest(PARSE_TESTS[i]); } + + // Custom test for Issue 1024 - renamedHeaders regression test + it('Issue 1024: renamedHeaders returned for simple duplicate headers (regression test)', function() { + var result = Papa.parse('Column,Column\n1-1,1-2\n2-1,2-2\n3-1,3-2', { header: true }); + + // Test data structure + assert.deepEqual(result.data, [ + {Column: '1-1', Column_1: '1-2'}, + {Column: '2-1', Column_1: '2-2'}, + {Column: '3-1', Column_1: '3-2'} + ]); + + // Test errors + assert.deepEqual(result.errors, []); + + // Test that renamedHeaders is present and correct + assert.isNotNull(result.meta.renamedHeaders, 'renamedHeaders should not be null'); + assert.isObject(result.meta.renamedHeaders, 'renamedHeaders should be an object'); + assert.deepEqual(result.meta.renamedHeaders, {Column_1: 'Column'}, 'renamedHeaders should contain the renamed header mapping'); + }); }); From b10b87ef8686c6f88299b50dd25e83606e9c36d4 Mon Sep 17 00:00:00 2001 From: Kai <2644614+Schweinepriester@users.noreply.github.com> Date: Wed, 20 Aug 2025 08:47:43 +0200 Subject: [PATCH 61/61] Use https on readme links (#1104) --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 19319385..28ab0f8f 100644 --- a/README.md +++ b/README.md @@ -42,12 +42,12 @@ const csv = Papa.unparse(data[, config]); Homepage & Demo ---------------- -- [Homepage](http://papaparse.com) -- [Demo](http://papaparse.com/demo) +- [Homepage](https://www.papaparse.com) +- [Demo](https://www.papaparse.com/demo) To learn how to use Papa Parse: -- [Documentation](http://papaparse.com/docs) +- [Documentation](https://www.papaparse.com/docs) The website is hosted on [Github Pages](https://pages.github.com/). Its content is also included in the docs folder of this repository. If you want to contribute on it just clone the master of this repository and open a pull request. @@ -62,7 +62,7 @@ Papa Parse can also parse in a node streaming style which makes `.pipe` availabl Get Started ----------- -For usage instructions, see the [homepage](http://papaparse.com) and, for more detail, the [documentation](http://papaparse.com/docs). +For usage instructions, see the [homepage](https://www.papaparse.com) and, for more detail, the [documentation](https://www.papaparse.com/docs). Tests -----