From 9c8aa1806c5579c9ed91027e0ae390da6974b6c7 Mon Sep 17 00:00:00 2001 From: tombatossals Date: Mon, 3 Mar 2014 23:58:20 +0100 Subject: [PATCH 1/3] fix(lexer): Remove the Byte Order Mark of the parsed buffer/string if present --- ragel/lexer.js.rl.erb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ragel/lexer.js.rl.erb b/ragel/lexer.js.rl.erb index c2bc7740..9e0f89c2 100644 --- a/ragel/lexer.js.rl.erb +++ b/ragel/lexer.js.rl.erb @@ -145,9 +145,14 @@ var Lexer = function(listener) { Lexer.prototype.scan = function(data) { var ending = "\n%_FEATURE_END_%"; if(typeof data == 'string') { + data = data.replace(/^\uFEFF/, ''); data = this.stringToBytes(data + ending); } else if(typeof Buffer != 'undefined' && Buffer.isBuffer(data)) { // Node.js + // Check for Byte Order Mark + if (data[0] === 239 && data[1] === 187 && data[2] === 191) { + data = data.slice(3, data.length); + } var buf = new Buffer(data.length + ending.length); data.copy(buf, 0, 0); new Buffer(ending).copy(buf, data.length, 0); From a90c9fc887a8d5039fdb7b701c28648193e71bfb Mon Sep 17 00:00:00 2001 From: tombatossals Date: Tue, 4 Mar 2014 00:00:34 +0100 Subject: [PATCH 2/3] Added comment for byte order mark removal code --- ragel/lexer.js.rl.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/ragel/lexer.js.rl.erb b/ragel/lexer.js.rl.erb index 9e0f89c2..1af29c4b 100644 --- a/ragel/lexer.js.rl.erb +++ b/ragel/lexer.js.rl.erb @@ -145,6 +145,7 @@ var Lexer = function(listener) { Lexer.prototype.scan = function(data) { var ending = "\n%_FEATURE_END_%"; if(typeof data == 'string') { + // Check for Byte Order Mark data = data.replace(/^\uFEFF/, ''); data = this.stringToBytes(data + ending); } else if(typeof Buffer != 'undefined' && Buffer.isBuffer(data)) { From e34ad2192b41ad7fb2b36b94cb92d5567a7f60fc Mon Sep 17 00:00:00 2001 From: tombatossals Date: Tue, 4 Mar 2014 15:47:01 +0100 Subject: [PATCH 3/3] Compare with Hex values, and reduced buf.slice call --- ragel/lexer.js.rl.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ragel/lexer.js.rl.erb b/ragel/lexer.js.rl.erb index 1af29c4b..249f90b3 100644 --- a/ragel/lexer.js.rl.erb +++ b/ragel/lexer.js.rl.erb @@ -151,8 +151,8 @@ Lexer.prototype.scan = function(data) { } else if(typeof Buffer != 'undefined' && Buffer.isBuffer(data)) { // Node.js // Check for Byte Order Mark - if (data[0] === 239 && data[1] === 187 && data[2] === 191) { - data = data.slice(3, data.length); + if (data[0] === 0xEF && data[1] === 0xBE && data[2] === 0xBB) { + data = data.slice(3); } var buf = new Buffer(data.length + ending.length); data.copy(buf, 0, 0);