From 2412ae2b6a7c091b2d72dd6a34e711cb7a3f40bc Mon Sep 17 00:00:00 2001 From: Borewit Date: Sat, 14 Dec 2019 13:51:33 +0100 Subject: [PATCH] Add floating-point number tokens: `Float32_BE` & `Float32_LE` --- README.md | 51 +++++++-- lib/index.ts | 187 ++++++++++++++++++++++++------- package.json | 16 ++- test/test-ieee-754-floats.ts | 206 +++++++++++++++++++++++++++++++++++ yarn.lock | 5 + 5 files changed, 410 insertions(+), 55 deletions(-) create mode 100644 test/test-ieee-754-floats.ts diff --git a/README.md b/README.md index 67f8771..778baae 100644 --- a/README.md +++ b/README.md @@ -9,28 +9,55 @@ [![DeepScan grade](https://deepscan.io/api/teams/5165/projects/6940/branches/61852/badge/grade.svg)](https://deepscan.io/dashboard#view=project&tid=5165&pid=6940&bid=61852) [![Known Vulnerabilities](https://snyk.io/test/github/Borewit/token-types/badge.svg?targetFile=package.json)](https://snyk.io/test/github/Borewit/token-types?targetFile=package.json) +# strtok3 + A primitive token library used to read from, and to write a node `Buffer`. +Although it is possible to use this module directly, it is designed to be used with [strtok3 tokenizer](https://github.com/Borewit/strtok3). + +## Installation + +```sh +npm add strtok3 +``` -### Tokens +## Tokens `node-strtok` supports a wide variety of numerical tokens out of the box: -* `UINT8` -* `UINT16_BE`, `UINT16_LE` -* `UINT24_BE`, `UINT24_LE` -* `UINT32_BE`, `UINT32_LE` -* `UINT64_BE`, `UINT64_LE`* -* `INT8` -* `INT16_BE`, `INT16_LE` -* `INT24_BE`, `INT24_LE` -* `INT32_BE`, `INT32_LE` -* `INT64_BE`, `UINT64_LE`* +| Token | Number | Bits | Endianness | +|---------------|------------------|------|----------------| +| `UINT8` | Unsigned integer | 8 | n/a | +| `UINT16_BE` | Unsigned integer | 16 | big endian | +| `UINT16_LE` | Unsigned integer | 16 | little endian | +| `UINT24_BE` | Unsigned integer | 24 | big endian | +| `UINT24_LE` | Unsigned integer | 24 | little endian | +| `UINT32_BE` | Unsigned integer | 32 | big endian | +| `UINT32_LE` | Unsigned integer | 32 | little endian | +| `UINT64_BE` | Unsigned integer | 64 | big endian | +| `UINT64_LE`* | Unsigned integer | 64 | little endian | +| `INT8` | Signed integer | 8 | n/a | +| `INT16_BE` | Signed integer | 16 | big endian | +| `INT16_LE` | Signed integer | 16 | little endian | +| `INT24_BE` | Signed integer | 24 | big endian | +| `INT24_LE` | Signed integer | 24 | little endian | +| `INT32_BE` | Signed integer | 32 | big endian | +| `INT32_LE` | Signed integer | 32 | little endian | +| `INT64_BE` | Signed integer | 64 | big endian | +| `INT64_LE`* | Signed integer | 64 | little endian | +| `Float16_BE` | IEEE 754 float | 16 | big endian | +| `Float16_LE` | IEEE 754 float | 16 | little endian | +| `Float32_BE` | IEEE 754 float | 32 | big endian | +| `Float32_LE` | IEEE 754 float | 32 | little endian | +| `Float64_BE` | IEEE 754 float | 64 | big endian | +| `Float64_LE` | IEEE 754 float | 64 | little endian | +| `Float80_BE`* | IEEE 754 float | 80 | big endian | +| `Float80_LE`* | IEEE 754 float | 80 | little endian | String types: * Windows-1252 * ISO-8859-1 -*) The 64-bit tokens are best effort based, since JavaScript limit value size to less than 2^64. +*) The tokens exceed the JavaScript IEEE 754 64-bit Floating Point precision, decoding and encoding is best effort based. Complex tokens can be added, which makes very suitable for reading binary files or network messages: ```javascript diff --git a/lib/index.ts b/lib/index.ts index d03b670..cee1b0f 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -1,13 +1,14 @@ // A fast streaming parser library. -import * as assert from "assert"; +import * as assert from 'assert'; +import * as ieee754 from 'ieee754'; // Possibly call flush() const maybeFlush = (b, o, len, flush) => { if (o + len > b.length) { - if (typeof(flush) !== "function") { + if (typeof (flush) !== 'function') { throw new Error( - "Buffer out of space and no valid flush() function found" + 'Buffer out of space and no valid flush() function found' ); } @@ -61,8 +62,8 @@ export const UINT8: IToken = { }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= 0 && v <= 0xff); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -86,8 +87,8 @@ export const UINT16_LE: IToken = { }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= 0 && v <= 0xffff); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -111,8 +112,8 @@ export const UINT16_BE: IToken = { }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= 0 && v <= 0xffff); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -136,8 +137,8 @@ export const UINT24_LE: IToken = { }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= 0 && v <= 0xffffff); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -158,8 +159,8 @@ export const UINT24_BE: IToken = { return buf.readUIntBE(off, 3); }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= 0 && v <= 0xffffff); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -183,8 +184,8 @@ export const UINT32_LE: IToken = { }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= 0 && v <= 0xffffffff); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -208,8 +209,8 @@ export const UINT32_BE: IToken = { }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= 0 && v <= 0xffffffff); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -233,8 +234,8 @@ export const INT8: IToken = { }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= -128 && v <= 127); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -255,8 +256,8 @@ export const INT16_BE: IToken = { return buf.readInt16BE(off); }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= -32768 && v <= 32767); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -277,8 +278,8 @@ export const INT16_LE: IToken = { return buf.readInt16LE(off); }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= -32768 && v <= 32767); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -299,8 +300,8 @@ export const INT24_LE: IToken = { return buf.readIntLE(off, 3); }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= -0x800000 && v <= 0x7fffff); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -321,8 +322,8 @@ export const INT24_BE: IToken = { return buf.readIntBE(off, 3); }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= -0x800000 && v <= 0x7fffff); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -343,8 +344,8 @@ export const INT32_BE: IToken = { return buf.readInt32BE(off); }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= -2147483648 && v <= 2147483647); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -365,8 +366,8 @@ export const INT32_LE: IToken = { return buf.readInt32LE(off); }, put(b: Buffer, o: number, v: number, flush?: IFlush): number { - assert.equal(typeof o, "number"); - assert.equal(typeof v, "number"); + assert.equal(typeof o, 'number'); + assert.equal(typeof v, 'number'); assert.ok(v >= -2147483648 && v <= 2147483647); assert.ok(o >= 0); assert.ok(this.len <= b.length); @@ -387,7 +388,7 @@ export const UINT64_LE: IToken = { return readUIntLE(buf, off, this.len); }, put(b: Buffer, o: number, v: number): number { - return writeUIntLE(b, v, o, this.len); + return writeUIntLE(b, v, o, this.len); } }; @@ -400,7 +401,7 @@ export const INT64_LE: IToken = { return readIntLE(buf, off, this.len); }, put(b: Buffer, off: number, v: number): number { - return writeIntLE(b, v, off, this.len); + return writeIntLE(b, v, off, this.len); } }; @@ -409,11 +410,11 @@ export const INT64_LE: IToken = { */ export const UINT64_BE: IToken = { len: 8, - get(buf: Buffer, off: number): number { - return readUIntBE(buf, off, this.len); + get(b: Buffer, off: number): number { + return readUIntBE(b, off, this.len); }, put(b: Buffer, o: number, v: number): number { - return writeUIntBE(b, v, o, this.len); + return writeUIntBE(b, v, o, this.len); } }; @@ -422,11 +423,115 @@ export const UINT64_BE: IToken = { */ export const INT64_BE: IToken = { len: 8, - get(buf: Buffer, off: number): number { - return readIntBE(buf, off, this.len); + get(b: Buffer, off: number): number { + return readIntBE(b, off, this.len); + }, + put(b: Buffer, off: number, v: number): number { + return writeIntBE(b, v, off, this.len); + } +}; + +/** + * IEEE 754 16-bit (half precision) float, big endian + */ +export const Float16_BE: IToken = { + len: 2, + get(b: Buffer, off: number): number { + return ieee754.read(b, off, false, 10, this.len); + }, + put(b: Buffer, off: number, v: number): number { + return ieee754.write(b, v, off, false, 10, this.len); + } +}; + +/** + * IEEE 754 16-bit (half precision) float, little endian + */ +export const Float16_LE: IToken = { + len: 2, + get(b: Buffer, off: number): number { + return ieee754.read(b, off, true, 10, this.len); + }, + put(b: Buffer, off: number, v: number): number { + return ieee754.write(b, v, off, true, 10, this.len); + } +}; + +/** + * IEEE 754 32-bit (single precision) float, big endian + */ +export const Float32_BE: IToken = { + len: 4, + get(b: Buffer, off: number): number { + return b.readFloatBE(off); + }, + put(b: Buffer, off: number, v: number): number { + return b.writeFloatBE(v, off); + } +}; + +/** + * IEEE 754 32-bit (single precision) float, little endian + */ +export const Float32_LE: IToken = { + len: 4, + get(b: Buffer, off: number): number { + return b.readFloatLE(off); + }, + put(b: Buffer, off: number, v: number): number { + return b.writeFloatLE(v, off); + } +}; + +/** + * IEEE 754 64-bit (double precision) float, big endian + */ +export const Float64_BE: IToken = { + len: 8, + get(b: Buffer, off: number): number { + return b.readDoubleBE(off); + }, + put(b: Buffer, off: number, v: number): number { + return b.writeDoubleBE(v, off); + } +}; + +/** + * IEEE 754 64-bit (double precision) float, little endian + */ +export const Float64_LE: IToken = { + len: 8, + get(b: Buffer, off: number): number { + return b.readDoubleLE(off); + }, + put(b: Buffer, off: number, v: number): number { + return b.writeDoubleLE(v, off); + } +}; + +/** + * IEEE 754 80-bit (extended precision) float, big endian + */ +export const Float80_BE: IToken = { + len: 10, + get(b: Buffer, off: number): number { + return ieee754.read(b, off, false, 63, this.len); + }, + put(b: Buffer, off: number, v: number): number { + return ieee754.write(b, v, off, false, 63, this.len); + } +}; + +/** + * IEEE 754 80-bit (extended precision) float, little endian + */ +export const Float80_LE: IToken = { + len: 10, + get(b: Buffer, off: number): number { + return ieee754.read(b, off, true, 63, this.len); }, put(b: Buffer, off: number, v: number): number { - return writeIntBE(b, v, off, this.len); + return ieee754.write(b, v, off, true, 63, this.len); } }; @@ -486,7 +591,7 @@ export class AnsiStringType implements IGetToken { 248, 249, 250, 251, 252, 253, 254, 255]; private static decode(buffer: Uint8Array, off: number, until: number): string { - let str = ""; + let str = ''; for (let i = off; i < until; ++i) { str += AnsiStringType.codePointToString(AnsiStringType.singleByteDecoder(buffer[i])); } @@ -513,7 +618,7 @@ export class AnsiStringType implements IGetToken { const codePoint = AnsiStringType.windows1252[bite - 0x80]; if (codePoint === null) { - throw Error("invaliding encoding"); + throw Error('invaliding encoding'); } return codePoint; diff --git a/package.json b/package.json index 1ec1cd3..bb9e533 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,9 @@ "tslint": "^5.20.1", "typescript": "^3.7.2" }, - "dependencies": {}, + "dependencies": { + "ieee754": "^1.1.13" + }, "remarkConfig": { "plugins": [ "preset-lint-recommended" @@ -73,5 +75,15 @@ "text" ], "report-dir": "coverage" - } + }, + "keywords": [ + "token", + "integer", + "unsigned", + "numeric", + "float", + "IEEE", + "754", + "strtok3" + ] } diff --git a/test/test-ieee-754-floats.ts b/test/test-ieee-754-floats.ts new file mode 100644 index 0000000..a960939 --- /dev/null +++ b/test/test-ieee-754-floats.ts @@ -0,0 +1,206 @@ +// Test writing and reading uint8 values. + +import {} from 'mocha'; +import { assert } from 'chai'; +import * as Token from '../lib'; +import * as util from './util'; + +describe('IEEE 754 floats', () => { + + describe('16-bit (half precision)', () => { + + describe('big-endian', () => { + + it('should encode', () => { + + const buf = Buffer.alloc(2); + + Token.Float16_BE.put(buf, 0, 0.0); + util.checkBuffer(buf, '0000'); + + Token.Float16_BE.put(buf, 0, 85.125); + util.checkBuffer(buf, '5552'); + + Token.Float16_BE.put(buf, 0, -1); + util.checkBuffer(buf, 'bc00'); + }); + + it('should decode', () => { + const buf = Buffer.from('\x55\x52', 'binary'); + assert.equal(Token.Float16_BE.get(buf, 0), 85.125); + }); + + }); + + describe('little-endian', () => { + + it('should encode', () => { + + const buf = Buffer.alloc(2); + + Token.Float16_LE.put(buf, 0, 0.0); + util.checkBuffer(buf, '0000'); + + Token.Float16_LE.put(buf, 0, 85.125); + util.checkBuffer(buf, '5255'); + + Token.Float16_LE.put(buf, 0, -1); + util.checkBuffer(buf, '00bc'); + }); + + it('should decode', () => { + const buf = Buffer.from('\x52\x55', 'binary'); + assert.equal(Token.Float16_LE.get(buf, 0), 85.125); + }); + + }); + }); + + describe('32-bit (single precision)', () => { + + describe('big-endian', () => { + + it('should encode', () => { + + const buf = Buffer.alloc(4); + + Token.Float32_BE.put(buf, 0, 0.0); + util.checkBuffer(buf, '00000000'); + + Token.Float32_BE.put(buf, 0, 85.125); + util.checkBuffer(buf, '42aa4000'); + + Token.Float32_BE.put(buf, 0, -1); + util.checkBuffer(buf, 'bf800000'); + }); + + it('should decode', () => { + const buf = Buffer.from('\x42\xAA\x40\x00', 'binary'); + assert.equal(Token.Float32_BE.get(buf, 0), 85.125); + }); + + }); + + describe('little-endian', () => { + + it('should encode', () => { + + const buf = Buffer.alloc(4); + + Token.Float32_LE.put(buf, 0, 0.0); + util.checkBuffer(buf, '00000000'); + + Token.Float32_LE.put(buf, 0, 85.125); + util.checkBuffer(buf, '0040aa42'); + + Token.Float32_LE.put(buf, 0, -1); + util.checkBuffer(buf, '000080bf'); + }); + + it('should decode', () => { + const buf = Buffer.from('\x00\x40\xAA\x42', 'binary'); + assert.equal(Token.Float32_LE.get(buf, 0), 85.125); + }); + + }); + }); + + describe('64-bit (double precision)', () => { + + describe('big-endian', () => { + + it('should encode', () => { + + const buf = Buffer.alloc(8); + + Token.Float64_BE.put(buf, 0, 0.0); + util.checkBuffer(buf, '0000000000000000'); + + Token.Float64_BE.put(buf, 0, 85.125); + util.checkBuffer(buf, '4055480000000000'); + + Token.Float64_BE.put(buf, 0, -1); + util.checkBuffer(buf, 'bff0000000000000'); + }); + + it('should decode', () => { + const buf = Buffer.from('\x40\x55\x48\x00\x00\x00\x00\x00', 'binary'); + assert.equal(Token.Float64_BE.get(buf, 0), 85.125); + }); + + }); + + describe('little-endian', () => { + + it('should encode', () => { + + const buf = Buffer.alloc(8); + + Token.Float64_LE.put(buf, 0, 0.0); + util.checkBuffer(buf, '0000000000000000'); + + Token.Float64_LE.put(buf, 0, 85.125); + util.checkBuffer(buf, '0000000000485540'); + + Token.Float64_LE.put(buf, 0, -1); + util.checkBuffer(buf, '000000000000f0bf'); + }); + + it('should decode', () => { + const buf = Buffer.from('\x00\x00\x00\x00\x00\x48\x55\x40', 'binary'); + assert.equal(Token.Float64_LE.get(buf, 0), 85.125); + }); + + }); + }); + + describe('80-bit (extended precision)', () => { + + describe('big-endian', () => { + + it('should encode', () => { + + const buf = Buffer.alloc(10); + + Token.Float80_BE.put(buf, 0, 0.0); + util.checkBuffer(buf, '00000000000000000000'); + + Token.Float80_BE.put(buf, 0, 85.125); + util.checkBuffer(buf, '4002aa40000000000000'); + + Token.Float80_BE.put(buf, 0, -1); + util.checkBuffer(buf, 'bfff8000000000000000'); + }); + + it('should decode', () => { + const buf = Buffer.from('\x40\x02\xAA\x40\x00\x00\x00\x00\x00\x00\x00\x00', 'binary'); + assert.equal(Token.Float80_BE.get(buf, 0), 85.125); + }); + + }); + + describe('little-endian', () => { + + it('should encode', () => { + + const buf = Buffer.alloc(10); + + Token.Float80_LE.put(buf, 0, 0.0); + util.checkBuffer(buf, '00000000000000000000'); + + Token.Float80_LE.put(buf, 0, 85.125); + util.checkBuffer(buf, '00000000000040aa0240'); + + Token.Float80_LE.put(buf, 0, -1); + util.checkBuffer(buf, '0000000000000080ffbf'); + }); + + it.skip('should decode', () => { + const buf = Buffer.from('\x00\x00\x00\x00\x00\x00\x00\x00\x40\xAA\x02\x40', 'binary'); + assert.equal(Token.Float80_LE.get(buf, 0), 85.125); + }); + + }); + }); + +}); diff --git a/yarn.lock b/yarn.lock index c1724b7..911574e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -994,6 +994,11 @@ http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.7.0" +ieee754@^1.1.13: + version "1.1.13" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" + integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== + ignore@^5.0.0, ignore@^5.1.1: version "5.1.4" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.4.tgz#84b7b3dbe64552b6ef0eca99f6743dbec6d97adf"