diff --git a/.changeset/eleven-chicken-live.md b/.changeset/eleven-chicken-live.md new file mode 100644 index 0000000..dff139f --- /dev/null +++ b/.changeset/eleven-chicken-live.md @@ -0,0 +1,5 @@ +--- +"toml-eslint-parser": patch +--- + +fix: wrong values for floats and datetimes. diff --git a/.eslintrc.js b/.eslintrc.js index 7fdd9a7..756f83b 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -64,6 +64,10 @@ module.exports = { selector: "method", format: null, }, + { + selector: "import", + format: null, + }, ], "no-implicit-globals": "off", "@typescript-eslint/no-non-null-assertion": "off", diff --git a/src/ast/token.ts b/src/ast/token.ts index 2a12d50..88994c2 100644 --- a/src/ast/token.ts +++ b/src/ast/token.ts @@ -71,4 +71,5 @@ export interface BooleanToken extends BaseTOMLToken { export interface DateTimeToken extends BaseTOMLToken { type: "OffsetDateTime" | "LocalDateTime" | "LocalDate" | "LocalTime"; value: string; + date: Date; } diff --git a/src/tokenizer/tokenizer.ts b/src/tokenizer/tokenizer.ts index 67df5ff..e696dc7 100644 --- a/src/tokenizer/tokenizer.ts +++ b/src/tokenizer/tokenizer.ts @@ -122,11 +122,12 @@ const ESCAPES: Record = { }; type ExponentData = { - left: number; + minus: boolean; + left: number[]; }; type FractionalData = { minus: boolean; - absInt: number; + absInt: number[]; }; type DateTimeData = { hasDate: boolean; @@ -137,6 +138,9 @@ type DateTimeData = { hour: number; minute: number; second: number; + + frac?: number[]; + offsetSign?: number; }; /** @@ -283,7 +287,7 @@ export class Tokenizer { } private endToken( - type: BareToken["type"] | Comment["type"] | DateTimeToken["type"], + type: BareToken["type"] | Comment["type"], pos: "start" | "end", ): void; @@ -313,6 +317,12 @@ export class Tokenizer { value: number, ): void; + private endToken( + type: DateTimeToken["type"], + pos: "start" | "end", + value: Date, + ): void; + private endToken( type: BooleanToken["type"], pos: "start" | "end", @@ -325,7 +335,7 @@ export class Tokenizer { private endToken( type: TokenType | Comment["type"], pos: "start" | "end", - option1?: number[] | number | boolean, + option1?: number[] | number | boolean | Date, option2?: 16 | 10 | 8 | 2, ): void { const { tokenStart } = this; @@ -396,6 +406,19 @@ export class Tokenizer { range, loc, }; + } else if ( + type === "LocalDate" || + type === "LocalTime" || + type === "LocalDateTime" || + type === "OffsetDateTime" + ) { + token = { + type, + value, + date: option1! as Date, + range, + loc, + }; } else { token = { type, @@ -764,7 +787,8 @@ export class Tokenizer { if (cp === LATIN_SMALL_E || cp === LATIN_CAPITAL_E) { const data: ExponentData = { // Float values -0.0 and +0.0 are valid and should map according to IEEE 754. - left: sign === DASH ? -0 : 0, + minus: sign === DASH, + left: [DIGIT_0], }; this.data = data; return "EXPONENT_RIGHT"; @@ -772,7 +796,7 @@ export class Tokenizer { if (cp === DOT) { const data: FractionalData = { minus: sign === DASH, - absInt: 0, + absInt: [DIGIT_0], }; this.data = data; return "FRACTIONAL_RIGHT"; @@ -821,9 +845,9 @@ export class Tokenizer { } if (nextCp === LATIN_SMALL_E || nextCp === LATIN_CAPITAL_E) { - const absNum = Number(String.fromCodePoint(...codePoints)); const data: ExponentData = { - left: sign === DASH ? -absNum : absNum, + minus: sign === DASH, + left: codePoints, }; this.data = data; return "EXPONENT_RIGHT"; @@ -831,7 +855,7 @@ export class Tokenizer { if (nextCp === DOT) { const data: FractionalData = { minus: sign === DASH, - absInt: Number(String.fromCodePoint(...codePoints)), + absInt: codePoints, }; this.data = data; return "FRACTIONAL_RIGHT"; @@ -869,34 +893,42 @@ export class Tokenizer { private FRACTIONAL_RIGHT(cp: number): TokenizerState { const { minus, absInt } = this.data! as FractionalData; const { codePoints, nextCp } = this.parseDigits(cp, isDigit); - const absNum = - absInt + - Number(String.fromCodePoint(...codePoints)) * - Math.pow(10, -codePoints.length); + const absNum = [...absInt, DOT, ...codePoints]; if (nextCp === LATIN_SMALL_E || nextCp === LATIN_CAPITAL_E) { const data: ExponentData = { - left: minus ? -absNum : absNum, + minus, + left: absNum, }; this.data = data; return "EXPONENT_RIGHT"; } - this.endToken("Float", "start", minus ? -absNum : absNum); + const value = Number( + minus + ? String.fromCodePoint(DASH, ...absNum) + : String.fromCodePoint(...absNum), + ); + this.endToken("Float", "start", value); return this.back("DATA"); } private EXPONENT_RIGHT(cp: number): TokenizerState { - const { left } = this.data! as ExponentData; + const { left, minus: leftMinus } = this.data! as ExponentData; let minus = false; if (cp === DASH || cp === PLUS_SIGN) { minus = cp === DASH; cp = this.nextCode(); } const { codePoints } = this.parseDigits(cp, isDigit); - let right = Number(String.fromCodePoint(...codePoints)); + let right = codePoints; if (minus) { - right = 0 - right; + right = [DASH, ...right]; } - this.endToken("Float", "start", left * Math.pow(10, right)); + const value = Number( + leftMinus + ? String.fromCodePoint(DASH, ...left, LATIN_SMALL_E, ...right) + : String.fromCodePoint(...left, LATIN_SMALL_E, ...right), + ); + this.endToken("Float", "start", value); return this.back("DATA"); } @@ -990,7 +1022,8 @@ export class Tokenizer { return "TIME_HOUR"; } } - this.endToken("LocalDate", "start"); + const dateValue = getDateFromDateTimeData(data, "Z"); + this.endToken("LocalDate", "start", dateValue); return this.back("DATA"); } @@ -1063,16 +1096,20 @@ export class Tokenizer { } if (data.hasDate) { if (cp === DASH || cp === PLUS_SIGN) { + data.offsetSign = cp; return "TIME_OFFSET"; } if (cp === LATIN_CAPITAL_Z || cp === LATIN_SMALL_Z) { - this.endToken("OffsetDateTime", "end"); + const dateValue = getDateFromDateTimeData(data, "Z"); + this.endToken("OffsetDateTime", "end", dateValue); return "DATA"; } - this.endToken("LocalDateTime", "start"); + const dateValue = getDateFromDateTimeData(data, ""); + this.endToken("LocalDateTime", "start", dateValue); return this.back("DATA"); } - this.endToken("LocalTime", "start"); + const dateValue = getDateFromDateTimeData(data, "Z"); + this.endToken("LocalTime", "start", dateValue); return this.back("DATA"); } @@ -1080,22 +1117,29 @@ export class Tokenizer { if (!isDigit(cp)) { return this.reportParseError("unexpected-char"); } + const codePoints = []; while (isDigit(cp)) { + codePoints.push(cp); cp = this.nextCode(); } const data: DateTimeData = this.data! as DateTimeData; + data.frac = codePoints; if (data.hasDate) { if (cp === DASH || cp === PLUS_SIGN) { + data.offsetSign = cp; return "TIME_OFFSET"; } if (cp === LATIN_CAPITAL_Z || cp === LATIN_SMALL_Z) { - this.endToken("OffsetDateTime", "end"); + const dateValue = getDateFromDateTimeData(data, "Z"); + this.endToken("OffsetDateTime", "end", dateValue); return "DATA"; } - this.endToken("LocalDateTime", "start"); + const dateValue = getDateFromDateTimeData(data, ""); + this.endToken("LocalDateTime", "start", dateValue); return this.back("DATA"); } - this.endToken("LocalTime", "start"); + const dateValue = getDateFromDateTimeData(data, "Z"); + this.endToken("LocalTime", "start", dateValue); return this.back("DATA"); } @@ -1133,7 +1177,15 @@ export class Tokenizer { return this.reportParseError("invalid-time"); } - this.endToken("OffsetDateTime", "end"); + const data: DateTimeData = this.data! as DateTimeData; + const dateValue = getDateFromDateTimeData( + data, + `${String.fromCodePoint(data.offsetSign!)}${padStart(hour, 2)}:${padStart( + minute, + 2, + )}`, + ); + this.endToken("OffsetDateTime", "end", dateValue); return "DATA"; } @@ -1233,3 +1285,32 @@ function isValidTime(h: number, m: number, s: number): boolean { } return true; } + +/** + * Get date from DateTimeData + */ +function getDateFromDateTimeData(data: DateTimeData, timeZone: string): Date { + const year = padStart(data.year, 4); + const month = data.month ? padStart(data.month, 2) : "01"; + const day = data.day ? padStart(data.day, 2) : "01"; + const hour = padStart(data.hour, 2); + const minute = padStart(data.minute, 2); + const second = padStart(data.second, 2); + const textDate = `${year}-${month}-${day}`; + const frac = data.frac ? `.${String.fromCodePoint(...data.frac)}` : ""; + const dateValue = new Date( + `${textDate}T${hour}:${minute}:${second}${frac}${timeZone}`, + ); + if (!isNaN(dateValue.getTime()) || data.second !== 60) { + return dateValue; + } + // leap seconds? + return new Date(`${textDate}T${hour}:${minute}:59${frac}${timeZone}`); +} + +/** + * Pad with zeros. + */ +function padStart(num: number, maxLength: number): string { + return String(num).padStart(maxLength, "0"); +} diff --git a/src/toml-parser/index.ts b/src/toml-parser/index.ts index df5a03f..6ff7f31 100644 --- a/src/toml-parser/index.ts +++ b/src/toml-parser/index.ts @@ -411,18 +411,10 @@ export class TOMLParser { ctx: Context, ): ParserState[] { const valueContainer = ctx.consumeValueContainer(); - let textDate = - token.type !== "LocalTime" ? token.value : `0000-01-01T${token.value}Z`; - let dateValue = new Date(textDate); - if (isNaN(dateValue.getTime())) { - // leap seconds? - textDate = textDate.replace(/(\d{2}:\d{2}):60/u, "$1:59"); - dateValue = new Date(textDate); - } const node: TOMLDateTimeValue = { type: "TOMLValue", kind: DATETIME_VALUE_KIND_MAP[token.type], - value: dateValue, + value: token.date, datetime: token.value, parent: valueContainer.parent, range: cloneRange(token.range), diff --git a/tests/fixtures/parser/ast/date-time03-min-input.toml b/tests/fixtures/parser/ast/date-time03-min-input.toml new file mode 100644 index 0000000..30cdcd6 --- /dev/null +++ b/tests/fixtures/parser/ast/date-time03-min-input.toml @@ -0,0 +1,3 @@ +first-offset = 0001-01-01 00:00:00Z +first-local = 0001-01-01 00:00:00 +first-date = 0001-01-01 diff --git a/tests/fixtures/parser/ast/date-time03-min-output.json b/tests/fixtures/parser/ast/date-time03-min-output.json new file mode 100644 index 0000000..544b7c8 --- /dev/null +++ b/tests/fixtures/parser/ast/date-time03-min-output.json @@ -0,0 +1,426 @@ +{ + "type": "Program", + "body": [ + { + "type": "TOMLTopLevelTable", + "body": [ + { + "type": "TOMLKeyValue", + "key": { + "type": "TOMLKey", + "keys": [ + { + "type": "TOMLBare", + "name": "first-offset", + "range": [ + 0, + 12 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 12 + } + } + } + ], + "range": [ + 0, + 12 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 12 + } + } + }, + "value": { + "type": "TOMLValue", + "kind": "offset-date-time", + "value": "0001-01-01T00:00:00.000Z", + "datetime": "0001-01-01 00:00:00Z", + "range": [ + 15, + 35 + ], + "loc": { + "start": { + "line": 1, + "column": 15 + }, + "end": { + "line": 1, + "column": 35 + } + } + }, + "range": [ + 0, + 35 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 35 + } + } + }, + { + "type": "TOMLKeyValue", + "key": { + "type": "TOMLKey", + "keys": [ + { + "type": "TOMLBare", + "name": "first-local", + "range": [ + 36, + 47 + ], + "loc": { + "start": { + "line": 2, + "column": 0 + }, + "end": { + "line": 2, + "column": 11 + } + } + } + ], + "range": [ + 36, + 47 + ], + "loc": { + "start": { + "line": 2, + "column": 0 + }, + "end": { + "line": 2, + "column": 11 + } + } + }, + "value": { + "type": "TOMLValue", + "kind": "local-date-time", + "value": "0000-12-31T14:41:01.000Z", + "datetime": "0001-01-01 00:00:00", + "range": [ + 51, + 70 + ], + "loc": { + "start": { + "line": 2, + "column": 15 + }, + "end": { + "line": 2, + "column": 34 + } + } + }, + "range": [ + 36, + 70 + ], + "loc": { + "start": { + "line": 2, + "column": 0 + }, + "end": { + "line": 2, + "column": 34 + } + } + }, + { + "type": "TOMLKeyValue", + "key": { + "type": "TOMLKey", + "keys": [ + { + "type": "TOMLBare", + "name": "first-date", + "range": [ + 71, + 81 + ], + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 10 + } + } + } + ], + "range": [ + 71, + 81 + ], + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 10 + } + } + }, + "value": { + "type": "TOMLValue", + "kind": "local-date", + "value": "0001-01-01T00:00:00.000Z", + "datetime": "0001-01-01", + "range": [ + 86, + 96 + ], + "loc": { + "start": { + "line": 3, + "column": 15 + }, + "end": { + "line": 3, + "column": 25 + } + } + }, + "range": [ + 71, + 96 + ], + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 25 + } + } + } + ], + "range": [ + 0, + 96 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 25 + } + } + } + ], + "sourceType": "module", + "tokens": [ + { + "type": "Bare", + "value": "first-offset", + "range": [ + 0, + 12 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 12 + } + } + }, + { + "type": "Punctuator", + "value": "=", + "range": [ + 13, + 14 + ], + "loc": { + "start": { + "line": 1, + "column": 13 + }, + "end": { + "line": 1, + "column": 14 + } + } + }, + { + "type": "OffsetDateTime", + "value": "0001-01-01 00:00:00Z", + "range": [ + 15, + 35 + ], + "loc": { + "start": { + "line": 1, + "column": 15 + }, + "end": { + "line": 1, + "column": 35 + } + } + }, + { + "type": "Bare", + "value": "first-local", + "range": [ + 36, + 47 + ], + "loc": { + "start": { + "line": 2, + "column": 0 + }, + "end": { + "line": 2, + "column": 11 + } + } + }, + { + "type": "Punctuator", + "value": "=", + "range": [ + 49, + 50 + ], + "loc": { + "start": { + "line": 2, + "column": 13 + }, + "end": { + "line": 2, + "column": 14 + } + } + }, + { + "type": "LocalDateTime", + "value": "0001-01-01 00:00:00", + "range": [ + 51, + 70 + ], + "loc": { + "start": { + "line": 2, + "column": 15 + }, + "end": { + "line": 2, + "column": 34 + } + } + }, + { + "type": "Bare", + "value": "first-date", + "range": [ + 71, + 81 + ], + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 10 + } + } + }, + { + "type": "Punctuator", + "value": "=", + "range": [ + 84, + 85 + ], + "loc": { + "start": { + "line": 3, + "column": 13 + }, + "end": { + "line": 3, + "column": 14 + } + } + }, + { + "type": "LocalDate", + "value": "0001-01-01", + "range": [ + 86, + 96 + ], + "loc": { + "start": { + "line": 3, + "column": 15 + }, + "end": { + "line": 3, + "column": 25 + } + } + } + ], + "comments": [], + "range": [ + 0, + 97 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 4, + "column": 0 + } + } +} \ No newline at end of file diff --git a/tests/fixtures/parser/ast/date-time03-min-value.json b/tests/fixtures/parser/ast/date-time03-min-value.json new file mode 100644 index 0000000..11b69a6 --- /dev/null +++ b/tests/fixtures/parser/ast/date-time03-min-value.json @@ -0,0 +1,5 @@ +{ + "first-offset": "0001-01-01T00:00:00.000Z", + "first-local": "0000-12-31T14:41:01.000Z", + "first-date": "0001-01-01T00:00:00.000Z" +} \ No newline at end of file diff --git a/tests/fixtures/parser/ast/float03-e-input.toml b/tests/fixtures/parser/ast/float03-e-input.toml new file mode 100644 index 0000000..98ed2e5 --- /dev/null +++ b/tests/fixtures/parser/ast/float03-e-input.toml @@ -0,0 +1 @@ +key1 = 1.11e1 diff --git a/tests/fixtures/parser/ast/float03-e-output.json b/tests/fixtures/parser/ast/float03-e-output.json new file mode 100644 index 0000000..3adb132 --- /dev/null +++ b/tests/fixtures/parser/ast/float03-e-output.json @@ -0,0 +1,171 @@ +{ + "type": "Program", + "body": [ + { + "type": "TOMLTopLevelTable", + "body": [ + { + "type": "TOMLKeyValue", + "key": { + "type": "TOMLKey", + "keys": [ + { + "type": "TOMLBare", + "name": "key1", + "range": [ + 0, + 4 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 4 + } + } + } + ], + "range": [ + 0, + 4 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 4 + } + } + }, + "value": { + "type": "TOMLValue", + "kind": "float", + "value": 11.1, + "number": "1.11e1", + "range": [ + 7, + 13 + ], + "loc": { + "start": { + "line": 1, + "column": 7 + }, + "end": { + "line": 1, + "column": 13 + } + } + }, + "range": [ + 0, + 13 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 13 + } + } + } + ], + "range": [ + 0, + 13 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 13 + } + } + } + ], + "sourceType": "module", + "tokens": [ + { + "type": "Bare", + "value": "key1", + "range": [ + 0, + 4 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 4 + } + } + }, + { + "type": "Punctuator", + "value": "=", + "range": [ + 5, + 6 + ], + "loc": { + "start": { + "line": 1, + "column": 5 + }, + "end": { + "line": 1, + "column": 6 + } + } + }, + { + "type": "Float", + "value": "1.11e1", + "number": 11.1, + "range": [ + 7, + 13 + ], + "loc": { + "start": { + "line": 1, + "column": 7 + }, + "end": { + "line": 1, + "column": 13 + } + } + } + ], + "comments": [], + "range": [ + 0, + 14 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 2, + "column": 0 + } + } +} \ No newline at end of file diff --git a/tests/fixtures/parser/ast/float03-e-value.json b/tests/fixtures/parser/ast/float03-e-value.json new file mode 100644 index 0000000..694b7dd --- /dev/null +++ b/tests/fixtures/parser/ast/float03-e-value.json @@ -0,0 +1,3 @@ +{ + "key1": 11.1 +} \ No newline at end of file diff --git a/tests/src/parser/parser.ts b/tests/src/parser/parser.ts index 0166368..8a78ff8 100644 --- a/tests/src/parser/parser.ts +++ b/tests/src/parser/parser.ts @@ -106,6 +106,7 @@ describe("Check for AST.", () => { "local-date-time-sample01-input.toml", "table-sample11-top-level-table-input.toml", "date01-leading-zero-input.toml", + "date-time03-min-input.toml", "leap-year01-input.toml", "leap-year02-input.toml", "spec-time-1.toml", diff --git a/tests/src/parser/utils.ts b/tests/src/parser/utils.ts index 51b30ac..ae0c217 100644 --- a/tests/src/parser/utils.ts +++ b/tests/src/parser/utils.ts @@ -22,6 +22,10 @@ function replacer(key: string, value: any) { return `# ${String(value)} #`; } } + if (key === "date" && /^\d{4}-\d{2}-\d{2}T/u.test(value)) { + // Backward compatibility + return undefined; + } return value; }