diff --git a/scripts/bnf.js b/scripts/bnf.js index c11b619..e7f33d9 100755 --- a/scripts/bnf.js +++ b/scripts/bnf.js @@ -84,7 +84,7 @@ try { let tokens; try { - tokens = bnf.tokenize(contents); + tokens = bnf.tokenize(contents, input); } catch (e) { console.error(is_issue(e) ? stringify_issue(e, options) : e.message); process.exit(1); @@ -125,7 +125,7 @@ function include(path) { } } -let config = bnf.create_config(ast, { log: logger(verbose), include }); +let config = bnf.create_config(ast, { log: logger(verbose), include, id: input }); if (options.compress) config = compress_config(config); diff --git a/src/bnf.ts b/src/bnf.ts index c0c6c3b..c6e9f85 100644 --- a/src/bnf.ts +++ b/src/bnf.ts @@ -14,14 +14,14 @@ export { bnf_config as config }; /** * Shortcut for tokenize(source, bnf.literals); */ -function tokenizeBnf(source: string): Token[] { - return tokenize(source, bnf_config.literals); +function tokenizeBnf(source: string, unit?: string): Token[] { + return tokenize(source, bnf_config.literals, unit); } export { tokenizeBnf as tokenize }; -export function parseSource(source: string, log?: LogFn): AST { - return parse({ ...bnf_config, log, source }); +export function parseSource(source: string, log?: LogFn, unit?: string): AST { + return parse({ ...bnf_config, log, source, id: unit }); } function parseBnf(tokens: Token[], log?: LogFn): AST { @@ -356,10 +356,11 @@ export function create_config(ast: AST, options: CreateConfigOptions): Config { logger(node?: Node): [Logger, (level: IssueLevel, message: string) => Issue] { const _log = logger(this.log, { depth: this.depth, kind: node?.kind || 'node' }); - const shared_issue_info = { line: 0, position: 0, column: 0, ...node, id: this.id, source: ast.source }; + const shared_issue_info: Omit = { location: node, source: ast.source }; function _log_issue(level: IssueLevel, message: string): Issue { - const issue = { ...shared_issue_info, level, message }; + const { stack } = new Error(); + const issue = { ...shared_issue_info, level, message, stack }; _log(issue); return issue; } @@ -374,7 +375,7 @@ export function create_config(ast: AST, options: CreateConfigOptions): Config { } if (!config.root_nodes?.length) { - context.logger()[1](0, 'No root nodes are defined! You will need to add root node(s) manually.'); + context.logger()[1](1, 'No root nodes are defined! You will need to add root node(s) manually.'); } return config; diff --git a/src/issue.ts b/src/issue.ts index ede8997..0307123 100644 --- a/src/issue.ts +++ b/src/issue.ts @@ -1,8 +1,7 @@ +import type { Location } from './tokens.js'; + export interface Issue { - id?: string; - line: number; - column: number; - position: number; + location?: Location; source: string; message?: string; level: IssueLevel; @@ -35,7 +34,7 @@ function extract_location(stack: string = ''): string { } export function is_issue(i: unknown): i is Issue { - return typeof i == 'object' && i != null && 'line' in i && 'column' in i && 'position' in i && 'source' in i && 'level' in i; + return typeof i == 'object' && i != null && 'source' in i && 'level' in i; } export interface IssueFormatting { @@ -48,9 +47,13 @@ export function stringify_issue(i: Issue, options: Partial): st const trace = options.trace ? ' ' + extract_location(i.stack) : ''; - const line_text = i.source.split('\n')[i.line - 1]; + const base_message = `${level}: ${i.message}${trace}`; + + if (!i.location) return base_message; + + const line_text = i.source.split('\n')[i.location.line - 1]; - let { column } = i, + let { column } = i.location, excerpt = line_text; // Max 80 chars, 40 before and 40 after @@ -61,5 +64,5 @@ export function stringify_issue(i: Issue, options: Partial): st column -= offset; } - return `${i.id ? i.id + ':' : ''}${i.line}:${column}\n\t${excerpt}\n\t${' '.repeat(column)}^\n${level}: ${i.message}${trace}`; + return `${i.location.unit ? i.location.unit + ':' : ''}${i.location.line}:${column}\n\t${excerpt}\n\t${' '.repeat(column)}^\n${base_message}`; } diff --git a/src/parser.ts b/src/parser.ts index 97b7628..61b0b23 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -141,7 +141,7 @@ export function parse(options: ParseOptions): AST { let position = 0, dirtyPosition = 0; - const raw_tokens = 'tokens' in options ? options.tokens : tokenize(options.source, options.literals); + const raw_tokens = 'tokens' in options ? options.tokens : tokenize(options.source, options.literals, id); const source = options.source ?? raw_tokens.map(token => token.text).join(''); @@ -159,7 +159,7 @@ export function parse(options: ParseOptions): AST { function _issue(level: IssueLevel, message?: string): Issue { const token = tokens[position]; const { stack } = new Error(); - return { id, line: token.line, column: token.column, position: token.position, source, level, message, stack }; + return { location: token, source, level, message, stack }; } function parseNode(kind: string, parents: string[] = []): Node | null { diff --git a/src/tokens.ts b/src/tokens.ts index 427e3dd..6f70bd6 100644 --- a/src/tokens.ts +++ b/src/tokens.ts @@ -1,11 +1,21 @@ import type { Issue } from './issue.js'; -export interface Token { - kind: string; - text: string; +/** + * A location in source text + */ +export interface Location { line: number; column: number; position: number; + /** + * The file, internal module, shared object, etc. + */ + unit?: string; +} + +export interface Token extends Location { + kind: string; + text: string; } export interface TokenDefinition { @@ -13,7 +23,13 @@ export interface TokenDefinition { pattern: RegExp; } -export function tokenize(source: string, definitions: Iterable): Token[] { +export interface TokenizeOptions { + source: string; + unit?: string; + definitions: Iterable; +} + +export function tokenize(source: string, definitions: Iterable, unit?: string): Token[] { const tokens: Token[] = []; let line = 1; @@ -27,12 +43,12 @@ export function tokenize(source: string, definitions: Iterable) for (const { name, pattern } of definitions) { const match = pattern.exec(slice); if (match && match[0].length > (token?.text.length || 0)) { - token = { kind: name, text: match[0], line, column, position }; + token = { kind: name, text: match[0], line, column, position, unit }; } } if (!token) { - throw { line, column, position, source, message: 'Unexpected token: ' + source[position], level: 0, stack: new Error().stack } satisfies Issue; + throw { location: { line, column, position, unit }, source, message: 'Unexpected token: ' + source[position], level: 0, stack: new Error().stack } satisfies Issue; } tokens.push(token);