Added issues

scripts/bnf.js

@@ -2,9 +2,10 @@
 import { readFileSync, writeFileSync } from 'node:fs';
 import { inspect, parseArgs } from 'node:util';
 import * as bnf from '../dist/bnf.js';
-import { stringifyNode } from '../dist/parser.js';
+import { stringify_node } from '../dist/parser.js';
+import { is_issue, stringify_issue } from '../dist/issue.js';
 import { dirname, resolve } from 'node:path/posix';
-import { compress as compressConfig } from '../dist/config.js';
+import { compress as compress_config } from '../dist/config.js';
 
 const {
 	positionals: [input],
@@ -66,28 +67,19 @@ try {
 	process.exit(1);
 }
 
-const token_error_reasons = {
-	unexpected: 'Unexpected token',
-};
-
 let tokens;
 try {
 	tokens = bnf.tokenize(contents);
 } catch (e) {
-	if (!('reason' in e)) throw e;
-
-	const { line, column, position, source, reason } = e;
+	if (!is_issue(e)) throw e;
 
-	console.error(`${input}:${line}:${column}
-${source.split('\n')[line - 1]}
-${' '.repeat(column)}^
-Error: ${token_error_reasons[reason]}: ${source[position]}`);
+	console.error(stringify_issue(e));
 	process.exit(1);
 }
 
 if (options.tokens) {
 	for (const token of tokens) {
-		console.log(stringifyNode(token));
+		console.log(stringify_node(token));
 	}
 	if (options.tokens == 'only') process.exit(0);
 }
@@ -98,7 +90,7 @@ const ast = bnf.parse(tokens, parseLogger);
 
 if (options.ast) {
 	for (const node of ast) {
-		console.log(stringifyNode(node));
+		console.log(stringify_node(node));
 	}
 	if (options.parser == 'only') process.exit(0);
 }
@@ -116,7 +108,7 @@ function include(path) {
 
 let config = bnf.ast_to_config(ast, logger(verbose), include);
 
-if (options.compress) config = compressConfig(config);
+if (options.compress) config = compress_config(config);
 
 const write = data => (options.output ? writeFileSync(options.output, data) : console.log);
 

scripts/dump.js

@@ -1,9 +1,10 @@
 #!/usr/bin/env node
 import { readFileSync } from 'node:fs';
 import { parseArgs } from 'node:util';
-import { parse, parseInfo, stringifyNode } from '../dist/parser.js';
-import { parseJSON as parseJSONConfig } from '../dist/config.js';
+import { parse, parse_info, stringify_node } from '../dist/parser.js';
+import { parse_json as parseJSONConfig } from '../dist/config.js';
 import { tokenize } from '../dist/tokens.js';
+import { is_issue, stringify_issue } from '../dist/issue.js';
 
 const {
 	values: options,
@@ -91,7 +92,7 @@ if (options.tokens) {
 
 	for (const token of tokens) {
 		if (options.mode != 'all' && config.ignoreLiterals.includes(token.kind)) continue;
-		console.log(stringifyNode(token));
+		console.log(stringify_node(token));
 	}
 
 	process.exit(0);
@@ -100,7 +101,7 @@ if (options.tokens) {
 function dump_info() {
 	if (!options.info) return;
 
-	const info = parseInfo.get(input);
+	const info = parse_info.get(input);
 
 	for (const [k, v] of Object.entries(info)) {
 		console.error(k + ': ', v);
@@ -124,13 +125,13 @@ try {
 	ast = parse({ ...config, tokens, log, id: input });
 	dump_info();
 } catch (e) {
-	console.error('Error: parsing failed:', e);
+	console.error(is_issue(e) ? stringify_issue(e, true) : e.message);
 	dump_info();
 	process.exit(1);
 }
 
 if (options.quiet) process.exit(0);
 
 for (const node of ast) {
-	console.log(stringifyNode(node));
+	console.log(stringify_node(node));
 }

src/bnf.ts

@@ -5,7 +5,7 @@ import { logger, parse } from './parser.js';
 import type { Token, TokenDefinition } from './tokens.js';
 import { tokenize } from './tokens.js';
 
-export const { literals, definitions, ignoreLiterals, rootNodes } = config.parseJSON(rawConfig as config.Json);
+export const { literals, definitions, ignoreLiterals, rootNodes } = config.parse_json(rawConfig as config.Json);
 
 /**
  * Shortcut for tokenize(source, bnf.literals);

src/config.ts

@@ -21,7 +21,7 @@ export interface Config {
 	ignoreLiterals: string[];
 }
 
-export function parseLiteral(literal: TokenDefinitionJSON): TokenDefinition {
+export function parse_json_literal(literal: TokenDefinitionJSON): TokenDefinition {
 	const $ = literal.pattern.endsWith('$');
 
 	if ($) literal.pattern = literal.pattern.slice(0, -1);
@@ -35,10 +35,10 @@ export function parseLiteral(literal: TokenDefinitionJSON): TokenDefinition {
 /**
  * Parses a JSON configuration into a normal configuration
  */
-export function parseJSON(config: Json): Config {
+export function parse_json(config: Json): Config {
 	return {
 		...config,
-		literals: config.literals.map(parseLiteral),
+		literals: config.literals.map(parse_json_literal),
 	};
 }
 

src/issue.ts

@@ -0,0 +1,47 @@
+export interface SourceIssue {
+	id?: string;
+	line: number;
+	column: number;
+	position: number;
+	source: string;
+	message?: string;
+	level: IssueLevel;
+}
+
+export enum IssueLevel {
+	Error = 0,
+	Warning = 1,
+	Note = 2,
+}
+
+/**
+ * Placed into \x1b[<here>m
+ */
+const colors = {
+	[IssueLevel.Error]: 31,
+	[IssueLevel.Warning]: 33,
+	[IssueLevel.Note]: 36,
+};
+
+export function is_issue(i: unknown): i is SourceIssue {
+	return typeof i == 'object' && i != null && 'line' in i && 'column' in i && 'position' in i && 'source' in i && 'level' in i;
+}
+
+export function stringify_issue(i: SourceIssue, colorize: boolean): string {
+	const level = colorize ? `\x1b[1;${colors[i.level]}m${IssueLevel[i.level]}\x1b[0m` : IssueLevel[i.level];
+
+	const line_text = i.source.split('\n')[i.line - 1];
+
+	let { column } = i,
+		excerpt = line_text;
+
+	// Max 80 chars, 40 before and 40 after
+
+	if (line_text.length > 80) {
+		const offset = Math.max(0, column - 40);
+		excerpt = line_text.slice(offset, column + 40);
+		column -= offset;
+	}
+
+	return `${i.id}:${i.line}:${column}\n\t${excerpt}\n\t${' '.repeat(column)}^\n${level}: ${i.message}`;
+}

src/parser.ts

@@ -1,3 +1,4 @@
+import type { IssueLevel, SourceIssue } from './issue.js';
 import { tokenize, type Token, type TokenDefinition } from './tokens.js';
 
 export interface DefinitionPart {
@@ -23,10 +24,10 @@ export interface Node extends Token {
 	children?: Node[];
 }
 
-export function stringifyNode(node: Node, depth = 0): string {
+export function stringify_node(node: Node, depth = 0): string {
 	return (
 		`${node.kind}${node?.children?.length ? '' : ` "${node.text.replaceAll('\n', '\\n').replaceAll('\t', '\\t')}"`} ${node.line}:${node.column}` +
-		(node?.children?.map(child => '\n' + '    '.repeat(depth + 1) + stringifyNode(child, depth + 1)).join('') || '')
+		(node?.children?.map(child => '\n' + '    '.repeat(depth + 1) + stringify_node(child, depth + 1)).join('') || '')
 	);
 }
 
@@ -76,6 +77,7 @@ export interface ParseOptionsShared {
 	maxCycles?: number;
 	log?: Logger;
 	id?: string;
+	source?: string;
 }
 
 export interface ParseAndTokenize extends ParseOptionsShared {
@@ -109,7 +111,7 @@ interface ParseInfo {
 	ignoredLiterals: number;
 }
 
-export const parseInfo = new Map<string, ParseInfo>();
+export const parse_info = new Map<string, ParseInfo>();
 
 export function parse(options: ParseOptions): Node[] {
 	const max_depth = options.maxNodeDepth ?? 100;
@@ -118,13 +120,15 @@ export function parse(options: ParseOptions): Node[] {
 
 	const info: ParseInfo = { parseNodeCalls: 0, nodesParsed: 0, ignoredLiterals: 0 };
 
-	if (id) parseInfo.set(id, info);
+	if (id) parse_info.set(id, info);
 
 	let position = 0,
 		dirtyPosition = 0;
 
 	const raw_tokens = 'tokens' in options ? options.tokens : tokenize(options.source, options.literals);
 
+	const source = options.source ?? raw_tokens.map(token => token.text).join('');
+
 	const tokens: Token[] = [];
 
 	for (let i = 0; i < raw_tokens.length; i++) {
@@ -136,12 +140,17 @@ export function parse(options: ParseOptions): Node[] {
 
 	const attempts = new Map<string, Node | null>();
 
+	function parseIssue(level: IssueLevel, message?: string): SourceIssue {
+		const token = tokens[position];
+		return { id, line: token.line, column: token.column, position: token.position, source, level, message };
+	}
+
 	function parseNode(kind: string, parents: string[] = []): Node | null {
 		if (id) info.parseNodeCalls++;
 
 		const depth = parents.length;
 
-		if (depth >= max_depth) throw 'Max depth exceeded when parsing ' + kind;
+		if (depth >= max_depth) throw parseIssue(0, 'Max depth exceeded while parsing ' + kind);
 
 		const log = logger(options.log, { kind, depth });
 
@@ -162,8 +171,7 @@ export function parse(options: ParseOptions): Node[] {
 		const loop = find_loop(parents, max_cycles);
 
 		if (loop) {
-			const node = tokens[position];
-			throw `Possible infinite loop: ${loop.join(' -> ')} -> ... at ${node.line}:${node.column}`;
+			throw parseIssue(0, `Possible infinite loop: ${loop.join(' -> ')} -> ...`);
 		}
 
 		if (id) info.nodesParsed++;
@@ -186,10 +194,7 @@ export function parse(options: ParseOptions): Node[] {
 		}
 
 		const definition = options.definitions.find(def => def.name === kind);
-		if (!definition) {
-			log(1, `Error: Definition for node "${kind}" not found`);
-			throw `Definition for node "${kind}" not found`;
-		}
+		if (!definition) throw parseIssue(0, `Definition for "${kind}" not found`);
 
 		const pattern = definition.pattern.map(part => (typeof part === 'string' ? { kind: part, type: 'required' } : part));
 
@@ -262,7 +267,7 @@ export function parse(options: ParseOptions): Node[] {
 		if (!node) {
 			if (position >= tokens.length && options.ignoreLiterals.includes(tokens.at(-1)!.kind)) break;
 			const token = tokens[dirtyPosition || position];
-			throw `Unexpected ${token.kind} "${token.text}" at ${token.line}:${token.column}`;
+			throw parseIssue(0, 'Unexpected ' + token.kind);
 		}
 		nodes.push(node);
 	}

src/tokens.ts

@@ -1,3 +1,5 @@
+import type { SourceIssue } from './issue.js';
+
 export interface Token {
 	kind: string;
 	text: string;
@@ -11,14 +13,6 @@ export interface TokenDefinition {
 	pattern: RegExp;
 }
 
-export interface TokenError {
-	line: number;
-	column: number;
-	position: number;
-	source: string;
-	reason: string;
-}
-
 export function tokenize(source: string, definitions: Iterable<TokenDefinition>): Token[] {
 	const tokens: Token[] = [];
 
@@ -38,7 +32,7 @@ export function tokenize(source: string, definitions: Iterable<TokenDefinition>)
 		}
 
 		if (!token) {
-			throw { line, column, position, source, reason: 'unexpected' };
+			throw { line, column, position, source, message: 'Unexpected token: ' + source[position], level: 0 } satisfies SourceIssue;
 		}
 
 		tokens.push(token);