Added Location

Improved issues across multiple files
james-pre · Jan 5, 2025 · 9cb8019 · 9cb8019
1 parent 0b78674
commit 9cb8019
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 25 deletions.
diff --git a/scripts/bnf.js b/scripts/bnf.js
@@ -84,7 +84,7 @@ try {
 
 let tokens;
 try {
-	tokens = bnf.tokenize(contents);
+	tokens = bnf.tokenize(contents, input);
 } catch (e) {
 	console.error(is_issue(e) ? stringify_issue(e, options) : e.message);
 	process.exit(1);
@@ -125,7 +125,7 @@ function include(path) {
 	}
 }
 
-let config = bnf.create_config(ast, { log: logger(verbose), include });
+let config = bnf.create_config(ast, { log: logger(verbose), include, id: input });
 
 if (options.compress) config = compress_config(config);
 

diff --git a/src/bnf.ts b/src/bnf.ts
@@ -14,14 +14,14 @@ export { bnf_config as config };
 /**
  * Shortcut for tokenize(source, bnf.literals);
  */
-function tokenizeBnf(source: string): Token[] {
-	return tokenize(source, bnf_config.literals);
+function tokenizeBnf(source: string, unit?: string): Token[] {
+	return tokenize(source, bnf_config.literals, unit);
 }
 
 export { tokenizeBnf as tokenize };
 
-export function parseSource(source: string, log?: LogFn): AST {
-	return parse({ ...bnf_config, log, source });
+export function parseSource(source: string, log?: LogFn, unit?: string): AST {
+	return parse({ ...bnf_config, log, source, id: unit });
 }
 
 function parseBnf(tokens: Token[], log?: LogFn): AST {
@@ -356,10 +356,11 @@ export function create_config(ast: AST, options: CreateConfigOptions): Config {
 		logger(node?: Node): [Logger, (level: IssueLevel, message: string) => Issue] {
 			const _log = logger(this.log, { depth: this.depth, kind: node?.kind || 'node' });
 
-			const shared_issue_info = { line: 0, position: 0, column: 0, ...node, id: this.id, source: ast.source };
+			const shared_issue_info: Omit<Issue, 'level' | 'message' | 'stack'> = { location: node, source: ast.source };
 
 			function _log_issue(level: IssueLevel, message: string): Issue {
-				const issue = { ...shared_issue_info, level, message };
+				const { stack } = new Error();
+				const issue = { ...shared_issue_info, level, message, stack };
 				_log(issue);
 				return issue;
 			}
@@ -374,7 +375,7 @@ export function create_config(ast: AST, options: CreateConfigOptions): Config {
 	}
 
 	if (!config.root_nodes?.length) {
-		context.logger()[1](0, 'No root nodes are defined! You will need to add root node(s) manually.');
+		context.logger()[1](1, 'No root nodes are defined! You will need to add root node(s) manually.');
 	}
 
 	return config;

diff --git a/src/issue.ts b/src/issue.ts
@@ -1,8 +1,7 @@
+import type { Location } from './tokens.js';
+
 export interface Issue {
-	id?: string;
-	line: number;
-	column: number;
-	position: number;
+	location?: Location;
 	source: string;
 	message?: string;
 	level: IssueLevel;
@@ -35,7 +34,7 @@ function extract_location(stack: string = ''): string {
 }
 
 export function is_issue(i: unknown): i is Issue {
-	return typeof i == 'object' && i != null && 'line' in i && 'column' in i && 'position' in i && 'source' in i && 'level' in i;
+	return typeof i == 'object' && i != null && 'source' in i && 'level' in i;
 }
 
 export interface IssueFormatting {
@@ -48,9 +47,13 @@ export function stringify_issue(i: Issue, options: Partial<IssueFormatting>): st
 
 	const trace = options.trace ? ' ' + extract_location(i.stack) : '';
 
-	const line_text = i.source.split('\n')[i.line - 1];
+	const base_message = `${level}: ${i.message}${trace}`;
+
+	if (!i.location) return base_message;
+
+	const line_text = i.source.split('\n')[i.location.line - 1];
 
-	let { column } = i,
+	let { column } = i.location,
 		excerpt = line_text;
 
 	// Max 80 chars, 40 before and 40 after
@@ -61,5 +64,5 @@ export function stringify_issue(i: Issue, options: Partial<IssueFormatting>): st
 		column -= offset;
 	}
 
-	return `${i.id ? i.id + ':' : ''}${i.line}:${column}\n\t${excerpt}\n\t${' '.repeat(column)}^\n${level}: ${i.message}${trace}`;
+	return `${i.location.unit ? i.location.unit + ':' : ''}${i.location.line}:${column}\n\t${excerpt}\n\t${' '.repeat(column)}^\n${base_message}`;
 }
diff --git a/src/parser.ts b/src/parser.ts
@@ -141,7 +141,7 @@ export function parse(options: ParseOptions): AST {
 	let position = 0,
 		dirtyPosition = 0;
 
-	const raw_tokens = 'tokens' in options ? options.tokens : tokenize(options.source, options.literals);
+	const raw_tokens = 'tokens' in options ? options.tokens : tokenize(options.source, options.literals, id);
 
 	const source = options.source ?? raw_tokens.map(token => token.text).join('');
 
@@ -159,7 +159,7 @@ export function parse(options: ParseOptions): AST {
 	function _issue(level: IssueLevel, message?: string): Issue {
 		const token = tokens[position];
 		const { stack } = new Error();
-		return { id, line: token.line, column: token.column, position: token.position, source, level, message, stack };
+		return { location: token, source, level, message, stack };
 	}
 
 	function parseNode(kind: string, parents: string[] = []): Node | null {

diff --git a/src/tokens.ts b/src/tokens.ts
@@ -1,19 +1,35 @@
 import type { Issue } from './issue.js';
 
-export interface Token {
-	kind: string;
-	text: string;
+/**
+ * A location in source text
+ */
+export interface Location {
 	line: number;
 	column: number;
 	position: number;
+	/**
+	 * The file, internal module, shared object, etc.
+	 */
+	unit?: string;
+}
+
+export interface Token extends Location {
+	kind: string;
+	text: string;
 }
 
 export interface TokenDefinition {
 	name: string;
 	pattern: RegExp;
 }
 
-export function tokenize(source: string, definitions: Iterable<TokenDefinition>): Token[] {
+export interface TokenizeOptions {
+	source: string;
+	unit?: string;
+	definitions: Iterable<TokenDefinition>;
+}
+
+export function tokenize(source: string, definitions: Iterable<TokenDefinition>, unit?: string): Token[] {
 	const tokens: Token[] = [];
 
 	let line = 1;
@@ -27,12 +43,12 @@ export function tokenize(source: string, definitions: Iterable<TokenDefinition>)
 		for (const { name, pattern } of definitions) {
 			const match = pattern.exec(slice);
 			if (match && match[0].length > (token?.text.length || 0)) {
-				token = { kind: name, text: match[0], line, column, position };
+				token = { kind: name, text: match[0], line, column, position, unit };
 			}
 		}
 
 		if (!token) {
-			throw { line, column, position, source, message: 'Unexpected token: ' + source[position], level: 0, stack: new Error().stack } satisfies Issue;
+			throw { location: { line, column, position, unit }, source, message: 'Unexpected token: ' + source[position], level: 0, stack: new Error().stack } satisfies Issue;
 		}
 
 		tokens.push(token);