add ast.FilePos && ast.FileLoc; improve tokenizer

src/compiler/ast/File.v

@@ -9,37 +9,73 @@ import os
 @[heap]
 pub struct File {
 pub:
-	file    string
-	content string
-pub mut:
+	filename string
+	content  string
+mut:
 	lines ?[]string
 }
 
+pub struct FilePos {
+pub mut:
+	file  &File = unsafe { nil }
+	begin FileLoc
+	end   FileLoc
+}
+
+pub fn (fp &FilePos) contains(loc &FileLoc) bool {
+	if loc.line > fp.begin.line && loc.line < fp.end.line {
+		return true
+	} else if loc.line == fp.begin.line && loc.line == fp.end.line {
+		return loc.col >= fp.begin.col && loc.col <= fp.end.col
+	} else if loc.line == fp.begin.line {
+		return loc.col >= fp.begin.col
+	} else if loc.line == fp.end.line {
+		return loc.col <= fp.end.col
+	}
+	return false
+}
+
+pub fn (fp &FilePos) str() string {
+	if fp.begin.line == fp.end.line {
+		return '${fp.file.filename}:${fp.begin.line + 1}:${fp.begin.col}'
+	}
+	return '${fp.file.filename}:${fp.begin.line + 1}:${fp.begin.col}-${fp.end.line + 1}:${fp.end.col}'
+}
+
+pub struct FileLoc {
+pub:
+	pos  int
+	line int
+	col  int
+}
+
 pub fn File.new(file string) &File {
 	content := read_file(file)
 	return &File{
-		file:    file
-		content: content
+		filename: file
+		content:  content
 	}
 }
 
 pub fn File.from_memory(content string) &File {
 	return &File{
-		file:    '<memory>'
-		content: content
+		filename: '<memory>'
+		content:  content
 	}
 }
 
-pub fn (mut file File) get_lines() []string {
+pub fn (file &File) get_lines() []string {
 	if file.lines != none {
 		return file.lines
 	}
 	lines := file.content.split_into_lines()
-	file.lines = lines
+	unsafe {
+		file.lines = lines
+	}
 	return lines
 }
 
-pub fn (mut file File) get_line(line int) ?string {
+pub fn (file &File) get_line(line int) ?string {
 	lines := file.get_lines()
 	return lines[line] or { none }
 }

src/compiler/context/mod.v

@@ -41,7 +41,7 @@ pub mut:
 
 pub fn (ctx CContext) get_file(filename string) ?&ast.File {
 	for file in ctx.files {
-		if file.file == filename {
+		if file.filename == filename {
 			return file
 		}
 	}

src/compiler/context/report.v

@@ -6,7 +6,7 @@ module context
 
 import term
 import strings
-import compiler.token
+import compiler.ast
 
 pub struct Report {
 mut:
@@ -46,12 +46,12 @@ pub struct Hint {
 pub:
 	kind HintKind
 	msg  string
-	pos  ?token.Pos
+	pos  ?ast.FilePos
 }
 
 @[params]
 struct ReportParams {
-	pos   ?token.Pos
+	pos   ?ast.FilePos
 	msg   string
 	type  ReportType
 	hints []Hint
@@ -60,28 +60,26 @@ struct ReportParams {
 const backtick = `\``
 const border = term.bold(term.blue('      | '))
 
-fn report_source(pos token.Pos, prefix string, mut sb strings.Builder) {
-	if mut file := get().get_file(pos.file) {
-		if offending_line := file.get_line(pos.line_nr) {
-			sb.writeln(border)
-			sb.write_string(term.bold(term.blue('${pos.line_nr + 1:5d} | ')))
-			sb.writeln(offending_line)
-			sb.write_string(border)
-			for jdx in 0 .. offending_line.len {
-				if offending_line[jdx] == `\t` {
-					sb.write_string('\t')
-					continue
-				}
-				if jdx == pos.col {
-					sb.write_string(term.green(term.bold('^')))
-				} else if jdx > pos.col && jdx < pos.col + pos.len {
-					sb.write_string(term.green(term.bold('~')))
-				} else {
-					sb.write_string(' ')
-				}
+fn report_source(pos ast.FilePos, prefix string, mut sb strings.Builder) {
+	if offending_line := pos.file.get_line(pos.begin.line) {
+		sb.writeln(border)
+		sb.write_string(term.bold(term.blue('${pos.begin.line + 1:5d} | ')))
+		sb.writeln(offending_line)
+		sb.write_string(border)
+		for jdx in 0 .. offending_line.len {
+			if offending_line[jdx] == `\t` {
+				sb.write_string('\t')
+				continue
+			}
+			if jdx == pos.begin.col {
+				sb.write_string(term.green(term.bold('^')))
+			} else if jdx > pos.begin.col && jdx < pos.end.col {
+				sb.write_string(term.green(term.bold('~')))
+			} else {
+				sb.write_string(' ')
 			}
-			sb.write_u8(`\n`)
 		}
+		sb.write_u8(`\n`)
 	}
 }
 
@@ -129,10 +127,10 @@ fn (r Report) print_message(params ReportParams) {
 	sb.write_string(term.bold(params.type.colorize()))
 	sb.write_u8(` `)
 
-	// `invalid character literal`
+	// invalid character `\`
 	print_highlighted_message(params.msg, mut sb)
 
-	// 8 | /
+	// 8 | '\'
 	if params.pos != none {
 		sb.write_u8(`\n`)
 		report_source(params.pos, '', mut sb)
@@ -177,12 +175,12 @@ fn (mut r Report) ic_fatal(msg string) {
 	for {}
 }
 
-fn (mut r Report) warn(msg string, pos token.Pos, hints ...Hint) {
+fn (mut r Report) warn(msg string, pos ast.FilePos, hints ...Hint) {
 	r.warnings++
 	r.print_message(pos: pos, msg: msg, type: .warn, hints: hints)
 }
 
-fn (mut r Report) error(msg string, pos token.Pos, hints ...Hint) {
+fn (mut r Report) error(msg string, pos ast.FilePos, hints ...Hint) {
 	r.errors++
 	r.print_message(pos: pos, msg: msg, type: .error, hints: hints)
 }
@@ -214,13 +212,13 @@ pub fn ic_fatal(msg string) {
 }
 
 @[inline]
-pub fn error(msg string, pos token.Pos, hints ...Hint) {
+pub fn error(msg string, pos ast.FilePos, hints ...Hint) {
 	mut r := get().report
 	r.error(msg, pos, ...hints)
 }
 
 @[inline]
-pub fn warn(msg string, pos token.Pos, hints ...Hint) {
+pub fn warn(msg string, pos ast.FilePos, hints ...Hint) {
 	mut r := get().report
 	r.warn(msg, pos, ...hints)
 }

src/compiler/parser/mod.v

@@ -5,7 +5,6 @@
 module parser
 
 import compiler.ast
-import compiler.token
 import compiler.context
 import compiler.tokenizer
 
@@ -16,9 +15,9 @@ mut:
 	file &ast.File = unsafe { nil }
 
 	tokenizer tokenizer.Tokenizer
-	prev_tok  token.Token
-	tok       token.Token
-	next_tok  token.Token
+	prev_tok  tokenizer.Token
+	tok       tokenizer.Token
+	next_tok  tokenizer.Token
 }
 
 pub fn new(ctx &context.CContext) &Parser {