fix tokenizer test

src/compiler/context/report.v

@@ -61,25 +61,75 @@ const backtick = `\``
 const border = term.bold(term.blue('      | '))
 
 fn report_source(pos ast.FilePos, prefix string, mut sb strings.Builder) {
-	if offending_line := pos.file.get_line(pos.begin.line) {
-		sb.writeln(border)
-		sb.write_string(term.bold(term.blue('${pos.begin.line + 1:5d} | ')))
-		sb.writeln(offending_line)
-		sb.write_string(border)
-		for jdx in 0 .. offending_line.len {
-			if offending_line[jdx] == `\t` {
-				sb.write_string('\t')
-				continue
-			}
-			if jdx == pos.begin.col {
-				sb.write_string(term.green(term.bold('^')))
-			} else if jdx > pos.begin.col && jdx < pos.end.col {
-				sb.write_string(term.green(term.bold('~')))
-			} else {
-				sb.write_string(' ')
+	for idx := pos.begin.line; idx <= pos.end.line; idx++ {
+		if offending_line := pos.file.get_line(idx) {
+			sb.writeln(border)
+			sb.write_string(term.bold(term.blue('${pos.begin.line + 1:5d} | ')))
+			sb.writeln(offending_line)
+			sb.write_string(border)
+			for jdx in 0 .. offending_line.len {
+				if offending_line[jdx] == `\t` {
+					sb.write_u8(`\t`)
+					continue
+				}
+				/*
+				println("${jdx} == ${pos.begin.col}")
+				if jdx == pos.begin.col-1 {
+					sb.write_string(term.green(term.bold('^')))
+				} else if jdx > pos.begin.col-1 && jdx < pos.end.col-1 {
+					sb.write_string(term.green(term.bold('~')))
+				} else {
+					sb.write_string(' ')
+				}
+				
+				
+				bool caret = false;
+				unowned SourceLocation begin = source.begin;
+				unowned SourceLocation end = source.end;
+				if (begin.line == idx && end.line == idx) {
+					if (begin.column <= jdx + 1 <= end.column) {
+						caret = true;
+					}
+				} else if (begin.line == idx && begin.column <= jdx + 1) {
+					caret = true;
+				} else if (begin.line < idx < end.line) {
+					caret = true;
+				} else if (end.line == idx && end.column >= jdx + 1) {
+					caret = true;
+				}
+				if (caret) {
+					if (begin.line == idx && begin.column == jdx + 1) {
+						stderr.putc ('^');
+					} else {
+						stderr.putc ('~');
+					}
+				} else {
+					stderr.putc (' ');
+				}*/
+				mut caret := false
+				if pos.begin.line == idx && pos.end.line == idx {
+					if pos.begin.col <= jdx + 1 && jdx + 1 <= pos.end.col {
+						caret = true
+					}
+				} else if pos.begin.line == idx && pos.begin.col <= jdx + 1 {
+					caret = true
+				} else if pos.begin.line < idx && idx < pos.end.line {
+					caret = true
+				} else if pos.end.line == idx && pos.end.col >= jdx + 1 {
+					caret = true
+				}
+				if caret {
+					if pos.begin.line == idx && pos.begin.col == jdx + 1 {
+						sb.write_string(term.green(term.bold('^')))
+					} else {
+						sb.write_string(term.green(term.bold('~')))
+					}
+				} else {
+					sb.write_u8(` `)
+				}
 			}
+			sb.write_u8(`\n`)
 		}
-		sb.write_u8(`\n`)
 	}
 }
 

src/compiler/tokenizer/mod.v

@@ -69,9 +69,11 @@ fn (mut t Tokenizer) tokenize_remaining_text() {
 
 @[inline]
 fn (t &Tokenizer) current_pos() ast.FilePos {
+	cur_loc := t.current_loc()
 	return ast.FilePos{
 		file:  t.file
-		begin: t.current_loc()
+		begin: cur_loc
+		end:   cur_loc
 	}
 }
 
@@ -114,7 +116,7 @@ fn (mut t Tokenizer) inc_line_number() {
 fn (mut t Tokenizer) skip_whitespace() {
 	for t.pos < t.text.len {
 		c := t.text[t.pos]
-		if c == 9 {
+		if c == 8 {
 			t.pos++
 			continue
 		}
@@ -228,10 +230,10 @@ fn (mut t Tokenizer) read_number_mode(mode NumberMode) string {
 		if !mode.is_valid(ch) && ch != num_sep {
 			if mode == .dec && (!ch.is_letter() || ch in [`e`, `E`]) {
 				break
-			} else if !ch.is_digit() && !ch.is_letter() {
+			} else if mode != .dec && (!ch.is_digit() && !ch.is_letter()) {
 				break
 			}
-			context.error('${mode} number has unsuitable digit `{self.current_char()}`',
+			context.error('${mode} number has unsuitable digit `${t.text[t.pos].ascii_str()}`',
 				t.current_pos())
 		}
 		t.pos++
@@ -265,7 +267,8 @@ fn (mut t Tokenizer) read_number_mode(mode NumberMode) string {
 								}
 								break
 							} else {
-								context.error('number has unsuitable digit `${c}`', t.current_pos())
+								context.error('number has unsuitable digit `${c.ascii_str()}`',
+									t.current_pos())
 							}
 						}
 						t.pos++
@@ -311,7 +314,8 @@ fn (mut t Tokenizer) read_number_mode(mode NumberMode) string {
 						}
 						break
 					} else {
-						context.error('this number has unsuitable digit `${c}`', t.current_pos())
+						context.error('this number has unsuitable digit `${c.ascii_str()}`',
+							t.current_pos())
 					}
 				}
 				t.pos++
@@ -469,9 +473,9 @@ fn (mut t Tokenizer) internal_next() Token {
 		if t.pos >= t.text.len {
 			return t.end_of_file()
 		}
-		mut pos := t.current_pos()
 		ch := t.text[t.pos]
 		nextc := t.look_ahead(1)
+		mut pos := t.current_pos()
 		if util.is_valid_name(ch) {
 			lit := t.read_ident()
 			pos.end = t.current_loc()
@@ -518,6 +522,7 @@ fn (mut t Tokenizer) internal_next() Token {
 							nest_count--
 						}
 					}
+					t.pos++
 					continue
 				}
 			}

src/compiler/tokenizer/mod_test.v

@@ -10,7 +10,7 @@ struct ExpectedToken {
 }
 
 const source = '
-fn main 123 123.0 0b0101 0o12345678 0x123456ABCDEF
+fn main 123 123.0 0b0101 0o1234567 0x123456ABCDEF
 // inline comment
 
 fn other
@@ -31,7 +31,7 @@ const expected_tokens = [
 	ExpectedToken{.number, '123'},
 	ExpectedToken{.number, '123.0'},
 	ExpectedToken{.number, '0b0101'},
-	ExpectedToken{.number, '0o12345678'},
+	ExpectedToken{.number, '0o1234567'},
 	ExpectedToken{.number, '0x123456ABCDEF'},
 	ExpectedToken{.kw_fn, 'fn'},
 	ExpectedToken{.ident, 'other'},

test.ri

@@ -1,9 +1,13 @@
-fn
-fn main 123 123.0 0x123abc 0b10101 0o1234 ()
+fn main 123 123.0 0b0101 0o1234567 0x123456ABCDEF
+// inline comment
 
-fn main
-/* multi-line comment
-    usb - pc - laptop
+fn other
+
+/*
+	multiline comment
 */
 
-fn set_current_dir
+fn other2
+
+"my string :)\nhello!"
+'a' 'b'