From 4160ae22230c019264e553756ec47b0fa269b1db Mon Sep 17 00:00:00 2001
From: StunxFS <mendozadiazjose77@gmail.com>
Date: Sun, 24 Nov 2024 20:28:24 -0400
Subject: [PATCH] tokenizer: support comments

---
 src/compiler/tokenizer/mod.v                  | 24 +++++++++++++++++++
 .../{tokenizer_test.v => mod_test.v}          |  0
 test.ri                                       |  8 ++++++-
 3 files changed, 31 insertions(+), 1 deletion(-)
 rename src/compiler/tokenizer/{tokenizer_test.v => mod_test.v} (100%)

diff --git a/src/compiler/tokenizer/mod.v b/src/compiler/tokenizer/mod.v
index 9f3fef4c1..51388e37f 100644
--- a/src/compiler/tokenizer/mod.v
+++ b/src/compiler/tokenizer/mod.v
@@ -451,6 +451,30 @@ fn (mut t Tokenizer) internal_next() token.Token {
 			}
 		}
 		match ch {
+			`/` {
+				if nextc == `/` {
+					t.ignore_line()
+					continue
+				} else if nextc == `*` {
+					start_pos := t.pos
+					t.pos++
+					for t.pos < t.text.len - 1 {
+						t.pos++
+						if t.current_char() == lf {
+							t.inc_line_number()
+							continue
+						} else if t.matches('*/', t.pos) {
+							t.pos++
+							break
+						}
+					}
+					if t.pos >= t.text.len {
+						t.pos = start_pos
+						report.error('comment not terminated', t.current_pos())
+					}
+					continue
+				}
+			}
 			`'` {
 				return token.Token{
 					lit:  t.read_char()
diff --git a/src/compiler/tokenizer/tokenizer_test.v b/src/compiler/tokenizer/mod_test.v
similarity index 100%
rename from src/compiler/tokenizer/tokenizer_test.v
rename to src/compiler/tokenizer/mod_test.v
diff --git a/test.ri b/test.ri
index c480c5677..52a90737f 100644
--- a/test.ri
+++ b/test.ri
@@ -2,4 +2,10 @@ fn main 123 123.0 0x123abc 0b10101 0o1234
 
 // Test file
 
-fn main() {}
\ No newline at end of file
+fn main() {}
+
+/* multi-line comment
+    usb - pc - laptop
+*/
+
+fn set_current_dir(){}