feat(rivet.tokenizer): tokenize comments

StunxFS · StunxFS · commit 9841f023b02e · 2023-11-12T16:41:36.000-04:00
diff --git a/lib/rivet/src/ast/Decl.ri b/lib/rivet/src/ast/Decl.ri
@@ -21,14 +21,14 @@ public enum Decl {
         pos: token.Pos;
     },
     Extern {
-        docs: DocComment;
+        docs: []Comment;
         attributes: Attributes;
         abi: ABI;
         decls: []Decl;
         pos: token.Pos;
     },
     Alias {
-        docs: DocComment;
+        docs: []Comment;
         attributes: Attributes;
         is_public: bool;
         name: string;
@@ -39,7 +39,7 @@ public enum Decl {
         mut sym: Sym;
     },
     Trait {
-        docs: DocComment;
+        docs: []Comment;
         attributes: Attributes;
         is_public: bool;
         name: string;
@@ -49,7 +49,7 @@ public enum Decl {
         mut sym: Sym;
     },
     Enum {
-        docs: DocComment;
+        docs: []Comment;
         attributes: Attributes;
         is_public: bool;
         name: string;
@@ -62,7 +62,7 @@ public enum Decl {
         mut sym: Sym;
     },
     Struct {
-        docs: DocComment;
+        docs: []Comment;
         attributes: Attributes;
         is_public: bool;
         name: string;
@@ -73,7 +73,7 @@ public enum Decl {
         mut sym: Sym;
     },
     Field {
-        docs: DocComment;
+        docs: []Comment;
         attributes: Attributes;
         is_public: bool;
         is_mut: bool;
@@ -91,7 +91,7 @@ public enum Decl {
         pos: token.Pos;
     },
     Const {
-        docs: DocComment;
+        docs: []Comment;
         attributes: Attributes;
         is_public: bool;
         name: string;
@@ -102,7 +102,7 @@ public enum Decl {
         mut sym: Sym;
     },
     Var {
-        docs: DocComment;
+        docs: []Comment;
         attributes: Attributes;
         is_public: bool;
         is_extern: bool;
@@ -114,7 +114,7 @@ public enum Decl {
         mut scope: Scope;
     },
     Func {
-        docs: DocComment;
+        docs: []Comment;
         attributes: Attributes;
         is_public: bool;
         is_extern: bool;
@@ -141,7 +141,7 @@ public enum Decl {
         mut defer_stmts: []Stmt.Defer;
     },
     Test {
-        docs: DocComment;
+        docs: []Comment;
         attributes: Attributes;
         name: string;
         stmts: []mut Stmt;
@@ -174,10 +174,16 @@ public struct EnumVariantDecl {
 }
 
 #[boxed]
-public struct DocComment {
+public struct Comment {
+    mut is_doc: bool;
     mut lines: []string;
     pos: token.Pos;
 
+    #[inline]
+    func is_multiline(self) -> bool {
+        return self.lines.len > 1;
+    }
+
     #[inline]
     func is_empty(self) -> bool {
         return self.lines.is_empty();
diff --git a/lib/rivet/src/parser/decls.ri b/lib/rivet/src/parser/decls.ri
@@ -9,14 +9,18 @@ import ../report;
 import ../token;
 
 extend Parser {
-    func parse_doc_comment(mut self) -> ast.DocComment {
-        mut pos := self.tok.pos;
-        mut lines := @vec(string);
-        while self.accept(.DocComment) {
-            lines.push(self.prev_tok.lit);
-            pos += self.prev_tok.pos;
+    func parse_doc_comment(mut self) -> []ast.Comment {
+        mut comments := @vec(ast.Comment);
+        while self.accept(.DocComment) or self.accept(.Comment) {
+            comments.push(
+                ast.Comment(
+                    self.prev_tok.kind == .DocComment,
+                    self.prev_tok.lit.split_into_lines(),
+                    self.prev_tok.pos
+                )
+            );
         }
-        return ast.DocComment(lines, pos);
+        return comments;
     }
 
     public func parse_attributes(
@@ -527,7 +531,7 @@ extend Parser {
     }
 
     func parse_func_decl(
-        mut self, docs: ast.DocComment, attributes: ast.Attributes,
+        mut self, docs: []ast.Comment, attributes: ast.Attributes,
         is_public: bool, is_unsafe: bool, abi: ast.ABI
     ) -> ast.Decl {
         mut is_operator := false;
diff --git a/lib/rivet/src/prefs/mod.ri b/lib/rivet/src/prefs/mod.ri
@@ -59,6 +59,10 @@ public struct Prefs {
     public mut mod_dir: string;
     public mut mod_output: string;
 
+    public mut is_fmt: bool;
+    public mut is_docs: bool;
+    public mut is_test: bool;
+
     public mut library_path: []string := [rivetcLibDir, libDir];
     public mut libraries_to_link: []string;
     public mut objects_to_link: []string;
@@ -71,12 +75,10 @@ public struct Prefs {
     public mut all_warns_are_errors: bool;
     public mut hide_all_warns: bool;
     public mut is_verbose: bool;
-    public mut is_fmt: bool; 
 
     // Output info
     public mut output_mode: OutputMode;
     public mut optimize_mode: OptimizeMode;
-    public mut is_test: bool;
     public mut run_output: bool;
     public mut run_output_args: []string;
 
diff --git a/lib/rivet/src/token/Kind.ri b/lib/rivet/src/token/Kind.ri
@@ -14,6 +14,7 @@ static kindStrings := [
     KindMap(.Unknown, "unknown"),
     KindMap(.EndOfFile, "end of file"),
     KindMap(.DocComment, "doc-comment"),
+    KindMap(.Comment, "comment"),
     KindMap(.Name, "name"),
     KindMap(.Number, "number"),
     KindMap(.Char, "character"),
@@ -111,6 +112,7 @@ public enum Kind < traits.Stringable {
     Unknown, // unknown
     EndOfFile, // end of file
     DocComment, // doc-comment
+    Comment, // comment
     Name, // name
     Number, // number
     Char, // character
diff --git a/lib/rivet/src/tokenizer/next.ri b/lib/rivet/src/tokenizer/next.ri
@@ -85,14 +85,23 @@ extend Tokenizer {
                         self.pos += 1;
                         return self.new_token(.DivAssign, len: 2);
                     } else if nextc == b'/' {
+                        is_doc_comment := self.look_ahead(2) == b'/';
+                        start := self.pos + if is_doc_comment { 3 } else { 0 };
                         self.ignore_line();
+                        end := self.pos;
                         if self.text[self.pos - 1] != CR {
                             self.pos -= 1;
                             self.line -= 1;
                         }
+                        if (is_doc_comment and self.prefs.is_docs) or self.prefs.is_fmt {
+                            line := self.text[start..end].trim_space();
+                            return self.new_token(.Comment, line, line.len);
+                        }
                         continue;
                     } else if nextc == b'*' {
                         start_pos := self.pos;
+                        start_line := self.line;
+                        start_col := utils.max(1, self.current_column()) - 1;
                         self.pos += 1;
                         mut nest_count := 1;
                         while nest_count > 0 and self.pos < self.text.len - 1 {
@@ -109,12 +118,19 @@ extend Tokenizer {
                                 nest_count -= 1;
                             }
                         }
+                        end_pos := self.pos;
                         self.pos += 1;
                         if self.pos >= self.text.len {
                             self.pos = start_pos;
                             self.error("comment not terminated");
                             break;
                         }
+                        if self.prefs.is_fmt {
+                            line := self.text[start_pos..end_pos].trim_space();
+                            return self.new_multiline_token(
+                                .Comment, line, line.len, start_line, self.line, start_col
+                            );
+                        }
                         continue;
                     }
                     return self.new_token(.Div);
