feat(rivet/parser): parse comments

Author: StunxFS

lib/rivet/src/ast/Stmt.ri

@@ -6,6 +6,7 @@ import ../token;
 
 public enum Stmt {
     Empty: token.Pos,
+    Comment: Comment,
     Expr: Expr,
     VarDecl {
         lefts: []ObjectData;
@@ -46,6 +47,7 @@ public enum Stmt {
     public func position(self) -> token.Pos {
         return match self is {
             .Empty as empty_pos -> empty_pos,
+            .Comment as comment -> comment.pos,
             .Expr as expr -> expr.position(),
             .VarDecl as var_decl -> var_decl.pos,
             .While as while_stmt -> while_stmt.pos,

lib/rivet/src/parser/decls.ri

@@ -9,21 +9,15 @@ import ../report;
 import ../token;
 
 extend Parser {
-    func parse_doc_comment(mut self) -> []ast.Comment {
+    func parse_doc_comments(mut self) -> []ast.Comment {
         mut comments := @vec(ast.Comment);
-        while self.accept(.DocComment) or self.accept(.Comment) {
-            comments.push(
-                ast.Comment(
-                    self.prev_tok.kind == .DocComment,
-                    self.prev_tok.lit.split_into_lines(),
-                    self.prev_tok.pos
-                )
-            );
+        while self.tok.kind == .DocComment {
+            comments.push(self.parse_comment(true));
         }
         return comments;
     }
 
-    public func parse_attributes(
+    func parse_attributes(
         mut self, parse_mod_attributes: bool := false
     ) -> ast.Attributes {
         mut attributes := ast.Attributes();
@@ -97,7 +91,7 @@ extend Parser {
     }
 
     func parse_decl(mut self) -> ast.Decl {
-        doc_comment := self.parse_doc_comment();
+        doc_comment := self.parse_doc_comments();
         attributes := self.parse_attributes(
             self.tok.kind == .Hash and self.peek_tok.kind == .Bang
         );

lib/rivet/src/parser/mod.ri

@@ -155,4 +155,35 @@ public struct Parser {
         }
         return false;
     }
+
+    func parse_comment(mut self, parse_doc_comment: bool := false) -> ast.Comment {
+        if !((parse_doc_comment and self.accept(.DocComment)) or self.accept(.Comment)) {
+            return ast.Comment();
+        }
+        return ast.Comment(
+            self.prev_tok.kind == .DocComment,
+            self.prev_tok.lit.split_into_lines(),
+            self.prev_tok.pos
+        );
+    }
+
+    // NOTE:
+    //   same_line: only eat comments on the same line as the previous token.
+    //   follow_up: comments directly below the previous token as long as there
+    //   is no empty line.
+    func eat_comments(mut self, same_line: bool := false, follow_up: bool := false) -> []ast.Comment {
+        mut comments := @vec(ast.Comment);
+        mut line := self.prev_tok.pos.line;
+        while {
+            if self.tok.kind != .Comment or (same_line and self.tok.pos.line > line)
+                or (follow_up and (self.tok.pos.line > line + 1 or self.tok.lit.contains("\n"))) {
+                break;
+            }
+            comments.push(self.parse_comment());
+            if follow_up {
+                line = self.prev_tok.pos.line;
+            }
+        }
+        return comments;
+    }
 }

lib/rivet/src/parser/stmts.ri

@@ -6,8 +6,14 @@ import ../ast;
 import ../report;
 
 extend Parser {
+    #[inline]
+    func parse_comment_stmt(mut self) -> ast.Stmt {
+        return .Comment(self.parse_comment());
+    }
+
     func parse_stmt(mut self) -> ast.Stmt {
         return match {
+            self.tok.kind == .Comment -> self.parse_comment_stmt(),
             self.accept(.KwWhile) -> {
                 pos := self.prev_tok.pos;
                 mut is_inf := false;

lib/rivet/src/tokenizer/next.ri

@@ -94,7 +94,7 @@ extend Tokenizer {
                             self.line -= 1;
                         }
                         if (is_doc_comment and self.prefs.is_docs) or self.prefs.is_fmt {
-                            line := self.text[start..end].trim_space();
+                            line := self.text[start..end];
                             return self.new_token(.Comment, line, line.len);
                         }
                         continue;
@@ -126,7 +126,7 @@ extend Tokenizer {
                             break;
                         }
                         if self.prefs.is_fmt {
-                            line := self.text[start_pos..end_pos].trim_space();
+                            line := self.text[start_pos..end_pos];
                             return self.new_multiline_token(
                                 .Comment, line, line.len, start_line, self.line, start_col
                             );