From 5cefda01b20fb3355cfb84463ea901f32c17443a Mon Sep 17 00:00:00 2001
From: Zev Averbach <zev@averba.ch>
Date: Mon, 16 Dec 2024 16:31:06 +0100
Subject: [PATCH 1/2] fix: small bug in whitespace_tokens, added a few tests
 for such bugs.

---
 crates/pg_lexer/src/lib.rs | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/crates/pg_lexer/src/lib.rs b/crates/pg_lexer/src/lib.rs
index df24f8d85..6f64cc9e4 100644
--- a/crates/pg_lexer/src/lib.rs
+++ b/crates/pg_lexer/src/lib.rs
@@ -89,7 +89,7 @@ fn whitespace_tokens(input: &str) -> VecDeque<Token> {
         } else if let Some(tab) = cap.name("tab") {
             tokens.push_back(Token {
                 token_type: TokenType::Whitespace,
-                kind: SyntaxKind::Newline,
+                kind: SyntaxKind::Tab,
                 text: tab.as_str().to_string(),
                 span: TextRange::new(
                     TextSize::from(u32::try_from(tab.start()).unwrap()),
@@ -177,6 +177,27 @@ pub fn lex(text: &str) -> Vec<Token> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_tab_tokens() {
+        let input = "select\t1";
+        let tokens = lex(input);
+        assert_eq!(tokens[1].kind, SyntaxKind::Tab);
+    }
+
+    #[test]
+    fn test_newline_tokens() {
+        let input = "select\n1";
+        let tokens = lex(input);
+        assert_eq!(tokens[1].kind, SyntaxKind::Newline);
+    }
+
+    #[test]
+    fn test_whitespace_tokens() {
+        let input = "select 1";
+        let tokens = lex(input);
+        assert_eq!(tokens[1].kind, SyntaxKind::Whitespace);
+    }
+
     #[test]
     fn test_lexer() {
         let input = "select 1; \n -- some comment \n select 2";

From e5ef1c4bdf7a9648ee42ca0bb9975e7e05fb0018 Mon Sep 17 00:00:00 2001
From: Zev Averbach <zev@averba.ch>
Date: Mon, 16 Dec 2024 16:36:23 +0100
Subject: [PATCH 2/2] fix: added a tab to the test_lexer test as well

---
 crates/pg_lexer/src/lib.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/crates/pg_lexer/src/lib.rs b/crates/pg_lexer/src/lib.rs
index 6f64cc9e4..0bf933e97 100644
--- a/crates/pg_lexer/src/lib.rs
+++ b/crates/pg_lexer/src/lib.rs
@@ -200,7 +200,7 @@ mod tests {
 
     #[test]
     fn test_lexer() {
-        let input = "select 1; \n -- some comment \n select 2";
+        let input = "select 1; \n -- some comment \n select 2\t";
 
         let tokens = lex(input);
         let mut tokens_iter = tokens.iter();
@@ -248,5 +248,8 @@ mod tests {
         let token = tokens_iter.next().unwrap();
         assert_eq!(token.kind, SyntaxKind::Iconst);
         assert_eq!(token.text, "2");
+
+        let token = tokens_iter.next().unwrap();
+        assert_eq!(token.kind, SyntaxKind::Tab);
     }
 }