🐛 Add keyword support + fix number search

Author: gabriel-dp

include/token.hpp

@@ -1,6 +1,7 @@
 #ifndef TOKEN_HPP
 #define TOKEN_HPP
 
+#include <iostream>
 #include <regex>
 #include <string>
 #include <vector>
@@ -9,6 +10,7 @@ using namespace std;
 
 enum TokenType {
     NONE,
+    KEYWORD,
     IDENTIFICATOR,
     OPERATOR,
     SEPARATOR,
@@ -26,9 +28,10 @@ const string LITERAL_INCOMPLETE_PATTERN = R"(^\"([^\"\\\n]|\\.)*$)";
 const string COMMENT_LINE_PATTERN = R"(^//.*)";
 const string COMMENT_BLOCK_PATTERN = R"(/\*[\s\S]*?\*/)";
 const string COMMENT_BLOCK_INCOMPLETE_PATTERN = R"(/\*[\s\S]*)";
-const vector<string> OPERATORS = {"*", "/", "%", "+", "-", "==", "!=", "&&", "||", "=", "+=", "-=", "*=", "/=", "%=", "&", "|", ".", ">", "<", ">=", "<=", "!"};
+const vector<string> OPERATORS = {"*", "/", "%", "+", "-", "==", "!=", "&&", "||", "=", "+=", "-=", "*=", "/=", "%=", "&", "|", ".", ">", "<", ">=", "<=", "!", "++", "--"};
 const vector<string> SEPARATORS_IGNORED = {" ", "\n", "\t"};
 const vector<string> SEPARATORS_IMPORTANT = {"(", ")", "{", "}", "[", "]", ";", ",", "."};
+const vector<string> KEYWORDS = {"break", "case", "char", "const", "continue", "default", "do", "double", "else", "enum", "float", "for", "goto", "if", "int", "long", "return", "short", "sizeof", "static", "struct", "switch", "typedef", "void", "while"};
 
 class Token {
    public:
@@ -39,13 +42,40 @@ class Token {
 
     Token() {};
     Token(string content, TokenType type, pair<int, int> position) : content(move(content)), type(type), position(position) {};
+
+    void print() {
+        cout << "(" << position.first << "," << position.second << ") " << enum_type_string(type) << endl;
+        cout << content << "\n\n";
+    }
+
+    static string enum_type_string(TokenType type) {
+        switch (type) {
+            case NONE:
+                return "NONE";
+            case IDENTIFICATOR:
+                return "IDENTIFICATOR";
+            case NUMBER:
+                return "NUMBER";
+            case SEPARATOR:
+                return "SEPARATOR";
+            case OPERATOR:
+                return "OPERATOR";
+            case LITERAL:
+                return "LITERAL";
+            case COMMENT:
+                return "COMMENT";
+            case KEYWORD:
+                return "KEYWORD";
+            default:
+                return "UNKNOWN";
+        }
+    }
 };
 
 void lexical_error(Token token);
 vector<Token>
 tokenize(string& buffer);
 Token get_token(string::iterator& sentinel, string::iterator end, pair<int, int>& position);
 regex create_regex(vector<string> tokens);
-string enum_type_string(TokenType type);
 
 #endif

src/main.cpp

@@ -25,8 +25,7 @@ int main(int argc, char* argv[]) {
     string buffer = get_content(argc, argv);
     vector<Token> tokens = tokenize(buffer);
     for (Token t : tokens) {
-        cout << "(" << t.position.first << "," << t.position.second << ") " << enum_type_string(t.type) << endl;
-        cout << t.content << "\n\n";
+        t.print();
     }
     return 0;
 }

src/token.cpp

@@ -18,6 +18,7 @@ const regex COMMENT_BLOCK_INCOMPLETE_REGEX = regex(COMMENT_BLOCK_INCOMPLETE_PATT
 const regex OPERATORS_REGEX = create_regex(OPERATORS);
 const regex SEPARATORS_IGNORED_REGEX = create_regex(SEPARATORS_IGNORED);
 const regex SEPARATORS_IMPORTANT_REGEX = create_regex(SEPARATORS_IMPORTANT);
+const regex KEYWORDS_REGEX = create_regex(KEYWORDS);
 
 string escape_regex(const string& str) {
     static const regex special_chars(R"([.^$|()\\[*+?{}])");
@@ -67,7 +68,7 @@ Token get_token(string::iterator& sentinel, string::iterator end, pair<int, int>
                 break;
             } else if (regex_search(actual_char, OPERATORS_REGEX)) {
                 token.type = OPERATOR;
-            } else if (regex_search(actual_char, NUMERIC_INCOMPLETE_REGEX)) {
+            } else if (regex_search(actual_char, NUMERIC_REGEX)) {
                 token.type = NUMBER;
             } else if (regex_search(actual_char, LITERAL_INCOMPLETE_REGEX)) {
                 token.type = LITERAL;
@@ -79,6 +80,18 @@ Token get_token(string::iterator& sentinel, string::iterator end, pair<int, int>
                 }
                 token.content = content_updated;
                 lexical_error(token);
+            } else if (regex_search(content_updated, KEYWORDS_REGEX)) {
+                token.type = KEYWORD;
+            }
+        } else if (token.type == KEYWORD) {
+            if (!regex_search(content_updated, KEYWORDS_REGEX)) {
+                if (regex_search(content_updated, ALPHA_NUMERIC_REGEX)) {
+                    token.type = IDENTIFICATOR;
+                } else if (regex_search(actual_char, SEPARATORS_IMPORTANT_REGEX) || regex_search(actual_char, OPERATORS_REGEX)) {
+                    break;
+                }
+                token.content = content_updated;
+                lexical_error(token);
             }
         } else if (token.type == OPERATOR) {
             if (regex_search(content_updated, COMMENT_LINE_REGEX) || regex_search(content_updated, COMMENT_BLOCK_INCOMPLETE_REGEX)) {
@@ -87,6 +100,7 @@ Token get_token(string::iterator& sentinel, string::iterator end, pair<int, int>
                 if (regex_search(actual_char, SEPARATORS_IMPORTANT_REGEX) || regex_search(actual_char, ALPHA_REGEX) || regex_search(actual_char, NUMERIC_REGEX) || regex_search(actual_char, LITERAL_INCOMPLETE_REGEX)) {
                     break;
                 }
+                break;
                 token.content = content_updated;
                 lexical_error(token);
             }
@@ -141,27 +155,6 @@ vector<Token> tokenize(string& buffer) {
     return tokens;
 }
 
-string enum_type_string(TokenType type) {
-    switch (type) {
-        case NONE:
-            return "NONE";
-        case IDENTIFICATOR:
-            return "IDENTIFICATOR";
-        case NUMBER:
-            return "NUMBER";
-        case SEPARATOR:
-            return "SEPARATOR";
-        case OPERATOR:
-            return "OPERATOR";
-        case LITERAL:
-            return "LITERAL";
-        case COMMENT:
-            return "COMMENT";
-        default:
-            return "UNKNOWN";
-    }
-}
-
 void lexical_error(Token token) {
     cout << "Invalid token " << token.content << " at line " << token.position.second << " (" << token.position.first << ")\n";
     exit(1);