Basic lexer

.clang-format

@@ -1,8 +1,14 @@
 ---
 AlignAfterOpenBracket: AlwaysBreak
 AllowShortBlocksOnASingleLine: "Always"
+AlignConsecutiveShortCaseStatements:
+  Enabled: true
+AllowShortEnumsOnASingleLine: 'false'
 AllowShortIfStatementsOnASingleLine: WithoutElse
+AllowShortLambdasOnASingleLine: 'true'
+AllowShortLoopsOnASingleLine: 'true'
 BasedOnStyle: Chromium
+BinPackParameters: 'OnePerLine'
 ColumnLimit: '100'
 ConstructorInitializerAllOnOneLineOrOnePerLine: 'true'
 Cpp11BracedListStyle: 'true'

.github/workflows/main.yml

@@ -9,10 +9,16 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
       - uses: pre-commit/action@v3.0.0
-  # unit_tests:
-  #   runs-on: ubuntu-24.04
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - run: sudo apt-get install clang-19
-  #     - run: cmake -DCMAKE_CXX_COMPILER=clang++-19 -DRUN_TESTS=true -B build && cmake --build build
-  #     - run: ./build/tests/test_exe
+  unit_tests:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba
+          eval "$(bin/micromamba shell hook --shell bash)"
+          micromamba create -n main -c conda-forge gxx ninja libstdcxx meson
+      - run: bin/micromamba run -n main meson setup build
+        env:
+          CXX: /home/runner/.local/share/mamba/envs/main/bin/g++
+      - run: bin/micromamba run -n main meson compile -C build
+      - run: ./build/test_exe

.gitignore

@@ -2,3 +2,6 @@ iris.log
 build/
 .cache/
 .gdb_history
+
+/subprojects/*
+!/subprojects/*.wrap

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 ---
-exclude: ^include|^build
+exclude: ^build|^subprojects
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0
@@ -13,10 +13,11 @@ repos:
     rev: v21.1.8
     hooks:
       - id: clang-format
-  - repo: https://github.com/cmake-lint/cmake-lint
-    rev: 1.4.3
+  - repo: https://github.com/trim21/pre-commit-mirror-meson
+    rev: v1.10.1
     hooks:
-      - id: cmakelint
+      - id: meson-fmt
+        args: ['--inplace']
   - repo: https://github.com/rhysd/actionlint
     rev: v1.7.10
     hooks:

meson.build

@@ -1,8 +1,17 @@
 project('winter', 'cpp', default_options: ['cpp_std=c++23'])
 cpp_flags = ['-Wall', '-Wextra', '-Wconversion', '-Wimplicit-fallthrough']
 
-src_files = [
-  'src/compiler.cpp'
-]
+src_files = ['src/compiler.cpp', 'src/lexer.cpp']
 winter_src = static_library('winter_src', src_files, cpp_args: cpp_flags)
 executable('winter', 'src/main.cpp', cpp_args: cpp_flags, link_with: winter_src)
+
+# tests
+willow = dependency('willow', method: 'cmake')
+executable(
+    'test_exe',
+    'tests/test.cpp',
+    link_with: winter_src,
+    cpp_args: cpp_flags,
+    dependencies: willow,
+    include_directories: 'src',
+)

src/error.h

@@ -0,0 +1,28 @@
+#ifndef WINTER_ERROR_H
+#define WINTER_ERROR_H
+
+#include <cstdint>
+#include <expected>
+#include <string>
+
+namespace Winter {
+    // pre-defined
+    struct Token;
+
+    enum class ErrType : uint8_t {
+        Lexer,
+        NotImplemented
+    };
+
+    struct Error {
+        ErrType type;
+        std::string msg;
+
+        explicit constexpr Error(ErrType t, std::string m) : type(t), msg(m) {}
+    };
+
+    using token_result_t = std::expected<Token, Error>;
+
+}  // namespace Winter
+
+#endif  // WINTER_ERROR_H

src/lexer.cpp

@@ -0,0 +1,174 @@
+#include "lexer.h"
+
+#include <algorithm>
+#include <array>
+#include <format>
+
+namespace Winter {
+    auto Lexer::skipWhitespace() -> void {
+        static constexpr std::array<char, 3> whitespace = {' ', '\n', '\t'};
+        auto cmp = [&](const char c) { return c == src.at(playhead); };
+        while (std::any_of(whitespace.begin(), whitespace.end(), cmp)) { playhead++; }
+    }
+
+    // TODO: utility?
+    [[nodiscard]] auto between(int min, int max, int val) -> bool {
+        return (min <= val && val <= max);
+    }
+
+    [[nodiscard]] auto Lexer::isNumeric() -> bool {
+        if (playhead >= src.size()) { return false; }
+        static constexpr std::array<char, 11> digits = {'0', '1', '2', '3', '4', '5',
+                                                        '6', '7', '8', '9', '.'};
+
+        auto cmp = [&](const char c) { return c == src.at(playhead); };
+        return std::any_of(digits.begin(), digits.end(), cmp);
+    }
+
+    [[nodiscard]] auto Lexer::isLetter() -> bool {
+        if (playhead >= src.size()) { return false; }
+        char c = src.at(playhead);
+        // [A-Za-z0-9_]
+        return (between(65, 90, c) || between(97, 122, c) || between(48, 57, c) || c == '_');
+    }
+
+    [[nodiscard]] auto Lexer::lexNumeric() -> token_result_t {
+        Token t = Token(TokenType::NUM_LITERAL, playhead);
+        while (isNumeric()) {
+            t.len++;
+            playhead++;
+            if (playhead >= src.size()) { break; }
+        }
+
+        if (t.len == 0) {
+            return std::unexpected(
+                Error(ErrType::Lexer, std::format("Invalid numeric found at {}", playhead)));
+        }
+
+        return t;
+    }
+
+    [[nodiscard]] auto Lexer::lexSingle(const TokenType type) -> token_result_t {
+        playhead++;
+        return Token(type, playhead - 1, 1);
+    }
+
+    [[nodiscard]] auto Lexer::lexDouble(const char c1, const TokenType single, const TokenType pair)
+        -> token_result_t {
+        playhead++;
+        if (src.at(playhead) == c1) {
+            playhead++;
+            return Token(pair, playhead - 2, 2);
+        }
+        return Token(single, playhead - 1, 1);
+    }
+
+    [[nodiscard]] auto Lexer::lexChar() -> token_result_t {
+        playhead += 2;
+        if (playhead >= src.size() || src.at(playhead) != '\'') {
+            return std::unexpected(
+                Error(ErrType::Lexer, std::format("Malformed char at pos {}", playhead)));
+        }
+
+        playhead++;
+        return Token(TokenType::CHAR_LITERAL, playhead - 3, 3);
+    }
+
+    [[nodiscard]] auto Lexer::lexString() -> token_result_t {
+        if (src.at(playhead) != '"') {
+            return std::unexpected(
+                Error(ErrType::Lexer, "Parsing string started at invalid location"));
+        }
+
+        std::size_t strlen = 1;
+        playhead++;
+        while (src.at(playhead) != '"') {
+            if (playhead >= src.size()) { break; }
+
+            strlen++;
+            playhead++;
+
+            if (playhead > src.size()) {
+                return std::unexpected(Error(ErrType::Lexer, "Unclosed string"));
+            }
+        }
+
+        // Include the closing quote
+        strlen++;
+        playhead++;
+
+        return Token(TokenType::STR_LITERAL, playhead - strlen, strlen);
+    }
+
+    [[nodiscard]] auto Lexer::lexIdentKeyword() -> std::expected<Token, Error> {
+        const std::size_t start = playhead;
+        while (isLetter()) { playhead++; }
+
+        TokenType type = TokenType::IDENT;
+        if (keywords.contains(src.substr(start, playhead - start))) {
+            type = keywords.at(src.substr(start, playhead - start));
+        }
+
+        if (types.contains(src.substr(start, playhead - start))) { type = TokenType::TYPE_LITERAL; }
+
+        return Token(type, start, playhead - start);
+    }
+
+    [[nodiscard]] auto Lexer::operator()(std::string_view source) -> token_result_t {
+        src = source;
+        skipWhitespace();
+
+        switch (src.at(playhead)) {
+            case '(':
+                return lexSingle(TokenType::LPAREN);
+            case ')':
+                return lexSingle(TokenType::RPAREN);
+            case '{':
+                return lexSingle(TokenType::LBRACE);
+            case '}':
+                return lexSingle(TokenType::RBRACE);
+            case '[':
+                return lexSingle(TokenType::LSQUACKET);
+            case ']':
+                return lexSingle(TokenType::RSQUACKET);
+            case ':':
+                return lexSingle(TokenType::COLON);
+            case ';':
+                return lexSingle(TokenType::SEMICOLON);
+            case '+':
+                return lexSingle(TokenType::PLUS);
+            case '-':
+                return lexSingle(TokenType::MINUS);
+            case '*':
+                return lexSingle(TokenType::STAR);
+            case '/':
+                return lexSingle(TokenType::SLASH);
+            case ',':
+                return lexSingle(TokenType::COMMA);
+            case '.':
+                return lexDouble('.', TokenType::DOT, TokenType::DOT_DOT);
+            case '>':
+                return lexDouble('=', TokenType::GREATER, TokenType::GREATER_EQ);
+            case '<':
+                return lexDouble('=', TokenType::LESS, TokenType::LESS_EQ);
+            case '!':
+                return lexDouble('=', TokenType::NOT, TokenType::NOT_EQ);
+            case '&':
+                return lexDouble('&', TokenType::ERROR, TokenType::AND);
+            case '|':
+                return lexDouble('|', TokenType::ERROR, TokenType::OR);
+            case '\'':
+                return lexChar();
+            case '"':
+                return lexString();
+            default:
+                break;
+        }
+
+        if (isNumeric()) { return lexNumeric(); }
+        if (isLetter()) { return lexIdentKeyword(); }
+
+        return std::unexpected(
+            Error(ErrType::Lexer, std::format("Invalid token found at {}", playhead)));
+    }
+};  // namespace Winter