Feat/better parser position tracking

include/Ark/Ark.hpp

@@ -11,7 +11,7 @@
 #ifndef ARK_ARK_HPP
 #define ARK_ARK_HPP
 
-#include <Ark/Exceptions.hpp>
+#include <Ark/Error/Exceptions.hpp>
 #include <Ark/Constants.hpp>
 #include <Ark/Utils/Utils.hpp>
 #include <Ark/VM/VM.hpp>

include/Ark/Compiler/AST/BaseParser.hpp

@@ -6,7 +6,7 @@
 #include <initializer_list>
 
 #include <Ark/Utils/Platform.hpp>
-#include <Ark/Exceptions.hpp>
+#include <Ark/Error/Exceptions.hpp>
 #include <Ark/Compiler/AST/Predicates.hpp>
 #include <Ark/Compiler/AST/utf8_char.hpp>
 
@@ -30,8 +30,9 @@ namespace Ark::internal
         std::string m_str;
         std::vector<std::pair<std::string::iterator, std::size_t>> m_it_to_row;  ///< A crude map of \n position to line number to speed up line number computing
         std::string::iterator m_it, m_next_it;
-        utf8_char_t m_sym;       ///< The current utf8 character we're on
-        FilePosition m_filepos;  ///< The position of the cursor in the file
+        utf8_char_t m_sym;                ///< The current utf8 character we're on
+        FilePosition m_filepos;           ///< The position of the cursor in the file
+        FilePosition m_previous_filepos;  ///< The previous position of the cursor in the file
 
         /**
          * @brief Register the position of a new line, with an iterator pointing to the new line and the row number
@@ -52,16 +53,18 @@ namespace Ark::internal
         void initParser(const std::string& filename, const std::string& code);
 
         [[nodiscard]] FilePosition getCursor() const;
+        [[nodiscard]] FilePosition getPreviousCursor() const;
 
-        [[nodiscard]] CodeErrorContext generateErrorContext(const std::string& expr);
+        [[nodiscard]] CodeErrorContext generateErrorContextAtCurrentPosition() const;
 
         /**
+         * @brief Create an error context and throw an error containing said context
          *
          * @param error an error message
-         * @param exp the expression causing the error
+         * @param start_at position in the file where the parsing for the erroneous token started
          * @param additional_context optional context created when a node is being parsed
          */
-        void error(const std::string& error, std::string exp, const std::optional<CodeErrorContext>& additional_context = std::nullopt);
+        void error(const std::string& error, FilePosition start_at, const std::optional<CodeErrorContext>& additional_context = std::nullopt) const;
 
         /**
          * @brief Fetch the next token (space and paren delimited) to generate an error
@@ -129,8 +132,8 @@ namespace Ark::internal
         bool space(std::string* s = nullptr);
         bool inlineSpace(std::string* s = nullptr);
         bool comment(std::string* s = nullptr);
-        bool spaceComment(std::string* s = nullptr);
-        bool newlineOrComment(std::string* s = nullptr);
+        [[nodiscard]] std::string spaceComment();
+        [[nodiscard]] std::string newlineOrComment();
         bool prefix(char c);
         bool number(std::string* s = nullptr);
         bool signedNumber(std::string* s = nullptr);

include/Ark/Compiler/AST/Node.hpp

@@ -20,6 +20,7 @@
 #include <Ark/Compiler/AST/Namespace.hpp>
 #include <Ark/Compiler/Common.hpp>
 #include <Ark/Utils/Platform.hpp>
+#include <Ark/Utils/Position.hpp>
 
 namespace Ark::internal
 {
@@ -162,19 +163,11 @@ namespace Ark::internal
         void setString(const std::string& value) noexcept;
 
         /**
-         * @brief Set the Position of the node in the text
+         * @brief Position the current node at a given span in a file
          *
-         * @param line
-         * @param col
+         * @param source node to copy filename and position from
          */
-        void setPos(std::size_t line, std::size_t col) noexcept;
-
-        /**
-         * @brief Set the original Filename where the node was
-         *
-         * @param filename
-         */
-        void setFilename(const std::string& filename) noexcept;
+        void setPositionFrom(const Node& source) noexcept;
 
         /**
          * @brief Set the comment field with the nearest comment before this node
@@ -216,18 +209,11 @@ namespace Ark::internal
         [[nodiscard]] bool isAltSyntax() const;
 
         /**
-         * @brief Get the line at which this node was created
-         *
-         * @return std::size_t
-         */
-        [[nodiscard]] std::size_t line() const noexcept;
-
-        /**
-         * @brief Get the column at which this node was created
+         * @brief Get the span of the node (start and end)
          *
-         * @return std::size_t
+         * @return const FileSpan
          */
-        [[nodiscard]] std::size_t col() const noexcept;
+        [[nodiscard]] FileSpan position() const noexcept;
 
         /**
          * @brief Return the filename in which this node was created
@@ -263,13 +249,14 @@ namespace Ark::internal
 
         friend bool operator==(const Node& A, const Node& B);
         friend bool operator<(const Node& A, const Node& B);
+        friend class Parser;
 
     private:
         NodeType m_type { NodeType::Unused };
         Value m_value;
         std::optional<std::string> m_unqualified_name { std::nullopt };  ///< Used by Capture nodes, to have the FQN in the value, and the captured name here
         // position of the node in the original code, useful when it comes to parser errors
-        std::size_t m_line = 0, m_col = 0;
+        FileSpan m_pos;
         std::string m_filename;
         std::string m_comment;
         std::string m_after_comment;          ///< Comment after node

include/Ark/Compiler/AST/Parser.hpp

@@ -63,198 +63,32 @@ namespace Ark::internal
         std::vector<Import> m_imports;
         unsigned m_allow_macro_behavior;  ///< Toggled on when inside a macro definition, off afterward
         std::size_t m_nested_nodes;       ///< Nested node counter
-        std::vector<std::function<std::optional<Node>()>> m_parsers;
+        std::vector<std::function<std::optional<Node>(FilePosition)>> m_parsers;
 
-        /**
-         * @brief Update a node given a file position
-         * @param node node to update
-         * @param cursor the node position in file
-         * @return Node& the modified node
-         */
-        Node& setNodePosAndFilename(Node& node, const std::optional<FilePosition>& cursor = std::nullopt) const;
+        [[nodiscard]] Node positioned(Node node, FilePosition cursor) const;
+        [[nodiscard]] std::optional<Node>& positioned(std::optional<Node>& node, FilePosition cursor) const;
 
         std::optional<Node> node();
-        std::optional<Node> letMutSet();
-        std::optional<Node> del();
-        std::optional<Node> condition();
-        std::optional<Node> loop();
-        std::optional<Node> import_();
-        std::optional<Node> block();
-        std::optional<Node> functionArgs();
-        std::optional<Node> function();
-        std::optional<Node> macroCondition();
-        std::optional<Node> macroArgs();
-        std::optional<Node> macro();
-        std::optional<Node> functionCall();
-        std::optional<Node> list();
-
-        std::optional<Node> number()
-        {
-            auto pos = getCount();
-
-            std::string res;
-            if (signedNumber(&res))
-            {
-                double output;
-                if (Utils::isDouble(res, &output))
-                    return std::optional<Node>(output);
-                backtrack(pos);
-                error("Is not a valid number", res);
-            }
-            return std::nullopt;
-        }
-
-        std::optional<Node> string()
-        {
-            std::string res;
-            if (accept(IsChar('"')))
-            {
-                while (true)
-                {
-                    if (accept(IsChar('\\')))
-                    {
-                        if (!m_interpret)
-                            res += '\\';
-
-                        if (accept(IsChar('"')))
-                            res += '"';
-                        else if (accept(IsChar('\\')))
-                            res += '\\';
-                        else if (accept(IsChar('n')))
-                            res += m_interpret ? '\n' : 'n';
-                        else if (accept(IsChar('t')))
-                            res += m_interpret ? '\t' : 't';
-                        else if (accept(IsChar('v')))
-                            res += m_interpret ? '\v' : 'v';
-                        else if (accept(IsChar('r')))
-                            res += m_interpret ? '\r' : 'r';
-                        else if (accept(IsChar('a')))
-                            res += m_interpret ? '\a' : 'a';
-                        else if (accept(IsChar('b')))
-                            res += m_interpret ? '\b' : 'b';
-                        else if (accept(IsChar('f')))
-                            res += m_interpret ? '\f' : 'f';
-                        else if (accept(IsChar('u')))
-                        {
-                            std::string seq;
-                            if (hexNumber(4, &seq))
-                            {
-                                if (m_interpret)
-                                {
-                                    char utf8_str[5];
-                                    utf8::decode(seq.c_str(), utf8_str);
-                                    if (*utf8_str == '\0')
-                                        error("Invalid escape sequence", "\\u" + seq);
-                                    res += utf8_str;
-                                }
-                                else
-                                    res += "u" + seq;
-                            }
-                            else
-                                error("Invalid escape sequence", "\\u");
-                        }
-                        else if (accept(IsChar('U')))
-                        {
-                            std::string seq;
-                            if (hexNumber(8, &seq))
-                            {
-                                if (m_interpret)
-                                {
-                                    std::size_t begin = 0;
-                                    for (; seq[begin] == '0'; ++begin)
-                                        ;
-                                    char utf8_str[5];
-                                    utf8::decode(seq.c_str() + begin, utf8_str);
-                                    if (*utf8_str == '\0')
-                                        error("Invalid escape sequence", "\\U" + seq);
-                                    res += utf8_str;
-                                }
-                                else
-                                    res += "U" + seq;
-                            }
-                            else
-                                error("Invalid escape sequence", "\\U");
-                        }
-                        else
-                        {
-                            backtrack(getCount() - 1);
-                            error("Unknown escape sequence", "\\");
-                        }
-                    }
-                    else
-                        accept(IsNot(IsEither(IsChar('\\'), IsChar('"'))), &res);
-
-                    if (accept(IsChar('"')))
-                        break;
-                    if (isEOF())
-                        expectSuffixOrError('"', "after string");
-                }
-
-                return { Node(NodeType::String, res) };
-            }
-            return std::nullopt;
-        }
-
-        std::optional<Node> field()
-        {
-            std::string sym;
-            if (!name(&sym))
-                return std::nullopt;
-
-            std::optional<Node> leaf { Node(NodeType::Field) };
-            setNodePosAndFilename(leaf.value());
-            leaf->push_back(Node(NodeType::Symbol, sym));
-
-            while (true)
-            {
-                if (leaf->list().size() == 1 && !accept(IsChar('.')))  // Symbol:abc
-                    return std::nullopt;
-
-                if (leaf->list().size() > 1 && !accept(IsChar('.')))
-                    break;
-                std::string res;
-                if (!name(&res))
-                    errorWithNextToken("Expected a field name: <symbol>.<field>");
-                leaf->push_back(Node(NodeType::Symbol, res));
-            }
-
-            return leaf;
-        }
-
-        std::optional<Node> symbol()
-        {
-            std::string res;
-            if (!name(&res))
-                return std::nullopt;
-            return { Node(NodeType::Symbol, res) };
-        }
-
-        std::optional<Node> spread()
-        {
-            std::string res;
-            if (sequence("..."))
-            {
-                if (!name(&res))
-                    errorWithNextToken("Expected a name for the variadic");
-                return { Node(NodeType::Spread, res) };
-            }
-            return std::nullopt;
-        }
-
-        std::optional<Node> nil()
-        {
-            if (!accept(IsChar('(')))
-                return std::nullopt;
-
-            std::string comment;
-            newlineOrComment(&comment);
-            if (!accept(IsChar(')')))
-                return std::nullopt;
-
-            if (m_interpret)
-                return { Node(NodeType::Symbol, "nil").attachNearestCommentBefore(comment) };
-            return { Node(NodeType::List).attachNearestCommentBefore(comment) };
-        }
+        std::optional<Node> letMutSet(FilePosition filepos);
+        std::optional<Node> del(FilePosition filepos);
+        std::optional<Node> condition(FilePosition filepos);
+        std::optional<Node> loop(FilePosition filepos);
+        std::optional<Node> import_(FilePosition filepos);
+        std::optional<Node> block(FilePosition filepos);
+        std::optional<Node> functionArgs(FilePosition filepos);
+        std::optional<Node> function(FilePosition filepos);
+        std::optional<Node> macroCondition(FilePosition filepos);
+        std::optional<Node> macroArgs(FilePosition filepos);
+        std::optional<Node> macro(FilePosition filepos);
+        std::optional<Node> functionCall(FilePosition filepos);
+        std::optional<Node> list(FilePosition filepos);
+
+        std::optional<Node> number(FilePosition filepos);
+        std::optional<Node> string(FilePosition filepos);
+        std::optional<Node> field(FilePosition filepos);
+        std::optional<Node> symbol(FilePosition filepos);
+        std::optional<Node> spread(FilePosition filepos);
+        std::optional<Node> nil(FilePosition filepos);
 
         /**
          * @brief Try to parse an atom (number, string, spread, field, symbol, nil)
@@ -264,7 +98,7 @@ namespace Ark::internal
 
         /**
          * @brief Try to parse an atom, if any, match its type against the given list
-         * @param types autorized types
+         * @param types authorized types
          * @return std::optional<Node> std::nullopt if the parsed atom didn't match the given types
          */
         std::optional<Node> anyAtomOf(std::initializer_list<NodeType> types);
@@ -281,7 +115,7 @@ namespace Ark::internal
          * @param name construction name, eg "let", "condition"
          * @return std::optional<Node> std::nullopt if the parser didn't match
          */
-        std::optional<Node> wrapped(std::optional<Node> (Parser::*parser)(), const std::string& name);
+        std::optional<Node> wrapped(std::optional<Node> (Parser::*parser)(FilePosition), const std::string& name);
     };
 }
 

include/Ark/Compiler/Package/ImportSolver.hpp

@@ -83,10 +83,10 @@ namespace Ark::internal
          * @brief Search for an import file, using the root file path
          *
          * @param file path to the file containing the import
-         * @param import current import directive
+         * @param import_ current import directive
          * @return std::filesystem::path
          */
-        [[nodiscard]] std::filesystem::path findFile(const std::filesystem::path& file, const Import& import) const;
+        [[nodiscard]] std::filesystem::path findFile(const std::filesystem::path& file, const Import& import_) const;
     };
 }