Feat/various improvements

.gitignore

@@ -12,7 +12,7 @@ __arkscript__/
 *.arkc
 *.arkm
 /*.ark
-/*.ark.ir
+*.ark.ir
 !tests/unittests/resources/BytecodeReaderSuite/*.arkc
 
 # Generated files

CHANGELOG.md

@@ -41,9 +41,15 @@
 - `STORE_FROM_INDEX` and `SET_VAL_FROM_INDEX` instructions for parity with the super instructions not using load by index
 - `INCREMENT_BY_INDEX` and `DECREMENT_BY_INDEX` instructions for parity with the super instructions not using load by index
 - `STORE_TAIL_BY_INDEX`, `STORE_HEAD_BY_INDEX`, `SET_VAL_TAIL_BY_INDEX`, `SET_VAL_HEAD_BY_INDEX` super instructions added for parity with the super instructions not using load by index
-- `RESET_SCOPE` instruction emitted at the end of a while loop to reset a scope so that we can create multiple variables and use `LOAD_SYMBOL_BY_INDEX`
+- `RESET_SCOPE_JUMP` instruction emitted at the end of a while loop to reset a scope so that we can create multiple variables and use `LOAD_SYMBOL_BY_INDEX`
 - instruction source location ; two new bytecode tables were added: one for filenames, another for (page pointer, instruction pointer, file id, line), allowing the VM to display better error messages when the source is available
 - show source location when a runtime error is thrown in the VM
+- `LT_CONST_JUMP_IF_FALSE` and `LT_SYM_JUMP_IF_FALSE` to compare a symbol to a const and a symbol to a symbol (respectively), then jump to an address if false (useful for while loops that check a simple `(< x n)` condition)
+- `CALL_SYMBOL` super instruction to load and call a symbol in a single instruction
+- `GET_FIELD_FROM_SYMBOL` and `GET_FIELD_FROM_SYMBOL_INDEX` super instructions to get a field from a closure and push it to the stack
+- `EQ_CONST_JUMP_IF_TRUE` and `EQ_SYM_INDEX_JUMP_IF_TRUE` to compare a symbol to a const and a symbol to a symbol (respectively), then jump to an address if true (useful for conditions that check a simple `(= x n)` condition)
+- `NEQ_CONST_JUMP_IF_TRUE` as a super instruction counterpart to `EQ_CONST_JUMP_IF_TRUE`
+- `AT_SYM_SYM` and `AT_SYM_INDEX_SYM_INDEX` super instructions, to get an element from a list in a single instruction, avoiding 2 push and 2 pop
 
 ### Changed
 - instructions are on 4 bytes: 1 byte for the instruction, 1 byte of padding, 2 bytes for an immediate argument
@@ -54,7 +60,7 @@
 - more documentation about the compiler implementation
 - more documentation about the virtual machine
 - closures can be now be compared field per field: `(= closure1 closure2)` will work only if they have the same fields (name) and if the values match
-- macros are now defined like `($ name value)` / `($ name (args args args) body)` / `($if cond then else)`
+- macros are now defined like `(macro name value)` / `(macro name (args args args) body)` / `($if cond then else)`
 - upgraded from C++17 to C++20
 - new parser, new syntax for imports: `(import package.sub.file)`
 - allow nodes to be empty when dumping the AST to JSON
@@ -67,7 +73,7 @@
 - fixed a bug in the compiler where one could pass something other than a list to `(fun)` as the argument block, resulting in a crash
 - fixed a bug in the compiler generating not callable functions
 - fixed a bug in the macro processor generating invalid `let` / `mut` / `set` nodes
-- fixed a bug in the macro processor allowing out of bounds access with `($ test (@ [1 2 3] -5))`
+- fixed a bug in the macro processor allowing out of bounds access with `(macro test (@ [1 2 3] -5))`
 - fixed a bug in the vm which wrongfully allowed self concat in place: `(concat! lst lst)`
 - fixed a bug in the compiler where one could "use" operators without calling them: `(print nil?)`
 - fixed a bug in the compiler allowing the use of operators without any argument: `(+)`

CMakeLists.txt

@@ -264,6 +264,9 @@ if (ARK_BENCHMARKS)
 endif ()
 
 if (ARK_BUILD_EXE)
+    # ArkScript lib needs to know if we are building an exe to enable/disable specific code for embedding
+    target_compile_definitions(ArkReactor PRIVATE ARK_BUILD_EXE=1)
+
     # additional files needed for the exe (repl, command line and stuff)
     file(GLOB_RECURSE EXE_SOURCES
             ${ark_SOURCE_DIR}/src/arkscript/*.cpp

examples/fizz_buzz.ark

@@ -5,9 +5,10 @@
   r
   (fun (e)
     (if (= 0 (mod e 15))
-      (print "FizzBuzz")
+      (puts "FizzBuzz,")
       (if (= 0 (mod e 3))
-        (print "Fizz")
+        (puts "Fizz,")
         (if (= 0 (mod e 5))
-          (print "Buzz")
-          (print e))))))
+          (puts "Buzz,")
+          (puts e ","))))))
+(puts "\n")

examples/macros.ark

@@ -1,9 +1,9 @@
-($ suffix-dup (sym x) {
+(macro suffix-dup (sym x) {
   ($if (> x 1) (suffix-dup sym (- x 1)))
   ($symcat sym x) })
 
-($ partial (func ...defargs) {
-  ($ bloc (suffix-dup a (- ($argcount func) (len defargs))))
+(macro partial (func ...defargs) {
+  (macro bloc (suffix-dup a (- ($argcount func) (len defargs))))
   (fun (bloc) (func ...defargs bloc))
   ($undef bloc) })
 
@@ -15,10 +15,10 @@
 (print "Expected arguments for test_func1: " ($argcount test_func1) ", expected " 2)
 (print "Calling them: " (test_func 1 2 3) " " (test_func1 2 3))
 
-($ foo (a b) (+ a b))
+(macro foo (a b) (+ a b))
 (print "Using macro foo (a b) => (+ a b): " (foo 1 2))
 
-($ var 12)
+(macro var 12)
 (print "Using macro constant var=12: " var)
 
 ($if (= var 12)
@@ -29,42 +29,42 @@
   (print "This was executed in a if macro, testing (and true true)")
   (print "You shouldn't see this (bis)"))
 
-($ defun (name args body) (let name (fun args body)))
+(macro defun (name args body) (let name (fun args body)))
 (defun a_func (a b) (+ a b))
 (print "Generated a function with a macro, a_func (a b) => (+ a b)")
 (print "Calling (a_func 1 2): " (a_func 1 2))
 
-($ one (...args) (print "Macro 'one', returns the 2nd argument given in " args " => " (@ args 1)))
+(macro one (...args) (print "Macro 'one', returns the 2nd argument given in " args " => " (@ args 1)))
 (one 1 2)
 (one 1 3 4)
 (one 1 5 6 7 8)
 
-($ last (...args) (print "Macro 'last', returns the last argument given in " args " => " (@ args -1)))
+(macro last (...args) (print "Macro 'last', returns the last argument given in " args " => " (@ args -1)))
 (last 1 2)
 (last 1 3 4)
 (last 1 5 6 7 8)
 
 {
   (print "Testing macros in scopes and macro shadowing")
 
-  ($ test (+ 1 2 3))
+  (macro test (+ 1 2 3))
   (print "(global) Reading macro 'test', expected 6, " test)
 
   ((fun () {
-    ($ test (- 1 2 3))
+    (macro test (- 1 2 3))
     (print "(sub scope) Reading macro 'test', expected -4, " test) }))
 
   (print "(global) Reading macro 'test', expected 6, " test)
 
   {
-    ($ test 555)
+    (macro test 555)
     (print "(subscope) Reading macro 'test', expected 555, " test)
-    ($ undef test)
+    (macro undef test)
     (print "(subscope, undef test) Reading macro 'test', expected 6, " test)
-    ($ undef a) } }
+    (macro undef a) } }
 (print "Demonstrating a threading macro")
 
-($ -> (arg fn1 ...fn) {
+(macro -> (arg fn1 ...fn) {
   ($if (> (len fn) 0)
     (-> (fn1 arg) ...fn)
     (fn1 arg)) })

include/Ark/Compiler/AST/Parser.hpp

@@ -19,8 +19,9 @@
 #include <Ark/Platform.hpp>
 
 #include <string>
-#include <optional>
 #include <vector>
+#include <optional>
+#include <functional>
 
 #include <utf8.hpp>
 
@@ -61,6 +62,8 @@ namespace Ark::internal
         Node m_ast;
         std::vector<Import> m_imports;
         unsigned m_allow_macro_behavior;  ///< Toggled on when inside a macro definition, off afterward
+        std::size_t m_nested_nodes;       ///< Nested node counter
+        std::vector<std::function<std::optional<Node>()>> m_parsers;
 
         /**
          * @brief Update a node given a file position

include/Ark/Compiler/Instructions.hpp

@@ -44,7 +44,7 @@ namespace Ark::internal
         // @role Load a symbol from the locals stack by its index (starting from the end of the current scope)
         LOAD_SYMBOL_BY_INDEX = 0x02,
 
-        // @args symbol id
+        // @args constant id
         // @role Load a constant from its ID onto the stack
         LOAD_CONST = 0x03,
 
@@ -143,8 +143,8 @@ namespace Ark::internal
         // @role Create a new local scope
         CREATE_SCOPE = 0x1d,
 
-        // @role Reset the current scope so that it is empty
-        RESET_SCOPE = 0x1e,
+        // @role Reset the current scope so that it is empty, and jump to a given location
+        RESET_SCOPE_JUMP = 0x1e,
 
         // @role Destroy the last local scope
         POP_SCOPE = 0x1f,
@@ -303,6 +303,41 @@ namespace Ark::internal
         // @role Call a builtin by its id in #[code primary], with #[code secondary] arguments. Bypass the stack size check because we do not push IP/PP since builtins calls do not alter the stack
         CALL_BUILTIN = 0x4b,
 
+        // @args constant id, absolute address to jump to
+        // @role Compare #[code TS < constant], if the comparison fails, jump to the given address. Otherwise, does nothing
+        LT_CONST_JUMP_IF_FALSE = 0x4c,
+
+        // @args symbol id, absolute address to jump to
+        // @role Compare #[code TS < symbol], if the comparison fails, jump to the given address. Otherwise, does nothing
+        LT_SYM_JUMP_IF_FALSE = 0x4d,
+
+        // @args constant id, absolute address to jump to
+        // @role Compare #[code TS == constant], if the comparison succeeds, jump to the given address. Otherwise, does nothing
+        EQ_CONST_JUMP_IF_TRUE = 0x4e,
+
+        // @args symbol index, absolute address to jump to
+        // @role Compare #[code TS == symbol], if the comparison succeeds, jump to the given address. Otherwise, does nothing
+        EQ_SYM_INDEX_JUMP_IF_TRUE = 0x4f,
+
+        // @args constant id, absolute address to jump to
+        // @role Compare #[code TS != constant], if the comparison succeeds, jump to the given address. Otherwise, does nothing
+        NEQ_CONST_JUMP_IF_TRUE = 0x50,
+
+        // @args symbol id, argument count
+        // @role Call a symbol by its id in #[code primary], with #[code secondary] arguments
+        CALL_SYMBOL = 0x51,
+
+        // @args symbol id, field id in symbols table
+        // @role Push the field of a given symbol (which has to be a closure) on the stack
+        GET_FIELD_FROM_SYMBOL = 0x52,
+
+        // @args symbol index, field id in symbols table
+        // @role Push the field of a given symbol (which has to be a closure) on the stack
+        GET_FIELD_FROM_SYMBOL_INDEX = 0x53,
+
+        AT_SYM_SYM = 0x54,
+        AT_SYM_INDEX_SYM_INDEX = 0x55,
+
         InstructionsCount
     };
 
@@ -337,7 +372,7 @@ namespace Ark::internal
         "POP",
         "DUP",
         "CREATE_SCOPE",
-        "RESET_SCOPE",
+        "RESET_SCOPE_JUMP",
         "POP_SCOPE",
         // operators
         "ADD",
@@ -384,7 +419,17 @@ namespace Ark::internal
         "SET_VAL_TAIL_BY_INDEX",
         "SET_VAL_HEAD",
         "SET_VAL_HEAD_BY_INDEX",
-        "CALL_BUILTIN"
+        "CALL_BUILTIN",
+        "LT_CONST_JUMP_IF_FALSE",
+        "LT_SYM_JUMP_IF_FALSE",
+        "EQ_CONST_JUMP_IF_TRUE",
+        "EQ_SYM_INDEX_JUMP_IF_TRUE",
+        "NEQ_CONST_JUMP_IF_TRUE",
+        "CALL_SYMBOL",
+        "GET_FIELD_FROM_SYMBOL",
+        "GET_FIELD_FROM_SYMBOL_INDEX",
+        "AT_SYM_SYM",
+        "AT_SYM_INDEX_SYM_INDEX"
     };
 
     static_assert(InstructionNames.size() == static_cast<std::size_t>(Instruction::InstructionsCount) && "Some instruction names appear to be missing");

include/Ark/Compiler/IntermediateRepresentation/Entity.hpp

@@ -24,8 +24,7 @@ namespace Ark::internal::IR
     {
         Label,
         Goto,
-        GotoIfTrue,
-        GotoIfFalse,
+        GotoWithArg,
         Opcode,
         Opcode2Args
     };
@@ -46,7 +45,9 @@ namespace Ark::internal::IR
 
         static Entity Label(label_t value);
 
-        static Entity Goto(const Entity& label);
+        static Entity Goto(const Entity& label, Instruction inst = Instruction::JUMP);
+
+        static Entity GotoWithArg(const Entity& label, Instruction inst, uint16_t primary_arg);
 
         static Entity GotoIf(const Entity& label, bool cond);
 

include/Ark/Compiler/IntermediateRepresentation/IROptimizer.hpp

@@ -48,19 +48,34 @@ namespace Ark::internal
 
     private:
         using Entities = std::vector<IR::Entity>;
-        using DualArgs = std::pair<uint16_t, uint16_t>;
+        using Condition_t = std::function<bool(const Entities&)>;
+        using Replacement_t = std::function<IR::Entity(const Entities&)>;
 
         struct Rule
         {
             std::vector<Instruction> expected;
-            Instruction replacement;
-            std::function<bool(const Entities&)> condition = [](const Entities&) {
+            Condition_t condition;            ///< Additional condition to match
+            Replacement_t createReplacement;  ///< Create the replacement instructions from given context
+
+            constexpr static auto default_cond = [](const Entities&) {
                 return true;
-            };  ///< Additional condition to match
-            std::function<DualArgs(const Entities&)> createReplacement =
-                [](const Entities& entities) {
-                    return std::make_pair(entities[0].primaryArg(), entities[1].primaryArg());
-                };  ///< Create the replacement instructions from given context
+            };
+
+            Rule(std::vector<Instruction>&& input, Instruction replacement, Condition_t&& cond = default_cond) :
+                expected(std::move(input)), condition(std::move(cond))
+            {
+                createReplacement = [replacement](const Entities& e) {
+                    return IR::Entity(replacement, e[0].primaryArg(), e[1].primaryArg());
+                };
+            }
+
+            Rule(std::vector<Instruction>&& input, Condition_t&& cond, Replacement_t&& repl) :
+                expected(std::move(input)), condition(std::move(cond)), createReplacement(std::move(repl))
+            {}
+
+            Rule(std::vector<Instruction>&& input, Replacement_t&& repl) :
+                expected(std::move(input)), condition(default_cond), createReplacement(std::move(repl))
+            {}
         };
 
         std::vector<Rule> m_ruleset_two;

include/Ark/Constants.hpp.in

@@ -62,6 +62,7 @@ namespace Ark
         | FeatureIROptimizer
         | FeatureNameResolver;
 
+    constexpr std::size_t MaxNestedNodes = 1024;  ///< Maximum number of nodes that can be nested while parsing code
     constexpr std::size_t MaxMacroProcessingDepth = 256;  ///< Controls the number of recursive calls to MacroProcessor::processNode
     constexpr std::size_t MaxMacroUnificationDepth = 256;  ///< Controls the number of recursive calls to MacroProcessor::unify
     constexpr std::size_t VMStackSize = 4096;

include/Ark/VM/VM.hpp

@@ -148,6 +148,14 @@ namespace Ark
          */
         [[nodiscard]] bool forceReloadPlugins() const;
 
+        /**
+         * @brief Throw a VM error message
+         *
+         * @param kind type of VM error
+         * @param message
+         */
+        [[noreturn]] static void throwVMError(internal::ErrorKind kind, const std::string& message);
+
         friend class Value;
         friend class internal::Closure;
         friend class Repl;
@@ -229,6 +237,8 @@ namespace Ark
          */
         inline void setVal(uint16_t id, const Value* val, internal::ExecutionContext& context);
 
+        Value getField(Value* closure, uint16_t id, internal::ExecutionContext& context);
+
         // ================================================
         //                 stack related
         // ================================================
@@ -330,15 +340,7 @@ namespace Ark
          */
         uint16_t findNearestVariableIdWithValue(const Value& value, internal::ExecutionContext& context) const noexcept;
 
-        /**
-         * @brief Throw a VM error message
-         *
-         * @param kind type of VM error
-         * @param message
-         */
-        static void throwVMError(internal::ErrorKind kind, const std::string& message);
-
-        void throwArityError(std::size_t passed_arg_count, std::size_t expected_arg_count, internal::ExecutionContext& context);
+        [[noreturn]] void throwArityError(std::size_t passed_arg_count, std::size_t expected_arg_count, internal::ExecutionContext& context);
 
         void showBacktraceWithException(const std::exception& e, internal::ExecutionContext& context);
 
@@ -365,8 +367,9 @@ namespace Ark
          *
          * @param context
          * @param argc number of arguments already sent
+         * @param function_ptr optional pointer to the function to call. If not provided, obtain it from the stack
          */
-        inline void call(internal::ExecutionContext& context, uint16_t argc);
+        inline void call(internal::ExecutionContext& context, uint16_t argc, Value* function_ptr = nullptr);
 
         /**
          * @brief Builtin called when the CALL_BUILTIN instruction is met in the bytecode