Serialize/bytecode numbers

.github/workflows/benchmarks.yml

@@ -31,7 +31,8 @@ jobs:
         run: |
           cmake -Bbuild -DCMAKE_BUILD_TYPE=Release \
             -DCMAKE_C_COMPILER=clang-16 -DCMAKE_CXX_COMPILER=clang++-16 \
-            -DARK_SANITIZERS=Off -DARK_BUILD_EXE=On -DARK_BUILD_MODULES=Off
+            -DARK_SANITIZERS=Off -DARK_BUILD_EXE=On -DARK_BUILD_MODULES=Off \
+            -DARK_UNITY_BUILD=On
           cmake --build build --config Release -- -j 4
 
       - uses: actions/setup-node@v4

.github/workflows/ci.yml

@@ -274,7 +274,8 @@ jobs:
             -DCMAKE_CXX_COMPILER=/AFLplusplus/afl-c++ \
             -DCMAKE_BUILD_TYPE=Release \
             -DARK_BUILD_EXE=On \
-            -DARK_SANITIZERS=On
+            -DARK_SANITIZERS=On \
+            -DARK_UNITY_BUILD=On
           cmake --build ${BUILD_FOLDER} --config Release -j 4
 
       - name: Fuzz

.github/workflows/setup-compilers/action.yaml

@@ -102,7 +102,8 @@ runs:
          -DARK_COVERAGE=${{ inputs.coverage }} \
          -DARK_BUILD_EXE=On \
          -DARK_BUILD_MODULES=$ToggleModules -DARK_MOD_ALL=$ToggleModules -DARK_MOD_DRAFT=$ToggleModules \
-         -DARK_TESTS=On
+         -DARK_TESTS=On \
+         -DARK_UNITY_BUILD=On
         cmake --build build --config $BUILD_TYPE -j $(nproc)
 
     - name: Configure CMake Ark
@@ -121,5 +122,6 @@ runs:
           -DARK_BUILD_EXE=On \
           -DARK_BUILD_MODULES=$ToggleModules -DARK_MOD_ALL=$ToggleModules -DARK_MOD_DRAFT=$ToggleModules \
           -DARK_TESTS=On \
+          -DARK_UNITY_BUILD=On \
           $additional_args
         cmake --build build --config $BUILD_TYPE -j $(nproc)

CHANGELOG.md

@@ -104,6 +104,10 @@
 - loops have their own scope: variables created inside a loop won't leak outside it
 - upgraded fmtlib to 11.1.3-13
 - allow capture in nested scope (before it was targeting only the current scope)
+- `-bcr` option can be given a source file, it will then be compiled before its bytecode is shown
+- magic numbers for tables start in bytecode files have been changed from 0x01, 0x02, 0x03 to 0xA1, 0xA2, 0xA3 (symbols, values, code) to make them stand out in hex editors
+- magic numbers for value types in bytecode files have been changed from 0x01, 0x02, 0x03 to 0xF1, 0xF2, 0xF3 (number, string, function)
+- numbers in the values table in bytecode files are no longer stringified but their IEEE754 representation is now encoded on 12 bytes (4 for the exponent, 8 for the mantissa)
 
 ### Removed
 - removed unused `NodeType::Closure`

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.12)
+cmake_minimum_required(VERSION 3.16)
 
 project(ark CXX)
 
@@ -22,6 +22,7 @@ option(ARK_SANITIZERS "Enable ASAN and UBSAN" Off)
 option(ARK_TESTS "Build ArkScript unit tests" Off)
 option(ARK_BENCHMARKS "Build ArkScript benchmarks" Off)
 option(ARK_COVERAGE "Enable coverage while building (clang, gcc) (requires ARK_TESTS to be On)" Off)
+option(ARK_UNITY_BUILD "Enable unity build" Off)
 
 include(cmake/link_time_optimization.cmake)
 include(cmake/sanitizers.cmake)
@@ -53,6 +54,11 @@ target_include_directories(ArkReactor
         ${ark_SOURCE_DIR}/include)
 target_compile_features(ArkReactor PRIVATE cxx_std_20)
 
+if (ARK_UNITY_BUILD)
+    set_target_properties(ArkReactor PROPERTIES UNITY_BUILD ON UNITY_BUILD_MODE BATCH UNITY_BUILD_BATCH_SIZE 16)
+    set_source_files_properties(src/arkreactor/VM/VM.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION true)
+endif ()
+
 if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANG OR APPLE)
     message(STATUS "Enabling computed gotos")
     target_compile_definitions(ArkReactor PRIVATE ARK_USE_COMPUTED_GOTOS=1)
@@ -194,6 +200,10 @@ if (ARK_TESTS)
     target_compile_features(unittests PRIVATE cxx_std_20)
     target_compile_definitions(unittests PRIVATE ARK_TESTS_ROOT="${CMAKE_CURRENT_SOURCE_DIR}/")
 
+    if (ARK_UNITY_BUILD)
+        set_target_properties(unittests PROPERTIES UNITY_BUILD ON UNITY_BUILD_MODE BATCH UNITY_BUILD_BATCH_SIZE 16)
+    endif ()
+
     if (ARK_COVERAGE AND CMAKE_COMPILER_IS_CLANG)
         target_compile_options(unittests PRIVATE -coverage -fcoverage-mapping -fprofile-instr-generate)
         target_link_options(unittests PRIVATE -coverage -fcoverage-mapping -fprofile-instr-generate)
@@ -250,6 +260,10 @@ if (ARK_BUILD_EXE)
     target_link_libraries(arkscript PUBLIC ArkReactor replxx clipp)
     target_compile_features(arkscript PRIVATE cxx_std_20)
 
+    if (ARK_UNITY_BUILD)
+        set_target_properties(arkscript PROPERTIES UNITY_BUILD ON UNITY_BUILD_MODE BATCH UNITY_BUILD_BATCH_SIZE 16)
+    endif ()
+
     enable_lto(arkscript)
 
     # Installs the arkscript executable.

Dockerfile

@@ -33,7 +33,7 @@ COPY --from=submodule-initializor /out .
 COPY --from=submodule-initializor /rev .
 RUN cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
-    -DARK_BUILD_EXE=On -DARK_COMMIT="$(cat rev)" \
+    -DARK_BUILD_EXE=On -DARK_COMMIT="$(cat rev)" -DARK_UNITY_BUILD=On \
     && cmake --build build --target arkscript -- -j $(nproc)
 
 FROM alpine:3.19 AS organizer

README.md

@@ -172,23 +172,25 @@ DESCRIPTION
         ArkScript programming language
 
 SYNOPSIS
-        arkscript -h
-        arkscript -v
-        arkscript --dev-info
-        arkscript -e <expression>
+        arkscript -h 
+        arkscript -v 
+        arkscript --dev-info 
+        arkscript -e <expression> 
         arkscript -c <file> [-d] [-f(importsolver|no-importsolver)]
                   [-f(macroprocessor|no-macroprocessor)] [-f(optimizer|no-optimizer)]
+                  [-f(iroptimizer|no-iroptimizer)] [-fdump-ir] 
 
         arkscript <file> [-d] [-L <lib_dir>] [-f(importsolver|no-importsolver)]
                   [-f(macroprocessor|no-macroprocessor)] [-f(optimizer|no-optimizer)]
+                  [-f(iroptimizer|no-iroptimizer)] [-fdump-ir]  
 
-        arkscript -f <file> [--(dry-run|check)]
-        arkscript --ast <file> [-d] [-L <lib_dir>]
-        arkscript -bcr <file> -on
-        arkscript -bcr <file> -a [-s <start> <end>]
-        arkscript -bcr <file> -st [-s <start> <end>]
-        arkscript -bcr <file> -vt [-s <start> <end>]
-        arkscript -bcr <file> [-cs] [-p <page>] [-s <start> <end>]
+        arkscript -f <file> [--(dry-run|check)] 
+        arkscript --ast <file> [-d] [-L <lib_dir>] 
+        arkscript -bcr <file> -on 
+        arkscript -bcr <file> -a [-s <start> <end>] 
+        arkscript -bcr <file> -st [-s <start> <end>] 
+        arkscript -bcr <file> -vt [-s <start> <end>] 
+        arkscript -bcr <file> [-cs] [-p <page>] [-s <start> <end>] 
 
 OPTIONS
         -h, --help                  Display this message
@@ -206,6 +208,10 @@ OPTIONS
                                     Toggle on and off the macro processor pass
 
         -f(optimizer|no-optimizer)  Toggle on and off the optimizer pass
+        -f(iroptimizer|no-iroptimizer)
+                                    Toggle on and off the IR optimizer pass
+
+        -fdump-ir                   Dump IR to file.ark.ir
         -d, --debug...              Increase debug level (default: 0)
 
         -L, --lib                   Set the location of the ArkScript standard library. Paths can be
@@ -218,6 +224,10 @@ OPTIONS
                                     Toggle on and off the macro processor pass
 
         -f(optimizer|no-optimizer)  Toggle on and off the optimizer pass
+        -f(iroptimizer|no-iroptimizer)
+                                    Toggle on and off the IR optimizer pass
+
+        -fdump-ir                   Dump IR to file.ark.ir
         -f, --format                Format the given source file in place
         --dry-run                   Do not modify the file, only print out the changes
         --check                     Check if a file formating is correctly, without modifying it.
@@ -230,8 +240,8 @@ OPTIONS
                                     delimited by ';'
 
         -bcr, --bytecode-reader     Launch the bytecode reader
-        <file>                      If file isn't a bytecode file, the cached compiled will be
-                                    loaded ; if there are none, it will be compiled first
+        <file>                      .arkc bytecode file or .ark source file that will be compiled
+                                    first
 
         -on, --only-names           Display only the bytecode segments names and sizes
         -a, --all                   Display all the bytecode segments (default)
@@ -242,7 +252,7 @@ OPTIONS
         -s, --slice                 Select a slice of instructions in the bytecode
 
 VERSION
-        4.0.0-ff04fd55
+        4.0.0-c24c8f22
 
 LICENSE
         Mozilla Public License 2.0

harden.dockerfile

@@ -33,7 +33,7 @@ COPY --from=submodule-initializor /out .
 COPY --from=submodule-initializor /rev .
 RUN cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
-    -DARK_BUILD_EXE=On -DARK_ENABLE_SYSTEM=Off -DARK_COMMIT="$(cat rev)" \
+    -DARK_BUILD_EXE=On -DARK_ENABLE_SYSTEM=Off -DARK_COMMIT="$(cat rev)" -DARK_UNITY_BUILD=On \
     && cmake --build build --target arkscript -- -j $(nproc)
 
 FROM alpine:3.19 AS organizer

include/Ark/Compiler/Common.hpp

@@ -16,6 +16,7 @@
 #include <string_view>
 #include <vector>
 #include <cinttypes>
+#include <Ark/Constants.hpp>
 
 namespace Ark
 {
@@ -24,6 +25,21 @@ namespace Ark
 
 namespace Ark::internal
 {
+    namespace bytecode
+    {
+        constexpr std::array Magic = { 'a', 'r', 'k', '\0' };
+        constexpr std::array Version = {
+            ARK_VERSION_MAJOR & 0xff00,
+            ARK_VERSION_MAJOR & 0x00ff,
+            ARK_VERSION_MINOR & 0xff00,
+            ARK_VERSION_MINOR & 0x00ff,
+            ARK_VERSION_PATCH & 0xff00,
+            ARK_VERSION_PATCH & 0x00ff
+        };
+        constexpr std::size_t TimestampLength = 8;
+        constexpr std::size_t HeaderSize = Magic.size() + Version.size() + TimestampLength;
+    }
+
     /// The different node types available
     enum class NodeType
     {

include/Ark/Compiler/Compiler.hpp

@@ -85,7 +85,6 @@ namespace Ark::internal
         std::vector<IR::Block> m_temp_pages;  ///< we need temporary code pages for some compilations passes
         IR::label_t m_current_label = 0;
 
-        unsigned m_debug;  ///< the debug level of the compiler
         Logger m_logger;
 
         /**

include/Ark/Compiler/Instructions.hpp

@@ -28,12 +28,12 @@ namespace Ark::internal
     {
         // @role Does nothing, useful for padding
         NOP = 0x00,
-        SYM_TABLE_START = 0x01,
-        VAL_TABLE_START = 0x02,
-        NUMBER_TYPE = 0x01,
-        STRING_TYPE = 0x02,
-        FUNC_TYPE = 0x03,
-        CODE_SEGMENT_START = 0x03,
+        SYM_TABLE_START = 0xA1,
+        VAL_TABLE_START = 0xA2,
+        CODE_SEGMENT_START = 0xA3,
+        NUMBER_TYPE = 0xF1,
+        STRING_TYPE = 0xF2,
+        FUNC_TYPE = 0xF3,
 
         // @args symbol id
         // @role Load a symbol from its ID onto the stack

include/Ark/Compiler/Serialization/IEEE754Serializer.hpp

@@ -0,0 +1,60 @@
+#ifndef ARK_COMPILER_SERIALIZATION_IEEE754SERIALIZER_HPP
+#define ARK_COMPILER_SERIALIZATION_IEEE754SERIALIZER_HPP
+
+#include <climits>
+#include <cmath>
+#include <limits>
+#include <vector>
+#include <bit>
+
+namespace Ark::internal::ieee754
+{
+    // Narrowing conversion from long long to double, 9223372036854775807 becomes 9223372036854775808.
+    // This gives us an error margin of 1.08420217248550443400745280086994171142578125 * 10^-19,
+    // which is acceptable.
+    static constexpr auto MaxLong = static_cast<double>(std::numeric_limits<std::int64_t>::max());
+
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+    template <class T>
+    constexpr T bswap(T i)
+    {
+        return i;
+    }
+#else
+    // https://stackoverflow.com/a/36937049/21584
+    template <class T, std::size_t... N>
+    constexpr T bswap_impl(T i, std::index_sequence<N...>)
+    {
+        return (((i >> N * CHAR_BIT & static_cast<std::uint8_t>(-1)) << (sizeof(T) - 1 - N) * CHAR_BIT) | ...);
+    }
+    template <class T, class U = std::make_unsigned_t<T>>
+    constexpr T bswap(T i)
+    {
+        return std::bit_cast<T>(bswap_impl<U>(std::bit_cast<U>(i), std::make_index_sequence<sizeof(T)> {}));
+    }
+#endif
+
+    struct DecomposedDouble
+    {
+        int32_t exponent;
+        int64_t mantissa;
+    };
+
+    [[nodiscard]] inline DecomposedDouble serialize(const double n)
+    {
+        int exp = 0;
+        const auto mant = static_cast<std::int64_t>(MaxLong * std::frexp(n, &exp));
+
+        return DecomposedDouble {
+            .exponent = std::bit_cast<int32_t>(bswap(exp)),
+            .mantissa = bswap(mant)
+        };
+    }
+
+    [[nodiscard]] inline double deserialize(const DecomposedDouble d)
+    {
+        return std::ldexp(static_cast<double>(bswap(d.mantissa)) / MaxLong, std::bit_cast<int32_t>(bswap(d.exponent)));
+    }
+}
+
+#endif  // ARK_COMPILER_SERIALIZATION_IEEE754SERIALIZER_HPP

include/Ark/Compiler/Serialization/IntegerSerializer.hpp

@@ -0,0 +1,67 @@
+#ifndef ARK_COMPILER_SERIALIZATION_INTEGERSERIALIZER_HPP
+#define ARK_COMPILER_SERIALIZATION_INTEGERSERIALIZER_HPP
+
+#include <span>
+#include <vector>
+#include <concepts>
+
+namespace Ark::internal
+{
+    void serializeToVecLE(std::integral auto number, std::vector<uint8_t>& out)
+    {
+        constexpr auto mask = static_cast<decltype(number)>(0xff);
+
+        for (std::size_t i = 0; i < sizeof(decltype(number)); ++i)
+            out.push_back(static_cast<uint8_t>((number & (mask << (8 * i))) >> (8 * i)));
+    }
+
+    void serializeToVecBE(std::integral auto number, std::vector<uint8_t>& out)
+    {
+        constexpr auto pad = sizeof(decltype(number)) - 1;
+        constexpr auto mask = static_cast<decltype(number)>(0xff);
+
+        for (std::size_t i = 0; i < sizeof(decltype(number)); ++i)
+        {
+            const auto shift = 8 * (pad - i);
+            out.push_back(static_cast<uint8_t>((number & (mask << shift)) >> shift));
+        }
+    }
+
+    void serializeOn2BytesToVecLE(std::integral auto number, std::vector<uint8_t>& out)
+    {
+        constexpr auto mask = static_cast<decltype(number)>(0xff);
+        for (std::size_t i = 0; i < 2; ++i)
+            out.push_back(static_cast<uint8_t>((number & (mask << (8 * i))) >> (8 * i)));
+    }
+
+    void serializeOn2BytesToVecBE(std::integral auto number, std::vector<uint8_t>& out)
+    {
+        constexpr auto mask = static_cast<decltype(number)>(0xff);
+        for (std::size_t i = 0; i < 2; ++i)
+            out.push_back(static_cast<uint8_t>((number & (mask << (8 * (1 - i)))) >> (8 * (1 - i))));
+    }
+
+    template <std::integral T>
+    T deserializeLE(std::vector<uint8_t>::const_iterator begin, std::vector<uint8_t>::const_iterator end)
+    {
+        constexpr std::size_t length = sizeof(T);
+        T result {};
+        for (std::size_t i = 0; i < length && begin != end; ++i, ++begin)
+            result += static_cast<T>(*begin) << (8 * i);
+
+        return result;
+    }
+
+    template <std::integral T>
+    T deserializeBE(std::vector<uint8_t>::const_iterator begin, std::vector<uint8_t>::const_iterator end)
+    {
+        constexpr std::size_t length = sizeof(T) - 1;
+        T result {};
+        for (std::size_t i = 0; i < length && begin != end; ++i, ++begin)
+            result += static_cast<T>(*begin) << (8 * (length - i));
+
+        return result;
+    }
+}
+
+#endif  // ARK_COMPILER_SERIALIZATION_INTEGERSERIALIZER_HPP