From 4efb3a650f9c1235e11948bf3dcd4065e6c64e36 Mon Sep 17 00:00:00 2001
From: Philipp Grulich <philippgrulich@hotmail.de>
Date: Wed, 18 Dec 2024 00:41:08 +0100
Subject: [PATCH] add execution benchmarks

---
 nautilus/test/benchmark/CMakeLists.txt        | 11 ++--
 .../test/benchmark/ExecutionBenchmark.cpp     | 59 +++++++++++++++++++
 2 files changed, 65 insertions(+), 5 deletions(-)
 create mode 100644 nautilus/test/benchmark/ExecutionBenchmark.cpp

diff --git a/nautilus/test/benchmark/CMakeLists.txt b/nautilus/test/benchmark/CMakeLists.txt
index 39677bac..b449146a 100644
--- a/nautilus/test/benchmark/CMakeLists.txt
+++ b/nautilus/test/benchmark/CMakeLists.txt
@@ -1,15 +1,16 @@
 if (ENABLE_TRACING AND ENABLE_BENCHMARKS)
-    add_executable(nautilus-tracing-benchmarks
+    add_executable(nautilus-benchmarks
             TracingBenchmark.cpp
+            ExecutionBenchmark.cpp
     )
-    target_include_directories(nautilus-tracing-benchmarks PRIVATE
+    target_include_directories(nautilus-benchmarks PRIVATE
             $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../common>
             $<INSTALL_INTERFACE:src/nautilus/>)
-    target_include_directories(nautilus-tracing-benchmarks PRIVATE
+    target_include_directories(nautilus-benchmarks PRIVATE
             $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../../src>
             $<INSTALL_INTERFACE:src/nautilus/>)
 
-    target_link_libraries(nautilus-tracing-benchmarks PUBLIC nautilus Catch2::Catch2WithMain)
+    target_link_libraries(nautilus-benchmarks PUBLIC nautilus Catch2::Catch2WithMain)
     list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras)
-    catch_discover_tests(nautilus-tracing-benchmarks EXTRA_ARGS --allow-running-no-tests)
+    catch_discover_tests(nautilus-benchmarks EXTRA_ARGS --allow-running-no-tests)
 endif ()
\ No newline at end of file
diff --git a/nautilus/test/benchmark/ExecutionBenchmark.cpp b/nautilus/test/benchmark/ExecutionBenchmark.cpp
new file mode 100644
index 00000000..3da26186
--- /dev/null
+++ b/nautilus/test/benchmark/ExecutionBenchmark.cpp
@@ -0,0 +1,59 @@
+
+#include "TracingUtil.hpp"
+#include "nautilus/Engine.hpp"
+#include "nautilus/compiler/backends/mlir/MLIRCompilationBackend.hpp"
+#include "nautilus/compiler/ir/IRGraph.hpp"
+#include "nautilus/config.hpp"
+#include "nautilus/tracing/ExecutionTrace.hpp"
+#include "nautilus/tracing/TraceContext.hpp"
+#include "nautilus/tracing/phases/SSACreationPhase.hpp"
+#include "nautilus/tracing/phases/TraceToIRConversionPhase.hpp"
+#include <catch2/catch_all.hpp>
+
+namespace nautilus::engine {
+
+val<int8_t> int8AddExpression(val<int8_t> x);
+
+void runAddBenchmark(Catch::Benchmark::Chronometer& meter, Options& options) {
+	auto engine = engine::NautilusEngine(options);
+	auto func = engine.registerFunction(int8AddExpression);
+	meter.measure([&] { return func(42); });
+}
+
+static auto benchmarks = std::vector<std::tuple<std::string, std::function<void(Catch::Benchmark::Chronometer& meter, Options& options)>>> {
+    {"add", runAddBenchmark},
+};
+
+TEST_CASE("Execution Benchmark") {
+
+	std::vector<std::string> backends = {};
+#ifdef ENABLE_MLIR_BACKEND
+	backends.emplace_back("mlir");
+#endif
+#ifdef ENABLE_C_BACKEND
+	backends.emplace_back("cpp");
+#endif
+#ifdef ENABLE_BC_BACKEND
+	backends.emplace_back("bc");
+#endif
+#ifdef ENABLE_ASMJIT_BACKEND
+	backends.emplace_back("asmjit");
+#endif
+	for (auto& backend : backends) {
+		for (auto& test : benchmarks) {
+			auto func = std::get<1>(test);
+			auto name = std::get<0>(test);
+
+			Catch::Benchmark::Benchmark("exec_" + backend + "_" + name)
+			    .operator=([&func, backend](Catch::Benchmark::Chronometer meter) {
+				    auto op = engine::Options();
+				    // force compilation for the MLIR backend.
+				    op.setOption("mlir.eager_compilation", true);
+				    op.setOption("engine.backend", backend);
+				    func(meter, op);
+			    });
+		}
+	}
+}
+
+} // namespace nautilus::engine