Add compiler profile to prioritize memory speed

benches/README.md

@@ -64,15 +64,20 @@ A few observations:
 
 ## Memory usage:
 
-| rules set      | boreal  | yara   |
-| -------------- | ----    | ----   |
-| orion          | 12.8 MB | 12.3MB |
-| atr            | 12.6 MB | 14.0MB |
-| reversinglabs  | 14.9 MB | 15.8MB |
-| panopticon     | 10.9 MB | 13.4MB |
-| c0ffee         | 22.9 MB | 200MB  |
-| icewater       | 77.9 MB | 55.1MB |
-| signature-base | 78.9 MB | 27.8MB |
+In `boreal`, different compiler profiles can be used, with one prioritizing scanning speed,
+and the other one prioritzing memory usage. Those two profiles are presented separately to
+show the memory consumption impact.
+
+| rules set      | boreal (speed) | boreal (memory) | yara    |
+| -------------- | -------------- | --------------- | ------- |
+| orion          | 10.9 MB        | 9.71 MB         | 14.8 MB |
+| atr            | 10.7 MB        | 8.53 MB         | 15.7 MB |
+| reversinglabs  | 13.6 MB        | 12.2 MB         | 17.7 MB |
+| panopticon     | 9.07 MB        | 7.71 MB         | 15.3 MB |
+| c0ffee         | 16.7 MB        | 13.4 MB         | 198 MB  |
+| icewater       | 61.6 MB        | 55.6 MB         | 56.3 MB |
+| signature-base | 70.3 MB        | 44.2 MB         | 35.5 MB |
+| yara-rules     | 99.3 MB        | 74.0 MB         | 45.4 MB |
 
 Note that optimizing memory usage has not been a priority for the moment, as the focus was
 on optimizing performances. However, the next release will provide a way to proritize

benches/src/bench.rs

@@ -113,7 +113,9 @@ fn bench_scan_process(c: &mut Criterion) {
 }
 
 fn build_boreal_compiler() -> boreal::Compiler {
-    let mut boreal_compiler = boreal::Compiler::new();
+    let mut boreal_compiler = boreal::compiler::CompilerBuilder::new()
+        .profile(boreal::compiler::CompilerProfile::Speed)
+        .build();
     let _ = boreal_compiler.define_symbol("owner", "owner");
     let _ = boreal_compiler.define_symbol("filename", "filename");
     let _ = boreal_compiler.define_symbol("filepath", "filepath");

boreal-cli/src/main.rs

@@ -5,8 +5,8 @@ use std::process::ExitCode;
 use std::thread::JoinHandle;
 use std::time::Duration;
 
-use boreal::compiler::ExternalValue;
-use boreal::module::Value as ModuleValue;
+use boreal::compiler::{CompilerBuilder, CompilerProfile, ExternalValue};
+use boreal::module::{Console, Value as ModuleValue};
 use boreal::scanner::{FragmentedScanMode, ScanError, ScanParams, ScanResult};
 use boreal::{statistics, Compiler, Metadata, MetadataValue, Scanner};
 
@@ -58,6 +58,13 @@ fn build_command() -> Command {
                 .value_parser(value_parser!(usize))
                 .help("Number of threads to use when scanning directories"),
         )
+        .arg(
+            Arg::new("profile")
+                .long("profile")
+                .value_name("speed|memory")
+                .value_parser(parse_compiler_profile)
+                .help("Profile to use when compiling rules"),
+        )
         .arg(
             Arg::new("rules_file")
                 .value_parser(value_parser!(PathBuf))
@@ -252,17 +259,23 @@ fn main() -> ExitCode {
     let mut scanner = {
         let rules_file: PathBuf = args.remove_one("rules_file").unwrap();
 
-        let mut compiler = Compiler::new();
+        let mut builder = CompilerBuilder::new();
 
-        let no_console_logs = args.get_flag("no_console_logs");
         // Even if the console logs are disabled, add the module so that rules that use it
         // can still compile properly.
-        let _r = compiler.add_module(boreal::module::Console::with_callback(move |log| {
+        let no_console_logs = args.get_flag("no_console_logs");
+        builder = builder.add_module(Console::with_callback(move |log| {
             if !no_console_logs {
                 println!("{log}");
             }
         }));
 
+        if let Some(profile) = args.get_one::<CompilerProfile>("profile") {
+            builder = builder.profile(*profile);
+        }
+
+        let mut compiler = builder.build();
+
         compiler.set_params(
             boreal::compiler::CompilerParams::default()
                 .fail_on_warnings(args.get_flag("fail_on_warnings"))
@@ -482,6 +495,14 @@ fn parse_fragmented_scan_mode(scan_mode: &str) -> Result<FragmentedScanMode, Str
     }
 }
 
+fn parse_compiler_profile(profile: &str) -> Result<CompilerProfile, String> {
+    match profile {
+        "speed" => Ok(CompilerProfile::Speed),
+        "memory" => Ok(CompilerProfile::Memory),
+        _ => Err("invalid value".to_string()),
+    }
+}
+
 #[derive(Clone, Debug)]
 struct ScanOptions {
     print_module_data: bool,

boreal-cli/tests/cli.rs

@@ -813,6 +813,22 @@ fn test_invalid_fragmented_scan_mode() {
         .failure();
 }
 
+#[test]
+fn test_invalid_compiler_profile() {
+    cmd()
+        .arg("--profile")
+        .arg("bad_value")
+        .arg("rules.yar")
+        .arg("input")
+        .assert()
+        .stdout("")
+        .stderr(predicate::str::contains(
+            "invalid value 'bad_value' for \
+            '--profile <speed|memory>\': invalid value",
+        ))
+        .failure();
+}
+
 #[test]
 fn test_tags() {
     let rule_file = test_file(

boreal/src/compiler/builder.rs

@@ -0,0 +1,121 @@
+use std::{collections::HashMap, sync::Arc};
+
+use super::{AvailableModule, ModuleLocation};
+
+/// Configurable builder for the [`Compiler`] object.
+#[derive(Debug, Default)]
+pub struct CompilerBuilder {
+    /// Modules that can be imported when compiling rules.
+    modules: HashMap<&'static str, AvailableModule>,
+
+    /// Profile to use when compiling rules.
+    profile: super::CompilerProfile,
+}
+
+impl CompilerBuilder {
+    /// Create a new builder with sane default values.
+    ///
+    /// Modules enabled by default:
+    /// - `time`
+    /// - `math`
+    /// - `string`
+    /// - `hash` if the `hash` feature is enabled
+    /// - `elf`, `macho`, `pe`, `dotnet` and `dex` if the `object` feature is enabled
+    /// - `magic` if the `magic` feature is enabled
+    /// - `cuckoo` if the `cuckoo` feature is enabled
+    ///
+    /// Modules disabled by default:
+    /// - `console`
+    ///
+    /// To create a builder without any modules, use [`CompilerBuilder::default`] to
+    /// create a [`CompilerBuilder`] without any modules, then add back only the desired modules.
+    #[must_use]
+    pub fn new() -> Self {
+        let this = Self::default();
+
+        let this = this.add_module(crate::module::Time);
+        let this = this.add_module(crate::module::Math);
+        let this = this.add_module(crate::module::String_);
+
+        #[cfg(feature = "hash")]
+        let this = this.add_module(crate::module::Hash);
+
+        #[cfg(feature = "object")]
+        let this = this.add_module(crate::module::Pe);
+        #[cfg(feature = "object")]
+        let this = this.add_module(crate::module::Elf);
+        #[cfg(feature = "object")]
+        let this = this.add_module(crate::module::MachO);
+        #[cfg(feature = "object")]
+        let this = this.add_module(crate::module::Dotnet);
+        #[cfg(feature = "object")]
+        let this = this.add_module(crate::module::Dex);
+
+        #[cfg(feature = "magic")]
+        let this = this.add_module(crate::module::Magic);
+
+        #[cfg(feature = "cuckoo")]
+        let this = this.add_module(crate::module::Cuckoo);
+
+        this
+    }
+
+    /// Add a module that will be importable in rules.
+    ///
+    /// If the same module has already been added, it will be replaced by this one.
+    /// This can be useful to change the parameters of a module.
+    #[must_use]
+    pub fn add_module<M: crate::module::Module + 'static>(mut self, module: M) -> Self {
+        let compiled_module = Arc::new(super::module::compile_module(&module));
+
+        let _r = self.modules.insert(
+            compiled_module.name,
+            AvailableModule {
+                compiled_module,
+                location: ModuleLocation::Module(Box::new(module)),
+            },
+        );
+        self
+    }
+
+    /// Set the profile to use when compiling rules.
+    ///
+    /// By default, [`CompilerProfile::Speed`] is used.
+    #[must_use]
+    pub fn profile(mut self, profile: super::CompilerProfile) -> Self {
+        self.profile = profile;
+        self
+    }
+
+    /// Build a [`Compiler`] object with the configuration set on this builder.
+    #[must_use]
+    pub fn build(self) -> super::Compiler {
+        super::Compiler::build(self.modules, self.profile)
+    }
+
+    /// Get the profile to use when compiling rules.
+    #[must_use]
+    pub fn get_profile(&self) -> super::CompilerProfile {
+        self.profile
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::compiler::CompilerProfile;
+    use crate::test_helpers::test_type_traits_non_clonable;
+
+    #[test]
+    fn test_types_traits() {
+        test_type_traits_non_clonable(CompilerBuilder::default());
+    }
+
+    #[test]
+    fn test_getters() {
+        let builder = CompilerBuilder::default();
+
+        let builder = builder.profile(CompilerProfile::Memory);
+        assert_eq!(builder.get_profile(), CompilerProfile::Memory);
+    }
+}