Transition to the Component-Based Model

.github/workflows/pullrequest_check.yml

@@ -12,7 +12,8 @@ on:
 
 jobs:
   ts_and_rust_lint:
-    if: github.event.pull_request.draft == false
+    # TODO AAZ: Make sure this check works upon merging into master.
+    if: github.event.pull_request.draft == false && github.event.pull_request.base.ref == 'master'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -63,7 +64,8 @@ jobs:
         run: yarn run check
 
   integration_and_unit_tests:
-    if: github.event.pull_request.draft == false
+    # TODO AAZ: Make sure this check works upon merging into master.
+    if: github.event.pull_request.draft == false && github.event.pull_request.base.ref == 'master'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

README.md

@@ -2,7 +2,7 @@
 [![](https://github.com/esrlabs/chipmunk/actions/workflows/release_next.yml/badge.svg)](https://github.com/esrlabs/chipmunk/actions/workflows/release_next.yml)
 [![](https://github.com/esrlabs/chipmunk/actions/workflows/lint_master.yml/badge.svg)](https://github.com/esrlabs/chipmunk/actions/workflows/lint_master.yml)
 
-`chipmunk` is one of the fastest desktop applications for viewing log files, with no limitations on file size. 1 GB, 2 GB, 10 GB? `chipmunk` is limited only by your disk space — nothing more. With no caching and no unnecessary copying, files of any size open with the same speed. But `chipmunk` goes beyond just working with files: it also allows you to create network connections to collect logs via TCP, UDP, Serial, or from the output of a running command.
+`chipmunk` is one of the fastest desktop applications for viewing log files, with no limitations on file size. 1 GB, 2 GB, 10 GB? `chipmunk` is limited only by your disk space - nothing more. With no caching and no unnecessary copying, files of any size open with the same speed. But `chipmunk` goes beyond just working with files: it also allows you to create network connections to collect logs via TCP, UDP, Serial, or from the output of a running command.
 
 ## Automotive and Network Traces
 
@@ -19,7 +19,7 @@ Additionally, `chipmunk` allows you to work with DLT traces both as standalone f
 - Serial Port  
 - Output from a command or program
 
-For each source, you can assign a parser — for example, to collect DLT packets over a UDP connection for analysis or to save them as a standalone trace file.
+For each source, you can assign a parser - for example, to collect DLT packets over a UDP connection for analysis or to save them as a standalone trace file.
 
 Another key feature is the ability to launch any command or program and collect its output, which can be analyzed in real time as it's generated.
 
@@ -29,7 +29,7 @@ At its core, `chipmunk` is a log analysis tool. It goes beyond simple search que
 
 ![filters_create](./docs/assets/search/filters_create.gif)
 
-The search engine works dynamically — results are updated in real time as new data is added. If you're connected to a live data source, your active filters will continuously update the search results as new logs arrive.
+The search engine works dynamically - results are updated in real time as new data is added. If you're connected to a live data source, your active filters will continuously update the search results as new logs arrive.
 
 ## Metrics, Measurements, and Graphs
 
@@ -47,6 +47,6 @@ See more details in the [documentation](https://esrlabs.github.io/chipmunk/) abo
 
 ## Contributing
 
-We welcome contributions of all kinds — bug reports, performance improvements, documentation fixes, or new features.
+We welcome contributions of all kinds - bug reports, performance improvements, documentation fixes, or new features.
 
 [Click here to view it](https://esrlabs.github.io/chipmunk/contributing/welcome/)

application/apps/indexer/Cargo.toml

@@ -7,6 +7,8 @@ members = [
     "addons/file-tools",
     "addons/text_grep",
     "addons/bufread",
+    "descriptor",
+    "register",
     "indexer_base",
     "merging",
     "parsers",
@@ -42,6 +44,9 @@ envvars = "0.1"
 anyhow = "1.0"
 toml = "0.8"
 blake3 = "1.8"
+serialport = "4.6"
+etherparse = "0.18"
+indexmap = "2.10"
 
 ## Development Dependencies ##
 # Support for `html_reports` needs running the benchmarks via `cargo-criterion` tool.

application/apps/indexer/addons/dlt-tools/Cargo.toml

@@ -5,11 +5,15 @@ edition = "2024"
 
 [dependencies]
 dlt-core.workspace = true
-indexer_base = { path = "../../indexer_base" }
 log.workspace = true
-parsers = { path = "../../parsers" }
-sources = { path = "../../sources" }
 tokio = { workspace = true , features = ["full"] }
 tokio-util = { workspace = true, features = ["codec", "net"] }
+indexer_base = { path = "../../indexer_base" }
+parsers = { path = "../../parsers" }
+sources = { path = "../../sources" }
+processor = { path = "../../processor" }
+stypes = { path = "../../stypes" }
+session = { path = "../../session" }
+uuid.workspace = true
 
 [dev-dependencies]

application/apps/indexer/addons/dlt-tools/src/lib.rs

@@ -1,4 +1,4 @@
-#![deny(unused_crate_dependencies)]
+// #![deny(unused_crate_dependencies)]
 // Copyright (c) 2019 E.S.R.Labs. All rights reserved.
 //
 // NOTICE:  All information contained herein is, and remains
@@ -10,81 +10,104 @@
 // Dissemination of this information or reproduction of this material
 // is strictly forbidden unless prior written permission is obtained
 // from E.S.R.Labs.
-extern crate indexer_base;
-
 #[macro_use]
 extern crate log;
 
-use dlt_core::filtering::DltFilterConfig;
-use parsers::{Attachment, MessageStreamItem, ParseYield, dlt::DltParser};
-use sources::{binary::raw::BinaryByteSource, producer::MessageProducer};
+use parsers::api::*;
+use processor::producer::{MessageProducer, MessageStreamItem};
+use session::session::Session;
 use std::{
+    collections::HashMap,
     fs::File,
-    io::{BufReader, BufWriter, Write},
+    io::{BufWriter, Write},
     path::{Path, PathBuf},
 };
+use stypes::{SessionAction, SessionSetup};
 use tokio_util::sync::CancellationToken;
+use uuid::Uuid;
+
+#[derive(Default)]
+pub struct AttachmentsCollector {
+    pub attachments: Vec<Attachment>,
+    messages: usize,
+}
+
+impl LogRecordsBuffer for AttachmentsCollector {
+    fn append(&mut self, record: LogRecordOutput<'_>) {
+        match record {
+            LogRecordOutput::Raw(..)
+            | LogRecordOutput::Message(..)
+            | LogRecordOutput::Columns(..) => {
+                self.messages += 1;
+            }
+            LogRecordOutput::Multiple(inner) => {
+                for rec in inner {
+                    self.append(rec);
+                }
+            }
+            LogRecordOutput::Attachment(inner) => {
+                self.attachments.push(inner);
+            }
+        }
+    }
+
+    async fn flush(&mut self) -> Result<(), stypes::NativeError> {
+        Ok(())
+    }
+
+    fn get_source_id(&self) -> u16 {
+        0
+    }
+}
 
 pub async fn scan_dlt_ft(
-    input: PathBuf,
-    filter: Option<DltFilterConfig>,
-    with_storage_header: bool,
+    filename: PathBuf,
+    filters: Option<HashMap<String, Vec<String>>>,
     cancel: CancellationToken,
 ) -> Result<Vec<Attachment>, String> {
-    match File::open(input) {
-        Ok(input) => {
-            let reader = BufReader::new(&input);
-            let source = BinaryByteSource::new(reader);
-            let parser = DltParser::new(
-                filter.map(|f| f.into()),
-                None,
-                None,
-                None,
-                with_storage_header,
-            );
-
-            let mut producer = MessageProducer::new(parser, source);
-
-            let mut canceled = false;
-
-            let mut attachments = vec![];
-            loop {
-                tokio::select! {
-                    // Check on events in current order ensuring cancel will be checked at first
-                    // as it's defined in the current unit tests.
-                    biased;
-                    _ = cancel.cancelled() => {
-                        debug!("scan canceled");
-                        canceled = true;
+    let setup = SessionSetup {
+        origin: SessionAction::File(filename),
+        parser: parsers::dlt::descriptor::get_default_options(None, filters),
+        source: sources::binary::raw::get_default_options(),
+    };
+    let (session, receiver) = Session::new(Uuid::new_v4())
+        .await
+        .map_err(|err| err.to_string())?;
+    let (_, source, parser) = session
+        .register
+        .setup(&setup)
+        .map_err(|err| err.to_string())?;
+    let mut collector = AttachmentsCollector::default();
+    let mut producer = MessageProducer::new(parser, source, &mut collector);
+    let mut canceled = false;
+    loop {
+        tokio::select! {
+            // Check on events in current order ensuring cancel will be checked at first
+            // as it's defined in the current unit tests.
+            biased;
+            _ = cancel.cancelled() => {
+                debug!("scan canceled");
+                canceled = true;
+                break;
+            }
+            item = producer.read_next_segment() => {
+                match item {
+                    Some((_, MessageStreamItem::Done)) | None => {
                         break;
-                    }
-                    items = producer.read_next_segment() => {
-                        match items {
-                            Some(items) => {
-                                for (_, item) in items {
-                                    if let MessageStreamItem::Item(ParseYield::MessageAndAttachment((_msg, attachment))) = item {
-                                        attachments.push(attachment.to_owned());
-                                    } else if let MessageStreamItem::Item(ParseYield::Attachment(attachment)) = item {
-                                        attachments.push(attachment.to_owned());
-                                    }
-                                }
-                            }
-                            _ => {
-                                break;
-                            }
-                        }
-                    }
+                    },
+                    Some(..) => {
+                        continue;
+                    },
                 }
             }
-
-            if canceled {
-                return Ok(Vec::new());
-            }
-
-            Ok(attachments)
         }
-        Err(error) => Err(format!("failed to open file: {error}")),
     }
+    if canceled {
+        return Ok(Vec::new());
+    }
+    // Keep receiver alive until here
+    drop(receiver);
+    Ok(collector.attachments)
 }
 
 pub fn extract_dlt_ft(
@@ -126,14 +149,14 @@ mod tests {
     use super::*;
     use std::path::Path;
 
-    const DLT_FT_SAMPLE: &str = "../../../../../application/developing/resources/attachments.dlt";
+    const DLT_FT_SAMPLE: &str = "../../../../developing/resources/attachments.dlt";
 
     #[tokio::test]
     async fn test_scan_dlt_ft() {
         let input: PathBuf = Path::new(DLT_FT_SAMPLE).into();
 
         let cancel = CancellationToken::new();
-        match scan_dlt_ft(input, None, true, cancel).await {
+        match scan_dlt_ft(input, None, cancel).await {
             Ok(files) => {
                 assert_eq!(files.len(), 3);
                 assert_eq!("test1.txt", files.first().unwrap().name);
@@ -153,7 +176,7 @@ mod tests {
         let cancel = CancellationToken::new();
         cancel.cancel();
 
-        match scan_dlt_ft(input, None, true, cancel).await {
+        match scan_dlt_ft(input, None, cancel).await {
             Ok(files) => {
                 assert_eq!(files.len(), 0);
             }
@@ -166,18 +189,10 @@ mod tests {
     #[tokio::test]
     async fn test_scan_dlt_ft_with_filter() {
         let input: PathBuf = Path::new(DLT_FT_SAMPLE).into();
-
-        let filter = DltFilterConfig {
-            min_log_level: None,
-            app_ids: None,
-            ecu_ids: Some(vec!["ecu2".to_string()]),
-            context_ids: None,
-            app_id_count: 0,
-            context_id_count: 0,
-        };
-
+        let mut filters = HashMap::new();
+        filters.insert("ecu_ids".to_string(), vec!["ecu2".to_string()]);
         let cancel = CancellationToken::new();
-        match scan_dlt_ft(input, Some(filter), true, cancel).await {
+        match scan_dlt_ft(input, Some(filters), cancel).await {
             Ok(files) => {
                 assert_eq!(files.len(), 1);
                 assert_eq!("test2.txt", files.first().unwrap().name);
@@ -194,7 +209,7 @@ mod tests {
         let output = TempDir::new();
 
         let cancel = CancellationToken::new();
-        match scan_dlt_ft(input.clone(), None, true, cancel.clone()).await {
+        match scan_dlt_ft(input.clone(), None, cancel.clone()).await {
             Ok(files) => {
                 match extract_dlt_ft(
                     &output.dir,
@@ -225,7 +240,7 @@ mod tests {
         let output = TempDir::new();
 
         let cancel = CancellationToken::new();
-        match scan_dlt_ft(input.clone(), None, true, cancel).await {
+        match scan_dlt_ft(input.clone(), None, cancel).await {
             Ok(files) => {
                 let cancel = CancellationToken::new();
                 cancel.cancel();
@@ -248,18 +263,10 @@ mod tests {
     async fn test_extract_dlt_ft_with_filter() {
         let input: PathBuf = Path::new(DLT_FT_SAMPLE).into();
         let output = TempDir::new();
-
-        let filter = DltFilterConfig {
-            min_log_level: None,
-            app_ids: None,
-            ecu_ids: Some(vec!["ecu2".to_string()]),
-            context_ids: None,
-            app_id_count: 0,
-            context_id_count: 0,
-        };
-
+        let mut filters = HashMap::new();
+        filters.insert("ecu_ids".to_string(), vec!["ecu2".to_string()]);
         let cancel = CancellationToken::new();
-        match scan_dlt_ft(input.clone(), Some(filter), true, cancel).await {
+        match scan_dlt_ft(input.clone(), Some(filters), cancel).await {
             Ok(files) => {
                 let cancel = CancellationToken::new();
                 match extract_dlt_ft(&output.dir, FileExtractor::files_with_names(files), cancel) {
@@ -285,7 +292,7 @@ mod tests {
         let output = TempDir::new();
 
         let cancel = CancellationToken::new();
-        match scan_dlt_ft(input.clone(), None, true, cancel).await {
+        match scan_dlt_ft(input.clone(), None, cancel).await {
             Ok(files) => {
                 let cancel = CancellationToken::new();
                 match extract_dlt_ft(
@@ -313,18 +320,10 @@ mod tests {
     async fn test_extract_dlt_ft_with_filtered_index() {
         let input: PathBuf = Path::new(DLT_FT_SAMPLE).into();
         let output = TempDir::new();
-
-        let filter = DltFilterConfig {
-            min_log_level: None,
-            app_ids: None,
-            ecu_ids: Some(vec!["ecu2".to_string()]),
-            context_ids: None,
-            app_id_count: 0,
-            context_id_count: 0,
-        };
-
+        let mut filters = HashMap::new();
+        filters.insert("ecu_ids".to_string(), vec!["ecu2".to_string()]);
         let cancel = CancellationToken::new();
-        match scan_dlt_ft(input.clone(), Some(filter), true, cancel).await {
+        match scan_dlt_ft(input.clone(), Some(filters), cancel).await {
             Ok(files) => {
                 let cancel = CancellationToken::new();
                 match extract_dlt_ft(&output.dir, FileExtractor::files_with_names(files), cancel) {

application/apps/indexer/addons/file-tools/src/lib.rs

@@ -13,6 +13,11 @@ pub fn is_binary(file_path: String) -> Result<bool> {
     Ok(from_utf8(&buffer).map_or(true, |_file_content| false))
 }
 
+pub fn is_path_binary<P: AsRef<Path>>(file_path: P) -> Result<bool> {
+    let buffer = fetch_starting_chunk(file_path.as_ref())?;
+    Ok(from_utf8(&buffer).map_or(true, |_file_content| false))
+}
+
 fn fetch_starting_chunk(file_path: &Path) -> Result<Vec<u8>> {
     let bytes_to_read: u64 = (metadata(file_path)?.len().max(1) - 1).min(BYTES_TO_READ);