feat: add -X, --remove-duplicate-detections option to eid-metrics and logon-summary

CHANGELOG-Japanese.md

@@ -2,6 +2,10 @@
 
 ## 3.1.0 [2025/02/22] - Ninja Day Release
 
+**新機能:**
+
+- `eid-metrics`と`logon-summary`コマンドに`-X, --remove-duplicate-detections`オプションを追加した。 (#1552) (@fukusuket)
+
 **改善:**
 
 - `search`コマンドに`--timeline-start/--timeline-end`オプションを追加した。 (#1543) (@fukuseket)

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## 3.1.0 [2025/02/22] - Ninja Day Release
 
+**New Features:**
+
+- `-X, --remove-duplicate-detections` option to `eid-metrics` and `logon-summary` commands. (#1552) (@fukusuket)
+
 **Enhancements:**
 
 - Added `--timeline-start/--timeline-end` options to the `search` command. (#1543) (@fukuseket)

src/detections/configs.rs

@@ -138,6 +138,7 @@ pub struct StoredStatic {
     pub is_low_memory: bool,
     pub enable_all_rules: bool,
     pub scan_all_evtx_files: bool,
+    pub metrics_remove_duplication: bool,
 }
 impl StoredStatic {
     /// main.rsでパースした情報からデータを格納する関数
@@ -684,6 +685,11 @@ impl StoredStatic {
             Some(Action::JsonTimeline(opt)) => opt.output_options.scan_all_evtx_files,
             _ => false,
         };
+        let metrics_remove_duplication = match &input_config.as_ref().unwrap().action {
+            Some(Action::EidMetrics(opt)) => opt.remove_duplicate_detections,
+            Some(Action::LogonSummary(opt)) => opt.remove_duplicate_detections,
+            _ => false,
+        };
         let mut ret = StoredStatic {
             config: input_config.as_ref().unwrap().to_owned(),
             config_path: config_path.to_path_buf(),
@@ -806,6 +812,7 @@ impl StoredStatic {
             is_low_memory,
             enable_all_rules,
             scan_all_evtx_files,
+            metrics_remove_duplication,
         };
         ret.profiles = load_profile(
             check_setting_path(
@@ -1362,6 +1369,10 @@ pub struct EidMetricsOption {
     #[arg(help_heading = Some("Output"), short = 'o', long, value_name = "FILE", display_order = 410)]
     pub output: Option<PathBuf>,
 
+    /// Remove duplicate detections (default: disabled)
+    #[arg(help_heading = Some("Output"), short = 'X', long = "remove-duplicate-detections", requires = "sort_events", display_order = 409)]
+    pub remove_duplicate_detections: bool,
+
     #[clap(flatten)]
     pub common_options: CommonOptions,
 
@@ -1485,6 +1496,10 @@ pub struct LogonSummaryOption {
     #[arg(help_heading = Some("Output"), short = 'o', long, value_name = "FILENAME-PREFIX", display_order = 410)]
     pub output: Option<PathBuf>,
 
+    /// Remove duplicate detections (default: disabled)
+    #[arg(help_heading = Some("Output"), short = 'X', long = "remove-duplicate-detections", requires = "sort_events", display_order = 409)]
+    pub remove_duplicate_detections: bool,
+
     #[clap(flatten)]
     pub common_options: CommonOptions,
 
@@ -3276,6 +3291,7 @@ mod tests {
                     include_computer: None,
                     exclude_computer: None,
                 },
+                remove_duplicate_detections: false,
             })),
             debug: false,
         }));
@@ -3324,6 +3340,7 @@ mod tests {
                 },
                 end_timeline: None,
                 start_timeline: None,
+                remove_duplicate_detections: false,
             })),
             debug: false,
         }));

src/main.rs

@@ -3406,6 +3406,7 @@ mod tests {
                 utc: false,
             },
             clobber: false,
+            remove_duplicate_detections: false,
         });
         let config = Some(Config {
             action: Some(action),
@@ -3463,6 +3464,7 @@ mod tests {
                 utc: false,
             },
             clobber: true,
+            remove_duplicate_detections: false,
         });
         let config = Some(Config {
             action: Some(action),
@@ -3520,6 +3522,7 @@ mod tests {
             clobber: false,
             end_timeline: None,
             start_timeline: None,
+            remove_duplicate_detections: false,
         });
         let config = Some(Config {
             action: Some(action),
@@ -3578,6 +3581,7 @@ mod tests {
             clobber: true,
             end_timeline: None,
             start_timeline: None,
+            remove_duplicate_detections: false,
         });
         let config = Some(Config {
             action: Some(action),

src/timeline/metrics.rs

@@ -36,6 +36,7 @@ pub struct EventMetrics {
     pub stats_list: HashMap<(CompactString, CompactString), usize>,
     pub stats_login_list: HashMap<LoginEvent, [usize; 2]>,
     pub stats_logfile: Vec<LogMetrics>,
+    pub counted_rec: HashSet<(String, String)>,
 }
 /**
 * Windows Event Logの統計情報を出力する
@@ -57,6 +58,7 @@ impl EventMetrics {
             stats_list,
             stats_login_list,
             stats_logfile: Vec::new(),
+            counted_rec: HashSet::new(),
         }
     }
 
@@ -238,6 +240,17 @@ impl EventMetrics {
             if let Some(idnum) =
                 utils::get_event_value("EventID", &record.record, &stored_static.eventkey_alias)
             {
+                if stored_static.metrics_remove_duplication {
+                    let event = &record.record["Event"]["System"];
+                    let rec_id = event["EventRecordID"].to_string();
+                    let evt_time = event["TimeCreated_attributes"]["SystemTime"].to_string();
+                    let counted = (rec_id, evt_time);
+                    if self.counted_rec.contains(&counted) {
+                        continue;
+                    }
+                    self.counted_rec.insert(counted);
+                }
+
                 let count: &mut usize = self
                     .stats_list
                     .entry((
@@ -409,6 +422,16 @@ impl EventMetrics {
                             &stored_static.eventkey_alias,
                         ),
                     };
+                    if stored_static.metrics_remove_duplication {
+                        let event = &record.record["Event"]["System"];
+                        let rec_id = event["EventRecordID"].to_string();
+                        let evt_time = event["TimeCreated_attributes"]["SystemTime"].to_string();
+                        let counted = (rec_id, evt_time);
+                        if self.counted_rec.contains(&counted) {
+                            continue;
+                        }
+                        self.counted_rec.insert(counted);
+                    }
 
                     let countlist: [usize; 2] = [0, 0];
                     // この段階でEventIDは4624もしくは4625となるのでこの段階で対応するカウンターを取得する
@@ -542,6 +565,7 @@ mod tests {
                 },
                 output: None,
                 clobber: false,
+                remove_duplicate_detections: false,
             }));
 
         let mut timeline = Timeline::new();

src/timeline/timelines.rs

@@ -830,6 +830,7 @@ mod tests {
                 clobber: false,
                 end_timeline: None,
                 start_timeline: None,
+                remove_duplicate_detections: false,
             }));
         dummy_stored_static.logon_summary_flag = true;
         *STORED_EKEY_ALIAS.write().unwrap() = Some(dummy_stored_static.eventkey_alias.clone());
@@ -1016,6 +1017,7 @@ mod tests {
                 },
                 output: Some(Path::new("./test_tm_stats.csv").to_path_buf()),
                 clobber: false,
+                remove_duplicate_detections: false,
             }));
         *STORED_EKEY_ALIAS.write().unwrap() = Some(dummy_stored_static.eventkey_alias.clone());
         let mut timeline = Timeline::default();
@@ -1111,6 +1113,7 @@ mod tests {
                 clobber: false,
                 end_timeline: None,
                 start_timeline: None,
+                remove_duplicate_detections: false,
             }));
         dummy_stored_static.logon_summary_flag = true;
         *STORED_EKEY_ALIAS.write().unwrap() = Some(dummy_stored_static.eventkey_alias.clone());