diff --git a/Cargo.lock b/Cargo.lock index daf2af8..d8e4b1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -655,7 +655,7 @@ dependencies = [ [[package]] name = "ngxav" -version = "0.4.7" +version = "0.4.8" dependencies = [ "chrono", "clap", diff --git a/Cargo.toml b/Cargo.toml index dd77977..db2a641 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ workspace = { members = [ "tests","tests/obfuscate_access_log_ips"] } [package] name = "ngxav" -version = "0.4.7" +version = "0.4.8" edition = "2021" license = "MIT" description = "Search through NGINX logs with advanced filters and support for displaying analytics about your selected log entries" diff --git a/src/main.rs b/src/main.rs index ff6b283..723d611 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,7 +37,9 @@ fn main() { .build_global() .unwrap(); } - if !args.conserve_memory.is_none() && args.conserve_memory.unwrap() == true { + let file_md = metadata(args.file.clone()).unwrap(); + if !args.conserve_memory.is_none() && args.conserve_memory.unwrap() == true && file_md.is_file() + { if let Ok(lines) = read_line_by_line(args.file) { let mut occurrences: HashMap = HashMap::new(); for line in lines.flatten() { @@ -57,9 +59,13 @@ fn main() { return; } let mut lines = Vec::new(); - let file_md = metadata(args.file.clone()).unwrap(); if file_md.is_dir() { - lines = utils::read_folder::read_folder(args.file); + if args.conserve_memory.is_some() && args.conserve_memory.unwrap() == true { + utils::read_folder_conserve_memory::read_folder_conserve_memory(args.file, args.unique); + return; + } else { + lines = utils::read_folder::read_folder(args.file); + } } else { lines = lines_from_file(args.file).expect("should read"); } diff --git a/src/utils/mod.rs b/src/utils/mod.rs index b133a05..fb367d3 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -5,6 +5,7 @@ pub mod parse_line; pub mod parse_nginx_time_format; pub mod parse_user_agent; pub mod read_folder; +pub mod read_folder_conserve_memory; pub mod session_analytics; pub mod session_unique; pub mod sessionize; diff --git a/src/utils/read_folder_conserve_memory.rs b/src/utils/read_folder_conserve_memory.rs new file mode 100644 index 0000000..23b6bf3 --- /dev/null +++ b/src/utils/read_folder_conserve_memory.rs @@ -0,0 +1,65 @@ +use crate::utils::keep_line::keep_line; +use crate::utils::parse_line::parse_line; +use flate2::read::GzDecoder; +use std::collections::HashMap; +use std::fs; +use std::fs::metadata; +use std::fs::File; +use std::io::{BufRead, BufReader}; + +pub fn read_folder_conserve_memory(file_path: String, isUnique: Option) { + println!("{}", "step zero"); + let mut paths: Vec<_> = fs::read_dir(file_path).unwrap().collect(); + paths.sort_by_key(|x| { + metadata(x.as_ref().unwrap().path().to_str().unwrap()) + .unwrap() + .modified() + .unwrap() + }); + println!("{}", "step one"); + let mut occurrences: HashMap = HashMap::new(); + for path in paths { + let p: String = path.unwrap().path().to_str().unwrap().to_string(); + if p.contains(".gz") { + let file = File::open(p).expect("Ooops."); + let reader = BufReader::new(GzDecoder::new(file)); + for r in reader.lines() { + match r { + Ok(line) => { + let ip: String = + line.clone().split(" ").collect::>()[0].to_string(); + if keep_line(&parse_line(&line), true) { + if isUnique.is_some() && isUnique.unwrap() == true { + if !occurrences.contains_key(&ip) { + println!("{}", line.clone() + "\n"); + occurrences.insert(ip, true); + } + } else { + println!("{}", line.clone() + "\n"); + } + } + } + Err(error) => { + eprintln!("Error reading line: {}", error); + } + } + } + } else { + let file = File::open(p).expect("Ooops."); + let reader = BufReader::new(file).lines(); + for line in reader.flatten() { + let ip: String = line.clone().split(" ").collect::>()[0].to_string(); + if keep_line(&parse_line(&line), true) { + if isUnique.is_some() && isUnique.unwrap() == true { + if !occurrences.contains_key(&ip) { + println!("{}", line.clone() + "\n"); + occurrences.insert(ip, true); + } + } else { + println!("{}", line.clone() + "\n"); + } + } + } + } + } +}