Skip to content

Commit

Permalink
feat: enable data compression for import functionality and update dum…
Browse files Browse the repository at this point in the history
…p file handling
  • Loading branch information
Kremilly committed Nov 22, 2024
1 parent f2c222f commit 85a69d4
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 8 deletions.
2 changes: 1 addition & 1 deletion dumpsync.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
exports:
dump_data: true
compress_data: false
compress_data: true
insert_ignore_into: false
drop_table_if_exists: true
database_if_not_exists: true
Expand Down
31 changes: 24 additions & 7 deletions src/engine/import.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use regex::Regex;
use flate2::read::GzDecoder;

use mysql::{
*,
Expand All @@ -7,8 +8,12 @@ use mysql::{

use std::{
fs::File,
io::Read,
error::Error,

io::{
Read,
BufReader,
},
};

use crate::{
Expand Down Expand Up @@ -52,16 +57,28 @@ impl Import {
}.create_pool()?;

let mut conn = pool.get_conn()?;
let is_compressed = self.dump_file_path.ends_with(".sql.gz");

let dump_content = if is_compressed {
let file = File::open(&self.dump_file_path)?;

let mut decoder = GzDecoder::new(BufReader::new(file));
let mut content = String::new();

decoder.read_to_string(&mut content)?;
content
} else {
let mut file = File::open(&self.dump_file_path)?;
let mut content = String::new();

let mut dump_file = File::open(&self.dump_file_path)?;
let mut dump_content = String::new();
dump_file.read_to_string(&mut dump_content)?;
file.read_to_string(&mut content)?;
content
};

let create_table_regex = Regex::new(r"(?i)CREATE TABLE\s+`?(\w+)`?").unwrap();

for statement in dump_content.split(';') {
let trimmed = statement.trim();

if !trimmed.is_empty() {
match conn.query_drop(trimmed) {
Ok(_) => {
Expand All @@ -70,8 +87,7 @@ impl Import {
SuccessAlerts::table(table_name.as_str());
}
}
},

}
Err(e) => ErrorsAlerts::import(&self.dbname, trimmed, &e.to_string()),
}
}
Expand All @@ -80,4 +96,5 @@ impl Import {
SuccessAlerts::import(&self.dbname);
Ok(())
}

}

0 comments on commit 85a69d4

Please sign in to comment.