From 034624278efef135f9667ed8033e9ab5340084b4 Mon Sep 17 00:00:00 2001 From: Josh Comer Date: Mon, 20 Apr 2020 14:30:10 -0300 Subject: [PATCH 1/4] Add embedded db caching --- Cargo.lock | 81 ++++++++++++++++++++++++++++++++++++++++---- Cargo.toml | 2 ++ src/app/config.rs | 5 +++ src/app/db.rs | 86 +++++++++++++++++++++++++++++++++++++++++++++++ src/app/mod.rs | 1 + src/app/server.rs | 55 ++++++++++++++++++++---------- src/main.rs | 2 ++ 7 files changed, 207 insertions(+), 25 deletions(-) create mode 100644 src/app/db.rs diff --git a/Cargo.lock b/Cargo.lock index fd62558..463c6a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -392,6 +392,16 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" +[[package]] +name = "bincode" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" +dependencies = [ + "byteorder", + "serde", +] + [[package]] name = "bitflags" version = "1.2.1" @@ -568,6 +578,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "lazy_static 1.4.0", + "maybe-uninit", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.7.2" @@ -739,6 +764,16 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi 0.3.8", +] + [[package]] name = "fs_extra" version = "1.1.0" @@ -955,6 +990,7 @@ dependencies = [ "actix-service", "actix-web", "assert_cmd", + "bincode", "dir-diff", "dogstatsd", "failure", @@ -973,6 +1009,7 @@ dependencies = [ "serde_derive", "serde_json", "shellexpand", + "sled", "stderrlog", "structopt", "tempfile", @@ -1216,12 +1253,27 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + [[package]] name = "memchr" version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +[[package]] +name = "memoffset" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4fc2c02a7e374099d4ee95a193111f72d2110197fe200272371758f6c3643d8" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -1649,9 +1701,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.6" +version = "1.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6946991529684867e47d86474e3a6d0c0ab9b82d5821e314b1ede31fa3a4b3" +checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" dependencies = [ "aho-corasick", "memchr", @@ -1809,6 +1861,22 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +[[package]] +name = "sled" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb6824dde66ad33bf20c6e8476f5b82b871bc8bc3c129a10ea2f7dae5060fa3" +dependencies = [ + "crc32fast", + "crossbeam-epoch", + "crossbeam-utils", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot", +] + [[package]] name = "smallvec" version = "1.3.0" @@ -1978,12 +2046,11 @@ dependencies = [ [[package]] name = "time" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" dependencies = [ "libc", - "redox_syscall", "winapi 0.3.8", ] @@ -2244,9 +2311,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa515c5163a99cc82bab70fd3bfdd36d827be85de63737b40fcef2ce084a436e" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ "winapi 0.3.8", ] diff --git a/Cargo.toml b/Cargo.toml index f58379b..d3c708e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,8 @@ actix-rt = '1' actix-service = '1' futures = '0.3' parking_lot = '0.10' +sled = '0.31' +bincode = '1.2.1' [dependencies.git2] version = '0.13' diff --git a/src/app/config.rs b/src/app/config.rs index 2b75697..a2d7a2a 100644 --- a/src/app/config.rs +++ b/src/app/config.rs @@ -104,6 +104,11 @@ pub enum AppCommand { value_name = "PATTERN" )] environment_pattern: String, + + ///Filepath to the embedded db for storing environments. Will be created if it doesn't exist. If not provided a + /// random temp directory will be created + #[structopt(long = "db", value_name = "PATH")] + db_location: Option, }, } diff --git a/src/app/db.rs b/src/app/db.rs new file mode 100644 index 0000000..e017813 --- /dev/null +++ b/src/app/db.rs @@ -0,0 +1,86 @@ +use bincode; +use failure::Error; +use hogan::config::Environment; +use serde::Deserialize; +use serde::Serialize; +use sled; +pub use sled::Db; +use tempfile; + +pub fn open_db(db_name: Option) -> Result { + let path = match db_name { + Some(p) => p, + None => tempfile::tempdir()? + .into_path() + .to_str() + .unwrap() + .to_owned(), + }; + info!("Opening DB: {}", path); + sled::open(path).map_err(|e| e.into()) +} + +fn gen_key(sha: &str, env: &str) -> String { + format!("{}::{}", sha, env) +} + +#[derive(Default, Serialize, Deserialize, Debug)] +struct WritableEnvironment { + config_data: String, + environment: String, + environment_type: Option, +} + +impl From<&Environment> for WritableEnvironment { + fn from(environment: &Environment) -> Self { + WritableEnvironment { + config_data: environment.config_data.to_string(), + environment: environment.environment.to_owned(), + environment_type: environment.environment_type.to_owned(), + } + } +} + +impl From for Environment { + fn from(environment: WritableEnvironment) -> Self { + Environment { + config_data: serde_json::from_str(&environment.config_data).unwrap(), + environment: environment.environment.to_owned(), + environment_type: environment.environment_type.to_owned(), + } + } +} + +pub fn write_env(db: &sled::Db, env: &str, sha: &str, data: &Environment) -> Result<(), Error> { + write_key(db, &gen_key(sha, env), data) +} + +fn write_key(db: &sled::Db, key: &str, data: &Environment) -> Result<(), Error> { + let env: WritableEnvironment = data.into(); + let data: Vec = bincode::serialize(&env).unwrap(); + info!("Writing {} to db size: {}", key, data.len()); + db.insert(key, data)?; + Ok(()) +} + +pub fn read_env(db: &sled::Db, env: &str, sha: &str) -> Result, Error> { + let key = gen_key(sha, env); + match db.get(&key)? { + Some(data) => { + let decoded: WritableEnvironment = match bincode::deserialize(&data) { + Ok(environment) => environment, + Err(e) => { + warn!("Unable to deserialize env: {} {:?}", key, e); + return Err(e.into()); + } + }; + Ok(Some(decoded.into())) + } + None => Ok(None), + } +} + +pub fn remove_env(db: &sled::Db, env: &str, sha: &str) -> Result<(), Error> { + db.remove(gen_key(sha, env))?; + Ok(()) +} diff --git a/src/app/mod.rs b/src/app/mod.rs index 68f3021..8fdba25 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -1,4 +1,5 @@ pub mod cli; pub mod config; mod datadogstatsd; +mod db; pub mod server; diff --git a/src/app/server.rs b/src/app/server.rs index 3c63262..3f70027 100644 --- a/src/app/server.rs +++ b/src/app/server.rs @@ -1,7 +1,8 @@ use crate::app::config::AppCommon; use crate::app::datadogstatsd::{CustomMetrics, DdMetrics}; +use crate::app::db; use actix_service::Service; -use actix_web::{get, post, web, HttpResponse, HttpServer}; +use actix_web::{get, middleware, post, web, HttpResponse, HttpServer}; use failure::Error; use futures::future::FutureExt; use hogan; @@ -22,6 +23,7 @@ pub fn start_up_server( environments_regex: Regex, datadog: bool, environment_pattern: String, + db_location: Option, ) -> Result<(), Error> { let config_dir = ConfigDir::new(common.configs_url, &common.ssh_key)?; @@ -41,6 +43,9 @@ pub fn start_up_server( } else { None }; + + let db = Arc::new(db::open_db(db_location)?); + let state = ServerState { environments, environment_listings, @@ -49,6 +54,7 @@ pub fn start_up_server( strict: common.strict, dd_metrics, environment_pattern, + db, }; start_server(address, port, state, datadog)?; @@ -83,6 +89,7 @@ struct ServerState { strict: bool, dd_metrics: Option, environment_pattern: String, + db: Arc, } fn contextualize_path(path: &str) -> &str { @@ -101,6 +108,7 @@ async fn start_server( HttpServer::new(move || { actix_web::App::new() + .wrap(middleware::Compress::default()) .app_data(server_state.clone()) .wrap_fn(move |req, srv| { let start_time = if req.path() != "/ok" { @@ -320,24 +328,35 @@ fn get_env( if let Some(custom_metrics) = &state.dd_metrics { custom_metrics.incr(CustomMetrics::CacheMiss.metrics_name(), None); } - let repo = state.config_dir.lock(); - if let Some(sha) = repo.refresh(remote, Some(sha)) { - let filter = match hogan::config::build_env_regex(env, Some(&state.environment_pattern)) - { - Ok(filter) => filter, - Err(e) => { - warn!("Incompatible env name: {} {:?}", env, e); - //In an error scenario we'll still try and match against all configs - state.environments_regex.clone() - } - }; - if let Some(env) = repo.find(filter).iter().find(|e| e.environment == env) { - cache.insert(key.clone(), Arc::new(env.clone())) - } else { - debug!("Unable to find the env {} in {}", env, sha); - return None; + //Check embedded db before git repo + + if let Some(environment) = db::read_env(&state.db, env, sha).unwrap_or(None) { + info!("Found environment in the db {} {}", env, sha); + cache.insert(key.clone(), Arc::new(environment)); + } else { + let repo = state.config_dir.lock(); + if let Some(sha) = repo.refresh(remote, Some(sha)) { + let filter = + match hogan::config::build_env_regex(env, Some(&state.environment_pattern)) { + Ok(filter) => filter, + Err(e) => { + warn!("Incompatible env name: {} {:?}", env, e); + //In an error scenario we'll still try and match against all configs + state.environments_regex.clone() + } + }; + if let Some(environment) = repo.find(filter).iter().find(|e| e.environment == env) { + if let Err(e) = db::write_env(&state.db, env, &sha, environment) { + warn!("Unable to write env {} to db {:?}", key, e); + return None; + }; + cache.insert(key.clone(), Arc::new(environment.clone())) + } else { + debug!("Unable to find the env {} in {}", env, sha); + return None; + }; }; - }; + } if let Some(envs) = cache.get(&key) { Some(envs.clone()) } else { diff --git a/src/main.rs b/src/main.rs index 0f7c7f0..3163668 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,6 +45,7 @@ fn main() -> Result<(), Error> { environments_regex, datadog, environment_pattern, + db_location, } => { server::start_up_server( common, @@ -54,6 +55,7 @@ fn main() -> Result<(), Error> { environments_regex, datadog, environment_pattern, + db_location, )?; } } From 7c358ffef9c56f447e9bb0de22ed05f8e5db3477 Mon Sep 17 00:00:00 2001 From: Josh Comer Date: Tue, 21 Apr 2020 14:44:26 -0300 Subject: [PATCH 2/4] ci: only build once on windows --- appveyor.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 80be449..10ca5e1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -46,11 +46,8 @@ install: test_script: # we don't run the "test phase" when doing deploys - if [%APPVEYOR_REPO_TAG%]==[false] ( - cargo build --target %TARGET% && cargo build --target %TARGET% --release && - cargo test --target %TARGET% && cargo test --target %TARGET% --release && - cargo run --target %TARGET% -- --help && cargo run --target %TARGET% --release -- --help ) From 444617d5a348b66b2039dbf60131d503c734669d Mon Sep 17 00:00:00 2001 From: Josh Comer Date: Thu, 23 Apr 2020 14:36:54 -0300 Subject: [PATCH 3/4] Change to sqlite for embedded db --- Cargo.lock | 144 ++++++++++++++++++++-------------------------- Cargo.toml | 9 ++- src/app/config.rs | 4 +- src/app/db.rs | 110 +++++++++++++++++++---------------- src/app/server.rs | 39 ++++++------- src/main.rs | 4 +- 6 files changed, 153 insertions(+), 157 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 463c6a3..c68ea8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -284,9 +284,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d663a8e9a99154b5fb793032533f6328da35e23aac63d5c152279aa8ba356825" +checksum = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62" [[package]] name = "arrayref" @@ -378,9 +378,9 @@ dependencies = [ [[package]] name = "backtrace-sys" -version = "0.1.35" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118" +checksum = "78848718ee1255a2485d1309ad9cdecfc2e7d0362dd11c6829364c6b35ae1bc7" dependencies = [ "cc", "libc", @@ -509,9 +509,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.50" +version = "1.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" +checksum = "c3d87b23d6a92cd03af510a5ade527033f6aa6fa92161e2d5863a907d4c5e31d" dependencies = [ "jobserver", ] @@ -578,21 +578,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if", - "crossbeam-utils", - "lazy_static 1.4.0", - "maybe-uninit", - "memoffset", - "scopeguard", -] - [[package]] name = "crossbeam-utils" version = "0.7.2" @@ -737,6 +722,18 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "flate2" version = "1.0.14" @@ -764,16 +761,6 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi 0.3.8", -] - [[package]] name = "fs_extra" version = "1.1.0" @@ -918,9 +905,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.13.2" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfb93ca10f2934069c3aaafb753fbe0663f08ee009a01b6d62e062391447b15" +checksum = "2be62b002b81549c68d31792d7dce5752c3658ea85715f4032ebbf9a6fdb0bb6" dependencies = [ "bitflags", "libc", @@ -1005,11 +992,11 @@ dependencies = [ "parking_lot", "predicates", "regex", + "rusqlite", "serde", "serde_derive", "serde_json", "shellexpand", - "sled", "stderrlog", "structopt", "tempfile", @@ -1158,9 +1145,9 @@ checksum = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" [[package]] name = "libgit2-sys" -version = "0.12.3+1.0.0" +version = "0.12.4+1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7637dc15e7f05a16011723e0448655081fc01a374bcd368e2c9b9c7f5c5ab3ea" +checksum = "ef2870ecd7b50a76391b108edc2c62283ad2b62e5b1ab4d5263ef1cd04ef1c44" dependencies = [ "cc", "libc", @@ -1170,6 +1157,17 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libsqlite3-sys" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d90181c2904c287e5390186be820e5ef311a3c62edebb7d6ca3d6a48ce041d" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libssh2-sys" version = "0.2.16" @@ -1253,27 +1251,12 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "memchr" version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" -[[package]] -name = "memoffset" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4fc2c02a7e374099d4ee95a193111f72d2110197fe200272371758f6c3643d8" -dependencies = [ - "autocfg", -] - [[package]] name = "mime" version = "0.3.16" @@ -1397,9 +1380,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-src" -version = "111.8.1+1.1.1f" +version = "111.9.0+1.1.1g" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04f0299a91de598dde58d2e99101895498dcf3d58896a3297798f28b27c8b72" +checksum = "a2dbe10ddd1eb335aba3780eb2eaa13e1b7b441d2562fd962398740927f39ec4" dependencies = [ "cc", ] @@ -1430,9 +1413,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e136c1904604defe99ce5fd71a28d473fa60a12255d511aa78a9ddf11237aeb" +checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" dependencies = [ "cfg-if", "cloudabi", @@ -1519,9 +1502,9 @@ checksum = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae" [[package]] name = "pin-utils" -version = "0.1.0-alpha.4" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" @@ -1736,6 +1719,21 @@ dependencies = [ "quick-error", ] +[[package]] +name = "rusqlite" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57edf4c4cea4d7e0fab069acb5da9e8e8e5403c78abc81b1f37d83af02148ea5" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "libsqlite3-sys", + "lru-cache", + "memchr", + "time", +] + [[package]] name = "rust-argon2" version = "0.7.0" @@ -1756,9 +1754,9 @@ checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" [[package]] name = "ryu" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76" +checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1" [[package]] name = "same-file" @@ -1861,27 +1859,11 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" -[[package]] -name = "sled" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb6824dde66ad33bf20c6e8476f5b82b871bc8bc3c129a10ea2f7dae5060fa3" -dependencies = [ - "crc32fast", - "crossbeam-epoch", - "crossbeam-utils", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot", -] - [[package]] name = "smallvec" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05720e22615919e4734f6a99ceae50d00226c3c5aca406e102ebc33298214e0a" +checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" [[package]] name = "socket2" @@ -1916,9 +1898,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "structopt" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff6da2e8d107dfd7b74df5ef4d205c6aebee0706c647f6bc6a2d5789905c00fb" +checksum = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef" dependencies = [ "clap", "lazy_static 1.4.0", @@ -1927,9 +1909,9 @@ dependencies = [ [[package]] name = "structopt-derive" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a489c87c08fbaf12e386665109dd13470dcc9c4583ea3e10dd2b4523e5ebd9ac" +checksum = "d239ca4b13aee7a2142e6795cbd69e457665ff8037aed33b3effdc430d2f927a" dependencies = [ "heck", "proc-macro-error", diff --git a/Cargo.toml b/Cargo.toml index d3c708e..1c55846 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,14 +14,14 @@ edition = '2018' [dependencies] failure = '0.1' -lru_time_cache = '0.10.0' +lru_time_cache = '0.10' handlebars = '3' itertools = '0.9' json-patch = '0.2' log = '0.4' serde_derive = '1' serde_json = '1' -shellexpand = '2.0.0' +shellexpand = '2.0' stderrlog = '0.4' structopt = '0.3' tempfile = '3' @@ -34,9 +34,12 @@ actix-rt = '1' actix-service = '1' futures = '0.3' parking_lot = '0.10' -sled = '0.31' bincode = '1.2.1' +[dependencies.rusqlite] +version = '0.22' +features = ['bundled'] + [dependencies.git2] version = '0.13' features = ['vendored-openssl'] diff --git a/src/app/config.rs b/src/app/config.rs index a2d7a2a..141d55d 100644 --- a/src/app/config.rs +++ b/src/app/config.rs @@ -107,8 +107,8 @@ pub enum AppCommand { ///Filepath to the embedded db for storing environments. Will be created if it doesn't exist. If not provided a /// random temp directory will be created - #[structopt(long = "db", value_name = "PATH")] - db_location: Option, + #[structopt(long = "db", value_name = "PATH", default_value = "hogan.db")] + db_path: String, }, } diff --git a/src/app/db.rs b/src/app/db.rs index e017813..2b8ad0c 100644 --- a/src/app/db.rs +++ b/src/app/db.rs @@ -1,26 +1,74 @@ use bincode; use failure::Error; use hogan::config::Environment; +use rusqlite::{params, Connection, OpenFlags, Result, NO_PARAMS}; use serde::Deserialize; use serde::Serialize; -use sled; -pub use sled::Db; -use tempfile; -pub fn open_db(db_name: Option) -> Result { - let path = match db_name { - Some(p) => p, - None => tempfile::tempdir()? - .into_path() - .to_str() - .unwrap() - .to_owned(), +fn open_sql_db(db_path: &str, read_only: bool) -> Result { + let read_flag = if read_only { + OpenFlags::SQLITE_OPEN_READ_ONLY + } else { + OpenFlags::SQLITE_OPEN_READ_WRITE }; - info!("Opening DB: {}", path); - sled::open(path).map_err(|e| e.into()) + let conn = Connection::open_with_flags( + db_path, + read_flag | OpenFlags::SQLITE_OPEN_CREATE | OpenFlags::SQLITE_OPEN_SHARED_CACHE, + )?; + + conn.execute( + "CREATE TABLE IF NOT EXISTS hogan ( + key STRING PRIMARY KEY, + data BLOB, + timestamp DATETIME DEFAULT CURRENT_TIMESTAMP )", + NO_PARAMS, + )?; + + info!("Opened sqlite connection to {}", db_path); + + Ok(conn) +} + +pub fn read_sql_env(db_path: &str, env: &str, sha: &str) -> Result, Error> { + let conn = open_sql_db(db_path, false)?; + let mut query = conn.prepare("SELECT data FROM hogan WHERE key = ? LIMIT 1")?; + let key = gen_env_key(sha, env); + let data: Option>> = + query.query_map(params![key], |row| Ok(row.get(0)?))?.nth(0); + if let Some(data) = data { + let decoded: WritableEnvironment = match bincode::deserialize(&data?) { + Ok(environment) => environment, + Err(e) => { + warn!("Unable to deserialize env: {} {:?}", key, e); + return Err(e.into()); + } + }; + Ok(Some(decoded.into())) + } else { + info!("Unable to find {} in sqlite db", key); + Ok(None) + } } -fn gen_key(sha: &str, env: &str) -> String { +pub fn write_sql_env( + db_path: &str, + env: &str, + sha: &str, + data: &Environment, +) -> Result { + let conn = open_sql_db(db_path, false)?; + let key = gen_env_key(sha, env); + let env_data: WritableEnvironment = data.into(); + let data = bincode::serialize(&env_data)?; + + conn.execute( + "INSERT INTO hogan (key, data) VALUES (?1, ?2)", + params![key, data], + ) + .map_err(|e| e.into()) +} + +fn gen_env_key(sha: &str, env: &str) -> String { format!("{}::{}", sha, env) } @@ -50,37 +98,3 @@ impl From for Environment { } } } - -pub fn write_env(db: &sled::Db, env: &str, sha: &str, data: &Environment) -> Result<(), Error> { - write_key(db, &gen_key(sha, env), data) -} - -fn write_key(db: &sled::Db, key: &str, data: &Environment) -> Result<(), Error> { - let env: WritableEnvironment = data.into(); - let data: Vec = bincode::serialize(&env).unwrap(); - info!("Writing {} to db size: {}", key, data.len()); - db.insert(key, data)?; - Ok(()) -} - -pub fn read_env(db: &sled::Db, env: &str, sha: &str) -> Result, Error> { - let key = gen_key(sha, env); - match db.get(&key)? { - Some(data) => { - let decoded: WritableEnvironment = match bincode::deserialize(&data) { - Ok(environment) => environment, - Err(e) => { - warn!("Unable to deserialize env: {} {:?}", key, e); - return Err(e.into()); - } - }; - Ok(Some(decoded.into())) - } - None => Ok(None), - } -} - -pub fn remove_env(db: &sled::Db, env: &str, sha: &str) -> Result<(), Error> { - db.remove(gen_key(sha, env))?; - Ok(()) -} diff --git a/src/app/server.rs b/src/app/server.rs index 3f70027..342d37f 100644 --- a/src/app/server.rs +++ b/src/app/server.rs @@ -15,6 +15,20 @@ use serde::Serialize; use std::sync::Arc; use std::time::SystemTime; +type EnvCache = Mutex>>; +type EnvListingCache = Mutex>>>; + +struct ServerState { + environments: EnvCache, + environment_listings: EnvListingCache, + config_dir: Mutex, + environments_regex: Regex, + strict: bool, + dd_metrics: Option, + environment_pattern: String, + db_path: String, +} + pub fn start_up_server( common: AppCommon, port: u16, @@ -23,7 +37,7 @@ pub fn start_up_server( environments_regex: Regex, datadog: bool, environment_pattern: String, - db_location: Option, + db_path: String, ) -> Result<(), Error> { let config_dir = ConfigDir::new(common.configs_url, &common.ssh_key)?; @@ -44,8 +58,6 @@ pub fn start_up_server( None }; - let db = Arc::new(db::open_db(db_location)?); - let state = ServerState { environments, environment_listings, @@ -54,7 +66,7 @@ pub fn start_up_server( strict: common.strict, dd_metrics, environment_pattern, - db, + db_path, }; start_server(address, port, state, datadog)?; @@ -78,20 +90,6 @@ impl From<&hogan::config::Environment> for EnvDescription { } } -type EnvCache = Mutex>>; -type EnvListingCache = Mutex>>>; - -struct ServerState { - environments: EnvCache, - environment_listings: EnvListingCache, - config_dir: Mutex, - environments_regex: Regex, - strict: bool, - dd_metrics: Option, - environment_pattern: String, - db: Arc, -} - fn contextualize_path(path: &str) -> &str { path.split('/').nth(1).unwrap_or_else(|| &"route") } @@ -330,7 +328,7 @@ fn get_env( } //Check embedded db before git repo - if let Some(environment) = db::read_env(&state.db, env, sha).unwrap_or(None) { + if let Some(environment) = db::read_sql_env(&state.db_path, env, sha).unwrap_or(None) { info!("Found environment in the db {} {}", env, sha); cache.insert(key.clone(), Arc::new(environment)); } else { @@ -346,9 +344,8 @@ fn get_env( } }; if let Some(environment) = repo.find(filter).iter().find(|e| e.environment == env) { - if let Err(e) = db::write_env(&state.db, env, &sha, environment) { + if let Err(e) = db::write_sql_env(&state.db_path, env, &sha, environment) { warn!("Unable to write env {} to db {:?}", key, e); - return None; }; cache.insert(key.clone(), Arc::new(environment.clone())) } else { diff --git a/src/main.rs b/src/main.rs index 3163668..322386b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,7 +45,7 @@ fn main() -> Result<(), Error> { environments_regex, datadog, environment_pattern, - db_location, + db_path, } => { server::start_up_server( common, @@ -55,7 +55,7 @@ fn main() -> Result<(), Error> { environments_regex, datadog, environment_pattern, - db_location, + db_path, )?; } } From 972481393a0a532f5670632a457d4f58da90f4a3 Mon Sep 17 00:00:00 2001 From: Josh Comer Date: Thu, 23 Apr 2020 15:15:57 -0300 Subject: [PATCH 4/4] Release 0.8.0 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 1c55846..681f404 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ doc = false [package] name = 'hogan' -version = '0.7.4' +version = '0.8.0' authors = [ 'Jonathan Morley ', 'Josh Comer ',