diff --git a/Cargo.lock b/Cargo.lock index fd62558..c68ea8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -284,9 +284,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d663a8e9a99154b5fb793032533f6328da35e23aac63d5c152279aa8ba356825" +checksum = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62" [[package]] name = "arrayref" @@ -378,9 +378,9 @@ dependencies = [ [[package]] name = "backtrace-sys" -version = "0.1.35" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8aba10a69c8e8d7622c5710229485ec32e9d55fdad160ea559c086fdcd118" +checksum = "78848718ee1255a2485d1309ad9cdecfc2e7d0362dd11c6829364c6b35ae1bc7" dependencies = [ "cc", "libc", @@ -392,6 +392,16 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" +[[package]] +name = "bincode" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" +dependencies = [ + "byteorder", + "serde", +] + [[package]] name = "bitflags" version = "1.2.1" @@ -499,9 +509,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.50" +version = "1.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" +checksum = "c3d87b23d6a92cd03af510a5ade527033f6aa6fa92161e2d5863a907d4c5e31d" dependencies = [ "jobserver", ] @@ -712,6 +722,18 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "flate2" version = "1.0.14" @@ -883,9 +905,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.13.2" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfb93ca10f2934069c3aaafb753fbe0663f08ee009a01b6d62e062391447b15" +checksum = "2be62b002b81549c68d31792d7dce5752c3658ea85715f4032ebbf9a6fdb0bb6" dependencies = [ "bitflags", "libc", @@ -955,6 +977,7 @@ dependencies = [ "actix-service", "actix-web", "assert_cmd", + "bincode", "dir-diff", "dogstatsd", "failure", @@ -969,6 +992,7 @@ dependencies = [ "parking_lot", "predicates", "regex", + "rusqlite", "serde", "serde_derive", "serde_json", @@ -1121,9 +1145,9 @@ checksum = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" [[package]] name = "libgit2-sys" -version = "0.12.3+1.0.0" +version = "0.12.4+1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7637dc15e7f05a16011723e0448655081fc01a374bcd368e2c9b9c7f5c5ab3ea" +checksum = "ef2870ecd7b50a76391b108edc2c62283ad2b62e5b1ab4d5263ef1cd04ef1c44" dependencies = [ "cc", "libc", @@ -1133,6 +1157,17 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libsqlite3-sys" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d90181c2904c287e5390186be820e5ef311a3c62edebb7d6ca3d6a48ce041d" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libssh2-sys" version = "0.2.16" @@ -1345,9 +1380,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-src" -version = "111.8.1+1.1.1f" +version = "111.9.0+1.1.1g" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04f0299a91de598dde58d2e99101895498dcf3d58896a3297798f28b27c8b72" +checksum = "a2dbe10ddd1eb335aba3780eb2eaa13e1b7b441d2562fd962398740927f39ec4" dependencies = [ "cc", ] @@ -1378,9 +1413,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e136c1904604defe99ce5fd71a28d473fa60a12255d511aa78a9ddf11237aeb" +checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" dependencies = [ "cfg-if", "cloudabi", @@ -1467,9 +1502,9 @@ checksum = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae" [[package]] name = "pin-utils" -version = "0.1.0-alpha.4" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" @@ -1649,9 +1684,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.3.6" +version = "1.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6946991529684867e47d86474e3a6d0c0ab9b82d5821e314b1ede31fa3a4b3" +checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" dependencies = [ "aho-corasick", "memchr", @@ -1684,6 +1719,21 @@ dependencies = [ "quick-error", ] +[[package]] +name = "rusqlite" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57edf4c4cea4d7e0fab069acb5da9e8e8e5403c78abc81b1f37d83af02148ea5" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "libsqlite3-sys", + "lru-cache", + "memchr", + "time", +] + [[package]] name = "rust-argon2" version = "0.7.0" @@ -1704,9 +1754,9 @@ checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" [[package]] name = "ryu" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76" +checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1" [[package]] name = "same-file" @@ -1811,9 +1861,9 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "smallvec" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05720e22615919e4734f6a99ceae50d00226c3c5aca406e102ebc33298214e0a" +checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" [[package]] name = "socket2" @@ -1848,9 +1898,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "structopt" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff6da2e8d107dfd7b74df5ef4d205c6aebee0706c647f6bc6a2d5789905c00fb" +checksum = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef" dependencies = [ "clap", "lazy_static 1.4.0", @@ -1859,9 +1909,9 @@ dependencies = [ [[package]] name = "structopt-derive" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a489c87c08fbaf12e386665109dd13470dcc9c4583ea3e10dd2b4523e5ebd9ac" +checksum = "d239ca4b13aee7a2142e6795cbd69e457665ff8037aed33b3effdc430d2f927a" dependencies = [ "heck", "proc-macro-error", @@ -1978,12 +2028,11 @@ dependencies = [ [[package]] name = "time" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" dependencies = [ "libc", - "redox_syscall", "winapi 0.3.8", ] @@ -2244,9 +2293,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa515c5163a99cc82bab70fd3bfdd36d827be85de63737b40fcef2ce084a436e" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ "winapi 0.3.8", ] diff --git a/Cargo.toml b/Cargo.toml index f58379b..681f404 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ doc = false [package] name = 'hogan' -version = '0.7.4' +version = '0.8.0' authors = [ 'Jonathan Morley ', 'Josh Comer ', @@ -14,14 +14,14 @@ edition = '2018' [dependencies] failure = '0.1' -lru_time_cache = '0.10.0' +lru_time_cache = '0.10' handlebars = '3' itertools = '0.9' json-patch = '0.2' log = '0.4' serde_derive = '1' serde_json = '1' -shellexpand = '2.0.0' +shellexpand = '2.0' stderrlog = '0.4' structopt = '0.3' tempfile = '3' @@ -34,6 +34,11 @@ actix-rt = '1' actix-service = '1' futures = '0.3' parking_lot = '0.10' +bincode = '1.2.1' + +[dependencies.rusqlite] +version = '0.22' +features = ['bundled'] [dependencies.git2] version = '0.13' diff --git a/appveyor.yml b/appveyor.yml index 80be449..10ca5e1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -46,11 +46,8 @@ install: test_script: # we don't run the "test phase" when doing deploys - if [%APPVEYOR_REPO_TAG%]==[false] ( - cargo build --target %TARGET% && cargo build --target %TARGET% --release && - cargo test --target %TARGET% && cargo test --target %TARGET% --release && - cargo run --target %TARGET% -- --help && cargo run --target %TARGET% --release -- --help ) diff --git a/src/app/config.rs b/src/app/config.rs index 2b75697..141d55d 100644 --- a/src/app/config.rs +++ b/src/app/config.rs @@ -104,6 +104,11 @@ pub enum AppCommand { value_name = "PATTERN" )] environment_pattern: String, + + ///Filepath to the embedded db for storing environments. Will be created if it doesn't exist. If not provided a + /// random temp directory will be created + #[structopt(long = "db", value_name = "PATH", default_value = "hogan.db")] + db_path: String, }, } diff --git a/src/app/db.rs b/src/app/db.rs new file mode 100644 index 0000000..2b8ad0c --- /dev/null +++ b/src/app/db.rs @@ -0,0 +1,100 @@ +use bincode; +use failure::Error; +use hogan::config::Environment; +use rusqlite::{params, Connection, OpenFlags, Result, NO_PARAMS}; +use serde::Deserialize; +use serde::Serialize; + +fn open_sql_db(db_path: &str, read_only: bool) -> Result { + let read_flag = if read_only { + OpenFlags::SQLITE_OPEN_READ_ONLY + } else { + OpenFlags::SQLITE_OPEN_READ_WRITE + }; + let conn = Connection::open_with_flags( + db_path, + read_flag | OpenFlags::SQLITE_OPEN_CREATE | OpenFlags::SQLITE_OPEN_SHARED_CACHE, + )?; + + conn.execute( + "CREATE TABLE IF NOT EXISTS hogan ( + key STRING PRIMARY KEY, + data BLOB, + timestamp DATETIME DEFAULT CURRENT_TIMESTAMP )", + NO_PARAMS, + )?; + + info!("Opened sqlite connection to {}", db_path); + + Ok(conn) +} + +pub fn read_sql_env(db_path: &str, env: &str, sha: &str) -> Result, Error> { + let conn = open_sql_db(db_path, false)?; + let mut query = conn.prepare("SELECT data FROM hogan WHERE key = ? LIMIT 1")?; + let key = gen_env_key(sha, env); + let data: Option>> = + query.query_map(params![key], |row| Ok(row.get(0)?))?.nth(0); + if let Some(data) = data { + let decoded: WritableEnvironment = match bincode::deserialize(&data?) { + Ok(environment) => environment, + Err(e) => { + warn!("Unable to deserialize env: {} {:?}", key, e); + return Err(e.into()); + } + }; + Ok(Some(decoded.into())) + } else { + info!("Unable to find {} in sqlite db", key); + Ok(None) + } +} + +pub fn write_sql_env( + db_path: &str, + env: &str, + sha: &str, + data: &Environment, +) -> Result { + let conn = open_sql_db(db_path, false)?; + let key = gen_env_key(sha, env); + let env_data: WritableEnvironment = data.into(); + let data = bincode::serialize(&env_data)?; + + conn.execute( + "INSERT INTO hogan (key, data) VALUES (?1, ?2)", + params![key, data], + ) + .map_err(|e| e.into()) +} + +fn gen_env_key(sha: &str, env: &str) -> String { + format!("{}::{}", sha, env) +} + +#[derive(Default, Serialize, Deserialize, Debug)] +struct WritableEnvironment { + config_data: String, + environment: String, + environment_type: Option, +} + +impl From<&Environment> for WritableEnvironment { + fn from(environment: &Environment) -> Self { + WritableEnvironment { + config_data: environment.config_data.to_string(), + environment: environment.environment.to_owned(), + environment_type: environment.environment_type.to_owned(), + } + } +} + +impl From for Environment { + fn from(environment: WritableEnvironment) -> Self { + Environment { + config_data: serde_json::from_str(&environment.config_data).unwrap(), + environment: environment.environment.to_owned(), + environment_type: environment.environment_type.to_owned(), + } + } +} diff --git a/src/app/mod.rs b/src/app/mod.rs index 68f3021..8fdba25 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -1,4 +1,5 @@ pub mod cli; pub mod config; mod datadogstatsd; +mod db; pub mod server; diff --git a/src/app/server.rs b/src/app/server.rs index 3c63262..342d37f 100644 --- a/src/app/server.rs +++ b/src/app/server.rs @@ -1,7 +1,8 @@ use crate::app::config::AppCommon; use crate::app::datadogstatsd::{CustomMetrics, DdMetrics}; +use crate::app::db; use actix_service::Service; -use actix_web::{get, post, web, HttpResponse, HttpServer}; +use actix_web::{get, middleware, post, web, HttpResponse, HttpServer}; use failure::Error; use futures::future::FutureExt; use hogan; @@ -14,6 +15,20 @@ use serde::Serialize; use std::sync::Arc; use std::time::SystemTime; +type EnvCache = Mutex>>; +type EnvListingCache = Mutex>>>; + +struct ServerState { + environments: EnvCache, + environment_listings: EnvListingCache, + config_dir: Mutex, + environments_regex: Regex, + strict: bool, + dd_metrics: Option, + environment_pattern: String, + db_path: String, +} + pub fn start_up_server( common: AppCommon, port: u16, @@ -22,6 +37,7 @@ pub fn start_up_server( environments_regex: Regex, datadog: bool, environment_pattern: String, + db_path: String, ) -> Result<(), Error> { let config_dir = ConfigDir::new(common.configs_url, &common.ssh_key)?; @@ -41,6 +57,7 @@ pub fn start_up_server( } else { None }; + let state = ServerState { environments, environment_listings, @@ -49,6 +66,7 @@ pub fn start_up_server( strict: common.strict, dd_metrics, environment_pattern, + db_path, }; start_server(address, port, state, datadog)?; @@ -72,19 +90,6 @@ impl From<&hogan::config::Environment> for EnvDescription { } } -type EnvCache = Mutex>>; -type EnvListingCache = Mutex>>>; - -struct ServerState { - environments: EnvCache, - environment_listings: EnvListingCache, - config_dir: Mutex, - environments_regex: Regex, - strict: bool, - dd_metrics: Option, - environment_pattern: String, -} - fn contextualize_path(path: &str) -> &str { path.split('/').nth(1).unwrap_or_else(|| &"route") } @@ -101,6 +106,7 @@ async fn start_server( HttpServer::new(move || { actix_web::App::new() + .wrap(middleware::Compress::default()) .app_data(server_state.clone()) .wrap_fn(move |req, srv| { let start_time = if req.path() != "/ok" { @@ -320,24 +326,34 @@ fn get_env( if let Some(custom_metrics) = &state.dd_metrics { custom_metrics.incr(CustomMetrics::CacheMiss.metrics_name(), None); } - let repo = state.config_dir.lock(); - if let Some(sha) = repo.refresh(remote, Some(sha)) { - let filter = match hogan::config::build_env_regex(env, Some(&state.environment_pattern)) - { - Ok(filter) => filter, - Err(e) => { - warn!("Incompatible env name: {} {:?}", env, e); - //In an error scenario we'll still try and match against all configs - state.environments_regex.clone() - } - }; - if let Some(env) = repo.find(filter).iter().find(|e| e.environment == env) { - cache.insert(key.clone(), Arc::new(env.clone())) - } else { - debug!("Unable to find the env {} in {}", env, sha); - return None; + //Check embedded db before git repo + + if let Some(environment) = db::read_sql_env(&state.db_path, env, sha).unwrap_or(None) { + info!("Found environment in the db {} {}", env, sha); + cache.insert(key.clone(), Arc::new(environment)); + } else { + let repo = state.config_dir.lock(); + if let Some(sha) = repo.refresh(remote, Some(sha)) { + let filter = + match hogan::config::build_env_regex(env, Some(&state.environment_pattern)) { + Ok(filter) => filter, + Err(e) => { + warn!("Incompatible env name: {} {:?}", env, e); + //In an error scenario we'll still try and match against all configs + state.environments_regex.clone() + } + }; + if let Some(environment) = repo.find(filter).iter().find(|e| e.environment == env) { + if let Err(e) = db::write_sql_env(&state.db_path, env, &sha, environment) { + warn!("Unable to write env {} to db {:?}", key, e); + }; + cache.insert(key.clone(), Arc::new(environment.clone())) + } else { + debug!("Unable to find the env {} in {}", env, sha); + return None; + }; }; - }; + } if let Some(envs) = cache.get(&key) { Some(envs.clone()) } else { diff --git a/src/main.rs b/src/main.rs index 0f7c7f0..322386b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -45,6 +45,7 @@ fn main() -> Result<(), Error> { environments_regex, datadog, environment_pattern, + db_path, } => { server::start_up_server( common, @@ -54,6 +55,7 @@ fn main() -> Result<(), Error> { environments_regex, datadog, environment_pattern, + db_path, )?; } }