Merge pull request #60 from jjcomer/git-proper

Add native git support

Cargo.toml

@@ -5,7 +5,7 @@ doc = false
 
 [package]
 name = 'hogan'
-version = '0.11.1'
+version = '0.12.0'
 authors = [
     'Jonathan Morley <jmorley@cvent.com>',
     'Josh Comer <jcomer@cvent.com>',
@@ -31,7 +31,7 @@ walkdir = '2'
 zip = '0.5'
 dogstatsd = '0.6'
 actix-web = '3.3'
-actix-rt = '1'
+actix-rt = '2.1'
 actix-service = '1.0'
 futures = '0.3'
 parking_lot = '0.11'
@@ -40,13 +40,14 @@ lazy_static = '1'
 riker = '0.4'
 riker-patterns = '0.4'
 compression = '0.1'
+which = '4.0'
 
 [dependencies.tokio]
-version = '1.0'
+version = '1.1'
 features = ['full']
 
 [dependencies.rusqlite]
-version = '0.24'
+version = '0.25'
 features = ['bundled']
 
 [dependencies.git2]

Dockerfile

@@ -36,6 +36,8 @@ RUN apk --update add ca-certificates
 # because we already have a binary built
 FROM alpine:latest
 
+RUN apk --no-cache add git openssh
+
 # Copies standard SSL certs from the "build" stage
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 

src/app/cli.rs

@@ -24,7 +24,7 @@ pub fn cli(
     let mut templates = template_dir.find(templates_regex);
     println!("Loaded {} template file(s)", templates.len());
 
-    let config_dir = ConfigDir::new(common.configs_url, &common.ssh_key)?;
+    let config_dir = ConfigDir::new(common.configs_url, &common.ssh_key, common.native_git)?;
     let environments = config_dir.find(App::config_regex(&environments_regex)?);
     println!("Loaded {} config file(s)", environments.len());
 

src/app/config.rs

@@ -144,6 +144,10 @@ pub struct AppCommon {
     /// Throw errors if values do not exist in configs
     #[structopt(short = "s", long = "strict")]
     pub strict: bool,
+
+    /// When true the application will assume an external git application is installed and available in the execution environment
+    #[structopt(short = "g", long = "git")]
+    pub native_git: bool,
 }
 
 impl App {

src/app/datadogstatsd.rs

@@ -102,6 +102,7 @@ pub enum CustomMetrics {
     RequestTime,
     FetchTime,
     FetchCounter,
+    MaintenanceTime,
 }
 
 impl From<CustomMetrics> for &str {
@@ -111,6 +112,7 @@ impl From<CustomMetrics> for &str {
             CustomMetrics::RequestTime => &"hogan.requests",
             CustomMetrics::FetchTime => &"hogan.fetch",
             CustomMetrics::FetchCounter => &"hogan.fetchcounter",
+            CustomMetrics::MaintenanceTime => &"hogan.maintenance",
         }
     }
 }

src/app/db.rs

@@ -1,7 +1,7 @@
 use anyhow::Result;
 use compression::prelude::*;
 use hogan::config::Environment;
-use rusqlite::{params, Connection, OpenFlags, NO_PARAMS};
+use rusqlite::{params, Connection, OpenFlags};
 use serde::Deserialize;
 use serde::Serialize;
 
@@ -20,7 +20,7 @@ fn open_sql_db(db_path: &str, read_only: bool) -> Result<Connection> {
             key STRING PRIMARY KEY,
             data BLOB,
             timestamp DATETIME DEFAULT CURRENT_TIMESTAMP )",
-            NO_PARAMS,
+            [],
         )?;
     }
 

src/app/fetch_actor.rs

@@ -15,6 +15,7 @@ struct FetchActor {
     last_updated: SystemTime,
     metrics: Arc<DdMetrics>,
     fetch_delay: u64,
+    maintenance_count: u8,
 }
 
 impl ActorFactoryArgs<(Arc<ConfigDir>, Arc<DdMetrics>, u64)> for FetchActor {
@@ -24,6 +25,7 @@ impl ActorFactoryArgs<(Arc<ConfigDir>, Arc<DdMetrics>, u64)> for FetchActor {
             last_updated: SystemTime::now(),
             metrics,
             fetch_delay,
+            maintenance_count: 0,
         }
     }
 }
@@ -77,6 +79,33 @@ impl Receive<ExecuteFetch> for FetchActor {
             self.metrics
                 .incr(CustomMetrics::FetchCounter.into(), Some(counter_tags));
         }
+        if self.maintenance_count >= 10 {
+            self.maintenance_count = 0;
+            let maintenance_start = SystemTime::now();
+            let maintenance_result = self.config.perform_maintenance();
+            if let Ok(elapsed_time) = maintenance_start.elapsed() {
+                if let Err(e) = &maintenance_result {
+                    warn!(
+                        "Unable to perform maintenance on git repo. Took {} ms. Error: {:?}",
+                        elapsed_time.as_millis(),
+                        e
+                    );
+                } else {
+                    info!(
+                        "Performed maintenance on repo took: {} ms",
+                        elapsed_time.as_millis()
+                    );
+                }
+                self.metrics.time(
+                    CustomMetrics::MaintenanceTime.into(),
+                    None,
+                    elapsed_time.as_millis() as i64,
+                );
+            }
+        } else {
+            self.maintenance_count += 1
+        }
+
         self.last_updated = SystemTime::now();
     }
 }

src/app/server.rs

@@ -102,7 +102,11 @@ pub fn start_up_server(
 ) -> Result<()> {
     info!("datadog monitoring is setting: {}", datadog);
     let dd_metrics = Arc::new(DdMetrics::new(datadog));
-    let config_dir = Arc::new(ConfigDir::new(common.configs_url, &common.ssh_key)?);
+    let config_dir = Arc::new(ConfigDir::new(
+        common.configs_url,
+        &common.ssh_key,
+        common.native_git,
+    )?);
 
     let actor_system = ActorSystem::new()?;
     let head_request_actor =
@@ -166,7 +170,7 @@ fn contextualize_path(path: &str) -> &str {
     path.split('/').nth(1).unwrap_or_else(|| &"route")
 }
 
-#[actix_rt::main]
+#[actix_web::main]
 async fn start_server(address: String, port: u16, state: ServerState) -> Result<()> {
     let binding = format!("{}:{}", address, port);
     let dd_client = state.dd_metrics.clone();

src/config.rs

@@ -107,27 +107,33 @@ pub enum ConfigDir {
         ssh_key_path: PathBuf,
         temp_dir: TempDir,
         directory: PathBuf,
+        native_git: bool,
     },
 }
 
 impl ConfigDir {
-    pub fn new(url: ConfigUrl, ssh_key_path: &Path) -> Result<ConfigDir> {
+    pub fn new(url: ConfigUrl, ssh_key_path: &Path, native_git: bool) -> Result<ConfigDir> {
         let config_dir = match url {
             ConfigUrl::Git {
                 url,
-                branch,
                 internal_path,
+                branch,
             } => {
                 let temp_dir = tempfile::tempdir().map_err(|e| HoganError::GitError {
                     msg: format!("Unable to create temp directory {:?}", e),
                 })?;
 
-                let git_repo = git::clone(
-                    &url,
-                    branch.as_deref(),
-                    temp_dir.path(),
-                    Some(&ssh_key_path),
-                )?;
+                let git_repo = if native_git {
+                    git::ext_clone(&url, temp_dir.path())?;
+                    git::build_repo(temp_dir.path().to_str().unwrap())?
+                } else {
+                    git::clone(
+                        &url,
+                        branch.as_deref(),
+                        temp_dir.path(),
+                        Some(&ssh_key_path),
+                    )?
+                };
 
                 let head_sha = git::get_head_sha(&git_repo)?;
 
@@ -148,6 +154,7 @@ impl ConfigDir {
                     ssh_key_path,
                     temp_dir,
                     directory,
+                    native_git,
                 })
             }
             ConfigUrl::File { path } => Ok(ConfigDir::File { directory: path }),
@@ -171,14 +178,18 @@ impl ConfigDir {
     pub fn extend(&self, branch: &str) -> Result<ConfigDir> {
         match self {
             ConfigDir::Git {
-                url, ssh_key_path, ..
+                url,
+                ssh_key_path,
+                native_git,
+                ..
             } => ConfigDir::new(
                 ConfigUrl::Git {
                     url: url.clone(),
                     branch: Some(branch.to_owned()),
                     internal_path: PathBuf::new(),
                 },
                 ssh_key_path,
+                *native_git,
             ),
             ConfigDir::File { .. } => Err(HoganError::GitError {
                 msg: "Can not extend file config".to_string(),
@@ -320,6 +331,26 @@ impl ConfigDir {
         }
     }
 
+    pub fn perform_maintenance(&self) -> Result<()> {
+        match self {
+            ConfigDir::File { .. } => Err(HoganError::GitError {
+                msg: "Unable to perform git actions on a file".to_string(),
+            })
+            .context("Performing repo maintenance"),
+            ConfigDir::Git {
+                directory,
+                native_git,
+                ..
+            } => {
+                if *native_git {
+                    git::ext_maintenance(&directory.as_path())
+                        .with_context(|| "Performing Maintenance")?;
+                }
+                Ok(())
+            }
+        }
+    }
+
     pub fn fetch_only(&self, remote_name: &str) -> Result<()> {
         match self {
             ConfigDir::File { .. } => Err(HoganError::GitError {
@@ -330,13 +361,20 @@ impl ConfigDir {
                 directory,
                 ssh_key_path,
                 url,
+                native_git,
                 ..
             } => {
-                let git_repo = git::build_repo(directory.to_str().unwrap())
-                    .with_context(|| "Fetching git repo. Building repo")?;
-                git::fetch(&git_repo, remote_name, Some(ssh_key_path), Some(url))
-                    .with_context(|| "Fetching Repo")?;
-                Ok(())
+                if *native_git {
+                    git::ext_fetch(&directory.as_path(), remote_name)
+                        .with_context(|| "Fetching git repo")?;
+                    Ok(())
+                } else {
+                    let git_repo = git::build_repo(directory.to_str().unwrap())
+                        .with_context(|| "Fetching git repo. Building repo")?;
+                    git::fetch(&git_repo, remote_name, Some(ssh_key_path), Some(url))
+                        .with_context(|| "Fetching Repo")?;
+                    Ok(())
+                }
             }
         }
     }
@@ -590,6 +628,7 @@ mod tests {
         let config_dir = ConfigDir::new(
             "file://./tests/fixtures/configs".parse().unwrap(),
             Path::new(""),
+            true,
         )
         .unwrap();
         let environments = config_dir.find(build_regex("config\\..+\\.json$").unwrap());
@@ -601,6 +640,7 @@ mod tests {
         let config_dir = ConfigDir::new(
             "file://./tests/fixtures/configs".parse().unwrap(),
             Path::new(""),
+            true,
         )
         .unwrap();
         let environments = config_dir.find(build_regex(r#"config\.test\d?\.json"#).unwrap());

src/git.rs

@@ -3,9 +3,20 @@ use anyhow::{Context, Result};
 use git2::build::RepoBuilder;
 use git2::{AutotagOption, Cred, FetchOptions, Reference, RemoteCallbacks, Repository, ResetType};
 use std::path::Path;
+use std::process::Command;
 use std::str;
 use url::Url;
 
+pub fn ext_clone(url: &Url, path: &Path) -> Result<()> {
+    info!("Cloning {:?} to {:?}", url, path);
+    let mut clone = Command::new("git")
+        .args(&["clone", &url.to_string(), path.to_str().unwrap()])
+        .spawn()?;
+    let result = clone.wait()?;
+    info!("Clone output {}", result);
+    Ok(())
+}
+
 pub fn clone(
     url: &Url,
     branch: Option<&str>,
@@ -112,6 +123,27 @@ fn detach_head(repo: &Repository, sha: &str) -> Result<()> {
         .context(format!("Error detaching head to SHA {}", sha))
 }
 
+pub fn ext_fetch(path: &Path, remote: &str) -> Result<()> {
+    info!("Fetching {}", remote);
+    let mut fetch_cmd = Command::new("git")
+        .current_dir(path.to_str().unwrap())
+        .args(&["fetch", remote])
+        .spawn()?;
+
+    fetch_cmd.wait()?;
+    Ok(())
+}
+
+pub fn ext_maintenance(path: &Path) -> Result<()> {
+    info!("Performing maintenance");
+    let mut maintenance_cmd = Command::new("git")
+        .current_dir(path.to_str().unwrap())
+        .args(&["maintenance", "run", "--auto"])
+        .spawn()?;
+    maintenance_cmd.wait()?;
+    Ok(())
+}
+
 pub fn fetch(
     repo: &Repository,
     remote: &str,