From 5ef663faf85c2ee1b40b191ff1567a3f49694552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20K=C3=B6hl?= Date: Fri, 22 Dec 2023 13:42:08 +0100 Subject: [PATCH] feat: allow external repositories to be pulled (#14) --- Cargo.lock | 17 ++ crates/rugpi-bakery/Cargo.toml | 1 + crates/rugpi-bakery/src/config.rs | 4 + crates/rugpi-bakery/src/main.rs | 42 ++++ crates/rugpi-bakery/src/repositories.rs | 188 ++++++++++++++++++ .../rugpi-bakery/src/repositories/sources.rs | 162 +++++++++++++++ 6 files changed, 414 insertions(+) create mode 100644 crates/rugpi-bakery/src/repositories.rs create mode 100644 crates/rugpi-bakery/src/repositories/sources.rs diff --git a/Cargo.lock b/Cargo.lock index 83851a9..4ec3e5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,6 +250,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "cpufeatures" version = "0.2.11" @@ -496,6 +506,12 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.151" @@ -745,6 +761,7 @@ dependencies = [ "anyhow", "camino", "clap", + "colored", "hex", "nix 0.27.1", "rugpi-common", diff --git a/crates/rugpi-bakery/Cargo.toml b/crates/rugpi-bakery/Cargo.toml index b18888b..b2f17d1 100644 --- a/crates/rugpi-bakery/Cargo.toml +++ b/crates/rugpi-bakery/Cargo.toml @@ -11,6 +11,7 @@ homepage.workspace = true anyhow = "1.0.71" camino = { version = "1.1.4", features = ["serde"] } clap = { version = "4.3.8", features = ["derive"] } +colored = "2.1.0" hex = "0.4.3" nix = { version = "0.27.1", features = ["process"] } rugpi-common = { path = "../rugpi-common" } diff --git a/crates/rugpi-bakery/src/config.rs b/crates/rugpi-bakery/src/config.rs index 4a27d57..d97288c 100644 --- a/crates/rugpi-bakery/src/config.rs +++ b/crates/rugpi-bakery/src/config.rs @@ -11,12 +11,16 @@ use serde::{Deserialize, Serialize}; use crate::{ recipes::{ParameterValue, RecipeName}, + repositories::sources::Source, Args, }; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct BakeryConfig { + /// The repositories to use. + #[serde(default)] + pub repositories: HashMap, /// The recipes to include. #[serde(default)] pub recipes: HashSet, diff --git a/crates/rugpi-bakery/src/main.rs b/crates/rugpi-bakery/src/main.rs index 735d2eb..aad9423 100644 --- a/crates/rugpi-bakery/src/main.rs +++ b/crates/rugpi-bakery/src/main.rs @@ -1,9 +1,13 @@ use std::{ + env, ffi::{CStr, CString}, path::PathBuf, }; use clap::Parser; +use colored::Colorize; +use config::load_config; +use repositories::Repositories; use rugpi_common::Anyhow; use tasks::{ bake::{self, BakeTask}, @@ -13,6 +17,7 @@ use tasks::{ pub mod config; pub mod recipes; +pub mod repositories; pub mod tasks; pub mod utils; @@ -37,6 +42,8 @@ pub enum Task { /// Spawn a shell in the Rugpi Bakery Docker container. Shell, Update(UpdateTask), + /// Pull in external repositories. + Pull, } #[derive(Debug, Parser)] @@ -65,6 +72,41 @@ fn main() -> Anyhow<()> { println!("Switch Rugpi Bakery to version `{version}`..."); std::fs::write("run-bakery", interpolate_run_bakery(version))?; } + Task::Pull => { + let config = load_config(&args)?; + let root_dir = std::env::current_dir()?; + let mut repositories = Repositories::new(&root_dir); + repositories.load_root(config.repositories.clone(), true)?; + for (_, repository) in repositories.iter() { + println!( + "{} {} {}", + repository.source.id.as_short_str().blue(), + repository.config.name.as_deref().unwrap_or(""), + repository + .config + .description + .as_deref() + .unwrap_or("") + .bright_black(), + ); + match &repository.source.source { + repositories::sources::Source::Path(path_source) => { + println!( + " {}{}", + "source path ./".bright_black(), + path_source.path.to_string_lossy().bright_black() + ); + } + repositories::sources::Source::Git(git_source) => { + println!( + " {}{}", + "source git ".bright_black(), + git_source.url.bright_black() + ); + } + } + } + } } Ok(()) } diff --git a/crates/rugpi-bakery/src/repositories.rs b/crates/rugpi-bakery/src/repositories.rs new file mode 100644 index 0000000..121f697 --- /dev/null +++ b/crates/rugpi-bakery/src/repositories.rs @@ -0,0 +1,188 @@ +//! *Repositories* provide recipes, collections, and layers. +//! +//! Generally, a repository is a directory with the following structure: +//! +//! - `rugpi-repository.toml`: Configuration file of the repository (required). +//! - `recipes`: Directory containing recipes (optional). +//! - `collections`: Directory containing collections (optional). +//! - `layers`: Directory containing layers (optional). +//! +//! As an exception, a project's root directory is also treated as a repository, however, +//! in this case, the repository configuration file is not required/used. +//! Instead, the configuration is synthesized from `rugpi-bakery.toml`. +//! +//! ## Sources +//! +//! Repositories can be sourced from different *[sources]*: +//! +//! - [`sources::GitSource`]: Repository sourced from a Git repository. +//! - [`sources::PathSource`]: Repository sourced from a local path. +//! +//! Sources have to be *materialized* into a local directory before they can be used. + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; + +use anyhow::{bail, Context}; +use rugpi_common::Anyhow; +use serde::{Deserialize, Serialize}; + +use self::sources::{MaterializedSource, PathSource, SourceId}; +use crate::repositories::sources::Source; + +pub mod sources; + +/// A collection of repositories. +#[derive(Debug)] +pub struct Repositories { + /// The repositories of the collection. + repositories: Vec>, + /// Table for finding repositories by their source. + source_to_repository: HashMap, + /// Path to the project's root directory. + root_dir: PathBuf, +} + +impl Repositories { + /// Create an empty collection of repositories. + pub fn new(root_dir: impl AsRef) -> Self { + let root_dir = root_dir.as_ref(); + Self { + repositories: Vec::new(), + source_to_repository: HashMap::new(), + root_dir: root_dir.to_path_buf(), + } + } + + /// Iterator over the loaded repositories. + pub fn iter(&self) -> impl Iterator { + self.repositories + .iter() + .enumerate() + .map(|(idx, repository)| { + ( + RepositoryId(idx), + repository + .as_ref() + .expect("repository has not been fully loaded yet"), + ) + }) + } + + /// Load the repository from the project's root directory. + /// + /// The *update* flag indicates whether remote repositories should be updated. + pub fn load_root( + &mut self, + repositories: HashMap, + update: bool, + ) -> Anyhow { + self.load_repository( + Source::Path(PathSource { path: "".into() }).materialize(&self.root_dir, update)?, + RepositoryConfig { + name: Some("root".to_owned()), + description: None, + repositories, + }, + update, + ) + } + + /// Load a repository from the given source and return its id. + /// + /// The *update* flag indicates whether remote repositories should be updated. + pub fn load_source(&mut self, source: Source, update: bool) -> Anyhow { + let source_id = source.id(); + if let Some(id) = self.source_to_repository.get(&source_id).cloned() { + let Some(repository) = &self.repositories[id.0] else { + bail!("cycle while loading repository from:\n{:?}", source); + }; + if repository.source.source == source { + Ok(id) + } else { + bail!( + "incompatible repository sources:\n{:?}\n{:?}", + repository.source.source, + source, + ); + } + } else { + let source = source.materialize(&self.root_dir, update)?; + let config_path = source.dir.join("rugpi-repository.toml"); + let config = + toml::from_str(&std::fs::read_to_string(&config_path).with_context(|| { + format!("reading repository configuration from {config_path:?}") + })?)?; + self.load_repository(source, config, update) + } + } + + /// Load a repository from an already materialized source and given config. + fn load_repository( + &mut self, + source: MaterializedSource, + config: RepositoryConfig, + update: bool, + ) -> Anyhow { + if self.source_to_repository.contains_key(&source.id) { + bail!("repository from {} has already been loaded", source.id); + } + eprintln!("=> loading repository from source {}", source.id); + let id = RepositoryId(self.repositories.len()); + self.repositories.push(None); + self.source_to_repository.insert(source.id.clone(), id); + let mut repositories = HashMap::new(); + for (name, source) in &config.repositories { + repositories.insert(name.clone(), self.load_source(source.clone(), update)?); + } + let repository = Repository { + id, + source, + config, + repositories, + }; + self.repositories[id.0] = Some(repository); + Ok(id) + } +} + +impl std::ops::Index for Repositories { + type Output = Repository; + + fn index(&self, index: RepositoryId) -> &Self::Output { + self.repositories[index.0] + .as_ref() + .expect("repository has not been fully loaded yet") + } +} + +/// Uniquely identifies a repository. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct RepositoryId(usize); + +/// A repository. +#[derive(Debug)] +pub struct Repository { + /// The id of the repository. + pub id: RepositoryId, + /// The source of the repository. + pub source: MaterializedSource, + /// The configuration of the repository. + pub config: RepositoryConfig, + /// The repositories used by the repository. + pub repositories: HashMap, +} + +/// Repository configuration. +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct RepositoryConfig { + /// An optional name of the repository. + pub name: Option, + /// An optional description of the repository. + pub description: Option, + /// The repositories used by the repository. + #[serde(default)] + pub repositories: HashMap, +} diff --git a/crates/rugpi-bakery/src/repositories/sources.rs b/crates/rugpi-bakery/src/repositories/sources.rs new file mode 100644 index 0000000..f9a24dd --- /dev/null +++ b/crates/rugpi-bakery/src/repositories/sources.rs @@ -0,0 +1,162 @@ +//! Repository sources, e.g., a local path or a Git repository. + +use std::{ + fmt::Display, + os::unix::ffi::OsStrExt, + path::{Path, PathBuf}, + sync::Arc, +}; + +use rugpi_common::Anyhow; +use serde::{Deserialize, Serialize}; +use sha1::{Digest, Sha1}; +use xscript::{read_str, run, LocalEnv, Run}; + +/// A source which has been materialized in a local directory. +#[derive(Debug, Clone)] +pub struct MaterializedSource { + /// The id of the source. + pub id: SourceId, + /// The definition of the source. + pub source: Source, + /// The directory where the source has been materialized. + pub dir: PathBuf, +} + +/// Globally unique id of a source. +/// +/// The id is computed by hashing the path or URL of a source. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct SourceId(Arc); + +impl SourceId { + /// The string representation of the id. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// The short string representation of the id. + pub fn as_short_str(&self) -> &str { + &self.as_str()[..6] + } +} + +impl Display for SourceId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// The source of a repository. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum Source { + /// The repository is sourced from a local path. + Path(PathSource), + /// The repository is sourced from a Git repository. + Git(GitSource), +} + +impl Source { + /// The globally unique id of the source. + pub fn id(&self) -> SourceId { + let mut hasher = Sha1::new(); + match self { + Source::Path(path_source) => { + hasher.update(b"path"); + hasher.update(path_source.path.as_os_str().as_bytes()); + } + Source::Git(git_source) => { + hasher.update(b"git"); + hasher.update(git_source.url.as_bytes()); + if let Some(inner_path) = &git_source.dir { + hasher.update(inner_path.as_os_str().as_bytes()); + } + } + } + SourceId(hex::encode(&hasher.finalize()[..]).into()) + } + + /// Materialize the source within the given project root directory. + /// + /// The *update* flag indicates whether remote repositories should be updated. + pub fn materialize(self, root_dir: &Path, update: bool) -> Anyhow { + let id = self.id(); + eprintln!("=> materializing source {id}"); + let path = match &self { + Source::Path(path_source) => root_dir.join(&path_source.path), + Source::Git(git_source) => { + let mut path = root_dir.join(".rugpi/repositories"); + path.push(id.as_str()); + git_source.checkout(&path, update)?; + if let Some(repository_path) = &git_source.dir { + path.push(repository_path); + } + path + } + }; + Ok(MaterializedSource { + id, + source: self, + dir: path, + }) + } +} + +/// A repository sourced from a local path. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PathSource { + /// The path relative to the project's root directory. + pub path: PathBuf, +} + +/// A repository sourced from a Git repository. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct GitSource { + /// The URL of the Git repository. + #[serde(rename = "git")] + pub url: String, + /// Specifies the branch to use. + pub branch: Option, + /// Specifies the tag to use. + pub tag: Option, + /// Specifies the revision to use. + pub rev: Option, + /// The directory of the repository in the Git repository. + pub dir: Option, +} + +impl GitSource { + /// Checkout the Git repository in the given directory. + /// + /// The *fetch* flag indicates whether updates should be fetched from the remote. + fn checkout(&self, path: &Path, fetch: bool) -> Anyhow<()> { + if !path.exists() { + run!(["git", "clone", &self.url, path.to_string_lossy()])?; + } + let env = LocalEnv::new(&path); + if fetch { + run!(env, ["git", "fetch", "--all"])?; + } + macro_rules! rev_parse { + ($rev:literal) => { + read_str!(env, ["git", "rev-parse", "--verify", $rev]) + }; + } + let mut commit = rev_parse!("refs/remotes/origin/HEAD^{{commit}}")?; + if let Some(tag) = &self.tag { + commit = rev_parse!("refs/tags/{tag}^{{commit}}")?; + } + if let Some(branch) = &self.branch { + commit = rev_parse!("refs/remotes/origin/{branch}^{{commit}}")?; + } + if let Some(rev) = &self.rev { + commit = rev_parse!("{rev}^{{commit}}")?; + } + let head = rev_parse!("HEAD^{{commit}}")?; + if head != commit { + run!(env, ["git", "checkout", commit])?; + } + Ok(()) + } +}