From bd2a81781c2fd16dccc715ad74780984a719b7fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20K=C3=B6hl?= Date: Sun, 28 Jan 2024 18:10:37 +0100 Subject: [PATCH] chore: cleanup and better logging --- .vscode/cspell.json | 1 + Cargo.lock | 436 ++++++++++++++++++ crates/rugpi-bakery/Cargo.toml | 2 + crates/rugpi-bakery/src/bake/customize.rs | 34 +- crates/rugpi-bakery/src/bake/image.rs | 21 +- crates/rugpi-bakery/src/bake/mod.rs | 98 ++-- crates/rugpi-bakery/src/main.rs | 87 ++-- crates/rugpi-bakery/src/project/config.rs | 7 + crates/rugpi-bakery/src/project/layers.rs | 4 +- crates/rugpi-bakery/src/project/library.rs | 4 +- crates/rugpi-bakery/src/project/recipes.rs | 3 +- .../rugpi-bakery/src/project/repositories.rs | 12 +- crates/rugpi-bakery/src/utils/caching.rs | 59 ++- crates/rugpi-bakery/src/utils/logging.rs | 11 +- crates/rugpi-bakery/src/utils/prelude.rs | 2 + 15 files changed, 665 insertions(+), 116 deletions(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index d73f59d..f4062eb 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -25,6 +25,7 @@ "github", "GPIO", "Imager", + "indicatif", "linux", "losetup", "lowerdir", diff --git a/Cargo.lock b/Cargo.lock index fd5076b..e8a9611 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,19 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "async-compression" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-trait" version = "0.1.74" @@ -153,6 +166,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "bitflags" version = "1.3.2" @@ -174,6 +193,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + [[package]] name = "bytes" version = "1.5.0" @@ -251,6 +276,35 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.52.0", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + [[package]] name = "cpufeatures" version = "0.2.11" @@ -289,6 +343,12 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.33" @@ -320,6 +380,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -350,6 +420,18 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + [[package]] name = "futures-task" version = "0.3.29" @@ -363,9 +445,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-core", + "futures-io", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -378,12 +463,42 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "gimli" version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "h2" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.14.3" @@ -452,6 +567,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2", "http", "http-body", "httparse", @@ -465,6 +581,20 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "idna" version = "0.5.0" @@ -485,18 +615,55 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "indoc" version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "itoa" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "js-sys" +version = "0.3.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -640,6 +807,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.32.1" @@ -722,6 +895,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + [[package]] name = "proc-macro2" version = "1.0.70" @@ -749,6 +928,62 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "reqwest" +version = "0.11.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +dependencies = [ + "async-compression", + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "system-configuration", + "tokio", + "tokio-rustls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.48.0", +] + [[package]] name = "rugpi-admin" version = "0.1.0" @@ -769,7 +1004,9 @@ dependencies = [ "clap", "colored", "hex", + "indicatif", "nix 0.27.1", + "reqwest", "rugpi-common", "serde", "sha1", @@ -829,6 +1066,37 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls" +version = "0.21.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.14" @@ -847,6 +1115,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "serde" version = "1.0.193" @@ -938,6 +1216,15 @@ dependencies = [ "libc", ] +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.11.2" @@ -993,6 +1280,27 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.8.1" @@ -1081,6 +1389,30 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.7.8" @@ -1259,6 +1591,18 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.0" @@ -1304,6 +1648,88 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" + +[[package]] +name = "web-sys" +version = "0.3.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" + [[package]] name = "winapi" version = "0.3.9" @@ -1467,6 +1893,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "xscript" version = "0.2.0" diff --git a/crates/rugpi-bakery/Cargo.toml b/crates/rugpi-bakery/Cargo.toml index a289073..81366ff 100644 --- a/crates/rugpi-bakery/Cargo.toml +++ b/crates/rugpi-bakery/Cargo.toml @@ -12,7 +12,9 @@ anyhow = "1.0.71" clap = { version = "4.3.8", features = ["derive"] } colored = "2.1.0" hex = "0.4.3" +indicatif = "0.17.7" nix = { version = "0.27.1", features = ["process"] } +reqwest = { version = "0.11.23", features = ["blocking", "rustls-tls", "gzip", "deflate"], default-features = false } rugpi-common = { path = "../rugpi-common" } serde = { version = "1.0.171", features = ["derive", "rc"] } sha1 = "0.10.5" diff --git a/crates/rugpi-bakery/src/bake/customize.rs b/crates/rugpi-bakery/src/bake/customize.rs index 11b4b73..a78fe38 100644 --- a/crates/rugpi-bakery/src/bake/customize.rs +++ b/crates/rugpi-bakery/src/bake/customize.rs @@ -8,9 +8,7 @@ use std::{ sync::Arc, }; -use anyhow::{anyhow, bail}; -use clap::Parser; -use rugpi_common::{mount::Mounted, Anyhow}; +use rugpi_common::mount::Mounted; use tempfile::tempdir; use xscript::{cmd, run, vars, ParentEnv, Run}; @@ -23,18 +21,12 @@ use crate::{ repositories::RepositoryIdx, Project, }, - utils::caching::{mtime, mtime_recursive}, + utils::{ + caching::{mtime, mtime_recursive}, + prelude::*, + }, }; -/// The arguments of the `customize` command. -#[derive(Debug, Parser)] -pub struct CustomizeTask { - /// The source archive with the original system. - src: String, - /// The destination archive with the modified system. - dest: String, -} - pub fn customize( project: &Project, arch: Architecture, @@ -47,12 +39,16 @@ pub fn customize( // Collect the recipes to apply. let config = layer.config(arch).unwrap(); let jobs = recipe_schedule(layer.repo, config, library)?; + if jobs.is_empty() { + bail!("layer must have recipes") + } let mut last_modified = jobs .iter() .map(|job| job.recipe.modified) .max() .unwrap() - .max(layer.modified); + .max(layer.modified) + .max(mtime(src)?); let mut force_run = false; let used_files = project.dir.join(layer_path.join("rebuild-if-changed.txt")); if used_files.exists() { @@ -60,7 +56,7 @@ pub fn customize( if let Ok(modified) = mtime_recursive(&project.dir.join(line)) { last_modified = last_modified.max(modified) } else { - eprintln!("Error determining modification time for {line}."); + error!("error determining modification time for {line}"); force_run = true; } } @@ -71,10 +67,10 @@ pub fn customize( // Prepare system chroot. let root_dir = tempdir()?; let root_dir_path = root_dir.path(); - println!("Extracting system files..."); + info!("extracting system files"); run!(["tar", "-x", "-f", &src, "-C", root_dir_path])?; apply_recipes(project, arch, &jobs, root_dir_path, layer_path)?; - println!("Packing system files..."); + info!("packing system files"); run!(["tar", "-c", "-f", &target, "-C", root_dir_path, "."])?; Ok(()) } @@ -190,7 +186,7 @@ fn apply_recipes( for (idx, job) in jobs.iter().enumerate() { let recipe = &job.recipe; - println!( + info!( "[{:>2}/{}] {} {:?}", idx + 1, jobs.len(), @@ -204,7 +200,7 @@ fn apply_recipes( let _mounted_recipe = Mounted::bind(&recipe.path, &bakery_recipe_path)?; for step in &recipe.steps { - println!(" - {}", step.filename); + info!(" - {}", step.filename); match &step.kind { StepKind::Packages { packages } => { let mut cmd = cmd!("chroot", root_dir_path, "apt-get", "install", "-y"); diff --git a/crates/rugpi-bakery/src/bake/image.rs b/crates/rugpi-bakery/src/bake/image.rs index 68f5785..8ebe76d 100644 --- a/crates/rugpi-bakery/src/bake/image.rs +++ b/crates/rugpi-bakery/src/bake/image.rs @@ -12,21 +12,23 @@ use rugpi_common::{ use tempfile::tempdir; use xscript::{run, Run}; -use crate::project::{ - config::{Architecture, IncludeFirmware}, - images::ImageConfig, +use crate::{ + project::{ + config::{Architecture, IncludeFirmware}, + images::ImageConfig, + }, + utils::prelude::*, }; pub fn make_image(image_config: &ImageConfig, src: &Path, image: &Path) -> Anyhow<()> { let size = calculate_image_size(src)?; - println!("Size: {} bytes", size); fs::remove_file(image).ok(); - println!("Creating image..."); + info!("creating image (size: {} bytes)", size); run!(["fallocate", "-l", "{size}", image])?; sfdisk_apply_layout(image, sfdisk_image_layout())?; let disk_id = get_disk_id(image)?; let loop_device = LoopDevice::attach(image)?; - println!("Creating file systems..."); + info!("creating filesystems"); mkfs_vfat(loop_device.partition(1), "CONFIG")?; mkfs_vfat(loop_device.partition(2), "BOOT-A")?; mkfs_ext4(loop_device.partition(5), "system-a")?; @@ -52,9 +54,9 @@ pub fn make_image(image_config: &ImageConfig, src: &Path, image: &Path) -> Anyho }; run!(["tar", "-x", "-f", src, "-C", root_dir_path])?; - println!("Patching boot configuration..."); + info!("patching boot configuration"); patch_boot(ctx.mounted_boot.path(), format!("PARTUUID={disk_id}-05"))?; - println!("Patching `config.txt`..."); + info!("patching `config.txt`"); patch_config(boot_dir.join("config.txt"))?; match image_config.boot_flow { @@ -73,9 +75,6 @@ pub fn make_image(image_config: &ImageConfig, src: &Path, image: &Path) -> Anyho IncludeFirmware::Pi5 => include_pi5_firmware(ctx.mounted_config.path())?, } } - // Shrink root filesystem. - // run!(["resize2fs", "-M", loop_device.partition(5)])?; - // run!(["dumpe2fs", "-h", loop_device.partition(5)])?; Ok(()) } diff --git a/crates/rugpi-bakery/src/bake/mod.rs b/crates/rugpi-bakery/src/bake/mod.rs index e5b1a81..e9c91d3 100644 --- a/crates/rugpi-bakery/src/bake/mod.rs +++ b/crates/rugpi-bakery/src/bake/mod.rs @@ -11,8 +11,11 @@ use tempfile::tempdir; use xscript::{run, Run}; use crate::{ - project::{config::Architecture, Project}, - utils::caching::{download, sha1}, + project::{config::Architecture, library::LayerIdx, Project}, + utils::{ + caching::{download, Hasher}, + prelude::*, + }, }; pub mod customize; @@ -24,40 +27,67 @@ pub fn bake_image(project: &Project, image: &str, output: &Path) -> Anyhow<()> { .images .get(image) .ok_or_else(|| anyhow!("unable to find image {image}"))?; - let baked_layer = bake_layer(project, image_config.architecture, &image_config.layer)?; + let layer_bakery = LayerBakery::new(project, image_config.architecture); + let baked_layer = layer_bakery.bake_root(&image_config.layer)?; image::make_image(image_config, &baked_layer, output) } -pub fn bake_layer(project: &Project, arch: Architecture, layer_name: &str) -> Anyhow { - let library = project.library()?; - let layer = &library.layers[library - .lookup_layer(library.repositories.root_repository, layer_name) - .unwrap()]; - let layer_config = layer.config(arch).unwrap(); - if let Some(url) = &layer_config.url { - let layer_id = sha1(url); - let system_tar = project - .dir - .join(format!(".rugpi/layers/{layer_id}/system.tar")); - if !system_tar.exists() { - extract(url, &system_tar)?; +pub struct LayerBakery<'p> { + project: &'p Project, + arch: Architecture, +} + +impl<'p> LayerBakery<'p> { + pub fn new(project: &'p Project, arch: Architecture) -> Self { + Self { project, arch } + } + + pub fn bake_root(&self, layer: &str) -> Anyhow { + let library = self.project.library()?; + let Some(layer) = library.lookup_layer(library.repositories.root_repository, layer) else { + bail!("unable to find layer {layer}"); + }; + self.bake(layer) + } + + pub fn bake(&self, layer: LayerIdx) -> Anyhow { + let repositories = &self.project.repositories()?.repositories; + let library = self.project.library()?; + let layer = &library.layers[layer]; + info!("baking layer `{}`", layer.name); + let Some(config) = layer.config(self.arch) else { + bail!("no layer configuration for architecture `{}`", self.arch); + }; + let mut layer_id = Hasher::new(); + layer_id.push("layer", &layer.name); + layer_id.push("repository", repositories[layer.repo].source.id.as_str()); + layer_id.push("arch", self.arch.as_str()); + if let Some(url) = &config.url { + layer_id.push("url", url); + let layer_id = layer_id.finalize(); + let system_tar = self + .project + .dir + .join(format!(".rugpi/layers/{layer_id}/system.tar")); + if !system_tar.exists() { + extract(url, &system_tar)?; + } + Ok(system_tar) + } else if let Some(parent) = &config.parent { + layer_id.push("parent", parent); + let Some(parent) = library.lookup_layer(layer.repo, parent) else { + bail!("unable to find layer `{parent}`"); + }; + let src = self.bake(parent)?; + let layer_id = layer_id.finalize(); + let layer_path = PathBuf::from(format!(".rugpi/layers/{layer_id}")); + let target = self.project.dir.join(&layer_path).join("system.tar"); + fs::create_dir_all(target.parent().unwrap()).ok(); + customize::customize(self.project, self.arch, layer, &src, &target, &layer_path)?; + Ok(target) + } else { + bail!("invalid layer configuration") } - Ok(system_tar) - } else if let Some(parent) = &layer_config.parent { - let src = bake_layer(project, arch, parent)?; - let mut layer_string = layer_name.to_owned(); - layer_string.push('.'); - layer_string.push_str(arch.as_str()); - layer_string.push('.'); - layer_string.push_str(library.repositories[layer.repo].source.id.as_str()); - let layer_id = sha1(&layer_string); - let layer_path = PathBuf::from(format!(".rugpi/layers/{layer_id}")); - let target = project.dir.join(&layer_path).join("system.tar"); - fs::create_dir_all(target.parent().unwrap()).ok(); - customize::customize(project, arch, layer, &src, &target, &layer_path)?; - Ok(target) - } else { - bail!("invalid layer configuration") } } @@ -66,12 +96,12 @@ fn extract(image_url: &str, layer_path: &Path) -> Anyhow<()> { if image_path.extension() == Some("xz".as_ref()) { let decompressed_image_path = image_path.with_extension(""); if !decompressed_image_path.is_file() { - eprintln!("Decompressing XZ image..."); + info!("decompressing XZ image"); run!(["xz", "-d", "-k", image_path])?; } image_path = decompressed_image_path; } - eprintln!("Creating `.tar` archive with system files..."); + info!("creating `.tar` archive with system files"); if let Some(parent) = layer_path.parent() { if !parent.exists() { fs::create_dir_all(parent)?; diff --git a/crates/rugpi-bakery/src/main.rs b/crates/rugpi-bakery/src/main.rs index 27d74ec..b8e3050 100644 --- a/crates/rugpi-bakery/src/main.rs +++ b/crates/rugpi-bakery/src/main.rs @@ -1,11 +1,13 @@ use std::{ + convert::Infallible, ffi::{CStr, CString}, path::PathBuf, }; +use bake::LayerBakery; use clap::Parser; use colored::Colorize; -use project::{config::Architecture, repositories::Source, ProjectLoader}; +use project::{config::Architecture, repositories::Source, Project, ProjectLoader}; use rugpi_common::Anyhow; use utils::logging::init_logging; @@ -13,77 +15,83 @@ pub mod bake; pub mod project; pub mod utils; +/// Command line arguments. #[derive(Debug, Parser)] +#[command(author, about = None, long_about = None)] pub struct Args { - /// Path to `rugpi-bakery.toml` configuration file. + /// Path to the `rugpi-bakery.toml` configuration file. #[clap(long)] config: Option, - /// The task to execute. + /// The command to execute. #[clap(subcommand)] - task: Task, + command: Command, } +/// Commands of the CLI. +#[derive(Debug, Parser)] +pub enum Command { + /// Bake an image or a layer. + #[clap(subcommand)] + Bake(BakeCommand), + /// Spawn a shell in the Rugpi Bakery Docker container. + Shell, + /// Pull in external repositories. + Pull, + /// Update Rugpi Bakery itself. + Update(UpdateCommand), +} + +/// The `bake` command. #[derive(Debug, Parser)] pub enum BakeCommand { + /// Bake an image. Image { + /// The name of the image to bake. image: String, + /// The output path of the resulting image. output: PathBuf, }, + /// Bake a layer. Layer { + /// The architecture to bake the layer for. #[clap(long)] arch: Architecture, + /// The name of the layer to bake. layer: String, }, } +/// The `update` command. #[derive(Debug, Parser)] -pub enum Task { - // /// Extract all system files from a given base image. - // Extract(ExtractTask), - // /// Apply modification to the system. - // Customize(CustomizeTask), - /// Bake a final image for distribution. - #[clap(subcommand)] - Bake(BakeCommand), - /// Spawn a shell in the Rugpi Bakery Docker container. - Shell, - Update(UpdateTask), - /// Pull in external repositories. - Pull, -} - -#[derive(Debug, Parser)] -pub struct UpdateTask { +pub struct UpdateCommand { + /// The version to update to. version: Option, } +/// Entrypoint of the CLI. fn main() -> Anyhow<()> { - let args = Args::parse(); - init_logging(); - let project = ProjectLoader::current_dir()? - .with_config_file(args.config.as_deref()) - .load()?; - match &args.task { - Task::Bake(task) => match task { + let args = Args::parse(); + let project = load_project(&args)?; + match &args.command { + Command::Bake(command) => match command { BakeCommand::Image { image, output } => { bake::bake_image(&project, image, output)?; } BakeCommand::Layer { layer, arch } => { - bake::bake_layer(&project, *arch, layer)?; + LayerBakery::new(&project, *arch).bake_root(layer)?; } }, - Task::Shell => { - let zsh_prog = CString::new("/bin/zsh")?; - nix::unistd::execv::<&CStr>(&zsh_prog, &[])?; + Command::Shell => { + exec_shell()?; } - Task::Update(task) => { + Command::Update(task) => { let version = task.version.as_deref().unwrap_or("v0"); println!("Switch Rugpi Bakery to version `{version}`..."); std::fs::write("run-bakery", interpolate_run_bakery(version))?; } - Task::Pull => { + Command::Pull => { for (_, repository) in project.repositories()?.iter() { println!( "{} {} {}", @@ -121,3 +129,14 @@ fn main() -> Anyhow<()> { fn interpolate_run_bakery(version: &str) -> String { include_str!("../assets/run-bakery").replace("%%DEFAULT_VERSION%%", version) } + +fn load_project(args: &Args) -> Anyhow { + ProjectLoader::current_dir()? + .with_config_file(args.config.as_deref()) + .load() +} + +fn exec_shell() -> Anyhow { + let zsh = CString::new("/bin/zsh").unwrap(); + Ok(nix::unistd::execv::<&CStr>(&zsh, &[])?) +} diff --git a/crates/rugpi-bakery/src/project/config.rs b/crates/rugpi-bakery/src/project/config.rs index b3c29f4..b43fe18 100644 --- a/crates/rugpi-bakery/src/project/config.rs +++ b/crates/rugpi-bakery/src/project/config.rs @@ -1,5 +1,6 @@ //! Project configuration. +use core::fmt; use std::{collections::HashMap, fs, path::Path, str::FromStr}; use anyhow::Context; @@ -74,3 +75,9 @@ impl Architecture { } } } + +impl fmt::Display for Architecture { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} diff --git a/crates/rugpi-bakery/src/project/layers.rs b/crates/rugpi-bakery/src/project/layers.rs index 1e5a39a..262046e 100644 --- a/crates/rugpi-bakery/src/project/layers.rs +++ b/crates/rugpi-bakery/src/project/layers.rs @@ -46,6 +46,7 @@ impl LayerConfig { #[derive(Debug)] pub struct Layer { + pub name: String, pub repo: RepositoryIdx, pub modified: ModificationTime, pub default_config: Option, @@ -53,8 +54,9 @@ pub struct Layer { } impl Layer { - pub fn new(repo: RepositoryIdx, modified: ModificationTime) -> Self { + pub fn new(name: String, repo: RepositoryIdx, modified: ModificationTime) -> Self { Self { + name, repo, modified, default_config: None, diff --git a/crates/rugpi-bakery/src/project/library.rs b/crates/rugpi-bakery/src/project/library.rs index f4a3c2c..146d751 100644 --- a/crates/rugpi-bakery/src/project/library.rs +++ b/crates/rugpi-bakery/src/project/library.rs @@ -66,8 +66,8 @@ impl Library { let modified = mtime(&path)?; let layer_config = LayerConfig::load(&path)?; let layer_idx = *table - .entry(name) - .or_insert_with(|| layers.push(Layer::new(repo, modified))); + .entry(name.clone()) + .or_insert_with(|| layers.push(Layer::new(name, repo, modified))); layers[layer_idx].modified = layers[layer_idx].modified.max(modified); match arch { Some(arch) => { diff --git a/crates/rugpi-bakery/src/project/recipes.rs b/crates/rugpi-bakery/src/project/recipes.rs index c877bbb..f9e9273 100644 --- a/crates/rugpi-bakery/src/project/recipes.rs +++ b/crates/rugpi-bakery/src/project/recipes.rs @@ -11,6 +11,7 @@ use std::{ use anyhow::{anyhow, bail, Context}; use rugpi_common::Anyhow; use serde::{Deserialize, Serialize}; +use tracing::warn; use super::repositories::RepositoryIdx; use crate::utils::caching::{mtime_recursive, ModificationTime}; @@ -70,7 +71,7 @@ impl RecipeLoader { path, }; if recipe.info.default.is_some() { - eprintln!("default recipes have been deprecated"); + warn!("default recipes have been deprecated"); } Ok(recipe) } diff --git a/crates/rugpi-bakery/src/project/repositories.rs b/crates/rugpi-bakery/src/project/repositories.rs index 9629b87..3521ded 100644 --- a/crates/rugpi-bakery/src/project/repositories.rs +++ b/crates/rugpi-bakery/src/project/repositories.rs @@ -28,14 +28,16 @@ use std::{ sync::Arc, }; -use anyhow::{bail, Context}; -use rugpi_common::Anyhow; +use anyhow::Context; use serde::{Deserialize, Serialize}; use sha1::{Digest, Sha1}; use xscript::{read_str, run, LocalEnv, Run}; use super::Project; -use crate::utils::idx_vec::{new_idx_type, IdxVec}; +use crate::utils::{ + idx_vec::{new_idx_type, IdxVec}, + prelude::*, +}; #[derive(Debug)] #[non_exhaustive] @@ -156,7 +158,7 @@ impl RepositoriesLoader { if self.source_to_repository.contains_key(&source.id) { bail!("repository from {} has already been loaded", source.id); } - eprintln!("=> loading repository from source {}", source.id); + debug!("loading repository from source {}", source.id); let idx = RepositoryIdx(self.repositories.len()); self.repositories.push(None); self.source_to_repository.insert(source.id.clone(), idx); @@ -275,7 +277,7 @@ impl Source { /// The *update* flag indicates whether remote repositories should be updated. pub fn materialize(self, root_dir: &Path, update: bool) -> Anyhow { let id = self.id(); - eprintln!("=> materializing source {id}"); + debug!("materializing source {id}"); let path = match &self { Source::Path(path_source) => root_dir.join(&path_source.path), Source::Git(git_source) => { diff --git a/crates/rugpi-bakery/src/utils/caching.rs b/crates/rugpi-bakery/src/utils/caching.rs index 03f9da6..6b58bd4 100644 --- a/crates/rugpi-bakery/src/utils/caching.rs +++ b/crates/rugpi-bakery/src/utils/caching.rs @@ -1,16 +1,18 @@ //! Utilities for caching. use std::{ - fs, io, + fs, + io::{self, Read, Write}, path::{Path, PathBuf}, time::SystemTime, }; -use rugpi_common::Anyhow; +use indicatif::{ProgressBar, ProgressStyle}; use serde::{Deserialize, Serialize}; use sha1::{Digest, Sha1}; use url::Url; -use xscript::{run, Run}; + +use crate::utils::prelude::*; pub fn download(url: &str) -> Anyhow { let url = url.parse::()?; @@ -28,16 +30,57 @@ pub fn download(url: &str) -> Anyhow { } let cache_file_path = Path::new(".rugpi/cache").join(cache_file_name); if !cache_file_path.exists() { + info!("downloading `{url}`"); std::fs::create_dir_all(".rugpi/cache")?; - run!(["wget", "-O", &cache_file_path, url.as_str()])?; + let mut response = reqwest::blocking::get(url.clone())?; + if response.status().is_success() { + let Some(size) = response.content_length() else { + bail!("server did not send `Content-Length` header"); + }; + let progress = ProgressBar::new(size).with_style( + ProgressStyle::with_template( + "[{elapsed_precise}] {bar:40.cyan/blue} {bytes}/{total_bytes} [{bytes_per_sec}] {msg}", + ) + .unwrap(), + ); + let mut file = fs::File::create(&cache_file_path)?; + let mut buffer = vec![0u8; 8096]; + loop { + let chunk_size = response.read(&mut buffer)?; + if chunk_size > 0 { + file.write(&buffer[..chunk_size])?; + progress.inc(chunk_size as u64); + } else { + break; + } + } + } else { + bail!("error downloading file: {}", response.status()); + } } Ok(cache_file_path) } -pub fn sha1(string: &str) -> String { - let mut hasher = Sha1::new(); - hasher.update(string.as_bytes()); - hex::encode(hasher.finalize()) +#[derive(Debug, Default)] +pub struct Hasher { + hasher: Sha1, +} + +impl Hasher { + pub fn new() -> Self { + Self::default() + } + + pub fn push(&mut self, tag: &str, value: impl AsRef<[u8]>) { + self.hasher.update(tag.as_bytes()); + self.hasher.update(b":"); + self.hasher.update(value.as_ref()); + self.hasher.update(b"\n"); + } + + pub fn finalize(self) -> String { + hex::encode(self.hasher.finalize()) + } } /// Modification time in seconds since the UNIX epoch. diff --git a/crates/rugpi-bakery/src/utils/logging.rs b/crates/rugpi-bakery/src/utils/logging.rs index b039d4e..5b4c0e4 100644 --- a/crates/rugpi-bakery/src/utils/logging.rs +++ b/crates/rugpi-bakery/src/utils/logging.rs @@ -1,5 +1,12 @@ pub fn init_logging() { - tracing_subscriber::fmt::init(); + let format = tracing_subscriber::fmt::format() + .without_time() + .with_target(false) + .compact(); + tracing_subscriber::fmt() + .with_writer(io::stderr) + .event_format(format) + .init(); } #[macro_export] @@ -9,4 +16,6 @@ macro_rules! bug { }; } +use std::io; + pub use bug; diff --git a/crates/rugpi-bakery/src/utils/prelude.rs b/crates/rugpi-bakery/src/utils/prelude.rs index a3085e7..205a107 100644 --- a/crates/rugpi-bakery/src/utils/prelude.rs +++ b/crates/rugpi-bakery/src/utils/prelude.rs @@ -1,5 +1,7 @@ //! Custom prelude. +pub use anyhow::{anyhow, bail}; +pub use rugpi_common::Anyhow; pub use tracing::{debug, error, info, trace, warn}; pub use super::{logging::bug, once_cell_ext::OnceCellExt};