From 7ea57559fd39d404eff59091a5bc2f14c8f327f7 Mon Sep 17 00:00:00 2001 From: lovasoa Date: Tue, 28 May 2024 23:40:46 +0200 Subject: [PATCH] cache frontend dependencies individually locally and warn when re-downloading --- Cargo.lock | 20 +++++++-------- Cargo.toml | 2 +- build.rs | 71 ++++++++++++++++++++++++++++++++++++------------------ 3 files changed, 59 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dea7ef08..e75a1b33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,9 +264,9 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -643,9 +643,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" dependencies = [ "addr2line", "cc", @@ -1466,9 +1466,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "gloo-net" @@ -2107,9 +2107,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" dependencies = [ "memchr", ] @@ -3696,9 +3696,9 @@ checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winnow" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c52e9c97a68071b23e836c9380edae937f17b9c4667bd021973efc689f618d" +checksum = "86c949fede1d13936a99f14fafd3e76fd642b556dd2ce96287fbe2e0151bfac6" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 9dda2925..9746be45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,4 +55,4 @@ awc = { version = "3", features = ["rustls-0_22-webpki-roots"] } awc = { version = "3", features = ["rustls-0_22-webpki-roots"] } actix-rt = "2.8" libflate = "2" -futures-util = "0.3.21" +futures-util = {version = "0.3.21" , features=["io"]} diff --git a/build.rs b/build.rs index e3cd4269..284eec3a 100644 --- a/build.rs +++ b/build.rs @@ -1,5 +1,4 @@ use actix_rt::spawn; -use futures_util::StreamExt; use libflate::gzip; use std::collections::hash_map::DefaultHasher; use std::fs::File; @@ -7,24 +6,34 @@ use std::hash::Hasher; use std::io::Read; use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; +use std::rc::Rc; +use std::time::Duration; #[actix_rt::main] async fn main() { println!("cargo:rerun-if-changed=build.rs"); + let c = Rc::new(make_client()); for h in [ - spawn(download_deps("sqlpage.js")), - spawn(download_deps("sqlpage.css")), - spawn(download_deps("tabler-icons.svg")), - spawn(download_deps("apexcharts.js")), - spawn(download_deps("tomselect.js")), + spawn(download_deps(c.clone(), "sqlpage.js")), + spawn(download_deps(c.clone(), "sqlpage.css")), + spawn(download_deps(c.clone(), "tabler-icons.svg")), + spawn(download_deps(c.clone(), "apexcharts.js")), + spawn(download_deps(c.clone(), "tomselect.js")), ] { h.await.unwrap(); } } +fn make_client() -> awc::Client { + awc::ClientBuilder::new() + .timeout(Duration::from_secs(10)) + .no_default_headers() + .finish() +} + /// Creates a file with inlined remote files included -async fn download_deps(filename: &str) { +async fn download_deps(client: Rc, filename: &str) { let path_in = format!("sqlpage/{}", filename); let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); let path_out: PathBuf = out_dir.join(filename); @@ -33,7 +42,7 @@ async fn download_deps(filename: &str) { // the URL in the generated file. println!("cargo:rerun-if-changed={}", path_in); let original = File::open(path_in).unwrap(); - process_input_file(&path_out, original).await; + process_input_file(&client, &path_out, original).await; std::fs::write( format!("{}.filename.txt", path_out.display()), hashed_filename(&path_out), @@ -41,12 +50,7 @@ async fn download_deps(filename: &str) { .unwrap(); } -async fn process_input_file(path_out: &Path, original: File) { - let client = awc::ClientBuilder::new() - .timeout(core::time::Duration::from_secs(3)) - .max_http_version(awc::http::Version::HTTP_11) - .no_default_headers() - .finish(); +async fn process_input_file(client: &awc::Client, path_out: &Path, original: File) { let mut outfile = gzip::Encoder::new(File::create(path_out).unwrap()).unwrap(); for l in BufReader::new(original).lines() { let line = l.unwrap(); @@ -54,7 +58,7 @@ async fn process_input_file(path_out: &Path, original: File) { let url = line .trim_start_matches("/* !include ") .trim_end_matches(" */"); - download_url_to_opened_file(&client, url, &mut outfile).await; + copy_url_to_opened_file(client, url, &mut outfile).await; outfile.write_all(b"\n").unwrap(); } else { writeln!(outfile, "{}", line).unwrap(); @@ -66,27 +70,43 @@ async fn process_input_file(path_out: &Path, original: File) { .expect("Unable to write compressed frontend asset"); } -async fn download_url_to_opened_file( +async fn copy_url_to_opened_file( client: &awc::Client, url: &str, outfile: &mut impl std::io::Write, ) { + // If the file has been downloaded manually, use it + let cached_file_path = make_url_path(url); + if !cached_file_path.exists() { + println!("cargo:warning=Downloading {url} to cache file {cached_file_path:?}."); + download_url_to_path(client, url, &cached_file_path).await; + println!("cargo:rerun-if-changed={}", cached_file_path.display()); + } + copy_cached_to_opened_file(&cached_file_path, outfile); +} + +fn copy_cached_to_opened_file(source: &Path, outfile: &mut impl std::io::Write) { + let reader = std::fs::File::open(source).unwrap(); + let mut buf = std::io::BufReader::new(reader); + // Not async, but performance should not really matter here + std::io::copy(&mut buf, outfile).unwrap(); +} + +async fn download_url_to_path(client: &awc::Client, url: &str, path: &Path) { let mut resp = client.get(url).send().await.unwrap_or_else(|err| { + let path = make_url_path(url); panic!( "We need to download external frontend dependencies to build the static frontend. \ - Could not download {url} \ + Could not download {url}. You can manually download the file and place it in {path:?}\ {err}" ) }); if resp.status() != 200 { panic!("Received {} status code from {}", resp.status(), url); } - while let Some(b) = resp.next().await { - let chunk = b.unwrap_or_else(|err| panic!("Failed to read data from {url}: {err}")); - outfile - .write_all(&chunk) - .expect("Failed to write external frontend dependency to local file"); - } + let bytes = resp.body().limit(128 * 1024 * 1024).await.unwrap(); + std::fs::write(path, &bytes) + .expect("Failed to write external frontend dependency to local file"); } // Given a filename, creates a new unique filename based on the file contents @@ -111,3 +131,8 @@ fn hashed_filename(path: &Path) -> String { path.extension().unwrap().to_str().unwrap() ) } + +fn make_url_path(url: &str) -> PathBuf { + let filename = url.replace(|c: char| !c.is_ascii_alphanumeric() && c != '.', "_"); + Path::new(&std::env::var("OUT_DIR").unwrap()).join(filename) +}