From a721768bb3685cf1a0aabb4fb443215d8b0811a1 Mon Sep 17 00:00:00 2001 From: Fernthedev <15272073+Fernthedev@users.noreply.github.com> Date: Wed, 8 Mar 2023 21:01:29 -0400 Subject: [PATCH 1/3] ureq and reqwest implementation features --- Cargo.lock | 96 ++++++++++++++- Cargo.toml | 8 +- src/commands/publish/mod.rs | 2 +- src/commands/qmod/mod.rs | 2 +- src/main.rs | 8 ++ src/network/github.rs | 24 ++-- src/network/mod.rs | 130 ++++++++++++++++++++- src/network/{agent.rs => reqwest_agent.rs} | 0 src/network/ureq_agent.rs | 81 +++++++++++++ src/repository/qpackages.rs | 52 ++++----- src/utils/android.rs | 13 ++- src/utils/git.rs | 29 ++--- 12 files changed, 374 insertions(+), 71 deletions(-) rename src/network/{agent.rs => reqwest_agent.rs} (100%) create mode 100644 src/network/ureq_agent.rs diff --git a/Cargo.lock b/Cargo.lock index 8e020595..d0b4b58a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,6 +95,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.0" @@ -1585,6 +1591,8 @@ dependencies = [ "stopwatch", "symlink", "templatr", + "thiserror", + "ureq", "vergen", "walkdir", "zip", @@ -1741,7 +1749,7 @@ version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" dependencies = [ - "base64", + "base64 0.21.0", "bytes", "encoding_rs", "futures-core", @@ -1772,6 +1780,21 @@ dependencies = [ "winreg", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "rustc-demangle" version = "0.1.21" @@ -1813,6 +1836,18 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "rustls" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "rustversion" version = "1.0.11" @@ -1849,6 +1884,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "secret-service" version = "2.0.2" @@ -2030,6 +2075,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "static_assertions" version = "1.1.0" @@ -2358,6 +2409,30 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "ureq" +version = "2.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338b31dd1314f68f3aabf3ed57ab922df95ffcd902476ca7ba3c4ce7b908c46d" +dependencies = [ + "base64 0.13.1", + "flate2", + "log", + "once_cell", + "rustls", + "serde", + "serde_json", + "url", + "webpki", + "webpki-roots", +] + [[package]] name = "url" version = "2.3.1" @@ -2520,6 +2595,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + [[package]] name = "wepoll-ffi" version = "0.1.2" diff --git a/Cargo.toml b/Cargo.toml index febf25cc..e18c4a4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,8 +12,10 @@ name = "qpm-rust" path = "src/main.rs" [features] -default = ["templatr"] +default = ["templatr", "ureq"] templatr = ["dep:templatr"] +reqwest = ["dep:reqwest"] +ureq = ["dep:ureq"] [build-dependencies] vergen = "7" @@ -33,7 +35,8 @@ pbr = "*" #{ git = "https://github.com/a8m/pb.git" } git2 = "0.16" bytes = "*" -reqwest = { version = "0.11", features = ["blocking", "json"] } +reqwest = { version = "0.11", features = ["blocking", "json"], optional = true} +ureq = { version = "2", features = ["json"], optional = true} clap = { version = "4", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" @@ -42,6 +45,7 @@ semver = { version = "1.0", features = ["serde"] } cursed-semver-parser = { git = "https://github.com/raftario/cursed-semver-parser.git", features = [ "serde", ] } +thiserror = "1" pubgrub = "0.2" owo-colors = "3" dirs = "4.0.0" diff --git a/src/commands/publish/mod.rs b/src/commands/publish/mod.rs index 7af1a9ee..f760a2e0 100644 --- a/src/commands/publish/mod.rs +++ b/src/commands/publish/mod.rs @@ -8,7 +8,7 @@ use crate::{ config::get_publish_keyring, package::{PackageConfigExtensions}, }, - repository::{multi::MultiDependencyRepository, qpackages::QPMRepository, Repository}, + repository::{qpackages::QPMRepository, Repository}, terminal::colors::QPMColor, }; diff --git a/src/commands/qmod/mod.rs b/src/commands/qmod/mod.rs index aaada897..4073b803 100644 --- a/src/commands/qmod/mod.rs +++ b/src/commands/qmod/mod.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use clap::{Args, Subcommand}; use color_eyre::{eyre::ensure, Result}; -use qpm_package::models::{dependency::SharedPackageConfig, package::PackageConfig}; +use qpm_package::models::{dependency::SharedPackageConfig}; use qpm_qmod::models::mod_json::ModJson; use semver::Version; diff --git a/src/main.rs b/src/main.rs index ca69871e..45464222 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,6 +25,14 @@ mod benchmark; #[cfg(test)] mod tests; + +#[cfg(all(feature = "reqwest", feature = "ureq"))] +compile_error!("feature \"reqwest\" and feature \"ureq\" cannot be enabled at the same time"); + +#[cfg(not(any(feature = "reqwest", feature = "ureq")))] +compile_error!("feature \"reqwest\" or feature \"ureq\" must be enabled, though not both simultaneously"); + + fn main() -> Result<()> { color_eyre::config::HookBuilder::default() .panic_section(concat!( diff --git a/src/network/github.rs b/src/network/github.rs index 1ba27346..ca58e87e 100644 --- a/src/network/github.rs +++ b/src/network/github.rs @@ -1,7 +1,7 @@ use color_eyre::Result; use serde::{Deserialize, Serialize}; -use super::agent::get_agent; +use super::agent::{self}; const GITHUB_OWNER: &str = "QuestPackageManager"; const GITHUB_REPO: &str = "QPM.CLI"; @@ -46,24 +46,18 @@ pub struct GithubCommitDiffCommitDataResponse { pub message: String, } -pub fn get_github_branch(branch: &str) -> Result { - get_agent() - .get(format!( - "https://api.github.com/repos/{GITHUB_OWNER}/{GITHUB_REPO}/branches/{branch}" - )) - .send()? - .json() +pub fn get_github_branch(branch: &str) -> Result { + agent::get(&format!( + "https://api.github.com/repos/{GITHUB_OWNER}/{GITHUB_REPO}/branches/{branch}" + )) } pub fn get_github_commit_diff( old: &str, new: &str, -) -> Result { - get_agent() - .get(format!( - "https://api.github.com/repos/{GITHUB_OWNER}/{GITHUB_REPO}/compare/{old}...{new}" - )) - .send()? - .json() +) -> Result { + agent::get(&format!( + "https://api.github.com/repos/{GITHUB_OWNER}/{GITHUB_REPO}/compare/{old}...{new}" + )) } pub fn download_github_artifact_url(sha: &str) -> String { diff --git a/src/network/mod.rs b/src/network/mod.rs index c0b7ed38..ce572590 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -1,2 +1,128 @@ -pub mod agent; -pub mod github; \ No newline at end of file +pub mod github; + +#[cfg(feature = "ureq")] +mod ureq_agent; + +#[cfg(feature = "reqwest")] +mod reqwest_agent; + +#[cfg(feature = "ureq")] +pub mod agent { + use std::collections::HashMap; + + use thiserror::Error; + + use serde::de::DeserializeOwned; + + pub use super::ureq_agent::*; + + pub type AgentError = ureq::Error; + + #[derive(Error, Debug)] + + pub enum Error { + #[error("Agent error")] + AgentError(Box), + #[error("IO Error")] + IoError(std::io::Error), + #[error("Unauthorized")] + Unauthorized, + } + + pub fn get(url: &str) -> Result + where + T: DeserializeOwned, + { + let res = match get_agent().get(url).call() { + Ok(o) => Ok(o), + Err(e) => Err(Error::AgentError(Box::new(e))), + }?; + match res.into_json::() { + Ok(o) => Ok(o), + Err(e) => Err(Error::IoError(e)), + } + } + + pub fn get_bytes(url: &str) -> Result, Error> { + Ok(get_str(url)?.into_bytes()) + } + pub fn get_str(url: &str) -> Result { + let res = match get_agent().get(url).call() { + Ok(o) => Ok(o), + Err(e) => Err(Error::AgentError(Box::new(e))), + }?; + match res.into_string() { + Ok(o) => Ok(o), + Err(e) => Err(Error::IoError(e)), + } + } + pub fn post( + url: &str, + data: impl serde::Serialize, + headers: &HashMap<&str, &str>, + ) -> Result + where + T: DeserializeOwned, + { + let mut req = get_agent().post(url); + + for (key, val) in headers { + req = req.set(key, val); + } + + let res = match req.send_json(data) { + Ok(o) => Ok(o), + Err(e) => match e { + ureq::Error::Status(code, _) => { + if code == 401 { + Err(Error::Unauthorized) + } else { + Err(Error::AgentError(Box::new(e))) + } + } + ureq::Error::Transport(_) => Err(Error::AgentError(Box::new(e))), + }, + }?; + + match res.into_json::() { + Ok(o) => Ok(o), + Err(e) => Err(Error::IoError(e)), + } + } + + pub fn get_opt(url: &str) -> Result, Error> + where + T: DeserializeOwned, + { + match get::(url) { + Ok(o) => Ok(Some(o)), + Err(e) => Err(match &e { + Error::AgentError(u) => match u.as_ref() { + ureq::Error::Status(code, _) => { + if *code == 404u16 { + return Ok(None); + } + return Err(e); + } + _ => e, + }, + _ => e, + }), + } + } +} + +#[cfg(feature = "reqwest")] +pub mod agent { + pub use super::reqwest_agent::*; + + pub type Error = reqwest::Error; + pub type AgentError = reqwest::Error; + + fn get(url: &str) -> Result + where + T: DeserializeOwned, + { + get_agent().get(url).send()?.json() + } +} diff --git a/src/network/agent.rs b/src/network/reqwest_agent.rs similarity index 100% rename from src/network/agent.rs rename to src/network/reqwest_agent.rs diff --git a/src/network/ureq_agent.rs b/src/network/ureq_agent.rs new file mode 100644 index 00000000..57b535ec --- /dev/null +++ b/src/network/ureq_agent.rs @@ -0,0 +1,81 @@ +use std::{ + io::{BufReader, Read}, + sync, + time::Duration, +}; + +use color_eyre::{eyre::Context, Result}; +use ureq::{Agent, AgentBuilder}; + +use crate::models::config::get_combine_config; + +static AGENT: sync::OnceLock = sync::OnceLock::new(); + +pub fn get_agent() -> &'static Agent { + AGENT.get_or_init(|| { + AgentBuilder::new() + .timeout(Duration::from_millis( + get_combine_config().timeout.unwrap().into(), + )) + .https_only(true) + .user_agent(format!("questpackagemanager-rust2/{}", env!("CARGO_PKG_VERSION")).as_str()) + .build() + }) +} + +pub fn download_file(url: &str, _callback: F) -> Result> +where + F: FnMut(usize, usize), +{ + let request = get_agent().get(url).timeout(Duration::MAX); + + let response = request + .call() + .with_context(|| format!("Unable to download file {url}"))?; + + let mut bytes: Vec = Vec::with_capacity( + response + .header("Content-Length") + .unwrap_or("0") + .parse::() + .unwrap_or(0), + ); + + let mut reader = BufReader::new(response.into_reader()); + + reader.read_to_end(&mut bytes)?; + + Ok(bytes) + + // TODO: Fix + // let mut bytes = Vec::with_capacity(response.content_length().unwrap_or(0) as usize); + // let mut read_bytes = vec![0u8; 4 * 1024]; + + // loop { + // let read = response.read(&mut read_bytes)?; + // bytes.append(&mut read_bytes); + + // callback(bytes.len(), bytes.capacity()); + // if read == 0 { + // println!("Done!"); + // break; + // } + // } + + // Ok(bytes) +} + +#[inline(always)] +pub fn download_file_report(url: &str, mut callback: F) -> Result> +where + F: FnMut(usize, usize), +{ + // let mut progress_bar = ProgressBar::new(1000); + + // progress_bar.finish_println(""); + download_file(url, |current, expected| { + // progress_bar.set((current / expected) as u64 * 1000); + + callback(current, expected) + }) +} diff --git a/src/repository/qpackages.rs b/src/repository/qpackages.rs index f39fafab..7977bf3c 100644 --- a/src/repository/qpackages.rs +++ b/src/repository/qpackages.rs @@ -1,10 +1,9 @@ use color_eyre::{ - eyre::{bail, Context}, + eyre::{anyhow, bail, Context}, Result, }; use itertools::Itertools; use owo_colors::OwoColorize; -use reqwest::StatusCode; use semver::Version; use std::{ cell::UnsafeCell, @@ -26,7 +25,7 @@ use crate::{ config::get_combine_config, package::PackageConfigExtensions, package_metadata::PackageMetadataExtensions, }, - network::agent::{download_file_report, get_agent}, + network::agent::{self, download_file_report}, terminal::colors::QPMColor, utils::git, }; @@ -49,18 +48,9 @@ impl QPMRepository { { let url = format!("{API_URL}/{path}"); - let response = get_agent() - .get(url) - .send() - .context("Unable to make request to qpackages.com")?; - - if response.status() == StatusCode::NOT_FOUND { - return Ok(None); - } - - let result: T = response.json().context("Into json failed")?; - - Ok(Some(result)) + let response = + agent::get_opt::(&url).context("Unable to make request to qpackages.com")?; + Ok(response) } /// Requests the appriopriate package info from qpackage.com @@ -82,19 +72,21 @@ impl QPMRepository { API_URL, &package.config.info.id, &package.config.info.version ); - let resp = get_agent() - .post(url) - .header("Authorization", auth) - .json(&package) - .send()?; - - if resp.status() == StatusCode::UNAUTHORIZED { - bail!( - "Could not publish to {}: Unauthorized! Did you provide the correct key?", - API_URL - ); - } - resp.error_for_status()?; + let mut headers = HashMap::new(); + headers.insert("Authorization", auth); + + match agent::post(&url, package, &headers) { + Ok(o) => Ok(o), + Err(e) => match e { + agent::Error::Unauthorized => Err(anyhow!( + "Could not publish to {}: Unauthorized! Did you provide the correct key?", + API_URL + ) + .wrap_err(e)), + _ => Err(anyhow!("Error").wrap_err(e)), + }, + }?; + Ok(()) } @@ -160,9 +152,9 @@ impl QPMRepository { )?; } else { // not a github url, assume it's a zip - let response = get_agent().get(url).send()?; + let response = agent::get_bytes(url)?; - let buffer = Cursor::new(response.bytes()?); + let buffer = Cursor::new(response); // Extract to tmp folder ZipArchive::new(buffer)?.extract(&tmp_path)?; } diff --git a/src/utils/android.rs b/src/utils/android.rs index 783b55b3..8ee9d77d 100644 --- a/src/utils/android.rs +++ b/src/utils/android.rs @@ -1,4 +1,8 @@ -use std::{collections::HashMap, env, io::Cursor}; +use std::{ + collections::HashMap, + env, + io::{Cursor}, +}; use bytes::Bytes; use color_eyre::Result; @@ -12,7 +16,7 @@ use crate::{ android_repo::{AndroidRepositoryManifest, Archive, RemotePackage}, config::get_combine_config, }, - network::agent::{download_file_report, get_agent}, + network::agent::{self, download_file_report}, terminal::colors::QPMColor, }; @@ -20,10 +24,9 @@ const ANDROID_REPO_MANIFEST: &str = "https://dl.google.com/android/repository/re const ANDROID_DL_URL: &str = "https://dl.google.com/android/repository"; pub fn get_android_manifest() -> Result { - let response = get_agent().get(ANDROID_REPO_MANIFEST).send()?; - + let response = agent::get_str(ANDROID_REPO_MANIFEST)?; - Ok(serde_xml_rs::from_reader(response)?) + Ok(serde_xml_rs::from_str(&response)?) } pub fn get_ndk_packages(manifest: &AndroidRepositoryManifest) -> Vec<&RemotePackage> { diff --git a/src/utils/git.rs b/src/utils/git.rs index e87e95ed..128dde61 100644 --- a/src/utils/git.rs +++ b/src/utils/git.rs @@ -1,6 +1,6 @@ use std::{ fs::File, - io::BufReader, + io::{BufReader, BufWriter, Write}, path::Path, process::{Command, Stdio}, }; @@ -13,7 +13,10 @@ use owo_colors::OwoColorize; //use duct::cmd; use serde::{Deserialize, Serialize}; -use crate::{models::config::get_keyring, network::agent::get_agent}; +use crate::{ + models::config::get_keyring, + network::agent::{self}, +}; pub fn check_git() -> Result<()> { let mut git = std::process::Command::new("git"); @@ -55,11 +58,10 @@ pub fn get_release(url: &str, out: &std::path::Path) -> Result { } pub fn get_release_without_token(url: &str, out: &std::path::Path) -> Result { - let mut file = File::create(out).context("create so file failed")?; - get_agent() - .get(url) - .send()? - .copy_to(&mut file) + let file = File::create(out).context("create so file failed")?; + let mut writer = BufWriter::new(file); + writer + .write_all(&agent::get_bytes(url)?) .context("Failed to write to file")?; Ok(out.exists()) @@ -85,8 +87,8 @@ pub fn get_release_with_token(url: &str, out: &std::path::Path, token: &str) -> &token, &user, &repo, &tag ); - let data = match get_agent().get(&asset_data_link).send() { - Ok(o) => o.json::().unwrap(), + let data = match agent::get::(&asset_data_link) { + Ok(o) => o, Err(e) => { let error_string = e.to_string().replace(token, "***"); bail!("{}", error_string); @@ -100,12 +102,11 @@ pub fn get_release_with_token(url: &str, out: &std::path::Path, token: &str) -> .url .replace("api.github.com", &format!("{token}@api.github.com")); - let mut file = File::create(out).context("create so file failed")?; + let file = File::create(out).context("create so file failed")?; - get_agent() - .get(download) - .send()? - .copy_to(&mut file) + let mut writer = BufWriter::new(file); + writer + .write_all(&agent::get_bytes(&download)?) .context("Failed to write out downloaded bytes")?; break; } From 783a3590f8dc05674c382b3899c77bf78c069227 Mon Sep 17 00:00:00 2001 From: Fernthedev <15272073+Fernthedev@users.noreply.github.com> Date: Wed, 8 Mar 2023 21:51:08 -0400 Subject: [PATCH 2/3] Implement reqwest method proxies --- src/network/mod.rs | 86 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/src/network/mod.rs b/src/network/mod.rs index ce572590..54f650fe 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -114,15 +114,95 @@ pub mod agent { #[cfg(feature = "reqwest")] pub mod agent { + use std::collections::HashMap; + + use reqwest::{StatusCode}; + use serde::de::DeserializeOwned; + use thiserror::Error; + pub use super::reqwest_agent::*; - pub type Error = reqwest::Error; + #[derive(Error, Debug)] + pub enum Error { + #[error("Agent error")] + AgentError(Box), + #[error("IO Error")] + IoError(std::io::Error), + #[error("Unauthorized")] + Unauthorized, + } + pub type AgentError = reqwest::Error; - fn get(url: &str) -> Result + fn map_err(e: AgentError) -> Error { + Error::AgentError(Box::new(e)) + } + + pub fn get(url: &str) -> Result where T: DeserializeOwned, { - get_agent().get(url).send()?.json() + get_agent() + .get(url) + .send() + .map_err(map_err)? + .error_for_status() + .map_err(map_err)? + .json::() + .map_err(map_err) + } + + pub fn get_bytes(url: &str) -> Result, Error> { + get_agent() + .get(url) + .send() + .map_err(map_err)? + .bytes() + .map(|b| b.into()) + .map_err(map_err) + } + pub fn get_str(url: &str) -> Result { + get_agent() + .get(url) + .send() + .map_err(map_err)? + .text() + .map_err(map_err) + } + pub fn post( + url: &str, + data: impl serde::Serialize, + headers: &HashMap<&str, &str>, + ) -> Result + where + T: DeserializeOwned, + { + let mut req = get_agent().post(url); + + for (key, val) in headers { + req = req.header(*key, *val); + } + + let res = req.json(&data).send().map_err(map_err)?; + if res.status() == StatusCode::UNAUTHORIZED { + return Err(Error::Unauthorized); + } + + res.json::().map_err(map_err) + } + + pub fn get_opt(url: &str) -> Result, Error> + where + T: DeserializeOwned, + { + let req = get_agent().get(url).send().map_err(map_err)?; + + if req.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + + let res = req.json::().map_err(map_err)?; + + Ok(Some(res)) } } From edf0659ed67e9171f3f7e99318899de9bd94284b Mon Sep 17 00:00:00 2001 From: Fernthedev <15272073+Fernthedev@users.noreply.github.com> Date: Mon, 12 May 2025 11:31:06 -0400 Subject: [PATCH 3/3] Fixes --- Cargo.lock | 1 + Cargo.toml | 3 +- src/commands/qmod/mod.rs | 12 +- src/commands/scripts.rs | 2 +- src/main.rs | 38 ++++++- src/network/agent.rs | 0 src/network/agent_common.rs | 11 ++ src/network/github.rs | 4 +- src/network/mod.rs | 210 ++--------------------------------- src/network/reqwest_agent.rs | 86 +++++++++++++- src/network/ureq_agent.rs | 104 +++++++++++++++++ src/repository/qpackages.rs | 13 +-- src/utils/android.rs | 8 +- 13 files changed, 251 insertions(+), 241 deletions(-) create mode 100644 src/network/agent.rs create mode 100644 src/network/agent_common.rs diff --git a/Cargo.lock b/Cargo.lock index 697d6acc..0ca8db72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2567,6 +2567,7 @@ dependencies = [ "serde_json", "symlink", "templatr", + "thiserror 2.0.12", "trycmd", "ureq", "vergen", diff --git a/Cargo.toml b/Cargo.toml index 8d166baa..f103e3e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ required-features = [] [features] -default = ["templatr", "network_test", "cli", "gitoxide"] +default = ["templatr", "network_test", "cli", "gitoxide", "reqwest"] templatr = ["dep:templatr"] reqwest = ["dep:reqwest"] @@ -95,6 +95,7 @@ symlink = "0.1.0" fs_extra = "1.2" itertools = "0.14" schemars = { version = "0.8", features = ["semver"] } +thiserror = "2.0.12" [target.aarch64-apple-darwin.dependencies] # Allow cross compiles diff --git a/src/commands/qmod/mod.rs b/src/commands/qmod/mod.rs index 15ff9740..f07f63fa 100644 --- a/src/commands/qmod/mod.rs +++ b/src/commands/qmod/mod.rs @@ -1,15 +1,7 @@ use clap::{Args, Subcommand}; -use color_eyre::{eyre::ensure, Result}; -use qpm_package::models::{dependency::SharedPackageConfig}; -use qpm_qmod::models::mod_json::ModJson; -use semver::Version; +use color_eyre::Result; +use owo_colors::OwoColorize; -use crate::{ - models::{ - mod_json::{ModJsonExtensions, PreProcessingData}, - package::{PackageConfigExtensions}, - }, -}; use super::Command; diff --git a/src/commands/scripts.rs b/src/commands/scripts.rs index 1231d094..afaf4e0b 100644 --- a/src/commands/scripts.rs +++ b/src/commands/scripts.rs @@ -1,4 +1,4 @@ -use std::{path::Path, process::Stdio}; +use std::process::Stdio; use clap::Args; diff --git a/src/main.rs b/src/main.rs index 0c291f7f..dd41c9c1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,13 +28,41 @@ mod benchmark; #[cfg(test)] mod tests; +fn print_completions(generator: G, cmd: &mut clap::Command) { + generate( + generator, + cmd, + cmd.get_name().to_string(), + &mut io::stdout(), + ); +} -#[cfg(all(feature = "reqwest", feature = "ureq"))] -compile_error!("feature \"reqwest\" and feature \"ureq\" cannot be enabled at the same time"); +/// Suggests the location where to pipe the auto-generated completion script +/// based on the shell type. +fn suggest_completion_location(shell: Shell) { + eprintln!("To add this to your shell, you may use the following command:"); -#[cfg(not(any(feature = "reqwest", feature = "ureq")))] -compile_error!("feature \"reqwest\" or feature \"ureq\" must be enabled, though not both simultaneously"); + let file_name = shell.file_name("qpm"); + // powershell is unique so + // we make it its own suggestion + if shell == Shell::PowerShell { + eprintln!("\tqpm --generate {shell} | Set-Content \"$HOME\\qpm_autocomplete.ps1\""); + eprintln!( + "\t'if (Test-Path \"$HOME\\qpm_autocomplete.ps1\") {{ . \"$HOME\\qpm_autocomplete.ps1\" }}' | Add-Content -Path $PROFILE" + ); + } else { + let loc = match shell { + Shell::Bash => format!("/etc/bash_completion.d/{file_name}"), + Shell::Elvish => format!("~/.elvish/lib/completions/{file_name}"), + Shell::Fish => format!("~/.config/fish/completions/{file_name}"), + Shell::Zsh => format!("~/.zsh/completions/{file_name}"), + _ => todo!(), + }; + + eprintln!("\tqpm --generate {shell} > {loc}") + } +} fn main() -> Result<()> { color_eyre::config::HookBuilder::default() @@ -59,4 +87,4 @@ fn main() -> Result<()> { } Ok(()) -} +} \ No newline at end of file diff --git a/src/network/agent.rs b/src/network/agent.rs new file mode 100644 index 00000000..e69de29b diff --git a/src/network/agent_common.rs b/src/network/agent_common.rs new file mode 100644 index 00000000..105e9dad --- /dev/null +++ b/src/network/agent_common.rs @@ -0,0 +1,11 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum AgentError { + #[error("Agent error")] + AgentError(Box), + #[error("IO Error")] + IoError(std::io::Error), + #[error("Unauthorized")] + Unauthorized, +} \ No newline at end of file diff --git a/src/network/github.rs b/src/network/github.rs index 47af3327..e8a1b488 100644 --- a/src/network/github.rs +++ b/src/network/github.rs @@ -46,7 +46,7 @@ pub struct GithubCommitDiffCommitDataResponse { pub message: String, } -pub fn get_github_branch(branch: &str) -> Result { +pub fn get_github_branch(branch: &str) -> Result { agent::get(&format!( "https://api.github.com/repos/{GITHUB_OWNER}/{GITHUB_REPO}/branches/{branch}" )) @@ -54,7 +54,7 @@ pub fn get_github_branch(branch: &str) -> Result Result { +) -> Result { agent::get(&format!( "https://api.github.com/repos/{GITHUB_OWNER}/{GITHUB_REPO}/compare/{old}...{new}" )) diff --git a/src/network/mod.rs b/src/network/mod.rs index 54f650fe..416199ac 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -1,208 +1,14 @@ pub mod github; -#[cfg(feature = "ureq")] -mod ureq_agent; -#[cfg(feature = "reqwest")] -mod reqwest_agent; +#[cfg(all(feature = "reqwest", feature = "ureq"))] +compile_error!("feature \"reqwest\" and feature \"ureq\" cannot be enabled at the same time"); -#[cfg(feature = "ureq")] -pub mod agent { - use std::collections::HashMap; +#[cfg(not(any(feature = "reqwest", feature = "ureq")))] +compile_error!("feature \"reqwest\" or feature \"ureq\" must be enabled, though not both simultaneously"); - use thiserror::Error; +#[cfg_attr(feature = "ureq", path = "ureq_agent.rs")] +#[cfg_attr(feature = "reqwest", path = "reqwest_agent.rs")] +pub mod agent; - use serde::de::DeserializeOwned; - - pub use super::ureq_agent::*; - - pub type AgentError = ureq::Error; - - #[derive(Error, Debug)] - - pub enum Error { - #[error("Agent error")] - AgentError(Box), - #[error("IO Error")] - IoError(std::io::Error), - #[error("Unauthorized")] - Unauthorized, - } - - pub fn get(url: &str) -> Result - where - T: DeserializeOwned, - { - let res = match get_agent().get(url).call() { - Ok(o) => Ok(o), - Err(e) => Err(Error::AgentError(Box::new(e))), - }?; - match res.into_json::() { - Ok(o) => Ok(o), - Err(e) => Err(Error::IoError(e)), - } - } - - pub fn get_bytes(url: &str) -> Result, Error> { - Ok(get_str(url)?.into_bytes()) - } - pub fn get_str(url: &str) -> Result { - let res = match get_agent().get(url).call() { - Ok(o) => Ok(o), - Err(e) => Err(Error::AgentError(Box::new(e))), - }?; - match res.into_string() { - Ok(o) => Ok(o), - Err(e) => Err(Error::IoError(e)), - } - } - pub fn post( - url: &str, - data: impl serde::Serialize, - headers: &HashMap<&str, &str>, - ) -> Result - where - T: DeserializeOwned, - { - let mut req = get_agent().post(url); - - for (key, val) in headers { - req = req.set(key, val); - } - - let res = match req.send_json(data) { - Ok(o) => Ok(o), - Err(e) => match e { - ureq::Error::Status(code, _) => { - if code == 401 { - Err(Error::Unauthorized) - } else { - Err(Error::AgentError(Box::new(e))) - } - } - ureq::Error::Transport(_) => Err(Error::AgentError(Box::new(e))), - }, - }?; - - match res.into_json::() { - Ok(o) => Ok(o), - Err(e) => Err(Error::IoError(e)), - } - } - - pub fn get_opt(url: &str) -> Result, Error> - where - T: DeserializeOwned, - { - match get::(url) { - Ok(o) => Ok(Some(o)), - Err(e) => Err(match &e { - Error::AgentError(u) => match u.as_ref() { - ureq::Error::Status(code, _) => { - if *code == 404u16 { - return Ok(None); - } - return Err(e); - } - _ => e, - }, - _ => e, - }), - } - } -} - -#[cfg(feature = "reqwest")] -pub mod agent { - use std::collections::HashMap; - - use reqwest::{StatusCode}; - use serde::de::DeserializeOwned; - use thiserror::Error; - - pub use super::reqwest_agent::*; - - #[derive(Error, Debug)] - pub enum Error { - #[error("Agent error")] - AgentError(Box), - #[error("IO Error")] - IoError(std::io::Error), - #[error("Unauthorized")] - Unauthorized, - } - - pub type AgentError = reqwest::Error; - - fn map_err(e: AgentError) -> Error { - Error::AgentError(Box::new(e)) - } - - pub fn get(url: &str) -> Result - where - T: DeserializeOwned, - { - get_agent() - .get(url) - .send() - .map_err(map_err)? - .error_for_status() - .map_err(map_err)? - .json::() - .map_err(map_err) - } - - pub fn get_bytes(url: &str) -> Result, Error> { - get_agent() - .get(url) - .send() - .map_err(map_err)? - .bytes() - .map(|b| b.into()) - .map_err(map_err) - } - pub fn get_str(url: &str) -> Result { - get_agent() - .get(url) - .send() - .map_err(map_err)? - .text() - .map_err(map_err) - } - pub fn post( - url: &str, - data: impl serde::Serialize, - headers: &HashMap<&str, &str>, - ) -> Result - where - T: DeserializeOwned, - { - let mut req = get_agent().post(url); - - for (key, val) in headers { - req = req.header(*key, *val); - } - - let res = req.json(&data).send().map_err(map_err)?; - if res.status() == StatusCode::UNAUTHORIZED { - return Err(Error::Unauthorized); - } - - res.json::().map_err(map_err) - } - - pub fn get_opt(url: &str) -> Result, Error> - where - T: DeserializeOwned, - { - let req = get_agent().get(url).send().map_err(map_err)?; - - if req.status() == StatusCode::NOT_FOUND { - return Ok(None); - } - - let res = req.json::().map_err(map_err)?; - - Ok(Some(res)) - } -} +pub mod agent_common; \ No newline at end of file diff --git a/src/network/reqwest_agent.rs b/src/network/reqwest_agent.rs index 91ff8741..583f62c0 100644 --- a/src/network/reqwest_agent.rs +++ b/src/network/reqwest_agent.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashMap, env, io::{ErrorKind, Read, Write}, sync, @@ -6,13 +7,16 @@ use std::{ time::Duration, }; -use color_eyre::{ - Result, - eyre::{Context, ensure}, -}; +use color_eyre::eyre::{Context, ensure}; +use reqwest::StatusCode; +use serde::de::DeserializeOwned; use crate::models::config::get_combine_config; +use super::agent_common; + +pub type AgentError = agent_common::AgentError; + static AGENT: sync::OnceLock = sync::OnceLock::new(); pub fn get_agent() -> &'static reqwest::blocking::Client { @@ -30,7 +34,7 @@ pub fn get_agent() -> &'static reqwest::blocking::Client { }) } -pub fn download_file(url: &str, buffer: &mut impl Write, mut callback: F) -> Result +pub fn download_file(url: &str, buffer: &mut impl Write, mut callback: F) -> color_eyre::Result where F: FnMut(usize, usize), { @@ -89,7 +93,7 @@ where #[inline(always)] #[cfg(feature = "cli")] -pub fn download_file_report(url: &str, buffer: &mut impl Write, mut callback: F) -> Result +pub fn download_file_report(url: &str, buffer: &mut impl Write, mut callback: F) -> color_eyre::Result where F: FnMut(usize, usize), { @@ -114,3 +118,73 @@ where result } + +fn map_err(e: reqwest::Error) -> AgentError { + AgentError::AgentError(Box::new(e)) +} + +pub fn get(url: &str) -> Result +where + T: DeserializeOwned, +{ + get_agent() + .get(url) + .send() + .map_err(map_err)? + .error_for_status() + .map_err(map_err)? + .json::() + .map_err(map_err) +} + +pub fn get_bytes(url: &str) -> Result, AgentError> { + get_agent() + .get(url) + .send() + .map_err(map_err)? + .bytes() + .map(|b| b.into()) + .map_err(map_err) +} +pub fn get_str(url: &str) -> Result { + get_agent() + .get(url) + .send() + .map_err(map_err)? + .text() + .map_err(map_err) +} +pub fn post( + url: &str, + data: impl serde::Serialize, + headers: &HashMap<&str, &str>, +) -> Result<(), AgentError> { + let mut req = get_agent().post(url); + + for (key, val) in headers { + req = req.header(*key, *val); + } + + let res = req.json(&data).send().map_err(map_err)?; + if res.status() == StatusCode::UNAUTHORIZED { + return Err(AgentError::Unauthorized); + } + + res.error_for_status().map_err(map_err)?; + Ok(()) +} + +pub fn get_opt(url: &str) -> Result, AgentError> +where + T: DeserializeOwned, +{ + let req = get_agent().get(url).send().map_err(map_err)?; + + if req.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + + let res = req.json::().map_err(map_err)?; + + Ok(Some(res)) +} diff --git a/src/network/ureq_agent.rs b/src/network/ureq_agent.rs index 57b535ec..05d38ab3 100644 --- a/src/network/ureq_agent.rs +++ b/src/network/ureq_agent.rs @@ -23,6 +23,110 @@ pub fn get_agent() -> &'static Agent { }) } + +use std::collections::HashMap; + +use thiserror::Error; + +use serde::de::DeserializeOwned; + +pub use super::ureq_agent::*; + +pub type AgentError = ureq::Error; + +#[derive(Error, Debug)] + +pub enum Error { + #[error("Agent error")] + AgentError(Box), + #[error("IO Error")] + IoError(std::io::Error), + #[error("Unauthorized")] + Unauthorized, +} + +pub fn get(url: &str) -> Result +where + T: DeserializeOwned, +{ + let res = match get_agent().get(url).call() { + Ok(o) => Ok(o), + Err(e) => Err(Error::AgentError(Box::new(e))), + }?; + match res.into_json::() { + Ok(o) => Ok(o), + Err(e) => Err(Error::IoError(e)), + } +} + +pub fn get_bytes(url: &str) -> Result, Error> { + Ok(get_str(url)?.into_bytes()) +} +pub fn get_str(url: &str) -> Result { + let res = match get_agent().get(url).call() { + Ok(o) => Ok(o), + Err(e) => Err(Error::AgentError(Box::new(e))), + }?; + match res.into_string() { + Ok(o) => Ok(o), + Err(e) => Err(Error::IoError(e)), + } +} +pub fn post( + url: &str, + data: impl serde::Serialize, + headers: &HashMap<&str, &str>, +) -> Result +where + T: DeserializeOwned, +{ + let mut req = get_agent().post(url); + + for (key, val) in headers { + req = req.set(key, val); + } + + let res = match req.send_json(data) { + Ok(o) => Ok(o), + Err(e) => match e { + ureq::Error::Status(code, _) => { + if code == 401 { + Err(Error::Unauthorized) + } else { + Err(Error::AgentError(Box::new(e))) + } + } + ureq::Error::Transport(_) => Err(Error::AgentError(Box::new(e))), + }, + }?; + + match res.into_json::() { + Ok(o) => Ok(o), + Err(e) => Err(Error::IoError(e)), + } +} + +pub fn get_opt(url: &str) -> Result, Error> +where + T: DeserializeOwned, +{ + match get::(url) { + Ok(o) => Ok(Some(o)), + Err(e) => Err(match &e { + Error::AgentError(u) => match u.as_ref() { + ureq::Error::Status(code, _) => { + if *code == 404u16 { + return Ok(None); + } + return Err(e); + } + _ => e, + }, + _ => e, + }), + } +} + pub fn download_file(url: &str, _callback: F) -> Result> where F: FnMut(usize, usize), diff --git a/src/repository/qpackages.rs b/src/repository/qpackages.rs index 85b8b484..3e2ab231 100644 --- a/src/repository/qpackages.rs +++ b/src/repository/qpackages.rs @@ -1,12 +1,12 @@ -use bytes::{BufMut, BytesMut}; use color_eyre::{ - eyre::{anyhow, bail, Context}, - Result, + Result, Section, + eyre::{Context, OptionExt, anyhow, bail}, }; use itertools::Itertools; use owo_colors::OwoColorize; use semver::Version; use std::{ + collections::HashMap, fs::{self, File}, io::{BufWriter, Cursor}, path::Path, @@ -21,10 +21,7 @@ use qpm_package::{ }; use crate::{ - models::{ - config::get_combine_config, package::PackageConfigExtensions, - package_metadata::PackageMetadataExtensions, - }, + models::{config::get_combine_config, package::PackageConfigExtensions}, network::agent::{self, download_file_report}, terminal::colors::QPMColor, utils::git, @@ -84,7 +81,7 @@ impl QPMRepository { match agent::post(&url, package, &headers) { Ok(o) => Ok(o), Err(e) => match e { - agent::Error::Unauthorized => Err(anyhow!( + agent::AgentError::Unauthorized => Err(anyhow!( "Could not publish to {}: Unauthorized! Did you provide the correct key?", API_URL ) diff --git a/src/utils/android.rs b/src/utils/android.rs index 8b9a181b..e2964073 100644 --- a/src/utils/android.rs +++ b/src/utils/android.rs @@ -1,8 +1,4 @@ -use std::{ - collections::HashMap, - env, - io::{Cursor}, -}; +use std::{collections::HashMap, env, fs, io::Cursor, path::PathBuf}; use bytes::{BufMut, BytesMut}; use color_eyre::Result; @@ -16,7 +12,7 @@ use crate::{ android_repo::{AndroidRepositoryManifest, Archive, RemotePackage}, config::get_combine_config, }, - network::agent::{self, download_file_report}, + network::agent::{self, download_file, download_file_report}, terminal::colors::QPMColor, };