diff --git a/Cargo.lock b/Cargo.lock index 05369d2..9fe0a29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,6 +173,7 @@ dependencies = [ "clap", "const_format", "easy-logging", + "file-format", "flate2", "futures-util", "globset", @@ -503,6 +504,12 @@ dependencies = [ "log", ] +[[package]] +name = "file-format" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eab8aa2fba5f39f494000a22f44bf3c755b7d7f8ffad3f36c6d507893074159" + [[package]] name = "filetime" version = "0.2.26" diff --git a/Cargo.toml b/Cargo.toml index 38c81f3..40c48cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ chrono = "0.4.42" clap = "4.5.51" const_format = "0.2.35" easy-logging = "1" +file-format = { version = "0.28.0", features = ["reader"] } flate2 = "1.1" futures-util = "0.3.31" globset = "0.4.18" diff --git a/src/download.rs b/src/download.rs index cceccc4..99defe9 100644 --- a/src/download.rs +++ b/src/download.rs @@ -9,73 +9,127 @@ use url::Url; use crate::core::{EmptyResult, GenericResult}; use crate::util; +pub enum FileType { + Single, + Archived {mode: u32}, +} + pub trait Installer { - fn on_file(&mut self, path: &Path, mode: u32, data: &mut dyn Read) -> EmptyResult; + fn has_binary_matcher(&self) -> bool; + fn on_file(&mut self, path: &Path, file_type: FileType, data: &mut dyn Read) -> EmptyResult; } pub fn download(url: &Url, name: &str, installer: &mut dyn Installer) -> EmptyResult { - let reader = ReleaseReaderBuilder::new(name)?; - let client = ClientBuilder::new().user_agent(util::USER_AGENT).build()?; + let (stripped_name, decompressor_builder) = get_decompressor_builder(name)?; + let archive_type = stripped_name.rsplit_once('.').map(|(_, extension)| extension); + + let reader_builder: ReaderBuilder = match archive_type.unwrap_or_default() { + "tar" => Box::new(|| Box::new(TarReader::new())), + "7z" | "apk" | "deb" | "dmg" | "msi" | "pkg" | "rar" | "rpm" | "zip" => { + return Err!("Unsupported file type: {name:?}") + }, + _ => { + if installer.has_binary_matcher() { + return Err!(concat!( + "The release file {:?} looks like a simple binary (not an archive), ", + "but a binary matcher is specified", + ), name); + } + + let binary_name = stripped_name.to_owned(); + Box::new(move || Box::new(BinaryReader::new(&binary_name))) + }, + }; debug!("Downloading {url}..."); + let client = ClientBuilder::new().user_agent(util::USER_AGENT).build()?; let response = client.get(url.to_owned()).send()?; if !response.status().is_success() { - return Err!("The server returned and error: {}", response.status()) + return Err!("The server returned an error: {}", response.status()) } - let mut archive = reader.build(response); + let decompressor = decompressor_builder(Box::new(response)); + let reader = reader_builder(); + + reader.read(decompressor, installer) +} - for (index, entry) in archive.entries()?.enumerate() { - let mut entry = entry?; +type DecompressorBuilder = Box) -> Box>; +pub const COMPRESSION_EXTENSION_REGEX: &str = r"\.(?:bz2|gz|lz|lz4|lzma|lzo|xz|z|zst)"; - let header = entry.header(); - let path = entry.path()?; - let entry_type = header.entry_type(); +fn get_decompressor_builder(name: &str) -> GenericResult<(&str, DecompressorBuilder)> { + let Some((stripped_name, extension)) = name.rsplit_once('.') else { + return Ok((name, Box::new(|reader| reader))); + }; - if index == 0 { - debug!("Processing the archive:") - } - debug!("* {path:?} ({entry_type:?})"); + let builder: DecompressorBuilder = match extension { + "bz2" => Box::new(|reader| Box::new(bzip2::read::BzDecoder::new(reader))), + "gz" => Box::new(|reader| Box::new(flate2::read::GzDecoder::new(reader))), + "xz" => Box::new(|reader| Box::new(xz2::read::XzDecoder::new(reader))), + "lz" | "lz4" | "lzma" | "lzo" | "z" | "zst" => return Err!("Unsupported file type: {name:?}"), + _ => return Ok((name, Box::new(|reader| reader))), + }; + + Ok((stripped_name, builder)) +} + +type ReaderBuilder = Box Box>; + +trait ReleaseReader { + fn read(self: Box, reader: Box, installer: &mut dyn Installer) -> EmptyResult; +} - if matches!(entry_type, EntryType::Regular | EntryType::Continuous) { - let path = path.to_path_buf(); - let mode = header.mode()?; - installer.on_file(&path, mode, &mut entry)?; +struct BinaryReader { + name: String, +} + +impl BinaryReader { + fn new(name: &str) -> BinaryReader { + BinaryReader { + name: name.to_owned(), } } +} - Ok(()) +impl ReleaseReader for BinaryReader { + fn read(self: Box, mut reader: Box, installer: &mut dyn Installer) -> EmptyResult { + installer.on_file(Path::new(self.name.as_str()), FileType::Single, &mut reader) + } } -type DecoderBuilder = Box) -> Box>; +struct TarReader { +} -struct ReleaseReaderBuilder { - decoder_builder: DecoderBuilder, +impl TarReader { + fn new() -> TarReader { + TarReader {} + } } -impl ReleaseReaderBuilder { - fn new(name: &str) -> GenericResult { - let decoder_builder = name.rsplit_once('.').and_then(|(name, extension)| { - let decoder: DecoderBuilder = match extension { - "bz2" => Box::new(|reader| Box::new(bzip2::read::BzDecoder::new(reader))), - "gz" => Box::new(|reader| Box::new(flate2::read::GzDecoder::new(reader))), - "xz" => Box::new(|reader| Box::new(xz2::read::XzDecoder::new(reader))), - _ => return None, - }; - - if name.rsplit_once('.')?.1 != "tar" { - return None; - } +impl ReleaseReader for TarReader { + fn read(self: Box, reader: Box, installer: &mut dyn Installer) -> EmptyResult { + let mut archive = Archive::new(reader); - Some(decoder) - }).ok_or_else(|| format!("Unsupported file type: {name:?}"))?; + for (index, entry) in archive.entries()?.enumerate() { + let mut entry = entry?; - Ok(ReleaseReaderBuilder {decoder_builder}) - } + let header = entry.header(); + let path = entry.path()?; + let entry_type = header.entry_type(); + + if index == 0 { + debug!("Processing the archive:") + } + debug!("* {path:?} ({entry_type:?})"); + + if matches!(entry_type, EntryType::Regular | EntryType::Continuous) { + let path = path.to_path_buf(); + let file_type = FileType::Archived {mode: header.mode()?}; + installer.on_file(&path, file_type, &mut entry)?; + } + } - fn build(self, reader: R) -> Archive { - let reader = (self.decoder_builder)(Box::new(reader)); - Archive::new(reader) + Ok(()) } } \ No newline at end of file diff --git a/src/file_types.rs b/src/file_types.rs new file mode 100644 index 0000000..b4027cd --- /dev/null +++ b/src/file_types.rs @@ -0,0 +1,47 @@ +use std::io::{Read, Seek, SeekFrom}; +use std::env::consts; +use std::str::FromStr; + +use file_format::{FileFormat, Kind}; +use platforms::OS; + +use crate::core::GenericResult; + +pub fn is_executable(mut reader: R) -> GenericResult<(String, bool)> { + let format = { + reader.seek(SeekFrom::Start(0))?; + FileFormat::from_reader(reader)? + }; + + let description = format!( + "{full_name}{short_name} ({kind:?}, {media_type})", + full_name=format.name(), short_name=format.short_name().map(|name| format!(" / {name}")).unwrap_or_default(), + kind=format.kind(), media_type=format.media_type(), + ); + + let executable = get_os_specific_executable_types().unwrap_or_default().contains(&format) + || format.kind() == Kind::Other && format.name().ends_with(" Script"); + + Ok((description, executable)) +} + +fn get_os_specific_executable_types() -> Option> { + Some(match OS::from_str(consts::OS).ok()? { + OS::Linux => vec![FileFormat::ExecutableAndLinkableFormat], + OS::MacOS => vec![FileFormat::MachO], + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn os_support() { + assert!( + get_os_specific_executable_types().is_some(), + "Unsupported OS: {}", consts::OS, + ); + } +} \ No newline at end of file diff --git a/src/install.rs b/src/install.rs index 52b5c07..31c5e75 100644 --- a/src/install.rs +++ b/src/install.rs @@ -12,7 +12,8 @@ use url::Url; use crate::config::Config; use crate::core::{EmptyResult, GenericResult}; -use crate::download; +use crate::download::{self, FileType}; +use crate::file_types; use crate::github::{self, Github}; use crate::matcher::Matcher; use crate::release::{self, Release}; @@ -270,38 +271,56 @@ impl Installer { impl Drop for Installer { fn drop(&mut self) { - if let Some(temp_path) = self.temp_path.take() && let Err(err) = fs::remove_file(&temp_path) { + if let Some(temp_path) = self.temp_path.take() + && let Err(err) = fs::remove_file(&temp_path) { error!("Unable to delete {temp_path:?}: {err}."); } } } impl download::Installer for Installer { - fn on_file(&mut self, path: &Path, mode: u32, data: &mut dyn Read) -> EmptyResult { - let is_executable = mode & 0o100 != 0; + fn has_binary_matcher(&self) -> bool { + !self.automatic_matcher + } - if is_executable { - self.binaries.push(path.to_owned()); - } + fn on_file(&mut self, path: &Path, file_type: FileType, data: &mut dyn Read) -> EmptyResult { + let mut check_file_type = false; - if self.matcher.matches(path) { - debug!("{path:?} matches binary matcher."); + match file_type { + FileType::Single => { + assert!(!self.has_binary_matcher()); // It must be checked before release downloading + self.binaries.push(path.to_owned()); + self.matches.push(path.to_owned()); + check_file_type = true; + }, - self.matches.push(path.to_owned()); - if self.matches.len() > 1 { - return Ok(()); // We'll return error later when collect all matches - } + FileType::Archived {mode} => { + let is_executable = mode & 0o100 != 0; - if !is_executable { - return Err!("{path:?} in the archive is not executable"); - } - } else if self.automatic_matcher && is_executable && self.temp_path.is_none() { - debug!(concat!( - "Got first executable in archive: {:?}. ", - "Download it for the case if it's the only one executable in archive.", - ), path); - } else { - return Ok(()); + if is_executable { + self.binaries.push(path.to_owned()); + } + + if self.matcher.matches(path) { + debug!("{path:?} matches binary matcher."); + + self.matches.push(path.to_owned()); + if self.matches.len() > 1 { + return Ok(()); // We'll return error later when collect all matches + } + + if !is_executable { + return Err!("{path:?} in the archive is not executable"); + } + } else if self.automatic_matcher && is_executable && self.temp_path.is_none() { + debug!(concat!( + "Got first executable in archive: {:?}. ", + "Download it for the case if it's the only one executable in archive.", + ), path); + } else { + return Ok(()); + } + }, } let temp_path = match self.temp_path.as_ref() { @@ -320,6 +339,7 @@ impl download::Installer for Installer { let mut file = OpenOptions::new() .create(true) .mode(0o755) + .read(check_file_type) .write(true) .truncate(true) .custom_flags(libc::O_NOFOLLOW) @@ -329,8 +349,21 @@ impl download::Installer for Installer { io::copy(data, &mut file)?; file.set_modified(self.time)?; - file.sync_all()?; + if check_file_type { + let (description, executable) = file_types::is_executable(&mut file).map_err(|e| format!( + "Failed to determine {path:?} file type: {e}"))?; + + if !executable { + return Err!( + "{} doesn't look like an executable for current operating system and identified as {}", + path.display(), description) + } + + debug!("{} is identified as {description}.", path.display()); + } + + file.sync_all()?; Ok(()) } } diff --git a/src/main.rs b/src/main.rs index 5462496..e383987 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ mod cli; mod config; mod download; +mod file_types; mod github; mod install; mod list; diff --git a/src/release.rs b/src/release.rs index 9b93a12..619567f 100644 --- a/src/release.rs +++ b/src/release.rs @@ -3,11 +3,13 @@ use std::env::consts; use std::str::FromStr; use chrono::{DateTime, Utc}; +use itertools::Itertools; use platforms::{Arch, OS}; use regex::{self, Regex}; use url::Url; use crate::core::GenericResult; +use crate::download::COMPRESSION_EXTENSION_REGEX; use crate::matcher::Matcher; use crate::project::Project; use crate::util; @@ -100,17 +102,37 @@ fn generate_release_matchers(binary_name: &str, project_name: &str, os: &str, ar let any_fields_regex = format!("(?:{separator_regex}[^/]+)?"); let platform_regex = format!("(?:{os_regex}[-_]{arch_regex}|{arch_regex}[-_]{os_regex})"); - let basic_regex = format!( + let optional_compression_extension_regex = format!(r"(?:{COMPRESSION_EXTENSION_REGEX})?"); + let archive_regex = format!( r"{separator_regex}{platform_regex}{any_fields_regex}\.tar\.[^/.]+$", ); + // Prioritized list of matchers let mut matchers = Vec::new(); + let name_regexes = [binary_name, project_name].into_iter().dedup().map(get_name_matcher).collect_vec(); - for name in [binary_name, project_name] { - let name_regex = get_name_matcher(name); - matchers.push(Regex::new(&format!("^{name_regex}{any_fields_regex}{basic_regex}")).unwrap()); + for name_regex in &name_regexes { + // Archive with strict name and platform spec + matchers.push(Regex::new(&format!( + r"^{name_regex}{any_fields_regex}{archive_regex}")).unwrap()); + + // Binary with strict name and platform spec + matchers.push(Regex::new(&format!( + r"^{name_regex}{separator_regex}{platform_regex}{optional_compression_extension_regex}$")).unwrap()); + } + + // Archive with strict platform spec and relaxed name spec + matchers.push(Regex::new(&archive_regex).unwrap()); + + for name_regex in &name_regexes { + // Binary with strict name spec and relaxed platform spec (example: a Linux-only tool) + matchers.push(Regex::new(&format!( + r"^{name_regex}{separator_regex}{arch_regex}{optional_compression_extension_regex}$")).unwrap()); + + // Binary with strict name spec and with no platform spec (example: a Linux-only tool for a single architecture) + matchers.push(Regex::new(&format!( + r"^{name_regex}{optional_compression_extension_regex}$")).unwrap()); } - matchers.push(Regex::new(&basic_regex).unwrap()); Some(matchers.into_iter().map(Matcher::Regex).collect()) } @@ -160,6 +182,7 @@ mod tests { } #[rstest(binary_name, project_name, assets, matches, matcher_index, + // https://github.com/KonishchevDmitry/binup case("binup", "binup", &[ "binup-v1.1.0-linux-x64.tar.bz2", "binup-v1.1.0-macos-arm64.tar.bz2", @@ -170,6 +193,7 @@ mod tests { (OS::MacOS, Arch::AArch64, "binup-v1.1.0-macos-arm64.tar.bz2"), ], 0), + // https://github.com/DNSCrypt/dnscrypt-proxy case("dnscrypt-proxy", "dnscrypt-proxy", &[ "dnscrypt-proxy-android_arm-2.1.5.zip", "dnscrypt-proxy-android_arm-2.1.5.zip.minisig", @@ -231,6 +255,7 @@ mod tests { // (OS::MacOS, Arch::AArch64, "dnscrypt-proxy-macos_arm64-2.1.5.zip"), ], 0), + // https://github.com/martin-helmich/prometheus-nginxlog-exporter case("prometheus-nginxlog-exporter", "prometheus-nginxlog-exporter", &[ "checksums.txt", "prometheus-nginxlog-exporter_1.11.0_darwin_amd64.tar.gz", @@ -247,6 +272,7 @@ mod tests { (OS::MacOS, Arch::AArch64, "prometheus-nginxlog-exporter_1.11.0_darwin_arm64.tar.gz"), ], 0), + // https://github.com/prometheus/node_exporter case("prometheus-node-exporter", "node_exporter", &[ "node_exporter-1.8.2.darwin-amd64.tar.gz", "node_exporter-1.8.2.darwin-arm64.tar.gz", @@ -272,8 +298,9 @@ mod tests { (OS::Linux, Arch::X86_64, "node_exporter-1.8.2.linux-amd64.tar.gz"), (OS::MacOS, Arch::X86_64, "node_exporter-1.8.2.darwin-amd64.tar.gz"), (OS::MacOS, Arch::AArch64, "node_exporter-1.8.2.darwin-arm64.tar.gz"), - ], 1), + ], 2), + // https://github.com/shadowsocks/shadowsocks-rust case("ssservice", "shadowsocks-rust", &[ "shadowsocks-v1.20.3.aarch64-apple-darwin.tar.xz", "shadowsocks-v1.20.3.aarch64-apple-darwin.tar.xz.sha256", @@ -310,7 +337,34 @@ mod tests { // (OS::Linux, Arch::X86_64, "shadowsocks-v1.20.3.x86_64-unknown-linux-gnu.tar.xz"), (OS::MacOS, Arch::X86_64, "shadowsocks-v1.20.3.x86_64-apple-darwin.tar.xz"), (OS::MacOS, Arch::AArch64, "shadowsocks-v1.20.3.aarch64-apple-darwin.tar.xz"), - ], 2), + ], 4), + + // https://github.com/tsl0922/ttyd/releases + case("ttyd", "ttyd", &[ + "SHA256SUMS", + "ttyd.aarch64", + "ttyd.arm", + "ttyd.armhf", + "ttyd.i686", + "ttyd.mips", + "ttyd.mips64", + "ttyd.mips64el", + "ttyd.mipsel", + "ttyd.s390x", + "ttyd.win32.exe", + "ttyd.x86_64", + ], &[ + (OS::Linux, Arch::X86_64, "ttyd.x86_64"), + (OS::MacOS, Arch::X86_64, "ttyd.x86_64"), + (OS::MacOS, Arch::AArch64, "ttyd.aarch64"), + ], 3), + + // https://github.com/KonishchevDmitry/binup/issues/2#issuecomment-3222682495 + case("rapidgzip", "indexed_bzip2", &["rapidgzip"], &[ + (OS::Linux, Arch::X86_64, "rapidgzip"), + (OS::MacOS, Arch::X86_64, "rapidgzip"), + (OS::MacOS, Arch::AArch64, "rapidgzip"), + ], 6), )] fn release_matcher(binary_name: &str, project_name: &str, assets: &[&str], matches: &[(OS, Arch, &str)], matcher_index: usize) { for (os, arch, expected) in matches {