diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..9cafc76 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +rustflags = "--cfg reqwest_unstable" \ No newline at end of file diff --git a/.github/workflows/cli-release.yaml b/.github/workflows/cli-release.yaml new file mode 100644 index 0000000..3513072 --- /dev/null +++ b/.github/workflows/cli-release.yaml @@ -0,0 +1,155 @@ +# This file was 1:1 taken from https://github.com/houseabsolute/precious. Check it out! +name: "[impit-cli] Test & Build" + +on: + push: + branches: + - "**" + tags-ignore: + - "impit-cli-*" + paths: + - "impit-cli/**" + - "impit/**" + pull_request: + +env: + CRATE_NAME: impit-cli + GITHUB_TOKEN: ${{ github.token }} + RUST_BACKTRACE: 1 + +jobs: + test: + name: ${{ matrix.platform.os-name }} with rust ${{ matrix.toolchain }} + runs-on: ${{ matrix.platform.runs-on }} + strategy: + fail-fast: false + matrix: + platform: + # Platforms that don't work: + # + # - sparc64-unknown-linux-gnu - cannot compile openssl-sys + # - x86_64-unknown-illumos - weird error compiling openssl - "bin/sh: 1: granlib: not found" + + # - os-name: FreeBSD-x86_64 + # runs-on: ubuntu-20.04 + # target: x86_64-unknown-freebsd + # bin: impit-cli + # name: impit-cli-FreeBSD-x86_64.tar.gz + # skip_tests: true + - os-name: Linux-x86_64 + runs-on: ubuntu-20.04 + target: x86_64-unknown-linux-musl + bin: impit-cli + name: impit-cli-Linux-x86_64-musl.tar.gz + - os-name: Linux-aarch64 + runs-on: ubuntu-20.04 + target: aarch64-unknown-linux-musl + bin: impit-cli + name: impit-cli-Linux-aarch64-musl.tar.gz + - os-name: Linux-arm + runs-on: ubuntu-20.04 + target: arm-unknown-linux-musleabi + bin: impit-cli + name: impit-cli-Linux-arm-musl.tar.gz + # - os-name: Linux-i686 + # runs-on: ubuntu-20.04 + # target: i686-unknown-linux-musl + # bin: impit-cli + # name: impit-cli-Linux-i686-musl.tar.gz + # skip_tests: true + # - os-name: Linux-powerpc + # runs-on: ubuntu-20.04 + # target: powerpc-unknown-linux-gnu + # bin: impit-cli + # name: impit-cli-Linux-powerpc-gnu.tar.gz + # skip_tests: true + # - os-name: Linux-powerpc64 + # runs-on: ubuntu-20.04 + # target: powerpc64-unknown-linux-gnu + # bin: impit-cli + # name: impit-cli-Linux-powerpc64-gnu.tar.gz + # skip_tests: true + # - os-name: Linux-powerpc64le + # runs-on: ubuntu-20.04 + # target: powerpc64le-unknown-linux-gnu + # bin: impit-cli + # name: impit-cli-Linux-powerpc64le.tar.gz + # skip_tests: true + # - os-name: Linux-riscv64 + # runs-on: ubuntu-20.04 + # target: riscv64gc-unknown-linux-gnu + # bin: impit-cli + # name: impit-cli-Linux-riscv64gc-gnu.tar.gz + # - os-name: Linux-s390x + # runs-on: ubuntu-20.04 + # target: s390x-unknown-linux-gnu + # bin: impit-cli + # name: impit-cli-Linux-s390x-gnu.tar.gz + # skip_tests: true + # - os-name: NetBSD-x86_64 + # runs-on: ubuntu-20.04 + # target: x86_64-unknown-netbsd + # bin: impit-cli + # name: impit-cli-NetBSD-x86_64.tar.gz + # skip_tests: true + - os-name: Windows-aarch64 + runs-on: windows-latest + target: aarch64-pc-windows-msvc + bin: impit-cli.exe + name: impit-cli-Windows-aarch64.zip + skip_tests: true + # - os-name: Windows-i686 + # runs-on: windows-latest + # target: i686-pc-windows-msvc + # bin: impit-cli.exe + # name: impit-cli-Windows-i686.zip + # skip_tests: true + - os-name: Windows-x86_64 + runs-on: windows-latest + target: x86_64-pc-windows-msvc + bin: impit-cli.exe + name: impit-cli-Windows-x86_64.zip + - os-name: macOS-x86_64 + runs-on: macOS-latest + target: x86_64-apple-darwin + bin: impit-cli + name: impit-cli-Darwin-x86_64.tar.gz + - os-name: macOS-aarch64 + runs-on: macOS-latest + target: aarch64-apple-darwin + bin: impit-cli + name: impit-cli-Darwin-aarch64.tar.gz + toolchain: + - stable + steps: + - uses: actions/checkout@v4 + - name: Cache cargo & target directories + uses: Swatinem/rust-cache@v2 + - name: Configure Git + run: | + git config --global user.email "jdoe@example.com" + git config --global user.name "J. Doe" + - name: Build binary + uses: houseabsolute/actions-rust-cross@v0 + with: + command: "build" + target: ${{ matrix.platform.target }} + toolchain: ${{ matrix.toolchain }} + args: "--locked --release --manifest-path impit-cli/Cargo.toml" + strip: true + # - name: Run tests + # uses: houseabsolute/actions-rust-cross@v0 + # with: + # command: "test" + # target: ${{ matrix.platform.target }} + # toolchain: ${{ matrix.toolchain }} + # args: "--locked --release" + # if: ${{ !matrix.platform.skip_tests }} + - name: Publish artifacts and release + uses: houseabsolute/actions-rust-release@v0.0.4 + if: matrix.toolchain == 'stable' + with: + executable-name: impit-cli + target: ${{ matrix.platform.target }} + changes-file: "" + \ No newline at end of file diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 35d1efe..b974a4c 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -2,6 +2,7 @@ name: Build docs on: push: branches: [master] + paths: ['impit/**'] permissions: contents: read pages: write @@ -26,11 +27,11 @@ jobs: - name: Clean docs folder run: cargo clean --doc - name: Build docs - run: RUSTFLAGS='--cfg reqwest_unstable' cargo doc --no-deps + run: RUSTFLAGS='--cfg reqwest_unstable' cargo doc --no-deps --manifest-path impit/Cargo.toml - name: Remove lock file run: rm target/doc/.lock - name: Copy redirect - run: cp docs/index.html target/doc/index.html + run: cp impit/docs/index.html target/doc/index.html - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: diff --git a/Cargo.lock b/Cargo.lock index 58d425f..2e16f4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,56 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys 0.59.0", +] + [[package]] name = "async-compression" version = "0.4.18" @@ -80,14 +130,32 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-lc-fips-sys" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59057b878509d88952425fe694a2806e468612bde2d71943f3cd8034935b5032" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", + "regex", +] + [[package]] name = "aws-lc-rs" version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f409eb70b561706bf8abba8ca9c112729c481595893fd06a2dd9af8ed8441148" dependencies = [ + "aws-lc-fips-sys", "aws-lc-sys", "paste", + "untrusted 0.7.1", "zeroize", ] @@ -237,6 +305,46 @@ dependencies = [ "libloading", ] +[[package]] +name = "clap" +version = "4.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + [[package]] name = "cmake" version = "0.1.52" @@ -246,6 +354,12 @@ dependencies = [ "cc", ] +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "cookie" version = "0.18.1" @@ -1096,6 +1210,17 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "impit-cli" +version = "0.1.0" +dependencies = [ + "aws-lc-rs", + "clap", + "impit", + "openssl", + "tokio", +] + [[package]] name = "indexmap" version = "2.7.0" @@ -1112,6 +1237,12 @@ version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.12.1" @@ -1389,6 +1520,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-src" +version = "300.4.1+3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faa4eac4138c62414b5622d1b31c5c304f34b406b013c079c2bbc652fdd6678c" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.104" @@ -1397,6 +1537,7 @@ checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] @@ -1789,7 +1930,7 @@ dependencies = [ "getrandom", "libc", "spin", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -1868,7 +2009,7 @@ dependencies = [ "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -2098,6 +2239,12 @@ dependencies = [ "quote", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2415,6 +2562,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -2450,6 +2603,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/Cargo.toml b/Cargo.toml index 95ed6d2..2ac5e1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,17 @@ [workspace] +resolver = "2" members = [ - "impit" + "impit", + "impit-cli" ] [patch.crates-io] h2 = { git = "https://github.com/apify/h2", branch = "master" } rustls = { git = "https://github.com/apify/rustls", branch = "main" } + +[profile.release] +strip = true # Automatically strip symbols from the binary. +opt-level = "z" # Optimize for size. +lto = true +codegen-units = 1 diff --git a/impit-cli/Cargo.toml b/impit-cli/Cargo.toml new file mode 100644 index 0000000..c5163c6 --- /dev/null +++ b/impit-cli/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "impit-cli" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = { version = "4.5.21", features = ["derive"] } +impit = { path="../impit" } +tokio = { version="1.41.1", features = ["full"] } +aws-lc-rs = { version = "1.11.1", features = ["bindgen"] } + +[target.x86_64-unknown-linux-musl.dependencies] +openssl = { version = "*", features = ["vendored"] } + +[target.aarch64-unknown-linux-musl.dependencies] +openssl = { version = "*", features = ["vendored"] } + +[target.arm-unknown-linux-musleabi.dependencies] +openssl = { version = "*", features = ["vendored"] } diff --git a/impit-cli/src/headers.rs b/impit-cli/src/headers.rs new file mode 100644 index 0000000..aee0932 --- /dev/null +++ b/impit-cli/src/headers.rs @@ -0,0 +1,14 @@ +use std::collections::HashMap; + +pub(crate) fn process_headers(headers: Vec) -> HashMap { + let mut map = HashMap::new(); + + for header in headers { + let mut parts = header.splitn(2, ':'); + let key = parts.next().unwrap().trim().to_string(); + let value = parts.next().unwrap().trim().to_string(); + map.insert(key, value); + } + + map +} \ No newline at end of file diff --git a/impit-cli/src/main.rs b/impit-cli/src/main.rs new file mode 100644 index 0000000..da095e1 --- /dev/null +++ b/impit-cli/src/main.rs @@ -0,0 +1,142 @@ +use std::ffi::OsString; + +use clap::{Parser, ValueEnum}; +use impit::{impit::{Impit, RedirectBehavior}, emulation::Browser as ImpitBrowser, request::RequestOptions}; + +mod headers; + +#[derive(Parser, Debug, Clone, Copy, ValueEnum)] +enum Browser { + Chrome, + Firefox, + Impit +} + +#[derive(Parser, Debug, Clone, Copy, ValueEnum)] +enum Method { + GET, + POST, + PUT, + DELETE, + PATCH, + HEAD, + OPTIONS, + TRACE +} + +/// CLI interface for the impit library. +/// Something like CURL for libcurl, for making impersonated HTTP(2) requests. +#[derive(Parser, Debug)] +#[command(about, long_about = None)] +struct CliArgs { + /// Method to use for the request. + #[arg(short='X', long, default_value = "get")] + method: Method, + + /// HTTP headers to add to the request. + #[arg(short='H', long)] + headers: Vec, + + /// What browser to use for the request. + #[arg(short='A', long, default_value = "impit")] + impersonate: Browser, + + /// If set, impit will ignore TLS errors. + #[arg(short='k', long, action)] + ignore_tls_errors: bool, + + /// If set, impit will fallback to vanilla HTTP if the impersonated browser fails. + #[arg(short='f', long, action)] + fallback: bool, + + /// Proxy to use for the request. + #[arg(short='x', long="proxy")] + proxy: Option, + + /// Maximum time in seconds to wait for the request to complete. + #[arg(short='m', long="max-time")] + max_time: Option, + + /// Data to send with the request. + #[arg(short, long)] + data: Option, + + /// Enforce the use of HTTP/3 for the request. Note that if the server does not support HTTP/3, the request will fail. + #[arg(long="http3-only", action)] + http3_prior_knowledge: bool, + + /// Enable the use of HTTP/3. This will attempt to use HTTP/3, but fall back to earlier versions of HTTP if the server does not support it. + #[arg(long="http3", action)] + enable_http3: bool, + + /// Follow redirects + #[arg(short='L', long="location", action)] + follow_redirects: bool, + + /// Follow redirects + #[arg(long="max-redirs", default_value = "50")] + maximum_redirects: usize, + + /// URL of the request to make + url: String, +} + +#[tokio::main] +async fn main() { + let args = CliArgs::parse(); + + let mut client = Impit::builder() + .with_ignore_tls_errors(args.ignore_tls_errors) + .with_fallback_to_vanilla(args.fallback); + + client = match args.impersonate { + Browser::Chrome => client.with_browser(ImpitBrowser::Chrome), + Browser::Firefox => client.with_browser(ImpitBrowser::Firefox), + Browser::Impit => client + }; + + if args.proxy.is_some() { + client = client.with_proxy(args.proxy.unwrap()) + } + + if args.enable_http3 || args.http3_prior_knowledge { + client = client.with_http3() + } + + if args.follow_redirects { + client = client.with_redirect(RedirectBehavior::FollowRedirect(args.maximum_redirects)); + } else { + client = client.with_redirect(RedirectBehavior::ManualRedirect); + } + + let body: Option> = match args.data { + Some(data) => Some(data.into_string().unwrap().into_bytes()), + None => None + }; + + let mut client = client.build(); + + let timeout = match args.max_time { + Some(time) => Some(std::time::Duration::from_secs(time)), + None => None + }; + + let options = RequestOptions { + headers: headers::process_headers(args.headers), + http3_prior_knowledge: args.http3_prior_knowledge, + timeout, + }; + + let response = match args.method { + Method::GET => client.get(args.url, Some(options)).await.unwrap(), + Method::POST => client.post(args.url, body, Some(options)).await.unwrap(), + Method::PUT => client.put(args.url, body, Some(options)).await.unwrap(), + Method::DELETE => client.delete(args.url, Some(options)).await.unwrap(), + Method::PATCH => client.patch(args.url, body, Some(options)).await.unwrap(), + Method::HEAD => client.head(args.url, Some(options)).await.unwrap(), + Method::OPTIONS => client.options(args.url, Some(options)).await.unwrap(), + Method::TRACE => client.trace(args.url, Some(options)).await.unwrap(), + }; + + print!("{}", response.text().await.unwrap()); +} \ No newline at end of file diff --git a/impit/Cargo.toml b/impit/Cargo.toml index ec12631..080229a 100644 --- a/impit/Cargo.toml +++ b/impit/Cargo.toml @@ -18,6 +18,3 @@ tokio = { version="1.40.0", features = ["full"] } url = "2.5.2" webpki-roots = "0.26.6" -[patch.crates-io] -h2 = { git = "https://github.com/apify/h2", branch = "master" } -rustls = { git = "https://github.com/apify/rustls", branch = "main" } diff --git a/impit/src/http_headers/mod.rs b/impit/src/http_headers/mod.rs index 8d7e5e1..28c03ed 100644 --- a/impit/src/http_headers/mod.rs +++ b/impit/src/http_headers/mod.rs @@ -37,7 +37,7 @@ impl Into for HttpHeaders { }; if pseudo_headers_order.len() != 0 { - std::env::set_var("RETCH_H2_PSEUDOHEADERS_ORDER", pseudo_headers_order.join(",")); + std::env::set_var("IMPIT_H2_PSEUDOHEADERS_ORDER", pseudo_headers_order.join(",")); } let mut used_custom_headers: Vec = vec![];