From 405c2ac79a96bc729ce8cb19b0da2470740d277d Mon Sep 17 00:00:00 2001 From: HiGarfield Date: Sat, 7 Dec 2024 16:28:07 +0800 Subject: [PATCH] apt-fast: update --- .github/backup/apt-fast | 174 +++++++++++++++++++++++++++------------- 1 file changed, 119 insertions(+), 55 deletions(-) diff --git a/.github/backup/apt-fast b/.github/backup/apt-fast index 354a68e97..020189c59 100755 --- a/.github/backup/apt-fast +++ b/.github/backup/apt-fast @@ -1,6 +1,6 @@ #!/bin/bash # -# apt-fast v1.9.12 +# apt-fast v1.10.0 # Use this just like aptitude or apt-get for faster package downloading. # # Copyright: 2008-2012 Matt Parnell, http://www.mattparnell.com @@ -10,6 +10,9 @@ # Public License as published by the Free Software Foundation; either # version 3 of the License, or (at your option) any later version. # + +shopt -s nullglob + [ -n "$DEBUG" ] && set -xv # Print colored messages. @@ -34,7 +37,7 @@ msg(){ } # Search for known options and decide if root privileges are needed. -root=1 # default value: we need root privileges +root=$# option= for argument in "$@"; do case "$argument" in @@ -52,7 +55,7 @@ for argument in "$@"; do option="source" root=0 ;; - changelog | show) + changelog | list | show | help | --help | -h) root=0 ;; esac @@ -70,7 +73,6 @@ TMP__APTMGR="${_APTMGR-${TMP_RANDOM}}" TMP_APTCACHE="${APTCACHE-${TMP_RANDOM}}" TMP_DLDIR="${DLDIR-${TMP_RANDOM}}" TMP_DLLIST="${DLLIST-${TMP_RANDOM}}" -TMP_LISTDIR="${LISTDIR-${TMP_RANDOM}}" TMP__MAXNUM="${MAXNUM-${TMP_RANDOM}}" TMP__MAXCONPERSRV="${MAXCONPERSRV-${TMP_RANDOM}}" TMP__SPLITCON="${SPLITCON-${TMP_RANDOM}}" @@ -78,6 +80,7 @@ TMP__MINSPLITSZ=${MINSPLITSZ-${TMP_RANDOM}} TMP__PIECEALGO=${PIECEALGO-${TMP_RANDOM}} TMP_aptfast_prefix="${aptfast_prefix-${TMP_RANDOM}}" TMP_APT_FAST_TIMEOUT="${APT_FAST_TIMEOUT-${TMP_RANDOM}}" +TMP_APT_FAST_APT_AUTH="${APT_FAST_APT_AUTH-${TMP_RANDOM}}" TMP_VERBOSE_OUTPUT="${VERBOSE_OUTPUT-${TMP_RANDOM}}" TMP_ftp_proxy="${ftp_proxy-${TMP_RANDOM}}" TMP_http_proxy="${http_proxy-${TMP_RANDOM}}" @@ -85,7 +88,7 @@ TMP_https_proxy="${https_proxy-${TMP_RANDOM}}" # Check for proper privileges. # Call explicitly with environment variables to get them into root conext. -if [ "$root" = 1 ] && [ "$UID" != 0 ]; then +if [ "$root" -ne 0 ] && [ "$UID" != 0 ]; then exec sudo DEBUG="$DEBUG" \ LCK_FILE="$TMP_LCK_FILE" \ DOWNLOADBEFORE="$TMP_DOWNLOADBEFORE" \ @@ -93,7 +96,6 @@ if [ "$root" = 1 ] && [ "$UID" != 0 ]; then APTCACHE="$TMP_APTCACHE" \ DLDIR="$TMP_DLDIR" \ DLLIST="$TMP_DLLIST" \ - LISTDIR="$TMP_LISTDIR" \ _MAXNUM="$TMP__MAXNUM" \ _MAXCONPERSRV="$TMP__MAXCONPERSRV" \ _SPLITCON="$TMP__SPLITCON" \ @@ -101,6 +103,7 @@ if [ "$root" = 1 ] && [ "$UID" != 0 ]; then _PIECEALGO="$TMP__PIECEALGO" \ aptfast_prefix="$TMP_aptfast_prefix" \ APT_FAST_TIMEOUT="$TMP_APT_FAST_TIMEOUT" \ + APT_FAST_APT_AUTH="$TMP_APT_FAST_APT_AUTH" \ VERBOSE_OUTPUT="$TMP_VERBOSE_OUTPUT" \ ftp_proxy="$TMP_ftp_proxy" \ http_proxy="$TMP_http_proxy" \ @@ -127,14 +130,23 @@ if [ -z "$apt_fast_partial" ]; then else DLDIR="${apt_fast_partial}" fi -# Currently not needed. -eval "$(apt-config shell LISTDIR Dir::State::lists/d)" + +# Check for apt auth files +eval "$(apt-config shell NETRC Dir::Etc::netrc/f)" +eval "$(apt-config shell NETRCDIR Dir::Etc::netrcparts/d)" +APTAUTHFILES=() +if [ -f "$NETRC" ]; then + APTAUTHFILES=("$NETRC") +fi +APTAUTHFILES+=("$NETRCDIR"*) + DLLIST="/tmp/apt-fast.list" _MAXNUM=5 _MAXCONPERSRV=10 _SPLITCON=8 _MINSPLITSZ="1M" _PIECEALGO="default" +MIRRORS=() # Prefix in front of apt-fast output: aptfast_prefix= @@ -153,6 +165,9 @@ APT_FAST_TIMEOUT=60 # Ask for download confirmation if unset DOWNLOADBEFORE= +# Enable APT authentication support +APT_FAST_APT_AUTH=1 + # Formatted package list in download confirmation if unset VERBOSE_OUTPUT= @@ -181,7 +196,6 @@ https_proxy= [ "$TMP_APTCACHE" = "$TMP_RANDOM" ] || APTCACHE="$TMP_APTCACHE" [ "$TMP_DLDIR" = "$TMP_RANDOM" ] || DLDIR="$TMP_DLDIR" [ "$TMP_DLLIST" = "$TMP_RANDOM" ] || DLLIST="$TMP_DLLIST" -[ "$TMP_LISTDIR" = "$TMP_RANDOM" ] || LISTDIR="$TMP_LISTDIR" [ "$TMP__MAXNUM" = "$TMP_RANDOM" ] || _MAXNUM="$TMP__MAXNUM" [ "$TMP__MAXCONPERSRV" = "$TMP_RANDOM" ] || _MAXCONPERSRV="$TMP__MAXCONPERSRV" [ "$TMP__SPLITCON" = "$TMP_RANDOM" ] || _SPLITCON="$TMP__SPLITCON" @@ -189,6 +203,7 @@ https_proxy= [ "$TMP__PIECEALGO" = "$TMP_RANDOM" ] || _PIECEALGO="$TMP__PIECEALGO" [ "$TMP_aptfast_prefix" = "$TMP_RANDOM" ] || aptfast_prefix="$TMP_aptfast_prefix" [ "$TMP_APT_FAST_TIMEOUT" = "$TMP_RANDOM" ] || APT_FAST_TIMEOUT="$TMP_APT_FAST_TIMEOUT" +[ "$TMP_APT_FAST_APT_AUTH" = "$TMP_RANDOM" ] || APT_FAST_APT_AUTH="$TMP_APT_FAST_APT_AUTH" [ "$TMP_VERBOSE_OUTPUT" = "$TMP_RANDOM" ] || VERBOSE_OUTPUT="$TMP_VERBOSE_OUTPUT" [ "$TMP_ftp_proxy" = "$TMP_RANDOM" ] || ftp_proxy="$TMP_ftp_proxy" [ "$TMP_http_proxy" = "$TMP_RANDOM" ] || http_proxy="$TMP_http_proxy" @@ -284,7 +299,7 @@ get_mirrors(){ for mirror in "${mirrors[@]}"; do # Real expension. if [[ "$1" == "$mirror"* ]]; then - filepath=${1#${mirror}} + filepath="${1#"${mirror}"}" # Build list for aria download list. list="${mirrors[*]}" echo -e "${list// /${filepath}\\t}$filepath\n" @@ -296,6 +311,57 @@ get_mirrors(){ echo "$1" } +AUTH_INFO_PARSED=() +# Parse apt authentication files. +# Undefined behavior on whitespaces in host, username or password. +prepare_auth(){ + if [ "$APT_FAST_APT_AUTH" -eq 0 ]; then + return + fi + for auth_file in "${APTAUTHFILES[@]}"; do + # auth files have netrc syntax, possible multiline entries starting with "machine" + auth_info="$(tr '\n' ' ' < "$auth_file" | sed 's/\(\\)/\n\1/g' | sed '1d')" + while IFS= read -r auth; do + machine="$(echo "$auth" | sed 's/.*\[ \t]\+\([^ \t]\+\).*/\1/')" + login="$(echo "$auth" | sed 's/.*\[ \t]\+\([^ \t]\+\).*/\1/')" + password="$(echo "$auth" | sed 's/.*\[ \t]\+\([^ \t]\+\).*/\1/')" + # if machine does not have protocol, try https:// + if ! [[ "$machine" =~ ^.*:// ]]; then + machine="https://$machine" + fi + if [ -z "$machine" ] || [ -z "$login" ] || [ -z "$password" ]; then + msg "Could not parse apt authentication (skipping): $auth ($auth_file)" "warning" + continue + fi + # use space separated string to convert back to array later + AUTH_INFO_PARSED+=("$machine $login $password") + done <<< "$auth_info" + done +} + +# Gets URI as parameter and tries to add basic http credentials. Will fail on +# credentials that contain characters that need URL-encoding. +get_auth(){ + if [ "$APT_FAST_APT_AUTH" -eq 0 ]; then + echo "$1" + return + fi + for auth_info in "${AUTH_INFO_PARSED[@]}"; do + # convert to array, don't escape variable here + auth_info_arr=($auth_info) + machine="${auth_info_arr[0]}" + # takes first match + if [[ "$1" == "$machine"* ]]; then + login="${auth_info_arr[1]}" + password="${auth_info_arr[2]}" + uri="$(echo "$1" | sed "s|^\([^:]\+://\)|\1$login:$password@|")" + echo "$uri" + return + fi + done + echo "$1" +} + # Globals to save package name, version, size and overall size. DOWNLOAD_DISPLAY= DOWNLOAD_SIZE=0 @@ -333,21 +399,18 @@ get_uris(){ msg "Package manager quit with exit code." "warning" exit fi - while read -r pkg_uri_info + prepare_auth + ## --print-uris format is: + # 'fileurl' filename filesize checksum_hint:filechecksum + while IFS=' ' read -r uri filename filesize checksum_string _ do - [ -z "$pkg_uri_info" ] && continue - ## --print-uris format is: - # 'fileurl' filename filesize checksum_hint:filechecksum - uri="$(echo "$pkg_uri_info" | cut -d' ' -f1 | tr -d "'")" - filename="$(echo "$pkg_uri_info" | cut -d' ' -f2)" - filesize="$(echo "$pkg_uri_info" | cut -d' ' -f3)" - checksum_string="$(echo "$pkg_uri_info" | cut -d' ' -f4)" - hash_algo="$(echo "$checksum_string" | cut -d':' -f1)" - checksum="$(echo "$checksum_string" | cut -d':' -f2)" + [ -z "$uri" ] && continue + uri="$(get_auth "${uri//"'"/}")" + IFS=':' read -r hash_algo checksum _ <<<"$checksum_string" filename_decoded="$(urldecode "$filename")" - DOWNLOAD_DISPLAY="${DOWNLOAD_DISPLAY}$(echo "$filename_decoded" | cut -d'_' -f1)" - DOWNLOAD_DISPLAY="${DOWNLOAD_DISPLAY} $(echo "$filename_decoded" | cut -d'_' -f2)" + IFS='_' read -r pkg_name_decoded pkg_version_decoded _ <<<"$filename_decoded" + DOWNLOAD_DISPLAY="${DOWNLOAD_DISPLAY}$pkg_name_decoded $pkg_version_decoded" DOWNLOAD_DISPLAY="${DOWNLOAD_DISPLAY} $(echo "$filesize" | numfmt --to=iec-i --suffix=B)\n" DOWNLOAD_SIZE=$((DOWNLOAD_SIZE + filesize)) @@ -370,35 +433,38 @@ get_uris(){ # Using apt-cache show package=version to ensure recover single and # correct package version. # Warning: assuming that package naming uses '_' as field separator. - # Therefore, this code expects package-name_version_arch.deb Otherways + # Therefore, this code expects package-name_version_arch.deb Otherwise # below code will fail resoundingly if [ -z "$hash_algo" ]; then - pkg_name="$(echo "$filename" | cut -d'_' -f1)" - pkg_version="$(echo "$filename" | cut -d'_' -f2)" - pkg_version="$(urldecode "$pkg_version")" - package_info="$(apt-cache show "$pkg_name=$pkg_version")" - - patch_checksum= - if [ -n "$SHA512_SUPPORTED" ]; then - patch_checksum="$(echo "$package_info" | grep SHA512 | head -n 1)" - [ -n "$patch_checksum" ] && hash_algo="sha-512" - fi - if [ -z "$patch_checksum" ] && [ -n "$SHA256_SUPPORTED" ]; then - patch_checksum="$(echo "$package_info" | grep SHA256 | head -n 1)" - [ -n "$patch_checksum" ] && hash_algo="sha-256" - fi - if [ -z "$patch_checksum" ] && [ -n "$SHA1_SUPPORTED" ]; then - patch_checksum="$(echo "$package_info" | grep SHA1 | head -n 1)" - [ -n "$patch_checksum" ] && hash_algo="sha-1" - fi - if [ -z "$patch_checksum" ] && [ -n "$MD5sum_SUPPORTED" ]; then - patch_checksum="$(echo "$package_info" | grep MD5sum | head -n 1)" - [ -n "$patch_checksum" ] && hash_algo="md5" - fi - - if [ -n "$patch_checksum" ]; then - checksum="$(echo "$patch_checksum" | cut -d' ' -f2)" - else + IFS='_' read -r pkg_name _ <<<"$filename" + pkg_version="$pkg_version_decoded" + # Transform multi-line field output from apt-cache to single line and sort checksums, strongest first + package_info="$(apt-cache show "$pkg_name=$pkg_version" | sed ':r;$!{N;br};s/\n / /g' | sort -r)" + + while IFS=': ' read -r field checksum _ + do + case "$field" in + SHA512) + [ -n "$SHA512_SUPPORTED" ] || continue + hash_algo="sha-512" + break ;; + SHA256) + [ -n "$SHA256_SUPPORTED" ] || continue + hash_algo="sha-256" + break ;; + SHA1) + [ -n "$SHA1_SUPPORTED" ] || continue + hash_algo="sha-1" + break ;; + MD5sum) + [ -n "$MD5sum_SUPPORTED" ] || continue + hash_algo="md5" + break ;; + esac + done <<<"$package_info" + + if [ -z "$hash_algo" ]; then + checksum= msg "Couldn't get supported checksum for $pkg_name ($pkg_version)." "warning" REMOVE_WORKING_MESSAGE= fi @@ -418,7 +484,6 @@ get_uris(){ done <<<"$(echo "$uris_full" | grep -E "^'(http(s|)|(s|)ftp)://")" #cat "$DLLIST" - #LCK_RM #exit } @@ -429,11 +494,8 @@ display_downloadfile(){ DISPLAY_SORT_OPTIONS=(-k 1,1) # Sort output after package download size (decreasing): #DISPLAY_SORT_OPTIONS=(-k 3,3 -hr) - while read -r line; do - [ -z "$line" ] && continue - pkg="$(echo "$line" | cut -d' ' -f1)" - ver="$(echo "$line" | cut -d' ' -f2)" - size="$(echo "$line" | cut -d' ' -f3)" + while IFS=' ' read -r pkg ver size _; do + [ -z "$pkg" ] && continue printf '%s%-40s %-20s %10s\n' "$aptfast_prefix" "$pkg" "$ver" "$size" done <<<"$(echo -e "$DOWNLOAD_DISPLAY" | sort "${DISPLAY_SORT_OPTIONS[@]}")" fi @@ -508,6 +570,8 @@ while true; do d) DOWNLOAD_ONLY=true ;; + *) + ;; esac done ((OPTIND++))