From d5fc45af76da2e3844678074615fb98f392c11db Mon Sep 17 00:00:00 2001 From: Sergio Durigan Junior Date: Sat, 22 Nov 2025 23:23:39 -0500 Subject: [PATCH 1/2] Introduce --curl-binary option It may be important to be able to specify a different curl binary to be invoked (e.g., when performing tests against different versions of curl, or when curl isn't installed in PATH). This commit introduces a new "--curl-binary" option. If it's not specified, wcurl defaults to using "curl". Signed-off-by: Sergio Durigan Junior --- tests/curl-mock-version | 34 ++++++++++++++++++++++++++++++++++ tests/tests.sh | 13 +++++++++++++ wcurl | 23 +++++++++++++++++++---- wcurl.1 | 6 ++++-- wcurl.md | 8 ++++++-- 5 files changed, 76 insertions(+), 8 deletions(-) create mode 100755 tests/curl-mock-version diff --git a/tests/curl-mock-version b/tests/curl-mock-version new file mode 100755 index 0000000..2b9e67a --- /dev/null +++ b/tests/curl-mock-version @@ -0,0 +1,34 @@ +#!/bin/sh + +# wcurl - a simple wrapper around curl to easily download files. +# +# This is wcurl's mock curl binary. +# +# Copyright (C) Samuel Henrique , Sergio Durigan +# Junior and many contributors, see the AUTHORS +# file. +# +# Permission to use, copy, modify, and distribute this software for any purpose +# with or without fee is hereby granted, provided that the above copyright +# notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN +# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +# OR OTHER DEALINGS IN THE SOFTWARE. +# +# Except as contained in this notice, the name of a copyright holder shall not be +# used in advertising or otherwise to promote the sale, use or other dealings in +# this Software without prior written authorization of the copyright holder. +# +# SPDX-License-Identifier: curl + +# This is a simple binary to mock curl's version string. It will +# print a version specified in the CURL_MOCK_VERSION variable and just +# exit. If CURL_MOCK_VERSION is empty, print 0.0.0. + +printf "%s\n" "curl ${CURL_MOCK_VERSION:-0.0.0}" +exit 0 diff --git a/tests/tests.sh b/tests/tests.sh index ccfc23b..2b44d5c 100755 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -233,6 +233,19 @@ testUrlDecodingNonLatinLanguages() assertContains "Verify whether 'wcurl' successfully decodes percent-encoded Korean in URLs" "${ret}" '퍼센트_인코딩' } +testCurlBinaryOption() +{ + url='example.com' + ret=$(${WCURL_CMD} --dry-run "${url}") + bin=$(printf "%s" "${ret}" | head -n1) + assertEquals "Verify whether 'wcurl' invokes 'curl' when '--curl-binary' is not provided" "${bin}" "curl" + + curlbin="${ROOTDIR}/tests/curl-mock-version" + ret=$(${WCURL_CMD} --dry-run --curl-binary "${curlbin}" "${url}") + bin=$(printf "%s" "${ret}" | head -n1) + assertContains "Verify whether 'wcurl' invokes the binary specified by '--curl-binary'" "${bin}" "${curlbin}" +} + ## Ideas for tests: ## ## - URL with whitespace diff --git a/wcurl b/wcurl index c5dd6d1..70df992 100755 --- a/wcurl +++ b/wcurl @@ -49,8 +49,8 @@ usage() ${PROGRAM_NAME} -- a simple wrapper around curl to easily download files. Usage: ${PROGRAM_NAME} ... - ${PROGRAM_NAME} [--curl-options ]... [--no-decode-filename] [-o|-O|--output ] [--dry-run] [--] ... - ${PROGRAM_NAME} [--curl-options=]... [--no-decode-filename] [--output=] [--dry-run] [--] ... + ${PROGRAM_NAME} [--curl-options ]... [--curl-binary ] [--no-decode-filename] [-o|-O|--output ] [--dry-run] [--] ... + ${PROGRAM_NAME} [--curl-options=]... [--curl-binary=] [--no-decode-filename] [--output=] [--dry-run] [--] ... ${PROGRAM_NAME} -h|--help ${PROGRAM_NAME} -V|--version @@ -59,6 +59,8 @@ Options: --curl-options : Specify extra options to be passed when invoking curl. May be specified more than once. + --curl-binary : Specify the curl binary to be used. By default, "curl" is used. + -o, -O, --output : Use the provided output path instead of getting it from the URL. If multiple URLs are provided, resulting files share the same name with a number appended to the end (curl >= 7.83.0). If this option is provided @@ -89,6 +91,9 @@ error() exit 1 } +# The curl binary to be used. +CURL_BINARY="curl" + # Extra curl options provided by the user. # This is set per-URL for every URL provided. # Some options are global, but we are erroring on the side of needlessly setting @@ -130,7 +135,7 @@ sanitize() error "You must provide at least one URL to download." fi - readonly CURL_OPTIONS URLS DRY_RUN HAS_USER_SET_OUTPUT + readonly CURL_OPTIONS CURL_BINARY URLS DRY_RUN HAS_USER_SET_OUTPUT } # Indicate via exit code whether the string given in the first parameter @@ -201,7 +206,7 @@ get_url_filename() # Execute curl with the list of URLs provided by the user. exec_curl() { - CMD="curl " + CMD="${CURL_BINARY} " # Store version to check if it supports --no-clobber, --parallel and --parallel-max-host. curl_version=$($CMD --version | cut -f2 -d' ' | head -n1) @@ -285,6 +290,16 @@ while [ -n "${1-}" ]; do CURL_OPTIONS="${CURL_OPTIONS} ${1}" ;; + --curl-binary=*) + opt=$(printf "%s\n" "${1}" | sed 's/^--curl-binary=//') + CURL_BINARY="${opt}" + ;; + + --curl-binary) + shift + CURL_BINARY="${1}" + ;; + --dry-run) DRY_RUN="true" ;; diff --git a/wcurl.1 b/wcurl.1 index 99d0ab2..a5d6b02 100644 --- a/wcurl.1 +++ b/wcurl.1 @@ -5,9 +5,9 @@ .SH SYNOPSIS \fBwcurl ...\fP -\fBwcurl [\--curl\-options ]... [\--dry\-run] [\--no\-decode\-filename] [\-o|\-O|\--output ] [\--] ...\fP +\fBwcurl [\--curl\-options ]... [\--curl\-binary ] [\--dry\-run] [\--no\-decode\-filename] [\-o|\-O|\--output ] [\--] ...\fP -\fBwcurl [\--curl\-options=]... [\--dry\-run] [\--no\-decode\-filename] [\--output=] [\--] ...\fP +\fBwcurl [\--curl\-options=]... [\--curl\-binary=] [\--dry\-run] [\--no\-decode\-filename] [\--output=] [\--] ...\fP \fBwcurl \-V|\--version\fP @@ -61,6 +61,8 @@ if there is none in the URL. .IP "--curl-options, --curl-options=\..." Specify extra options to be passed when invoking curl. May be specified more than once. +.IP "--curl-binary, --curl-binary=\..." +Specify the curl binary to be used. By default, "curl" is used. .IP "-o, -O, --output, --output=\" Use the provided output path instead of getting it from the URL. If multiple URLs are provided, resulting files share the same name with a number appended to diff --git a/wcurl.md b/wcurl.md index ab5c3aa..c91c9b3 100644 --- a/wcurl.md +++ b/wcurl.md @@ -18,9 +18,9 @@ Added-in: n/a **wcurl \...** -**wcurl [--curl-options \]... [--dry-run] [--no-decode-filename] [-o|-O|--output \] [--] \...** +**wcurl [--curl-options \]... [--curl-binary \] [--dry-run] [--no-decode-filename] [-o|-O|--output \] [--] \...** -**wcurl [--curl-options=\]... [--dry-run] [--no-decode-filename] [--output=\] [--] \...** +**wcurl [--curl-options=\]... [--curl-binary=\] [--dry-run] [--no-decode-filename] [--output=\] [--] \...** **wcurl -V|--version** @@ -78,6 +78,10 @@ By default, **wcurl** does: Specify extra options to be passed when invoking curl. May be specified more than once. +## --curl-binary, --curl-binary=\ + +Specify the curl binary to be used. By default, "curl" is used. + ## -o, -O, --output, --output=\ Use the provided output path instead of getting it from the URL. If multiple From 87ec4a9b3cedd3468765bc3169bf279bd0657135 Mon Sep 17 00:00:00 2001 From: Sergio Durigan Junior Date: Fri, 21 Nov 2025 14:45:38 -0500 Subject: [PATCH 2/2] Improve wcurl version comparison mechanism We've been having a some problems with version comparison in the script. The approach we've taken so far is to decompose the current curl version into two variables (major and minor), and then perform comparisons against these components separately. It works, but it's confusing. Another possible approach would be to use a C-style version normalization (basically a printf "%02d%02d%02d") and then perform comparisons against the versions we want. The problem is that these versions need also to be normalized, which can be confusing as well. I decided to implement the second approach but abstract it as a simple function that can take a regular version string like "8.16.0" as well as a comparison operator that will then be passed onto to "test". This reads much nicer and abstracts the complexities of version normalization away. Unfortunately due to the limitations of shell scripting it's not easy to deduplicate code in this scenario, but that should be OK for now. Signed-off-by: Sergio Durigan Junior --- tests/tests.sh | 29 ++++++++++++++++++ wcurl | 81 ++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 88 insertions(+), 22 deletions(-) diff --git a/tests/tests.sh b/tests/tests.sh index 2b44d5c..8ea3c83 100755 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -246,6 +246,35 @@ testCurlBinaryOption() assertContains "Verify whether 'wcurl' invokes the binary specified by '--curl-binary'" "${bin}" "${curlbin}" } +testCurlVersionComparison() +{ + # Verify that using a very old curl version correctly omits + # --parallel, --no-clobber and --parallel-max-host. + url='example.com' + curlbin="${ROOTDIR}/tests/curl-mock-version" + ret=$(CURL_MOCK_VERSION="1.0.0" ${WCURL_CMD} --dry-run --curl-binary "${curlbin}" "${url}" "${url}") + assertNotContains "Verify whether 'wcurl' correctly omits --parallel when using very old curl" "${ret}" '--parallel' + assertNotContains "Verify whether 'wcurl' correctly omits --no-clobber when using very old curl" "${ret}" '--no-clobber' + assertNotContains "Verify whether 'wcurl' correctly omits --parallel-max-host when using very old curl" "${ret}" '--parallel-max-host' + + # Verify that using a curl version that's >= 7.66.0 and < 7.83.0 + # correctly adds --parallel but omits --no-clobber and --parallel-max-host. + ret=$(CURL_MOCK_VERSION="7.66.0" ${WCURL_CMD} --dry-run --curl-binary "${curlbin}" "${url}" "${url}") + assertContains "Verify whether 'wcurl' correctly adds --parallel when using curl >= 7.66.0" "${ret}" '--parallel' + assertNotContains "Verify whether 'wcurl' correctly omits --no-clobber when using curl >= 7.66.0 && curl < 7.83.0" "${ret}" '--no-clobber' + assertNotContains "Verify whether 'wcurl' correctly omits --parallel-max-host when using curl >= 7.66.0 && curl < 8.16.0" "${ret}" '--parallel-max-host' + + # Verify that using a curl version that's >= 7.83.0 and < 8.16.0 correctly adds --no-clobber but omits --parallel-max-host. + ret=$(CURL_MOCK_VERSION="7.83.0" ${WCURL_CMD} --dry-run --curl-binary "${curlbin}" "${url}" "${url}") + assertContains "Verify whether 'wcurl' correctly adds --no-clobber when using curl >= 7.83.0" "${ret}" '--no-clobber' + assertNotContains "Verify whether 'wcurl' correctly omits --parallel-max-host when using curl >= 7.83.0 && curl < 8.16.0" "${ret}" '--parallel-max-host' + + # Verify that using a curl version that's >= 8.16.0 correctly adds + # --parallel-max-host. + ret=$(CURL_MOCK_VERSION="8.16.0" ${WCURL_CMD} --dry-run --curl-binary "${curlbin}" "${url}" "${url}") + assertContains "Verify whether 'wcurl' correctly adds --parallel-max-host. when using curl >= 8.16.0" "${ret}" '--parallel-max-host' +} + ## Ideas for tests: ## ## - URL with whitespace diff --git a/wcurl b/wcurl index 70df992..060f3bf 100755 --- a/wcurl +++ b/wcurl @@ -128,6 +128,13 @@ readonly UNSAFE_PERCENT_ENCODE="%2F %5C" # Whether to invoke curl or not. DRY_RUN="false" +# The current version of curl. +CURL_VERSION="" +# The normalized curl version, in the format "XXYYZZ", where "XX" is +# the zero-padded major version, "YY" is the zero-padded minor +# version and "ZZ" is the zero-padded patch version. +CURL_NORMALIZED_VERSION="" + # Sanitize parameters. sanitize() { @@ -135,7 +142,35 @@ sanitize() error "You must provide at least one URL to download." fi - readonly CURL_OPTIONS CURL_BINARY URLS DRY_RUN HAS_USER_SET_OUTPUT + CURL_VERSION=$(${CURL_BINARY} --version | head -n1 | cut -f2 -d' ') + if [ -z "${CURL_VERSION}" ]; then + error "Unable to determine curl version. Is curl installed?" + fi + + CURL_NORMALIZED_VERSION=$(normalize_version "${CURL_VERSION}") + + readonly CURL_OPTIONS \ + CURL_BINARY \ + URLS \ + DRY_RUN \ + HAS_USER_SET_OUTPUT \ + CURL_VERSION \ + CURL_NORMALIZED_VERSION +} + +# Print the normalized format of a version specified as the first argument. +# +# The normalized version has the format "XXYYZZ", where "XX" is the +# zero-padded major version, "YY" is the zero-padded minor version and +# "ZZ" is the zero-padded patch version. +normalize_version() +{ + version="${1}" + vermaj=$(printf "%s" "${version}" | cut -f1 -d.) + vermin=$(printf "%s" "${version}" | cut -f2 -d.) + verpatch=$(printf "%s" "${version}" | cut -f3 -d.) + + printf "%02d%02d%02d" "${vermaj}" "${vermin}" "${verpatch}" } # Indicate via exit code whether the string given in the first parameter @@ -203,36 +238,38 @@ get_url_filename() # No slash means there was just a hostname and no path; return empty string. } +# Given a version (in the format MAJOR.MINOR) as the first argument +# and an operator as the second argument, perform a comparison against +# the current curl version. +compare_curl_version() +{ + # Any of: -lt, -le, -eq, -gt, -ge + operator="${1}" + version="${2}" + version_to_compare=$(normalize_version "${version}") + + test "${CURL_NORMALIZED_VERSION}" "${operator}" "${version_to_compare}" +} + # Execute curl with the list of URLs provided by the user. exec_curl() { CMD="${CURL_BINARY} " - # Store version to check if it supports --no-clobber, --parallel and --parallel-max-host. - curl_version=$($CMD --version | cut -f2 -d' ' | head -n1) - curl_version_major=$(echo "$curl_version" | cut -f1 -d.) - curl_version_minor=$(echo "$curl_version" | cut -f2 -d.) - CURL_NO_CLOBBER="" CURL_PARALLEL="" - if [ "${curl_version_major}" -ge 8 ]; then + # --parallel is only supported since 7.66.0. + if compare_curl_version -ge "7.66.0"; then + CURL_PARALLEL="--parallel" + fi + # --no-clobber is only supported since 7.83.0. + if compare_curl_version -ge "7.83.0"; then CURL_NO_CLOBBER="--no-clobber" - CURL_PARALLEL="--parallel --parallel-max-host 5" - - # --parallel-max-host is only supported since 8.16.0. - if [ "${curl_version_major}" -eq 8 ] && [ "${curl_version_minor}" -lt 16 ]; then - CURL_PARALLEL="--parallel" - fi - elif [ "${curl_version_major}" -eq 7 ]; then - # --no-clobber is only supported since 7.83.0. - if [ "${curl_version_minor}" -ge 83 ]; then - CURL_NO_CLOBBER="--no-clobber" - fi - # --parallel is only supported since 7.66.0. - if [ "${curl_version_minor}" -ge 66 ]; then - CURL_PARALLEL="--parallel" - fi + fi + # --parallel-max-host is only supported since 8.16.0. + if compare_curl_version -ge "8.16.0"; then + CURL_PARALLEL="${CURL_PARALLEL} --parallel-max-host 5" fi # Detecting whether we need --parallel. It is easier to rely on