Added download cache

uniget-org · Sep 16, 2024 · cfc074d · cfc074d
1 parent ee60baf
commit cfc074d
Show file tree

Hide file tree

Showing 808 changed files with 7,163 additions and 3,558 deletions.
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -24,6 +24,11 @@ jobs:
       tools: ${{ steps.collect.outputs.tools }}
     steps:
 
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
     - name: Install uniget
       uses: uniget-org/uniget-action@v1
       with:
@@ -56,7 +61,44 @@ jobs:
             | helper/usr/local/bin/jq --slurp --compact-output '.'
         )"
         echo "    JSON: ${TOOLS_JSON}"
+
+        TOOLS_JSON_LEN="$(
+            echo -n "${TOOLS}" \
+            | jq 'length'
+        )"
+        echo "    TOOLS_JSON_LEN=${TOOLS_JSON_LEN}"
+
+        if test "${TOOLS_JSON_LEN}" -ge 256; then
+            echo "### Too many tools to process: ${TOOLS_JSON_LEN}."
+            echo "### Fallback to last commit"
+
+            TOOLS="$(
+                git log -1 --pretty=format: --name-only --diff-filter=d \
+                | grep ^tools/ \
+                | cut -d/ -f2 \
+                | sort \
+                | uniq \
+                | xargs echo
+            )"
+            echo "### Got tools=${TOOLS}."
+            
+            TOOLS_JSON="$(
+                echo -n "${TOOLS}" \
+                | tr ' ' '\n' \
+                | helper/usr/local/bin/jq --raw-input '.' \
+                | helper/usr/local/bin/jq --slurp --compact-output '.'
+            )"
+            echo "    JSON: ${TOOLS_JSON}"
+
+            TOOLS_JSON_LEN="$(
+                echo -n "${TOOLS}" \
+                | jq 'length'
+            )"
+            echo "    TOOLS_JSON_LEN=${TOOLS_JSON_LEN}"
+        fi
+
         echo "tools=${TOOLS_JSON}" >>"${GITHUB_OUTPUT}"
+        echo "tools_len=${TOOLS_JSON_LEN}" >>"${GITHUB_OUTPUT}"
 
   build:
     name: Build container image

diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml
@@ -66,7 +66,7 @@ jobs:
             CHANGED_TOOLS="$(
                 git log --pretty=format: --name-only --diff-filter=d ${OLD_COMMIT_SHA}..${GITHUB_SHA} \
                 | sort \
-                | grep -E "^tools/[^/]+/" \
+                | grep "^tools/" \
                 | cut -d/ -f2 \
                 | uniq \
                 | while read -r TOOL; do test -d "tools/${TOOL}" && echo "${TOOL}"; done \

diff --git a/@template/Dockerfile.template b/@template/Dockerfile.template
@@ -18,7 +18,7 @@ ARG name
 ARG version
 
 # tarball
-#RUN <<EOF
+#RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
 #echo "### Setting architecture string for ${TARGETPLATFORM}"
 #case "${arch}" in
 #    x86_64)
@@ -34,14 +34,20 @@ ARG version
 #esac
 #echo "    Using ${arch_suffix}"
 #
-#check-github-release-asset "foo/bar" "v${version}" "foo-${version}-linux-amd64.tar.gz"
-#check-download ""
-#curl --silent --show-error --location --fail "" \
-#| tar --extract --gzip --directory="${prefix}/bin/" --no-same-owner
+#url=""
+#filename="$( basename "${url}" )"
+#
+#check-download "${url}"
+#check-github-release-asset "foo/bar" "v${version}" "${filename}"
+#curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+#    "${url}"
+#
+#tar --file="${uniget_cache_download}/${filename}" --list
+#tar --file="${uniget_cache_download}/${filename}" --extract --gzip --directory="${prefix}/bin" --no-same-owner
 #EOF
 
 # executable
-#RUN <<EOF
+#RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
 #echo "### Setting architecture string for ${TARGETPLATFORM}"
 #case "${arch}" in
 #    x86_64)
@@ -57,15 +63,21 @@ ARG version
 #esac
 #echo "    Using ${arch_suffix}"
 #
-#check-github-release-asset "foo/bar" "v${version}" "foo-${version}-linux-amd64.tar.gz"
-#check-download ""
-#curl --silent --show-error --location --fail --output "${prefix}/bin/foo" \
-#    ""
-#chmod +x "${prefix}/bin/foo"
+#url=""
+#filename="$( basename "${url}" )"
+#
+#check-download "${url}"
+#check-github-release-asset "foo/bar" "v${version}" "${filename}"
+#curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+#    "${url}"
+#
+#install --mode=0755 \
+#    "${uniget_cache_download}/${filename}" \
+#    "${prefix}/bin/foo"
 #EOF
 
 # zip
-#RUN <<EOF
+#RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
 #echo "### Setting architecture string for ${TARGETPLATFORM}"
 #case "${arch}" in
 #    x86_64)
@@ -81,19 +93,33 @@ ARG version
 #esac
 #echo "    Using ${arch_suffix}"
 #
-#check-github-release-asset "foo/bar" "v${version}" "foo-${version}-linux-amd64.tar.gz"
-#check-download ""
 #url=""
 #filename="$(basename "${url}")"
-#curl --silent --show-error --location --fail --remote-name "${url}"
-#unzip -q -o -d "${prefix}/bin" "${filename}"
+#
+#check-download "${url}"
+#check-github-release-asset "foo/bar" "v${version}" "${filename}"
+#curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+#    "${url}"
+#
+#unzip -q -o -d "${prefix}/bin" "${uniget_cache_download}/${filename}"
 #EOF
 
 # go
+#RUN --mount=type=cache,target=/var/cache/apt <<EOF
+#apt-get update
+#apt-get -y install --no-install-recommends \
+#    foo
+#EOF
+#RUN --mount=type=cache,target=/var/cache/apk <<EOF
+#apk add --update-cache \
+#    foo
+#EOF
 #COPY --link --from=go / /usr/local/
 #COPY --link --from=make / /usr/local/
 #WORKDIR /go/src/github.com/foo/bar
-#RUN <<EOF
+#ARG name
+#ARG version
+#RUN --mount=type=cache,target=/root/go/pkg/mod <<EOF
 #check-clone "https://github.com/foo/bar" "v${version}"
 #git clone -q --config advice.detachedHead=false --depth 1 --branch "v${version}" https://github.com/foo/bar .
 #make
@@ -103,7 +129,7 @@ ARG version
 # rust
 #COPY --link --from=rust / /usr/local/
 #WORKDIR /tmp/github.com/foo/bar
-#RUN <<EOF
+#RUN <<EOF # no download cache
 #git clone -q --config advice.detachedHead=false --depth 1 --branch "${version}" https://github.com/foo/bar .
 #export CARGO_HOME=/usr/local/cargo
 #export RUSTUP_HOME=/usr/local/rustup
@@ -115,14 +141,7 @@ ARG version
 # python
 #COPY --link --from=python / /usr/local/
 #COPY --link --from=shiv / /usr/local/
-#RUN <<EOF
-#apt-get update
-#apt-get -y install --no-install-recommends \
-#    python3-pip \
-#    python3-venv \
-#    python-is-python3
-#EOF
-#RUN <<EOF
+#RUN --mount=type=cache,target=/root/.cache/pip <<EOF # no download cache
 #shiv --output-file "${prefix}/bin/foo" --console-script foo "foo==${version}"
 #EOF
 
@@ -132,30 +151,39 @@ ARG version
 #WORKDIR /uniget_bootstrap/libexec/foo
 #ARG name
 #ARG version
-#RUN <<EOF
+#RUN <<EOF # no download cache
 #npm install \
 #    --omit=dev \
 #    "foo@${version}"
 #ln --symbolic --relative --force "${prefix}/libexec/foo/node_modules/.bin/foo" "${prefix}/bin/"
 #EOF
 
 # c
+#RUN --mount=type=cache,target=/var/cache/apt <<EOF
+#apt-get update
+#apt-get -y install --no-install-recommends \
+#    foo
+#EOF
+#RUN --mount=type=cache,target=/var/cache/apk <<EOF
+#apk add --update-cache \
+#    foo
+#EOF
 #COPY --link --from=make / /usr/local/
 #WORKDIR /tmp/foo
-#RUN <<EOF
+#RUN <<EOF # no download cache
 #git clone -q --config advice.detachedHead=false --depth 1 --branch "${version}" https://github.com/foo/bar .
 #./configure --prefix="${prefix}"
 #make LDFLAGS=-static
 #make install
 #EOF
 
 # dummy for testing
-#RUN <<EOF
+#RUN <<EOF # no download cache
 #touch "${prefix}/foo"
 #EOF
 
 # completion
-#RUN <<EOF
+#RUN <<EOF # no download cache
 #"${prefix}/bin/foo" completion bash >"${prefix}/share/bash-completion/completions/foo"
 #"${prefix}/bin/foo" completion fish >"${prefix}/share/fish/vendor_completions.d/foo.fish"
 #"${prefix}/bin/foo" completion zsh >"${prefix}/share/zsh/vendor-completions/_foo"

diff --git a/scripts/rewrite-add-download-cache.sh b/scripts/rewrite-add-download-cache.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+set -o errexit
+
+find tools -type f -name Dockerfile.template \
+| sort \
+| while read -r FILENAME; do
+
+    if grep -P -z -q 'RUN <<EOF # no download cache\n' "${FILENAME}"; then
+        #echo "Ignoring ${FILENAME}"
+        continue
+    fi
+
+    if grep -P -z -q 'RUN <<EOF\nnpm install ' "${FILENAME}"; then
+        #echo "Ignoring ${FILENAME}"
+        continue
+    fi
+
+    if grep -P -z -q 'RUN <<EOF\nshiv ' "${FILENAME}"; then
+        #echo "Ignoring ${FILENAME}"
+        continue
+    fi
+
+    if grep -q "^unzip -q " "${FILENAME}"; then
+        sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?url="([^"]+)"\nfilename=".+"\ncurl --silent --show-error --location --fail --remote-name "([^"]+)"\nunzip -q -o -d "([^"]+)" "([^"]+)"@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\4"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n    "${url}"\n\nunzip -t "${uniget_cache_download}/${filename}"\nunzip -q -o -d "\6" "${uniget_cache_download}/${filename}"@' "${FILENAME}"
+        continue
+    fi
+
+    if grep -P -z -q 'RUN <<EOF\ncurl ' "${FILENAME}"; then
+        echo "Missing handler ${FILENAME}"
+        continue
+    fi
+
+    if grep -P -z -q 'RUN <<EOF\ngit clone ' "${FILENAME}"; then
+        #sed -E -z -i 's@RUN <<EOF\ngit clone -q --config advice.detachedHead=false --depth 1 --branch "([^"]+)" https://github.com/([^ ]+) .@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="https://github.com/\2/archive/refs/tags/\1.tar.gz"\nfilename="$( basename "${url}" )"\n\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n    "${url}"\n@' "${FILENAME}"
+        #echo "Ignoring ${FILENAME}"
+        continue
+    fi
+
+    if grep -P -z -q 'RUN <<EOF\ncheck-github-release-asset "[^"]+" "[^"]+" "[^"]+"\n\n?curl --silent --show-error --location --fail "[^"]+" \\\n| tar' "${FILENAME}"; then
+        sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?curl --silent --show-error --location --fail "([^"]+)" \\\n\| tar @RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\4"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n    "${url}"\n\ntar --file "${uniget_cache_download}/${filename}" --list\ntar --file "${uniget_cache_download}/${filename}" @' "${FILENAME}"
+        continue
+    fi
+
+    if grep -P -z -q 'RUN <<EOF\ncheck-github-release-asset "[^"]+" "[^"]+" "[^"]+"\n\n?curl --silent --show-error --location --fail --output "[^"]+" \\\n\s+"[^"]+"' "${FILENAME}"; then
+        sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?curl --silent --show-error --location --fail --output "([^"]+)" \\\n\s+"([^"]+)"\nchmod \+x "([^"]+)"@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\5"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n    "${url}"\n\ninstall --mode=0755 \\\n    "${uniget_cache_download}/${filename}" \\\n    "\6"\n@' "${FILENAME}"
+        continue
+    fi
+
+    if grep -P -z -q 'RUN <<EOF\ncheck-github-release-asset "[^"]+" "[^"]+" "[^"]+"\n' "${FILENAME}"; then
+        sed -E -z -i 's@RUN <<EOF\ncheck-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\ncheck-github-release-asset "\1" "\2" "\3"\n@' "${FILENAME}"
+        continue
+    fi
+
+    if grep -P -z -q 'RUN <<EOF\ncheck-download "[^"]+"\n\n?curl --silent --show-error --location --fail --output "[^"]+" \\\n\s+"[^"]+"' "${FILENAME}"; then
+        sed -E -z -i 's@RUN <<EOF\ncheck-download "([^"]+)"\n\n?curl --silent --show-error --location --fail --output "([^"]+)" \\\n\s+"([^"]+)"\nchmod \+x "([^"]+)"@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\nurl="\1"\nfilename="$( basename "${url}" )"\n\ncheck-download "${url}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n    "${url}"\n\ninstall --mode=0755 \\\n    "${uniget_cache_download}/${filename}" \\\n    "\4"\n@' "${FILENAME}"
+        continue
+    fi
+
+    if grep -P -z -q 'RUN <<EOF\necho "### Setting' "${FILENAME}" && grep -P -z -q 'check-github-release-asset' "${FILENAME}"; then
+        sed -E -z -i 's@echo "### Downloading [^"]+"\n@@; s@RUN <<EOF\n@RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF\n@; s@check-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?curl --silent --show-error --location --fail "([^"]+)" \\\n\| tar @url="\4"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n    "${url}"\n\ntar --file "${uniget_cache_download}/${filename}" --list\ntar --file "${uniget_cache_download}/${filename}" @; s@check-github-release-asset "([^"]+)" "([^"]+)" "([^"]+)"\n\n?curl --silent --show-error --location --fail --output "([^"]+)" \\\n\s+"([^"]+)"\nchmod \+x "([^"]+)"@url="\5"\nfilename="$( basename "${url}" )"\n\ncheck-github-release-asset "\1" "\2" "${filename}"\ncurl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \\\n    "${url}"\n\ninstall --mode=0755 \\\n    "${uniget_cache_download}/${filename}" \\\n    "\6"\n@' "${FILENAME}"
+        continue
+    fi
+
+    if grep -q "RUN <<EOF" "${FILENAME}"; then
+        echo "Unhandled ${FILENAME}"
+    fi
+
+done
diff --git a/tools/acorn/Dockerfile.template b/tools/acorn/Dockerfile.template
@@ -7,11 +7,16 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
 SHELL [ "bash", "-clo", "errexit" ]
 ARG name
 ARG version
-RUN <<EOF
-check-github-release-asset "acorn-io/runtime" "v${version}" "acorn-v${version}-linux-${alt_arch}.tar.gz"
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://github.com/acorn-io/runtime/releases/download/v${version}/acorn-v${version}-linux-${alt_arch}.tar.gz"
+filename="$( basename "${url}" )"
 
-curl --silent --show-error --location --fail "https://github.com/acorn-io/runtime/releases/download/v${version}/acorn-v${version}-linux-${alt_arch}.tar.gz" \
-| tar --extract --gzip --directory="${prefix}/bin/" --no-same-owner \
+check-github-release-asset "acorn-io/runtime" "v${version}" "${filename}"
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+
+tar --file="${uniget_cache_download}/${filename}" --list
+tar --file="${uniget_cache_download}/${filename}" --extract --gzip --directory="${prefix}/bin/" --no-same-owner \
     acorn
 
 "${prefix}/bin/acorn" completion bash >"${prefix}/share/bash-completion/completions/acorn"

diff --git a/tools/acr/Dockerfile.template b/tools/acr/Dockerfile.template
@@ -8,12 +8,12 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
     /etc/profile.d/ \
     /etc/profile.d/
 SHELL [ "bash", "-clo", "errexit" ]
-ARG name
-ARG version
 COPY --link --from=go / /usr/local/
 COPY --link --from=make / /usr/local/
 WORKDIR /go/src/github.com/Azure/acr-cli
-RUN <<EOF
+ARG name
+ARG version
+RUN <<EOF # no download cache
 git clone -q --config advice.detachedHead=false --depth 1 --branch "v${version}" https://github.com/Azure/acr-cli .
 make -j$(nproc) binaries GO_LDFLAGS="-ldflags '-s -w -linkmode external -extldflags -static'"
 cp bin/acr "${prefix}/bin/"

diff --git a/tools/act/Dockerfile.template b/tools/act/Dockerfile.template
@@ -7,9 +7,15 @@ COPY --from=ghcr.io/uniget-org/tools/uniget-build:latest \
 SHELL [ "bash", "-clo", "errexit" ]
 ARG name
 ARG version
-RUN <<EOF
-check-github-release-asset "nektos/act" "v${version}" "act_Linux_x86_64.tar.gz"
-curl --silent --show-error --location --fail "https://github.com/nektos/act/releases/download/v${version}/act_Linux_${arch}.tar.gz" \
-| tar --extract --gzip --directory="${prefix}/bin" --no-same-owner \
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
+url="https://github.com/nektos/act/releases/download/v${version}/act_Linux_${arch}.tar.gz"
+filename="$( basename "${url}" )"
+
+check-github-release-asset "nektos/act" "v${version}" "${filename}"
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+
+tar --file="${uniget_cache_download}/${filename}" --list
+tar --file="${uniget_cache_download}/${filename}" --extract --gzip --directory="${prefix}/bin" --no-same-owner \
     act
 EOF
diff --git a/tools/actions-runner/Dockerfile.template b/tools/actions-runner/Dockerfile.template
@@ -8,7 +8,7 @@ SHELL [ "bash", "-clo", "errexit" ]
 WORKDIR /uniget_bootstrap/opt/actions-runner
 ARG name
 ARG version
-RUN <<EOF
+RUN --mount=type=cache,target=/var/cache/uniget/download <<EOF
 echo "### Setting architecture string for ${TARGETPLATFORM}"
 case "${arch}" in
     x86_64)
@@ -24,9 +24,13 @@ case "${arch}" in
 esac
 echo "    Using ${arch_suffix}"
 
-check-github-release-asset "actions/runner" "v${version}" "actions-runner-linux-${arch_suffix}-${version}.tar.gz"
+url="https://github.com/actions/runner/releases/download/v${version}/actions-runner-linux-${arch_suffix}-${version}.tar.gz"
+filename="$( basename "${url}" )"
 
-echo "### Downloading actions-runner ${version}"
-curl --silent --show-error --location --fail "https://github.com/actions/runner/releases/download/v${version}/actions-runner-linux-${arch_suffix}-${version}.tar.gz" \
-| tar --extract --gzip --no-same-owner
+check-github-release-asset "actions/runner" "v${version}" "${filename}"
+curl --silent --show-error --location --fail --output "${uniget_cache_download}/${filename}" \
+    "${url}"
+
+tar --file="${uniget_cache_download}/${filename}" --list
+tar --file="${uniget_cache_download}/${filename}" --extract --gzip --no-same-owner
 EOF