dockerhub2oci

#!/bin/bash

#
# dockerhub2oci - Simple shell tool to pull from DockerHub and create an OCI image.
# Copyright (C) 2018  Oliver Freyermuth
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

function help() {
cat <<EOF
Tool to pull an image from a docker registry and extract the contents
to a folder to be used as OCI image.

Example usage:
$0 --repo gliderlabs --image alpine --tag latest

Options:

--strip-headers: (false) if true, use token for first request and don't forward
--writable-dirs: (false) if true, make directories writable in blob2oci
--registry: Docker Hub registry (default registry.hub.docker.com)
--repo: registry repository (default gliderlabs)
--image: image name (default alpine)
--tag: image tag (default latest)
EOF
exit 1
}

# Ensure functions available

HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

if [ ! -f "${HERE}/blob2oci" ]; then
    echo "Cannot find blob2oci, exiting."
    exit 1 
fi

# Aria2c

if ! [ -x "$(command -v aria2c)" ]; then
  echo '# Missing aria2c, install with:
sudo apt-get install aria2
sudo yum install aria2
sudo pacman -Sy aria2
sudo emerge -av aria2' >&2
  exit 1
fi

# Defaults

registry="registry.hub.docker.com"
repo="gliderlabs"
token="unset"
image="alpine"
tag="latest"

cachedir=${TMPDIR:-/tmp}/docker2oci

strip_headers=false
writable_dirs=false

# Token can come from environment, second priority to set on command line
if [ ! -z "${DOCKERHUB2OCI_TOKEN}" ]; then
    echo "Found token in environment."
    token=${DOCKERHUB2OCI_TOKEN}; 
fi

TEMP=$(getopt -o h --long repo:,image:,tag:,cachedir:,registry:,token:,writable-dirs,strip-headers,help -n $0 -- "$@")
if [ $? != 0 ] ; then usage; exit 1; fi
eval set -- "$TEMP"
while true ; do
	case "$1" in
		--repo)          repo=$2;            shift 2;;
		--image)         image=$2;           shift 2;;
		--tag)           tag=$2;             shift 2;;
		--cachedir)      cachedir=$2;        shift 2;;
		--registry)      registry=$2;        shift 2;;
		--token)         token=$2;           shift 2;;
		--writable-dirs) writable_dirs=true; shift 1;;
		--strip-headers) strip_headers=true; shift 1;;
		-h|--help)       help;               exit 0;;
		--)              shift;              break;;
		*) echo "Internal error!";           exit 1;;
	esac
done

mkdir -p ${cachedir}

imgroot=${repo}/${image}/${tag}
rm -rf ${imgroot}
mkdir -p ${imgroot}

img_full="${repo}/${image}"

# If the token is unset, we use default auth_uri and auth_uri full. If a 
#     different client is needed (e.g., Nvidia Cloud) this logic will be handled
#     in a calling script, and then token passed forward

if [ "${token}" == "unset" ]; then

    auth_uri="https://auth.docker.io/token"
    auth_uri_full="${auth_uri}?service=registry.docker.io&scope=repository:${img_full}:pull"

    if [ "${registry}" = "ghcr.io" ]; then
        auth_uri="https://ghcr.io/token"
        auth_uri_full="${auth_uri}?service=ghcr.io&expires_in=900&scope=repository:${img_full}:pull"
    fi

    # If the user is logged in with Docker, this should return the Auth token?
    token=$(curl -s ${auth_uri_full} | jq -r .token)

    # grep -Po '"'"token"'"\s*:\s*"\K([^"]*)')
    #echo $token
fi

reg_uri_manifest="https://${registry}/v2/${img_full}/manifests/${tag}"
reg_uri_blobs="https://${registry}/v2/${img_full}/blobs"

# Layers
# Try classic docker-style manifest first.
layers_raw=$(curl -s -H "Authorization: Bearer ${token}" "${reg_uri_manifest}" | jq -r .fsLayers[].blobSum 2>/dev/null)
if [ ! $? -eq 0 ]; then
    # Retry assuming OCI manifest.
    layers_raw=$(curl -s -H "Authorization: Bearer ${token}" "${reg_uri_manifest}" -H "Accept: application/vnd.oci.image.manifest.v1+json"| jq -r .layers[].digest)
    if [ ! $? -eq 0 ]; then
      echo "Error fetching layers!"
      exit 1
    fi
fi
# grep -Po '"'"blobSum"'"\s*:\s*"\K([^"]*)')
#echo "$reply"

layers=$(echo ${layers_raw} | tr ' ' '\n' | tac | uniq)
# Layers may be strangely stacked and hashes may repeat further up,
# download only fully unique layer IDs.
layers_dl=$(echo ${layers_raw} | tr ' ' '\n' | sort | uniq)


##############################################
# Download
##############################################

# The aria2c command that doesn't strip headers is given Auth header directly

echo "Collecting download links..."

if [ "${strip_headers}" == true ]; then

    # Only pass token to first request, use response with aria (without auth)
    for HASHLAYER in ${layers_dl}; do
        hashtype=${HASHLAYER%:*}
        layer=${HASHLAYER#*:}
        hashtype_aria=$(echo ${hashtype} | sed 's/^\([a-z]*\)\([0-9]*\)$/\1-\2/')

        # Decide on the final uri (with or without Auth, and
        hashlayer_final_uri=${reg_uri_blobs}/${HASHLAYER}
        hashlayer_final_uri=$(curl -H "Authorization: Bearer ${token}" -Ls -o /dev/null -w %{url_effective} --max-filesize 1024 ${reg_uri_blobs}/${HASHLAYER})
        echo ${hashlayer_final_uri}
        echo "  checksum=${hashtype_aria}=${layer}"
        echo "  out=${HASHLAYER}.tar.gz"
    done |  aria2c -i - -d ${cachedir} -V
        
else

    # Pass Auth header to aria2c and thus all following requests
    for HASHLAYER in ${layers_dl}; do
 	hashtype=${HASHLAYER%:*}
 	layer=${HASHLAYER#*:}
 	hashtype_aria=$(echo ${hashtype} | sed 's/^\([a-z]*\)\([0-9]*\)$/\1-\2/')
 	echo ${reg_uri_blobs}/${HASHLAYER}
 	echo "  checksum=${hashtype_aria}=${layer}"
 	echo "  out=${HASHLAYER}.tar.gz"
    done | aria2c -i - -d ${cachedir} --header="Authorization: Bearer ${token}" -V
fi

##############################################
# Extract and Clean
##############################################

for HASHLAYER in ${layers}; do

	layer="${cachedir}/${HASHLAYER}.tar.gz"

        # Call blob2oci to handle the layer
        if [ "${writable_dirs}" == true ]; then
            "${HERE}/blob2oci" --layer "$layer" --extract "${imgroot}" --writable-dirs
        else
            "${HERE}/blob2oci" --layer "$layer" --extract "${imgroot}"
        fi
done