diff --git a/LICENSE b/LICENSE index 5ccb3e9..ab75d54 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 Nao YONASHIRO +Copyright (c) 2019 Nao YONASHIRO Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/rget.sh b/rget.sh index 7102654..d90a4c0 100755 --- a/rget.sh +++ b/rget.sh @@ -1,22 +1,52 @@ #!/bin/sh set -e -WORKDIR=`mktemp -d` +URL= +OUTPUT= PROCESS=4 -BLOCKSIZE=8388608 -while getopts "P:b:o:u:" OPTION +BLOCK_SIZE= + +usage() { + echo "Usage: rget.sh -o OUTPUT -u URL [-P PROCESS] [-b BLOCK_SIZE]" >&2 +} + +while getopts "o:u:P:b:" OPTION do - case $OPTION in - P) PROCESS=${OPTARG};; - b) BLOCKSIZE=${OPTARG};; + case ${OPTION} in o) OUTPUT=${OPTARG};; - *) exit $E_OPTERROR;; + u) URL=${OPTARG};; + P) PROCESS=${OPTARG};; + b) BLOCK_SIZE=${OPTARG};; + *) usage; exit 2;; esac done -shift $(($OPTIND - 1)) -CONTENT_LENGTH=`curl -IsSL -X GET $1 | grep Content-Length: | awk '$0=$2'` -echo $CONTENT_LENGTH -seq 0 $BLOCKSIZE $CONTENT_LENGTH | awk -v BS=$BLOCKSIZE -v WD=$WORKDIR '{print "-SsL -r " sprintf("%d-%d", $0, $0 + BS - 1) " -o " sprintf("%s/%05d", WD, NR)}' | xargs -n 5 -P $PROCESS -I{} sh -c "curl {} $1" -cat $WORKDIR/* > $OUTPUT -rm -rf $WORKDIR +[ -z ${OUTPUT} ] && { usage; echo "OUTPUT must be required" >&2; exit 2; } +[ -z ${URL} ] && { usage; echo "URL must be required" >&2; exit 2; } + +WORK_DIR=`mktemp -d` +atexit() { + rm -rf ${WORK_DIR} || true +} +trap 'rc=$?; trap - EXIT; atexit; exit $?' INT PIPE TERM +trap atexit EXIT + +CONTENT_LENGTH=`wget --spider -S ${URL} 2>&1 | grep Content-Length | awk '$0=$2'` +[ -z ${BLOCK_SIZE} ] && BLOCK_SIZE=$(( (${CONTENT_LENGTH} + ${PROCESS} - 1) / ${PROCESS} )) + +GEN_PARAM=`cat<<'EOF' +{ + block_path = sprintf("%s/%05d", WORK_DIR, NR); + printf("echo > %s", block_path); + printf(" && "); + printf("wget -q -c --header \"Range: bytes=%d-%d\" -O %s", $0, $0 + BLOCK_SIZE - 1, block_path) + printf("\n"); +} +EOF` + +seq 0 ${BLOCK_SIZE} $((${CONTENT_LENGTH} - 1)) | awk -v BLOCK_SIZE=${BLOCK_SIZE} -v WORK_DIR=${WORK_DIR} "${GEN_PARAM}" | xargs -n 12 -P ${PROCESS} -I{} sh -c "{} ${URL}" + +rm -rf ${OUTPUT} || true +for block in ${WORK_DIR}/*; do + dd if=${block} skip=1 iflag=skip_bytes bs=${BLOCK_SIZE} 2>/dev/null >> ${OUTPUT} +done