From f85d1b27bb26fe6871d1bfc71af8df7b1bdb4872 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Sat, 1 Sep 2018 01:43:33 +0900 Subject: [PATCH 1/7] refactor: rget shell version --- rget.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rget.sh b/rget.sh index 7102654..f5ff21a 100755 --- a/rget.sh +++ b/rget.sh @@ -16,7 +16,6 @@ done shift $(($OPTIND - 1)) CONTENT_LENGTH=`curl -IsSL -X GET $1 | grep Content-Length: | awk '$0=$2'` -echo $CONTENT_LENGTH -seq 0 $BLOCKSIZE $CONTENT_LENGTH | awk -v BS=$BLOCKSIZE -v WD=$WORKDIR '{print "-SsL -r " sprintf("%d-%d", $0, $0 + BS - 1) " -o " sprintf("%s/%05d", WD, NR)}' | xargs -n 5 -P $PROCESS -I{} sh -c "curl {} $1" +seq 0 $BLOCKSIZE $CONTENT_LENGTH | awk -v BS=$BLOCKSIZE -v WD=$WORKDIR '{printf("-SsL -r %d-%d -o %s/%05d\n", $0, $0 + BS - 1, WD, NR)}' | xargs -n 5 -P $PROCESS -I{} sh -c "curl {} $1" cat $WORKDIR/* > $OUTPUT rm -rf $WORKDIR From b4d90b4657cff4d95cc9cf3655888a48e75d82b5 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Sun, 2 Sep 2018 23:15:53 +0900 Subject: [PATCH 2/7] feat: use wget --- rget.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rget.sh b/rget.sh index f5ff21a..ee84145 100755 --- a/rget.sh +++ b/rget.sh @@ -15,7 +15,7 @@ do done shift $(($OPTIND - 1)) -CONTENT_LENGTH=`curl -IsSL -X GET $1 | grep Content-Length: | awk '$0=$2'` +CONTENT_LENGTH=`wget --spider -S ${1} 2>&1 | grep Content-Length | awk '$0=$2'` seq 0 $BLOCKSIZE $CONTENT_LENGTH | awk -v BS=$BLOCKSIZE -v WD=$WORKDIR '{printf("-SsL -r %d-%d -o %s/%05d\n", $0, $0 + BS - 1, WD, NR)}' | xargs -n 5 -P $PROCESS -I{} sh -c "curl {} $1" cat $WORKDIR/* > $OUTPUT rm -rf $WORKDIR From c4cb1f839f167a19151e4d01549bf338f849b1ac Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Tue, 4 Sep 2018 23:47:02 +0900 Subject: [PATCH 3/7] feat: remove dependency(curl) --- rget.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/rget.sh b/rget.sh index ee84145..203ab17 100755 --- a/rget.sh +++ b/rget.sh @@ -1,21 +1,24 @@ #!/bin/sh set -e -WORKDIR=`mktemp -d` PROCESS=4 BLOCKSIZE=8388608 -while getopts "P:b:o:u:" OPTION +while getopts "P:b:o:" OPTION do case $OPTION in P) PROCESS=${OPTARG};; b) BLOCKSIZE=${OPTARG};; o) OUTPUT=${OPTARG};; - *) exit $E_OPTERROR;; + *) exit 2;; esac done shift $(($OPTIND - 1)) +WORKDIR=`mktemp -d` + CONTENT_LENGTH=`wget --spider -S ${1} 2>&1 | grep Content-Length | awk '$0=$2'` -seq 0 $BLOCKSIZE $CONTENT_LENGTH | awk -v BS=$BLOCKSIZE -v WD=$WORKDIR '{printf("-SsL -r %d-%d -o %s/%05d\n", $0, $0 + BS - 1, WD, NR)}' | xargs -n 5 -P $PROCESS -I{} sh -c "curl {} $1" -cat $WORKDIR/* > $OUTPUT +seq 0 $BLOCKSIZE $(($CONTENT_LENGTH - 1)) | awk -v BS=$BLOCKSIZE -v WD=$WORKDIR '{f=sprintf("%s/%05d",WD,NR);printf("echo > %s && wget -q -c --header \"Range: bytes=%d-%d\" -O %s\n", f, $0, $0 + BS - 1, f)}' | xargs -t -n 12 -P ${PROCESS} -I{} sh -c "{} $1" +for f in $WORKDIR/*; do + tail -c +2 $f >> $OUTPUT +done rm -rf $WORKDIR From 17158f19de241df0672c383d58b42530e34fb46d Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Tue, 4 Sep 2018 23:48:14 +0900 Subject: [PATCH 4/7] chore: remove t flag --- rget.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rget.sh b/rget.sh index 203ab17..7206dd8 100755 --- a/rget.sh +++ b/rget.sh @@ -17,7 +17,7 @@ shift $(($OPTIND - 1)) WORKDIR=`mktemp -d` CONTENT_LENGTH=`wget --spider -S ${1} 2>&1 | grep Content-Length | awk '$0=$2'` -seq 0 $BLOCKSIZE $(($CONTENT_LENGTH - 1)) | awk -v BS=$BLOCKSIZE -v WD=$WORKDIR '{f=sprintf("%s/%05d",WD,NR);printf("echo > %s && wget -q -c --header \"Range: bytes=%d-%d\" -O %s\n", f, $0, $0 + BS - 1, f)}' | xargs -t -n 12 -P ${PROCESS} -I{} sh -c "{} $1" +seq 0 $BLOCKSIZE $(($CONTENT_LENGTH - 1)) | awk -v BS=$BLOCKSIZE -v WD=$WORKDIR '{f=sprintf("%s/%05d",WD,NR);printf("echo > %s && wget -q -c --header \"Range: bytes=%d-%d\" -O %s\n", f, $0, $0 + BS - 1, f)}' | xargs -n 12 -P ${PROCESS} -I{} sh -c "{} $1" for f in $WORKDIR/*; do tail -c +2 $f >> $OUTPUT done From f6655dffb4c0b5b4d9c489525cbd87e5ae5bbd6c Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Fri, 7 Sep 2018 01:11:43 +0900 Subject: [PATCH 5/7] refactor: shell version --- rget.sh | 56 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/rget.sh b/rget.sh index 7206dd8..1805f22 100755 --- a/rget.sh +++ b/rget.sh @@ -1,24 +1,52 @@ #!/bin/sh set -e +URL= +OUTPUT= PROCESS=4 -BLOCKSIZE=8388608 -while getopts "P:b:o:" OPTION +BLOCK_SIZE= + +usage() { + echo "Usage: rget.sh -o OUTPUT -u URL [-P PROCESS] [-b BLOCK_SIZE]" >&2 +} + +while getopts "o:u:P:b:" OPTION do - case $OPTION in - P) PROCESS=${OPTARG};; - b) BLOCKSIZE=${OPTARG};; + case ${OPTION} in o) OUTPUT=${OPTARG};; - *) exit 2;; + u) URL=${OPTARG};; + P) PROCESS=${OPTARG};; + b) BLOCK_SIZE=${OPTARG};; + *) usage; exit 2;; esac done -shift $(($OPTIND - 1)) -WORKDIR=`mktemp -d` +[ -z ${OUTPUT} ] && { usage; echo "OUTPUT must be required" >&2; exit 2; } +[ -z ${URL} ] && { usage; echo "URL must be required" >&2; exit 2; } -CONTENT_LENGTH=`wget --spider -S ${1} 2>&1 | grep Content-Length | awk '$0=$2'` -seq 0 $BLOCKSIZE $(($CONTENT_LENGTH - 1)) | awk -v BS=$BLOCKSIZE -v WD=$WORKDIR '{f=sprintf("%s/%05d",WD,NR);printf("echo > %s && wget -q -c --header \"Range: bytes=%d-%d\" -O %s\n", f, $0, $0 + BS - 1, f)}' | xargs -n 12 -P ${PROCESS} -I{} sh -c "{} $1" -for f in $WORKDIR/*; do - tail -c +2 $f >> $OUTPUT -done -rm -rf $WORKDIR +WORK_DIR=`mktemp -d` +atexit() { + rm -rf ${WORK_DIR} || true +} +trap 'rc=$?; trap - EXIT; atexit; exit $?' INT PIPE TERM +trap atexit EXIT + +CONTENT_LENGTH=`wget --spider -S ${URL} 2>&1 | grep Content-Length | awk '$0=$2'` +[ -z ${BLOCK_SIZE} ] && BLOCK_SIZE=$(( (${CONTENT_LENGTH} + ${PROCESS} - 1) / ${PROCESS} )) + +GEN_PARAM=`cat<<'EOF' +{ + block_path = sprintf("%s/%05d", WORK_DIR, NR); + printf("echo > %s", block_path); + printf(" && "); + printf("wget -q -c --header \"Range: bytes=%d-%d\" -O %s", $0, $0 + BLOCK_SIZE - 1, block_path) + printf("\n"); +} +EOF` + +seq 0 ${BLOCK_SIZE} $(($CONTENT_LENGTH - 1)) | awk -v BLOCK_SIZE=${BLOCK_SIZE} -v WORK_DIR=${WORK_DIR} "${GEN_PARAM}" | xargs -n 12 -P ${PROCESS} -I{} sh -c "{} ${URL}" + +rm -rf ${OUTPUT} || true +for block in ${WORK_DIR}/*; do + dd if=${block} skip=1 iflag=skip_bytes bs=${BLOCK_SIZE} 2>/dev/null >> ${OUTPUT} +done \ No newline at end of file From e69c04b962f4210f988967323d6e3c18eabc512c Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Sat, 8 Sep 2018 00:38:27 +0900 Subject: [PATCH 6/7] refactor: use {} --- rget.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rget.sh b/rget.sh index 1805f22..d90a4c0 100755 --- a/rget.sh +++ b/rget.sh @@ -44,9 +44,9 @@ GEN_PARAM=`cat<<'EOF' } EOF` -seq 0 ${BLOCK_SIZE} $(($CONTENT_LENGTH - 1)) | awk -v BLOCK_SIZE=${BLOCK_SIZE} -v WORK_DIR=${WORK_DIR} "${GEN_PARAM}" | xargs -n 12 -P ${PROCESS} -I{} sh -c "{} ${URL}" +seq 0 ${BLOCK_SIZE} $((${CONTENT_LENGTH} - 1)) | awk -v BLOCK_SIZE=${BLOCK_SIZE} -v WORK_DIR=${WORK_DIR} "${GEN_PARAM}" | xargs -n 12 -P ${PROCESS} -I{} sh -c "{} ${URL}" rm -rf ${OUTPUT} || true for block in ${WORK_DIR}/*; do dd if=${block} skip=1 iflag=skip_bytes bs=${BLOCK_SIZE} 2>/dev/null >> ${OUTPUT} -done \ No newline at end of file +done From a46753acb7d5ca0ed2b00a6269e0bd740a66ce47 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Fri, 11 Jan 2019 23:34:59 +0900 Subject: [PATCH 7/7] chore: happy new year --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 5ccb3e9..ab75d54 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 Nao YONASHIRO +Copyright (c) 2019 Nao YONASHIRO Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal