Merge pull request #10 from infosiftr/deploy-retries

Retry each crane command on 429 after a sleep

Jenkinsfile.deploy

@@ -64,21 +64,8 @@ node('multiarch-' + env.BASHBREW_ARCH) { ansiColor('xterm') {
 				set -Eeuo pipefail -x
 
 				jq -L.scripts -r '
-					include "meta";
-					reduce (.[] | select(.build.resolved and .build.arch == env.BASHBREW_ARCH)) as $i ({};
-						.[ $i.source.arches[].archTags[] ] += [
-							$i.build.resolved
-							| .index.ref // .manifest.ref
-						]
-					)
-					| to_entries[]
-					| .key as $target
-					| .value
-					| if length == 1 then
-						@sh "crane copy \\(.) \\($target)"
-					else
-						@sh "crane index append --tag \\($target) " + (map("--manifest " + @sh) | join(" ")) + " --flatten"
-					end
+					include "jenkins";
+					crane_deploy_commands | @sh
 				' builds.json
 			''').trim()
 
@@ -89,7 +76,24 @@ node('multiarch-' + env.BASHBREW_ARCH) { ansiColor('xterm') {
 			sh """#!/usr/bin/env bash
 				set -Eeuo pipefail
 
-				${ shell }
+				commands=( ${ shell } )
+				for c in "${commands[@]}"; do
+					tries=3
+					while ! output="$( { $c; } |& tee /dev/stderr )"; then
+						# check to see if we hit the docker hub rate limit
+						if grep -q 'TOOMANYREQUESTS' <<<"$output"; then
+							if [ "$(( --tries ))" -le 0 ]; then
+								echo >&2 'failed to copy image 3 times'
+								exit 1
+							fi
+							echo 'sleeping before retrying...'
+							sleep 60
+						else
+							# non 429 error
+							exit 1
+						fi
+					fi
+				done
 			"""
 		}
 	}

jenkins.jq

@@ -0,0 +1,18 @@
+# input: list of build objects i.e., builds.json
+# output: stream of crane copy command strings
+def crane_deploy_commands:
+	reduce (.[] | select(.build.resolved and .build.arch == env.BASHBREW_ARCH)) as $i ({};
+		.[ $i.source.arches[].archTags[] ] += [
+			$i.build.resolved
+			| .index.ref // .manifest.ref
+		]
+	)
+	| to_entries[]
+	| .key as $target
+	| .value
+	| if length == 1 then
+		@sh "crane copy \(.) \($target)"
+	else
+		@sh "crane index append --tag \($target) " + (map("--manifest " + @sh) | join(" ")) + " --flatten"
+	end
+;