Jenkinsfile,testsuite: add rclone tests

This is the same set up as uplink rclone tests except we are testing
the s3 backend of rclone instead of the native Storj DC integration.

Additionally, this disables the metainfo rate limiter for testing
Gateway-ST, just as Gateway-MT currently does. rclone tests throw
a lot of requests at the test Satellite, so we need to disable this.

Updates storj/edge#90

Change-Id: I54462f14a251db562375025cf5faabf22ba86673

Jenkinsfile

@@ -149,6 +149,13 @@ pipeline {
                         sh 'psql -U postgres -c \'create database integration;\''
                         sh 'cd ./testsuite/integration && ./run.sh'
                     }
+                    post {
+                        always {
+                            zip zipFile: 'rclone-integration-tests.zip', archive: true, dir: '.build/rclone-integration-tests'
+                            archiveArtifacts artifacts: 'rclone-integration-tests.zip'
+                            sh 'rm rclone-integration-tests.zip'
+                        }
+                    }
                 }
 
                 stage('Cross Compile') {

testsuite/integration/rclone.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -Eueo pipefail
+
+log_error() {
+    rc=$?
+    echo "error code $rc in $(caller) line $LINENO :: ${BASH_COMMAND}"
+    exit $rc
+}
+trap log_error ERR
+
+[ -n "${GATEWAY_0_ACCESS_KEY}" ]
+[ -n "${GATEWAY_0_SECRET_KEY}" ]
+[ -n "${GATEWAY_0_ADDR}" ]
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+# setup tmpdir for testfiles and cleanup
+TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXX)
+cleanup() {
+    rm -rf "$TMPDIR"
+}
+trap cleanup EXIT
+
+cd "$TMPDIR"; git clone https://github.com/rclone/rclone
+RCLONE=$TMPDIR/rclone
+
+pushd "$RCLONE"
+    git fetch --tags
+    latest_version=$(git tag -l --sort -version:refname | head -1)
+    git checkout "$latest_version"
+
+    go build ./fstest/test_all
+    go build
+
+    ./rclone config create TestS3 s3 \
+        env_auth false \
+        provider Minio \
+        endpoint "http://${GATEWAY_0_ADDR}" \
+        access_key_id "$GATEWAY_0_ACCESS_KEY" \
+        secret_access_key "$GATEWAY_0_SECRET_KEY" \
+        chunk_size 64M \
+        upload_cutoff 64M
+
+    # only run "fs/sync" for the moment, as the other main test suite
+    # "fs/operations" has modification time window test failures.
+    # see https://github.com/storj/gateway-st/issues/46
+    ./test_all \
+        -backends s3 \
+        -remotes TestS3: \
+        -tests "fs/sync" \
+        -maxtries 1 \
+        -verbose \
+        -output "$SCRIPTDIR"/../../.build/rclone-integration-tests
+popd

testsuite/integration/run.sh

@@ -37,11 +37,15 @@ else
 	storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network --postgres=$STORJ_SIM_POSTGRES setup
 fi
 
+# disable metainfo rate limiter for tests
+sed -i 's/# metainfo.rate-limiter.enabled: true/metainfo.rate-limiter.enabled: false/g' "$(storj-sim network env SATELLITE_0_DIR)/config.yaml"
+
 # run aws-cli tests
 storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/awscli.sh
 storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/awscli_multipart.sh
 storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/duplicity.sh
 storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/duplicati.sh
+storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/rclone.sh
 storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network destroy
 
 # setup the network with ipv6