From adb9dbeabb801443edf14cfa620e82ca5bd1d3ed Mon Sep 17 00:00:00 2001
From: Sean Harvey <sean@storj.io>
Date: Tue, 7 Dec 2021 16:40:56 +1300
Subject: [PATCH] Jenkinsfile,testsuite: add rclone tests

This is the same set up as uplink rclone tests except we are testing
the s3 backend of rclone instead of the native Storj DC integration.

Additionally, this disables the metainfo rate limiter for testing
Gateway-ST, just as Gateway-MT currently does. rclone tests throw
a lot of requests at the test Satellite, so we need to disable this.

Updates storj/gateway-mt#90

Change-Id: I54462f14a251db562375025cf5faabf22ba86673
---
 Jenkinsfile                     |  7 +++++
 testsuite/integration/rclone.sh | 54 +++++++++++++++++++++++++++++++++
 testsuite/integration/run.sh    |  4 +++
 3 files changed, 65 insertions(+)
 create mode 100644 testsuite/integration/rclone.sh

diff --git a/Jenkinsfile b/Jenkinsfile
index dcacf34..354a410 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -149,6 +149,13 @@ pipeline {
                         sh 'psql -U postgres -c \'create database integration;\''
                         sh 'cd ./testsuite/integration && ./run.sh'
                     }
+                    post {
+                        always {
+                            zip zipFile: 'rclone-integration-tests.zip', archive: true, dir: '.build/rclone-integration-tests'
+                            archiveArtifacts artifacts: 'rclone-integration-tests.zip'
+                            sh 'rm rclone-integration-tests.zip'
+                        }
+                    }
                 }
 
                 stage('Cross Compile') {
diff --git a/testsuite/integration/rclone.sh b/testsuite/integration/rclone.sh
new file mode 100644
index 0000000..6351b03
--- /dev/null
+++ b/testsuite/integration/rclone.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -Eueo pipefail
+
+log_error() {
+    rc=$?
+    echo "error code $rc in $(caller) line $LINENO :: ${BASH_COMMAND}"
+    exit $rc
+}
+trap log_error ERR
+
+[ -n "${GATEWAY_0_ACCESS_KEY}" ]
+[ -n "${GATEWAY_0_SECRET_KEY}" ]
+[ -n "${GATEWAY_0_ADDR}" ]
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+# setup tmpdir for testfiles and cleanup
+TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXX)
+cleanup() {
+    rm -rf "$TMPDIR"
+}
+trap cleanup EXIT
+
+cd "$TMPDIR"; git clone https://github.com/rclone/rclone
+RCLONE=$TMPDIR/rclone
+
+pushd "$RCLONE"
+    git fetch --tags
+    latest_version=$(git tag -l --sort -version:refname | head -1)
+    git checkout "$latest_version"
+
+    go build ./fstest/test_all
+    go build
+
+    ./rclone config create TestS3 s3 \
+        env_auth false \
+        provider Minio \
+        endpoint "http://${GATEWAY_0_ADDR}" \
+        access_key_id "$GATEWAY_0_ACCESS_KEY" \
+        secret_access_key "$GATEWAY_0_SECRET_KEY" \
+        chunk_size 64M \
+        upload_cutoff 64M
+
+    # only run "fs/sync" for the moment, as the other main test suite
+    # "fs/operations" has modification time window test failures.
+    # see https://github.com/storj/gateway-st/issues/46
+    ./test_all \
+        -backends s3 \
+        -remotes TestS3: \
+        -tests "fs/sync" \
+        -maxtries 1 \
+        -verbose \
+        -output "$SCRIPTDIR"/../../.build/rclone-integration-tests
+popd
\ No newline at end of file
diff --git a/testsuite/integration/run.sh b/testsuite/integration/run.sh
index fcc49cd..2b32384 100755
--- a/testsuite/integration/run.sh
+++ b/testsuite/integration/run.sh
@@ -37,11 +37,15 @@ else
 	storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network --postgres=$STORJ_SIM_POSTGRES setup
 fi
 
+# disable metainfo rate limiter for tests
+sed -i 's/# metainfo.rate-limiter.enabled: true/metainfo.rate-limiter.enabled: false/g' "$(storj-sim network env SATELLITE_0_DIR)/config.yaml"
+
 # run aws-cli tests
 storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/awscli.sh
 storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/awscli_multipart.sh
 storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/duplicity.sh
 storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/duplicati.sh
+storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/rclone.sh
 storj-sim -x --satellites 1 --host $STORJ_NETWORK_HOST4 network destroy
 
 # setup the network with ipv6