Upgrade MinIO

.github/workflows/main.yml

@@ -19,7 +19,27 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Install dependencies
-        run: sudo apt-get install -y wait-for-it
+        run: sudo apt-get update -qq && sudo apt-get install -y curl wait-for-it
+      - name: Restore cached binaries
+        id: cache-binaries-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: .bin
+          key: ${{ runner.os }}-binaries
+      - name: Install MinIO Client
+        run: |
+          mkdir .bin || exit 0
+          cd .bin
+          curl --retry 6 --fail --silent --location --output mc.RELEASE.2023-06-19T19-31-19Z "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/mc.RELEASE.2023-06-19T19-31-19Z"
+          curl --retry 6 --fail --silent --location "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/mc.RELEASE.2023-06-19T19-31-19Z.sha256sum" | sha256sum --check -
+          mv mc.RELEASE.2023-06-19T19-31-19Z mc
+          chmod +x mc
+      - name: Save cached binaries
+        id: cache-binaries-save
+        uses: actions/cache/save@v3
+        with:
+          path: .bin
+          key: ${{ steps.cache-binaries-restore.outputs.cache-primary-key }}
       - name: Run tests - latest njs version
         run: ./test.sh --latest-njs --type oss
       - name: Run tests - stable njs version

.gitignore

@@ -110,3 +110,5 @@ modules.xml
 test-settings.*
 s3-requests.http
 httpRequests/
+
+.bin/

docs/development.md

@@ -55,7 +55,7 @@ additional modules.
 
 ## Testing
 
-Automated tests require `docker`, `docker-compose`, `curl` and `md5sum` to be
+Automated tests require `docker`, `docker-compose`, `curl`, `mc` and `md5sum` to be
 installed. To run all unit tests and integration tests, run the following command.
 If you invoke the test script with a plus parameter, you will need to add your
 NGINX repository keys to the `plus/etc/ssl/nginx` directory

test.sh

@@ -22,7 +22,7 @@ set -o pipefail  # don't hide errors within pipes
 nginx_server_proto="http"
 nginx_server_host="localhost"
 nginx_server_port="8989"
-minio_server="http://localhost:9090"
+
 test_server="${nginx_server_proto}://${nginx_server_host}:${nginx_server_port}"
 test_fail_exit_code=2
 no_dep_exit_code=3
@@ -31,6 +31,12 @@ test_dir="${script_dir}/test"
 test_compose_config="${test_dir}/docker-compose.yaml"
 test_compose_project="ngt"
 
+minio_server="http://localhost:9090"
+minio_user="AKIAIOSFODNN7EXAMPLE"
+minio_passwd="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+minio_name="${test_compose_project}_minio_1"
+minio_bucket="bucket-1"
+
 is_windows="0"
 if [ -z "${OS}" ] && [ "${OS}" == "Windows_NT" ]; then
   is_windows="1"
@@ -137,6 +143,21 @@ if ! [ -x "${curl_cmd}" ]; then
   exit ${no_dep_exit_code}
 fi
 
+if command -v mc > /dev/null; then
+  mc_cmd="$(command -v mc)"
+elif [ -x "${script_dir}/.bin/mc" ]; then
+  mc_cmd="${script_dir}/.bin/mc"
+else
+  e "required dependency not found: mc not found in the path or not executable"
+  exit ${no_dep_exit_code}
+fi
+e "Using MinIO Client: ${mc_cmd}"
+
+if ! [ -x "${mc_cmd}" ]; then
+  e "required dependency not found: mc not found in the path or not executable"
+  exit ${no_dep_exit_code}
+fi
+
 wait_for_it_cmd="$(command -v wait-for-it || true)"
 if [ -x "${wait_for_it_cmd}" ]; then
   wait_for_it_installed=1
@@ -168,16 +189,7 @@ compose() {
   ${docker_compose_cmd} -f "${test_compose_config}" -p "${test_compose_project}" "$@"
 }
 
-integration_test() {
-  printf "\033[34;1m▶\033[0m"
-  printf "\e[1m Integration test suite for v%s signatures\e[22m\n" "$1"
-  printf "\033[34;1m▶\033[0m"
-  printf "\e[1m Integration test suite with ALLOW_DIRECTORY_LIST=%s\e[22m\n" "$2"
-  printf "\033[34;1m▶\033[0m"
-  printf "\e[1m Integration test suite with PROVIDE_INDEX_PAGE=%s\e[22m\n" "$3"
-  printf "\033[34;1m▶\033[0m"
-  printf "\e[1m Integration test suite with APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=%s\e[22m\n" "$4"
-
+integration_test_data() {
 
   # Write problematic files to disk if we are not on Windows. Originally,
   # these files were checked in, but that prevented the git repository from
@@ -187,23 +199,10 @@ integration_test() {
     echo 'We are but selling water next to a river.' > "${test_dir}"'/data/bucket-1/a/%@!*()=$#^&|.txt'
   fi
 
-  # See if Minio is already running, if it isn't then we don't need to build it
-  # COMPOSE_COMPATIBILITY=true Supports older style compose filenames with _ vs -
-
-  if [ -z "$(docker ps -q -f name=${test_compose_project}_minio-_1)" ]; then
-    p "Building Docker Compose environment"
-    COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 compose up --no-start
-
-    p "Adding test data to container"
-    echo "Copying contents of ${test_dir}/data to Docker container ${test_compose_project}_minio_1:/"
-    "${docker_cmd}" cp "${test_dir}/data" "${test_compose_project}"_minio_1:/
-    echo "Docker diff output:"
-    "${docker_cmd}" diff "${test_compose_project}"_minio_1
-  fi
-
   p "Starting Docker Compose Environment"
-  COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 compose up -d
-
+  # COMPOSE_COMPATIBILITY=true Supports older style compose filenames with _ vs -
+  COMPOSE_COMPATIBILITY=true compose up -d
+
   if [ "${wait_for_it_installed}" ]; then
     # Hit minio's health check end point to see if it has started up
     for (( i=1; i<=3; i++ ))
@@ -216,7 +215,34 @@ integration_test() {
         sleep 2
       fi
     done
+  fi
+
+  p "Adding test data to container"
+  "${mc_cmd}" alias set "$minio_name" "$minio_server" "$minio_user" "$minio_passwd"
+  "${mc_cmd}" mb "$minio_name/$minio_bucket"
+  echo "Copying contents of ${test_dir}/data/$minio_bucket to Docker container $minio_name"
+  for file in "${test_dir}/data/$minio_bucket"/*; do
+    "${mc_cmd}" cp -r "${file}" "$minio_name/$minio_bucket"
+  done
+  echo "Docker diff output:"
+  "${docker_cmd}" diff "$minio_name"
+}
+
+integration_test() {
+  printf "\033[34;1m▶\033[0m"
+  printf "\e[1m Integration test suite for v%s signatures\e[22m\n" "$1"
+  printf "\033[34;1m▶\033[0m"
+  printf "\e[1m Integration test suite with ALLOW_DIRECTORY_LIST=%s\e[22m\n" "$2"
+  printf "\033[34;1m▶\033[0m"
+  printf "\e[1m Integration test suite with PROVIDE_INDEX_PAGE=%s\e[22m\n" "$3"
+  printf "\033[34;1m▶\033[0m"
+  printf "\e[1m Integration test suite with APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=%s\e[22m\n" "$4"
 
+  p "Starting Docker Compose Environment"
+  # COMPOSE_COMPATIBILITY=true Supports older style compose filenames with _ vs -
+  COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 compose up -d
+
+  if [ "${wait_for_it_installed}" ]; then
     if [ -x "${wait_for_it_cmd}" ]; then
       "${wait_for_it_cmd}" -h "${nginx_server_host}" -p "${nginx_server_port}"
     fi
@@ -247,7 +273,8 @@ finish() {
 
   p "Cleaning up Docker compose environment"
   compose stop
-  compose rm -f
+  compose rm -f -v
+  "${mc_cmd}" alias rm "$minio_name"
 
   exit ${result}
 }
@@ -352,6 +379,8 @@ runUnitTestWithSessionToken "s3gateway_test.js"
 
 ### INTEGRATION TESTS
 
+integration_test_data
+
 p "Testing API with AWS Signature V2 and allow directory listing off"
 integration_test 2 0 0 0
 

test/docker-compose.yaml

@@ -35,14 +35,15 @@ services:
       PROXY_CACHE_VALID_FORBIDDEN: "30s"
 
   minio:
-    image: "minio/minio:RELEASE.2021-02-19T04-38-02Z"
+    image: quay.io/minio/minio:RELEASE.2023-06-09T07-32-12Z
     ports:
       - "9090:9000/tcp"
     restart: "no"
-    command: "server --address :9000 /data"
+    command: minio server /data
     environment:
-      MINIO_ACCESS_KEY: "AKIAIOSFODNN7EXAMPLE"
-      MINIO_SECRET_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+      MINIO_ADDRESS: :9000
+      MINIO_ROOT_USER: "AKIAIOSFODNN7EXAMPLE"
+      MINIO_ROOT_PASSWORD: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
       MINIO_REGION_NAME: "us-east-1"
       MINIO_DOMAIN: "minio"
       MINIO_BROWSER: "off"