diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dfd96396..00359809 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,27 @@ jobs: - uses: actions/checkout@v2 - name: Install dependencies - run: sudo apt-get install -y wait-for-it + run: sudo apt-get update -qq && sudo apt-get install -y curl wait-for-it + - name: Restore cached binaries + id: cache-binaries-restore + uses: actions/cache/restore@v3 + with: + path: .bin + key: ${{ runner.os }}-binaries + - name: Install MinIO Client + run: | + mkdir .bin || exit 0 + cd .bin + curl --retry 6 --fail --silent --location --output mc.RELEASE.2023-06-19T19-31-19Z "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/mc.RELEASE.2023-06-19T19-31-19Z" + curl --retry 6 --fail --silent --location "https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/mc.RELEASE.2023-06-19T19-31-19Z.sha256sum" | sha256sum --check - + mv mc.RELEASE.2023-06-19T19-31-19Z mc + chmod +x mc + - name: Save cached binaries + id: cache-binaries-save + uses: actions/cache/save@v3 + with: + path: .bin + key: ${{ steps.cache-binaries-restore.outputs.cache-primary-key }} - name: Run tests - latest njs version run: ./test.sh --latest-njs --type oss - name: Run tests - stable njs version diff --git a/.gitignore b/.gitignore index 731c0b03..c6de281a 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,5 @@ modules.xml test-settings.* s3-requests.http httpRequests/ + +.bin/ \ No newline at end of file diff --git a/docs/development.md b/docs/development.md index e690be43..f3763475 100644 --- a/docs/development.md +++ b/docs/development.md @@ -55,7 +55,7 @@ additional modules. ## Testing -Automated tests require `docker`, `docker-compose`, `curl` and `md5sum` to be +Automated tests require `docker`, `docker-compose`, `curl`, `mc` and `md5sum` to be installed. To run all unit tests and integration tests, run the following command. If you invoke the test script with a plus parameter, you will need to add your NGINX repository keys to the `plus/etc/ssl/nginx` directory diff --git a/test.sh b/test.sh index 73825cf9..507fa0f8 100755 --- a/test.sh +++ b/test.sh @@ -22,7 +22,7 @@ set -o pipefail # don't hide errors within pipes nginx_server_proto="http" nginx_server_host="localhost" nginx_server_port="8989" -minio_server="http://localhost:9090" + test_server="${nginx_server_proto}://${nginx_server_host}:${nginx_server_port}" test_fail_exit_code=2 no_dep_exit_code=3 @@ -31,6 +31,12 @@ test_dir="${script_dir}/test" test_compose_config="${test_dir}/docker-compose.yaml" test_compose_project="ngt" +minio_server="http://localhost:9090" +minio_user="AKIAIOSFODNN7EXAMPLE" +minio_passwd="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +minio_name="${test_compose_project}_minio_1" +minio_bucket="bucket-1" + is_windows="0" if [ -z "${OS}" ] && [ "${OS}" == "Windows_NT" ]; then is_windows="1" @@ -137,6 +143,21 @@ if ! [ -x "${curl_cmd}" ]; then exit ${no_dep_exit_code} fi +if command -v mc > /dev/null; then + mc_cmd="$(command -v mc)" +elif [ -x "${script_dir}/.bin/mc" ]; then + mc_cmd="${script_dir}/.bin/mc" +else + e "required dependency not found: mc not found in the path or not executable" + exit ${no_dep_exit_code} +fi +e "Using MinIO Client: ${mc_cmd}" + +if ! [ -x "${mc_cmd}" ]; then + e "required dependency not found: mc not found in the path or not executable" + exit ${no_dep_exit_code} +fi + wait_for_it_cmd="$(command -v wait-for-it || true)" if [ -x "${wait_for_it_cmd}" ]; then wait_for_it_installed=1 @@ -168,16 +189,7 @@ compose() { ${docker_compose_cmd} -f "${test_compose_config}" -p "${test_compose_project}" "$@" } -integration_test() { - printf "\033[34;1m▶\033[0m" - printf "\e[1m Integration test suite for v%s signatures\e[22m\n" "$1" - printf "\033[34;1m▶\033[0m" - printf "\e[1m Integration test suite with ALLOW_DIRECTORY_LIST=%s\e[22m\n" "$2" - printf "\033[34;1m▶\033[0m" - printf "\e[1m Integration test suite with PROVIDE_INDEX_PAGE=%s\e[22m\n" "$3" - printf "\033[34;1m▶\033[0m" - printf "\e[1m Integration test suite with APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=%s\e[22m\n" "$4" - +integration_test_data() { # Write problematic files to disk if we are not on Windows. Originally, # these files were checked in, but that prevented the git repository from @@ -187,23 +199,10 @@ integration_test() { echo 'We are but selling water next to a river.' > "${test_dir}"'/data/bucket-1/a/%@!*()=$#^&|.txt' fi - # See if Minio is already running, if it isn't then we don't need to build it - # COMPOSE_COMPATIBILITY=true Supports older style compose filenames with _ vs - - - if [ -z "$(docker ps -q -f name=${test_compose_project}_minio-_1)" ]; then - p "Building Docker Compose environment" - COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 compose up --no-start - - p "Adding test data to container" - echo "Copying contents of ${test_dir}/data to Docker container ${test_compose_project}_minio_1:/" - "${docker_cmd}" cp "${test_dir}/data" "${test_compose_project}"_minio_1:/ - echo "Docker diff output:" - "${docker_cmd}" diff "${test_compose_project}"_minio_1 - fi - p "Starting Docker Compose Environment" - COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 compose up -d - + # COMPOSE_COMPATIBILITY=true Supports older style compose filenames with _ vs - + COMPOSE_COMPATIBILITY=true compose up -d + if [ "${wait_for_it_installed}" ]; then # Hit minio's health check end point to see if it has started up for (( i=1; i<=3; i++ )) @@ -216,7 +215,34 @@ integration_test() { sleep 2 fi done + fi + + p "Adding test data to container" + "${mc_cmd}" alias set "$minio_name" "$minio_server" "$minio_user" "$minio_passwd" + "${mc_cmd}" mb "$minio_name/$minio_bucket" + echo "Copying contents of ${test_dir}/data/$minio_bucket to Docker container $minio_name" + for file in "${test_dir}/data/$minio_bucket"/*; do + "${mc_cmd}" cp -r "${file}" "$minio_name/$minio_bucket" + done + echo "Docker diff output:" + "${docker_cmd}" diff "$minio_name" +} + +integration_test() { + printf "\033[34;1m▶\033[0m" + printf "\e[1m Integration test suite for v%s signatures\e[22m\n" "$1" + printf "\033[34;1m▶\033[0m" + printf "\e[1m Integration test suite with ALLOW_DIRECTORY_LIST=%s\e[22m\n" "$2" + printf "\033[34;1m▶\033[0m" + printf "\e[1m Integration test suite with PROVIDE_INDEX_PAGE=%s\e[22m\n" "$3" + printf "\033[34;1m▶\033[0m" + printf "\e[1m Integration test suite with APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=%s\e[22m\n" "$4" + p "Starting Docker Compose Environment" + # COMPOSE_COMPATIBILITY=true Supports older style compose filenames with _ vs - + COMPOSE_COMPATIBILITY=true AWS_SIGS_VERSION=$1 ALLOW_DIRECTORY_LIST=$2 PROVIDE_INDEX_PAGE=$3 APPEND_SLASH_FOR_POSSIBLE_DIRECTORY=$4 compose up -d + + if [ "${wait_for_it_installed}" ]; then if [ -x "${wait_for_it_cmd}" ]; then "${wait_for_it_cmd}" -h "${nginx_server_host}" -p "${nginx_server_port}" fi @@ -247,7 +273,8 @@ finish() { p "Cleaning up Docker compose environment" compose stop - compose rm -f + compose rm -f -v + "${mc_cmd}" alias rm "$minio_name" exit ${result} } @@ -352,6 +379,8 @@ runUnitTestWithSessionToken "s3gateway_test.js" ### INTEGRATION TESTS +integration_test_data + p "Testing API with AWS Signature V2 and allow directory listing off" integration_test 2 0 0 0 diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml index a3414d18..df54b0d8 100644 --- a/test/docker-compose.yaml +++ b/test/docker-compose.yaml @@ -35,14 +35,15 @@ services: PROXY_CACHE_VALID_FORBIDDEN: "30s" minio: - image: "minio/minio:RELEASE.2021-02-19T04-38-02Z" + image: quay.io/minio/minio:RELEASE.2023-06-09T07-32-12Z ports: - "9090:9000/tcp" restart: "no" - command: "server --address :9000 /data" + command: minio server /data environment: - MINIO_ACCESS_KEY: "AKIAIOSFODNN7EXAMPLE" - MINIO_SECRET_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + MINIO_ADDRESS: :9000 + MINIO_ROOT_USER: "AKIAIOSFODNN7EXAMPLE" + MINIO_ROOT_PASSWORD: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" MINIO_REGION_NAME: "us-east-1" MINIO_DOMAIN: "minio" MINIO_BROWSER: "off"