From 2991a2b178ff6642411c60d3b4a32127518a547f Mon Sep 17 00:00:00 2001 From: bkuschel Date: Wed, 17 Aug 2022 08:44:53 -0400 Subject: [PATCH 1/5] test branch --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 05fd337e57c5..2b1734c87d53 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -11,7 +11,7 @@ on: # yamllint disable-line rule:truthy - reopened - opened branches: - - master + - oss_runners paths-ignore: - 'docker/docs/**' - 'docs/**' From 54da333a87dc27c8ce3a7d33f8bf633e8ffaa70d Mon Sep 17 00:00:00 2001 From: bkuschel Date: Fri, 19 Aug 2022 12:32:17 -0400 Subject: [PATCH 2/5] Fix fast test --- docker/test/fasttest/Dockerfile | 6 ++++-- docker/test/fasttest/run.sh | 2 +- tests/ci/fast_test_check.py | 15 ++++++++++++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index aa209f197dfd..97c9b4c3705c 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -1,7 +1,9 @@ # rebuild in #33610 # docker build -t clickhouse/fasttest . -ARG FROM_TAG=latest -FROM clickhouse/test-util:$FROM_TAG +ENV GITHUB_SERVER_URL=https://github.com +ENV GITHUB_REPOSITORY=ClickHouse/ClickHouse +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 2bbdd978e5e7..e4b08e98523d 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -73,7 +73,7 @@ function start_server function clone_root { git config --global --add safe.directory "$FASTTEST_SOURCE" - git clone --depth 1 https://github.com/ClickHouse/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" + git clone --depth 1 $GITHUB_SERVER_URL/$GITHUB_REPOSITORY -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt" ( cd "$FASTTEST_SOURCE" diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index b756a5c709f8..eab7933bd416 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -8,10 +8,10 @@ from github import Github -from env_helper import CACHES_PATH, TEMP_PATH, GITHUB_SERVER_URL, GITHUB_REPOSITORY +from env_helper import CACHES_PATH, TEMP_PATH, GITHUB_SERVER_URL, GITHUB_REPOSITORY, DOCKER_REPO, DOCKER_USER from pr_info import FORCE_TESTS_LABEL, PRInfo, SKIP_SIMPLE_CHECK_LABEL from s3_helper import S3Helper -from get_robot_token import get_best_robot_token +from get_robot_token import get_best_robot_token, get_parameter_from_ssm from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version from commit_status_helper import post_commit_status, get_commit @@ -37,6 +37,8 @@ def get_fasttest_cmd( f"-e FASTTEST_SOURCE=/ClickHouse --cap-add=SYS_PTRACE " f"-e PULL_REQUEST_NUMBER={pr_number} -e COMMIT_SHA={commit_sha} " f"-e COPY_CLICKHOUSE_BINARY_TO_OUTPUT=1 " + f"-e GITHUB_SERVER_URL={GITHUB_SERVER_URL} " + f"-e GITHUB_REPOSITORY={GITHUB_REPOSITORY} " f"--volume={workspace}:/fasttest-workspace --volume={repo_path}:/ClickHouse " f"--volume={output_path}:/test_output " f"--volume={ccache_path}:/fasttest-workspace/ccache {image}" @@ -90,6 +92,13 @@ def process_results(result_folder): pr_info = PRInfo() + subprocess.check_output( # pylint: disable=unexpected-keyword-arg + "docker login {} --username '{}' --password-stdin".format(DOCKER_REPO, DOCKER_USER), + input=get_parameter_from_ssm("dockerhub_robot_password"), + encoding="utf-8", + shell=True, + ) + gh = Github(get_best_robot_token()) rerun_helper = RerunHelper(gh, pr_info, NAME) @@ -97,7 +106,7 @@ def process_results(result_folder): logging.info("Check is already finished according to github status, exiting") sys.exit(0) - docker_image = get_image_with_version(temp_path, "clickhouse/fasttest") + docker_image = get_image_with_version(temp_path, "{}/clickhouse/fasttest".format(DOCKER_REPO)) s3_helper = S3Helper() From fd8b88a7b8a4b71e273016d01b127d4c1e525fa8 Mon Sep 17 00:00:00 2001 From: bkuschel Date: Fri, 19 Aug 2022 15:04:46 -0400 Subject: [PATCH 3/5] Fast test fix --- docker/test/fasttest/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 97c9b4c3705c..47594730a9d8 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -1,9 +1,11 @@ # rebuild in #33610 # docker build -t clickhouse/fasttest . +ARG FROM_TAG=latest +ARG DOCKER_REPO=docker.io +FROM $DOCKER_REPO/clickhouse/fasttest:$FROM_TAG + ENV GITHUB_SERVER_URL=https://github.com ENV GITHUB_REPOSITORY=ClickHouse/ClickHouse -ARG DOCKER_REPO=docker.io -FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" From d25ff582d38be06f15cc1d0fe21344479560af93 Mon Sep 17 00:00:00 2001 From: bkuschel Date: Fri, 19 Aug 2022 15:11:41 -0400 Subject: [PATCH 4/5] Fast test fix 2 --- docker/test/fasttest/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 47594730a9d8..b9b0cf07d2ae 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -2,7 +2,7 @@ # docker build -t clickhouse/fasttest . ARG FROM_TAG=latest ARG DOCKER_REPO=docker.io -FROM $DOCKER_REPO/clickhouse/fasttest:$FROM_TAG +FROM $DOCKER_REPO/clickhouse/test-util::$FROM_TAG ENV GITHUB_SERVER_URL=https://github.com ENV GITHUB_REPOSITORY=ClickHouse/ClickHouse From b138fa3d5b3abc3721d36abab977b50fe3c6f952 Mon Sep 17 00:00:00 2001 From: bkuschel Date: Fri, 19 Aug 2022 15:34:18 -0400 Subject: [PATCH 5/5] Fast test fix 3 --- docker/test/fasttest/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index b9b0cf07d2ae..bc79bea416c9 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -2,7 +2,7 @@ # docker build -t clickhouse/fasttest . ARG FROM_TAG=latest ARG DOCKER_REPO=docker.io -FROM $DOCKER_REPO/clickhouse/test-util::$FROM_TAG +FROM $DOCKER_REPO/clickhouse/test-util:$FROM_TAG ENV GITHUB_SERVER_URL=https://github.com ENV GITHUB_REPOSITORY=ClickHouse/ClickHouse