From efe38f8b70a9602f7f4c5004b1a83eb3da09fe50 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:12:16 +0100 Subject: [PATCH 01/15] fuck github actions --- .github/workflows/backport.yaml | 4 ++-- scripts/backport.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 2101abb93d7..47037816bd9 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -39,7 +39,7 @@ jobs: - name: Run the Backport Script env: - GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} + GITHUB_TOKEN: ${{ secrets.TIMESCALEDB_AUTOMATION_TOKEN_2 }} run: | git remote --verbose - scripts/backport.py + scripts/backport.py 2>&1 diff --git a/scripts/backport.py b/scripts/backport.py index d1d8c3549d1..9622c9abe8b 100755 --- a/scripts/backport.py +++ b/scripts/backport.py @@ -330,6 +330,9 @@ def should_backport_by_labels(number, title, labels): # commits in forward order. prs_to_backport[pull.number].pygithub_commits.insert(0, pygithub_commit) + # FIXME + break + def branch_has_open_pr(repo, branch): """Check whether the given branch has an open PR.""" @@ -509,7 +512,7 @@ def report_backport_not_done(original_pr, reason, details=None): ) # Push the backport branch. - git_check(f"push --quiet {target_remote} @:refs/heads/{backport_branch}") + git_check(f"push {target_remote} @:refs/heads/{backport_branch}") # Prepare description for the backport PR. backport_description = ( From f0a4f1fbe4a512faa13e246f6691f308a674b3d6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:26:26 +0100 Subject: [PATCH 02/15] checkout a fork --- .github/workflows/backport.yaml | 8 +++++++- scripts/backport.py | 12 ++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 47037816bd9..7ecfc874799 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -36,10 +36,16 @@ jobs: - name: Checkout TimescaleDB uses: actions/checkout@v4 + with: + repository: timescale-automation/timescaledb - name: Run the Backport Script env: - GITHUB_TOKEN: ${{ secrets.TIMESCALEDB_AUTOMATION_TOKEN_2 }} + # GITHUB_TOKEN: ${{ secrets.TIMESCALEDB_AUTOMATION_TOKEN_2 }} + GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | + git remote add upstream https://github.com/timescale/timescaledb.git + git fetch upstream + git checkout upstream/main git remote --verbose scripts/backport.py 2>&1 diff --git a/scripts/backport.py b/scripts/backport.py index 9622c9abe8b..7a2147f23ea 100755 --- a/scripts/backport.py +++ b/scripts/backport.py @@ -139,7 +139,7 @@ def git_returncode(command): # The token has to have the "access public repositories" permission, or else creating a PR returns 404. github = Github(os.environ.get("GITHUB_TOKEN")) -source_remote = "origin" +source_remote = "upstream" source_repo_name = os.environ.get("GITHUB_REPOSITORY") # This is set in GitHub Actions. if not source_repo_name: source_repo_name = "timescale/timescaledb" @@ -413,11 +413,11 @@ def report_backport_not_done(original_pr, reason, details=None): # If we use the default credentials of the GitHub Actions to push, the workflows # in the backport PRs don't start. To keep the things more fun for us, GitHub # does this only for the merge commits, but not for the new PRs. -target_remote = "backport-target-remote" -git_returncode(f"remote remove {target_remote}") -git_check( - f'remote add {target_remote} https://{os.environ["GITHUB_TOKEN"]}@github.com/{token_user.login}/{source_repo.name}.git' -) +target_remote = "origin" +#git_returncode(f"remote remove {target_remote}") +#git_check( +# f'remote add {target_remote} https://{os.environ["GITHUB_TOKEN"]}@github.com/{token_user.login}/{source_repo.name}.git' +#) # Fetch all branches from the target repository, because we use the presence # of the backport branches to determine that a backport exists. It's not convenient From f890a3c2e4f6a6ddba9c64ec6f420819c6b3d96b Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:34:05 +0100 Subject: [PATCH 03/15] fetch the correct ref --- .github/workflows/backport.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 7ecfc874799..21c07f72f7e 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -44,8 +44,10 @@ jobs: # GITHUB_TOKEN: ${{ secrets.TIMESCALEDB_AUTOMATION_TOKEN_2 }} GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | - git remote add upstream https://github.com/timescale/timescaledb.git + git remote add upstream "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" git fetch upstream - git checkout upstream/main + git fetch upstream "${GITHUB_REF}" + git checkout "${GITHUB_REF}" + git log -2 git remote --verbose scripts/backport.py 2>&1 From e0bc5060c5ac5f3a7175d7a7d35670608e408a71 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:40:19 +0100 Subject: [PATCH 04/15] sha --- .github/workflows/backport.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 21c07f72f7e..d973025d1f7 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -47,7 +47,7 @@ jobs: git remote add upstream "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" git fetch upstream git fetch upstream "${GITHUB_REF}" - git checkout "${GITHUB_REF}" + git checkout "${GITHUB_SHA}" git log -2 git remote --verbose scripts/backport.py 2>&1 From b781a24f59ef83e4284085f2c0df59b54698b3a4 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:51:09 +0100 Subject: [PATCH 05/15] tokens --- .github/workflows/backport.yaml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index d973025d1f7..2046d55cb6c 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -18,6 +18,10 @@ on: # The workflow needs the permission to push branches permissions: contents: write + pull_requests: write + issues: write + actions: write + statuses: write jobs: backport: @@ -41,8 +45,8 @@ jobs: - name: Run the Backport Script env: - # GITHUB_TOKEN: ${{ secrets.TIMESCALEDB_AUTOMATION_TOKEN_2 }} - GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} + GITHUB_TOKEN_2: ${{ secrets.TIMESCALEDB_AUTOMATION_TOKEN_2 }} + GITHUB_TOKEN_3: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | git remote add upstream "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" git fetch upstream @@ -50,4 +54,20 @@ jobs: git checkout "${GITHUB_SHA}" git log -2 git remote --verbose + git remote add pat1 "https://timesacle-automation:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git remote add pat11 "https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git remote add pat2 "https://timesacle-automation:${GITHUB_TOKEN_2}@github.com/${GITHUB_REPOSITORY}.git" + git remote add pat21 "https://${GITHUB_TOKEN_2}@github.com/${GITHUB_REPOSITORY}.git" + git remote add pat3 "https://timesacle-automation:${GITHUB_TOKEN_3}@github.com/${GITHUB_REPOSITORY}.git" + git remote add pat31 "https://${GITHUB_TOKEN_3}@github.com/${GITHUB_REPOSITORY}.git" + + git push origin @:refs/heads/backport/test1 -f + git push upstream @:refs/heads/backport/test2 -f + git push pat1 @:refs/heads/backport/test4 -f + git push pat11 @:refs/heads/backport/test5 -f + git push pat2 @:refs/heads/backport/test6 -f + git push pat21 @:refs/heads/backport/test7 -f + git push pat3 @:refs/heads/backport/test8 -f + git push pat31 @:refs/heads/backport/test9 -f + scripts/backport.py 2>&1 From e69c8132ce2859a34030c28c6725cd323f804c28 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:51:55 +0100 Subject: [PATCH 06/15] github... --- .github/workflows/backport.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 2046d55cb6c..0333c9648ee 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -18,7 +18,7 @@ on: # The workflow needs the permission to push branches permissions: contents: write - pull_requests: write + pull-requests: write issues: write actions: write statuses: write From f97ff65a1cb14236fe9feb85f2a9ae7ad91f78e6 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:53:04 +0100 Subject: [PATCH 07/15] fix --- .github/workflows/backport.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 0333c9648ee..3612469206d 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -29,11 +29,6 @@ jobs: runs-on: ubuntu-latest steps: - - name: Install Linux Dependencies - run: | - sudo apt-get update - sudo apt-get install pip - - name: Install Python Dependencies run: | pip install PyGithub requests @@ -48,6 +43,8 @@ jobs: GITHUB_TOKEN_2: ${{ secrets.TIMESCALEDB_AUTOMATION_TOKEN_2 }} GITHUB_TOKEN_3: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | + set -x + set +e git remote add upstream "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" git fetch upstream git fetch upstream "${GITHUB_REF}" From 2ae1d5bebd9abe803239254eaa6b181f9bb68e6b Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 15:57:41 +0100 Subject: [PATCH 08/15] test more things --- .github/workflows/backport.yaml | 1 + scripts/backport.py | 86 ++++++++++++++++----------------- 2 files changed, 43 insertions(+), 44 deletions(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 3612469206d..74ce13796f2 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -40,6 +40,7 @@ jobs: - name: Run the Backport Script env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN_2: ${{ secrets.TIMESCALEDB_AUTOMATION_TOKEN_2 }} GITHUB_TOKEN_3: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | diff --git a/scripts/backport.py b/scripts/backport.py index 7a2147f23ea..5670687d5a3 100755 --- a/scripts/backport.py +++ b/scripts/backport.py @@ -90,6 +90,7 @@ def get_referenced_issue(pr_number): def set_auto_merge(pr_number): """Enable auto-merge for the given PR""" + owner, name = target_repo_name.split("/") # We first have to find out the PR id, which is some base64 string, different # from its number. query = string.Template( @@ -100,9 +101,7 @@ def set_auto_merge(pr_number): } } }""" - ).substitute( - pr_number=pr_number, owner=source_repo.owner.login, name=source_repo.name - ) + ).substitute(pr_number=pr_number, owner=owner, name=name) result = run_query(query) pr_id = result["data"]["repository"]["pullRequest"]["id"] @@ -139,16 +138,33 @@ def git_returncode(command): # The token has to have the "access public repositories" permission, or else creating a PR returns 404. github = Github(os.environ.get("GITHUB_TOKEN")) -source_remote = "upstream" -source_repo_name = os.environ.get("GITHUB_REPOSITORY") # This is set in GitHub Actions. +# If we are running inside Github Action, will modify the main repo. +source_remote = "origin" +source_repo_name = os.environ.get("GITHUB_REPOSITORY") +target_remote = source_remote +target_repo_name = source_repo_name + if not source_repo_name: + # We are running manually for debugging, probably want to modify a fork. source_repo_name = "timescale/timescaledb" + target_repo_name = os.environ.get("BACKPORT_TARGET_REPO") + target_remote = os.environ.get("BACKPORT_TARGET_REMOTE") + if not target_repo_name or not target_remote: + print( + "Please specify the target repositories for debugging, using the " + "environment variables BACKPORT_TARGET_REPO (e.g. `timescale/timescaledb`) " + "and BACKPORT_TARGET_REMOTE (e.g. `origin`).", + file=sys.stderr, + ) + sys.exit(1) print( - f"Will look at '{source_repo_name}' (git remote '{source_remote}') for bug fixes." + f"Will look at '{source_repo_name}' (git remote '{source_remote}') for bug fixes, " + f"and create the backport PRs in '{target_repo_name}' (git remote '{target_remote}')." ) source_repo = github.get_repo(source_repo_name) +target_repo = github.get_repo(target_repo_name) # Fetch the main branch. Apparently the local repo can be shallow in some cases # in Github Actions, so specify the depth. --unshallow will complain on normal @@ -181,16 +197,21 @@ def git_returncode(command): # in Github Actions, so specify the depth. --unshallow will complain on normal # repositories, this is why we don't use it here. git_check( - f"fetch --quiet --depth={HISTORY_DEPTH} {source_remote} {backport_target}:refs/remotes/{source_remote}/{backport_target}" + f"fetch --quiet --depth={HISTORY_DEPTH} {target_remote} {backport_target}:refs/remotes/{target_remote}/{backport_target}" ) +# Also fetch all branches from the target repository, because we use the presence +# of the backport branches to determine that a backport exists. It's not convenient +# to query for branch existence through the PyGithub API. +git_check(f"fetch {target_remote}") + # Find out which commits are unique to main and target branch. Also build sets of # the titles of these commits. We will compare the titles to check whether a # commit was backported. main_commits = [ line.split("\t") for line in git_output( - f'log -{HISTORY_DEPTH} --pretty="format:%h\t%s" {source_remote}/{backport_target}..{source_remote}/main' + f'log -{HISTORY_DEPTH} --pretty="format:%h\t%s" {target_remote}/{backport_target}..{source_remote}/main' ).splitlines() if line ] @@ -200,7 +221,7 @@ def git_returncode(command): branch_commits = [ line.split("\t") for line in git_output( - f'log -{HISTORY_DEPTH} --pretty="format:%h\t%s" {source_remote}/main..{source_remote}/{backport_target}' + f'log -{HISTORY_DEPTH} --pretty="format:%h\t%s" {source_remote}/main..{target_remote}/{backport_target}' ).splitlines() if line ] @@ -330,19 +351,15 @@ def should_backport_by_labels(number, title, labels): # commits in forward order. prs_to_backport[pull.number].pygithub_commits.insert(0, pygithub_commit) - # FIXME - break - def branch_has_open_pr(repo, branch): """Check whether the given branch has an open PR.""" - # There's no way to search by branch name + fork name, but in the case the - # branch name is probably unique. We'll bail out if we find more than one PR. template = """query { repository(name: "$repo_name", owner: "$repo_owner") { - pullRequests(headRefName: "$branch", first: 2) { - nodes { closed }}}}""" + ref(qualifiedName: "$branch") { + associatedPullRequests(first: 1) { + nodes { closed }}}}}""" params = { "branch": branch, @@ -350,14 +367,12 @@ def branch_has_open_pr(repo, branch): "repo_owner": repo.owner.login, } - query = string.Template(template).substitute(params) - - result = run_query(query) + result = run_query(string.Template(template).substitute(params)) # This returns: - # {'data': {'repository': {'pullRequests': {'nodes': [{'closed': True}]}}}} + # {'data': {'repository': {'ref': {'associatedPullRequests': {'nodes': [{'closed': True}]}}}}} - prs = result["data"]["repository"]["pullRequests"]["nodes"] + prs = result["data"]["repository"]["ref"]["associatedPullRequests"]["nodes"] if not prs or len(prs) != 1 or not prs[0]: return None @@ -409,20 +424,6 @@ def report_backport_not_done(original_pr, reason, details=None): f"Will commit as {os.environ['GIT_COMMITTER_NAME']} <{os.environ['GIT_COMMITTER_EMAIL']}>" ) -# We have to push using the personal access token of the automation user. -# If we use the default credentials of the GitHub Actions to push, the workflows -# in the backport PRs don't start. To keep the things more fun for us, GitHub -# does this only for the merge commits, but not for the new PRs. -target_remote = "origin" -#git_returncode(f"remote remove {target_remote}") -#git_check( -# f'remote add {target_remote} https://{os.environ["GITHUB_TOKEN"]}@github.com/{token_user.login}/{source_repo.name}.git' -#) - -# Fetch all branches from the target repository, because we use the presence -# of the backport branches to determine that a backport exists. It's not convenient -# to query for branch existence through the PyGithub API. -git_check(f"fetch {target_remote}") # Now, go over the list of PRs that we have collected, and try to backport # each of them. @@ -439,8 +440,6 @@ def report_backport_not_done(original_pr, reason, details=None): original_pr = pr_info.pygithub_pr backport_branch = f"backport/{backport_target}/{original_pr.number}" - # We're creating the backport PR from the token user's fork. - backport_pr_head = f"{token_user.login}:{backport_branch}" # If there is already a backport branch for this PR, this probably means # that we already created the backport PR. Update it, because the PR might @@ -455,7 +454,7 @@ def report_backport_not_done(original_pr, reason, details=None): f'Backport branch {backport_branch} for PR #{original_pr.number}: "{original_pr.title}" already exists.' ) - if not branch_has_open_pr(source_repo, backport_branch): + if not branch_has_open_pr(target_repo, backport_branch): # The PR can be closed manually when the backport is not needed, or # can not exist when there was some error. We are only interested in # the most certain case when there is an open backport PR. @@ -469,7 +468,7 @@ def report_backport_not_done(original_pr, reason, details=None): ) # Use merge and no force-push, so that the simultaneous changes made by # other users are not accidentally overwritten. - git_check(f"merge --quiet --no-edit {source_remote}/{backport_target}") + git_check(f"merge --quiet {target_remote}/{backport_target}") git_check(f"push --quiet {target_remote} @:{backport_branch}") continue @@ -477,7 +476,7 @@ def report_backport_not_done(original_pr, reason, details=None): git_check("reset --hard") git_check("clean -xfd") git_check( - f"checkout --quiet --force --detach {source_remote}/{backport_target} > /dev/null" + f"checkout --quiet --force --detach {target_remote}/{backport_target} > /dev/null" ) commit_shas = [commit.sha for commit in pr_info.pygithub_commits] @@ -512,7 +511,7 @@ def report_backport_not_done(original_pr, reason, details=None): ) # Push the backport branch. - git_check(f"push {target_remote} @:refs/heads/{backport_branch}") + git_check(f"push --quiet {target_remote} @:refs/heads/{backport_branch}") # Prepare description for the backport PR. backport_description = ( @@ -571,11 +570,10 @@ def report_backport_not_done(original_pr, reason, details=None): ) # Create the backport PR. - backport_pr = source_repo.create_pull( + backport_pr = target_repo.create_pull( title=f"Backport to {backport_target}: #{original_pr.number}: {original_pr.title}", body=backport_description, - # We're creating PR from the token user's fork. - head=backport_pr_head, + head=backport_branch, base=backport_target, ) backport_pr.add_to_labels("is-auto-backport") From 56f85f61f6dffb651ce672195a3f243867de0080 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 16:00:24 +0100 Subject: [PATCH 09/15] main repo --- .github/workflows/backport.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 74ce13796f2..4280899cc2b 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -35,8 +35,8 @@ jobs: - name: Checkout TimescaleDB uses: actions/checkout@v4 - with: - repository: timescale-automation/timescaledb + # with: + # repository: timescale-automation/timescaledb - name: Run the Backport Script env: From 481d989c830eab91117395047a4d451013c91f85 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 16:02:59 +0100 Subject: [PATCH 10/15] the script must still push through PAT --- scripts/backport.py | 79 ++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/scripts/backport.py b/scripts/backport.py index 5670687d5a3..d1d8c3549d1 100755 --- a/scripts/backport.py +++ b/scripts/backport.py @@ -90,7 +90,6 @@ def get_referenced_issue(pr_number): def set_auto_merge(pr_number): """Enable auto-merge for the given PR""" - owner, name = target_repo_name.split("/") # We first have to find out the PR id, which is some base64 string, different # from its number. query = string.Template( @@ -101,7 +100,9 @@ def set_auto_merge(pr_number): } } }""" - ).substitute(pr_number=pr_number, owner=owner, name=name) + ).substitute( + pr_number=pr_number, owner=source_repo.owner.login, name=source_repo.name + ) result = run_query(query) pr_id = result["data"]["repository"]["pullRequest"]["id"] @@ -138,33 +139,16 @@ def git_returncode(command): # The token has to have the "access public repositories" permission, or else creating a PR returns 404. github = Github(os.environ.get("GITHUB_TOKEN")) -# If we are running inside Github Action, will modify the main repo. source_remote = "origin" -source_repo_name = os.environ.get("GITHUB_REPOSITORY") -target_remote = source_remote -target_repo_name = source_repo_name - +source_repo_name = os.environ.get("GITHUB_REPOSITORY") # This is set in GitHub Actions. if not source_repo_name: - # We are running manually for debugging, probably want to modify a fork. source_repo_name = "timescale/timescaledb" - target_repo_name = os.environ.get("BACKPORT_TARGET_REPO") - target_remote = os.environ.get("BACKPORT_TARGET_REMOTE") - if not target_repo_name or not target_remote: - print( - "Please specify the target repositories for debugging, using the " - "environment variables BACKPORT_TARGET_REPO (e.g. `timescale/timescaledb`) " - "and BACKPORT_TARGET_REMOTE (e.g. `origin`).", - file=sys.stderr, - ) - sys.exit(1) print( - f"Will look at '{source_repo_name}' (git remote '{source_remote}') for bug fixes, " - f"and create the backport PRs in '{target_repo_name}' (git remote '{target_remote}')." + f"Will look at '{source_repo_name}' (git remote '{source_remote}') for bug fixes." ) source_repo = github.get_repo(source_repo_name) -target_repo = github.get_repo(target_repo_name) # Fetch the main branch. Apparently the local repo can be shallow in some cases # in Github Actions, so specify the depth. --unshallow will complain on normal @@ -197,21 +181,16 @@ def git_returncode(command): # in Github Actions, so specify the depth. --unshallow will complain on normal # repositories, this is why we don't use it here. git_check( - f"fetch --quiet --depth={HISTORY_DEPTH} {target_remote} {backport_target}:refs/remotes/{target_remote}/{backport_target}" + f"fetch --quiet --depth={HISTORY_DEPTH} {source_remote} {backport_target}:refs/remotes/{source_remote}/{backport_target}" ) -# Also fetch all branches from the target repository, because we use the presence -# of the backport branches to determine that a backport exists. It's not convenient -# to query for branch existence through the PyGithub API. -git_check(f"fetch {target_remote}") - # Find out which commits are unique to main and target branch. Also build sets of # the titles of these commits. We will compare the titles to check whether a # commit was backported. main_commits = [ line.split("\t") for line in git_output( - f'log -{HISTORY_DEPTH} --pretty="format:%h\t%s" {target_remote}/{backport_target}..{source_remote}/main' + f'log -{HISTORY_DEPTH} --pretty="format:%h\t%s" {source_remote}/{backport_target}..{source_remote}/main' ).splitlines() if line ] @@ -221,7 +200,7 @@ def git_returncode(command): branch_commits = [ line.split("\t") for line in git_output( - f'log -{HISTORY_DEPTH} --pretty="format:%h\t%s" {source_remote}/main..{target_remote}/{backport_target}' + f'log -{HISTORY_DEPTH} --pretty="format:%h\t%s" {source_remote}/main..{source_remote}/{backport_target}' ).splitlines() if line ] @@ -355,11 +334,12 @@ def should_backport_by_labels(number, title, labels): def branch_has_open_pr(repo, branch): """Check whether the given branch has an open PR.""" + # There's no way to search by branch name + fork name, but in the case the + # branch name is probably unique. We'll bail out if we find more than one PR. template = """query { repository(name: "$repo_name", owner: "$repo_owner") { - ref(qualifiedName: "$branch") { - associatedPullRequests(first: 1) { - nodes { closed }}}}}""" + pullRequests(headRefName: "$branch", first: 2) { + nodes { closed }}}}""" params = { "branch": branch, @@ -367,12 +347,14 @@ def branch_has_open_pr(repo, branch): "repo_owner": repo.owner.login, } - result = run_query(string.Template(template).substitute(params)) + query = string.Template(template).substitute(params) + + result = run_query(query) # This returns: - # {'data': {'repository': {'ref': {'associatedPullRequests': {'nodes': [{'closed': True}]}}}}} + # {'data': {'repository': {'pullRequests': {'nodes': [{'closed': True}]}}}} - prs = result["data"]["repository"]["ref"]["associatedPullRequests"]["nodes"] + prs = result["data"]["repository"]["pullRequests"]["nodes"] if not prs or len(prs) != 1 or not prs[0]: return None @@ -424,6 +406,20 @@ def report_backport_not_done(original_pr, reason, details=None): f"Will commit as {os.environ['GIT_COMMITTER_NAME']} <{os.environ['GIT_COMMITTER_EMAIL']}>" ) +# We have to push using the personal access token of the automation user. +# If we use the default credentials of the GitHub Actions to push, the workflows +# in the backport PRs don't start. To keep the things more fun for us, GitHub +# does this only for the merge commits, but not for the new PRs. +target_remote = "backport-target-remote" +git_returncode(f"remote remove {target_remote}") +git_check( + f'remote add {target_remote} https://{os.environ["GITHUB_TOKEN"]}@github.com/{token_user.login}/{source_repo.name}.git' +) + +# Fetch all branches from the target repository, because we use the presence +# of the backport branches to determine that a backport exists. It's not convenient +# to query for branch existence through the PyGithub API. +git_check(f"fetch {target_remote}") # Now, go over the list of PRs that we have collected, and try to backport # each of them. @@ -440,6 +436,8 @@ def report_backport_not_done(original_pr, reason, details=None): original_pr = pr_info.pygithub_pr backport_branch = f"backport/{backport_target}/{original_pr.number}" + # We're creating the backport PR from the token user's fork. + backport_pr_head = f"{token_user.login}:{backport_branch}" # If there is already a backport branch for this PR, this probably means # that we already created the backport PR. Update it, because the PR might @@ -454,7 +452,7 @@ def report_backport_not_done(original_pr, reason, details=None): f'Backport branch {backport_branch} for PR #{original_pr.number}: "{original_pr.title}" already exists.' ) - if not branch_has_open_pr(target_repo, backport_branch): + if not branch_has_open_pr(source_repo, backport_branch): # The PR can be closed manually when the backport is not needed, or # can not exist when there was some error. We are only interested in # the most certain case when there is an open backport PR. @@ -468,7 +466,7 @@ def report_backport_not_done(original_pr, reason, details=None): ) # Use merge and no force-push, so that the simultaneous changes made by # other users are not accidentally overwritten. - git_check(f"merge --quiet {target_remote}/{backport_target}") + git_check(f"merge --quiet --no-edit {source_remote}/{backport_target}") git_check(f"push --quiet {target_remote} @:{backport_branch}") continue @@ -476,7 +474,7 @@ def report_backport_not_done(original_pr, reason, details=None): git_check("reset --hard") git_check("clean -xfd") git_check( - f"checkout --quiet --force --detach {target_remote}/{backport_target} > /dev/null" + f"checkout --quiet --force --detach {source_remote}/{backport_target} > /dev/null" ) commit_shas = [commit.sha for commit in pr_info.pygithub_commits] @@ -570,10 +568,11 @@ def report_backport_not_done(original_pr, reason, details=None): ) # Create the backport PR. - backport_pr = target_repo.create_pull( + backport_pr = source_repo.create_pull( title=f"Backport to {backport_target}: #{original_pr.number}: {original_pr.title}", body=backport_description, - head=backport_branch, + # We're creating PR from the token user's fork. + head=backport_pr_head, base=backport_target, ) backport_pr.add_to_labels("is-auto-backport") From 0c25499d26ba8fb1064edaada34792a1e9af3e51 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 16:09:38 +0100 Subject: [PATCH 11/15] other token --- .github/workflows/backport.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 4280899cc2b..e1c30e51ec2 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -40,9 +40,9 @@ jobs: - name: Run the Backport Script env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN_2: ${{ secrets.TIMESCALEDB_AUTOMATION_TOKEN_2 }} - GITHUB_TOKEN_3: ${{ secrets.ORG_AUTOMATION_TOKEN }} + GITHUB_TOKEN_2: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN_3: ${{ secrets.TIMESCALEDB_AUTOMATION_TOKEN_2 }} + GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | set -x set +e From e4c5eba0910170d6c5b85f1b9ff5d4162e75df78 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 16:44:30 +0100 Subject: [PATCH 12/15] try to push to the main repo --- .github/workflows/backport.yaml | 2 ++ scripts/backport.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index e1c30e51ec2..a466ad5891f 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -69,3 +69,5 @@ jobs: git push pat31 @:refs/heads/backport/test9 -f scripts/backport.py 2>&1 + + git remote --verbose diff --git a/scripts/backport.py b/scripts/backport.py index d1d8c3549d1..c1e850f88c1 100755 --- a/scripts/backport.py +++ b/scripts/backport.py @@ -413,7 +413,7 @@ def report_backport_not_done(original_pr, reason, details=None): target_remote = "backport-target-remote" git_returncode(f"remote remove {target_remote}") git_check( - f'remote add {target_remote} https://{os.environ["GITHUB_TOKEN"]}@github.com/{token_user.login}/{source_repo.name}.git' + f'remote add {target_remote} https://{os.environ["GITHUB_TOKEN"]}@github.com/{source_repo.owner.login}/{source_repo.name}.git' ) # Fetch all branches from the target repository, because we use the presence From f23725cca1d3e7bdde6adea538c95d411a79fd14 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 16:55:10 +0100 Subject: [PATCH 13/15] fix pr head --- scripts/backport.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/backport.py b/scripts/backport.py index c1e850f88c1..8814be36c78 100755 --- a/scripts/backport.py +++ b/scripts/backport.py @@ -436,8 +436,6 @@ def report_backport_not_done(original_pr, reason, details=None): original_pr = pr_info.pygithub_pr backport_branch = f"backport/{backport_target}/{original_pr.number}" - # We're creating the backport PR from the token user's fork. - backport_pr_head = f"{token_user.login}:{backport_branch}" # If there is already a backport branch for this PR, this probably means # that we already created the backport PR. Update it, because the PR might @@ -474,7 +472,7 @@ def report_backport_not_done(original_pr, reason, details=None): git_check("reset --hard") git_check("clean -xfd") git_check( - f"checkout --quiet --force --detach {source_remote}/{backport_target} > /dev/null" + f"checkout --quiet --force --detach {source_remote}/{backport_target}~ > /dev/null" ) commit_shas = [commit.sha for commit in pr_info.pygithub_commits] @@ -572,7 +570,7 @@ def report_backport_not_done(original_pr, reason, details=None): title=f"Backport to {backport_target}: #{original_pr.number}: {original_pr.title}", body=backport_description, # We're creating PR from the token user's fork. - head=backport_pr_head, + head=backport_branch, base=backport_target, ) backport_pr.add_to_labels("is-auto-backport") From c9b3707a5aed4aa8f98c3c1816454f435c4b8743 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 17:48:58 +0100 Subject: [PATCH 14/15] fixes --- .github/workflows/backport.yaml | 28 +--------------------------- scripts/backport.py | 6 +++--- 2 files changed, 4 insertions(+), 30 deletions(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index a466ad5891f..188e9466d8f 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -35,39 +35,13 @@ jobs: - name: Checkout TimescaleDB uses: actions/checkout@v4 - # with: - # repository: timescale-automation/timescaledb - name: Run the Backport Script env: - GITHUB_TOKEN_2: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN_3: ${{ secrets.TIMESCALEDB_AUTOMATION_TOKEN_2 }} GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | - set -x - set +e - git remote add upstream "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" - git fetch upstream - git fetch upstream "${GITHUB_REF}" - git checkout "${GITHUB_SHA}" - git log -2 git remote --verbose - git remote add pat1 "https://timesacle-automation:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git remote add pat11 "https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - git remote add pat2 "https://timesacle-automation:${GITHUB_TOKEN_2}@github.com/${GITHUB_REPOSITORY}.git" - git remote add pat21 "https://${GITHUB_TOKEN_2}@github.com/${GITHUB_REPOSITORY}.git" - git remote add pat3 "https://timesacle-automation:${GITHUB_TOKEN_3}@github.com/${GITHUB_REPOSITORY}.git" - git remote add pat31 "https://${GITHUB_TOKEN_3}@github.com/${GITHUB_REPOSITORY}.git" - - git push origin @:refs/heads/backport/test1 -f - git push upstream @:refs/heads/backport/test2 -f - git push pat1 @:refs/heads/backport/test4 -f - git push pat11 @:refs/heads/backport/test5 -f - git push pat2 @:refs/heads/backport/test6 -f - git push pat21 @:refs/heads/backport/test7 -f - git push pat3 @:refs/heads/backport/test8 -f - git push pat31 @:refs/heads/backport/test9 -f - + scripts/backport.py scripts/backport.py 2>&1 git remote --verbose diff --git a/scripts/backport.py b/scripts/backport.py index 8814be36c78..caa686f7d3f 100755 --- a/scripts/backport.py +++ b/scripts/backport.py @@ -465,14 +465,14 @@ def report_backport_not_done(original_pr, reason, details=None): # Use merge and no force-push, so that the simultaneous changes made by # other users are not accidentally overwritten. git_check(f"merge --quiet --no-edit {source_remote}/{backport_target}") - git_check(f"push --quiet {target_remote} @:{backport_branch}") + git_check(f"push {target_remote} @:{backport_branch}") continue # Try to cherry-pick the commits. git_check("reset --hard") git_check("clean -xfd") git_check( - f"checkout --quiet --force --detach {source_remote}/{backport_target}~ > /dev/null" + f"checkout --quiet --force --detach {source_remote}/{backport_target} > /dev/null" ) commit_shas = [commit.sha for commit in pr_info.pygithub_commits] @@ -507,7 +507,7 @@ def report_backport_not_done(original_pr, reason, details=None): ) # Push the backport branch. - git_check(f"push --quiet {target_remote} @:refs/heads/{backport_branch}") + git_check(f"push {target_remote} @:refs/heads/{backport_branch}") # Prepare description for the backport PR. backport_description = ( From b805a590001b3084300e146367d1948a10c93448 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 21 Feb 2025 23:59:35 +0100 Subject: [PATCH 15/15] remove extra --- .github/workflows/backport.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yaml b/.github/workflows/backport.yaml index 188e9466d8f..30470878c50 100644 --- a/.github/workflows/backport.yaml +++ b/.github/workflows/backport.yaml @@ -41,7 +41,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | git remote --verbose - scripts/backport.py + scripts/backport.py 2>&1 git remote --verbose