diff --git a/cronjobs/src/commands/_git_export_git_tools.py b/cronjobs/src/commands/_git_export_git_tools.py index 48b70ad1..986aabb9 100644 --- a/cronjobs/src/commands/_git_export_git_tools.py +++ b/cronjobs/src/commands/_git_export_git_tools.py @@ -35,32 +35,47 @@ def clone_or_fetch( print("Head was at", repo.head.target) print(f"Fetching from {repo_url}...") remote.fetch(callbacks=callbacks, prune=True) - reset_repo(repo, callbacks=callbacks) else: # Clone remote repository into work dir. print(f"Clone {repo_url} into {repo_path}...") pygit2.clone_repository(repo_url, repo_path, callbacks=callbacks) repo = pygit2.Repository(repo_path) + reset_repo(repo, callbacks=callbacks) return repo def reset_repo(repo: pygit2.Repository, callbacks: pygit2.RemoteCallbacks): print("Reset local content to remote content...") - # Reset all local branches to their remote - for branch_name in repo.branches.local: - remote_ref_name = f"{REMOTE_NAME}/{branch_name}" - if remote_ref_name not in repo.branches: - print(f"Delete local branch {branch_name}") - repo.branches.delete(branch_name) - else: - local_branch = repo.branches[branch_name] - remote_branch = repo.branches[remote_ref_name] - if local_branch.target != remote_branch.target: - # Reset local branch to remote target + # If the repo is freshly cloned, the remotes branches do not exist locally. Create them. + # If the repo was cloned from previous run, reset the local branches to the remote targets. + # Remote always wins. + + for branch_name in repo.branches.remote: + branch = repo.branches.remote[branch_name] + assert branch.remote_name == REMOTE_NAME + remote_target = branch.target + local_branch_name = branch_name.removeprefix(f"{REMOTE_NAME}/") + local_refname = f"refs/heads/{local_branch_name}" + if local_refname in repo.references: + local_ref = repo.lookup_reference(local_refname) + if local_ref.target != remote_target: print( - f"Resetting local branch {branch_name} to remote {remote_ref_name}" + f"Resetting local branch {local_branch_name} to remote {branch_name}" + ) + local_ref.set_target( + remote_target, + f"reset to {REMOTE_NAME}/{branch_name}", ) - local_branch.set_target(remote_branch.target) + else: + repo.create_reference(local_refname, remote_target) + + # Delete local branches that are not on remote + for branch_name in repo.branches.local: + remote_branch_name = f"{REMOTE_NAME}/{branch_name}" + if remote_branch_name not in repo.branches.remote: + ref = repo.lookup_reference(f"refs/heads/{branch_name}") + print(f"Delete local branch {branch_name}") + ref.delete() # Delete local tags that are not on remote origin = repo.remotes[REMOTE_NAME] diff --git a/cronjobs/tests/commands/test_git_export.py b/cronjobs/tests/commands/test_git_export.py index 8b7ff387..7055ea5f 100644 --- a/cronjobs/tests/commands/test_git_export.py +++ b/cronjobs/tests/commands/test_git_export.py @@ -223,7 +223,7 @@ def init_fake_repo(path): return repo -def create_branch_with_empty_commit(repo, branch_name): +def create_branch_with_empty_commit(repo, branch_name, set_as_repo_head=False): author = pygit2.Signature("Test User", "test@example.com") builder = repo.TreeBuilder() tree = builder.write() @@ -237,10 +237,10 @@ def create_branch_with_empty_commit(repo, branch_name): ) commit = repo[commit_id] - repo.branches.local.create(branch_name, commit) refname = f"refs/remotes/origin/{branch_name}" repo.references.create(refname, commit.id) - repo.set_head(f"refs/heads/{branch_name}") + if set_as_repo_head: + repo.set_head(branch_name) def simulate_pushed(repo, mock_ls_remotes): @@ -282,6 +282,7 @@ def test_remote_is_clone_if_dir_missing( mock_truncate_branch, mock_github_lfs, mock_git_push, + mock_ls_remotes, ): def _fake_clone(url, path, *args, **kwargs): return init_fake_repo(path) @@ -353,7 +354,7 @@ def test_repo_sync_does_nothing_if_up_to_date( mock_github_lfs, mock_git_push, ): - create_branch_with_empty_commit(repo, "v1/common") + create_branch_with_empty_commit(repo, "v1/common", set_as_repo_head=True) create_branch_with_empty_commit(repo, "v1/buckets/bid1") create_branch_with_empty_commit(repo, "v1/buckets/bid2") @@ -381,7 +382,7 @@ def test_repo_sync_can_be_forced_even_if_up_to_date( mock_github_lfs, mock_git_push, ): - create_branch_with_empty_commit(repo, "v1/common") + create_branch_with_empty_commit(repo, "v1/common", set_as_repo_head=True) create_branch_with_empty_commit(repo, "v1/buckets/bid1") create_branch_with_empty_commit(repo, "v1/buckets/bid2") @@ -408,13 +409,13 @@ def test_repo_sync_content_uses_previous_run_to_fetch_changes( mock_github_lfs, mock_git_push, ): - create_branch_with_empty_commit(repo, "v1/common") + create_branch_with_empty_commit(repo, "v1/common", set_as_repo_head=True) create_branch_with_empty_commit(repo, "v1/buckets/bid1") create_branch_with_empty_commit(repo, "v1/buckets/bid2") repo.create_tag( "v1/timestamps/common/1600000000000", - repo.lookup_reference("refs/heads/v1/common").target, + repo.head.target, pygit2.GIT_OBJECT_COMMIT, pygit2.Signature("Test User", "test@example.com"), "Test tag at 1600000000000", @@ -456,13 +457,13 @@ def test_repo_sync_content_ignores_previous_run_if_forced( mock_github_lfs, mock_git_push, ): - create_branch_with_empty_commit(repo, "v1/common") + create_branch_with_empty_commit(repo, "v1/common", set_as_repo_head=True) create_branch_with_empty_commit(repo, "v1/buckets/bid1") create_branch_with_empty_commit(repo, "v1/buckets/bid2") repo.create_tag( "v1/timestamps/common/1600000000000", - repo.lookup_reference("refs/heads/v1/common").target, + repo.head.target, pygit2.GIT_OBJECT_COMMIT, pygit2.Signature("Test User", "test@example.com"), "Test tag at 1600000000000", @@ -746,7 +747,7 @@ def test_repo_is_resetted_to_local_content_on_error( mock_git_push, mock_ls_remotes, ): - create_branch_with_empty_commit(repo, "v1/common") + create_branch_with_empty_commit(repo, "v1/common", set_as_repo_head=True) create_branch_with_empty_commit(repo, "v1/buckets/bid1") create_branch_with_empty_commit(repo, "v1/buckets/bid2") diff --git a/cronjobs/tests/commands/test_git_export_git_tools.py b/cronjobs/tests/commands/test_git_export_git_tools.py index 8f05d6c6..0584361c 100644 --- a/cronjobs/tests/commands/test_git_export_git_tools.py +++ b/cronjobs/tests/commands/test_git_export_git_tools.py @@ -1,11 +1,14 @@ import time +from unittest import mock import pygit2 import pytest +from commands import git_export from commands._git_export_git_tools import ( delete_old_tags, iter_tree, parse_lfs_pointer, + reset_repo, tree_upsert_blobs, truncate_branch, ) @@ -33,9 +36,17 @@ def tmp_repo(tmp_path): tree_oid, [], ) + repo.remotes.create("origin", git_export.GIT_REMOTE_URL) return repo +@pytest.fixture +def mock_ls_remotes(): + with mock.patch("pygit2.Remote.ls_remotes") as mock_ls: + mock_ls.return_value = [] + yield mock_ls + + def test_valid_lfs_pointer(): data = b"""version https://git-lfs.github.com/spec/v1 oid sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef @@ -71,6 +82,80 @@ def test_iter_tree_single_file(tmp_repo): ] +def test_reset_repo_creates_local_branches(tmp_repo, mock_ls_remotes): + repo = tmp_repo + commit = repo.revparse_single("main") + # remote branch can be created via reference only. + repo.create_reference( + "refs/remotes/origin/v1/buckets/main", + commit.id, + force=True, + ) + assert "v1/buckets/main" not in repo.branches.local + + reset_repo(repo, callbacks=None) + + local_ref = repo.lookup_reference("refs/heads/v1/buckets/main") + remote_ref = repo.lookup_reference("refs/remotes/origin/v1/buckets/main") + assert local_ref.target == remote_ref.target + + +def test_reset_repo_resets_local_branches_to_remote(tmp_repo, mock_ls_remotes): + repo = tmp_repo + commit = repo.revparse_single("main") + author = committer = pygit2.Signature("Test", "test@example.com") + repo.create_commit( + "refs/heads/v1/buckets/main", + author, + committer, + "diverging commit", + repo.TreeBuilder().write(), + [commit.id], + ) + # remote branch can be created via reference only. + repo.create_reference( + "refs/remotes/origin/v1/buckets/main", + commit.id, + force=True, + ) + # With this local commit, the branches have diverged. + local_ref = repo.lookup_reference("refs/heads/v1/buckets/main") + remote_ref = repo.lookup_reference("refs/remotes/origin/v1/buckets/main") + assert local_ref.target != remote_ref.target + + reset_repo(repo, callbacks=None) + + # Now they match. + local_ref = repo.lookup_reference("refs/heads/v1/buckets/main") + remote_ref = repo.lookup_reference("refs/remotes/origin/v1/buckets/main") + assert local_ref.target == remote_ref.target + + +def test_reset_repo_deletes_extra_local_branches_and_tags(tmp_repo, mock_ls_remotes): + repo = tmp_repo + some_target = repo.references["refs/heads/main"].target + repo.create_reference("refs/remotes/origin/v1/buckets/main", some_target) + repo.create_reference("refs/heads/v1/buckets/main", some_target) + assert "v1/buckets/main" in repo.branches.local + # Create another ref that wouldn't be repo's head (to allow delete) + author = committer = pygit2.Signature("Test", "test@example.com") + commit_id = repo.create_commit( + "refs/heads/v1/buckets/main", + author, + committer, + "initial commit", + repo.TreeBuilder().write(), + [some_target], + ) + # Create an extra branch. + repo.create_reference("refs/heads/v1/buckets/unknown", commit_id) + + reset_repo(repo, callbacks=None) + + assert "v1/buckets/main" in repo.branches.local + assert "v1/buckets/unknown" not in repo.branches.local + + def test_delete_old_tags(tmp_repo): repo = tmp_repo now_ts = int(time.time())