From ea06cf03c04dc4786af158d02a75c92d79df9f1d Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 25 Sep 2025 18:54:20 +0200 Subject: [PATCH 1/4] Add restat sync mechanism for both v2 and v1... compatible with upcoming py-client --- Mergin/projects_manager.py | 184 +++++++++++++++++++------------------ Mergin/sync_dialog.py | 12 +-- Mergin/utils.py | 6 +- 3 files changed, 103 insertions(+), 99 deletions(-) diff --git a/Mergin/projects_manager.py b/Mergin/projects_manager.py index 7ab057ea..65222ee6 100644 --- a/Mergin/projects_manager.py +++ b/Mergin/projects_manager.py @@ -34,6 +34,9 @@ UnsavedChangesStrategy, write_project_variables, bytes_to_human_size, + get_push_changes_batch, + SYNC_ATTEMPTS, + SYNC_ATTEMPT_WAIT, ) from .utils_auth import get_stored_mergin_server_url @@ -366,102 +369,105 @@ def sync_project(self, project_dir, project_name=None): ) return - dlg = SyncDialog() - dlg.pull_start(self.mc, project_dir, project_name) - - dlg.exec() # blocks until success, failure or cancellation - - if dlg.exception: - # pull failed for some reason - if isinstance(dlg.exception, LoginError): - login_error_message(dlg.exception) - elif isinstance(dlg.exception, ClientError): - QMessageBox.critical(None, "Project sync", "Client error: " + str(dlg.exception)) - elif isinstance(dlg.exception, AuthTokenExpiredError): - self.plugin.auth_token_expired() + has_push_changes = True + error_retries_attempts = 0 + while has_push_changes: + dlg = SyncDialog() + pull_timeout = 250 + if error_retries_attempts > 0: + pull_timeout = SYNC_ATTEMPT_WAIT * 1000 + dlg.labelStatus.setText("A sync conflict was detected. We are now retrying the synchronization to ensure your project is up to date.") else: - unhandled_exception_message( - dlg.exception_details(), - "Project sync", - f"Something went wrong while synchronising your project {project_name}.", - self.mc, - ) - return - - # after pull project might be in the unfinished pull state. So we - # have to check and if this is the case, try to close project and - # finish pull. As in the result we will have conflicted copies created - # we stop and ask user to examine them. - if self.mc.has_unfinished_pull(project_dir): - self.close_project_and_fix_pull(project_dir) - return - - if dlg.pull_conflicts: - self.report_conflicts(dlg.pull_conflicts) - return - - if not dlg.is_complete: - # we were cancelled - return + dlg.labelStatus.setText("Starting project synchronisation...") + dlg.pull_start(self.mc, project_dir, project_name, pull_timeout) + dlg.exec() # blocks until success, failure or cancellation + + if dlg.exception: + # pull failed for some reason + if isinstance(dlg.exception, LoginError): + login_error_message(dlg.exception) + elif isinstance(dlg.exception, ClientError): + QMessageBox.critical(None, "Project sync", "Client error: " + str(dlg.exception)) + elif isinstance(dlg.exception, AuthTokenExpiredError): + self.plugin.auth_token_expired() + else: + unhandled_exception_message( + dlg.exception_details(), + "Project sync", + f"Something went wrong while synchronising your project {project_name}.", + self.mc, + ) + return - # pull finished, start push - if any(push_changes.values()) and not self.mc.has_writing_permissions(project_name): - QMessageBox.information( - None, - "Project sync", - "You have no writing rights to this project", - QMessageBox.StandardButton.Close, - ) - return + # after pull project might be in the unfinished pull state. So we + # have to check and if this is the case, try to close project and + # finish pull. As in the result we will have conflicted copies created + # we stop and ask user to examine them. + if self.mc.has_unfinished_pull(project_dir): + self.close_project_and_fix_pull(project_dir) + return - dlg = SyncDialog() - dlg.push_start(self.mc, project_dir, project_name) - dlg.exec() # blocks until success, failure or cancellation + if dlg.pull_conflicts: + self.report_conflicts(dlg.pull_conflicts) + return - qgis_proj_filename = os.path.normpath(QgsProject.instance().fileName()) - qgis_proj_basename = os.path.basename(qgis_proj_filename) - qgis_proj_changed = False - for updated in pull_changes["updated"]: - if updated["path"] == qgis_proj_basename: - qgis_proj_changed = True - break - if qgis_proj_filename in find_qgis_files(project_dir) and qgis_proj_changed: - self.open_project(project_dir) + if not dlg.is_complete: + # we were cancelled + return - if dlg.exception: - # push failed for some reason - if isinstance(dlg.exception, LoginError): - login_error_message(dlg.exception) - elif isinstance(dlg.exception, ClientError): - if dlg.exception.http_error == 400 and "Another process" in dlg.exception.detail: - # To note we check for a string since error in flask doesn't return server error code - msg = "Somebody else is syncing, please try again later" - elif dlg.exception.server_code == ErrorCode.StorageLimitHit.value: - msg = f"{dlg.exception.detail}\nCurrent limit: {bytes_to_human_size(dlg.exception.server_response['storage_limit'])}" + dlg = SyncDialog() + dlg.labelStatus.setText("Preparing project upload...") + dlg.push_start(self.mc, project_dir, project_name) + dlg.exec() # blocks until success, failure or cancellation + + qgis_proj_filename = os.path.normpath(QgsProject.instance().fileName()) + qgis_proj_basename = os.path.basename(qgis_proj_filename) + qgis_proj_changed = False + for updated in pull_changes["updated"]: + if updated["path"] == qgis_proj_basename: + qgis_proj_changed = True + break + if qgis_proj_filename in find_qgis_files(project_dir) and qgis_proj_changed: + self.open_project(project_dir) + + if dlg.exception: + # push failed for some reason + if isinstance(dlg.exception, LoginError): + login_error_message(dlg.exception) + elif isinstance(dlg.exception, ClientError): + if error_retries_attempts < SYNC_ATTEMPTS - 1 and dlg.exception.is_retryable_sync(): + error_retries_attempts += 1 + continue # try again + if dlg.exception.http_error == 400 and "Another process" in dlg.exception.detail or dlg.exception.server_code == ErrorCode.AnotherUploadRunning.value: + # To note we check for a string since error in flask doesn't return server error code + msg = "Somebody else is syncing, please try again later" + elif dlg.exception.server_code == ErrorCode.StorageLimitHit.value: + msg = f"{dlg.exception.detail}\nCurrent limit: {bytes_to_human_size(dlg.exception.server_response['storage_limit'])}" + else: + msg = str(dlg.exception) + QMessageBox.critical(None, "Project sync", "Client error: \n" + msg) + elif isinstance(dlg.exception, AuthTokenExpiredError): + self.plugin.auth_token_expired() else: - msg = str(dlg.exception) - QMessageBox.critical(None, "Project sync", "Client error: \n" + msg) - elif isinstance(dlg.exception, AuthTokenExpiredError): - self.plugin.auth_token_expired() + unhandled_exception_message( + dlg.exception_details(), + "Project sync", + f"Something went wrong while synchronising your project {project_name}.", + self.mc, + ) + return + _, has_push_changes = get_push_changes_batch(self.mc, project_dir) + error_retries_attempts = 0 + if dlg.is_complete and not has_push_changes: + # TODO: report success only when we have actually done anything + msg = "Mergin Maps project {} synchronised successfully".format(project_name) + QMessageBox.information(None, "Project sync", msg, QMessageBox.StandardButton.Close) + # clear canvas cache so any changes become immediately visible to users + self.iface.mapCanvas().clearCache() + self.iface.mapCanvas().refresh() else: - unhandled_exception_message( - dlg.exception_details(), - "Project sync", - f"Something went wrong while synchronising your project {project_name}.", - self.mc, - ) - return - - if dlg.is_complete: - # TODO: report success only when we have actually done anything - msg = "Mergin Maps project {} synchronised successfully".format(project_name) - QMessageBox.information(None, "Project sync", msg, QMessageBox.StandardButton.Close) - # clear canvas cache so any changes become immediately visible to users - self.iface.mapCanvas().clearCache() - self.iface.mapCanvas().refresh() - else: - # we were cancelled - but no need to show a message box about that...? - pass + # we were cancelled - but no need to show a message box about that...? + pass def submit_logs(self, project_dir): logs_path = os.path.join(project_dir, ".mergin", "client-log.txt") diff --git a/Mergin/sync_dialog.py b/Mergin/sync_dialog.py index d016949a..a86dfca7 100644 --- a/Mergin/sync_dialog.py +++ b/Mergin/sync_dialog.py @@ -161,17 +161,15 @@ def download_cancel(self): else: self.cancel_sync_operation("Cancelling download...", download_project_cancel) - def push_start(self, mergin_client, target_dir, project_name): + def push_start(self, mergin_client, target_dir, project_name, timeout=250): self.operation = self.PUSH self.mergin_client = mergin_client self.target_dir = target_dir self.project_name = project_name - self.labelStatus.setText("Querying project...") - # we would like to get the dialog displayed at least for a bit # with low timeout (or zero) it may not even appear before it is closed - QTimer.singleShot(250, self.push_start_internal) + QTimer.singleShot(timeout, self.push_start_internal) def push_start_internal(self): with OverrideCursor(Qt.CursorShape.WaitCursor): @@ -227,17 +225,15 @@ def push_cancel(self): else: self.cancel_sync_operation("Cancelling sync...", push_project_cancel) - def pull_start(self, mergin_client, target_dir, project_name): + def pull_start(self, mergin_client, target_dir, project_name, timeout=250): self.operation = self.PULL self.mergin_client = mergin_client self.target_dir = target_dir self.project_name = project_name - self.labelStatus.setText("Querying project...") - # we would like to get the dialog displayed at least for a bit # with low timeout (or zero) it may not even appear before it is closed - QTimer.singleShot(250, self.pull_start_internal) + QTimer.singleShot(timeout, self.pull_start_internal) def pull_start_internal(self): with OverrideCursor(Qt.CursorShape.WaitCursor): diff --git a/Mergin/utils.py b/Mergin/utils.py index 51395f2a..2024cc9d 100644 --- a/Mergin/utils.py +++ b/Mergin/utils.py @@ -72,7 +72,7 @@ from .mergin.merginproject import MerginProject try: - from .mergin.common import ClientError, ErrorCode, LoginError, InvalidProject + from .mergin.common import ClientError, ErrorCode, LoginError, InvalidProject, SYNC_ATTEMPTS, SYNC_ATTEMPT_WAIT from .mergin.client import MerginClient, ServerType from .mergin.client_pull import ( download_project_async, @@ -91,6 +91,7 @@ push_project_is_running, push_project_finalize, push_project_cancel, + get_push_changes_batch ) from .mergin.report import create_report from .mergin.deps import pygeodiff @@ -101,7 +102,7 @@ path = os.path.join(this_dir, "mergin_client.whl") sys.path.append(path) from mergin.client import MerginClient, ServerType - from mergin.common import ClientError, InvalidProject, LoginError + from mergin.common import ClientError, InvalidProject, LoginError, PUSH_ATTEMPTS, PUSH_ATTEMPT_WAIT from mergin.client_pull import ( download_project_async, @@ -120,6 +121,7 @@ push_project_is_running, push_project_finalize, push_project_cancel, + get_push_changes_batch ) from .mergin.report import create_report from .mergin.deps import pygeodiff From ce8b30c89e43ba8acf8feb694459297581748e8b Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 26 Sep 2025 11:51:27 +0200 Subject: [PATCH 2/4] fix texts --- Mergin/projects_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Mergin/projects_manager.py b/Mergin/projects_manager.py index 65222ee6..183fd571 100644 --- a/Mergin/projects_manager.py +++ b/Mergin/projects_manager.py @@ -170,6 +170,7 @@ def create_project(self, project_name, project_dir, is_public, namespace): return True dlg = SyncDialog() + dlg.labelStatus.setText("Starting project upload...") dlg.push_start(self.mc, project_dir, full_project_name) dlg.exec() # blocks until success, failure or cancellation @@ -376,9 +377,7 @@ def sync_project(self, project_dir, project_name=None): pull_timeout = 250 if error_retries_attempts > 0: pull_timeout = SYNC_ATTEMPT_WAIT * 1000 - dlg.labelStatus.setText("A sync conflict was detected. We are now retrying the synchronization to ensure your project is up to date.") - else: - dlg.labelStatus.setText("Starting project synchronisation...") + dlg.labelStatus.setText("Starting project synchronisation...") dlg.pull_start(self.mc, project_dir, project_name, pull_timeout) dlg.exec() # blocks until success, failure or cancellation From c87a7dd99b944678038f4dedf8c8da0048fe71b1 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 30 Oct 2025 17:49:33 +0100 Subject: [PATCH 3/4] Black --- Mergin/projects_manager.py | 6 +++++- Mergin/utils.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Mergin/projects_manager.py b/Mergin/projects_manager.py index 183fd571..e42fc618 100644 --- a/Mergin/projects_manager.py +++ b/Mergin/projects_manager.py @@ -437,7 +437,11 @@ def sync_project(self, project_dir, project_name=None): if error_retries_attempts < SYNC_ATTEMPTS - 1 and dlg.exception.is_retryable_sync(): error_retries_attempts += 1 continue # try again - if dlg.exception.http_error == 400 and "Another process" in dlg.exception.detail or dlg.exception.server_code == ErrorCode.AnotherUploadRunning.value: + if ( + dlg.exception.http_error == 400 + and "Another process" in dlg.exception.detail + or dlg.exception.server_code == ErrorCode.AnotherUploadRunning.value + ): # To note we check for a string since error in flask doesn't return server error code msg = "Somebody else is syncing, please try again later" elif dlg.exception.server_code == ErrorCode.StorageLimitHit.value: diff --git a/Mergin/utils.py b/Mergin/utils.py index b3e21fed..07777d0a 100644 --- a/Mergin/utils.py +++ b/Mergin/utils.py @@ -95,7 +95,7 @@ push_project_is_running, push_project_finalize, push_project_cancel, - get_push_changes_batch + get_push_changes_batch, ) from .mergin.report import create_report from .mergin.deps import pygeodiff @@ -125,7 +125,7 @@ push_project_is_running, push_project_finalize, push_project_cancel, - get_push_changes_batch + get_push_changes_batch, ) from .mergin.report import create_report from .mergin.deps import pygeodiff From 5a7ac12537245bea78b5467fa155986ee3d03e8c Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Tue, 11 Nov 2025 17:33:09 +0100 Subject: [PATCH 4/4] Cancel job if cancelled from button --- Mergin/projects_manager.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Mergin/projects_manager.py b/Mergin/projects_manager.py index e42fc618..b9f8a6fe 100644 --- a/Mergin/projects_manager.py +++ b/Mergin/projects_manager.py @@ -459,9 +459,13 @@ def sync_project(self, project_dir, project_name=None): self.mc, ) return + + if not dlg.is_complete: + # we were cancelled + return _, has_push_changes = get_push_changes_batch(self.mc, project_dir) error_retries_attempts = 0 - if dlg.is_complete and not has_push_changes: + if not has_push_changes: # TODO: report success only when we have actually done anything msg = "Mergin Maps project {} synchronised successfully".format(project_name) QMessageBox.information(None, "Project sync", msg, QMessageBox.StandardButton.Close)