From 67e56572d7c819be99a5979ded5e0bfcc53e4ac3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 3 Sep 2023 12:18:05 +0200 Subject: [PATCH 1/3] Revert "Load data tables in Celery worker" This reverts commit 362bf76a0c92b4b5ab418287147ce1178bf272c4. --- lib/galaxy/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 5cb61d592f24..b7c6a6266b2f 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -576,8 +576,6 @@ def __init__(self, configure_logging=True, use_converters=True, use_display_appl self._configure_tool_shed_registry() self._register_singleton(tool_shed_registry.Registry, self.tool_shed_registry) - # Tool Data Tables - self._configure_tool_data_tables(from_shed_config=False) def _configure_tool_shed_registry(self) -> None: # Set up the tool sheds registry @@ -630,6 +628,8 @@ def __init__(self, **kwargs) -> None: ) self.api_keys_manager = self._register_singleton(ApiKeyManager) + # Tool Data Tables + self._configure_tool_data_tables(from_shed_config=False) # Load dbkey / genome build manager self._configure_genome_builds(data_table_name="__dbkeys__", load_old_style=True) From 9a5954690d764068a783f03c8fab35766061f3d8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 3 Sep 2023 12:18:26 +0200 Subject: [PATCH 2/3] Setup tool data tables lazily tool data table loading is very slow on usegalaxy.eu, so this limits the throughput significantly on usegalaxy.eu --- lib/galaxy/celery/tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index 976e0ebab442..b3b7dc4d2bf4 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -58,6 +58,11 @@ log = get_logger(__name__) +@lru_cache() +def setup_data_table_manager(app): + app._configure_tool_data_tables(from_shed_config=False) + + @lru_cache() def cached_create_tool_from_representation(app, raw_tool_source): return create_tool_from_representation( @@ -369,6 +374,7 @@ def compute_dataset_hash( @galaxy_task(action="import a data bundle") def import_data_bundle( + app: MinimalManagerApp, hda_manager: HDAManager, ldda_manager: LDDAManager, tool_data_import_manager: ToolDataImportManager, @@ -378,6 +384,7 @@ def import_data_bundle( id: Optional[int] = None, tool_data_file_path: Optional[str] = None, ): + setup_data_table_manager(app) if src == "uri": assert uri tool_data_import_manager.import_data_bundle_by_uri(config, uri, tool_data_file_path=tool_data_file_path) From 5684562f37b3d549a5e66ae81ff0ee08250657de Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 3 Sep 2023 18:24:28 +0200 Subject: [PATCH 3/3] Send reload message after data bundle import is complete --- lib/galaxy/celery/tasks.py | 3 +++ test/integration/test_tool_data_delete.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/lib/galaxy/celery/tasks.py b/lib/galaxy/celery/tasks.py index b3b7dc4d2bf4..842ab9fc87cc 100644 --- a/lib/galaxy/celery/tasks.py +++ b/lib/galaxy/celery/tasks.py @@ -34,6 +34,7 @@ from galaxy.managers.tool_data import ToolDataImportManager from galaxy.metadata.set_metadata import set_metadata_portable from galaxy.model.scoped_session import galaxy_scoped_session +from galaxy.queue_worker import GalaxyQueueWorker from galaxy.schema.tasks import ( ComputeDatasetHashTaskRequest, GenerateHistoryContentDownload, @@ -396,6 +397,8 @@ def import_data_bundle( else: dataset = ldda_manager.by_id(id) tool_data_import_manager.import_data_bundle_by_dataset(config, dataset, tool_data_file_path=tool_data_file_path) + queue_worker = GalaxyQueueWorker(app) + queue_worker.send_control_task("reload_tool_data_tables") @galaxy_task(action="pruning history audit table") diff --git a/test/integration/test_tool_data_delete.py b/test/integration/test_tool_data_delete.py index 32f6430fc41a..6104aef2bfdf 100644 --- a/test/integration/test_tool_data_delete.py +++ b/test/integration/test_tool_data_delete.py @@ -45,6 +45,8 @@ def _testbase_fields(self): return show_response.json()["fields"] def _testbeta_field_count(self) -> int: + # We need to wait for the reload message to reach the control message consumer + time.sleep(1) return len(self._testbase_fields())