Merge pull request galaxyproject#16640 from mvdbeek/lazy_tool_data_ta…

…bles

[23.0] Lazy load tool data tables in celery worker

lib/galaxy/app.py

@@ -576,8 +576,6 @@ def __init__(self, configure_logging=True, use_converters=True, use_display_appl
 
         self._configure_tool_shed_registry()
         self._register_singleton(tool_shed_registry.Registry, self.tool_shed_registry)
-        # Tool Data Tables
-        self._configure_tool_data_tables(from_shed_config=False)
 
     def _configure_tool_shed_registry(self) -> None:
         # Set up the tool sheds registry
@@ -630,6 +628,8 @@ def __init__(self, **kwargs) -> None:
         )
         self.api_keys_manager = self._register_singleton(ApiKeyManager)
 
+        # Tool Data Tables
+        self._configure_tool_data_tables(from_shed_config=False)
         # Load dbkey / genome build manager
         self._configure_genome_builds(data_table_name="__dbkeys__", load_old_style=True)
 

lib/galaxy/celery/tasks.py

@@ -34,6 +34,7 @@
 from galaxy.managers.tool_data import ToolDataImportManager
 from galaxy.metadata.set_metadata import set_metadata_portable
 from galaxy.model.scoped_session import galaxy_scoped_session
+from galaxy.queue_worker import GalaxyQueueWorker
 from galaxy.schema.tasks import (
     ComputeDatasetHashTaskRequest,
     GenerateHistoryContentDownload,
@@ -58,6 +59,11 @@
 log = get_logger(__name__)
 
 
+@lru_cache()
+def setup_data_table_manager(app):
+    app._configure_tool_data_tables(from_shed_config=False)
+
+
 @lru_cache()
 def cached_create_tool_from_representation(app, raw_tool_source):
     return create_tool_from_representation(
@@ -369,6 +375,7 @@ def compute_dataset_hash(
 
 @galaxy_task(action="import a data bundle")
 def import_data_bundle(
+    app: MinimalManagerApp,
     hda_manager: HDAManager,
     ldda_manager: LDDAManager,
     tool_data_import_manager: ToolDataImportManager,
@@ -378,6 +385,7 @@ def import_data_bundle(
     id: Optional[int] = None,
     tool_data_file_path: Optional[str] = None,
 ):
+    setup_data_table_manager(app)
     if src == "uri":
         assert uri
         tool_data_import_manager.import_data_bundle_by_uri(config, uri, tool_data_file_path=tool_data_file_path)
@@ -389,6 +397,8 @@ def import_data_bundle(
         else:
             dataset = ldda_manager.by_id(id)
         tool_data_import_manager.import_data_bundle_by_dataset(config, dataset, tool_data_file_path=tool_data_file_path)
+    queue_worker = GalaxyQueueWorker(app)
+    queue_worker.send_control_task("reload_tool_data_tables")
 
 
 @galaxy_task(action="pruning history audit table")

test/integration/test_tool_data_delete.py

@@ -45,6 +45,8 @@ def _testbase_fields(self):
         return show_response.json()["fields"]
 
     def _testbeta_field_count(self) -> int:
+        # We need to wait for the reload message to reach the control message consumer
+        time.sleep(1)
         return len(self._testbase_fields())