Run unsupervised clustering with asynchronous framework.

docker-compose.yml

@@ -11,6 +11,7 @@ services:
     depends_on:
       - database
       - object_storage
+      - docker_orchestrator
     environment:
       DB_URL_GROUP: postgresql://postgres:password@database:5432/group_alpha_group_db
       DB_URL_APP: postgresql://postgres:password@database:5432/app_a_app_db
@@ -24,6 +25,7 @@ services:
       JOB_INPUTS_BUCKET_NAME: inputs
       JOB_OUTPUTS_BUCKET_NAME: outputs
       DATA_OBJECTS_BUCKET_NAME: objects
+      DOCKER_ORCHESTRATOR_URL: http://docker_orchestrator:8080
       APP_USER: ${USER}  # Delete this (and modify get_current_username()) in the future when we implement running the container as non-root.
       APP_PLATFORM: local
       APP_NAME: mawa
@@ -67,9 +69,11 @@ services:
     build:
       context: .
       dockerfile: docker_orchestrator/Dockerfile
+    ports:
+      - "8080:8080"
     environment:
       # Worker image to launch (reuses frontend image for parity)
-      WORKER_IMAGE: frontend  ## ENSURE THIS MATCHES FRONTEND IMAGE NAME
+      WORKER_IMAGE: frontend:${IMAGE_TAG}  ## ENSURE THIS MATCHES FRONTEND IMAGE NAME
       WORKER_CPUS: "8"  # Note some machines may not have 8 CPUs available so we need to change this; ensure this is changed elsewhere in code!
       WORKER_MEM: "8g"
       # Pass-through envs required by the worker

source/basic_phenotyper_lib.py

@@ -684,7 +684,8 @@ def perform_density_calc(spatial_umap, calc_areas, cpu_pool_size = 1, area_thres
 
     return spatial_umap
 
-def perform_spatialUMAP(spatial_umap, bc, umap_subset_per_fit, umap_subset_toggle, umap_subset_per):
+#def perform_spatialUMAP(spatial_umap, bc, umap_subset_per_fit, umap_subset_toggle, umap_subset_per):
+def perform_spatialUMAP(spatial_umap, umap_subset_per_fit, umap_subset_toggle, umap_subset_per):
     '''
     Perform the spatial UMAP analysis
 
@@ -708,12 +709,12 @@ def perform_spatialUMAP(spatial_umap, bc, umap_subset_per_fit, umap_subset_toggl
     # fit umap on training cells
     print('Fitting Model')
     spatial_umap.umap_fit = umap.UMAP().fit(spatial_umap.density[spatial_umap.cells['umap_train'].values].reshape((spatial_umap.cells['umap_train'].sum(), -1)))
-    bc.printElapsedTime(f'      Fitting {np.sum(spatial_umap.cells["umap_train"] == 1)} points to a model', split = True)
+    #bc.printElapsedTime(f'      Fitting {np.sum(spatial_umap.cells["umap_train"] == 1)} points to a model', split = True)
 
     # Transform test cells based on fitted model
     print('Transforming Data')
     spatial_umap.umap_test = spatial_umap.umap_fit.transform(spatial_umap.density[spatial_umap.cells['umap_test'].values].reshape((spatial_umap.cells['umap_test'].sum(), -1)))
-    bc.printElapsedTime(f'      Transforming {np.sum(spatial_umap.cells["umap_test"] == 1)} points with the model', split = True)
+    #bc.printElapsedTime(f'      Transforming {np.sum(spatial_umap.cells["umap_test"] == 1)} points with the model', split = True)
 
     spatial_umap.umap_completed = True
 

source/framework/utils.py

@@ -26,13 +26,9 @@ def session_dir():
     if ST_KEY_PREFIX_STARTUP + "app_session_id" not in st.session_state:
         st.error("Session ID not found in session state; cannot return the session directory.")
         return None
-
     app_session_id = st.session_state[ST_KEY_PREFIX_STARTUP + "app_session_id"]
-
     session_dir = f"/tmp/{_app_title_simple()}/app_session_data/{app_session_id}"
-
     os.makedirs(session_dir, exist_ok=True)
-
     return session_dir
 
 

source/neighborhood_profiles.py

@@ -18,8 +18,7 @@
 from scipy import ndimage as ndi
 
 import basic_phenotyper_lib as bpl  # Useful functions for cell phenotyping
-import nidap_dashboard_lib as ndl   # Useful functions for dashboards connected to NIDAP
-from benchmark_collector import benchmark_collector # Benchmark Collector Class
+#import nidap_dashboard_lib as ndl   # Useful functions for dashboards connected to NIDAP
 from SpatialUMAP import SpatialUMAP
 import PlottingTools as umPT
 import utils
@@ -29,16 +28,12 @@ class NeighborhoodProfiles:
     Organization of the methods and attributes that are required to run
     the neighborhood profiles analysis
     '''
-    def __init__(self, bc = None):
+    def __init__(self):
 
 
         self.clust_minmax = [0, 40]
         self.reset_neigh_profile_settings()
 
-        if bc is None:
-            bc = benchmark_collector()
-        self.bc = bc
-
         # Spectrogram Plotting Settings
         self.n_bins = 100
         self.n_pad  = 0
@@ -183,7 +178,6 @@ def perform_density_calc(self, calc_areas, cpu_pool_size = 1, area_threshold = 0
 
         Args:
             spatial_umap (SpatialUMAP): SpatialUMAP object
-            bc (benchmark_collector): Benchmark Collector object
             cpu_pool_size (int): Number of CPUs to use for parallel processing
 
         Returns:
@@ -199,15 +193,13 @@ def perform_density_calc(self, calc_areas, cpu_pool_size = 1, area_threshold = 0
 
         # get the counts per cell and save to pickle file
         print('Starting Cell Counts process')
-        self.bc.startTimer()
         self.spatial_umap.get_counts_And(cpu_pool_size=cpu_pool_size)
-        self.bc.printElapsedTime(f'Calculating Counts for {len(self.spatial_umap.cells)} cells')
+        print(f'Completed calculating counts for {len(self.spatial_umap.cells)} cells')
 
         # get the areas of cells and save to pickle file
         print(f'\nStarting Cell Areas process with area threshold of {area_threshold}')
-        self.bc.startTimer()
         self.spatial_umap.get_areas(calc_areas, area_threshold, pool_size=cpu_pool_size)
-        self.bc.printElapsedTime(f'Calculating Areas for {len(self.spatial_umap.cells)} cells')
+        print(f'Completed calculating areas for {len(self.spatial_umap.cells)} cells')
 
         # calculate density based on counts of cells / area of each arc examine
         self.spatial_umap.calc_densities(area_threshold)
@@ -222,7 +214,6 @@ def perform_spatial_umap(self, session_state, umap_subset_per_fit, umap_subset_t
 
         Args:
             spatial_umap (spatial_umap): spatial_umap object
-            bc (benchmark_collector): Benchmark Collector object
             UMAPStyle (str): Style of UMAP to use
 
         Returns:
@@ -238,16 +229,14 @@ def perform_spatial_umap(self, session_state, umap_subset_per_fit, umap_subset_t
         self.spatial_umap.set_train_test(n_fit=n_fit, n_tra = n_tra, groupby_label = 'TMA_core_id', seed=54321, umap_subset_toggle = umap_subset_toggle)
 
         # fit umap on training cells
-        self.bc.startTimer()
         print('Fitting Model')
         self.spatial_umap.umap_fit = umap.UMAP().fit(self.spatial_umap.density[self.spatial_umap.cells['umap_train'].values].reshape((self.spatial_umap.cells['umap_train'].sum(), -1)))
-        self.bc.printElapsedTime(f'      Fitting {np.sum(self.spatial_umap.cells["umap_train"] == 1)} points to a model')
+        print(f'      Completed fitting {np.sum(self.spatial_umap.cells["umap_train"] == 1)} points to a model')
 
         # Transform test cells based on fitted model
-        self.bc.startTimer()
         print('Transforming Data')
         self.spatial_umap.umap_test = self.spatial_umap.umap_fit.transform(self.spatial_umap.density[self.spatial_umap.cells['umap_test'].values].reshape((self.spatial_umap.cells['umap_test'].sum(), -1)))
-        self.bc.printElapsedTime(f'      Transforming {np.sum(self.spatial_umap.cells["umap_test"] == 1)} points with the model')
+        print(f'      Completed transforming {np.sum(self.spatial_umap.cells["umap_test"] == 1)} points with the model')
 
         self.spatial_umap.umap_completed = True
 
@@ -434,6 +423,7 @@ def filter_and_plot(self, session_state):
         '''
         if self.umap_completed:
             self.df_umap_filt = self.df_umap.loc[self.df_umap['Slide ID'] == session_state['selSlide ID'], :]
+            import nidap_dashboard_lib as ndl
             session_state = ndl.setFigureObjs_UMAP(session_state)
 
         return session_state