From b0fc41680c119110c695272de5cc5abb06ac53a6 Mon Sep 17 00:00:00 2001 From: Heinrich Date: Mon, 15 Jul 2024 19:05:01 +0200 Subject: [PATCH 1/3] implemented spinna --- picasso_workflow/analyse.py | 114 +++++++++++++++++++++++++++++++++ picasso_workflow/confluence.py | 41 ++++++++++++ picasso_workflow/util.py | 5 ++ 3 files changed, 160 insertions(+) diff --git a/picasso_workflow/analyse.py b/picasso_workflow/analyse.py index 189f61d..85d327e 100644 --- a/picasso_workflow/analyse.py +++ b/picasso_workflow/analyse.py @@ -1866,6 +1866,120 @@ def _save_datasets_agg(self, folder): allfps.append(filepath) return allfps + @module_decorator + def spinna(self, parameters, results): + """Direct implementation of spinna batch analysis. + The current locs file(s) are saved into the results folder, and + a template csv file is created. This csv needs to be filled out by the + user in a manual step before the spinna analysis is carried out. + + Args: + i : int + the index of the module + parameters: dict + with required keys: + labeling_efficiency : dict of float, range 0-100 + labeling efficiency percentage, for all targets + labeling_uncertainty : float or dict of floats + labeling uncertainty [nm]; good value is e.g. 5 + assumed the same value for all targets + N_simulate : int + number of target molecules to simulated; + good value is e.g. 50000 + # density : dict of float + # density to simulate; + # area density if 2D; volume density if 3D + # used proposed value in spinna_config.csv and can be + # altered manually after the first run of this module + structures : str or list of dict + if str: filepath to a yaml file with the structures. + if list of dict: + SPINNA structures. Each structure dict has + "Molecular targets": list of str, + "Structure title": str, + "TARGET_x": list of float, + "TARGET_y": list of float, + "TARGET_z": list of float, + where TARGET is one each of the target names in + "Molecular targets" + fp_mask_dict : str + the filepath to the mask_dict file + width, height, depth : float + the width, height, and depth of the simulation space + depth is optional + random_rot_mode : '2D', or '3D' + Mode of molecule rotation in simulation + sim_repeats : int + number of simulation repeats + fit_NND_bin : float + bin size of fits + fit_NND_maxdist : float + max of histogram + n_nearest_neighbors : int + number of nearest neighbors to evaluate + """ + if isinstance(parameters["structures"], str): + with open(parameters["structures"], "r") as f: + structures = yaml.read_all(f) + + if isinstance(parameters["labeling_uncertainty"], (int, float)): + labeling_uncertainty = { + tgt: parameters["labeling_uncertainty"] + for tgt in self.channel_tags + } + else: + labeling_uncertainty = parameters["labeling_uncertainty"] + + if parameters.get("fp_mask_dict"): + with open(parameters["fp_mask_dict"], "rb") as f: + mask_dict = pickle.load(f) + else: + mask_dict = None + + # locs, but as np.ndarray + pixelsize = self.analysis_config["camera_info"].get("pixelsize") + exp_data = {} + for i, target in enumerate(self.channel_tags): + locs = self.channel_locs[i] + if hasattr(locs, "z"): + exp_data[target] = np.stack( + (locs.x * pixelsize, locs.y * pixelsize, locs.z) + ).T + # dim = 3 + else: + exp_data[target] = np.stack( + (locs.x * pixelsize, locs.y * pixelsize) + ).T + + spinna_pars = { + "structures": structures, + "label_unc": labeling_uncertainty, + "le": parameters["labeling_efficiency"], + "mask_dict": mask_dict, + "width": parameters["width"], + "height": parameters["height"], + "depth": parameters.get("depth"), + "random_rot_mode": parameters["random_rot_mode"], + "exp_data": exp_data, + "sim_repeats": parameters["sim_repeats"], + "fit_NND_bin": parameters["fit_NND_bin"], + "fit_NND_maxdist": parameters["fit_NND_maxdist"], + "N_structures": parameters["N_simulate"], + "save_filename": os.path.join(results["folder"], "spinna-run"), + "asynch": True, + "targets": self.channel_tags, + "apply_mask": mask_dict is not None, + "nn_plotted": parameters["n_nearest_neighbors"], + "result_dir": results["folder"], + } + + result_dir, fp_fig = picasso_outpost.spinna_temp(spinna_pars) + plt.close("all") + results["fp_fig"] = fp_fig + results["result_dir"] = result_dir + + return parameters, results + @module_decorator def spinna_manual(self, i, parameters, results): """Direct implementation of spinna batch analysis. diff --git a/picasso_workflow/confluence.py b/picasso_workflow/confluence.py index 0fe52f7..0e495d3 100644 --- a/picasso_workflow/confluence.py +++ b/picasso_workflow/confluence.py @@ -762,6 +762,47 @@ def spinna_manual(self, i, parameters, results): os.path.split(fp)[1], ) + def spinna(self, i, parameters, results): + """ """ + logger.debug("Reporting spinna_manual.") + text = f""" + +

SPINNA-Manual

+
  • file present: {results.get('success')}
    • +
  • Start Time: {results['start time']}
    • +
  • Duration: {results["duration"] // 60:.0f} min + {(results["duration"] % 60):.02f} s
    • +
  • Labeling Efficiency: {parameters["labeling_efficiency"]} %
    • +
  • Labeling Uncertainty: {parameters["labeling_uncertainty"]} nm
    • +
  • # simulated structures: {parameters["n_simulate"]}
    • +
  • Nearest Neighbors to evaluate: {parameters["n_nearest_neighbors"]} +
    • +
  • Using Mask: {parameters.get("fp_mask_dict") is not None}
    • +
  • Width, Height, Depth: {parameters["width"]}, + {parameters["height"]}, {parameters.get("depth")}
    • +
  • Random Rotation Mode: {parameters["random_rot_mode"]}
    • +
  • # simulation repeats: {parameters["sim_repeats"]}
    • +
  • Histogram Bin Size: {parameters["fit_NND_bin"]}
    • +
  • Histogram Max value: {parameters["fit_NND_maxdist"]}
    • + """ + if fp_fig := results.get("fp_fig"): + try: + self.ci.upload_attachment(self.report_page_id, fp_fig) + except ConfluenceInterfaceError: + pass + _, fp_fig = os.path.split(fp_fig) + text += ( + "
      ' + + "
    " + ) + text += """
+ + """ + self.ci.update_page_content( + self.report_page_name, self.report_page_id, text + ) + def ripleysk(self, i, parameters, results): logger.debug("Reporting ripleysk.") text = f""" diff --git a/picasso_workflow/util.py b/picasso_workflow/util.py index 1750dbc..09c8fbd 100644 --- a/picasso_workflow/util.py +++ b/picasso_workflow/util.py @@ -180,6 +180,11 @@ def spinna_manual(self): """Direct implementation of spinna batch analysis.""" pass + @abc.abstractmethod + def spinna(self): + """implementation of a single spinna run.""" + pass + @abc.abstractmethod def ripleysk(self): pass From 243209e2aba407c56bcf60323d9dd5c2466b7ada Mon Sep 17 00:00:00 2001 From: Heinrich Date: Wed, 17 Jul 2024 13:23:44 +0200 Subject: [PATCH 2/3] testing --- picasso_workflow/tests/test_analyse.py | 42 +++++++++++++++++++++++ picasso_workflow/tests/test_confluence.py | 28 +++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/picasso_workflow/tests/test_analyse.py b/picasso_workflow/tests/test_analyse.py index 021948b..455ee7a 100644 --- a/picasso_workflow/tests/test_analyse.py +++ b/picasso_workflow/tests/test_analyse.py @@ -560,6 +560,48 @@ def fit_csr(self): shutil.rmtree(os.path.join(self.results_folder, "00_fit_csr")) + @patch( + "picasso_workflow.analyse.picasso_outpost.spinna_sgl_temp", MagicMock + ) + def spinna(self): + info = [{"Width": 1000, "Height": 1000}] + locs_dtype = [ + ("frame", "u4"), + ("photons", "f4"), + ("x", "f4"), + ("y", "f4"), + ("lpx", "f4"), + ("lpy", "f4"), + ] + locs = np.rec.array( + [ + tuple([i] + list(np.random.rand(len(locs_dtype) - 1))) + for i in range(len(self.ap.movie)) + ], + dtype=locs_dtype, + ) + self.ap.channel_locs = [locs] + self.ap.channel_info = [info] + self.ap.channel_tags = ["CD86"] + + parameters = { + "labeling_efficiency": {"CD86": 0.34}, + "labeling_uncertainty": {"CD86": 5}, + "n_simulate": 5000, + "fp_mask_dict": None, + "width": 512, + "height": 256, + "depth": 4, + "random_rot_mode": "3D", + "n_nearest_neighbors": 4, + "sim_repeats": 50, + "fit_NND_bin": 0.5, + "fit_NND_maxdist": 30, + } + parameters, results = self.ap.spinna(0, parameters) + + shutil.rmtree(os.path.join(self.results_folder, "00_spinna")) + # @patch("picasso_workflow.analyse.picasso_outpost.spinna_temp", MagicMock) def spinna_manual(self): info = [{"Width": 1000, "Height": 1000}] diff --git a/picasso_workflow/tests/test_confluence.py b/picasso_workflow/tests/test_confluence.py index a6c0154..8cc434c 100644 --- a/picasso_workflow/tests/test_confluence.py +++ b/picasso_workflow/tests/test_confluence.py @@ -545,6 +545,34 @@ def spinna_manual(self): ) self.cr.ci.delete_page(pgid) + # @unittest.skip("") + def spinna(self): + parameters = { + "labeling_efficiency": {"A": 0.34, "B": 0.56}, + "labeling_uncertainty": {"A": 5, "B": 5}, + "n_simulate": 5000, + "fp_mask_dict": None, + "width": 512, + "height": 256, + "depth": 4, + "random_rot_mode": "3D", + "n_nearest_neighbors": 4, + "sim_repeats": 50, + "fit_NND_bin": 0.5, + "fit_NND_maxdist": 30, + } + results = { + "start time": "now", + "duration": 4.12, + } + self.cr.spinna(0, parameters, results) + + # clean up + pgid, pgtitle = self.cr.ci.get_page_properties( + self.cr.report_page_name + ) + self.cr.ci.delete_page(pgid) + # @unittest.skip("") def ripleysk(self): parameters = { From 08c193526510d2899df90e5743d4df9c92d0ce7d Mon Sep 17 00:00:00 2001 From: Heinrich Date: Wed, 24 Jul 2024 16:03:53 +0200 Subject: [PATCH 3/3] spinna auto debug --- picasso_workflow/analyse.py | 65 ++++++++++++++++------- picasso_workflow/confluence.py | 26 ++++----- picasso_workflow/tests/test_analyse.py | 41 +++++++++----- picasso_workflow/tests/test_confluence.py | 3 +- picasso_workflow/workflow.py | 1 + 5 files changed, 91 insertions(+), 45 deletions(-) diff --git a/picasso_workflow/analyse.py b/picasso_workflow/analyse.py index 85d327e..97b7226 100644 --- a/picasso_workflow/analyse.py +++ b/picasso_workflow/analyse.py @@ -1867,7 +1867,7 @@ def _save_datasets_agg(self, folder): return allfps @module_decorator - def spinna(self, parameters, results): + def spinna(self, i, parameters, results): """Direct implementation of spinna batch analysis. The current locs file(s) are saved into the results folder, and a template csv file is created. This csv needs to be filled out by the @@ -1878,19 +1878,14 @@ def spinna(self, parameters, results): the index of the module parameters: dict with required keys: - labeling_efficiency : dict of float, range 0-100 - labeling efficiency percentage, for all targets + labeling_efficiency : dict of float, range 0-1 + labeling efficiency, for all targets labeling_uncertainty : float or dict of floats labeling uncertainty [nm]; good value is e.g. 5 assumed the same value for all targets - N_simulate : int + n_simulate : int number of target molecules to simulated; good value is e.g. 50000 - # density : dict of float - # density to simulate; - # area density if 2D; volume density if 3D - # used proposed value in spinna_config.csv and can be - # altered manually after the first run of this module structures : str or list of dict if str: filepath to a yaml file with the structures. if list of dict: @@ -1904,9 +1899,9 @@ def spinna(self, parameters, results): "Molecular targets" fp_mask_dict : str the filepath to the mask_dict file - width, height, depth : float - the width, height, and depth of the simulation space - depth is optional + density : list of float + density to simulate in 1/nm^d; + area density if 2D; volume density if 3D random_rot_mode : '2D', or '3D' Mode of molecule rotation in simulation sim_repeats : int @@ -1917,10 +1912,16 @@ def spinna(self, parameters, results): max of histogram n_nearest_neighbors : int number of nearest neighbors to evaluate + res_factor : float + the spinna res_factor """ + from picasso_workflow.spinna_main import load_structures_from_dict + if isinstance(parameters["structures"], str): with open(parameters["structures"], "r") as f: structures = yaml.read_all(f) + else: + structures = parameters["structures"] if isinstance(parameters["labeling_uncertainty"], (int, float)): labeling_uncertainty = { @@ -1939,8 +1940,10 @@ def spinna(self, parameters, results): # locs, but as np.ndarray pixelsize = self.analysis_config["camera_info"].get("pixelsize") exp_data = {} + exp_n_targets = np.zeros(len(self.channel_tags)) for i, target in enumerate(self.channel_tags): locs = self.channel_locs[i] + exp_n_targets[i] = len(locs) if hasattr(locs, "z"): exp_data[target] = np.stack( (locs.x * pixelsize, locs.y * pixelsize, locs.z) @@ -1951,20 +1954,46 @@ def spinna(self, parameters, results): (locs.x * pixelsize, locs.y * pixelsize) ).T + data_2d = "z" not in self.channel_locs[0].dtype.names + if data_2d: + area = parameters["n_simulate"] / sum(parameters["density"]) + width = np.sqrt(area) + height = np.sqrt(area) + depth = None + # d = 2 + else: + z_range = int(self.locs["z"].max() - self.locs["z"].min()) + volume = parameters["n_simulate"] / sum(parameters["density"]) + width = np.sqrt(volume / z_range) + height = np.sqrt(volume / z_range) + depth = z_range + # d = 3 + + structures, targets = load_structures_from_dict(structures) + + exp_frac_targets = exp_n_targets / np.sum(exp_n_targets) + n_sim_targets = { + tgt: exp_frac_targets[i] * parameters["n_simulate"] + for i, tgt in enumerate(self.channel_tags) + } + N_structures = picasso_outpost.generate_N_structures( + structures, n_sim_targets, parameters["res_factor"] + ) + spinna_pars = { "structures": structures, "label_unc": labeling_uncertainty, "le": parameters["labeling_efficiency"], "mask_dict": mask_dict, - "width": parameters["width"], - "height": parameters["height"], - "depth": parameters.get("depth"), + "width": width, + "height": height, + "depth": depth, "random_rot_mode": parameters["random_rot_mode"], "exp_data": exp_data, "sim_repeats": parameters["sim_repeats"], "fit_NND_bin": parameters["fit_NND_bin"], "fit_NND_maxdist": parameters["fit_NND_maxdist"], - "N_structures": parameters["N_simulate"], + "N_structures": N_structures, "save_filename": os.path.join(results["folder"], "spinna-run"), "asynch": True, "targets": self.channel_tags, @@ -1973,9 +2002,9 @@ def spinna(self, parameters, results): "result_dir": results["folder"], } - result_dir, fp_fig = picasso_outpost.spinna_temp(spinna_pars) + result_dir, fp_figs = picasso_outpost.spinna_sgl_temp(spinna_pars) plt.close("all") - results["fp_fig"] = fp_fig + results["fp_figs"] = fp_figs results["result_dir"] = result_dir return parameters, results diff --git a/picasso_workflow/confluence.py b/picasso_workflow/confluence.py index 0e495d3..fb78225 100644 --- a/picasso_workflow/confluence.py +++ b/picasso_workflow/confluence.py @@ -778,24 +778,24 @@ def spinna(self, i, parameters, results):
  • Nearest Neighbors to evaluate: {parameters["n_nearest_neighbors"]}
  • Using Mask: {parameters.get("fp_mask_dict") is not None}
    • -
  • Width, Height, Depth: {parameters["width"]}, - {parameters["height"]}, {parameters.get("depth")}
    • +
  • Density: {parameters["density"]} [1/nm^d]
  • Random Rotation Mode: {parameters["random_rot_mode"]}
  • # simulation repeats: {parameters["sim_repeats"]}
  • Histogram Bin Size: {parameters["fit_NND_bin"]}
  • Histogram Max value: {parameters["fit_NND_maxdist"]}
    • """ - if fp_fig := results.get("fp_fig"): - try: - self.ci.upload_attachment(self.report_page_id, fp_fig) - except ConfluenceInterfaceError: - pass - _, fp_fig = os.path.split(fp_fig) - text += ( - "
      ' - + "
    " - ) + if fp_figs := results.get("fp_figs"): + for fp_fig in fp_figs: + try: + self.ci.upload_attachment(self.report_page_id, fp_fig) + except ConfluenceInterfaceError: + pass + _, fp_fig = os.path.split(fp_fig) + text += ( + "
      ' + + "
    " + ) text += """ """ diff --git a/picasso_workflow/tests/test_analyse.py b/picasso_workflow/tests/test_analyse.py index 455ee7a..6a78413 100644 --- a/picasso_workflow/tests/test_analyse.py +++ b/picasso_workflow/tests/test_analyse.py @@ -218,6 +218,7 @@ def undrift_aim(self, mock_undrift_aim): nspots = 5 mock_undrift_aim.return_value = ( np.random.rand(2, len(self.ap.movie)), + [{"name": "info"}], np.rec.array( [ tuple(np.random.rand(len(self.locs_dtype))) @@ -560,10 +561,9 @@ def fit_csr(self): shutil.rmtree(os.path.join(self.results_folder, "00_fit_csr")) - @patch( - "picasso_workflow.analyse.picasso_outpost.spinna_sgl_temp", MagicMock - ) - def spinna(self): + @patch("picasso_workflow.analyse.picasso_outpost.spinna_sgl_temp") + def spinna(self, mock_sptmp): + mock_sptmp.return_value = (0, 1) info = [{"Width": 1000, "Height": 1000}] locs_dtype = [ ("frame", "u4"), @@ -585,18 +585,33 @@ def spinna(self): self.ap.channel_tags = ["CD86"] parameters = { - "labeling_efficiency": {"CD86": 0.34}, + "labeling_efficiency": {"CD86": 0.54}, "labeling_uncertainty": {"CD86": 5}, - "n_simulate": 5000, + "n_simulate": 50000, "fp_mask_dict": None, - "width": 512, - "height": 256, - "depth": 4, - "random_rot_mode": "3D", + "density": [0.00009], + "random_rot_mode": "2D", "n_nearest_neighbors": 4, - "sim_repeats": 50, - "fit_NND_bin": 0.5, - "fit_NND_maxdist": 30, + "sim_repeats": 5, + "fit_NND_bin": 5, + "fit_NND_maxdist": 300, + "res_factor": 10, + "structures": [ + { + "Molecular targets": ["CD86"], + "Structure title": "monomer", + "CD86_x": [0], + "CD86_y": [0], + "CD86_z": [0], + }, + { + "Molecular targets": ["CD86"], + "Structure title": "dimer", + "CD86_x": [-10, 10], + "CD86_y": [0, 0], + "CD86_z": [0, 0], + }, + ], } parameters, results = self.ap.spinna(0, parameters) diff --git a/picasso_workflow/tests/test_confluence.py b/picasso_workflow/tests/test_confluence.py index 8cc434c..3f06e5a 100644 --- a/picasso_workflow/tests/test_confluence.py +++ b/picasso_workflow/tests/test_confluence.py @@ -552,7 +552,7 @@ def spinna(self): "labeling_uncertainty": {"A": 5, "B": 5}, "n_simulate": 5000, "fp_mask_dict": None, - "width": 512, + "density": [8e-5], "height": 256, "depth": 4, "random_rot_mode": "3D", @@ -560,6 +560,7 @@ def spinna(self): "sim_repeats": 50, "fit_NND_bin": 0.5, "fit_NND_maxdist": 30, + "res_factor": 10, } results = { "start time": "now", diff --git a/picasso_workflow/workflow.py b/picasso_workflow/workflow.py index 25811d2..805e8e4 100644 --- a/picasso_workflow/workflow.py +++ b/picasso_workflow/workflow.py @@ -214,6 +214,7 @@ def _initialize_confluence_interface( def run(self): """individualize the aggregation workflow and run.""" + self.save(self.result_folder) # First, run the individual analysis sgl_ds_workflow_parameters = self.aggregation_workflow[ "single_dataset_modules"