Merge branch 'feature/autospinna' into develop

picasso_workflow/analyse.py

@@ -1866,6 +1866,149 @@ def _save_datasets_agg(self, folder):
             allfps.append(filepath)
         return allfps
 
+    @module_decorator
+    def spinna(self, i, parameters, results):
+        """Direct implementation of spinna batch analysis.
+        The current locs file(s) are saved into the results folder, and
+        a template csv file is created. This csv needs to be filled out by the
+        user in a manual step before the spinna analysis is carried out.
+
+        Args:
+            i : int
+                the index of the module
+            parameters: dict
+                with required keys:
+                    labeling_efficiency : dict of float, range 0-1
+                        labeling efficiency, for all targets
+                    labeling_uncertainty : float or dict of floats
+                        labeling uncertainty [nm]; good value is e.g. 5
+                        assumed the same value for all targets
+                    n_simulate : int
+                        number of target molecules to simulated;
+                        good value is e.g. 50000
+                    structures : str or list of dict
+                        if str: filepath to a yaml file with the structures.
+                        if list of dict:
+                        SPINNA structures. Each structure dict has
+                            "Molecular targets": list of str,
+                            "Structure title": str,
+                            "TARGET_x": list of float,
+                            "TARGET_y": list of float,
+                            "TARGET_z": list of float,
+                        where TARGET is one each of the target names in
+                        "Molecular targets"
+                    fp_mask_dict : str
+                        the filepath to the mask_dict file
+                    density : list of float
+                        density to simulate in 1/nm^d;
+                        area density if 2D; volume density if 3D
+                    random_rot_mode : '2D', or '3D'
+                        Mode of molecule rotation in simulation
+                    sim_repeats : int
+                        number of simulation repeats
+                    fit_NND_bin : float
+                        bin size of fits
+                    fit_NND_maxdist : float
+                        max of histogram
+                    n_nearest_neighbors : int
+                        number of nearest neighbors to evaluate
+                    res_factor : float
+                    the spinna res_factor
+        """
+        from picasso_workflow.spinna_main import load_structures_from_dict
+
+        if isinstance(parameters["structures"], str):
+            with open(parameters["structures"], "r") as f:
+                structures = yaml.read_all(f)
+        else:
+            structures = parameters["structures"]
+
+        if isinstance(parameters["labeling_uncertainty"], (int, float)):
+            labeling_uncertainty = {
+                tgt: parameters["labeling_uncertainty"]
+                for tgt in self.channel_tags
+            }
+        else:
+            labeling_uncertainty = parameters["labeling_uncertainty"]
+
+        if parameters.get("fp_mask_dict"):
+            with open(parameters["fp_mask_dict"], "rb") as f:
+                mask_dict = pickle.load(f)
+        else:
+            mask_dict = None
+
+        # locs, but as np.ndarray
+        pixelsize = self.analysis_config["camera_info"].get("pixelsize")
+        exp_data = {}
+        exp_n_targets = np.zeros(len(self.channel_tags))
+        for i, target in enumerate(self.channel_tags):
+            locs = self.channel_locs[i]
+            exp_n_targets[i] = len(locs)
+            if hasattr(locs, "z"):
+                exp_data[target] = np.stack(
+                    (locs.x * pixelsize, locs.y * pixelsize, locs.z)
+                ).T
+                # dim = 3
+            else:
+                exp_data[target] = np.stack(
+                    (locs.x * pixelsize, locs.y * pixelsize)
+                ).T
+
+        data_2d = "z" not in self.channel_locs[0].dtype.names
+        if data_2d:
+            area = parameters["n_simulate"] / sum(parameters["density"])
+            width = np.sqrt(area)
+            height = np.sqrt(area)
+            depth = None
+            # d = 2
+        else:
+            z_range = int(self.locs["z"].max() - self.locs["z"].min())
+            volume = parameters["n_simulate"] / sum(parameters["density"])
+            width = np.sqrt(volume / z_range)
+            height = np.sqrt(volume / z_range)
+            depth = z_range
+            # d = 3
+
+        structures, targets = load_structures_from_dict(structures)
+
+        exp_frac_targets = exp_n_targets / np.sum(exp_n_targets)
+        n_sim_targets = {
+            tgt: exp_frac_targets[i] * parameters["n_simulate"]
+            for i, tgt in enumerate(self.channel_tags)
+        }
+        N_structures = picasso_outpost.generate_N_structures(
+            structures, n_sim_targets, parameters["res_factor"]
+        )
+
+        spinna_pars = {
+            "structures": structures,
+            "label_unc": labeling_uncertainty,
+            "le": parameters["labeling_efficiency"],
+            "mask_dict": mask_dict,
+            "width": width,
+            "height": height,
+            "depth": depth,
+            "random_rot_mode": parameters["random_rot_mode"],
+            "exp_data": exp_data,
+            "sim_repeats": parameters["sim_repeats"],
+            "fit_NND_bin": parameters["fit_NND_bin"],
+            "fit_NND_maxdist": parameters["fit_NND_maxdist"],
+            "N_structures": N_structures,
+            "save_filename": os.path.join(results["folder"], "spinna-run"),
+            "asynch": True,
+            "targets": self.channel_tags,
+            "apply_mask": mask_dict is not None,
+            "nn_plotted": parameters["n_nearest_neighbors"],
+            "result_dir": results["folder"],
+        }
+
+        result_dir, fp_figs = picasso_outpost.spinna_sgl_temp(spinna_pars)
+        plt.close("all")
+        results["fp_figs"] = fp_figs
+        results["result_dir"] = result_dir
+
+        return parameters, results
+
     @module_decorator
     def spinna_manual(self, i, parameters, results):
         """Direct implementation of spinna batch analysis.

picasso_workflow/confluence.py

@@ -762,6 +762,47 @@ def spinna_manual(self, i, parameters, results):
                     os.path.split(fp)[1],
                 )
 
+    def spinna(self, i, parameters, results):
+        """ """
+        logger.debug("Reporting spinna_manual.")
+        text = f"""
+        <ac:layout><ac:layout-section ac:type="single"><ac:layout-cell>
+        <p><strong>SPINNA-Manual</strong></p>
+        <ul><li>file present: {results.get('success')}</li>
+        <li>Start Time: {results['start time']}</li>
+        <li>Duration: {results["duration"] // 60:.0f} min
+        {(results["duration"] % 60):.02f} s</li>
+        <li>Labeling Efficiency: {parameters["labeling_efficiency"]} %</li>
+        <li>Labeling Uncertainty: {parameters["labeling_uncertainty"]} nm</li>
+        <li># simulated structures: {parameters["n_simulate"]}</li>
+        <li>Nearest Neighbors to evaluate: {parameters["n_nearest_neighbors"]}
+        </li>
+        <li>Using Mask: {parameters.get("fp_mask_dict") is not None}</li>
+        <li>Density: {parameters["density"]} [1/nm^d]</li>
+        <li>Random Rotation Mode: {parameters["random_rot_mode"]}</li>
+        <li># simulation repeats: {parameters["sim_repeats"]}</li>
+        <li>Histogram Bin Size: {parameters["fit_NND_bin"]}</li>
+        <li>Histogram Max value: {parameters["fit_NND_maxdist"]}</li>
+        """
+        if fp_figs := results.get("fp_figs"):
+            for fp_fig in fp_figs:
+                try:
+                    self.ci.upload_attachment(self.report_page_id, fp_fig)
+                except ConfluenceInterfaceError:
+                    pass
+                _, fp_fig = os.path.split(fp_fig)
+                text += (
+                    "<ul><ac:image><ri:attachment "
+                    + f'ri:filename="{fp_fig}" />'
+                    + "</ac:image></ul>"
+                )
+        text += """</ul>
+        </ac:layout-cell></ac:layout-section></ac:layout>
+        """
+        self.ci.update_page_content(
+            self.report_page_name, self.report_page_id, text
+        )
+
     def ripleysk(self, i, parameters, results):
         logger.debug("Reporting ripleysk.")
         text = f"""

picasso_workflow/tests/test_analyse.py

@@ -218,6 +218,7 @@ def undrift_aim(self, mock_undrift_aim):
         nspots = 5
         mock_undrift_aim.return_value = (
             np.random.rand(2, len(self.ap.movie)),
+            [{"name": "info"}],
             np.rec.array(
                 [
                     tuple(np.random.rand(len(self.locs_dtype)))
@@ -560,6 +561,62 @@ def fit_csr(self):
 
         shutil.rmtree(os.path.join(self.results_folder, "00_fit_csr"))
 
+    @patch("picasso_workflow.analyse.picasso_outpost.spinna_sgl_temp")
+    def spinna(self, mock_sptmp):
+        mock_sptmp.return_value = (0, 1)
+        info = [{"Width": 1000, "Height": 1000}]
+        locs_dtype = [
+            ("frame", "u4"),
+            ("photons", "f4"),
+            ("x", "f4"),
+            ("y", "f4"),
+            ("lpx", "f4"),
+            ("lpy", "f4"),
+        ]
+        locs = np.rec.array(
+            [
+                tuple([i] + list(np.random.rand(len(locs_dtype) - 1)))
+                for i in range(len(self.ap.movie))
+            ],
+            dtype=locs_dtype,
+        )
+        self.ap.channel_locs = [locs]
+        self.ap.channel_info = [info]
+        self.ap.channel_tags = ["CD86"]
+
+        parameters = {
+            "labeling_efficiency": {"CD86": 0.54},
+            "labeling_uncertainty": {"CD86": 5},
+            "n_simulate": 50000,
+            "fp_mask_dict": None,
+            "density": [0.00009],
+            "random_rot_mode": "2D",
+            "n_nearest_neighbors": 4,
+            "sim_repeats": 5,
+            "fit_NND_bin": 5,
+            "fit_NND_maxdist": 300,
+            "res_factor": 10,
+            "structures": [
+                {
+                    "Molecular targets": ["CD86"],
+                    "Structure title": "monomer",
+                    "CD86_x": [0],
+                    "CD86_y": [0],
+                    "CD86_z": [0],
+                },
+                {
+                    "Molecular targets": ["CD86"],
+                    "Structure title": "dimer",
+                    "CD86_x": [-10, 10],
+                    "CD86_y": [0, 0],
+                    "CD86_z": [0, 0],
+                },
+            ],
+        }
+        parameters, results = self.ap.spinna(0, parameters)
+
+        shutil.rmtree(os.path.join(self.results_folder, "00_spinna"))
+
     # @patch("picasso_workflow.analyse.picasso_outpost.spinna_temp", MagicMock)
     def spinna_manual(self):
         info = [{"Width": 1000, "Height": 1000}]

picasso_workflow/tests/test_confluence.py

@@ -545,6 +545,35 @@ def spinna_manual(self):
         )
         self.cr.ci.delete_page(pgid)
 
+    # @unittest.skip("")
+    def spinna(self):
+        parameters = {
+            "labeling_efficiency": {"A": 0.34, "B": 0.56},
+            "labeling_uncertainty": {"A": 5, "B": 5},
+            "n_simulate": 5000,
+            "fp_mask_dict": None,
+            "density": [8e-5],
+            "height": 256,
+            "depth": 4,
+            "random_rot_mode": "3D",
+            "n_nearest_neighbors": 4,
+            "sim_repeats": 50,
+            "fit_NND_bin": 0.5,
+            "fit_NND_maxdist": 30,
+            "res_factor": 10,
+        }
+        results = {
+            "start time": "now",
+            "duration": 4.12,
+        }
+        self.cr.spinna(0, parameters, results)
+
+        # clean up
+        pgid, pgtitle = self.cr.ci.get_page_properties(
+            self.cr.report_page_name
+        )
+        self.cr.ci.delete_page(pgid)
+
     # @unittest.skip("")
     def ripleysk(self):
         parameters = {

picasso_workflow/util.py

@@ -180,6 +180,11 @@ def spinna_manual(self):
         """Direct implementation of spinna batch analysis."""
         pass
 
+    @abc.abstractmethod
+    def spinna(self):
+        """implementation of a single spinna run."""
+        pass
+
     @abc.abstractmethod
     def ripleysk(self):
         pass

picasso_workflow/workflow.py

@@ -214,6 +214,7 @@ def _initialize_confluence_interface(
 
     def run(self):
         """individualize the aggregation workflow and run."""
+        self.save(self.result_folder)
         # First, run the individual analysis
         sgl_ds_workflow_parameters = self.aggregation_workflow[
             "single_dataset_modules"