Skip to content

Commit

Permalink
Use cache to avoid dupe entries. Mod Notebook
Browse files Browse the repository at this point in the history
  • Loading branch information
@CBroz1
CBroz1 committed Oct 14, 2024
1 parent 49b774f commit 82e83f4
Show file tree
Hide file tree
Showing 7 changed files with 302 additions and 105 deletions.
12 changes: 12 additions & 0 deletions docs/src/Features/Export.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,18 @@ This is designed to capture any way that Spyglass is accessed, including
restricting one table via a join with another table. If this process seems to be
missing a way that Spyglass is accessed in your pipeline, please let us know.

Note that logging all restrictions may log more than is necessary. For example,
`MyTable & restr1 & restr2` will log `MyTable & restr1` and `MyTable & restr2`,
despite returning the combined restriction. Logging will treat compound
restrictions as 'OR' instead of 'AND' statements. This can be avoided by
combining restrictions before using the `&` operator.

```python
MyTable & "a = b" & "c > 5" # Will capture 'a = b' OR 'c > 5'
MyTable & "a = b AND c > 5" # Will capture 'a = b AND c > 5'
MyTable & dj.AndList(["a = b", "c > 5"]) # Will capture 'a = b AND c > 5'
```

If this process captures too much, you can either run a process with logging
disabled, or delete these entries from `ExportSelection` after the export is
logged.
Expand Down
207 changes: 155 additions & 52 deletions notebooks/05_Export.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,14 +107,8 @@
"name": "stderr",
"output_type": "stream",
"text": [
"[2024-05-29 14:56:01,787][INFO]: Connecting sambray@lmf-db.cin.ucsf.edu:3306\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[2024-05-29 14:56:01,872][INFO]: Connected sambray@lmf-db.cin.ucsf.edu:3306\n"
"[2024-10-14 16:47:00,520][INFO]: Connecting root@localhost:3308\n",
"[2024-10-14 16:47:00,529][INFO]: Connected root@localhost:3308\n"
]
}
],
Expand All @@ -128,7 +122,7 @@
"dj.config.load(\"dj_local_conf.json\") # load config for database connection info\n",
"\n",
"from spyglass.common.common_usage import Export, ExportSelection\n",
"from spyglass.lfp.analysis.v1 import LFPBandV1\n",
"from spyglass.common.common_ephys import Electrode\n",
"from spyglass.position.v1 import TrodesPosV1\n",
"from spyglass.spikesorting.v1.curation import CurationV1"
]
Expand Down Expand Up @@ -223,17 +217,26 @@
" <p id=\"nonprimary\">time</p>\n",
" <span class=\"djtooltiptext\"></span>\n",
" </div> </th> </tr> </thead>\n",
" <tbody> <tr> </tr> </tbody>\n",
" <tbody> <tr> <td>1</td>\n",
"<td>paper1</td>\n",
"<td>analysis1</td>\n",
"<td>0.5.1</td>\n",
"<td>2024-10-14 21:45:22</td></tr><tr><td>2</td>\n",
"<td>paper1</td>\n",
"<td>analysis2</td>\n",
"<td>0.5.1</td>\n",
"<td>2024-10-14 21:46:18</td> </tr> </tbody>\n",
" </table>\n",
" \n",
" <p>Total: 0</p></div>\n",
" <p>Total: 2</p></div>\n",
" "
],
"text/plain": [
"*export_id paper_id analysis_id spyglass_versi time \n",
"+-----------+ +----------+ +------------+ +------------+ +------+\n",
"\n",
" (Total: 0)"
"*export_id paper_id analysis_id spyglass_versi time \n",
"+-----------+ +----------+ +------------+ +------------+ +------------+\n",
"1 paper1 analysis1 0.5.1 2024-10-14 21:\n",
"2 paper1 analysis2 0.5.1 2024-10-14 21:\n",
" (Total: 2)"
]
},
"execution_count": 2,
Expand Down Expand Up @@ -321,17 +324,44 @@
" <p id=\"nonprimary\">restriction</p>\n",
" <span class=\"djtooltiptext\"></span>\n",
" </div> </th> </tr> </thead>\n",
" <tbody> <tr> </tr> </tbody>\n",
" <tbody> <tr> <td>1</td>\n",
"<td>1</td>\n",
"<td>`common_ephys`.`_electrode`</td>\n",
"<td>((nwb_file_name LIKE 'min%%'))AND((electrode_id=1))</td></tr><tr><td>1</td>\n",
"<td>2</td>\n",
"<td>`common_ephys`.`_electrode`</td>\n",
"<td>(electrode_id > 125)</td></tr><tr><td>2</td>\n",
"<td>3</td>\n",
"<td>`spikesorting_v1_curation`.`curation_v1`</td>\n",
"<td>(curation_id = 1)</td></tr><tr><td>2</td>\n",
"<td>4</td>\n",
"<td>`common_nwbfile`.`analysis_nwbfile`</td>\n",
"<td>(analysis_file_name in ('minirec20230622_JKTSSFUIL3.nwb'))</td></tr><tr><td>2</td>\n",
"<td>5</td>\n",
"<td>`position_v1_trodes_position`.`__trodes_pos_v1`</td>\n",
"<td>(trodes_pos_params_name = 'single_led')</td></tr><tr><td>2</td>\n",
"<td>6</td>\n",
"<td>`common_nwbfile`.`analysis_nwbfile`</td>\n",
"<td>(analysis_file_name='minirec20230622_JKTSSFUIL3.nwb')</td></tr><tr><td>2</td>\n",
"<td>7</td>\n",
"<td>`common_nwbfile`.`nwbfile`</td>\n",
"<td>(nwb_file_name LIKE '%%minirec20230622_.nwb%%')</td> </tr> </tbody>\n",
" </table>\n",
" \n",
" <p>Total: 0</p></div>\n",
" <p>Total: 7</p></div>\n",
" "
],
"text/plain": [
"*export_id *table_id table_name restriction \n",
"+-----------+ +----------+ +------------+ +------------+\n",
"\n",
" (Total: 0)"
"1 1 `common_ephys` ((nwb_file_nam\n",
"1 2 `common_ephys` (electrode_id \n",
"2 3 `spikesorting_ (curation_id =\n",
"2 4 `common_nwbfil (analysis_file\n",
"2 5 `position_v1_t (trodes_pos_pa\n",
"2 6 `common_nwbfil (analysis_file\n",
"2 7 `common_nwbfil (nwb_file_name\n",
" (Total: 7)"
]
},
"execution_count": 3,
Expand Down Expand Up @@ -413,17 +443,18 @@
" <p id=\"primary\">analysis_file_name</p>\n",
" <span class=\"djtooltiptext\">name of the file</span>\n",
" </div> </th> </tr> </thead>\n",
" <tbody> <tr> </tr> </tbody>\n",
" <tbody> <tr> <td>2</td>\n",
"<td>minirec20230622_JKTSSFUIL3.nwb</td> </tr> </tbody>\n",
" </table>\n",
" \n",
" <p>Total: 0</p></div>\n",
" <p>Total: 1</p></div>\n",
" "
],
"text/plain": [
"*export_id *analysis_file\n",
"+-----------+ +------------+\n",
"\n",
" (Total: 0)"
"2 minirec2023062\n",
" (Total: 1)"
]
},
"execution_count": 4,
Expand Down Expand Up @@ -469,7 +500,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"[16:32:51][INFO] Spyglass: Starting {'export_id': 1}\n"
"[16:47:01][INFO] Spyglass: Resuming {'export_id': 1}\n"
]
}
],
Expand All @@ -478,12 +509,25 @@
"\n",
"ExportSelection().start_export(**paper_key, analysis_id=\"analysis1\")\n",
"my_lfp_data = (\n",
" LFPBandV1 # Logging this table\n",
" & \"nwb_file_name LIKE 'med%'\" # using a string restriction\n",
" & {\"filter_name\": \"Theta 5-11 Hz\"} # and a dictionary restriction\n",
" Electrode # Logging this table\n",
" & dj.AndList(\n",
" [\n",
" \"nwb_file_name LIKE 'min%'\", # using a string restrictionshared\n",
" {\"electrode_id\": 1}, # and a dictionary restriction\n",
" ]\n",
" )\n",
").fetch()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Note**: Compound resrictions (e.g., `Table & a & b`) will be logged separately\n",
"as `Table & a` or `Table & b`. To fully restrict, use strings (e.g.,\n",
"`Table & 'a AND b'`) or `dj.AndList([a,b])`."
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down Expand Up @@ -571,18 +615,46 @@
" </div> </th> </tr> </thead>\n",
" <tbody> <tr> <td>1</td>\n",
"<td>1</td>\n",
"<td>`lfp_band_v1`.`__l_f_p_band_v1`</td>\n",
"<td> (( ((nwb_file_name LIKE 'med%%%%%%%%')))AND( ((`filter_name`=\"Theta 5-11 Hz\"))))</td> </tr> </tbody>\n",
"<td>`common_ephys`.`_electrode`</td>\n",
"<td>((nwb_file_name LIKE 'min%%'))AND((electrode_id=1))</td></tr><tr><td>1</td>\n",
"<td>2</td>\n",
"<td>`common_ephys`.`_electrode`</td>\n",
"<td>(electrode_id > 125)</td></tr><tr><td>1</td>\n",
"<td>8</td>\n",
"<td>`common_ephys`.`_electrode`</td>\n",
"<td>((nwb_file_name LIKE 'min%%'))AND((electrode_id=1))</td></tr><tr><td>2</td>\n",
"<td>3</td>\n",
"<td>`spikesorting_v1_curation`.`curation_v1`</td>\n",
"<td>(curation_id = 1)</td></tr><tr><td>2</td>\n",
"<td>4</td>\n",
"<td>`common_nwbfile`.`analysis_nwbfile`</td>\n",
"<td>(analysis_file_name in ('minirec20230622_JKTSSFUIL3.nwb'))</td></tr><tr><td>2</td>\n",
"<td>5</td>\n",
"<td>`position_v1_trodes_position`.`__trodes_pos_v1`</td>\n",
"<td>(trodes_pos_params_name = 'single_led')</td></tr><tr><td>2</td>\n",
"<td>6</td>\n",
"<td>`common_nwbfile`.`analysis_nwbfile`</td>\n",
"<td>(analysis_file_name='minirec20230622_JKTSSFUIL3.nwb')</td></tr><tr><td>2</td>\n",
"<td>7</td>\n",
"<td>`common_nwbfile`.`nwbfile`</td>\n",
"<td>(nwb_file_name LIKE '%%minirec20230622_.nwb%%')</td> </tr> </tbody>\n",
" </table>\n",
" \n",
" <p>Total: 1</p></div>\n",
" <p>Total: 8</p></div>\n",
" "
],
"text/plain": [
"*export_id *table_id table_name restriction \n",
"+-----------+ +----------+ +------------+ +------------+\n",
"1 1 `lfp_band_v1`. (( ((nwb_file\n",
" (Total: 1)"
"1 1 `common_ephys` ((nwb_file_nam\n",
"1 2 `common_ephys` (electrode_id \n",
"1 8 `common_ephys` ((nwb_file_nam\n",
"2 3 `spikesorting_ (curation_id =\n",
"2 4 `common_nwbfil (analysis_file\n",
"2 5 `position_v1_t (trodes_pos_pa\n",
"2 6 `common_nwbfil (analysis_file\n",
"2 7 `common_nwbfil (nwb_file_name\n",
" (Total: 8)"
]
},
"execution_count": 6,
Expand All @@ -598,7 +670,10 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"And log more under the same analysis ...\n"
"If there seem to be redundant entries in this part table, we can ignore them. \n",
"They'll be merged later during `Export.populate`.\n",
"\n",
"Let's log more under the same analysis ...\n"
]
},
{
Expand All @@ -607,21 +682,15 @@
"metadata": {},
"outputs": [],
"source": [
"my_other_lfp_data = (\n",
" LFPBandV1\n",
" & {\n",
" \"nwb_file_name\": \"mediumnwb20230802_.nwb\",\n",
" \"filter_name\": \"Theta 5-10 Hz\",\n",
" }\n",
").fetch()"
"my_other_lfp_data = (Electrode & \"electrode_id > 125\").fetch()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Since these restrictions are mutually exclusive, we can check that the will\n",
"be combined appropriately by priviewing the logged tables...\n"
"be combined appropriately by previewing the logged tables...\n"
]
},
{
Expand All @@ -632,12 +701,37 @@
{
"data": {
"text/plain": [
"[FreeTable(`lfp_band_v1`.`__l_f_p_band_v1`)\n",
" *lfp_merge_id *filter_name *filter_sampli *nwb_file_name *target_interv *lfp_band_samp analysis_file_ interval_list_ lfp_band_objec\n",
" +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n",
" 0f3bb01e-0ef6- Theta 5-10 Hz 1000 mediumnwb20230 pos 0 valid ti 100 mediumnwb20230 pos 0 valid ti 44e38dc1-3779-\n",
" 0f3bb01e-0ef6- Theta 5-11 Hz 1000 mediumnwb20230 pos 0 valid ti 100 mediumnwb20230 pos 0 valid ti c9b93111-decb-\n",
" (Total: 2)]"
"[FreeTable(`common_ephys`.`_electrode`)\n",
" *nwb_file_name *electrode_gro *electrode_id probe_id probe_shank probe_electrod region_id name original_refer x y z filtering impedance bad_channel x_warped y_warped z_warped contacts \n",
" +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +-----------+ +------+ +------------+ +-----+ +-----+ +-----+ +-----------+ +-----------+ +------------+ +----------+ +----------+ +----------+ +----------+\n",
" minirec2023062 0 0 tetrode_12.5 0 0 1 0 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 \n",
" minirec2023062 0 1 tetrode_12.5 0 1 1 1 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 \n",
" minirec2023062 0 2 tetrode_12.5 0 2 1 2 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 \n",
" minirec2023062 0 3 tetrode_12.5 0 3 1 3 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 \n",
" minirec2023062 31 126 tetrode_12.5 0 2 1 126 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 \n",
" minirec2023062 31 127 tetrode_12.5 0 3 1 127 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 \n",
" (Total: 6),\n",
" FreeTable(`spikesorting_v1_curation`.`curation_v1`)\n",
" *sorting_id *curation_id parent_curatio analysis_file_ object_id merges_applied description \n",
" +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n",
" 3b909e6b-9cec- 1 0 minirec2023062 9bcad28e-3b7f- 0 after metric c\n",
" (Total: 1),\n",
" FreeTable(`common_nwbfile`.`analysis_nwbfile`)\n",
" *analysis_file nwb_file_name analysis_f analysis_file_ analysis_p\n",
" +------------+ +------------+ +--------+ +------------+ +--------+\n",
" minirec2023062 minirec2023062 =BLOB= =BLOB= \n",
" minirec2023062 minirec2023062 =BLOB= =BLOB= \n",
" (Total: 2),\n",
" FreeTable(`position_v1_trodes_position`.`__trodes_pos_v1`)\n",
" *nwb_file_name *interval_list *trodes_pos_pa analysis_file_ position_objec orientation_ob velocity_objec\n",
" +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n",
" minirec2023062 pos 0 valid ti single_led minirec2023062 5f48f897-2d13- 06d39e03-51cc- 210c92b5-5719-\n",
" (Total: 1),\n",
" FreeTable(`common_nwbfile`.`nwbfile`)\n",
" *nwb_file_name nwb_file_a\n",
" +------------+ +--------+\n",
" minirec2023062 =BLOB= \n",
" (Total: 1)]"
]
},
"execution_count": 8,
Expand All @@ -649,6 +743,15 @@
"ExportSelection().preview_tables(**paper_key)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"For a more comprehensive view of what would be in the export, you can look at\n",
"`ExportSelection().show_all_tables(**paper_key)`. This may take some time to\n",
"generate."
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -666,8 +769,8 @@
"name": "stderr",
"output_type": "stream",
"text": [
"[16:32:51][INFO] Spyglass: Export 1 in progress. Starting new.\n",
"[16:32:51][INFO] Spyglass: Starting {'export_id': 2}\n"
"[16:47:02][INFO] Spyglass: Export 1 in progress. Starting new.\n",
"[16:47:02][INFO] Spyglass: Resuming {'export_id': 2}\n"
]
}
],
Expand All @@ -692,8 +795,8 @@
{
"data": {
"text/plain": [
"[{'file_path': '/home/cb/wrk/alt/data/raw/mediumnwb20230802_.nwb'},\n",
" {'file_path': '/home/cb/wrk/alt/data/analysis/mediumnwb20230802/mediumnwb20230802_ALNN6TZ4L7.nwb'}]"
"[{'file_path': '/home/cb/wrk/spyglass/tests/_data/raw/minirec20230622_.nwb'},\n",
" {'file_path': '/home/cb/wrk/spyglass/tests/_data/analysis/minirec20230622/minirec20230622_JKTSSFUIL3.nwb'}]"
]
},
"execution_count": 10,
Expand Down Expand Up @@ -1065,7 +1168,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.16"
"version": "3.9.20"
}
},
"nbformat": 4,
Expand Down
Loading

0 comments on commit 82e83f4

Please sign in to comment.