Skip to content

Commit 6ce6f56

Browse files
j1cj1c
authored
Figures (#422)
* mean connectomes * data processing notebook * figure 2 notebook * index update
1 parent 7c3fc69 commit 6ce6f56

25 files changed

+505
-3
lines changed

‎docs/paper/data.ipynb

+261
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"# Data Preprocessing\n",
8+
"\n",
9+
"The outputs from the `m2g` pipeline is available in our open-access AWS S3 bucket: `s3://open-neurodata/m2`. You can use the file tree to browse the outputs [http://open-neurodata.s3-website-us-east-1.amazonaws.com/](http://open-neurodata.s3-website-us-east-1.amazonaws.com/)."
10+
]
11+
},
12+
{
13+
"cell_type": "code",
14+
"execution_count": 1,
15+
"metadata": {},
16+
"outputs": [
17+
{
18+
"name": "stderr",
19+
"output_type": "stream",
20+
"text": [
21+
"/Users/j1c/miniconda3/envs/m2g/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
22+
" from .autonotebook import tqdm as notebook_tqdm\n"
23+
]
24+
}
25+
],
26+
"source": [
27+
"import boto3\n",
28+
"from botocore import UNSIGNED\n",
29+
"from botocore.client import Config\n",
30+
"\n",
31+
"from pathlib import Path\n",
32+
"import numpy as np\n",
33+
"\n",
34+
"from graspologic.utils import import_edgelist, pass_to_ranks"
35+
]
36+
},
37+
{
38+
"cell_type": "code",
39+
"execution_count": null,
40+
"metadata": {},
41+
"outputs": [],
42+
"source": [
43+
"modalities = [\"Diffusion\", \"Functional\"]\n",
44+
"diffusion_datasets = [\n",
45+
" \"SWU4\",\n",
46+
" \"HNU1\",\n",
47+
" \"NKIENH\",\n",
48+
" \"XHCUMS\",\n",
49+
" \"BNU1\",\n",
50+
" \"BNU3\",\n",
51+
" \"NKI1\",\n",
52+
" \"NKI24\",\n",
53+
" \"IPCAS8\",\n",
54+
" \"MRN_1\",\n",
55+
"]\n",
56+
"functional_datasets = [\n",
57+
" \"NYU_2\",\n",
58+
" \"SWU4\",\n",
59+
" \"HNU1\",\n",
60+
" \"XHCUMS\",\n",
61+
" \"UPSM_1\",\n",
62+
" \"BNU3\",\n",
63+
" \"IPCAS7\",\n",
64+
" \"SWU1\",\n",
65+
" \"IPCAS1\",\n",
66+
" \"BNU1\",\n",
67+
"]\n",
68+
"\n",
69+
"datasets = {\"Diffusion\": diffusion_datasets, \"Functional\": functional_datasets}"
70+
]
71+
},
72+
{
73+
"cell_type": "markdown",
74+
"metadata": {},
75+
"source": [
76+
"## Fetch from S3 and Download to Local\n",
77+
"\n",
78+
"The files will be stored at `m2g/docs/paper/data/` directory."
79+
]
80+
},
81+
{
82+
"cell_type": "code",
83+
"execution_count": 5,
84+
"metadata": {},
85+
"outputs": [
86+
{
87+
"name": "stdout",
88+
"output_type": "stream",
89+
"text": [
90+
"Downloading m2g/Diffusion/SWU4-8-27-20-m2g-native-csa-det/... Total files: 422\n",
91+
"Downloading m2g/Diffusion/HNU1-8-27-20-m2g-native-csa-det/... Total files: 300\n",
92+
"Downloading m2g/Diffusion/NKIENH-11-01-20-m2g-native-csa-det/... Total files: 129\n",
93+
"Downloading m2g/Diffusion/XHCUMS-8-27-20-m2g-native-csa-det/... Total files: 117\n",
94+
"Downloading m2g/Diffusion/BNU1-8-27-20-m2g-native-csa-det/... Total files: 114\n",
95+
"Downloading m2g/Diffusion/BNU3-11-01-20-m2g-native-csa-det/... Total files: 47\n",
96+
"Downloading m2g/Diffusion/NKI1-8-24-20-m2g-native-csa-det/... Total files: 40\n",
97+
"Downloading m2g/Diffusion/NKI24-11-01-20-m2g-native-csa-det/... Total files: 38\n",
98+
"Downloading m2g/Diffusion/IPCAS8-8-27-20-m2g-native-csa-det/... Total files: 26\n",
99+
"Downloading m2g/Diffusion/MRN_1-8-27-20-m2g-native-csa-det/... Total files: 19\n",
100+
"Downloading m2g/Functional/NYU_2-11-27-20-m2g-func/... Total files: 494\n",
101+
"Downloading m2g/Functional/SWU4-11-12-20-m2g-func/... Total files: 425\n",
102+
"Downloading m2g/Functional/HNU1-11-12-20-m2g-func/... Total files: 300\n",
103+
"Downloading m2g/Functional/XHCUMS-11-27-20-m2g-func/... Total files: 247\n",
104+
"Downloading m2g/Functional/UPSM_1-11-27-20-m2g-func/... Total files: 230\n",
105+
"Downloading m2g/Functional/BNU3-11-12-20-m2g-func/... Total files: 144\n",
106+
"Downloading m2g/Functional/IPCAS7-11-27-20-m2g-func/... Total files: 144\n",
107+
"Downloading m2g/Functional/SWU1-11-27-20-m2g-func/... Total files: 119\n",
108+
"Downloading m2g/Functional/IPCAS1-11-27-20-m2g-func/... Total files: 118\n",
109+
"Downloading m2g/Functional/BNU1-11-12-20-m2g-func/... Total files: 106\n"
110+
]
111+
}
112+
],
113+
"source": [
114+
"parcellation = \"DKT_space-MNI152NLin6_res-2x2x2\"\n",
115+
"bucket = \"open-neurodata\"\n",
116+
"\n",
117+
"for modality in modalities:\n",
118+
" if modality == \"Diffusion\":\n",
119+
" parcellation = \"DKT_space-MNI152NLin6_res-2x2x2\"\n",
120+
" else:\n",
121+
" parcellation = \"DKT_space-MNI152NLin6_res-2x2x2.nii.gz\"\n",
122+
"\n",
123+
" prefix = f\"m2g/{modality}/\"\n",
124+
"\n",
125+
" s3 = boto3.client(\"s3\", config=Config(signature_version=UNSIGNED))\n",
126+
" resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter=\"/\")\n",
127+
"\n",
128+
" dataset_fullnames = []\n",
129+
" for dset in datasets[modality]:\n",
130+
" for r in resp.get(\"CommonPrefixes\"):\n",
131+
" if dset in r.get(\"Prefix\"):\n",
132+
" dataset_fullnames.append(r.get(\"Prefix\"))\n",
133+
"\n",
134+
" for dset, dset_abbrev in zip(dataset_fullnames, datasets[modality]):\n",
135+
" prefix = f\"{dset}Connectomes/{parcellation}/\"\n",
136+
"\n",
137+
" resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter=\"/\")\n",
138+
" contents = resp[\"Contents\"]\n",
139+
"\n",
140+
" files = []\n",
141+
" for obj in contents:\n",
142+
" key = obj[\"Key\"]\n",
143+
" if modality == \"Functional\":\n",
144+
" if key.endswith(\".csv\") and \"abs\" in key:\n",
145+
" files.append(key)\n",
146+
" else:\n",
147+
" if key.endswith(\".csv\"):\n",
148+
" files.append(key)\n",
149+
"\n",
150+
" print(f\"Downloading {dset}... Total files: {len(files)}\")\n",
151+
"\n",
152+
" # Save to data folder\n",
153+
" p = Path(f\"./data/{modality}/{dset_abbrev}\")\n",
154+
" p.mkdir(parents=True, exist_ok=True)\n",
155+
"\n",
156+
" # Download files\n",
157+
" for f in files:\n",
158+
" out = p / Path(f).name\n",
159+
" if not out.exists():\n",
160+
" s3.download_file(bucket, f, out)"
161+
]
162+
},
163+
{
164+
"cell_type": "markdown",
165+
"metadata": {},
166+
"source": [
167+
"# Compute mean connectomes\n",
168+
"\n",
169+
"This data will be used for plotting in Figure 2."
170+
]
171+
},
172+
{
173+
"cell_type": "code",
174+
"execution_count": 10,
175+
"metadata": {},
176+
"outputs": [
177+
{
178+
"name": "stdout",
179+
"output_type": "stream",
180+
"text": [
181+
"Computing mean graph for Diffusion SWU4... Total files: 422\n",
182+
"Computing mean graph for Diffusion HNU1... Total files: 300\n",
183+
"Computing mean graph for Diffusion NKIENH... Total files: 129\n",
184+
"Computing mean graph for Diffusion XHCUMS... Total files: 117\n",
185+
"Computing mean graph for Diffusion BNU1... Total files: 114\n",
186+
"Computing mean graph for Diffusion BNU3... Total files: 47\n",
187+
"Computing mean graph for Diffusion NKI1... Total files: 40\n",
188+
"Computing mean graph for Diffusion NKI24... Total files: 38\n",
189+
"Computing mean graph for Diffusion IPCAS8... Total files: 26\n",
190+
"Computing mean graph for Diffusion MRN_1... Total files: 19\n",
191+
"Computing mean graph for Functional NYU_2... Total files: 494\n",
192+
"Computing mean graph for Functional SWU4... Total files: 425\n",
193+
"Computing mean graph for Functional HNU1... Total files: 300\n",
194+
"Computing mean graph for Functional XHCUMS... Total files: 247\n",
195+
"Computing mean graph for Functional UPSM_1... Total files: 230\n",
196+
"Computing mean graph for Functional BNU3... Total files: 144\n",
197+
"Computing mean graph for Functional IPCAS7... Total files: 144\n",
198+
"Computing mean graph for Functional SWU1... Total files: 119\n",
199+
"Computing mean graph for Functional IPCAS1... Total files: 118\n",
200+
"Computing mean graph for Functional BNU1... Total files: 106\n"
201+
]
202+
}
203+
],
204+
"source": [
205+
"out_dir = Path(f\"./data/mean_connectomes/\")\n",
206+
"out_dir.mkdir(parents=True, exist_ok=True)\n",
207+
"\n",
208+
"for modality, dsets in datasets.items():\n",
209+
" if modality == \"Functional\":\n",
210+
" keyword = \"*abs*\"\n",
211+
" else:\n",
212+
" keyword = \"*\"\n",
213+
"\n",
214+
" for dset in dsets:\n",
215+
" p = Path(f\"./data/{modality}/{dset}\")\n",
216+
" files = list(p.glob(keyword))\n",
217+
"\n",
218+
" print(\n",
219+
" f\"Computing mean graph for {modality} {dset}... Total files: {len(files)}\"\n",
220+
" )\n",
221+
"\n",
222+
" graphs = import_edgelist(files, \"csv\")\n",
223+
" graphs = [pass_to_ranks(g) for g in graphs]\n",
224+
"\n",
225+
" # Compute mean graph\n",
226+
" mean_graph = np.array(graphs).mean(axis=0)\n",
227+
"\n",
228+
" # Save mean graph\n",
229+
" np.save(out_dir / f\"{len(files):>03}_{modality}_{dset}\", mean_graph)"
230+
]
231+
},
232+
{
233+
"cell_type": "code",
234+
"execution_count": null,
235+
"metadata": {},
236+
"outputs": [],
237+
"source": []
238+
}
239+
],
240+
"metadata": {
241+
"kernelspec": {
242+
"display_name": "m2g",
243+
"language": "python",
244+
"name": "python3"
245+
},
246+
"language_info": {
247+
"codemirror_mode": {
248+
"name": "ipython",
249+
"version": 3
250+
},
251+
"file_extension": ".py",
252+
"mimetype": "text/x-python",
253+
"name": "python",
254+
"nbconvert_exporter": "python",
255+
"pygments_lexer": "ipython3",
256+
"version": "3.10.14"
257+
}
258+
},
259+
"nbformat": 4,
260+
"nbformat_minor": 2
261+
}

‎docs/paper/data/mean_connectomes/019_Diffusion_MRN_1.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/026_Diffusion_IPCAS8.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/038_Diffusion_NKI24.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/040_Diffusion_NKI1.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/047_Diffusion_BNU3.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/106_Functional_BNU1.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/114_Diffusion_BNU1.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/117_Diffusion_XHCUMS.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/118_Functional_IPCAS1.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/119_Functional_SWU1.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/129_Diffusion_NKIENH.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/144_Functional_BNU3.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/144_Functional_IPCAS7.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/230_Functional_UPSM_1.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/247_Functional_XHCUMS.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/300_Diffusion_HNU1.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/300_Functional_HNU1.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/422_Diffusion_SWU4.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/425_Functional_SWU4.npy

53.9 KB
Binary file not shown.

‎docs/paper/data/mean_connectomes/494_Functional_NYU_2.npy

53.9 KB
Binary file not shown.

‎docs/paper/figure2.ipynb

+243-3
Large diffs are not rendered by default.

‎docs/paper/figures/figure2.pdf

3.6 MB
Binary file not shown.

‎docs/paper/figures/figure2.png

839 KB
Loading

‎docs/paper/index.rst

+1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ Figures
1515
:maxdepth: 1
1616
:titlesonly:
1717

18+
data
1819
figure2
1920
figure3
2021
figure4

0 commit comments

Comments
 (0)