Skip to content

Commit e0c91ff

Browse files
patrickscholzpatrickscholz
committed
further update the entire code, towards parallelisation with dask, close memory leakages and minimize dask task graph
1 parent 0b799ed commit e0c91ff

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+16916
-6355
lines changed

‎templates_notebooks/template_hovm.ipynb

+6-1
Original file line numberDiff line numberDiff line change
@@ -1935,6 +1935,9 @@
19351935
],
19361936
"source": [
19371937
"t0 = clock.time()\n",
1938+
"# clean up garabage on workers before the party starts!\n",
1939+
"if client is not None: client.run(gc.collect)\n",
1940+
"\n",
19381941
"data_list = list()\n",
19391942
"year_list = list(range(year[0], year[1]+1))\n",
19401943
"chnk_size = 5 # -> you can use larger chunks here if you have enough memory\n",
@@ -1978,7 +1981,8 @@
19781981
" \n",
19791982
" #_________________________________________________________________________________________________\n",
19801983
" # create hovmoeller from chunked data piece\n",
1981-
" data_hovm_chnk = tpv.load_index_fesom2(mesh, data, box, do_harithm='wmean')\n",
1984+
" data_hovm_chnk = tpv.load_index_fesom2(mesh, data, box, \n",
1985+
" do_harithm='wmean', client=client)\n",
19821986
" del(data)\n",
19831987
"\n",
19841988
" #__________________________________________________________________________________________________\n",
@@ -1989,6 +1993,7 @@
19891993
" data_hovm[ii] = xr.concat([data_hovm[ii], data_hovm_chnk[ii]], dim='time')\n",
19901994
" del(data_hovm_chnk)\n",
19911995
" print(' --> elasped time: {:3.2f} min.'.format( (clock.time()-ts)/60 ), end='\\n ')\n",
1996+
" if client is not None: client.run(gc.collect)\n",
19921997
" \n",
19931998
" #______________________________________________________________________________________________________\n",
19941999
" print('')\n",

‎templates_notebooks/template_hovm_clim.ipynb

+7-1
Original file line numberDiff line numberDiff line change
@@ -3180,6 +3180,9 @@
31803180
],
31813181
"source": [
31823182
"t0 = clock.time()\n",
3183+
"# clean up garabage on workers before the party starts!\n",
3184+
"if client is not None: client.run(gc.collect)\n",
3185+
"\n",
31833186
"#___LOAD CLIMATOLOGY___________________________________________________________________________________\n",
31843187
"clim_vname = vname\n",
31853188
"if (vname in ['temp', 'salt', 'pdens'] or 'sigma' in vname): \n",
@@ -3230,7 +3233,9 @@
32303233
" \n",
32313234
" #__________________________________________________________________________________________________\n",
32323235
" # create hovmoeller from data\n",
3233-
" data_hovm_chnk = tpv.load_index_fesom2(mesh, data, box, do_harithm='wmean')\n",
3236+
" data_hovm_chnk = tpv.load_index_fesom2(mesh, data, box, \n",
3237+
" do_harithm='wmean',\n",
3238+
" client=client)\n",
32343239
" del(data)\n",
32353240
" \n",
32363241
" # compute anomalous hovmoeller with respect to climatology\n",
@@ -3244,6 +3249,7 @@
32443249
" data_hovm[ii] = xr.concat([data_hovm[ii], data_hovm_chnk[ii]], dim='time')\n",
32453250
" del(data_hovm_chnk)\n",
32463251
" print(' --> elasped time: {:3.2f} min.'.format( (clock.time()-ts)/60 ), end='\\n ')\n",
3252+
" if client is not None: client.run(gc.collect)\n",
32473253
" #______________________________________________________________________________________________________\n",
32483254
" for ii, data_ii in enumerate(data_hovm):\n",
32493255
" print(' --> data_hovm[{:s}] uses {:3.2f} Mb:'.format(data_hovm[ii][list(data_hovm[ii].keys())[0]].attrs['boxname'], data_hovm[ii].nbytes/(1024**2)))\n",

‎templates_notebooks/template_hquiver.ipynb

+6-1
Original file line numberDiff line numberDiff line change
@@ -2625,6 +2625,10 @@
26252625
}
26262626
],
26272627
"source": [
2628+
"t0 = clock.time()\n",
2629+
"# clean up garabage on workers before the party starts!\n",
2630+
"if client is not None: client.run(gc.collect)\n",
2631+
" \n",
26282632
"data_list = list()\n",
26292633
"#___LOAD FESOM2 DATA___________________________________________________________________________________\n",
26302634
"for ii, (datapath, descript) in enumerate(zip(input_paths, input_names)):\n",
@@ -2662,7 +2666,8 @@
26622666
" else:\n",
26632667
" data_list.append(data) \n",
26642668
" del(data)\n",
2665-
"if (ref_path != None): del(data_ref)"
2669+
"if (ref_path != None): del(data_ref)\n",
2670+
"print(' --> total elasped time: {:3.2f} min.'.format( (clock.time()-t0)/60 )) "
26662671
]
26672672
},
26682673
{

‎templates_notebooks/template_hslice.ipynb

+544-1,821
Large diffs are not rendered by default.

‎templates_notebooks/template_hslice_clim.ipynb

+6-1
Original file line numberDiff line numberDiff line change
@@ -2614,6 +2614,10 @@
26142614
}
26152615
],
26162616
"source": [
2617+
"t0 = clock.time()\n",
2618+
"# clean up garabage on workers before the party starts!\n",
2619+
"if client is not None: client.run(gc.collect)\n",
2620+
"\n",
26172621
"#___LOAD Climatology DATA______________________________________________________________________________\n",
26182622
"clim_vname = vname\n",
26192623
"if (vname in ['temp', 'sst', 'salt', 'sss', 'pdens'] or 'sigma' in vname) and (depth != 'bottom'): \n",
@@ -2642,7 +2646,8 @@
26422646
" \n",
26432647
" #__________________________________________________________________________________________________ \n",
26442648
" data_list.append(tpv.do_anomaly(data, clim))\n",
2645-
" del data "
2649+
" del data \n",
2650+
"print(' --> total elasped time: {:3.2f} min.'.format( (clock.time()-t0)/60 )) "
26462651
]
26472652
},
26482653
{

‎templates_notebooks/template_hslice_isotdep.ipynb

+3-1
Original file line numberDiff line numberDiff line change
@@ -2018,8 +2018,10 @@
20182018
],
20192019
"source": [
20202020
"t0 = clock.time()\n",
2021+
"# clean up garabage on workers before the party starts!\n",
2022+
"if client is not None: client.run(gc.collect)\n",
2023+
" \n",
20212024
"data_list = list()\n",
2022-
"\n",
20232025
"#___APPEND ABS CLIMATOLOGY_____________________________________________________________________________ \n",
20242026
"if (depth != 'bottom') and do_clim and (ref_path is None): \n",
20252027
" ts = clock.time()\n",

‎templates_notebooks/template_hslice_quiver.ipynb

+3
Original file line numberDiff line numberDiff line change
@@ -2717,6 +2717,9 @@
27172717
],
27182718
"source": [
27192719
"t0 = clock.time()\n",
2720+
"# clean up garabage on workers before the party starts!\n",
2721+
"if client is not None: client.run(gc.collect)\n",
2722+
" \n",
27202723
"#______________________________________________________________________________________________________________\n",
27212724
"data_list, data_v_list = list(), list()\n",
27222725
"for ii, (datapath, descript) in enumerate(zip(input_paths, input_names)):\n",

‎templates_notebooks/template_hslice_streamf.ipynb

+3
Original file line numberDiff line numberDiff line change
@@ -2702,6 +2702,9 @@
27022702
],
27032703
"source": [
27042704
"t0 = clock.time()\n",
2705+
"# clean up garabage on workers before the party starts!\n",
2706+
"if client is not None: client.run(gc.collect)\n",
2707+
" \n",
27052708
"#______________________________________________________________________________________________________________\n",
27062709
"data_list, data_v_list = list(), list()\n",
27072710
"for ii, (datapath, descript) in enumerate(zip(input_paths, input_names)):\n",

‎templates_notebooks/template_transect.ipynb

+3
Original file line numberDiff line numberDiff line change
@@ -2696,6 +2696,9 @@
26962696
],
26972697
"source": [
26982698
"t0 = clock.time()\n",
2699+
"# clean up garabage on workers before the party starts!\n",
2700+
"if client is not None: client.run(gc.collect)\n",
2701+
"\n",
26992702
"data_list = list()\n",
27002703
"#___LOAD FESOM2 DATA___________________________________________________________________________________\n",
27012704
"for ii, (datapath, descript) in enumerate(zip(input_paths, input_names)):\n",

‎templates_notebooks/template_transect_clim.ipynb

+3
Original file line numberDiff line numberDiff line change
@@ -2724,6 +2724,9 @@
27242724
],
27252725
"source": [
27262726
"t0 = clock.time()\n",
2727+
"# clean up garabage on workers before the party starts!\n",
2728+
"if client is not None: client.run(gc.collect)\n",
2729+
" \n",
27272730
"#___LOAD CLIMATOLOGY_____________________________________________________________________________________\n",
27282731
"clim_vname= vname\n",
27292732
"if vname in ['temp', 'salt', 'pdens'] or 'sigma' in vname: \n",

‎templates_notebooks/template_transect_hflx.ipynb

+5-1
Original file line numberDiff line numberDiff line change
@@ -4641,6 +4641,9 @@
46414641
],
46424642
"source": [
46434643
"t0 = clock.time()\n",
4644+
"# clean up garabage on workers before the party starts!\n",
4645+
"if client is not None: client.run(gc.collect)\n",
4646+
" \n",
46444647
"data_list = list()\n",
46454648
"# if you do more than 1 transect at ones its better to pre load the data before extracting the transect\n",
46464649
"# but might need a bit more memory\n",
@@ -4734,7 +4737,8 @@
47344737
" # computre absolute \n",
47354738
" else:\n",
47364739
" data_list.append(csect)\n",
4737-
"if (ref_path != None): del(csect_ref) "
4740+
"if (ref_path != None): del(csect_ref) \n",
4741+
"print(' --> total elasped time: {:3.2f} min.'.format( (clock.time()-t0)/60 )) "
47384742
]
47394743
},
47404744
{

‎templates_notebooks/template_transect_hflx_t.ipynb

+6-4
Original file line numberDiff line numberDiff line change
@@ -4633,9 +4633,11 @@
46334633
}
46344634
],
46354635
"source": [
4636-
"data_list = list()\n",
4637-
"data_list_plus = list()\n",
4638-
"data_list_minus= list()\n",
4636+
"t0 = clock.time()\n",
4637+
"# clean up garabage on workers before the party starts!\n",
4638+
"if client is not None: client.run(gc.collect)\n",
4639+
"\n",
4640+
"data_list, data_list_plus, data_list_minus = list(), list(), list()\n",
46394641
"# if you do more than 1 transect at ones its better to pre load the data before extracting the transect\n",
46404642
"# but might need a bit more memory\n",
46414643
"do_persist = len(transects) > 1\n",
@@ -4645,7 +4647,6 @@
46454647
"chnk_yrs_list= year_list[::chnk_size]\n",
46464648
"\n",
46474649
"#___LOAD FESOM2 DATA___________________________________________________________________________________\n",
4648-
"t0 = clock.time()\n",
46494650
"for datapath, descript in zip(input_paths, input_names): \n",
46504651
" print(datapath, descript)\n",
46514652
" print(' > year:', end='')\n",
@@ -4736,6 +4737,7 @@
47364737
" #____________________________________________________________________________________________________\n",
47374738
" del(transp)\n",
47384739
" print(' --> elasped time: {:3.2f} min.'.format( (clock.time()-ts)/60 ), end='\\n ') \n",
4740+
" if client is not None: client.run(gc.collect)\n",
47394741
" #____________________________________________________________________________________________________\n",
47404742
" # moctseries_list.append([time, moc_t]) \n",
47414743
" data_list.append(transp_t) \n",

‎templates_notebooks/template_transect_mmean.ipynb

+3
Original file line numberDiff line numberDiff line change
@@ -2597,6 +2597,9 @@
25972597
],
25982598
"source": [
25992599
"t0 = clock.time()\n",
2600+
"# clean up garabage on workers before the party starts!\n",
2601+
"if client is not None: client.run(gc.collect)\n",
2602+
"\n",
26002603
"data_list = list()\n",
26012604
"#___LOAD FESOM2 DATA___________________________________________________________________________________\n",
26022605
"for ii, (datapath, descript) in enumerate(zip(input_paths, input_names)):\n",

‎templates_notebooks/template_transect_mmean_clim.ipynb

+3
Original file line numberDiff line numberDiff line change
@@ -4530,6 +4530,9 @@
45304530
],
45314531
"source": [
45324532
"t0 = clock.time()\n",
4533+
"# clean up garabage on workers before the party starts!\n",
4534+
"if client is not None: client.run(gc.collect)\n",
4535+
"\n",
45334536
"data_list = list()\n",
45344537
"#___LOAD CLIMATOLOGY_____________________________________________________________________________________\n",
45354538
"clim_vname= vname\n",

‎templates_notebooks/template_transect_transp.ipynb

+3
Original file line numberDiff line numberDiff line change
@@ -4641,6 +4641,9 @@
46414641
],
46424642
"source": [
46434643
"t0 = clock.time()\n",
4644+
"# clean up garabage on workers before the party starts!\n",
4645+
"if client is not None: client.run(gc.collect)\n",
4646+
"\n",
46444647
"data_list = list()\n",
46454648
"# if you do more than 1 transect at ones its better to pre load the data before extracting the transect\n",
46464649
"# but might need a bit more memory\n",

‎templates_notebooks/template_transect_transp_t.ipynb

+7-5
Original file line numberDiff line numberDiff line change
@@ -4649,9 +4649,11 @@
46494649
}
46504650
],
46514651
"source": [
4652-
"data_list = list()\n",
4653-
"data_list_plus = list()\n",
4654-
"data_list_minus= list()\n",
4652+
"t0 = clock.time()\n",
4653+
"# clean up garabage on workers before the party starts!\n",
4654+
"if client is not None: client.run(gc.collect)\n",
4655+
" \n",
4656+
"data_list, data_list_plus, data_list_minus = list(), list(), list()\n",
46554657
"# if you do more than 1 transect at ones its better to pre load the data before extracting the transect\n",
46564658
"# but might need a bit more memory\n",
46574659
"do_persist = len(transects) > 1\n",
@@ -4661,7 +4663,6 @@
46614663
"chnk_yrs_list= year_list[::chnk_size]\n",
46624664
"\n",
46634665
"#___LOAD FESOM2 DATA___________________________________________________________________________________\n",
4664-
"t0 = clock.time()\n",
46654666
"for datapath, descript in zip(input_paths, input_names): \n",
46664667
" print(datapath, descript)\n",
46674668
" print(' > year:', end='')\n",
@@ -4725,7 +4726,8 @@
47254726
" \n",
47264727
" #____________________________________________________________________________________________________\n",
47274728
" del(transp)\n",
4728-
" print(' --> elasped time: {:3.2f} min.'.format( (clock.time()-ts)/60 ), end='\\n ') \n",
4729+
" print(' --> elasped time: {:3.2f} min.'.format( (clock.time()-ts)/60 ), end='\\n ') \n",
4730+
" if client is not None: client.run(gc.collect)\n",
47294731
" #________________________________________________________________________________________________________\n",
47304732
" # moctseries_list.append([time, moc_t]) \n",
47314733
" data_list.append(transp_t) \n",

‎templates_notebooks/template_transect_zmean.ipynb

+3
Original file line numberDiff line numberDiff line change
@@ -2598,6 +2598,9 @@
25982598
],
25992599
"source": [
26002600
"t0 = clock.time()\n",
2601+
"# clean up garabage on workers before the party starts!\n",
2602+
"if client is not None: client.run(gc.collect)\n",
2603+
"\n",
26012604
"data_list = list()\n",
26022605
"#___LOAD FESOM2 DATA___________________________________________________________________________________\n",
26032606
"for ii, (datapath, descript) in enumerate(zip(input_paths, input_names)):\n",

‎templates_notebooks/template_transect_zmean_clim.ipynb

+3
Original file line numberDiff line numberDiff line change
@@ -2619,6 +2619,9 @@
26192619
],
26202620
"source": [
26212621
"t0 = clock.time()\n",
2622+
"# clean up garabage on workers before the party starts!\n",
2623+
"if client is not None: client.run(gc.collect)\n",
2624+
" \n",
26222625
"data_list = list()\n",
26232626
"#___LOAD CLIMATOLOGY_____________________________________________________________________________________\n",
26242627
"clim_vname= vname\n",

0 commit comments

Comments
 (0)