Skip to content

Commit b977bb8

Browse files
whitelightning450whitelightning450
authored
Merge pull request #66 from whitelightning450/MonitoringSitesUpdate
Updated the files for getting monitoring sites, removed sites that ha…
2 parents 0d299e7 + dc89388 commit b977bb8

File tree

4 files changed

+262
-107
lines changed

4 files changed

+262
-107
lines changed

‎Model/Neural_Network/SWEML_Hindcast.ipynb

Lines changed: 171 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -30,18 +30,29 @@
3030
},
3131
{
3232
"cell_type": "code",
33-
"execution_count": null,
33+
"execution_count": 1,
3434
"id": "13048ad4",
3535
"metadata": {
3636
"tags": []
3737
},
38-
"outputs": [],
38+
"outputs": [
39+
{
40+
"name": "stderr",
41+
"output_type": "stream",
42+
"text": [
43+
"2024-04-23 15:14:17.606517: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
44+
"2024-04-23 15:14:17.606541: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
45+
]
46+
}
47+
],
3948
"source": [
4049
"import os\n",
4150
"import sys\n",
4251
"from tqdm import tqdm\n",
4352
"sys.path.insert(0, '..') #sys allows for the .ipynb file to connect to the shared folder files\n",
4453
"from shared_scripts import Hindcast_Initialization, NSM_SCA\n",
54+
"import warnings\n",
55+
"warnings.filterwarnings('ignore')\n",
4556
"\n",
4657
"#Set working directories\n",
4758
"cwd = os.getcwd()\n",
@@ -86,19 +97,19 @@
8697
},
8798
{
8899
"cell_type": "code",
89-
"execution_count": null,
100+
"execution_count": 2,
90101
"id": "d82649fa-4b3e-4a34-b4ac-a5504298b827",
91102
"metadata": {},
92-
"outputs": [],
103+
"outputs": [
104+
{
105+
"name": "stdout",
106+
"output_type": "stream",
107+
"text": [
108+
"Creating files for a historical simulation within 'N_Sierras', 'S_Sierras_High', 'S_Sierras_Low', 'Greater_Yellowstone', 'N_Co_Rockies', 'SW_Mont', 'SW_Co_Rockies', 'GBasin', 'N_Wasatch', 'N_Cascade', 'S_Wasatch', 'SW_Mtns', 'E_WA_N_Id_W_Mont', 'S_Wyoming', 'SE_Co_Rockies', 'Sawtooth', 'Ca_Coast', 'E_Or', 'N_Yellowstone', 'S_Cascade', 'Wa_Coast', 'Greater_Glacier', 'Or_Coast' regions for water year 2022\n"
109+
]
110+
}
111+
],
93112
"source": [
94-
"import os\n",
95-
"import sys\n",
96-
"from tqdm import tqdm\n",
97-
"sys.path.insert(0, '..') #sys allows for the .ipynb file to connect to the shared folder files\n",
98-
"from shared_scripts import Hindcast_Initialization, NSM_SCA\n",
99-
"import warnings\n",
100-
"warnings.filterwarnings('ignore')\n",
101-
"\n",
102113
"#To create .netrc file\n",
103114
"#import earthaccess\n",
104115
"#earthaccess.login(persist=True)\n",
@@ -132,12 +143,132 @@
132143
},
133144
{
134145
"cell_type": "code",
135-
"execution_count": null,
146+
"execution_count": 4,
136147
"id": "c0062b00",
137148
"metadata": {
138149
"tags": []
139150
},
140-
"outputs": [],
151+
"outputs": [
152+
{
153+
"name": "stderr",
154+
"output_type": "stream",
155+
"text": [
156+
" 0%| | 0/268 [00:00<?, ?it/s]"
157+
]
158+
},
159+
{
160+
"name": "stdout",
161+
"output_type": "stream",
162+
"text": [
163+
"Getting California Data Exchange Center SWE data from sites\n",
164+
"Getting NRCS SNOTEL SWE data from sites\n",
165+
"Getting VIIRS fSCA data and calculating the spatial average NSDI\n",
166+
"VIIRS fSCA files found locally\n"
167+
]
168+
},
169+
{
170+
"name": "stderr",
171+
"output_type": "stream",
172+
"text": [
173+
"100%|██████████| 694/694 [00:00<00:00, 2317.83it/s]\n",
174+
"100%|██████████| 694/694 [00:00<00:00, 2332.76it/s]\n",
175+
"100%|██████████| 2458/2458 [00:00<00:00, 7050.47it/s]\n",
176+
"100%|██████████| 15/15 [00:00<00:00, 6246.48it/s]\n",
177+
"100%|██████████| 6642/6642 [00:00<00:00, 8171.68it/s]\n",
178+
"100%|██████████| 1/1 [00:00<00:00, 2711.25it/s]\n",
179+
"100%|██████████| 8/8 [00:00<00:00, 6558.72it/s]\n",
180+
"100%|██████████| 6/6 [00:00<00:00, 6277.33it/s]\n",
181+
"100%|██████████| 10/10 [00:00<00:00, 6878.16it/s]\n",
182+
"100%|██████████| 6448/6448 [00:00<00:00, 8463.61it/s]\n",
183+
"100%|██████████| 5/5 [00:00<00:00, 5925.83it/s]\n",
184+
"100%|██████████| 3/3 [00:00<00:00, 4907.53it/s]\n",
185+
"100%|██████████| 13/13 [00:00<00:00, 7237.32it/s]\n",
186+
"100%|██████████| 12/12 [00:00<00:00, 6986.63it/s]\n",
187+
"100%|██████████| 3/3 [00:00<00:00, 5148.49it/s]\n",
188+
"100%|██████████| 1707/1707 [00:00<00:00, 8434.73it/s]\n",
189+
"100%|██████████| 12/12 [00:00<00:00, 7064.09it/s]\n",
190+
"100%|██████████| 1/1 [00:00<00:00, 3079.52it/s]\n",
191+
"100%|██████████| 25/25 [00:00<00:00, 7829.28it/s]\n",
192+
"100%|██████████| 16/16 [00:00<00:00, 7470.65it/s]\n",
193+
"100%|██████████| 7/7 [00:00<00:00, 6442.86it/s]\n",
194+
"100%|██████████| 9/9 [00:00<00:00, 6677.65it/s]\n",
195+
"100%|██████████| 9/9 [00:00<00:00, 6670.57it/s]\n",
196+
"100%|██████████| 3349/3349 [00:00<00:00, 8484.64it/s]\n"
197+
]
198+
},
199+
{
200+
"name": "stdout",
201+
"output_type": "stream",
202+
"text": [
203+
"Regional data QA/QC\n"
204+
]
205+
},
206+
{
207+
"name": "stderr",
208+
"output_type": "stream",
209+
"text": [
210+
"100%|██████████| 22/22 [00:00<00:00, 25.34it/s]\n",
211+
"100%|██████████| 22/22 [00:15<00:00, 1.38it/s]\n"
212+
]
213+
},
214+
{
215+
"name": "stdout",
216+
"output_type": "stream",
217+
"text": [
218+
"Calculating mean SCA for each geometry in each region...\n"
219+
]
220+
},
221+
{
222+
"name": "stderr",
223+
"output_type": "stream",
224+
"text": [
225+
"100%|██████████| 23/23 [00:34<00:00, 1.51s/it]\n",
226+
"2024-04-23 15:16:19.098846: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
227+
"2024-04-23 15:16:19.098875: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n",
228+
"2024-04-23 15:16:19.098893: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (compute004.ual-ciroh.cluster): /proc/driver/nvidia/version does not exist\n",
229+
"2024-04-23 15:16:19.099147: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n",
230+
"To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
231+
]
232+
},
233+
{
234+
"name": "stdout",
235+
"output_type": "stream",
236+
"text": [
237+
"WARNING:tensorflow:5 out of the last 57 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f7105411820> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
238+
]
239+
},
240+
{
241+
"name": "stderr",
242+
"output_type": "stream",
243+
"text": [
244+
"5 out of the last 57 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f7105411820> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
245+
]
246+
},
247+
{
248+
"name": "stdout",
249+
"output_type": "stream",
250+
"text": [
251+
"WARNING:tensorflow:6 out of the last 58 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f716364cd30> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
252+
]
253+
},
254+
{
255+
"name": "stderr",
256+
"output_type": "stream",
257+
"text": [
258+
"6 out of the last 58 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f716364cd30> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
259+
" 0%| | 1/268 [01:26<6:26:13, 86.79s/it]"
260+
]
261+
},
262+
{
263+
"name": "stdout",
264+
"output_type": "stream",
265+
"text": [
266+
"No snow in region: Or_Coast\n",
267+
"Getting California Data Exchange Center SWE data from sites\n",
268+
"Getting NRCS SNOTEL SWE data from sites\n"
269+
]
270+
}
271+
],
141272
"source": [
142273
"#run the model through all time (data acqusition already completed)\n",
143274
"model = 'Neural_Network'\n",
@@ -146,6 +277,7 @@
146277
"NewSim = True\n",
147278
"\n",
148279
"for day in tqdm(datelist):\n",
280+
" print(day)\n",
149281
" #connect interactive script to Wasatch Snow module\n",
150282
" Snow = NSM_SCA.NSM_SCA(day, threshold=threshold, Regions = Region_list, modelname = model, frequency = frequency, fSCA = fSCA, NewSim = NewSim)\n",
151283
" \n",
@@ -166,24 +298,6 @@
166298
" Snow.SWE_Predict(NewSim = NewSim, Corrections = False, fSCA = fSCA)\n"
167299
]
168300
},
169-
{
170-
"cell_type": "code",
171-
"execution_count": null,
172-
"id": "7dafffb6",
173-
"metadata": {},
174-
"outputs": [],
175-
"source": [
176-
"#Go through snotel/CDEC sites and remove ones that are no longer active..."
177-
]
178-
},
179-
{
180-
"cell_type": "code",
181-
"execution_count": null,
182-
"id": "59712933",
183-
"metadata": {},
184-
"outputs": [],
185-
"source": []
186-
},
187301
{
188302
"cell_type": "markdown",
189303
"id": "1e2290d9",
@@ -198,20 +312,35 @@
198312
},
199313
{
200314
"cell_type": "code",
201-
"execution_count": null,
202-
"id": "95050821",
203-
"metadata": {},
204-
"outputs": [],
205-
"source": []
206-
},
207-
{
208-
"cell_type": "code",
209-
"execution_count": null,
315+
"execution_count": 3,
210316
"id": "0b64a8b9-47ac-45ef-885f-98339b0983ad",
211317
"metadata": {
212318
"tags": []
213319
},
214-
"outputs": [],
320+
"outputs": [
321+
{
322+
"name": "stderr",
323+
"output_type": "stream",
324+
"text": [
325+
" 0% | |\r"
326+
]
327+
},
328+
{
329+
"name": "stdout",
330+
"output_type": "stream",
331+
"text": [
332+
"['submission_format.h5', '2019_predictions.h5', '2022_predictions.h5', 'RegionWYTest.h5']\n",
333+
"Pushing files to AWS\n"
334+
]
335+
},
336+
{
337+
"name": "stderr",
338+
"output_type": "stream",
339+
"text": [
340+
"100% |########################################################################|\n"
341+
]
342+
}
343+
],
215344
"source": [
216345
"modelname= 'Neural_Network'\n",
217346
"folderpath = 'Predictions/Hold_Out_Year/Daily/fSCA_True/'\n",

0 commit comments

Comments
 (0)