Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updated the files for getting monitoring sites, removed sites that ha… #66

Merged
merged 1 commit into from
Apr 23, 2024
Merged

Updated the files for getting monitoring sites, removed sites that ha… #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
213 changes: 171 additions & 42 deletions Model/Neural_Network/SWEML_Hindcast.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,29 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"id": "13048ad4",
"metadata": {
"tags": []
},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-04-23 15:14:17.606517: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
"2024-04-23 15:14:17.606541: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"from tqdm import tqdm\n",
"sys.path.insert(0, '..') #sys allows for the .ipynb file to connect to the shared folder files\n",
"from shared_scripts import Hindcast_Initialization, NSM_SCA\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"#Set working directories\n",
"cwd = os.getcwd()\n",
Expand Down Expand Up @@ -86,19 +97,19 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"id": "d82649fa-4b3e-4a34-b4ac-a5504298b827",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Creating files for a historical simulation within 'N_Sierras', 'S_Sierras_High', 'S_Sierras_Low', 'Greater_Yellowstone', 'N_Co_Rockies', 'SW_Mont', 'SW_Co_Rockies', 'GBasin', 'N_Wasatch', 'N_Cascade', 'S_Wasatch', 'SW_Mtns', 'E_WA_N_Id_W_Mont', 'S_Wyoming', 'SE_Co_Rockies', 'Sawtooth', 'Ca_Coast', 'E_Or', 'N_Yellowstone', 'S_Cascade', 'Wa_Coast', 'Greater_Glacier', 'Or_Coast' regions for water year 2022\n"
]
}
],
"source": [
"import os\n",
"import sys\n",
"from tqdm import tqdm\n",
"sys.path.insert(0, '..') #sys allows for the .ipynb file to connect to the shared folder files\n",
"from shared_scripts import Hindcast_Initialization, NSM_SCA\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"#To create .netrc file\n",
"#import earthaccess\n",
"#earthaccess.login(persist=True)\n",
Expand Down Expand Up @@ -132,12 +143,132 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"id": "c0062b00",
"metadata": {
"tags": []
},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
" 0%| | 0/268 [00:00<?, ?it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Getting California Data Exchange Center SWE data from sites\n",
"Getting NRCS SNOTEL SWE data from sites\n",
"Getting VIIRS fSCA data and calculating the spatial average NSDI\n",
"VIIRS fSCA files found locally\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 694/694 [00:00<00:00, 2317.83it/s]\n",
"100%|██████████| 694/694 [00:00<00:00, 2332.76it/s]\n",
"100%|██████████| 2458/2458 [00:00<00:00, 7050.47it/s]\n",
"100%|██████████| 15/15 [00:00<00:00, 6246.48it/s]\n",
"100%|██████████| 6642/6642 [00:00<00:00, 8171.68it/s]\n",
"100%|██████████| 1/1 [00:00<00:00, 2711.25it/s]\n",
"100%|██████████| 8/8 [00:00<00:00, 6558.72it/s]\n",
"100%|██████████| 6/6 [00:00<00:00, 6277.33it/s]\n",
"100%|██████████| 10/10 [00:00<00:00, 6878.16it/s]\n",
"100%|██████████| 6448/6448 [00:00<00:00, 8463.61it/s]\n",
"100%|██████████| 5/5 [00:00<00:00, 5925.83it/s]\n",
"100%|██████████| 3/3 [00:00<00:00, 4907.53it/s]\n",
"100%|██████████| 13/13 [00:00<00:00, 7237.32it/s]\n",
"100%|██████████| 12/12 [00:00<00:00, 6986.63it/s]\n",
"100%|██████████| 3/3 [00:00<00:00, 5148.49it/s]\n",
"100%|██████████| 1707/1707 [00:00<00:00, 8434.73it/s]\n",
"100%|██████████| 12/12 [00:00<00:00, 7064.09it/s]\n",
"100%|██████████| 1/1 [00:00<00:00, 3079.52it/s]\n",
"100%|██████████| 25/25 [00:00<00:00, 7829.28it/s]\n",
"100%|██████████| 16/16 [00:00<00:00, 7470.65it/s]\n",
"100%|██████████| 7/7 [00:00<00:00, 6442.86it/s]\n",
"100%|██████████| 9/9 [00:00<00:00, 6677.65it/s]\n",
"100%|██████████| 9/9 [00:00<00:00, 6670.57it/s]\n",
"100%|██████████| 3349/3349 [00:00<00:00, 8484.64it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Regional data QA/QC\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 22/22 [00:00<00:00, 25.34it/s]\n",
"100%|██████████| 22/22 [00:15<00:00, 1.38it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Calculating mean SCA for each geometry in each region...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 23/23 [00:34<00:00, 1.51s/it]\n",
"2024-04-23 15:16:19.098846: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
"2024-04-23 15:16:19.098875: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n",
"2024-04-23 15:16:19.098893: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (compute004.ual-ciroh.cluster): /proc/driver/nvidia/version does not exist\n",
"2024-04-23 15:16:19.099147: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA\n",
"To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"WARNING:tensorflow:5 out of the last 57 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f7105411820> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"5 out of the last 57 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f7105411820> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"WARNING:tensorflow:6 out of the last 58 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f716364cd30> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"6 out of the last 58 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7f716364cd30> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n",
" 0%| | 1/268 [01:26<6:26:13, 86.79s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"No snow in region: Or_Coast\n",
"Getting California Data Exchange Center SWE data from sites\n",
"Getting NRCS SNOTEL SWE data from sites\n"
]
}
],
"source": [
"#run the model through all time (data acqusition already completed)\n",
"model = 'Neural_Network'\n",
Expand All @@ -146,6 +277,7 @@
"NewSim = True\n",
"\n",
"for day in tqdm(datelist):\n",
" print(day)\n",
" #connect interactive script to Wasatch Snow module\n",
" Snow = NSM_SCA.NSM_SCA(day, threshold=threshold, Regions = Region_list, modelname = model, frequency = frequency, fSCA = fSCA, NewSim = NewSim)\n",
" \n",
Expand All @@ -166,24 +298,6 @@
" Snow.SWE_Predict(NewSim = NewSim, Corrections = False, fSCA = fSCA)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7dafffb6",
"metadata": {},
"outputs": [],
"source": [
"#Go through snotel/CDEC sites and remove ones that are no longer active..."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "59712933",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"id": "1e2290d9",
Expand All @@ -198,20 +312,35 @@
},
{
"cell_type": "code",
"execution_count": null,
"id": "95050821",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"id": "0b64a8b9-47ac-45ef-885f-98339b0983ad",
"metadata": {
"tags": []
},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
" 0% | |\r"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"['submission_format.h5', '2019_predictions.h5', '2022_predictions.h5', 'RegionWYTest.h5']\n",
"Pushing files to AWS\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100% |########################################################################|\n"
]
}
],
"source": [
"modelname= 'Neural_Network'\n",
"folderpath = 'Predictions/Hold_Out_Year/Daily/fSCA_True/'\n",
Expand Down
Loading
Loading