Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updated .netrc method, SWEML_Hindcast.ipynb, shared scripts for predi… #64

Merged
merged 1 commit into from
Apr 22, 2024
Merged

Updated .netrc method, SWEML_Hindcast.ipynb, shared scripts for predi… #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
0 ...Old/Pre_Processed/grid_cells_eval.geojson → ...Old/Pre_Processed/grid_cells_eval.geojson
View file
Open in desktop
File renamed without changes.
0 Data_Old/Provided_Data/grid_cells.geojson → Data-Old/Provided_Data/grid_cells.geojson
View file
Open in desktop
File renamed without changes.
Binary file removed Data_Old/Optimal_Features.pkl
View file
Open in desktop
Binary file not shown.
Binary file removed Data_Old/PreProcessed/RegionVal.pkl
View file
Open in desktop
Binary file not shown.
Binary file removed Data_Old/PreProcessed/RegionVal2.pkl
View file
Open in desktop
Binary file not shown.
Binary file removed Data_Old/Pre_Processed/SWE.h5
View file
Open in desktop
Binary file not shown.
10,879 changes: 0 additions & 10,879 deletions Data_Old/Provided_Data/Prediction_Location_Metadata.csv
View file
Open in desktop

This file was deleted.

10,879 changes: 0 additions & 10,879 deletions Data_Old/Provided_Data/Prediction_Location_Observations.csv
View file
Open in desktop

This file was deleted.

Binary file removed Data_Old/Provided_Data/SWE.h5
View file
Open in desktop
Binary file not shown.
Binary file removed Data_Old/Provided_Data/TestSWE.h5
View file
Open in desktop
Binary file not shown.
Binary file removed Data_Old/Provided_Data/TrainSWE.h5
View file
Open in desktop
Binary file not shown.
701 changes: 0 additions & 701 deletions Data_Old/Provided_Data/ground_measures_metadata.csv
View file
Open in desktop

This file was deleted.

701 changes: 0 additions & 701 deletions Data_Old/Provided_Data/ground_measures_test_features.csv
View file
Open in desktop

This file was deleted.

701 changes: 0 additions & 701 deletions Data_Old/Provided_Data/ground_measures_train_features.csv
View file
Open in desktop

This file was deleted.

9,067 changes: 0 additions & 9,067 deletions Data_Old/Provided_Data/submission_format.csv
View file
Open in desktop

This file was deleted.

Binary file removed Data_Old/RegionSnotel_Train.pkl
View file
Open in desktop
Binary file not shown.
Binary file removed Data_Old/RegionTrain.pkl
View file
Open in desktop
Binary file not shown.
Binary file removed Data_Old/RegionTrain_SCA.pkl
View file
Open in desktop
Binary file not shown.
14 changes: 0 additions & 14 deletions Data_Processing_NSMv2.0/NSIDC_Data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
filename_filter = ''
url_list = [] """

CMR_URL = 'https://cmr.earthdata.nasa.gov'
URS_URL = 'https://urs.earthdata.nasa.gov'
CMR_PAGE_SIZE = 2000
CMR_FILE_URL = ('{0}/search/granules.json?provider=NSIDC_ECS'
Expand All @@ -54,19 +53,6 @@
#load access key
HOME = os.path.expanduser('~')

def get_login_credentials():
try:
info = netrc.netrc()
username,none, password = info.authenticators(URS_URL)
credentials = f'{username}:{password}'
credentials = base64.b64encode(credentials.encode('ascii')).decode('ascii')
except Exception:
username = input("Earthdata Login Username: ")
password = getpass.getpass("Earthdata Login Password: ")
credentials = f'{username}:{password}'
credentials = base64.b64encode(credentials.encode('ascii')).decode('ascii')
return credentials

def build_version_query_params(version):
desired_pad_length = 3
if len(version) > desired_pad_length:
Expand Down
265 changes: 33 additions & 232 deletions Model/NSM_SWE_Historical_SCA.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

Binary file added ...twork/Predictions/Hold_Out_Year/Daily/fSCA_True/Prediction_DF_SCA_2021-09-30.pkl.4f9fbB4d
View file
Open in desktop
Binary file not shown.
92 changes: 13 additions & 79 deletions Model/Neural_Network/SSM_Hindcast_2019.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,21 +30,12 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"id": "13048ad4",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-03-19 15:54:59.895941: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
"2024-03-19 15:54:59.895967: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
]
}
],
"outputs": [],
"source": [
"import os\n",
"import sys\n",
Expand All @@ -68,27 +59,12 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"id": "e80aea1d-de20-4d76-a170-2740096f7813",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Downloading files from AWS to local\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 274/274 [07:39<00:00, 1.68s/it]\n"
]
}
],
"outputs": [],
"source": [
"# frequency = 'Daily'\n",
"# fSCA = True\n",
Expand Down Expand Up @@ -147,49 +123,30 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"id": "c0062b00",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
" 0%| | 0/268 [00:00<?, ?it/s]\n"
]
},
{
"ename": "AttributeError",
"evalue": "'NSM_SCA' object has no attribute 'fSCA'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[3], line 26\u001b[0m\n\u001b[1;32m 10\u001b[0m Snow \u001b[38;5;241m=\u001b[39m NSM_SCA\u001b[38;5;241m.\u001b[39mNSM_SCA(day, threshold\u001b[38;5;241m=\u001b[39mthreshold, Regions \u001b[38;5;241m=\u001b[39m Region_list, modelname \u001b[38;5;241m=\u001b[39m model, frequency \u001b[38;5;241m=\u001b[39m frequency, fSCA \u001b[38;5;241m=\u001b[39m fSCA)\n\u001b[1;32m 12\u001b[0m \u001b[38;5;66;03m#Go get SNOTEL observations - all data currently loaded, set to True to download\u001b[39;00m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;66;03m#Snow.Get_Monitoring_Data_Threaded(getdata = True)\u001b[39;00m\n\u001b[1;32m 14\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;66;03m#Make predictions, set NewSim to False\u001b[39;00m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;66;03m#Look to multiprocess, each region can do a prediction to speed things up. set NewSim to true for New simulation, turn to false once all data has been proces and saved.\u001b[39;00m\n\u001b[0;32m---> 26\u001b[0m \u001b[43mSnow\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mSWE_Predict\u001b[49m\u001b[43m(\u001b[49m\u001b[43mNewSim\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mCorrections\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfSCA\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mfSCA\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/SWEML/Model/Neural_Network/../shared_scripts/NSM_SCA.py:257\u001b[0m, in \u001b[0;36mNSM_SCA.SWE_Predict\u001b[0;34m(self, SCA, NewSim, modelname, Corrections, fSCA)\u001b[0m\n\u001b[1;32m 254\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mSWE_Predict\u001b[39m(\u001b[38;5;28mself\u001b[39m, SCA\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, NewSim \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m, modelname \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mNeural_Network\u001b[39m\u001b[38;5;124m'\u001b[39m, Corrections \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m, fSCA \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m 255\u001b[0m \u001b[38;5;66;03m# load first SWE observation forecasting dataset with prev and delta swe for observations.\u001b[39;00m\n\u001b[1;32m 256\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodelname \u001b[38;5;241m=\u001b[39m modelname\n\u001b[0;32m--> 257\u001b[0m path \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./Predictions/Hold_Out_Year/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfrequency\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/fSCA_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfSCA\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/Prediction_DF_SCA_\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdate\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.pkl\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m NewSim \u001b[38;5;241m==\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m:\n\u001b[1;32m 260\u001b[0m \n\u001b[1;32m 261\u001b[0m \u001b[38;5;66;03m#set up predictions for next timestep\u001b[39;00m\n\u001b[1;32m 262\u001b[0m fdate \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mto_datetime(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdate)\u001b[38;5;241m+\u001b[39mtimedelta(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdelta)\n",
"\u001b[0;31mAttributeError\u001b[0m: 'NSM_SCA' object has no attribute 'fSCA'"
]
}
],
"outputs": [],
"source": [
"#run the model through all time (data acqusition already completed)\n",
"model = 'Neural_Network'\n",
"frequency = 'Daily'\n",
"threshold = 10\n",
"fSCA = True\n",
"NewSim = True\n",
"\n",
"for day in tqdm(datelist):\n",
" #print('Updating SWE predictions for ', day)\n",
" #connect interactive script to Wasatch Snow module\n",
" Snow = NSM_SCA.NSM_SCA(day, threshold=threshold, Regions = Region_list, modelname = model, frequency = frequency, fSCA = fSCA)\n",
" Snow = NSM_SCA.NSM_SCA(day, threshold=threshold, Regions = Region_list, modelname = model, frequency = frequency, fSCA = fSCA, NewSim = NewSim)\n",
" \n",
" #Go get SNOTEL observations - all data currently loaded, set to True to download\n",
" #Snow.Get_Monitoring_Data_Threaded(getdata = True)\n",
"\n",
" #Initialize/Download the granules, all data preprocessed for the SSM activRegion_listhange to True to use the functions.\n",
" # Snow.initializeGranules(getdata = True)\n",
" #Snow.initializeGranules(getdata = True)\n",
"\n",
" #Process observations into Model prediction ready format,\n",
" #Snow.Data_Processing(SCA = True)\n",
Expand All @@ -199,7 +156,7 @@
" \n",
" #Make predictions, set NewSim to False\n",
" #Look to multiprocess, each region can do a prediction to speed things up. set NewSim to true for New simulation, turn to false once all data has been proces and saved.\n",
" Snow.SWE_Predict(NewSim = False, Corrections = False, fSCA = fSCA)\n"
" Snow.SWE_Predict(NewSim = NewSim, Corrections = False, fSCA = fSCA)\n"
]
},
{
Expand All @@ -216,35 +173,12 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": null,
"id": "0b64a8b9-47ac-45ef-885f-98339b0983ad",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
" 0% | |\r"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"['submission_format.h5', '2019_predictions.h5']\n",
"Pushing files to AWS\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100% |########################################################################|\n"
]
}
],
"outputs": [],
"source": [
"modelname= 'Neural_Network'\n",
"folderpath = 'Predictions/Hold_Out_Year/Daily/fSCA_True/'\n",
Expand All @@ -266,7 +200,7 @@
"kernelspec": {
"display_name": "SWEML_env",
"language": "python",
"name": "sweml_env"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
Expand Down
Loading
Loading