Skip to content

Commit

Permalink
cache latitude/longitude as secret
Browse files Browse the repository at this point in the history
  • Loading branch information
@jimdowling
jimdowling committed May 6, 2024
1 parent 5e43ff3 commit ebeff87
Show file tree
Hide file tree
Showing 5 changed files with 182 additions and 176 deletions.
98 changes: 51 additions & 47 deletions notebooks/ch03/1_air_quality_feature_backfill.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,10 @@
"Connected. Call `.close()` to terminate connection gracefully.\n",
"Connected. Call `.close()` to terminate connection gracefully.\n",
"No air_quality_fv feature view found\n",
"No air_quality feature group found\n",
"Deleted air_quality/1\n",
"No weather feature group found\n",
"No aq_monitoring feature group found\n",
"Deleted model air_quality_xgboost_model/1\n"
"No air_quality_xgboost_model model found\n"
]
}
],
Expand Down Expand Up @@ -213,6 +213,8 @@
"# TODO: Change this path to point to the CSV file you downloaded for your sensor\n",
"csv_file=\"../../data/hells-kitchen.csv\"\n",
"\n",
"latitude, longitude = util.get_city_coordinates(city)\n",
"\n",
"today = datetime.date.today()"
]
},
Expand Down Expand Up @@ -813,7 +815,7 @@
"earliest_aq_date = earliest_aq_date.strftime('%Y-%m-%d')\n",
"earliest_aq_date\n",
"\n",
"weather_df = util.get_historical_weather(city, earliest_aq_date, str(today))"
"weather_df = util.get_historical_weather(city, earliest_aq_date, str(today), latitude, longitude)"
]
},
{
Expand All @@ -827,18 +829,18 @@
"output_type": "stream",
"text": [
"<class 'pandas.core.frame.DataFrame'>\n",
"Index: 1757 entries, 0 to 1756\n",
"Index: 1759 entries, 0 to 1758\n",
"Data columns (total 6 columns):\n",
" # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n",
" 0 date 1757 non-null datetime64[ns]\n",
" 1 temperature_2m_mean 1757 non-null float32 \n",
" 2 precipitation_sum 1757 non-null float32 \n",
" 3 wind_speed_10m_max 1757 non-null float32 \n",
" 4 wind_direction_10m_dominant 1757 non-null float32 \n",
" 5 city 1757 non-null object \n",
" 0 date 1759 non-null datetime64[ns]\n",
" 1 temperature_2m_mean 1759 non-null float32 \n",
" 2 precipitation_sum 1759 non-null float32 \n",
" 3 wind_speed_10m_max 1759 non-null float32 \n",
" 4 wind_direction_10m_dominant 1759 non-null float32 \n",
" 5 city 1759 non-null object \n",
"dtypes: datetime64[ns](1), float32(4), object(1)\n",
"memory usage: 68.6+ KB\n"
"memory usage: 68.7+ KB\n"
]
}
],
Expand Down Expand Up @@ -1027,11 +1029,11 @@
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>25.0</td>\n",
" <td>92.0</td>\n",
" <td>usa</td>\n",
" <td>new york</td>\n",
" <td>51st and 8th</td>\n",
" <td>2024-05-04</td>\n",
" <td>2024-05-06</td>\n",
" <td>https://api.waqi.info/feed/A36655</td>\n",
" </tr>\n",
" </tbody>\n",
Expand All @@ -1040,7 +1042,7 @@
],
"text/plain": [
" pm25 country city street date \\\n",
"0 25.0 usa new york 51st and 8th 2024-05-04 \n",
"0 92.0 usa new york 51st and 8th 2024-05-06 \n",
"\n",
" url \n",
"0 https://api.waqi.info/feed/A36655 "
Expand Down Expand Up @@ -1083,7 +1085,7 @@
"output_type": "stream",
"text": [
"SENSOR_LOCATION_JSON already exists. If you want to update it, delete the secret in the Hopworks UI and re-run.\n",
"{\"country\": \"usa\", \"city\": \"new york\", \"street\": \"51st and 8th\", \"aqicn_url\": \"https://api.waqi.info/feed/A36655\"}\n"
"{\"country\": \"usa\", \"city\": \"new york\", \"street\": \"51st and 8th\", \"aqicn_url\": \"https://api.waqi.info/feed/A36655\", \"latitude\": 40.71, \"longitude\": -74.01}\n"
]
}
],
Expand All @@ -1092,7 +1094,9 @@
" \"country\": country,\n",
" \"city\": city,\n",
" \"street\": street,\n",
" \"aqicn_url\": aqicn_url\n",
" \"aqicn_url\": aqicn_url,\n",
" \"latitude\": latitude,\n",
" \"longitude\": longitude\n",
"}\n",
"\n",
"# Convert the dictionary to a JSON string\n",
Expand Down Expand Up @@ -1166,16 +1170,16 @@
"output_type": "stream",
"text": [
"Feature Group created successfully, explore it at \n",
"https://c.app.hopsworks.ai:443/p/398/fs/335/fg/780658\n",
"2024-05-04 20:20:52,191 INFO: \t1 expectation(s) included in expectation_suite.\n",
"https://c.app.hopsworks.ai:443/p/398/fs/335/fg/786794\n",
"2024-05-06 17:53:24,649 INFO: \t1 expectation(s) included in expectation_suite.\n",
"Validation succeeded.\n",
"Validation Report saved successfully, explore a summary at https://c.app.hopsworks.ai:443/p/398/fs/335/fg/780658\n"
"Validation Report saved successfully, explore a summary at https://c.app.hopsworks.ai:443/p/398/fs/335/fg/786794\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "8153ba779cf04ef486c8ef6237e7e7bd",
"model_id": "2b220f198c014544b16ad7f148100f25",
"version_major": 2,
"version_minor": 0
},
Expand All @@ -1198,7 +1202,7 @@
{
"data": {
"text/plain": [
"(<hsfs.core.job.Job at 0x7fac74e4f370>,\n",
"(<hsfs.core.job.Job at 0x7f0577fb4670>,\n",
" {\n",
" \"success\": true,\n",
" \"results\": [\n",
Expand All @@ -1213,7 +1217,7 @@
" \"strict_min\": true\n",
" },\n",
" \"meta\": {\n",
" \"expectationId\": 489497\n",
" \"expectationId\": 491522\n",
" }\n",
" },\n",
" \"result\": {\n",
Expand All @@ -1224,7 +1228,7 @@
" },\n",
" \"meta\": {\n",
" \"ingestionResult\": \"INGESTED\",\n",
" \"validationTime\": \"2024-05-04T06:20:52.000191Z\"\n",
" \"validationTime\": \"2024-05-06T03:53:24.000648Z\"\n",
" },\n",
" \"exception_info\": {\n",
" \"raised_exception\": false,\n",
Expand All @@ -1245,14 +1249,14 @@
" \"expectation_suite_name\": \"aq_expectation_suite\",\n",
" \"run_id\": {\n",
" \"run_name\": null,\n",
" \"run_time\": \"2024-05-04T20:20:52.191787+02:00\"\n",
" \"run_time\": \"2024-05-06T17:53:24.648790+02:00\"\n",
" },\n",
" \"batch_kwargs\": {\n",
" \"ge_batch_id\": \"0a320c92-0a43-11ef-9455-00155d85a383\"\n",
" \"ge_batch_id\": \"c5798ed4-0bc0-11ef-937c-00155ddc5d7c\"\n",
" },\n",
" \"batch_markers\": {},\n",
" \"batch_parameters\": {},\n",
" \"validation_time\": \"20240504T182052.191726Z\",\n",
" \"validation_time\": \"20240506T155324.648672Z\",\n",
" \"expectation_suite_meta\": {\n",
" \"great_expectations_version\": \"0.18.12\"\n",
" }\n",
Expand Down Expand Up @@ -1286,7 +1290,7 @@
{
"data": {
"text/plain": [
"<hsfs.feature_group.FeatureGroup at 0x7fabe388a380>"
"<hsfs.feature_group.FeatureGroup at 0x7f0577f8c160>"
]
},
"execution_count": 17,
Expand Down Expand Up @@ -1354,21 +1358,21 @@
"output_type": "stream",
"text": [
"Feature Group created successfully, explore it at \n",
"https://c.app.hopsworks.ai:443/p/398/fs/335/fg/780659\n",
"2024-05-04 20:21:05,915 INFO: \t2 expectation(s) included in expectation_suite.\n",
"https://c.app.hopsworks.ai:443/p/398/fs/335/fg/786795\n",
"2024-05-06 17:53:37,708 INFO: \t2 expectation(s) included in expectation_suite.\n",
"Validation succeeded.\n",
"Validation Report saved successfully, explore a summary at https://c.app.hopsworks.ai:443/p/398/fs/335/fg/780659\n"
"Validation Report saved successfully, explore a summary at https://c.app.hopsworks.ai:443/p/398/fs/335/fg/786795\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "1bb7464fd5e24fd697729fa70c235f8c",
"model_id": "bd5a18e1e81e443084e55c6f89ae1278",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Uploading Dataframe: 0.00% | | Rows 0/1757 | Elapsed Time: 00:00 | Remaining Time: ?"
"Uploading Dataframe: 0.00% | | Rows 0/1759 | Elapsed Time: 00:00 | Remaining Time: ?"
]
},
"metadata": {},
Expand All @@ -1386,7 +1390,7 @@
{
"data": {
"text/plain": [
"(<hsfs.core.job.Job at 0x7fabe38f40a0>,\n",
"(<hsfs.core.job.Job at 0x7f0577f8d2a0>,\n",
" {\n",
" \"success\": true,\n",
" \"results\": [\n",
Expand All @@ -1395,24 +1399,24 @@
" \"expectation_config\": {\n",
" \"expectation_type\": \"expect_column_min_to_be_between\",\n",
" \"kwargs\": {\n",
" \"column\": \"precipitation_sum\",\n",
" \"column\": \"wind_speed_10m_max\",\n",
" \"min_value\": -0.1,\n",
" \"max_value\": 1000.0,\n",
" \"strict_min\": true\n",
" },\n",
" \"meta\": {\n",
" \"expectationId\": 489499\n",
" \"expectationId\": 491523\n",
" }\n",
" },\n",
" \"result\": {\n",
" \"observed_value\": 0.0,\n",
" \"element_count\": 1757,\n",
" \"observed_value\": 7.091177463531494,\n",
" \"element_count\": 1759,\n",
" \"missing_count\": null,\n",
" \"missing_percent\": null\n",
" },\n",
" \"meta\": {\n",
" \"ingestionResult\": \"INGESTED\",\n",
" \"validationTime\": \"2024-05-04T06:21:05.000915Z\"\n",
" \"validationTime\": \"2024-05-06T03:53:37.000707Z\"\n",
" },\n",
" \"exception_info\": {\n",
" \"raised_exception\": false,\n",
Expand All @@ -1425,24 +1429,24 @@
" \"expectation_config\": {\n",
" \"expectation_type\": \"expect_column_min_to_be_between\",\n",
" \"kwargs\": {\n",
" \"column\": \"wind_speed_10m_max\",\n",
" \"column\": \"precipitation_sum\",\n",
" \"min_value\": -0.1,\n",
" \"max_value\": 1000.0,\n",
" \"strict_min\": true\n",
" },\n",
" \"meta\": {\n",
" \"expectationId\": 489498\n",
" \"expectationId\": 491524\n",
" }\n",
" },\n",
" \"result\": {\n",
" \"observed_value\": 7.091177463531494,\n",
" \"element_count\": 1757,\n",
" \"observed_value\": 0.0,\n",
" \"element_count\": 1759,\n",
" \"missing_count\": null,\n",
" \"missing_percent\": null\n",
" },\n",
" \"meta\": {\n",
" \"ingestionResult\": \"INGESTED\",\n",
" \"validationTime\": \"2024-05-04T06:21:05.000915Z\"\n",
" \"validationTime\": \"2024-05-06T03:53:37.000707Z\"\n",
" },\n",
" \"exception_info\": {\n",
" \"raised_exception\": false,\n",
Expand All @@ -1463,14 +1467,14 @@
" \"expectation_suite_name\": \"weather_expectation_suite\",\n",
" \"run_id\": {\n",
" \"run_name\": null,\n",
" \"run_time\": \"2024-05-04T20:21:05.915532+02:00\"\n",
" \"run_time\": \"2024-05-06T17:53:37.707862+02:00\"\n",
" },\n",
" \"batch_kwargs\": {\n",
" \"ge_batch_id\": \"12600ca2-0a43-11ef-9455-00155d85a383\"\n",
" \"ge_batch_id\": \"cd422a9a-0bc0-11ef-937c-00155ddc5d7c\"\n",
" },\n",
" \"batch_markers\": {},\n",
" \"batch_parameters\": {},\n",
" \"validation_time\": \"20240504T182105.915429Z\",\n",
" \"validation_time\": \"20240506T155337.707750Z\",\n",
" \"expectation_suite_meta\": {\n",
" \"great_expectations_version\": \"0.18.12\"\n",
" }\n",
Expand Down Expand Up @@ -1505,7 +1509,7 @@
{
"data": {
"text/plain": [
"<hsfs.feature_group.FeatureGroup at 0x7fabe388b550>"
"<hsfs.feature_group.FeatureGroup at 0x7f0574e2f070>"
]
},
"execution_count": 20,
Expand Down
Loading

0 comments on commit ebeff87

Please sign in to comment.