Skip to content

Commit

Permalink
4.0 release
Browse files Browse the repository at this point in the history
  • Loading branch information
@Maxxx-zh
Maxxx-zh authored Dec 6, 2024
1 parent 724817d commit 64d29dd
Show file tree
Hide file tree
Showing 39 changed files with 2,435 additions and 1,268 deletions.
4 changes: 2 additions & 2 deletions advanced_tutorials/air_quality/1_air_quality_feature_backfill.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1131,7 +1131,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
Expand All @@ -1145,7 +1145,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.11"
"version": "3.8.18"
}
},
"nbformat": 4,
Expand Down
20 changes: 10 additions & 10 deletions advanced_tutorials/aml/2_aml_training_pipeline.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,16 +151,16 @@
"min_max_scaler = fs.get_transformation_function(name=\"min_max_scaler\")\n",
"\n",
"# Map features to transformations.\n",
"transformation_functions = {\n",
" \"monthly_in_count\": min_max_scaler,\n",
" \"monthly_in_total_amount\": min_max_scaler,\n",
" \"monthly_in_mean_amount\": min_max_scaler,\n",
" \"monthly_in_std_amount\": min_max_scaler,\n",
" \"monthly_out_count\": min_max_scaler,\n",
" \"monthly_out_total_amount\": min_max_scaler,\n",
" \"monthly_out_mean_amount\": min_max_scaler,\n",
" \"monthly_out_std_amount\": min_max_scaler,\n",
"}"
"transformation_functions = [\n",
" min_max_scaler(\"monthly_in_count\"),\n",
" min_max_scaler(\"monthly_in_total_amount\"),\n",
" min_max_scaler(\"monthly_in_mean_amount\"),\n",
" min_max_scaler(\"monthly_in_std_amount\"),\n",
" min_max_scaler(\"monthly_out_count\"),\n",
" min_max_scaler(\"monthly_out_total_amount\"),\n",
" min_max_scaler(\"monthly_out_mean_amount\"),\n",
" min_max_scaler(\"monthly_out_std_amount\"),\n",
"]"
]
},
{
Expand Down
28 changes: 14 additions & 14 deletions advanced_tutorials/bitcoin/3_bitcoin_training_pipeline.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@
"min_max_scaler = fs.get_transformation_function(name=\"min_max_scaler\")\n",
"\n",
"# Map features to transformation functions.\n",
"transformation_functions = {col: min_max_scaler for col in columns_to_transform}"
"transformation_functions = [min_max_scaler(col) for col in columns_to_transform]"
]
},
{
Expand Down Expand Up @@ -522,22 +522,22 @@
"metadata": {},
"outputs": [],
"source": [
"import inspect \n",
"# Recall that you applied transformation functions, such as min max scaler and laber encoder. \n",
"# Now you want to transform them back to human readable format.\n",
"# Initializing serving\n",
"feature_view.init_serving(1)\n",
"td_transformation_functions = feature_view._single_vector_server._transformation_functions\n",
"\n",
"# Accessing the transformation functions used in the serving configuration\n",
"fv_transformation_functions = feature_view._vector_server.model_dependent_transformation_functions\n",
"\n",
"y_pred = pd.DataFrame(y_pred_scaled, columns=[\"close\"])\n",
"\n",
"for feature_name in td_transformation_functions:\n",
" if feature_name == \"close\":\n",
" td_transformation_function = td_transformation_functions[feature_name]\n",
" sig, foobar_locals = inspect.signature(td_transformation_function.transformation_fn), locals()\n",
" param_dict = dict([(param.name, param.default) for param in sig.parameters.values() if param.default != inspect._empty])\n",
" if td_transformation_function.name == \"min_max_scaler\":\n",
" y_pred[feature_name] = y_pred[feature_name].map(lambda x: x*(param_dict[\"max_value\"]-param_dict[\"min_value\"])+param_dict[\"min_value\"])\n",
" y_test[feature_name] = y_test[feature_name].map(lambda x: x*(param_dict[\"max_value\"]-param_dict[\"min_value\"])+param_dict[\"min_value\"])"
"for transformation_function in fv_transformation_functions:\n",
" udf = transformation_function.hopsworks_udf\n",
" transformation_feature = udf.transformation_features[0]\n",
" transformed_feature = udf.transformation_features[0]\n",
" if transformed_feature == \"close\" and udf.function_name == \"min_max_scaler\":\n",
" stats = udf.transformation_statistics\n",
" y_pred[transformation_feature] = y_pred[transformation_feature].map(lambda x: x*(stats.feature.max-stats.feature.min)+stats.feature.min)\n",
" y_test[transformed_feature] = y_test[transformed_feature].map(lambda x: x*(stats.feature.max-stats.feature.min)+stats.feature.min)"
]
},
{
Expand Down Expand Up @@ -996,7 +996,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.12"
"version": "3.12.5"
},
"vscode": {
"interpreter": {
Expand Down
8 changes: 4 additions & 4 deletions advanced_tutorials/credit_scores/3_credit_scores_training_pipeline.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@
"outputs": [],
"source": [
"# Retrieving the names of all available transformation functions\n",
"[t_func.name for t_func in fs.get_transformation_functions()]"
"[t_func for t_func in fs.get_transformation_functions()]"
]
},
{
Expand Down Expand Up @@ -290,11 +290,11 @@
"label_encoder = fs.get_transformation_function(name='label_encoder') \n",
"\n",
"# Creating a dictionary of transformation functions, where each categorical column is associated with the Label Encoder\n",
"transformation_functions = {\n",
" col: label_encoder\n",
"transformation_functions = [\n",
" label_encoder(col)\n",
" for col \n",
" in cat_cols\n",
"}"
"]"
]
},
{
Expand Down
34 changes: 19 additions & 15 deletions advanced_tutorials/electricity/3_electricity_training_pipeline.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,21 +163,25 @@
"# List of price areas\n",
"price_areas = [\"se1\", \"se2\", \"se3\", \"se4\"]\n",
"\n",
"# Retrieving transformation functions\n",
"min_max_scaler = fs.get_transformation_function(name=\"min_max_scaler\")\n",
"label_encoder = fs.get_transformation_function(name='label_encoder')\n",
"\n",
"# Mapping features to their respective transformation functions\n",
"mapping_transformers = {}\n",
"transformation_functions = []\n",
"\n",
"# Iterate through each price area and map features to their transformation functions\n",
"for area in price_areas:\n",
" mapping_transformers[f\"price_{area}\"] = fs.get_transformation_function(name=\"min_max_scaler\")\n",
" mapping_transformers[f\"mean_temp_per_day_{area}\"] = fs.get_transformation_function(name=\"min_max_scaler\")\n",
" mapping_transformers[f\"mean_wind_speed_{area}\"] = fs.get_transformation_function(name=\"min_max_scaler\")\n",
" mapping_transformers[f\"precipitaton_amount_{area}\"] = fs.get_transformation_function(name=\"min_max_scaler\")\n",
" mapping_transformers[f\"total_sunshine_time_{area}\"] = fs.get_transformation_function(name=\"min_max_scaler\")\n",
" mapping_transformers[f\"mean_cloud_perc_{area}\"] = fs.get_transformation_function(name=\"min_max_scaler\") \n",
" mapping_transformers[f\"precipitaton_type_{area}\"] = fs.get_transformation_function(name='label_encoder')\n",
" transformation_functions.append(min_max_scaler(f\"price_{area}\"))\n",
" transformation_functions.append(min_max_scaler(f\"mean_temp_per_day_{area}\"))\n",
" transformation_functions.append(min_max_scaler(f\"mean_wind_speed_{area}\"))\n",
" transformation_functions.append(min_max_scaler(f\"precipitaton_amount_{area}\"))\n",
" transformation_functions.append(min_max_scaler(f\"total_sunshine_time_{area}\"))\n",
" transformation_functions.append(min_max_scaler(f\"mean_cloud_perc_{area}\"))\n",
" transformation_functions.append(label_encoder(f\"precipitaton_type_{area}\"))\n",
"\n",
"# Additional transformation for 'type_of_day'\n",
"mapping_transformers[\"type_of_day\"] = fs.get_transformation_function(name='label_encoder')"
"transformation_functions.append(label_encoder(\"type_of_day\"))"
]
},
{
Expand Down Expand Up @@ -215,7 +219,7 @@
" name='electricity_feature_view',\n",
" version=1,\n",
" labels=[], # you will define our 'y' later manualy\n",
" transformation_functions=mapping_transformers,\n",
" transformation_functions=transformation_functions,\n",
" query=selected_features,\n",
")"
]
Expand Down Expand Up @@ -304,9 +308,9 @@
"outputs": [],
"source": [
"# Define 'y_train', 'y_val' and 'y_test'\n",
"y_train = X_train[[\"price_se1\", \"price_se2\", \"price_se3\", \"price_se4\"]]\n",
"y_val = X_val[[\"price_se1\", \"price_se2\", \"price_se3\", \"price_se4\"]]\n",
"y_test = X_test[[\"price_se1\", \"price_se2\", \"price_se3\", \"price_se4\"]]"
"y_train = X_train[[\"min_max_scaler_price_se1_\", \"min_max_scaler_price_se2_\", \"min_max_scaler_price_se3_\", \"min_max_scaler_price_se4_\"]]\n",
"y_val = X_val[[\"min_max_scaler_price_se1_\", \"min_max_scaler_price_se2_\", \"min_max_scaler_price_se3_\", \"min_max_scaler_price_se4_\"]]\n",
"y_test = X_test[[\"min_max_scaler_price_se1_\", \"min_max_scaler_price_se2_\", \"min_max_scaler_price_se3_\", \"min_max_scaler_price_se4_\"]]"
]
},
{
Expand Down Expand Up @@ -494,7 +498,7 @@
" input_width=4, \n",
" label_width=4, \n",
" shift=1, \n",
" label_columns=[\"price_se1\", \"price_se2\", \"price_se3\", \"price_se4\"],\n",
" label_columns=[\"min_max_scaler_price_se1_\", \"min_max_scaler_price_se2_\", \"min_max_scaler_price_se3_\", \"min_max_scaler_price_se4_\"],\n",
")\n",
"\n",
"# Displaying the WindowGenerator instance\n",
Expand Down Expand Up @@ -710,7 +714,7 @@
"source": [
"# Plotting the time series data for the 'price_se4' column\n",
"n_step_window.plot(\n",
" plot_col=\"price_se4\", \n",
" plot_col=\"min_max_scaler_price_se4_\", \n",
" max_subplots=3, \n",
" model=model.predict,\n",
")"
Expand Down
19 changes: 8 additions & 11 deletions advanced_tutorials/electricity/4_electricity_batch_inference.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@
"feature_view.init_serving(1)\n",
"\n",
"# Accessing the transformation functions used in the serving configuration\n",
"td_transformation_functions = feature_view._batch_scoring_server._transformation_functions"
"fv_transformation_functions = feature_view._vector_server.model_dependent_transformation_functions"
]
},
{
Expand All @@ -268,15 +268,12 @@
"\n",
"# Extracting and decoding the transformation functions used in serving\n",
"res = {}\n",
"for feature_name in td_transformation_functions:\n",
" if feature_name in [\"price_se1\", \"price_se2\", \"price_se3\", \"price_se4\"]:\n",
" td_transformation_function = td_transformation_functions[feature_name]\n",
" sig, foobar_locals = inspect.signature(td_transformation_function.transformation_fn), locals()\n",
" param_dict = dict([(param.name, param.default) for param in sig.parameters.values() if param.default != inspect._empty])\n",
" if td_transformation_function.name == \"min_max_scaler\":\n",
" preds[feature_name] = preds[feature_name].map(\n",
" lambda x: x * (param_dict[\"max_value\"] - param_dict[\"min_value\"]) + param_dict[\"min_value\"]\n",
" )\n",
"for transformation_function in fv_transformation_functions:\n",
" udf = transformation_function.hopsworks_udf\n",
" transformed_features = udf.transformation_features[0]\n",
" if transformed_features in [\"price_se1\", \"price_se2\", \"price_se3\", \"price_se4\"] and udf.function_name == \"min_max_scaler\":\n",
" stats = udf.transformation_statistics\n",
" preds[transformed_features] = preds[transformed_features].map(lambda x: x*(stats.feature.max-stats.feature.min)+stats.feature.min)\n",
"\n",
"# Applying a transformation to reverse the sign of the decoded features\n",
"preds = preds.apply(lambda x: -x)\n",
Expand Down Expand Up @@ -336,7 +333,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.18"
"version": "3.12.5"
}
},
"nbformat": 4,
Expand Down
18 changes: 9 additions & 9 deletions advanced_tutorials/hospital_wait_time/2_training_pipeline.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@
"metadata": {},
"outputs": [],
"source": [
"[f.name for f in fs.get_transformation_functions()]"
"[f for f in fs.get_transformation_functions()]"
]
},
{
Expand All @@ -153,11 +153,11 @@
"source": [
"features_category = ['gender', 'age_cat', 'blood_gp', 'underlying_disease', 'gestation', 'prior_transplant', 'if_transplanted']\n",
"\n",
"transformation_functions_category = {\n",
" feature_name: label_encoder\n",
"transformation_functions_category = [\n",
" label_encoder(feature_name)\n",
" for feature_name\n",
" in features_category\n",
"}"
"]"
]
},
{
Expand All @@ -171,11 +171,11 @@
" 'age_at_list_registration', 'dialysis_duration', 'number_prior_transplant', 'cpra', 'hla_a1', 'hla_a2', 'hla_b1', 'hla_b2', 'hla_dr1', 'hla_dr2',\n",
"]\n",
"\n",
"transformation_functions_numerical = {\n",
" feature_name: standard_scaler\n",
"transformation_functions_numerical = [\n",
" standard_scaler(feature_name)\n",
" for feature_name\n",
" in features_numerical\n",
"}"
"]"
]
},
{
Expand All @@ -185,8 +185,8 @@
"metadata": {},
"outputs": [],
"source": [
"# Join transformation_functions_category and transformation_functions_numerical dictionaries into one\n",
"transformation_functions = transformation_functions_category | transformation_functions_numerical"
"# Join transformation_functions_category and transformation_functions_numerical lists into one\n",
"transformation_functions = transformation_functions_category + transformation_functions_numerical"
]
},
{
Expand Down
6 changes: 3 additions & 3 deletions advanced_tutorials/on_demand_feature/notebooks/3_feature_view_td_modelling.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,9 @@
"label_encoder = fs.get_transformation_function(name=\"label_encoder\")\n",
"\n",
"# Map features to transformation functions.\n",
"transformation_functions = {\n",
" \"ocean_proximity\": label_encoder,\n",
"}"
"transformation_functions = [\n",
" label_encoder(\"ocean_proximity\")\n",
"]"
]
},
{
Expand Down
Loading

0 comments on commit 64d29dd

Please sign in to comment.