From bb4e81a4541fe724910348f83b9bb09a5abd5093 Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Mon, 8 May 2023 13:00:07 -0700 Subject: [PATCH 01/18] Update dev_fortunatowheels-dev.yml --- .github/workflows/dev_fortunatowheels-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev_fortunatowheels-dev.yml b/.github/workflows/dev_fortunatowheels-dev.yml index dfcddcf..8569f69 100644 --- a/.github/workflows/dev_fortunatowheels-dev.yml +++ b/.github/workflows/dev_fortunatowheels-dev.yml @@ -2,7 +2,7 @@ # More GitHub Actions for Azure: https://github.com/Azure/actions # More info on Python, GitHub Actions, and Azure App Service: https://aka.ms/python-webapps-actions -name: Build and deploy Python app to Azure Web App - fortunatowheels-dev +name: Dev: Build/Deploy to Azure - fortunatowheels-dev on: push: From 4c9e4f73c933153af0c00bbcf029f6131f2c1a74 Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Mon, 8 May 2023 13:01:49 -0700 Subject: [PATCH 02/18] Update dev_fortunatowheels-dev.yml --- .github/workflows/dev_fortunatowheels-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev_fortunatowheels-dev.yml b/.github/workflows/dev_fortunatowheels-dev.yml index 8569f69..05b4924 100644 --- a/.github/workflows/dev_fortunatowheels-dev.yml +++ b/.github/workflows/dev_fortunatowheels-dev.yml @@ -2,7 +2,7 @@ # More GitHub Actions for Azure: https://github.com/Azure/actions # More info on Python, GitHub Actions, and Azure App Service: https://aka.ms/python-webapps-actions -name: Dev: Build/Deploy to Azure - fortunatowheels-dev +name: Dev-Build/Deploy to Azure-fortunatowheels-dev on: push: From 49d69bfaadda30530df85cfc1414e1628a4cd82b Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Mon, 8 May 2023 13:05:40 -0700 Subject: [PATCH 03/18] Update main_fortunatowheels.yml --- .github/workflows/main_fortunatowheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main_fortunatowheels.yml b/.github/workflows/main_fortunatowheels.yml index 8dbfd5e..45ac91d 100644 --- a/.github/workflows/main_fortunatowheels.yml +++ b/.github/workflows/main_fortunatowheels.yml @@ -2,7 +2,7 @@ # More GitHub Actions for Azure: https://github.com/Azure/actions # More info on Python, GitHub Actions, and Azure App Service: https://aka.ms/python-webapps-actions -name: Build and deploy Python app to Azure Web App - fortunatowheels +name: Prod - Build/Deploy to Azure-fortunatowheels on: push: From 1eb4ecbf91746bcd8157cecbe6207f21f13826bb Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Mon, 8 May 2023 13:06:24 -0700 Subject: [PATCH 04/18] Update dev_fortunatowheels-dev.yml --- .github/workflows/dev_fortunatowheels-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev_fortunatowheels-dev.yml b/.github/workflows/dev_fortunatowheels-dev.yml index 05b4924..3ad9fe7 100644 --- a/.github/workflows/dev_fortunatowheels-dev.yml +++ b/.github/workflows/dev_fortunatowheels-dev.yml @@ -2,7 +2,7 @@ # More GitHub Actions for Azure: https://github.com/Azure/actions # More info on Python, GitHub Actions, and Azure App Service: https://aka.ms/python-webapps-actions -name: Dev-Build/Deploy to Azure-fortunatowheels-dev +name: Dev - Build/Deploy to Azure-fortunatowheels-dev on: push: From 3890a453fa64fba89187b56ad0abfbcee35ebcc8 Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 11 May 2023 02:04:11 -0700 Subject: [PATCH 05/18] Revert "Basic Custom Google Analytics Logging Setup" --- src/analytics/google_analytics.py | 87 ------------------------------- src/app.py | 44 ++-------------- src/pages/explore_ads.py | 75 +++----------------------- 3 files changed, 10 insertions(+), 196 deletions(-) delete mode 100644 src/analytics/google_analytics.py diff --git a/src/analytics/google_analytics.py b/src/analytics/google_analytics.py deleted file mode 100644 index 319c0ff..0000000 --- a/src/analytics/google_analytics.py +++ /dev/null @@ -1,87 +0,0 @@ -# Author: Ty Andrews -# Date: 2023-05-11 - -import os, sys - -import os, sys -import requests -import json -from dotenv import load_dotenv, find_dotenv -import time - -from src.logs import get_logger - -logger = get_logger(__name__) - -load_dotenv(find_dotenv()) - -GA4_CLIENT_SECRET = os.getenv("GA4_CLIENT_SECRET") -GA4_MEASUREMENT_ID = "G-CKDC8LRCRB" - - -def log_to_GA_list_of_items(event_name: str, item_name: str, list_of_items: list): - """Sends a custom event to google analytics 4 - - Parameters - ---------- - list_of_items : list - list of items to be logged - list_name : str - name of the list to be logged - """ - success = True - client_id = get_ga4_client_id() - try: - for item in list_of_items: - event_params = {item_name: item} - custom_event_to_GA(client_id, event_name, event_params) - except Exception as e: - logger.error(f"Failed to log {event_name} to GA4: {e}") - success = False - - return success - - -def custom_event_to_GA(client_id: str, event_name: str, event_params: dict): - """Sends a custom event to google analytics 4 - - Parameters - ---------- - event_name : str - name of the event to be sent to google analytics 4 - event_params : dict - dictionary of event parameters to be sent to google analytics 4, must - contain only one instance of unique keys but can contain multiple key - value pairs. - - Raises - ------ - ValueError - if event_params does not contain only one key value pair - """ - - # placeholder until access to live gtag clinet id is setup - api_secret = GA4_CLIENT_SECRET - measurement_id = GA4_MEASUREMENT_ID - - url = f"https://www.google-analytics.com/mp/collect?measurement_id={measurement_id}&api_secret={api_secret}" - - payload = { - "client_id": client_id, - "non_personalized_ads": False, - "events": [{"name": event_name, "params": event_params}], - } - - r = requests.post(url, data=json.dumps(payload), verify=True) - - -def get_ga4_client_id(): - """Placeholder function to return a client id for google analytics - - Returns - ------- - str - client id for google analytics, currently a timestamp in ns. - """ - - return str(time.time_ns()) diff --git a/src/app.py b/src/app.py index 65d3105..754dd4d 100644 --- a/src/app.py +++ b/src/app.py @@ -10,6 +10,9 @@ from dash_iconify import DashIconify import time +from logs import get_logger +from data.azure_blob_storage import AzureBlob + # on launch ensure src is in path cur_dir = os.getcwd() try: @@ -21,10 +24,6 @@ if SRC_PATH not in sys.path: sys.path.append(SRC_PATH) -from logs import get_logger -from data.azure_blob_storage import AzureBlob -from analytics.google_analytics import custom_event_to_GA - # Create a custom logger logger = get_logger(__name__) @@ -218,58 +217,23 @@ def toggle_navbar_collapse(n, is_open): def load_data(first_load, price_summary, num_ads_summary, mileage_summary): # if any summary is None, then we need to load data if not price_summary or not num_ads_summary or not mileage_summary: - azure_blob = AzureBlob() - - # placeholder until proper gtag id's can be extracted - client_id = str(time.time_ns()) - start_time = time.time() + azure_blob = AzureBlob() ad_price_summary = azure_blob.load_parquet( "processed/avg_price_summary.parquet" ).to_dict() - custom_event_to_GA( - client_id, - "price_data_load_time", - {"time_ms": round((time.time() - start_time) * 1000, 0)}, - ) - logger - - start_time = time.time() mileage_summary = azure_blob.load_parquet( "processed/mileage_distribution_summary.parquet" ).to_dict() - custom_event_to_GA( - client_id, - "mileage_data_load_time", - {"time_ms": round((time.time() - start_time) * 1000, 0)}, - ) - - start_time = time.time() num_ads_summary = azure_blob.load_parquet( "processed/num_ads_summary.parquet" ).to_dict() - custom_event_to_GA( - client_id, - "num_ads_data_load_time", - {"time_ms": round((time.time() - start_time) * 1000, 0)}, - ) - - start_time = time.time() makes_models = azure_blob.load_parquet( "processed/makes_models.parquet" ).to_dict() - custom_event_to_GA( - client_id, - "makes_models_data_load_time", - {"time_ms": round((time.time() - start_time) * 1000, 0)}, - ) - logger.debug( f"Loaded data from Azure Blob Storage in {time.time() - start_time} seconds" ) - - custom_event_to_GA(time.time_ns(), "summary_data_load_time", {}) - return ad_price_summary, num_ads_summary, mileage_summary, makes_models else: logger.debug("Data already loaded, skipping") diff --git a/src/pages/explore_ads.py b/src/pages/explore_ads.py index 20e0f29..ccb1fd7 100644 --- a/src/pages/explore_ads.py +++ b/src/pages/explore_ads.py @@ -17,7 +17,6 @@ import dash_mantine_components as dmc from dash_iconify import DashIconify import pandas as pd -import time from src.visualizations.explore_ads_plots import ( plot_vehicle_prices_summary, @@ -27,11 +26,9 @@ from src.visualizations.utils import blank_placeholder_plot from src.pages.dash_styles import SIDEBAR_STYLE, CONTENT_STYLE from src.logs import get_logger -from src.analytics.google_analytics import log_to_GA_list_of_items INVALID_MODELS = ["other"] DEFAULT_MODELS = ["ghost", "model-x", "911", "m3"] -DEFAULT_MAKES = [] # Create a custom logger logger = get_logger(__name__) @@ -318,8 +315,6 @@ def update_filter_options( List of values of the price slider is set at, if the price range changes based on the selected makes and models, then the price slider is reset to the new range. """ - start_time = time.time() - make_model_df = pd.DataFrame.from_dict(makes_models_store, orient="columns") price_summary_df = pd.DataFrame.from_dict(price_summary_store, orient="columns") num_ads_summary_df = pd.DataFrame.from_dict(num_ads_summary_store, orient="columns") @@ -402,14 +397,6 @@ def update_filter_options( if price_slider_values[1] == price_slider_max: price_slider_values[1] = max_price - log_success = log_to_GA_list_of_items( - event_name="explore_update_filters_time", - item_name="time_ms", - list_of_items=[int((time.time() - start_time) * 1000)], - ) - logger.debug( - f"explore_update_filters_time log success - {log_success}: {int((time.time() - start_time) * 1000)}" - ) return model_options, make_options, max_price, price_slider_values @@ -427,11 +414,12 @@ def update_filter_options( def update_ad_filter_count( age_range, price_range, models, makes, num_ads_summary_dict, makes_models_dict ): - start_time = time.time() - num_ads_summary_df = pd.DataFrame.from_dict(num_ads_summary_dict, orient="columns") makes_models_df = pd.DataFrame.from_dict(makes_models_dict, orient="columns") + if price_range is None: + price_range = [0, vehicles_df.price.max()] + if (len(models) == 0) & (len(makes) == 0): # remove some models that are not really models models = ( @@ -449,6 +437,9 @@ def update_ad_filter_count( if len(makes) == 0: makes = makes_models_df.make.unique() + if age_range is None: + age_range = [vehicles_df.year.min(), vehicles_df.year.max()] + matching_ads = ( num_ads_summary_df.query("age >= @age_range[0] & age <= @age_range[1]")[models] .sum() @@ -469,14 +460,6 @@ def update_ad_filter_count( ), ] ) - log_success = log_to_GA_list_of_items( - event_name="explore_matching_ads_update_time", - item_name="time_ms", - list_of_items=[int((time.time() - start_time) * 1000)], - ) - logger.debug( - f"explore_matching_ads_update_time log success - {log_success}: {int((time.time() - start_time) * 1000)}" - ) return num_matching_entries @@ -496,27 +479,10 @@ def update_ad_filter_count( def update_price_summary_plot( n_clicks, age_range, price_range, models, makes, price_summary, makes_models ): - start_time = time.time() # if no models selected, display the default models if len(models) == 0: models = DEFAULT_MODELS - # log to GA the currently selected models & makes - log_model_success = log_to_GA_list_of_items( - event_name="explore_apply_filters_click", - item_name="model", - # remove default models from list of models - list_of_items=[model for model in models if model not in DEFAULT_MODELS], - ) - logger.debug(f"GA logging success for models: {log_model_success}") - - log_make_success = log_to_GA_list_of_items( - event_name="explore_apply_filters_click", - item_name="make", - list_of_items=[make for make in makes if make not in DEFAULT_MAKES], - ) - logger.debug(f"GA logging success for makes: {log_make_success}") - # convert models to lower case and replace spaces with dashes models = [model.lower().replace(" ", "-") for model in models] @@ -553,15 +519,6 @@ def update_price_summary_plot( price_summary_plot = plot_vehicle_prices_summary(price_summary_df) - log_success = log_to_GA_list_of_items( - event_name="explore_price_age_summary_update_time", - item_name="time_ms", - list_of_items=[int((time.time() - start_time) * 1000)], - ) - logger.debug( - f"explore_price_age_summary_update_time log success - {log_success}: {int((time.time() - start_time) * 1000)}" - ) - return price_summary_plot @@ -580,7 +537,6 @@ def update_price_summary_plot( def update_mileage_summary_plot( n_clicks, year_range, price_range, models, makes, mileage_summary, makes_models ): - start_time = time.time() # if no models selected, display the default models if len(models) == 0: models = DEFAULT_MODELS @@ -624,15 +580,6 @@ def update_mileage_summary_plot( mileage_summary_plot = plot_mileage_distribution_summary(mileage_summary_df) - log_success = log_to_GA_list_of_items( - event_name="explore_mileage_summary_update_time", - item_name="time_ms", - list_of_items=[int((time.time() - start_time) * 1000)], - ) - logger.debug( - f"explore_mileage_summary_update_time log success - {log_success}: {int((time.time() - start_time) * 1000)}" - ) - return mileage_summary_plot @@ -651,7 +598,6 @@ def update_mileage_summary_plot( def update_num_ads_summary_plot( n_clicks, age_range, price_range, models, makes, num_ads_summary, makes_models ): - start_time = time.time() # if no models selected, display the default models if len(models) == 0: models = DEFAULT_MODELS @@ -686,13 +632,4 @@ def update_num_ads_summary_plot( num_ads_summary_plot = plot_num_ads_summary(num_ads_per_model) - log_success = log_to_GA_list_of_items( - event_name="explore_num_ads_summary_update_time", - item_name="time_ms", - list_of_items=[int((time.time() - start_time) * 1000)], - ) - logger.debug( - f"explore_num_ads_summary_update_time log success - {log_success}: {int((time.time() - start_time) * 1000)}" - ) - return num_ads_summary_plot From 128cec27274f57e2e6a0fd713406bf457182efc7 Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 11 May 2023 22:42:08 -0700 Subject: [PATCH 06/18] feat: adding logging makes/models to GA (cherry picked from commit 7422529b5981796d433e72fc270f22585048e099) --- src/pages/explore_ads.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/pages/explore_ads.py b/src/pages/explore_ads.py index ccb1fd7..025968f 100644 --- a/src/pages/explore_ads.py +++ b/src/pages/explore_ads.py @@ -26,9 +26,11 @@ from src.visualizations.utils import blank_placeholder_plot from src.pages.dash_styles import SIDEBAR_STYLE, CONTENT_STYLE from src.logs import get_logger +from src.analytics.google_analytics import log_to_GA_list_of_items INVALID_MODELS = ["other"] DEFAULT_MODELS = ["ghost", "model-x", "911", "m3"] +DEFAULT_MAKES = [] # Create a custom logger logger = get_logger(__name__) @@ -483,6 +485,22 @@ def update_price_summary_plot( if len(models) == 0: models = DEFAULT_MODELS + # log to GA the currently selected models & makes + log_model_success = log_to_GA_list_of_items( + event_name="explore_apply_filters_click", + item_name="model", + # remove default models from list of models + list_of_items=[model for model in models if model not in DEFAULT_MODELS], + ) + logger.debug(f"GA logging success for models: {log_model_success}") + + log_make_success = log_to_GA_list_of_items( + event_name="explore_apply_filters_click", + item_name="make", + list_of_items=[make for make in makes if make not in DEFAULT_MAKES], + ) + logger.debug(f"GA logging success for makes: {log_make_success}") + # convert models to lower case and replace spaces with dashes models = [model.lower().replace(" ", "-") for model in models] From a44a25fb20b73dacfec62584304554230a31d76c Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 11 May 2023 22:42:35 -0700 Subject: [PATCH 07/18] feat: setup basic logging to google analytics (cherry picked from commit df6ad9aa10a137e05ee74db997549d4b000f5cf4) --- src/analytics/google_analytics.py | 89 +++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 src/analytics/google_analytics.py diff --git a/src/analytics/google_analytics.py b/src/analytics/google_analytics.py new file mode 100644 index 0000000..1cdfd1e --- /dev/null +++ b/src/analytics/google_analytics.py @@ -0,0 +1,89 @@ +# Author: Ty Andrews +# Date: 2023-05-11 + +import os, sys + +import os, sys +import requests +import json +from dotenv import load_dotenv, find_dotenv +import time + +from src.logs import get_logger + +logger = get_logger(__name__) + +load_dotenv(find_dotenv()) + +GA4_CLIENT_SECRET = os.getenv("GA4_CLIENT_SECRET") +GA4_MEASUREMENT_ID = "G-CKDC8LRCRB" + + +def log_to_GA_list_of_items(event_name: str, item_name: str, list_of_items: list): + """Sends a custom event to google analytics 4 + + Parameters + ---------- + list_of_items : list + list of items to be logged + list_name : str + name of the list to be logged + """ + success = True + client_id = get_ga4_client_id() + try: + for item in list_of_items: + event_params = {item_name: item} + custom_event_to_GA(client_id, event_name, event_params) + except Exception as e: + logger.error(f"Failed to log {event_name} to GA4: {e}") + success = False + + return success + + +def custom_event_to_GA(client_id: str, event_name: str, event_params: dict): + """Sends a custom event to google analytics 4 + + Parameters + ---------- + event_name : str + name of the event to be sent to google analytics 4 + event_params : dict + dictionary of event parameters to be sent to google analytics 4, must + contain only one instance of unique keys but can contain multiple key + value pairs. + + Raises + ------ + ValueError + if event_params does not contain only one key value pair + """ + + # placeholder until access to live gtag clinet id is setup + api_secret = GA4_CLIENT_SECRET + measurement_id = GA4_MEASUREMENT_ID + + url = f"https://www.google-analytics.com/mp/collect?measurement_id={measurement_id}&api_secret={api_secret}" + + payload = { + "client_id": client_id, + "non_personalized_ads": False, + "events": [{"name": event_name, "params": event_params}], + } + + r = requests.post(url, data=json.dumps(payload), verify=True) + + print(r.status_code) + + +def get_ga4_client_id(): + """Placeholder function to return a client id for google analytics + + Returns + ------- + str + client id for google analytics, currently a timestamp in ns. + """ + + return str(time.time_ns()) From fdf7469a2ebd5a018356a56fe5e6eeba94237d2c Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 11 May 2023 22:43:09 -0700 Subject: [PATCH 08/18] bug: removed print statement (cherry picked from commit 07739b1f1ac163325480c7c79a6c483ff3dad616) --- src/analytics/google_analytics.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/analytics/google_analytics.py b/src/analytics/google_analytics.py index 1cdfd1e..319c0ff 100644 --- a/src/analytics/google_analytics.py +++ b/src/analytics/google_analytics.py @@ -74,8 +74,6 @@ def custom_event_to_GA(client_id: str, event_name: str, event_params: dict): r = requests.post(url, data=json.dumps(payload), verify=True) - print(r.status_code) - def get_ga4_client_id(): """Placeholder function to return a client id for google analytics From 1c55ab621665a1c3bf627f250cedaa5cce85089c Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 11 May 2023 01:49:27 -0700 Subject: [PATCH 09/18] feat: added data loading time logging (cherry picked from commit 977f286afe329c30ae60e76a9a9f77974f7d165a) --- src/app.py | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/src/app.py b/src/app.py index 754dd4d..65d3105 100644 --- a/src/app.py +++ b/src/app.py @@ -10,9 +10,6 @@ from dash_iconify import DashIconify import time -from logs import get_logger -from data.azure_blob_storage import AzureBlob - # on launch ensure src is in path cur_dir = os.getcwd() try: @@ -24,6 +21,10 @@ if SRC_PATH not in sys.path: sys.path.append(SRC_PATH) +from logs import get_logger +from data.azure_blob_storage import AzureBlob +from analytics.google_analytics import custom_event_to_GA + # Create a custom logger logger = get_logger(__name__) @@ -217,23 +218,58 @@ def toggle_navbar_collapse(n, is_open): def load_data(first_load, price_summary, num_ads_summary, mileage_summary): # if any summary is None, then we need to load data if not price_summary or not num_ads_summary or not mileage_summary: - start_time = time.time() azure_blob = AzureBlob() + + # placeholder until proper gtag id's can be extracted + client_id = str(time.time_ns()) + + start_time = time.time() ad_price_summary = azure_blob.load_parquet( "processed/avg_price_summary.parquet" ).to_dict() + custom_event_to_GA( + client_id, + "price_data_load_time", + {"time_ms": round((time.time() - start_time) * 1000, 0)}, + ) + logger + + start_time = time.time() mileage_summary = azure_blob.load_parquet( "processed/mileage_distribution_summary.parquet" ).to_dict() + custom_event_to_GA( + client_id, + "mileage_data_load_time", + {"time_ms": round((time.time() - start_time) * 1000, 0)}, + ) + + start_time = time.time() num_ads_summary = azure_blob.load_parquet( "processed/num_ads_summary.parquet" ).to_dict() + custom_event_to_GA( + client_id, + "num_ads_data_load_time", + {"time_ms": round((time.time() - start_time) * 1000, 0)}, + ) + + start_time = time.time() makes_models = azure_blob.load_parquet( "processed/makes_models.parquet" ).to_dict() + custom_event_to_GA( + client_id, + "makes_models_data_load_time", + {"time_ms": round((time.time() - start_time) * 1000, 0)}, + ) + logger.debug( f"Loaded data from Azure Blob Storage in {time.time() - start_time} seconds" ) + + custom_event_to_GA(time.time_ns(), "summary_data_load_time", {}) + return ad_price_summary, num_ads_summary, mileage_summary, makes_models else: logger.debug("Data already loaded, skipping") From ef43b8b2314cfaeab53bc90c3d18360b38cd9a50 Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 11 May 2023 02:02:23 -0700 Subject: [PATCH 10/18] feat: added callback log timing (cherry picked from commit 3c9be0aedb9630426f9258c03059ecf8644ef275) --- src/pages/explore_ads.py | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/pages/explore_ads.py b/src/pages/explore_ads.py index 025968f..bd88542 100644 --- a/src/pages/explore_ads.py +++ b/src/pages/explore_ads.py @@ -17,6 +17,7 @@ import dash_mantine_components as dmc from dash_iconify import DashIconify import pandas as pd +import time from src.visualizations.explore_ads_plots import ( plot_vehicle_prices_summary, @@ -317,6 +318,8 @@ def update_filter_options( List of values of the price slider is set at, if the price range changes based on the selected makes and models, then the price slider is reset to the new range. """ + start_time = time.time() + make_model_df = pd.DataFrame.from_dict(makes_models_store, orient="columns") price_summary_df = pd.DataFrame.from_dict(price_summary_store, orient="columns") num_ads_summary_df = pd.DataFrame.from_dict(num_ads_summary_store, orient="columns") @@ -399,6 +402,14 @@ def update_filter_options( if price_slider_values[1] == price_slider_max: price_slider_values[1] = max_price + log_success = log_to_GA_list_of_items( + event_name="explore_update_filters_time", + item_name="time_ms", + list_of_items=[int((time.time() - start_time) * 1000)], + ) + logger.debug( + f"explore_update_filters_time log success - {log_success}: {int((time.time() - start_time) * 1000)}" + ) return model_options, make_options, max_price, price_slider_values @@ -416,6 +427,8 @@ def update_filter_options( def update_ad_filter_count( age_range, price_range, models, makes, num_ads_summary_dict, makes_models_dict ): + start_time = time.time() + num_ads_summary_df = pd.DataFrame.from_dict(num_ads_summary_dict, orient="columns") makes_models_df = pd.DataFrame.from_dict(makes_models_dict, orient="columns") @@ -462,6 +475,14 @@ def update_ad_filter_count( ), ] ) + log_success = log_to_GA_list_of_items( + event_name="explore_matching_ads_update_time", + item_name="time_ms", + list_of_items=[int((time.time() - start_time) * 1000)], + ) + logger.debug( + f"explore_matching_ads_update_time log success - {log_success}: {int((time.time() - start_time) * 1000)}" + ) return num_matching_entries @@ -481,6 +502,7 @@ def update_ad_filter_count( def update_price_summary_plot( n_clicks, age_range, price_range, models, makes, price_summary, makes_models ): + start_time = time.time() # if no models selected, display the default models if len(models) == 0: models = DEFAULT_MODELS @@ -537,6 +559,15 @@ def update_price_summary_plot( price_summary_plot = plot_vehicle_prices_summary(price_summary_df) + log_success = log_to_GA_list_of_items( + event_name="explore_price_age_summary_update_time", + item_name="time_ms", + list_of_items=[int((time.time() - start_time) * 1000)], + ) + logger.debug( + f"explore_price_age_summary_update_time log success - {log_success}: {int((time.time() - start_time) * 1000)}" + ) + return price_summary_plot @@ -555,6 +586,7 @@ def update_price_summary_plot( def update_mileage_summary_plot( n_clicks, year_range, price_range, models, makes, mileage_summary, makes_models ): + start_time = time.time() # if no models selected, display the default models if len(models) == 0: models = DEFAULT_MODELS @@ -598,6 +630,15 @@ def update_mileage_summary_plot( mileage_summary_plot = plot_mileage_distribution_summary(mileage_summary_df) + log_success = log_to_GA_list_of_items( + event_name="explore_mileage_summary_update_time", + item_name="time_ms", + list_of_items=[int((time.time() - start_time) * 1000)], + ) + logger.debug( + f"explore_mileage_summary_update_time log success - {log_success}: {int((time.time() - start_time) * 1000)}" + ) + return mileage_summary_plot @@ -616,6 +657,7 @@ def update_mileage_summary_plot( def update_num_ads_summary_plot( n_clicks, age_range, price_range, models, makes, num_ads_summary, makes_models ): + start_time = time.time() # if no models selected, display the default models if len(models) == 0: models = DEFAULT_MODELS @@ -650,4 +692,13 @@ def update_num_ads_summary_plot( num_ads_summary_plot = plot_num_ads_summary(num_ads_per_model) + log_success = log_to_GA_list_of_items( + event_name="explore_num_ads_summary_update_time", + item_name="time_ms", + list_of_items=[int((time.time() - start_time) * 1000)], + ) + logger.debug( + f"explore_num_ads_summary_update_time log success - {log_success}: {int((time.time() - start_time) * 1000)}" + ) + return num_ads_summary_plot From 21d1e45b6df5963825fd80930055f702f2e245dc Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 22 Jun 2023 11:07:52 -0700 Subject: [PATCH 11/18] Create testing-coverage.yml --- .github/workflows/testing-coverage.yml | 45 ++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/testing-coverage.yml diff --git a/.github/workflows/testing-coverage.yml b/.github/workflows/testing-coverage.yml new file mode 100644 index 0000000..5eb001f --- /dev/null +++ b/.github/workflows/testing-coverage.yml @@ -0,0 +1,45 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Testing & Code Coverage + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main", "dev" ] + +permissions: + contents: read + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pytest --cov=fortunato-wheels --cov-report=xml + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + files: ./coverage.xml # coverage report From 9d6d8180676692a37d97d9d6e4325b3662e9b4b8 Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 22 Jun 2023 12:02:23 -0700 Subject: [PATCH 12/18] Update requirements.txt for testing --- requirements.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b117681..bed4b9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,8 @@ dash-loading-spinners~=1.0 azure-storage-blob~=12.16 azure-identity~=1.12 python-dotenv~=1.0 -python-frontmatter~=1.0 \ No newline at end of file +python-frontmatter~=1.0 +pytest-cov~=4.1 +pytest~=6.2 +coverage~=7.2 +codecov~=2.1 From 272531300d80713ffe7b9b8d69753950a5366778 Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 22 Jun 2023 12:12:23 -0700 Subject: [PATCH 13/18] feat: add manual dispatch --- .github/workflows/testing-coverage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing-coverage.yml b/.github/workflows/testing-coverage.yml index 5eb001f..63ecf79 100644 --- a/.github/workflows/testing-coverage.yml +++ b/.github/workflows/testing-coverage.yml @@ -8,6 +8,7 @@ on: branches: [ "main" ] pull_request: branches: [ "main", "dev" ] + workflow_dispatch: permissions: contents: read From bbe9684f4ca72e29dc0c417c33927ebf9aa9a9e7 Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 22 Jun 2023 12:13:01 -0700 Subject: [PATCH 14/18] Create codecov.yml --- codecov.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..40a3e00 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,2 @@ +ignore: + - "*/tests/*” From f74d8ec182d4a49143a85beb963477b0517de35a Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 22 Jun 2023 12:26:51 -0700 Subject: [PATCH 15/18] bug: vehicles_df not used --- src/pages/explore_ads.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/pages/explore_ads.py b/src/pages/explore_ads.py index bd88542..1590b78 100644 --- a/src/pages/explore_ads.py +++ b/src/pages/explore_ads.py @@ -423,17 +423,19 @@ def update_filter_options( ], State("num-ads-summary-store", "data"), State("makes-models-store", "data"), + State("price-summary-store", "data"), ) def update_ad_filter_count( - age_range, price_range, models, makes, num_ads_summary_dict, makes_models_dict + age_range, price_range, models, makes, num_ads_summary_dict, makes_models_dict, price_summary_store ): start_time = time.time() num_ads_summary_df = pd.DataFrame.from_dict(num_ads_summary_dict, orient="columns") makes_models_df = pd.DataFrame.from_dict(makes_models_dict, orient="columns") + price_summary_df = pd.DataFrame.from_dict(price_summary_store, orient="columns") if price_range is None: - price_range = [0, vehicles_df.price.max()] + price_range = price_summary_df.max().max() if (len(models) == 0) & (len(makes) == 0): # remove some models that are not really models @@ -452,9 +454,6 @@ def update_ad_filter_count( if len(makes) == 0: makes = makes_models_df.make.unique() - if age_range is None: - age_range = [vehicles_df.year.min(), vehicles_df.year.max()] - matching_ads = ( num_ads_summary_df.query("age >= @age_range[0] & age <= @age_range[1]")[models] .sum() From de8fd77ccf3ec03ec738a5f94fa40446e41885ed Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 22 Jun 2023 12:55:55 -0700 Subject: [PATCH 16/18] bug: src imports path --- tests/visualizations/test_explore_ads_plots.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/visualizations/test_explore_ads_plots.py b/tests/visualizations/test_explore_ads_plots.py index 7a17aaf..5380b29 100644 --- a/tests/visualizations/test_explore_ads_plots.py +++ b/tests/visualizations/test_explore_ads_plots.py @@ -2,15 +2,9 @@ # Date: 2023-05-08 import os, sys import pytest - -# on launch ensure src is in path -cur_dir = os.getcwd() -try: - SRC_PATH = cur_dir[: cur_dir.index("fortunato-wheels") + len("fortunato-wheels")] -except ValueError: - # deal with Azure app service not working with relative imports - SRC_PATH = "" - pass + +# ensure that the parent directory is on the path for relative imports +SRC_PATH = sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) if SRC_PATH not in sys.path: sys.path.append(SRC_PATH) From 990709e282821d503800a86c770f815f5f9f5f52 Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 22 Jun 2023 13:01:10 -0700 Subject: [PATCH 17/18] bug: wrong cov target --- .github/workflows/testing-coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing-coverage.yml b/.github/workflows/testing-coverage.yml index 63ecf79..526383c 100644 --- a/.github/workflows/testing-coverage.yml +++ b/.github/workflows/testing-coverage.yml @@ -37,7 +37,7 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | - pytest --cov=fortunato-wheels --cov-report=xml + pytest --cov=src --cov-report=xml - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 env: From bd7b030599805b2c892c2e5a4fad75da0a2b2ec9 Mon Sep 17 00:00:00 2001 From: Ty Andrews Date: Thu, 22 Jun 2023 13:22:34 -0700 Subject: [PATCH 18/18] docs: added readme badges --- README.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9e96db5..407edb4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ +[![Contributors][contributors-shield]][contributors-url] +[![Forks][forks-shield]][forks-url] +[![Stargazers][stars-shield]][stars-url] +[![Issues][issues-shield]][issues-url] +[![MIT License][license-shield]][license-url] +[![codecov][codecov-shield]][codecov-url] + # Fortunato Wheels -(Fortunato: Latin word for "lucky") +(Fortunato: Latin for "lucky") Fortunato wheels takes the guesswork out of buying a used car. It does this by compiling datasets of used cars from multiple websites, analyzing the data to identify trends and baselines for car prices, then make this information available to users in an interactive tool to evaluate car prices. @@ -28,4 +35,17 @@ To start using Fortunato wheels you can browse through our database of used car 3. Check out what the distribution of vehicles condition/mileage looks like on the bottom two plots ## References -- Craigslist Used Cars Dataset, Austin Reese, https://www.kaggle.com/datasets/austinreese/craigslist-carstrucks-data \ No newline at end of file +- Craigslist Used Cars Dataset, Austin Reese, https://www.kaggle.com/datasets/austinreese/craigslist-carstrucks-data + +[contributors-shield]: https://img.shields.io/github/contributors/tieandrews/fortunato-wheels.svg?style=for-the-badge +[contributors-url]: https://github.com/tieandrews/fortunato-wheels/graphs/contributors +[forks-shield]: https://img.shields.io/github/forks/tieandrews/fortunato-wheels.svg?style=for-the-badge +[forks-url]: https://github.com/tieandrews/fortunato-wheels/network/members +[stars-shield]: https://img.shields.io/github/stars/tieandrews/fortunato-wheels.svg?style=for-the-badge +[stars-url]: https://github.com/tieandrews/fortunato-wheels/stargazers +[issues-shield]: https://img.shields.io/github/issues/tieandrews/fortunato-wheels.svg?style=for-the-badge +[issues-url]: https://github.com/tieandrews/fortunato-wheels/issues +[license-shield]: https://img.shields.io/github/license/tieandrews/fortunato-wheels.svg?style=for-the-badge +[license-url]: https://github.com/tieandrews/fortunato-wheels/blob/master/LICENSE.txt +[codecov-shield]: https://img.shields.io/codecov/c/github/tieandrews/fortunato-wheels?style=for-the-badge +[codecov-url]: https://codecov.io/gh/tieandrews/fortunato-wheels