User importer

.env_example

@@ -1,2 +1,3 @@
 HOST=0.0.0.0
-PORT=8050
+PORT=8050
+USER_FILE_PATH='file/path/user.json'

.gitignore

@@ -4,4 +4,6 @@ data/
 *__pycache__/
 *.pyc
 ./.pytest_cache
-.env
+.env
+.vscode
+.pytest_cache

app/analytics/MatchAnalytics.py

@@ -109,17 +109,6 @@ def phone_number_shares(events):
                                              columns=["Message Outcomes", "Count"])
     return phone_number_share_ratios
 
-
-def import_user_account_data(file_path="../data/app_uploaded_files/user.json"):
-    account_data = __import_user_data_by_key("account", file_path)
-    return account_data
-
-
-def import_user_device_data(file_path="../data/app_uploaded_files/user.json"):
-    device_data = __import_user_data_by_key("devices", file_path)
-    return device_data
-
-
 def __build_comments_list(events):
     likes_w_comments = []
     like_events = events["like"].dropna()
@@ -133,20 +122,6 @@ def __build_comments_list(events):
     return likes_w_comments
 
 
-def __import_user_data_by_key(key, file_path):
-    __validate_upload_file_type(file_path)
-    __validate_user_file_upload(file_path)
-
-    with open(file_path, 'r') as file:
-        raw_user_data = json.load(file)
-
-    user_data = []
-    if key in raw_user_data:
-        user_data = raw_user_data[key]
-
-    return user_data
-
-
 def __validate_upload_file_type(file_path):
     if not file_path.endswith('.json'):
         raise ValueError("Invalid file type. Please upload a JSON file.")
@@ -155,8 +130,3 @@ def __validate_upload_file_type(file_path):
 def __validate_match_file_upload(file_path):
     if 'match' not in file_path:
         raise ValueError("Invalid file name. Please upload a file with 'match' in the file name.")
-
-
-def __validate_user_file_upload(file_path):
-    if 'user' not in file_path:
-        raise ValueError("Invalid file name. Please upload a file with 'user' in the file name.")

app/analytics/UserAnalytics.py

@@ -1,47 +1,69 @@
 # from ip2geotools.databases.noncommercial import DbIpCity
 import json
 import pandas as pd
+import os
 
+class UserAnalytics:
+    def __init__(self):
+        self.user_file_path = os.environ.get("USER_FILE_PATH")
+        if self.user_file_path is None:
+            raise Exception("USER_FILE_PATH environment variable is not set.")
+
+        if '.json' not in self.user_file_path:
+            raise Exception("The user file needs to be a JSON file.")
 
-def parse_user_ip_addresses(file_path='data/export/user.json'):
-    """
-    Parses the IP addresses out of  the user data and gets latitude and longitude coordinates from the IP addresses.
-    This is only grabbing a subset of the IP addresses because the full set of data takes too long.
-    :return: a DataFrame with latitude and longitude coordinates
-    """
-    json_file_path = file_path
-
-    # opening json file
-    with open(json_file_path, 'r') as file:
-        # raw data is a list of dictionaries "list of interactions with a person"
-        raw_data = json.load(file)
-
-    device_value = []
-    # parse just the device records
-    if 'devices' in raw_data:
-        values = raw_data['devices']
-        device_value = values
-
-    # extract the IP addresses
-    ip_addresses = [entry['ip_address'] for entry in device_value]
-
-    lats = []
-    longs = []
-    # lookup the latitude and longitude coordinates of each IP address
-    # TODO: this API call doesn't work super well, replace it
-    # for ip in ip_addresses[:100]:
-    #     coord = DbIpCity.get(ip, api_key="free")
-    #     lats.append(coord.latitude)
-    #     longs.append(coord.longitude)
-
-    # define column names and create a DataFrame
-    coordinates = pd.DataFrame({'latitude': lats, 'longitude': longs})
-    return coordinates
-
-
-def __validate_user_file_upload(file_path):
-    if not file_path.endswith('.json'):
-        raise ValueError("Invalid file type. Please upload a JSON file.")
-
-    if 'user' not in file_path:
-        raise ValueError("Invalid file. Please upload a user file.")
+        with open(self.user_file_path, 'r') as file:
+            user_data = json.load(file)
+
+        self.user_data = user_data
+
+    def get_account_data(self):
+        return self.user_data["account"]
+
+    def get_devices_data(self):
+        return self.user_data["devices"]
+
+    def get_profile_data(self):
+        return self.user_data["profile"] 
+
+    def get_preferences_data(self):
+        return self.user_data["preferences"]
+
+    def get_location_data(self):
+        return self.user_data["location"]
+
+
+# def parse_user_ip_addresses(file_path='data/export/user.json'):
+#     """
+#     Parses the IP addresses out of  the user data and gets latitude and longitude coordinates from the IP addresses.
+#     This is only grabbing a subset of the IP addresses because the full set of data takes too long.
+#     :return: a DataFrame with latitude and longitude coordinates
+#     """
+#     json_file_path = file_path
+
+#     # opening json file
+#     with open(json_file_path, 'r') as file:
+#         # raw data is a list of dictionaries "list of interactions with a person"
+#         raw_data = json.load(file)
+
+#     device_value = []
+#     # parse just the device records
+#     if 'devices' in raw_data:
+#         values = raw_data['devices']
+#         device_value = values
+
+#     # extract the IP addresses
+#     ip_addresses = [entry['ip_address'] for entry in device_value]
+
+#     lats = []
+#     longs = []
+#     # lookup the latitude and longitude coordinates of each IP address
+#     # TODO: this API call doesn't work super well, replace it
+#     # for ip in ip_addresses[:100]:
+#     #     coord = DbIpCity.get(ip, api_key="free")
+#     #     lats.append(coord.latitude)
+#     #     longs.append(coord.longitude)
+
+#     # define column names and create a DataFrame
+#     coordinates = pd.DataFrame({'latitude': lats, 'longitude': longs})
+#     return coordinates

app/app.py

@@ -135,7 +135,7 @@ def parse_uploaded_file_contents(list_of_file_contents, list_of_file_names):
         uploaded_file_data = file_content.encode("utf8").split(b";base64,")[1]
 
         with open(os.path.join(USER_FILE_UPLOAD_DIRECTORY, file_name), "wb") as uploaded_file:
-            logger.info(f"Writing file: {file_name}...")
+            logger.info(f"Uploading user file: {file_name}...")
             uploaded_file.write(base64.decodebytes(uploaded_file_data))
 
         # return an html Div of the uploaded file names to display to the user

app/pages/UserPage.py

@@ -4,8 +4,9 @@
 import plotly.express as px
 from dash.exceptions import PreventUpdate
 
-import analytics.MatchAnalytics as MatchAnalytics
-import analytics.UserAnalytics as ua
+from analytics.UserAnalytics import UserAnalytics
+
+user_analytics = UserAnalytics()
 
 
 layout = html.Div([
@@ -45,27 +46,28 @@
 def update_comment_table(data):
     __check_for_live_update_data(data)
 
-    account_data = MatchAnalytics.import_user_account_data()
+    account_data =  user_analytics.get_account_data() 
     # passing in the account data as a list for the DataTable
     return [
         dash_table.DataTable(data=[account_data], page_size=5,
                              style_cell={'textAlign': 'left'})
        ]
 
 
-@callback(
-    Output('live-update-coords-graph', 'figure'),
-    [Input('refresh-page', 'n_clicks')]
-)
-def update_coords_graph_live(data):
-    __check_for_live_update_data(data)
+# TODO: commenting this out until there is an alternative
+# @callback(
+#     Output('live-update-coords-graph', 'figure'),
+#     [Input('refresh-page', 'n_clicks')]
+# )
+# def update_coords_graph_live(data):
+#     __check_for_live_update_data(data)
 
-    # initial setup of the global events
-    user_coordinates = ua.parse_user_ip_addresses()
-    # create the funnel graph
-    figure = px.scatter_geo(user_coordinates, locationmode="USA-states", lat="latitude", lon="longitude",
-                projection="orthographic")
-    return figure
+#     # initial setup of the global events
+#     user_coordinates = ua.parse_user_ip_addresses()
+#     # create the funnel graph
+#     figure = px.scatter_geo(user_coordinates, locationmode="USA-states", lat="latitude", lon="longitude",
+#                 projection="orthographic")
+#     return figure
 
 
 # TODO: I don't like this this is repeated in both files, consolidate at some point

tests/analytics/test_UserAnalytics.py

@@ -0,0 +1,116 @@
+import pytest
+import os
+import json
+from unittest.mock import mock_open, patch
+
+from app.analytics.UserAnalytics import UserAnalytics
+
+#########################################################################################
+# test values
+#########################################################################################
+USER_FILE_PATH = "fake/file/path/users.json"
+USER_DATA = '''
+{
+    "devices": [
+        {
+            "ip_address": "174.234.168.00",
+            "device_model": "unknown",
+            "device_platform": "ios",
+            "device_os_versions": "16.5.1"
+        }
+    ],
+    "account": {
+        "signup_time": "2001-06-29 03:27:17.539",
+        "last_pause_time": "2003-09-04 03:04:32",
+        "last_unpause_time": "2020-12-10 16:53:40",
+        "last_seen": "2024-01-17 04:07:39",
+        "device_platform": "ios",
+        "device_os": "16.6.1",
+        "device_model": "unknown",
+        "app_version": "9.30.0",
+        "push_notifications_enabled": false
+    },
+    "profile": {
+        "first_name": "Fake User",
+        "age": 99,
+        "height_centimeters": 213
+    },
+    "preferences": {
+        "distance_miles_max": 50,
+        "age_min": 98,
+        "age_max": 99
+    },
+    "location": {
+        "latitude": 65.00,
+        "longitude": 18.00,
+        "country": "Iceland"
+    }
+}
+'''
+
+#########################################################################################
+# pytest fixtures
+#########################################################################################
+@pytest.fixture
+def user_analytics(monkeypatch):
+    monkeypatch.setenv("USER_FILE_PATH", USER_FILE_PATH)
+
+    with patch("builtins.open", mock_open(read_data=USER_DATA)) as mock_file, \
+         patch("json.load", return_value=json.loads(USER_DATA)) as mock_json_load:
+
+        user_analytics = UserAnalytics()
+    return user_analytics
+
+#########################################################################################
+# unit tests
+#########################################################################################
+def test_exists(user_analytics):
+    assert user_analytics is not None
+
+def test_user_file_path_not_set():
+    if "USER_FILE_PATH" in os.environ:
+        del os.environ["USER_FILE_PATH"]
+
+    with pytest.raises(Exception, match="USER_FILE_PATH environment variable is not set."):
+        UserAnalytics()
+
+def test_user_file_not_json():
+    os.environ["USER_FILE_PATH"] = "invalid_file.txt"  # invalid value
+
+    with pytest.raises(Exception, match="The user file needs to be a JSON file."):
+        UserAnalytics()
+
+def test_user_analytics_loads_data(user_analytics):
+    assert isinstance(user_analytics.user_data, dict)
+    assert "devices" in user_analytics.user_data
+    assert "account" in user_analytics.user_data
+
+def test_device_data(user_analytics):
+    device = user_analytics.get_devices_data()[0]
+    assert device["ip_address"] == "174.234.168.00"
+    assert device["device_platform"] == "ios"
+    assert device["device_os_versions"] == "16.5.1"
+
+def test_account_data(user_analytics):
+    account = user_analytics.get_account_data()
+    assert account["device_os"] == "16.6.1"
+    assert account["app_version"] == "9.30.0"
+    assert account["push_notifications_enabled"] is False
+
+def test_profile_data(user_analytics):
+    profile = user_analytics.get_profile_data()
+    assert profile["first_name"] == "Fake User"
+    assert profile["age"] == 99
+    assert profile["height_centimeters"] == 213
+
+def test_preferences_data(user_analytics):
+    preferences = user_analytics.get_preferences_data()
+    assert preferences["distance_miles_max"] == 50
+    assert preferences["age_min"] == 98
+    assert preferences["age_max"] == 99
+
+def test_location_data(user_analytics):
+    locations = user_analytics.get_location_data()
+    assert locations["latitude"] == 65.00
+    assert locations["longitude"] == 18.00
+    assert locations["country"] == "Iceland"