From 9325b4fe8948564df1d6b55266b62dde9600b06a Mon Sep 17 00:00:00 2001 From: Valentin Porchet Date: Sun, 25 Aug 2024 15:27:59 +0200 Subject: [PATCH] feat: added last_updated_at in players search results and player summary (#184) --- app/commands/update_search_data_cache.py | 6 ++- app/common/enums.py | 1 + .../get_player_career_request_handler.py | 14 ++++-- .../search_players_request_handler.py | 1 + app/models/players.py | 18 ++++++++ app/parsers/search_data_parser.py | 9 ++++ app/routers/players.py | 2 +- pyproject.toml | 4 +- tests/fixtures/json/players/Dekk-2677.json | 3 +- tests/fixtures/json/players/JohnV1-1190.json | 3 +- tests/fixtures/json/players/KIRIKO-21253.json | 3 +- .../fixtures/json/players/Player-1112937.json | 3 +- tests/fixtures/json/players/TeKrop-2217.json | 3 +- .../json/players/copypasting-1216.json | 3 +- .../fixtures/json/players/quibble-11594.json | 3 +- .../search_players_api_result.json | 45 ++++++++++++------- tests/parsers/test_player_parser.py | 4 +- tests/views/test_player_career_route.py | 8 +++- tests/views/test_player_stats_route.py | 8 ++++ tests/views/test_player_summary_view.py | 8 +++- 20 files changed, 116 insertions(+), 33 deletions(-) diff --git a/app/commands/update_search_data_cache.py b/app/commands/update_search_data_cache.py index 6d769fb2..afe3d375 100644 --- a/app/commands/update_search_data_cache.py +++ b/app/commands/update_search_data_cache.py @@ -16,7 +16,7 @@ cache_manager = CacheManager() # Mapping between the search data type and the variable name in JS -variable_name_mapping: dict[SearchDataType, str] = { +variable_name_mapping: dict[SearchDataType, str | None] = { SearchDataType.PORTRAIT: "avatars", SearchDataType.NAMECARD: "namecards", SearchDataType.TITLE: "titles", @@ -37,6 +37,9 @@ def get_search_page() -> httpx.Response: def extract_search_data(html_content: str, data_type: SearchDataType) -> dict: variable_name = variable_name_mapping[data_type] + if not variable_name: + return None + data_regexp = rf"const {variable_name} = (\{{.*\}})\n" result = re.search(data_regexp, html_content) @@ -99,6 +102,7 @@ def update_search_data_cache(): search_data = { data_type: retrieve_search_data(data_type, search_page) for data_type in SearchDataType + if data_type != SearchDataType.LAST_UPDATED_AT } except SearchDataRetrievalError as error: raise SystemExit from error diff --git a/app/common/enums.py b/app/common/enums.py index 10dfd183..22268942 100644 --- a/app/common/enums.py +++ b/app/common/enums.py @@ -162,3 +162,4 @@ class SearchDataType(StrEnum): NAMECARD = "namecard" PORTRAIT = "portrait" TITLE = "title" + LAST_UPDATED_AT = "lastUpdatedAt" diff --git a/app/handlers/get_player_career_request_handler.py b/app/handlers/get_player_career_request_handler.py index 2de7208e..7ebfc1a0 100644 --- a/app/handlers/get_player_career_request_handler.py +++ b/app/handlers/get_player_career_request_handler.py @@ -2,9 +2,10 @@ from typing import ClassVar +from app.common.enums import SearchDataType from app.config import settings from app.parsers.player_parser import PlayerParser -from app.parsers.search_data_parser import NamecardParser +from app.parsers.search_data_parser import LastUpdatedAtParser, NamecardParser from .api_request_handler import APIRequestHandler @@ -15,12 +16,13 @@ class GetPlayerCareerRequestHandler(APIRequestHandler): PlayerParser class. """ - parser_classes: ClassVar[list] = [PlayerParser, NamecardParser] + parser_classes: ClassVar[list] = [PlayerParser, NamecardParser, LastUpdatedAtParser] timeout = settings.career_path_cache_timeout def merge_parsers_data(self, parsers_data: list[dict], **kwargs) -> dict: """Merge parsers data together : PlayerParser for statistics data, - and NamecardParser for namecard (not here in career page) + NamecardParser for namecard and LastUpdatedAtParser + for last time the player profile was updated """ # If the user asked for stats, no need to add the namecard @@ -31,7 +33,8 @@ def merge_parsers_data(self, parsers_data: list[dict], **kwargs) -> dict: summary = ( parsers_data[0] if kwargs.get("summary") else parsers_data[0]["summary"] ) - namecard_value = parsers_data[1]["namecard"] + namecard_value = parsers_data[1][SearchDataType.NAMECARD] + last_updated_at_value = parsers_data[2][SearchDataType.LAST_UPDATED_AT] # We want to insert the namecard before "title" key in "summary" title_pos = list(summary.keys()).index("title") @@ -39,6 +42,9 @@ def merge_parsers_data(self, parsers_data: list[dict], **kwargs) -> dict: summary_items.insert(title_pos, ("namecard", namecard_value)) summary = dict(summary_items) + # We can insert the last_updated_at at the end + summary["last_updated_at"] = last_updated_at_value + if kwargs.get("summary"): return summary diff --git a/app/handlers/search_players_request_handler.py b/app/handlers/search_players_request_handler.py index a21d5b82..891c18df 100644 --- a/app/handlers/search_players_request_handler.py +++ b/app/handlers/search_players_request_handler.py @@ -88,6 +88,7 @@ def apply_transformations(self, players: Iterable[dict]) -> list[dict]: "title": self.get_title(player, player_id), "career_url": f"{settings.app_base_url}/players/{player_id}", "blizzard_id": player["url"], + "last_updated_at": player["lastUpdated"], }, ) return transformed_players diff --git a/app/models/players.py b/app/models/players.py index c1506379..b71ba696 100644 --- a/app/models/players.py +++ b/app/models/players.py @@ -69,6 +69,15 @@ class PlayerShort(BaseModel): description="Blizzard unique identifier of the player (hexadecimal)", examples=["c65b8798bc61d6ffbba120%7Ccfe9dd77a4382165e2b920bdcc035949"], ) + last_updated_at: int | None = Field( + None, + title="Timestamp", + description=( + "Last time the player profile was updated on Blizzard (timestamp). " + "Can be null if couldn't retrieve any" + ), + examples=[1704209332], + ) class PlayerSearchResult(BaseModel): @@ -227,6 +236,15 @@ class PlayerSummary(BaseModel): "or if the player doesn't play competitive at all, it's null." ), ) + last_updated_at: int | None = Field( + None, + title="Timestamp", + description=( + "Last time the player profile was updated on Blizzard (timestamp). " + "Can be null if couldn't retrieve any" + ), + examples=[1704209332], + ) class HeroesComparisons(BaseModel): diff --git a/app/parsers/search_data_parser.py b/app/parsers/search_data_parser.py index a6f9fc02..f340b663 100644 --- a/app/parsers/search_data_parser.py +++ b/app/parsers/search_data_parser.py @@ -134,3 +134,12 @@ class TitleParser(SearchDataParser): """Title Parser class""" data_type = SearchDataType.TITLE + + +class LastUpdatedAtParser(SearchDataParser): + """LastUpdatedAt Parser class""" + + data_type = SearchDataType.LAST_UPDATED_AT + + def retrieve_data_value(self, player_data: dict) -> int: + return player_data["lastUpdated"] diff --git a/app/routers/players.py b/app/routers/players.py index 5d6f686b..8a0e1bc5 100644 --- a/app/routers/players.py +++ b/app/routers/players.py @@ -111,7 +111,7 @@ async def search_players( order_by: str = Query( "name:asc", title="Ordering field and the way it's arranged (asc[ending]/desc[ending])", - pattern=r"^(player_id|name):(asc|desc)$", + pattern=r"^(player_id|name|last_updated_at):(asc|desc)$", ), offset: int = Query(0, title="Offset of the results", ge=0), limit: int = Query(20, title="Limit of results per page", gt=0), diff --git a/pyproject.toml b/pyproject.toml index b3fff3c3..cf202acb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "overfast-api" -version = "2.34.1" +version = "2.35.0" description = "Overwatch API giving data about heroes, maps, and players statistics." license = {file = "LICENSE"} authors = [ @@ -125,4 +125,4 @@ allowed-confusables = ["（", "）", "："] [tool.ruff.lint.isort] # Consider app as first-party for imports in tests -known-first-party = ["app"] +known-first-party = ["app"] \ No newline at end of file diff --git a/tests/fixtures/json/players/Dekk-2677.json b/tests/fixtures/json/players/Dekk-2677.json index 40a5993e..c3a14794 100644 --- a/tests/fixtures/json/players/Dekk-2677.json +++ b/tests/fixtures/json/players/Dekk-2677.json @@ -17,7 +17,8 @@ "season": 8 }, "console": null - } + }, + "last_updated_at": null }, "stats": { "pc": { diff --git a/tests/fixtures/json/players/JohnV1-1190.json b/tests/fixtures/json/players/JohnV1-1190.json index ef8ba6ed..4b16c032 100644 --- a/tests/fixtures/json/players/JohnV1-1190.json +++ b/tests/fixtures/json/players/JohnV1-1190.json @@ -17,7 +17,8 @@ "open": null }, "console": null - } + }, + "last_updated_at": null }, "stats": { "pc": { diff --git a/tests/fixtures/json/players/KIRIKO-21253.json b/tests/fixtures/json/players/KIRIKO-21253.json index 4b8da76d..9c01a052 100644 --- a/tests/fixtures/json/players/KIRIKO-21253.json +++ b/tests/fixtures/json/players/KIRIKO-21253.json @@ -29,7 +29,8 @@ "open": null, "season": 1 } - } + }, + "last_updated_at": null }, "stats": { "pc": { diff --git a/tests/fixtures/json/players/Player-1112937.json b/tests/fixtures/json/players/Player-1112937.json index dae6e2d9..91a4bc01 100644 --- a/tests/fixtures/json/players/Player-1112937.json +++ b/tests/fixtures/json/players/Player-1112937.json @@ -35,7 +35,8 @@ "support": null, "open": null } - } + }, + "last_updated_at": null }, "stats": { "pc": { diff --git a/tests/fixtures/json/players/TeKrop-2217.json b/tests/fixtures/json/players/TeKrop-2217.json index e40dbb42..85971758 100644 --- a/tests/fixtures/json/players/TeKrop-2217.json +++ b/tests/fixtures/json/players/TeKrop-2217.json @@ -17,7 +17,8 @@ "open": null }, "console": null - } + }, + "last_updated_at": 1678536999 }, "stats": { "pc": { diff --git a/tests/fixtures/json/players/copypasting-1216.json b/tests/fixtures/json/players/copypasting-1216.json index 2550ca97..e6b4b7b7 100644 --- a/tests/fixtures/json/players/copypasting-1216.json +++ b/tests/fixtures/json/players/copypasting-1216.json @@ -29,7 +29,8 @@ "open": null }, "console": null - } + }, + "last_updated_at": null }, "stats": { "pc": { diff --git a/tests/fixtures/json/players/quibble-11594.json b/tests/fixtures/json/players/quibble-11594.json index b0a5f762..f616d7e1 100644 --- a/tests/fixtures/json/players/quibble-11594.json +++ b/tests/fixtures/json/players/quibble-11594.json @@ -35,7 +35,8 @@ "open": null }, "console": null - } + }, + "last_updated_at": null }, "stats": { "pc": { diff --git a/tests/fixtures/json/search_players/search_players_api_result.json b/tests/fixtures/json/search_players/search_players_api_result.json index 455b0717..6928a9c8 100644 --- a/tests/fixtures/json/search_players/search_players_api_result.json +++ b/tests/fixtures/json/search_players/search_players_api_result.json @@ -8,7 +8,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/DEKK-11775", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1665953911 }, { "player_id": "DEKK-21259", @@ -17,7 +18,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/DEKK-21259", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1676555200 }, { "player_id": "Dekk-11904", @@ -26,7 +28,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/Dekk-11904", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1666924000 }, { "player_id": "Dekk-11906", @@ -35,7 +38,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/Dekk-11906", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1667959199 }, { "player_id": "Dekk-1380", @@ -44,7 +48,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/Dekk-1380", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1666538443 }, { "player_id": "Dekk-1637", @@ -53,7 +58,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/Dekk-1637", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1670258731 }, { "player_id": "Dekk-1766", @@ -62,7 +68,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/Dekk-1766", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1665287787 }, { "player_id": "Dekk-21129", @@ -71,7 +78,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/Dekk-21129", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1668372404 }, { "player_id": "Dekk-21260", @@ -80,7 +88,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/Dekk-21260", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1668452370 }, { "player_id": "Dekk-21386", @@ -89,7 +98,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/Dekk-21386", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1677947164 }, { "player_id": "Dekk-2162", @@ -98,7 +108,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/Dekk-2162", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1671155709 }, { "player_id": "Dekk-21771", @@ -107,7 +118,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/Dekk-21771", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1666737903 }, { "player_id": "Dekk-21904", @@ -116,7 +128,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/Dekk-21904", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1676730323 }, { "player_id": "Dekk-2677", @@ -125,7 +138,8 @@ "namecard": "https://d15f34w2p8l1cc.cloudfront.net/overwatch/757219956129146d84617a7e713dfca1bc33ea27cf6c73df60a33d02a147edc1.png", "title": "Bytefixer", "career_url": "https://overfast-api.tekrop.fr/players/Dekk-2677", - "blizzard_id": "d65ba781fe23cdfabe%7C9b3063608098cdbf1b9825d8664f4d96" + "blizzard_id": "d65ba781fe23cdfabe%7C9b3063608098cdbf1b9825d8664f4d96", + "last_updated_at": 1678488893 }, { "player_id": "dekk-2548", @@ -134,7 +148,8 @@ "namecard": null, "title": null, "career_url": "https://overfast-api.tekrop.fr/players/dekk-2548", - "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05" + "blizzard_id": "d67b87a1fe23caffbca9%7Ce071d6c2df64c91dc7ad8f0eb73d5d05", + "last_updated_at": 1676811985 } ] } \ No newline at end of file diff --git a/tests/parsers/test_player_parser.py b/tests/parsers/test_player_parser.py index bbab99ca..465f0c62 100644 --- a/tests/parsers/test_player_parser.py +++ b/tests/parsers/test_player_parser.py @@ -25,9 +25,11 @@ async def test_player_page_parsing_with_filters( player_json_data: dict, kwargs_filter: dict, ): - # Remove "namecard" key from player_json_data, it's been added from another page + # Remove "namecard" and "last_updated_at" keys from player_json_data, + # it's been added from others parsers player_data = player_json_data.copy() del player_data["summary"]["namecard"] + del player_data["summary"]["last_updated_at"] parser = PlayerParser(player_id=player_id) update_parser_cache_last_update_mock = Mock() diff --git a/tests/views/test_player_career_route.py b/tests/views/test_player_career_route.py index 8625cc7e..433f26aa 100644 --- a/tests/views/test_player_career_route.py +++ b/tests/views/test_player_career_route.py @@ -35,7 +35,13 @@ def test_get_player_career( side_effect=[ # Player HTML page Mock(status_code=status.HTTP_200_OK, text=player_html_data), - # Search results related to the player + # Search results related to the player (for namecard) + Mock( + status_code=status.HTTP_200_OK, + text=json.dumps(search_tekrop_blizzard_json_data), + json=lambda: search_tekrop_blizzard_json_data, + ), + # Search results related to the player (for last_updated_at) Mock( status_code=status.HTTP_200_OK, text=json.dumps(search_tekrop_blizzard_json_data), diff --git a/tests/views/test_player_stats_route.py b/tests/views/test_player_stats_route.py index 50abb3c5..57dd7fc3 100644 --- a/tests/views/test_player_stats_route.py +++ b/tests/views/test_player_stats_route.py @@ -57,7 +57,15 @@ def test_get_player_stats( overfast_client, "get", side_effect=[ + # Player HTML page Mock(status_code=status.HTTP_200_OK, text=player_html_data), + # Search results related to the player (for namecard) + Mock( + status_code=status.HTTP_200_OK, + text=json.dumps(search_tekrop_blizzard_json_data), + json=lambda: search_tekrop_blizzard_json_data, + ), + # Search results related to the player (for last_updated_at) Mock( status_code=status.HTTP_200_OK, text=json.dumps(search_tekrop_blizzard_json_data), diff --git a/tests/views/test_player_summary_view.py b/tests/views/test_player_summary_view.py index b15a44fb..47eb4d91 100644 --- a/tests/views/test_player_summary_view.py +++ b/tests/views/test_player_summary_view.py @@ -35,7 +35,13 @@ def test_get_player_summary( side_effect=[ # Player HTML page Mock(status_code=status.HTTP_200_OK, text=player_html_data), - # Search results related to the player + # Search results related to the player (for namecard) + Mock( + status_code=status.HTTP_200_OK, + text=json.dumps(search_tekrop_blizzard_json_data), + json=lambda: search_tekrop_blizzard_json_data, + ), + # Search results related to the player (for last_updated_at) Mock( status_code=status.HTTP_200_OK, text=json.dumps(search_tekrop_blizzard_json_data),