From d3b709dbe001784fee3d42c3ebf1508ca3d99a84 Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Thu, 3 Oct 2024 18:03:20 +0300 Subject: [PATCH 01/14] Pass rom names in the delete function in order to avoid delete all media in case of multiple system having the same art directory --- src/app.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/app.py b/src/app.py index 89cada4..e0e260b 100644 --- a/src/app.py +++ b/src/app.py @@ -120,19 +120,20 @@ def get_roms(self, system: str) -> list[Rom]: roms.append(rom) return roms - def delete_all_files_in_directory(self, directory_path): + def delete_all_files_in_directory(self, filenames, directory_path): directory = Path(directory_path) if directory.is_dir(): for file in directory.iterdir(): - if file.is_file(): + if file.is_file() and file.stem in filenames: file.unlink() def delete_system_media(self) -> None: global selected_system system = self.systems_mapping.get(selected_system) if system: + roms = [rom.name for rom in self.get_roms(selected_system)] for media_type in ["box", "preview", "synopsis"]: - self.delete_all_files_in_directory(system.get(media_type, "")) + self.delete_all_files_in_directory(roms, system.get(media_type, "")) def draw_available_systems(self, available_systems: List[str]) -> None: max_elem = 11 From ea99b73ba4dcf57139da64bb4e62acd9c380faec Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Thu, 3 Oct 2024 18:05:20 +0300 Subject: [PATCH 02/14] Only delete the types that are enabled in the config --- src/app.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/app.py b/src/app.py index e0e260b..e4728a3 100644 --- a/src/app.py +++ b/src/app.py @@ -120,7 +120,7 @@ def get_roms(self, system: str) -> list[Rom]: roms.append(rom) return roms - def delete_all_files_in_directory(self, filenames, directory_path): + def delete_files_in_directory(self, filenames, directory_path): directory = Path(directory_path) if directory.is_dir(): for file in directory.iterdir(): @@ -132,8 +132,15 @@ def delete_system_media(self) -> None: system = self.systems_mapping.get(selected_system) if system: roms = [rom.name for rom in self.get_roms(selected_system)] - for media_type in ["box", "preview", "synopsis"]: - self.delete_all_files_in_directory(roms, system.get(media_type, "")) + media_types = [] + if self.box_enabled: + media_types.append("box") + if self.preview_enabled: + media_types.append("preview") + if self.synopsis_enabled: + media_types.append("synopsis") + for media_type in media_types: + self.delete_files_in_directory(roms, system.get(media_type, "")) def draw_available_systems(self, available_systems: List[str]) -> None: max_elem = 11 From ee0d99c252d5825f0cb312dd5284b7c1326a3fb7 Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Thu, 3 Oct 2024 18:08:41 +0300 Subject: [PATCH 03/14] Compare lower dir names with lower system names --- src/app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/app.py b/src/app.py index e4728a3..5634fb7 100644 --- a/src/app.py +++ b/src/app.py @@ -97,12 +97,16 @@ def update(self) -> None: def get_available_systems(self) -> List[str]: available_systems = [ - d.upper() + d.lower() for d in os.listdir(self.roms_path) if Path(self.roms_path, d).is_dir() ] return sorted( - [system for system in available_systems if system in self.systems_mapping] + [ + system + for system in available_systems + if system.lower() in map(str.lower, self.systems_mapping.keys()) + ], ) def get_roms(self, system: str) -> list[Rom]: From 3e22f6cac6d7ec4d17f89e28fed923a76fe01070 Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Fri, 4 Oct 2024 22:36:59 +0300 Subject: [PATCH 04/14] Guard against malformed url params --- src/scraper.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/scraper.py b/src/scraper.py index 24ccca1..6e43066 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -83,7 +83,11 @@ def parse_find_game_url(system_id, rom_path, dev_id, dev_password, username, pas "romnom": f"{clean_rom_name(rom_path)}.zip", "romtaille": str(file_size(rom_path)), } - return urlunparse(urlparse(BASE_URL)._replace(query=urlencode(params))) + try: + return urlunparse(urlparse(BASE_URL)._replace(query=urlencode(params))) + except UnicodeDecodeError as e: + logging.error(f"Error encoding URL: {e}. ROM params: {params}") + return None def find_media_url_by_region(medias, media_type, regions): From 95ba7f9103c6f09ec09956775210c62bed9ccb7b Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Fri, 4 Oct 2024 22:38:59 +0300 Subject: [PATCH 05/14] Add support for fetching user info data --- src/scraper.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/scraper.py b/src/scraper.py index 6e43066..9c127c6 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -9,7 +9,8 @@ import requests -BASE_URL = "https://www.screenscraper.fr/api2/jeuInfos.php" +GAME_INFO_URL = "https://www.screenscraper.fr/api2/jeuInfos.php" +USER_INFO_URL = "https://api.screenscraper.fr/api2/ssuserInfos.php" MAX_FILE_SIZE_BYTES = 104857600 # 100MB IMAGE_EXTENSIONS = [".jpg", ".jpeg", ".png"] VALID_MEDIA_TYPES = {"box-2D", "box-3D", "mixrbv1", "mixrbv2", "ss"} @@ -84,12 +85,28 @@ def parse_find_game_url(system_id, rom_path, dev_id, dev_password, username, pas "romtaille": str(file_size(rom_path)), } try: - return urlunparse(urlparse(BASE_URL)._replace(query=urlencode(params))) + return urlunparse(urlparse(GAME_INFO_URL)._replace(query=urlencode(params))) except UnicodeDecodeError as e: logging.error(f"Error encoding URL: {e}. ROM params: {params}") return None +def parse_user_info_url(dev_id, dev_password, username, password): + params = { + "devid": base64.b64decode(dev_id).decode(), + "devpassword": base64.b64decode(dev_password).decode(), + "softname": "crossmix", + "output": "json", + "ssid": username, + "sspassword": password, + } + try: + return urlunparse(urlparse(USER_INFO_URL)._replace(query=urlencode(params))) + except UnicodeDecodeError as e: + logging.error(f"Error encoding URL: {e}. User info params: {params}") + return None + + def find_media_url_by_region(medias, media_type, regions): for region in regions: for media in medias: From 6ed99f92719d9b6921b2c8cb2c9205f51e16fb6d Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Fri, 4 Oct 2024 22:40:32 +0300 Subject: [PATCH 06/14] Add pagination to systems as well --- src/app.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/app.py b/src/app.py index 5634fb7..b76ec3b 100644 --- a/src/app.py +++ b/src/app.py @@ -210,6 +210,22 @@ def load_emulators(self) -> None: skip_input_check = True time.sleep(self.LOG_WAIT) return + elif input.key_pressed("L1"): + if selected_position > 0: + selected_position = max(0, selected_position - max_elem) + elif input.key_pressed("R1"): + if selected_position < len(available_systems) - 1: + selected_position = min( + len(available_systems) - 1, selected_position + max_elem + ) + elif input.key_pressed("L2"): + if selected_position > 0: + selected_position = max(0, selected_position - 100) + elif input.key_pressed("R2"): + if selected_position < len(available_systems) - 1: + selected_position = min( + len(available_systems) - 1, selected_position + 100 + ) if len(available_systems) >= 1: self.draw_available_systems(available_systems) From 9a3659262fe3893780e89efa9c74ae9325c4b4f8 Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Fri, 4 Oct 2024 22:52:00 +0300 Subject: [PATCH 07/14] Fetch available threads from SS API --- src/app.py | 17 +++++++++++++++-- src/scraper.py | 50 +++++++++++++++++++++++++++----------------------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/app.py b/src/app.py index b76ec3b..f639d2f 100644 --- a/src/app.py +++ b/src/app.py @@ -13,9 +13,10 @@ fetch_box, fetch_preview, fetch_synopsis, - find_game, + get_game_data, get_image_files_without_extension, get_txt_files_without_extension, + get_user_data, ) selected_position = 0 @@ -266,9 +267,21 @@ def save_file_to_disk(self, data, destination): ) return True + def get_user_threads(self): + user_info = get_user_data( + self.dev_id, + self.dev_password, + self.username, + self.password, + ) + if not user_info: + self.threads = 1 + else: + self.threads = min(self.threads, user_info.get("maxthreads")) + def scrape(self, rom, system_id): scraped_box = scraped_preview = scraped_synopsis = None - game = find_game( + game = get_game_data( system_id, rom.path, self.dev_id, diff --git a/src/scraper.py b/src/scraper.py index 9c127c6..faac820 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -156,35 +156,39 @@ def get(url): return response.content -def find_game(system_id, rom_path, dev_id, dev_password, username, password): - game_url = parse_find_game_url( - system_id, rom_path, dev_id, dev_password, username, password - ) +def fetch_data(url): try: - body = get(game_url) - except Exception as e: - logging.error(f"Error fetching game data: {e}") - return None - - if not body: - return None + body = get(url) + if not body: + logging.error("Empty response body") + return None - body_str = body.decode("utf-8") - if "API closed" in body_str: - logging.error("API is closed") - return None - if "Erreur" in body_str: - logging.error("Game not found") - return None - if not body: - logging.error("Empty response body") - return None + body_str = body.decode("utf-8") + if "API closed" in body_str: + logging.error("API is closed") + return None + if "Erreur" in body_str: + logging.error("Error found in response: %s", body_str) + return None - try: return json.loads(body_str) except json.JSONDecodeError as e: logging.error(f"Error decoding JSON response: {e}") - return None + except Exception as e: + logging.error(f"Error fetching data from URL: {e}") + return None + + +def get_game_data(system_id, rom_path, dev_id, dev_password, username, password): + game_url = parse_find_game_url( + system_id, rom_path, dev_id, dev_password, username, password + ) + return fetch_data(game_url) + + +def get_user_data(dev_id, dev_password, username, password): + user_info_url = parse_user_info_url(dev_id, dev_password, username, password) + return fetch_data(user_info_url) def _fetch_media(medias, properties, regions): From 98a66f6becd7937d8b9993cf0f6def57806f3d26 Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Fri, 4 Oct 2024 22:54:42 +0300 Subject: [PATCH 08/14] Wrap scrape function in try/except --- src/app.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/app.py b/src/app.py index f639d2f..f818397 100644 --- a/src/app.py +++ b/src/app.py @@ -281,23 +281,26 @@ def get_user_threads(self): def scrape(self, rom, system_id): scraped_box = scraped_preview = scraped_synopsis = None - game = get_game_data( - system_id, - rom.path, - self.dev_id, - self.dev_password, - self.username, - self.password, - ) + try: + game = get_game_data( + system_id, + rom.path, + self.dev_id, + self.dev_password, + self.username, + self.password, + ) - if game: - content = self.content - if self.box_enabled: - scraped_box = fetch_box(game, content) - if self.preview_enabled: - scraped_preview = fetch_preview(game, content) - if self.synopsis_enabled: - scraped_synopsis = fetch_synopsis(game, content) + if game: + content = self.content + if self.box_enabled: + scraped_box = fetch_box(game, content) + if self.preview_enabled: + scraped_preview = fetch_preview(game, content) + if self.synopsis_enabled: + scraped_synopsis = fetch_synopsis(game, content) + except Exception as e: + print(f"Error scraping {rom.name}: {e}") return scraped_box, scraped_preview, scraped_synopsis From f92b49d730548c4269b10f804644ce23cdc78256 Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Fri, 4 Oct 2024 23:18:24 +0300 Subject: [PATCH 09/14] Remove test system --- config.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/config.json b/config.json index 69c2f7a..34b47ed 100644 --- a/config.json +++ b/config.json @@ -35,14 +35,6 @@ ] }, "systems": [ - { - "dir": "TEST", - "id": "-1", - "name": "test", - "box": "/mnt/mmc/MUOS/info/catalogue/ADVMAME/box/", - "preview": "/mnt/mmc/MUOS/info/catalogue/ADVMAME/preview/", - "synopsis": "/mnt/mmc/MUOS/info/catalogue/ADVMAME/text/" - }, { "dir": "ADVMAME", "id": "75", From fdf33fd68b566433606c9f039ef3ebf5fd938524 Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Sat, 5 Oct 2024 17:36:28 +0300 Subject: [PATCH 10/14] Make color configurable --- config.json | 7 ++++ src/app.py | 99 +++++++++++++++----------------------------------- src/graphic.py | 30 +++++++++------ 3 files changed, 54 insertions(+), 82 deletions(-) diff --git a/config.json b/config.json index 34b47ed..344ef65 100644 --- a/config.json +++ b/config.json @@ -1,6 +1,13 @@ { "roms": "/mnt/sdcard/ROMS", "logos": "assets/logos", + "colors": { + "primary": "#bb7200", + "primary_dark": "#7f4f00", + "secondary": "#292929", + "secondary_light": "#383838", + "secondary_dark": "#141414" + }, "screenscraper": { "username": "", "password": "", diff --git a/src/app.py b/src/app.py index f818397..86c6b9a 100644 --- a/src/app.py +++ b/src/app.py @@ -59,6 +59,7 @@ def load_config(self, config_file): self.config = json.loads(file_contents) self.roms_path = self.config.get("roms") self.systems_logo_path = self.config.get("logos") + self.colors = self.config.get("colors") self.dev_id = self.config.get("screenscraper").get("devid") self.dev_password = self.config.get("screenscraper").get("devpassword") self.username = self.config.get("screenscraper").get("username") @@ -71,6 +72,12 @@ def load_config(self, config_file): for system in self.config["screenscraper"]["systems"]: self.systems_mapping[system["dir"]] = system + self.gui.COLOR_PRIMARY = self.colors.get("primary") + self.gui.COLOR_PRIMARY_DARK = self.colors.get("primary_dark") + self.gui.COLOR_SECONDARY = self.colors.get("secondary") + self.gui.COLOR_SECONDARY_LIGHT = self.colors.get("secondary_light") + self.gui.COLOR_SECONDARY_DARK = self.colors.get("secondary_dark") + def start(self, config_file: str) -> None: self.load_config(config_file) self.gui.draw_start() @@ -164,17 +171,11 @@ def load_emulators(self) -> None: global selected_position, selected_system, current_window, skip_input_check self.gui.draw_clear() - self.gui.draw_rectangle_r( - [10, 40, 630, 440], 15, fill=self.gui.COLOR_GRAY_D2, outline=None - ) + self.gui.draw_rectangle_r([10, 40, 630, 440], 15) self.gui.draw_text((320, 20), "Artie Scraper v1.0.2", anchor="mm") if not Path(self.roms_path).exists() or not any(Path(self.roms_path).iterdir()): - self.gui.draw_log( - "Wrong Roms path, check config.json", - fill=self.gui.COLOR_BLUE, - outline=self.gui.COLOR_BLUE_D1, - ) + self.gui.draw_log("Wrong Roms path, check config.json") self.gui.draw_paint() time.sleep(self.LOG_WAIT) sys.exit() @@ -192,21 +193,14 @@ def load_emulators(self) -> None: elif input.key_pressed("A"): selected_system = available_systems[selected_position] current_window = "roms" - self.gui.draw_log( - "Checking existing media...", - fill=self.gui.COLOR_BLUE, - outline=self.gui.COLOR_BLUE_D1, - ) + self.gui.draw_log("Checking existing media...") self.gui.draw_paint() skip_input_check = True return elif input.key_pressed("X"): selected_system = available_systems[selected_position] self.delete_system_media() - self.gui.draw_log( - f"Deleting all existing {selected_system} media...", - fill=self.gui.COLOR_BLUE, - ) + self.gui.draw_log(f"Deleting all existing {selected_system} media...") self.gui.draw_paint() skip_input_check = True time.sleep(self.LOG_WAIT) @@ -260,11 +254,7 @@ def is_valid_rom(self, rom): def save_file_to_disk(self, data, destination): check_destination(destination) destination.write_bytes(data) - self.gui.draw_log( - "Scraping completed!", - fill=self.gui.COLOR_BLUE, - outline=self.gui.COLOR_BLUE_D1, - ) + self.gui.draw_log("Scraping completed!") return True def get_user_threads(self): @@ -310,11 +300,7 @@ def load_roms(self) -> None: exit_menu = False roms_list = self.get_roms(selected_system) if not roms_list: - self.gui.draw_log( - f"No roms found in {selected_system}...", - fill=self.gui.COLOR_BLUE, - outline=self.gui.COLOR_BLUE_D1, - ) + self.gui.draw_log(f"No roms found in {selected_system}...") self.gui.draw_paint() time.sleep(self.LOG_WAIT) self.gui.draw_clear() @@ -322,11 +308,7 @@ def load_roms(self) -> None: system = self.systems_mapping.get(selected_system) if not system: - self.gui.draw_log( - "System is unknown...", - fill=self.gui.COLOR_BLUE, - outline=self.gui.COLOR_BLUE_D1, - ) + self.gui.draw_log("System is unknown...") self.gui.draw_paint() time.sleep(self.LOG_WAIT) self.gui.draw_clear() @@ -376,11 +358,7 @@ def load_roms(self) -> None: if len(roms_to_scrape) < 1: current_window = "emulators" selected_system = "" - self.gui.draw_log( - "No roms with missing media found...", - fill=self.gui.COLOR_BLUE, - outline=self.gui.COLOR_BLUE_D1, - ) + self.gui.draw_log("No roms with missing media found...") self.gui.draw_paint() time.sleep(self.LOG_WAIT) self.gui.draw_clear() @@ -389,9 +367,7 @@ def load_roms(self) -> None: if input.key_pressed("B"): exit_menu = True elif input.key_pressed("A"): - self.gui.draw_log( - "Scraping...", fill=self.gui.COLOR_BLUE, outline=self.gui.COLOR_BLUE_D1 - ) + self.gui.draw_log("Scraping...") self.gui.draw_paint() rom = roms_to_scrape[roms_selected_position] scraped_box, scraped_preview, scraped_synopsis = self.scrape(rom, system_id) @@ -406,11 +382,7 @@ def load_roms(self) -> None: self.save_file_to_disk(scraped_synopsis.encode("utf-8"), destination) if not scraped_box and not scraped_preview and not scraped_synopsis: - self.gui.draw_log( - "Scraping failed!", - fill=self.gui.COLOR_BLUE, - outline=self.gui.COLOR_BLUE_D1, - ) + self.gui.draw_log("Scraping failed!") print(f"Failed to get screenshot for {rom.name}") self.gui.draw_paint() time.sleep(self.LOG_WAIT) @@ -419,11 +391,7 @@ def load_roms(self) -> None: progress: int = 0 success: int = 0 failure: int = 0 - self.gui.draw_log( - f"Scraping {progress} of {len(roms_to_scrape)}", - fill=self.gui.COLOR_BLUE, - outline=self.gui.COLOR_BLUE_D1, - ) + self.gui.draw_log(f"Scraping {progress} of {len(roms_to_scrape)}") self.gui.draw_paint() for rom in roms_to_scrape: scraped_box, scraped_preview, scraped_synopsis = self.scrape( @@ -443,25 +411,14 @@ def load_roms(self) -> None: if scraped_box or scraped_preview or scraped_synopsis: success += 1 else: - self.gui.draw_log( - "Scraping failed!", - fill=self.gui.COLOR_BLUE, - outline=self.gui.COLOR_BLUE_D1, - ) + self.gui.draw_log("Scraping failed!") print(f"Failed to get screenshot for {rom.name}") failure += 1 progress += 1 - self.gui.draw_log( - f"Scraping {progress} of {len(roms_to_scrape)}", - fill=self.gui.COLOR_BLUE, - outline=self.gui.COLOR_BLUE_D1, - ) + self.gui.draw_log(f"Scraping {progress} of {len(roms_to_scrape)}") self.gui.draw_paint() self.gui.draw_log( - f"Scraping completed! Success: {success} Errors: {failure}", - fill=self.gui.COLOR_BLUE, - outline=self.gui.COLOR_BLUE_D1, - width=800, + f"Scraping completed! Success: {success} Errors: {failure}" ) self.gui.draw_paint() time.sleep(self.LOG_WAIT) @@ -500,9 +457,7 @@ def load_roms(self) -> None: self.gui.draw_clear() - self.gui.draw_rectangle_r( - [10, 40, 630, 440], 15, fill=self.gui.COLOR_GRAY_D2, outline=None - ) + self.gui.draw_rectangle_r([10, 40, 630, 440], 15) rom_text = f"{selected_system} - Total Roms: {len(roms_list)}" @@ -578,7 +533,9 @@ def row_list( self.gui.draw_rectangle_r( [pos[0], pos[1], pos[0] + width, pos[1] + 32], 5, - fill=(self.gui.COLOR_BLUE if selected else self.gui.COLOR_GRAY_L1), + fill=( + self.gui.COLOR_PRIMARY if selected else self.gui.COLOR_SECONDARY_LIGHT + ), ) text_offset_x = pos[0] + 5 @@ -609,13 +566,15 @@ def row_list( self.gui.draw_text((text_offset_x, pos[1] + 5), text) def button_circle(self, pos: tuple[int, int], button: str, text: str) -> None: - self.gui.draw_circle(pos, 25, fill=self.gui.COLOR_BLUE_D1) + self.gui.draw_circle(pos, 25) self.gui.draw_text((pos[0] + 12, pos[1] + 12), button, anchor="mm") self.gui.draw_text((pos[0] + 30, pos[1] + 12), text, font=13, anchor="lm") def button_rectangle(self, pos: tuple[int, int], button: str, text: str) -> None: self.gui.draw_rectangle_r( - (pos[0], pos[1], pos[0] + 60, pos[1] + 25), 5, fill=self.gui.COLOR_GRAY_L1 + (pos[0], pos[1], pos[0] + 60, pos[1] + 25), + 5, + fill=self.gui.COLOR_SECONDARY_LIGHT, ) self.gui.draw_text((pos[0] + 30, pos[1] + 12), button, anchor="mm") self.gui.draw_text((pos[0] + 65, pos[1] + 12), text, font=13, anchor="lm") diff --git a/src/graphic.py b/src/graphic.py index 90a61fa..01ac6e6 100644 --- a/src/graphic.py +++ b/src/graphic.py @@ -6,6 +6,14 @@ class GUI: + COLOR_PRIMARY = "#bb7200" + COLOR_PRIMARY_DARK = "#7f4f00" + COLOR_SECONDARY = "#292929" + COLOR_SECONDARY_LIGHT = "#383838" + COLOR_SECONDARY_DARK = "#141414" + COLOR_WHITE = "#ffffff" + COLOR_BLACK = "#000000" + def __init__(self): self.fb = None self.mm = None @@ -20,12 +28,6 @@ def __init__(self): 11: ImageFont.truetype("assets/Roboto-Condensed.ttf", 11), } - self.COLOR_BLUE = "#bb7200" - self.COLOR_BLUE_D1 = "#7f4f00" - self.COLOR_GRAY = "#292929" - self.COLOR_GRAY_L1 = "#383838" - self.COLOR_GRAY_D2 = "#141414" - self.activeImage = None self.activeDraw = None @@ -49,7 +51,7 @@ def draw_end(self): def create_image(self): image = Image.new( - "RGBA", (self.screen_width, self.screen_height), color="black" + "RGBA", (self.screen_width, self.screen_height), color=self.COLOR_BLACK ) return image @@ -71,10 +73,10 @@ def draw_paint(self): def draw_clear(self): if self.activeDraw: self.activeDraw.rectangle( - (0, 0, self.screen_width, self.screen_height), fill="black" + (0, 0, self.screen_width, self.screen_height), fill=self.COLOR_BLACK ) - def draw_text(self, position, text, font=15, color="white", **kwargs): + def draw_text(self, position, text, font=15, color=COLOR_WHITE, **kwargs): if self.activeDraw: self.activeDraw.text( position, text, font=self.fontFile[font], fill=color, **kwargs @@ -84,13 +86,17 @@ def draw_rectangle(self, position, fill=None, outline=None, width=1): if self.activeDraw: self.activeDraw.rectangle(position, fill=fill, outline=outline, width=width) - def draw_rectangle_r(self, position, radius, fill=None, outline=None): + def draw_rectangle_r( + self, position, radius, fill=COLOR_SECONDARY_DARK, outline=None + ): if self.activeDraw: self.activeDraw.rounded_rectangle( position, radius, fill=fill, outline=outline ) - def draw_circle(self, position, radius, fill=None, outline="white"): + def draw_circle( + self, position, radius, fill=COLOR_PRIMARY_DARK, outline=COLOR_WHITE + ): if self.activeDraw: self.activeDraw.ellipse( [ @@ -103,7 +109,7 @@ def draw_circle(self, position, radius, fill=None, outline="white"): outline=outline, ) - def draw_log(self, text, fill="Black", outline="black", width=500): + def draw_log(self, text, fill=COLOR_PRIMARY, outline=COLOR_PRIMARY_DARK, width=500): # Center the rectangle horizontally x = (self.screen_width - width) / 2 # Center the rectangle vertically From 43c872e8638e12a5baae1f67e3613bd43a4685b5 Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Sat, 5 Oct 2024 17:53:05 +0300 Subject: [PATCH 11/14] Fix having all system names in lower letters --- src/app.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/app.py b/src/app.py index 86c6b9a..2fa0fba 100644 --- a/src/app.py +++ b/src/app.py @@ -70,7 +70,7 @@ def load_config(self, config_file): self.synopsis_enabled = self.content["synopsis"]["enabled"] self.threads = self.config.get("threads") for system in self.config["screenscraper"]["systems"]: - self.systems_mapping[system["dir"]] = system + self.systems_mapping[system["dir"].lower()] = system self.gui.COLOR_PRIMARY = self.colors.get("primary") self.gui.COLOR_PRIMARY_DARK = self.colors.get("primary_dark") @@ -110,11 +110,7 @@ def get_available_systems(self) -> List[str]: if Path(self.roms_path, d).is_dir() ] return sorted( - [ - system - for system in available_systems - if system.lower() in map(str.lower, self.systems_mapping.keys()) - ], + [system for system in available_systems if system in self.systems_mapping] ) def get_roms(self, system: str) -> list[Rom]: From d8659ee13bcc2484d8d6309a63579d60e96beb89 Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Sat, 5 Oct 2024 21:42:15 +0300 Subject: [PATCH 12/14] Skip dirs starting with . when discovering Roms to fix ScummVM random game files showing as roms --- src/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/app.py b/src/app.py index 2fa0fba..a283960 100644 --- a/src/app.py +++ b/src/app.py @@ -117,7 +117,9 @@ def get_roms(self, system: str) -> list[Rom]: roms = [] system_path = Path(self.roms_path) / system - for root, _, files in os.walk(system_path): + for root, dirs, files in os.walk(system_path): + dirs[:] = [d for d in dirs if not d.startswith(".")] + for file in files: file_path = Path(root) / file if file.startswith("."): From 5babcd975dd11330be78d697095ae39c9f1bd24a Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Sat, 5 Oct 2024 22:59:51 +0300 Subject: [PATCH 13/14] Fix SS api --- src/scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scraper.py b/src/scraper.py index faac820..45aad5f 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -9,7 +9,7 @@ import requests -GAME_INFO_URL = "https://www.screenscraper.fr/api2/jeuInfos.php" +GAME_INFO_URL = "https://api.screenscraper.fr/api2/jeuInfos.php" USER_INFO_URL = "https://api.screenscraper.fr/api2/ssuserInfos.php" MAX_FILE_SIZE_BYTES = 104857600 # 100MB IMAGE_EXTENSIONS = [".jpg", ".jpeg", ".png"] From 39cf1e01c1bf5ea7958f9032feda4fe3795b9d7a Mon Sep 17 00:00:00 2001 From: Michael Loukeris Date: Mon, 14 Oct 2024 22:34:24 +0300 Subject: [PATCH 14/14] Add custom logger --- config.json | 3 ++- src/app.py | 22 +++++++++++++++------ src/input.py | 10 +++++----- src/logger.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ src/scraper.py | 52 ++++++++++++++++++++++---------------------------- 5 files changed, 97 insertions(+), 41 deletions(-) create mode 100644 src/logger.py diff --git a/config.json b/config.json index 344ef65..b284059 100644 --- a/config.json +++ b/config.json @@ -1,13 +1,14 @@ { "roms": "/mnt/sdcard/ROMS", "logos": "assets/logos", + "log_level": "info", "colors": { "primary": "#bb7200", "primary_dark": "#7f4f00", "secondary": "#292929", "secondary_light": "#383838", "secondary_dark": "#141414" - }, + }, "screenscraper": { "username": "", "password": "", diff --git a/src/app.py b/src/app.py index a283960..4f9cc8a 100644 --- a/src/app.py +++ b/src/app.py @@ -1,4 +1,5 @@ import json +import logging import os import sys import time @@ -7,6 +8,7 @@ import input from graphic import GUI +from logger import LoggerSingleton as logger from PIL import Image from scraper import ( check_destination, @@ -19,13 +21,14 @@ get_user_data, ) +VERSION = "1.0.2" + selected_position = 0 roms_selected_position = 0 selected_system = "" current_window = "emulators" max_elem = 11 skip_input_check = False -scraping = False class Rom: @@ -78,8 +81,15 @@ def load_config(self, config_file): self.gui.COLOR_SECONDARY_LIGHT = self.colors.get("secondary_light") self.gui.COLOR_SECONDARY_DARK = self.colors.get("secondary_dark") + def setup_logging(self): + log_level_str = self.config.get("log_level", "INFO").upper() + log_level = getattr(logging, log_level_str, logging.INFO) + logger.setup_logger(log_level) + def start(self, config_file: str) -> None: self.load_config(config_file) + self.setup_logging() + logger.log_debug(f"Artie Scraper v{VERSION}") self.gui.draw_start() self.gui.screen_reset() main_gui = self.gui.create_image() @@ -253,6 +263,7 @@ def save_file_to_disk(self, data, destination): check_destination(destination) destination.write_bytes(data) self.gui.draw_log("Scraping completed!") + logger.log_debug(f"Saved file to {destination}") return True def get_user_threads(self): @@ -288,7 +299,7 @@ def scrape(self, rom, system_id): if self.synopsis_enabled: scraped_synopsis = fetch_synopsis(game, content) except Exception as e: - print(f"Error scraping {rom.name}: {e}") + logger.log_error(f"Error scraping {rom.name}: {e}") return scraped_box, scraped_preview, scraped_synopsis @@ -381,7 +392,7 @@ def load_roms(self) -> None: if not scraped_box and not scraped_preview and not scraped_synopsis: self.gui.draw_log("Scraping failed!") - print(f"Failed to get screenshot for {rom.name}") + logger.log_error(f"Failed to get screenshot for {rom.name}") self.gui.draw_paint() time.sleep(self.LOG_WAIT) exit_menu = True @@ -410,7 +421,7 @@ def load_roms(self) -> None: success += 1 else: self.gui.draw_log("Scraping failed!") - print(f"Failed to get screenshot for {rom.name}") + logger.log_error(f"Failed to get screenshot for {rom.name}") failure += 1 progress += 1 self.gui.draw_log(f"Scraping {progress} of {len(roms_to_scrape)}") @@ -558,9 +569,8 @@ def row_list( ) except Exception as e: - print(f"Error loading image from {image_path}: {e}") + logger.log_error(f"Error loading image from {image_path}: {e}") - # Draw the text self.gui.draw_text((text_offset_x, pos[1] + 5), text) def button_circle(self, pos: tuple[int, int], button: str, text: str) -> None: diff --git a/src/input.py b/src/input.py index 0612072..3022995 100644 --- a/src/input.py +++ b/src/input.py @@ -1,8 +1,6 @@ import struct -import logging -# Set up logging -logging.basicConfig(level=logging.ERROR) +from logger import LoggerSingleton as logger KEY_MAPPING = { 304: "A", @@ -40,7 +38,9 @@ def check_input(device_path="/dev/input/event1"): current_code = key_code current_code_name = KEY_MAPPING.get(current_code, str(current_code)) current_value = key_value - logging.debug(f"Key pressed: {current_code_name}, value: {current_value}") + logger.log_debug( + f"Key pressed: {current_code_name}, value: {current_value}" + ) return @@ -55,4 +55,4 @@ def reset_input(): global current_code_name, current_value current_code_name = "" current_value = 0 - logging.debug("Input reset") + logger.log_debug("Input reset") diff --git a/src/logger.py b/src/logger.py new file mode 100644 index 0000000..4c3b80d --- /dev/null +++ b/src/logger.py @@ -0,0 +1,51 @@ +import logging +from typing import Optional + + +class LoggerSingleton: + _logger_instance: Optional[logging.Logger] = None + _log_level = logging.INFO + + @classmethod + def setup_logger(cls, log_level=logging.INFO): + cls._log_level = log_level + if cls._logger_instance is None: + cls._initialize_logger() + else: + cls._logger_instance.setLevel(cls._log_level) + + @classmethod + def _initialize_logger(cls): + logging.basicConfig( + level=cls._log_level, format="%(asctime)s - %(levelname)s - %(message)s" + ) + cls._logger_instance = logging.getLogger("AppLogger") + cls._logger_instance.setLevel(cls._log_level) + + @classmethod + def get_logger(cls) -> logging.Logger: + if cls._logger_instance is None: + cls._initialize_logger() + + assert cls._logger_instance is not None, "Logger instance is not initialized" + return cls._logger_instance + + @classmethod + def log_info(cls, message: str): + logger = cls.get_logger() + logger.info(message) + + @classmethod + def log_debug(cls, message: str): + logger = cls.get_logger() + logger.debug(message) + + @classmethod + def log_warning(cls, message: str): + logger = cls.get_logger() + logger.warning(message) + + @classmethod + def log_error(cls, message: str): + logger = cls.get_logger() + logger.error(message) diff --git a/src/scraper.py b/src/scraper.py index 45aad5f..c3fd671 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -3,11 +3,11 @@ import json import os import re -import logging -from urllib.parse import parse_qs, urlencode, urlparse, urlunparse from pathlib import Path +from urllib.parse import parse_qs, urlencode, urlparse, urlunparse import requests +from logger import LoggerSingleton as logger GAME_INFO_URL = "https://api.screenscraper.fr/api2/jeuInfos.php" USER_INFO_URL = "https://api.screenscraper.fr/api2/ssuserInfos.php" @@ -16,15 +16,6 @@ VALID_MEDIA_TYPES = {"box-2D", "box-3D", "mixrbv1", "mixrbv2", "ss"} -def configure_logging(): - logging.basicConfig( - level=logging.ERROR, format="%(asctime)s - %(levelname)s - %(message)s" - ) - - -configure_logging() - - def get_image_files_without_extension(folder): return [ f.stem for f in Path(folder).glob("*") if f.suffix.lower() in IMAGE_EXTENSIONS @@ -38,7 +29,7 @@ def get_txt_files_without_extension(folder): def sha1sum(file_path): file_size = os.path.getsize(file_path) if file_size > MAX_FILE_SIZE_BYTES: - logging.warning(f"File {file_path} exceeds max file size limit.") + logger.log_warning(f"File {file_path} exceeds max file size limit.") return "" hash_sha1 = hashlib.sha1() @@ -47,7 +38,7 @@ def sha1sum(file_path): for chunk in iter(lambda: f.read(4096), b""): hash_sha1.update(chunk) except IOError as e: - logging.error(f"Error reading file {file_path}: {e}") + logger.log_error(f"Error reading file {file_path}: {e}") return "" return hash_sha1.hexdigest() @@ -66,7 +57,7 @@ def file_size(file_path): try: return os.path.getsize(file_path) except OSError as e: - logging.error(f"Error getting size of file {file_path}: {e}") + logger.log_error(f"Error getting size of file {file_path}: {e}") return None @@ -87,7 +78,8 @@ def parse_find_game_url(system_id, rom_path, dev_id, dev_password, username, pas try: return urlunparse(urlparse(GAME_INFO_URL)._replace(query=urlencode(params))) except UnicodeDecodeError as e: - logging.error(f"Error encoding URL: {e}. ROM params: {params}") + logger.log_debug("Params: %s") + logger.log_error(f"Error encoding URL: {e}. ROM params: {params}") return None @@ -103,7 +95,7 @@ def parse_user_info_url(dev_id, dev_password, username, password): try: return urlunparse(urlparse(USER_INFO_URL)._replace(query=urlencode(params))) except UnicodeDecodeError as e: - logging.error(f"Error encoding URL: {e}. User info params: {params}") + logger.log_error(f"Error encoding URL: {e}. User info params: {params}") return None @@ -112,7 +104,7 @@ def find_media_url_by_region(medias, media_type, regions): for media in medias: if media["type"] == media_type and media["region"] == region: return media["url"] - logging.error(f"Media not found for regions: {regions}") + logger.log_error(f"Media not found for regions: {regions}") return None @@ -125,20 +117,20 @@ def add_wh_to_media_url(media_url, width, height): def is_media_type_valid(media_type): if media_type not in VALID_MEDIA_TYPES: - logging.error(f"Unknown media type: {media_type}") + logger.log_error(f"Unknown media type: {media_type}") return False return True def check_destination(dest): if os.path.exists(dest): - logging.error(f"Destination file already exists: {dest}") + logger.log_error(f"Destination file already exists: {dest}") return None dest_dir = os.path.dirname(dest) try: os.makedirs(dest_dir, exist_ok=True) except OSError as e: - logging.error(f"Error creating directory {dest_dir}: {e}") + logger.log_error(f"Error creating directory {dest_dir}: {e}") return None @@ -148,10 +140,10 @@ def get(url): response = session.get(url, timeout=10) response.raise_for_status() except requests.Timeout: - logging.error("Request timed out") + logger.log_error("Request timed out") return None except requests.RequestException as e: - logging.error(f"Error making HTTP request: {e}") + logger.log_error(f"Error making HTTP request: {e}") return None return response.content @@ -160,22 +152,22 @@ def fetch_data(url): try: body = get(url) if not body: - logging.error("Empty response body") + logger.log_error("Empty response body") return None body_str = body.decode("utf-8") if "API closed" in body_str: - logging.error("API is closed") + logger.log_error("API is closed") return None if "Erreur" in body_str: - logging.error("Error found in response: %s", body_str) + logger.log_error("Error found in response: %s", body_str) return None return json.loads(body_str) except json.JSONDecodeError as e: - logging.error(f"Error decoding JSON response: {e}") + logger.log_error(f"Error decoding JSON response: {e}") except Exception as e: - logging.error(f"Error fetching data from URL: {e}") + logger.log_error(f"Error fetching data from URL: {e}") return None @@ -211,7 +203,7 @@ def fetch_box(game, config): regions = config.get("regions", ["us", "ame", "wor"]) box = _fetch_media(medias, config["box"], regions) if not box: - logging.error(f"Error downloading box: {game['response']['jeu']['medias']}") + logger.log_error(f"Error downloading box: {game['response']['jeu']['medias']}") return None return box @@ -221,7 +213,9 @@ def fetch_preview(game, config): regions = config.get("regions", ["us", "ame", "wor"]) preview = _fetch_media(medias, config["preview"], regions) if not preview: - logging.error(f"Error downloading preview: {game['response']['jeu']['medias']}") + logger.log_error( + f"Error downloading preview: {game['response']['jeu']['medias']}" + ) return None return preview