diff --git a/deemon/__init__.py b/deemon/__init__.py index ac76416..7084942 100644 --- a/deemon/__init__.py +++ b/deemon/__init__.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 from deemon.utils import startup -__version__ = '2.17.2' -__dbversion__ = '3.6' +__version__ = '2.18' +__dbversion__ = '3.7' appdata = startup.get_appdata_dir() startup.init_appdata_dir(appdata) diff --git a/deemon/cli.py b/deemon/cli.py index 82a7c3f..ce891fa 100644 --- a/deemon/cli.py +++ b/deemon/cli.py @@ -8,7 +8,7 @@ from packaging.version import parse as parse_version from deemon import __version__ -from deemon.cmd import download, rollback, backup, extra, tests +from deemon.cmd import download, rollback, backup, extra, tests, upgradelib from deemon.cmd.artistconfig import artist_lookup from deemon.cmd.monitor import Monitor from deemon.cmd.profile import ProfileConfig @@ -127,19 +127,24 @@ def test(email, exclusions): @run.command(name='download', no_args_is_help=True) @click.argument('artist', nargs=-1, required=False) +@click.option('-m', '--monitored', is_flag=True, help='Download all currently monitored artists') +@click.option('-i', '--artist-id', multiple=True, metavar='ID', type=int, help='Download by artist ID') @click.option('-A', '--album-id', multiple=True, metavar='ID', type=int, help='Download by album ID') +@click.option('-T', '--track-id', multiple=True, metavar='ID', type=int, help='Download by track ID') +@click.option('-u', '--url', metavar='URL', multiple=True, help='Download by URL of artist/album/track/playlist') +@click.option('-f', '--file', metavar='FILE', help='Download batch of artists or artist IDs from file', hidden=True) +@click.option('--artist-file', metavar='FILE', help='Download batch of artists or artist IDs from file') +@click.option('--album-file', metavar='FILE', help='Download batch of album IDs from file') +@click.option('--track-file', metavar='FILE', help='Download batch of track IDs from file') @click.option('-a', '--after', 'from_date', metavar="YYYY-MM-DD", type=str, help='Grab releases released after this date') @click.option('-B', '--before', 'to_date', metavar="YYYY-MM-DD", type=str, help='Grab releases released before this date') @click.option('-b', '--bitrate', metavar="BITRATE", help='Set custom bitrate for this operation') -@click.option('-f', '--file', metavar='FILE', help='Download batch of artists and/or artist IDs from file') -@click.option('-i', '--artist-id', multiple=True, metavar='ID', type=int, help='Download by artist ID') -@click.option('-m', '--monitored', is_flag=True, help='Download all currently monitored artists') @click.option('-o', '--download-path', metavar="PATH", type=str, help='Specify custom download directory') @click.option('-t', '--record-type', metavar="TYPE", type=str, help='Specify record types to download') -@click.option('-u', '--url', metavar='URL', multiple=True, help='Download by URL of artist/album/track/playlist') def download_command(artist, artist_id, album_id, url, file, bitrate, record_type, download_path, from_date, to_date, - monitored): + monitored, track_id, track_file, artist_file, + album_file): """ Download specific artist, album ID or by URL @@ -148,6 +153,7 @@ def download_command(artist, artist_id, album_id, url, file, bitrate, download Mozart download -i 100 -t album -b 9 """ + if bitrate: config.set('bitrate', bitrate) if download_path: @@ -155,13 +161,15 @@ def download_command(artist, artist_id, album_id, url, file, bitrate, if record_type: config.set('record_type', record_type) - if monitored: - artists, artist_ids, album_ids, urls = None, None, None, None - else: - artists = dataprocessor.csv_to_list(artist) if artist else None - artist_ids = [x for x in artist_id] if artist_id else None - album_ids = [x for x in album_id] if album_id else None - urls = [x for x in url] if url else None + artists = dataprocessor.csv_to_list(artist) if artist else None + artist_ids = [x for x in artist_id] if artist_id else None + album_ids = [x for x in album_id] if album_id else None + track_ids = [x for x in track_id] if track_id else None + urls = [x for x in url] if url else None + + if file: + logger.info("WARNING: -f/--file has been replaced with --artist-file and will be removed in future versions.") + artist_file = file if download_path and download_path != "": if Path(download_path).exists: @@ -172,7 +180,7 @@ def download_command(artist, artist_id, album_id, url, file, bitrate, dl = download.Download() dl.set_dates(from_date, to_date) - dl.download(artists, artist_ids, album_ids, urls, file) + dl.download(artists, artist_ids, album_ids, urls, artist_file, track_file, album_file, track_ids, monitored=monitored) @run.command(name='monitor', context_settings={"ignore_unknown_options": False}, no_args_is_help=True) @@ -184,12 +192,13 @@ def download_command(artist, artist_id, album_id, url, file, bitrate, @click.option('-I', '--import', 'im', metavar="PATH", help="Monitor artists/IDs from file or directory") @click.option('-i', '--artist-id', is_flag=True, help="Monitor artist by ID") @click.option('-p', '--playlist', is_flag=True, help='Monitor Deezer playlist by URL') +@click.option('--include-artists', is_flag=True, help='Also monitor artists from playlist') @click.option('-u', '--url', is_flag=True, help='Monitor artist by URL') @click.option('-R', '--remove', is_flag=True, help='Stop monitoring an artist') @click.option('-s', '--search', 'search_flag', is_flag=True, help='Show similar artist results to choose from') @click.option('-T', '--time-machine', type=str, metavar="YYYY-MM-DD", help="Refresh newly added artists on this date") @click.option('-t', '--record-type', metavar="TYPE", type=str, help='Specify record types to download') -def monitor_command(artist, im, playlist, bitrate, record_type, alerts, artist_id, +def monitor_command(artist, im, playlist, include_artists, bitrate, record_type, alerts, artist_id, dl, remove, url, download_path, search_flag, time_machine): """ Monitor artist for new releases by ID, URL or name. @@ -248,7 +257,7 @@ def monitor_command(artist, im, playlist, bitrate, record_type, alerts, artist_i if im: monitor.importer(im) elif playlist: - monitor.playlists(playlist_id) + monitor.playlists(playlist_id, include_artists) elif artist_id: monitor.artist_ids(dataprocessor.csv_to_list(artist)) elif artist: @@ -435,11 +444,13 @@ def profile_command(profile, add, clear, delete, edit): else: pc.show() + @run.command(name="extra") def extra_command(): """Fetch extra release info""" extra.main() + @run.command(name="search") @click.argument('query', nargs=-1, required=False) def search(query): @@ -469,3 +480,24 @@ def rollback_command(num, view): elif num: rollback.rollback_last(num) + +@click.group(name="library") +def library_command(): + """ + Library options such as upgrading from MP3 to FLAC + """ + + +@library_command.command(name="upgrade") +@click.argument('library', metavar='PATH') +@click.option('-A', '--album-only', is_flag=True, help="Get album IDs instead of track IDs (Fastest)") +@click.option('-E', '--allow-exclusions', is_flag=True, help="Allow exclusions to be applied") +@click.option('-O', '--output', metavar='PATH', help="Output file to save IDs (default: current directory)") +def library_upgrade_command(library, output, album_only, allow_exclusions): + """ (BETA) Scans MP3 files in PATH and generates a text file containing album/track IDs """ + if not output: + output = Path.cwd() + upgradelib.upgrade(library, output, album_only, allow_exclusions) + + +run.add_command(library_command) diff --git a/deemon/cmd/download.py b/deemon/cmd/download.py index bac8d81..6a06944 100644 --- a/deemon/cmd/download.py +++ b/deemon/cmd/download.py @@ -1,5 +1,7 @@ import logging import os +import sys + import requests from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -65,7 +67,7 @@ def __init__(self, artist=None, album=None, track=None, playlist=None, self.artist_name = track["artist"]["name"] self.track_id = track["id"] self.track_title = track["title"] - self.url = track["link"] + self.url = f"https://deezer.com/track/{self.track_id}" if playlist: self.url = playlist["url"] @@ -224,7 +226,8 @@ def download_queue(self, queue_list: list = None): refresh_plex(plex) return True - def download(self, artist, artist_id, album_id, url, input_file, auto=True, from_date: str = None): + def download(self, artist, artist_id, album_id, url, + artist_file, track_file, album_file, track_id, auto=True, monitored=False): def filter_artist_by_record_type(artist): album_api = self.api.get_artist_albums(query={'artist_name': '', 'artist_id': artist['id']}) @@ -263,7 +266,7 @@ def get_api_result(artist=None, artist_id=None, album_id=None, track_id=None): logger.error(f"Album ID {album_id} not found.") if track_id: try: - return self.dz.api.get_track(track_id) + return self.api.get_track(track_id) except (deezer.api.DataException, IndexError): logger.error(f"Track ID {track_id} not found.") @@ -305,7 +308,7 @@ def process_album_by_id(i): if not album_id_result: logger.debug(f"Album ID {i} was not found") return - logger.debug(f"Requested Album: {i}, " + logger.debug(f"Requested album: {i}, " f"Found: {album_id_result['artist']['name']} - {album_id_result['title']}") if album_id_result and not queue_item_exists(album_id_result['id']): self.queue_list.append(QueueItem(album=album_id_result)) @@ -320,6 +323,17 @@ def process_track_by_id(id): if track_id_result and not queue_item_exists(id): self.queue_list.append(QueueItem(track=track_id_result)) + def process_track_file(id): + if not queue_item_exists(id): + track_data = { + "artist": { + "name": "TRACK ID" + }, + "id": id, + "title": id + } + self.queue_list.append(QueueItem(track=track_data)) + def process_playlist_by_id(id): playlist_api = self.api.get_playlist(id) self.queue_list.append(QueueItem(playlist=playlist_api)) @@ -356,7 +370,7 @@ def extract_id_from_url(url): logger.info(":: Getting releases that were released before " f"{dates.ui_date(self.release_to)}") - if all(not x for x in [artist, artist_id, album_id, url, input_file]): + if monitored: artist_id = self.db.get_all_monitored_artist_ids() if artist: @@ -368,11 +382,35 @@ def extract_id_from_url(url): if album_id: [process_album_by_id(i) for i in album_id] - if input_file: - logger.info(f":: Reading from file {input_file}") - if Path(input_file).exists(): - artists_csv = utils.dataprocessor.read_file_as_csv(input_file) - artist_list = utils.dataprocessor.process_input_file(artists_csv) + if track_id: + [process_track_by_id(i) for i in track_id] + + if album_file: + logger.info(f":: Reading from file {album_file}") + if Path(album_file).exists(): + album_list = utils.dataprocessor.read_file_as_csv(album_file, split_new_line=False) + album_list = utils.dataprocessor.process_input_file(album_list) + if album_list: + if isinstance(album_list[0], int): + with ThreadPoolExecutor(max_workers=self.api.max_threads) as ex: + _api_results = list(tqdm(ex.map(process_album_by_id, album_list), + total=len(album_list), + desc=f"Fetching album data for {len(album_list)} " + f"album(s), please wait...", ascii=" #", + bar_format=ui.TQDM_FORMAT)) + else: + logger.debug(f"Invalid album ID: \"{album_list[0]}\"") + logger.error(f"Invalid album ID file detected.") + else: + logger.error(f"The file {album_file} could not be found") + sys.exit() + + if artist_file: + # TODO artist_file is in different format than album_file and track_file + # TODO is one continuous CSV line better than separate lines? + logger.info(f":: Reading from file {artist_file}") + if Path(artist_file).exists(): + artist_list = utils.dataprocessor.read_file_as_csv(artist_file) if artist_list: if isinstance(artist_list[0], int): with ThreadPoolExecutor(max_workers=self.api.max_threads) as ex: @@ -381,15 +419,38 @@ def extract_id_from_url(url): desc=f"Fetching artist release data for {len(artist_list)} " f"artist(s), please wait...", ascii=" #", bar_format=ui.TQDM_FORMAT)) - else: - if isinstance(artist_list[0], int): - with ThreadPoolExecutor(max_workers=self.api.max_threads) as ex: - _api_results = list(tqdm(ex.map(process_artist_by_name, artist_list), - total=len(artist_list), - desc=f"Fetching artist release data for {len(artist_list)} " - f"artist(s), please wait...", - ascii=" #", - bar_format=ui.TQDM_FORMAT)) + elif isinstance(artist_list[0], str): + with ThreadPoolExecutor(max_workers=self.api.max_threads) as ex: + _api_results = list(tqdm(ex.map(process_artist_by_name, artist_list), + total=len(artist_list), + desc=f"Fetching artist release data for {len(artist_list)} " + f"artist(s), please wait...", + ascii=" #", + bar_format=ui.TQDM_FORMAT)) + else: + logger.error(f"The file {artist_file} could not be found") + sys.exit() + + if track_file: + logger.info(f":: Reading from file {track_file}") + if Path(track_file).exists(): + track_list = utils.dataprocessor.read_file_as_csv(track_file, split_new_line=False) + try: + track_list = [int(x) for x in track_list] + except TypeError: + logger.info("Track file must only contain track IDs") + return + + if track_list: + with ThreadPoolExecutor(max_workers=self.api.max_threads) as ex: + _api_results = list(tqdm(ex.map(process_track_file, track_list), + total=len(track_list), + desc=f"Fetching track release data for {len(track_list)} " + f"track(s), please wait...", ascii=" #", + bar_format=ui.TQDM_FORMAT)) + else: + logger.error(f"The file {track_file} could not be found") + sys.exit() if url: logger.debug("Processing URLs") diff --git a/deemon/cmd/monitor.py b/deemon/cmd/monitor.py index c1ac172..768eb68 100644 --- a/deemon/cmd/monitor.py +++ b/deemon/cmd/monitor.py @@ -123,7 +123,11 @@ def build_artist_query(self, api_result: list): self.db.commit() return True - def build_playlist_query(self, api_result: list): + def build_playlist_query(self, api_result: list, include_artists: bool): + + if include_artists: + include_artists = '1' + existing = self.db.get_all_monitored_playlist_ids() or [] playlists_to_add = [] pbar = tqdm(api_result, total=len(api_result), desc="Setting up playlists for monitoring...", ascii=" #", @@ -134,8 +138,16 @@ def build_playlist_query(self, api_result: list): if playlist['id'] in existing: logger.info(f" Already monitoring {playlist['title']}, skipping...") else: - playlist.update({'bitrate': self.bitrate, 'alerts': self.alerts, 'download_path': self.download_path, - 'profile_id': config.profile_id(), 'trans_id': config.transaction_id()}) + playlist.update( + { + 'bitrate': self.bitrate, + 'alerts': self.alerts, + 'download_path': self.download_path, + 'profile_id': config.profile_id(), + 'trans_id': config.transaction_id(), + 'monitor_artists': include_artists + } + ) playlists_to_add.append(playlist) if len(playlists_to_add): logger.debug("New playlists have been monitored. Saving changes to the database...") @@ -214,7 +226,7 @@ def importer(self, import_path: str): return # @performance.timeit - def playlists(self, playlists: list): + def playlists(self, playlists: list, include_artists: bool): if self.remove: return self.purge_playlists(ids=playlists) ids = [int(x) for x in playlists] @@ -225,7 +237,7 @@ def playlists(self, playlists: list): desc=f"Fetching playlist data for {len(ids):,} playlist(s), please wait...", ascii=" #", bar_format=ui.TQDM_FORMAT)) - if self.build_playlist_query(api_result): + if self.build_playlist_query(api_result, include_artists): self.call_refresh() else: print("") diff --git a/deemon/cmd/refresh.py b/deemon/cmd/refresh.py index 0d34fc9..e4dc2c2 100644 --- a/deemon/cmd/refresh.py +++ b/deemon/cmd/refresh.py @@ -1,5 +1,6 @@ import logging import re +import time from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta @@ -248,12 +249,19 @@ def run(self, artists: list = None, playlists: list = None): for payload in payload_container: self.prep_payload(payload) + playlist_monitor_artists = [] for payload in api_result['playlists']: if payload and len(payload): self.seen = self.db.get_playlist_tracks(payload['id']) payload['tracks'] = self.remove_existing_releases(payload, self.seen) self.filter_playlist_releases(payload) + if payload['monitor_artists']: + logger.debug(f"Artists from this playlist ({payload['id']}) are to be monitored!") + for track in payload['tracks']: + playlist_monitor_artists.append(track['artist_id']) + playlist_monitor_artists = list(set(playlist_monitor_artists)) + if self.skip_download: logger.info(f" [!] You have opted to skip downloads, clearing {len(self.queue_list):,} item(s) from queue...") self.queue_list.clear() @@ -263,7 +271,6 @@ def run(self, artists: list = None, playlists: list = None): dl = Download() dl.download_queue(self.queue_list) - if len(self.new_playlist_releases) or len(self.new_releases): if len(self.new_playlist_releases): logger.debug("Updating playlist releases in database...") @@ -284,6 +291,14 @@ def run(self, artists: list = None, playlists: list = None): notification = notifier.Notify(self.new_releases_alert) notification.send() + if playlist_monitor_artists: + print("") + logger.info(":: New artists to monitor, stand by...") + time.sleep(2) + from deemon.cmd.monitor import Monitor + monitor = Monitor() + monitor.artist_ids(playlist_monitor_artists) + def db_stats(self): artists = len(self.db.get_all_monitored_artist_ids()) playlists = len(self.db.get_all_monitored_playlist_ids()) diff --git a/deemon/cmd/upgradelib.py b/deemon/cmd/upgradelib.py new file mode 100644 index 0000000..9d2e90e --- /dev/null +++ b/deemon/cmd/upgradelib.py @@ -0,0 +1,446 @@ +import sys +import time +import logging +from datetime import timedelta +from pathlib import Path +from mutagen.easyid3 import EasyID3 +from itertools import groupby +from operator import itemgetter +from deezer import Deezer +from concurrent.futures import ThreadPoolExecutor +from unidecode import unidecode +from tqdm import tqdm +from deemon.core.common import exclude_filtered_versions +from deemon.core.config import Config as config + +logger = logging.getLogger(__name__) + +dz = Deezer() + +LIBRARY_ROOT = None +ALBUM_ONLY = None +ALLOW_EXCLUSIONS = None + +# TODO - Add an 'exclusions' key to albums/tracks for count +# TODO - to improve album title matching, extract all a-zA-Z0-9 and compare (remove special chars) + +library_metadata = [] +performance = { + 'startID3': 0, + 'endID3': 0, + 'completeID3': 0, + 'startAPI': 0, + 'endAPI': 0, + 'completeAPI': 0 +} + + +class Performance: + def __init__(self): + self.startID3 = 0 + self.endID3 = 0 + self.completeID3 = 0 + self.startAPI = 0 + self.endAPI = 0 + self.completeAPI = 0 + + def start(self, module: str): + if module == 'ID3': + self.startID3 = time.time() + elif module == 'API': + self.startAPI = time.time() + + def end(self, module: str): + if module == 'ID3': + self.endID3 = time.time() + self.completeID3 = self.endID3 - self.startID3 + elif module == 'API': + self.endAPI = time.time() + self.completeAPI = self.endAPI - self.startAPI + + +def read_metadata(file): + metadata = { + 'abs_path': file, + 'rel_path': str(file).replace(LIBRARY_ROOT, ".."), + 'error': None + } + + try: + _audio = EasyID3(file) + + # Remove featured artists from artist tag + metadata['artist'] = _audio['artist'][0].split("/")[0].strip() + + # Remove special character replacement for search query + metadata['album'] = _audio['album'][0].replace("_", " ").strip() + metadata['title'] = _audio['title'][0].strip() + except Exception as e: + metadata['error'] = e + + return metadata + + +def get_time_from_secs(secs): + td_str = str(timedelta(seconds=secs)) + x = td_str.split(":") + x[2] = x[2].split(".")[0] + + if x[0] != "0": + friendly_time = f"{x[0]} Hours {x[1]} Minutes {x[2]} Seconds" + elif x[1] != "00": + friendly_time = f"{x[1]} Minutes {x[2]} Seconds" + elif x[2] == "00": + friendly_time = f"Less than 1 second" + else: + friendly_time = f"{x[2]} Seconds" + + return friendly_time + + +def invalid_metadata(track: dict) -> bool: + if not all([track['artist'], track['album'], track['title']]): + return True + else: + return False + + +def get_artist_api(name: str) -> list: + """ Get list of artists with exact name matches from API """ + artist_api = dz.gw.search(name)['ARTIST']['data'] + artist_matches = [] + + for artist in artist_api: + if artist['ART_NAME'].lower() == name.lower(): + artist_matches.append(artist) + + return artist_matches + + +def get_artist_discography_api(artist_name, artist_id) -> list: + """ Get list of albums with exact name matches from API """ + album_search = dz.gw.search(artist_name)['ALBUM']['data'] + album_gw = dz.gw.get_artist_discography(artist_id)['data'] + album_api = dz.api.get_artist_albums(artist_id)['data'] + + albums = [] + + for album in album_api: + if album['record_type'] == 'single': + album['record_type'] = '0' + elif album['record_type'] == 'album': + album['record_type'] = '1' + elif album['record_type'] == 'compilation': + album['record_type'] = '2' + elif album['record_type'] == 'ep': + album['record_type'] = '3' + + if album['explicit_lyrics']: + album['explicit_lyrics'] = '1' + else: + album['explicit_lyrics'] = '0' + + alb = { + 'ALB_ID': str(album['id']), + 'ALB_TITLE': album['title'], + 'EXPLICIT_LYRICS': album['explicit_lyrics'], + 'TYPE': album['record_type'] + } + albums.append(alb) + + for album in album_gw: + if album['ALB_ID'] not in [x['ALB_ID'] for x in albums]: + albums.append(album) + + for album in album_search: + if album['ART_ID'] == artist_id: + if album['ALB_ID'] not in [x['ALB_ID'] for x in albums]: + # Album returned via Search is missing EXPLICIT_LYRICS key + if not album.get('EXPLICIT_LYRICS'): + album['EXPLICIT_LYRICS'] = '0' + albums.append(album) + + return albums + + +def get_album_tracklist_api(album_id: str) -> list: + """ Get tracklist for album based on album_id """ + tracklist_api = dz.gw.get_album_tracks(album_id) + return tracklist_api + + +def retrieve_track_ids_per_artist(discography: tuple): + artist = discography[0] + albums = discography[1] + + # TODO - Need to implement this + duplicate_artists = False + found_artist = False + + track_ids = [] + album_ids = [] + + api_artists = get_artist_api(artist) + + tqdm.write(f"Getting track IDs for tracks by \"{artist}\"") + + if len(api_artists): + if len(api_artists) > 1: + tqdm.write(f"Duplicate artists detected for \"{artist}\"") + duplicate_artists = True + + for api_artist in api_artists: + + if found_artist: + tqdm.write("Found correct artist, skipping duplicates") + break + + if duplicate_artists: + tqdm.write(f"Searching with: {api_artist['ART_NAME']} ({api_artist['ART_ID']})") + + discog = get_artist_discography_api(api_artist['ART_NAME'], api_artist['ART_ID']) + + for album, tracks in groupby(albums, key=itemgetter('album')): + + # Convert itertools.groupby to list so we can use it more than once + tracks = [track for track in tracks] + + api_album_matches = [alb for alb in discog if alb['ALB_TITLE'].lower() == album.lower()] + + if ALLOW_EXCLUSIONS: + filtered_album_matches = exclude_filtered_versions(api_album_matches) + api_album = get_preferred_album(filtered_album_matches, len(tracks)) + else: + api_album = get_preferred_album(api_album_matches, len(tracks)) + + if ALBUM_ONLY: + if api_album: + album_ids.append(api_album) + continue + else: + album_ids.append( + { + 'artist': artist, + 'title': album, + 'info': "Album not found" + } + ) + continue + + if api_album: + tracklist = get_album_tracklist_api(api_album['ALB_ID']) + for track in tracks: + track_variations = [track['title'].lower(), unidecode(track['title']).lower()] + + for i, track_api in enumerate(tracklist, start=1): + if track_api['SNG_TITLE'].lower() in track_variations: + found_artist = True + track['id'] = track_api['SNG_ID'] + track_ids.append(track) + break + + elif f"{track_api['SNG_TITLE']} {track_api.get('VERSION', '')}".lower() in track_variations: + found_artist = True + track['id'] = track_api['SNG_ID'] + track_ids.append(track) + break + + if i == len(tracklist): + track['info'] = "Track not found" + tqdm.write(f"{track['info']}: {track['title']}") + track_ids.append(track) + break + else: + if duplicate_artists: + info = f"Album not found under artist ID {api_artist['ART_ID']}" + else: + info = f"Album not found" + tqdm.write(f"{info}: {album}") + for track in tracks: + track['info'] = info + track_ids.append(track) + else: + tqdm.write(f"Artist not found: {artist}") + for album, tracks in groupby(albums, key=itemgetter('album')): + for track in tracks: + track['info'] = "Artist not found" + track_ids.append(track) + + if ALBUM_ONLY: + return album_ids + else: + return track_ids + + +def get_preferred_album(api_albums: list, num_tracks: int): + """ Return preferred album order based on config.prefer_explicit() """ + preferred_album = None + + if num_tracks < 4: + preferred_album = [album for album in api_albums if album['EXPLICIT_LYRICS'] == '1' and album['TYPE'] == '0'] + if not preferred_album: + preferred_album = [album for album in api_albums if album['TYPE'] == '0'] + + if num_tracks >= 4 or not preferred_album: + preferred_album = [album for album in api_albums if album['EXPLICIT_LYRICS'] == '1' and album['TYPE'] in ['1', '2', '3']] + if not preferred_album: + preferred_album = [album for album in api_albums if album['TYPE'] in ['1', '2', '3']] + + if preferred_album: + return preferred_album[0] + + +def get_preferred_track_id(title: str, tracklist: list): + """ Return preferred track ID by comparing against title of local track """ + track_id = None + for track in tracklist: + if track.get('VERSION'): + api_title = f"{track['SNG_TITLE']} {track['VERSION']}".lower() + if title.lower() == api_title: + return track['SNG_ID'] + else: + if track['SNG_TITLE'].lower() == title.lower(): + track_id = track['SNG_ID'] + return track_id + + +def upgrade(library, output, albums=False, exclusions=False): + + global ALBUM_ONLY + global ALLOW_EXCLUSIONS + global LIBRARY_ROOT + + ALBUM_ONLY = albums + ALLOW_EXCLUSIONS = exclusions + LIBRARY_ROOT = library + + output_ids = Path(output) / "library_upgrade_ids.txt" + output_log = Path(output) / "library_upgrade.log" + + perf = Performance() + logger.info("Scanning library, standby...") + logger.debug(f"Library path: {LIBRARY_ROOT}") + library_files = Path(LIBRARY_ROOT).glob("**/*.mp3") + files = [file for file in library_files if not file.name.startswith(".")] + files.sort() + + if files: + print(f"Found {len(files)} MP3 files") + else: + print("No MP3 files found") + sys.exit() + + perf.start('ID3') + with ThreadPoolExecutor(10) as executor: + library_metadata = list( + tqdm( + executor.map(read_metadata, files), + total=(len(files)), + desc="Reading metadata", + ) + ) + perf.end('ID3') + + library_metadata_errors = [file for file in library_metadata if file.get('error')] + library_metadata = [file for file in library_metadata if not file.get('error')] + + artists = sorted(library_metadata, key=itemgetter('artist')) + artist_list = [(artist, list(albums)) for artist, albums in groupby(artists, key=itemgetter('artist'))] + + perf.start('API') + with ThreadPoolExecutor(20) as executor: + result = list( + tqdm( + executor.map(retrieve_track_ids_per_artist, artist_list), + total=len(artist_list), + desc="Processing tracks by artist" + ) + ) + if ALBUM_ONLY: + album_result = result + track_result = [] + else: + track_result = result + album_result = [] + perf.end('API') + + albums = [album for artist in album_result for album in artist] + album_ids = [album['ALB_ID'] for album in albums if album.get('ALB_ID')] + album_not_found = [album for album in albums if not album.get('ALB_ID')] + + tracks = [track for artist in track_result for track in artist] + track_ids = [track['id'] for track in tracks if track.get('id') and not track.get('error')] + track_not_found = [track for track in tracks if not track.get('id') and not track.get('error')] + + # TODO move this to function for reuse in f.write() below + print(f"Found: {len(track_ids)} | Not Found: {len(track_not_found)} | " + f"Errors: {len(library_metadata_errors)} | Total: {len(files)}\n\n") + print(f"Time to read metadata: {get_time_from_secs(perf.completeID3)}") + print(f"Time to retrieve API data: {get_time_from_secs(perf.completeAPI)}\n\n") + + if (ALBUM_ONLY and album_ids) or track_ids: + with open(output_ids, "w") as f: + if ALBUM_ONLY and album_ids: + f.write(', '.join(album_ids)) + elif track_ids: + f.write(', '.join(track_ids)) + + with open(output_log, "w") as f: + if ALBUM_ONLY: + f.write(f"Albums Found: {len(album_ids)} | Albums Not Found: {len(album_not_found)} | " + f"ID3 Errors: {len(library_metadata_errors)} | Total Files: {len(files)}\n\n") + f.write(f"Time to read metadata: {get_time_from_secs(perf.completeID3)}\n") + f.write(f"Time to retrieve API data: {get_time_from_secs(perf.completeAPI)}\n\n") + + if library_metadata_errors: + f.write("The following files had missing/invalid ID3 tag data:\n\n") + for track in library_metadata_errors: + if track.get('error'): + f.write(f"\tFile: {track['rel_path']}\n") + f.write(f"\t\tError: {track['error']}\n\n") + f.write("\n") + + if album_not_found: + + artists_not_found = [track['artist'] for track in tracks if track['info'] == 'Artist not found'] + if artists_not_found: + f.write("The following artists were not found:\n") + for artist in artists_not_found: + f.write("\n\t{track['artist']\n") + + f.write("The following albums were not found:\n") + for album in album_not_found: + f.write(f"\n\tArtist: {album['artist']}\n") + f.write(f"\tAlbum: {album['title']}\n") + if album.get('info'): + f.write(f"\tInfo: {album['info']}\n") + else: + f.write(f"Tracks Found: {len(track_ids)} | Tracks Not Found: {len(track_not_found)} | " + f"ID3 Errors: {len(library_metadata_errors)} | Total Files: {len(files)}\n\n") + f.write(f"Time to read metadata: {get_time_from_secs(perf.completeID3)}\n") + f.write(f"Time to retrieve API data: {get_time_from_secs(perf.completeAPI)}\n\n") + + if library_metadata_errors: + f.write("The following files had missing/invalid ID3 tag data:\n\n") + for track in library_metadata_errors: + if track.get('error'): + f.write(f"\tFile: {track['rel_path']}\n") + f.write(f"\t\tError: {track['error']}\n\n") + f.write("\n") + + if track_not_found: + + artists_not_found = [track['artist'] for track in tracks if track.get('info', '') == 'Artist not found'] + if artists_not_found: + f.write("The following artists were not found:\n") + for artist in artists_not_found: + f.write("\n\t{track['artist']\n") + + f.write("The following tracks were not found:\n") + for track in track_not_found: + f.write(f"\n\tArtist: {track['artist']}\n") + f.write(f"\tAlbum: {track['album']}\n") + f.write(f"\tTrack: {track['title']}\n") + f.write(f"\tFile: {track['rel_path']}\n") + if track.get('info'): + f.write(f"\tInfo: {track['info']}\n") \ No newline at end of file diff --git a/deemon/core/api.py b/deemon/core/api.py index 6b1623c..9a23480 100644 --- a/deemon/core/api.py +++ b/deemon/core/api.py @@ -137,8 +137,37 @@ def get_album(self, query: int) -> dict: return {} return {'id': int(result['ALB_ID']), 'title': result['ALB_TITLE'], 'artist': {'name': result['ART_NAME']}} else: - logger.warning("Please enable the fast_api for album downloads") - return {} + try: + result = self.api.get_album(query) + except deezer.errors.DataException as e: + logger.debug(f"API error: {e}") + return + else: + return result + + def get_track(self, query: int) -> dict: + """Return a dictionary from API containing album info""" + if self.platform == "deezer-gw": + try: + result = self.api.get_track(query) + except deezer.errors.GWAPIError as e: + logger.debug(f"API error: {e}") + return {} + except json.decoder.JSONDecodeError: + logger.error(f" [!] Empty response from API while getting data for album ID {query}, retrying...") + try: + result = self.api.get_album(query) + except json.decoder.JSONDecodeError: + logger.error(f" [!] API still sending empty response for album ID {query}") + return {} + return {'id': int(result['SNG_ID']), 'title': result['SNG_TITLE'], 'artist': {'name': result['ART_NAME']}} + else: + try: + result = self.api.get_track(query) + except deezer.errors.DataException as e: + logger.debug(f"API error: {e}") + else: + return result def get_extra_release_info(self, query: dict): album = {'id': query['album_id'], 'label': None} diff --git a/deemon/core/db.py b/deemon/core/db.py index b5a210a..0ea01ee 100644 --- a/deemon/core/db.py +++ b/deemon/core/db.py @@ -82,7 +82,8 @@ def create_new_database(self): "'profile_id' INTEGER DEFAULT 1," "'download_path' TEXT," "'refreshed' INTEGER DEFAULT 0," - "'trans_id' INTEGER)") + "'trans_id' INTEGER," + "'monitor_artists' INTEGER DEFAULT 0)") self.query("CREATE TABLE playlist_tracks (" "'track_id' INTEGER," @@ -169,19 +170,19 @@ def do_upgrade(self): if current_ver < parse_version("3.5.2"): self.query("CREATE TABLE releases_tmp (" - "'artist_id' INTEGER," - "'artist_name' TEXT," - "'album_id' INTEGER," - "'album_name' TEXT," - "'album_release' TEXT," - "'album_added' INTEGER," - "'explicit' INTEGER," - "'label' TEXT," - "'record_type' INTEGER," - "'profile_id' INTEGER DEFAULT 1," - "'future_release' INTEGER DEFAULT 0," - "'trans_id' INTEGER," - "unique(album_id, profile_id))") + "'artist_id' INTEGER," + "'artist_name' TEXT," + "'album_id' INTEGER," + "'album_name' TEXT," + "'album_release' TEXT," + "'album_added' INTEGER," + "'explicit' INTEGER," + "'label' TEXT," + "'record_type' INTEGER," + "'profile_id' INTEGER DEFAULT 1," + "'future_release' INTEGER DEFAULT 0," + "'trans_id' INTEGER," + "unique(album_id, profile_id))") self.query("INSERT OR REPLACE INTO releases_tmp(artist_id, artist_name, " "album_id, album_name, album_release, album_added, " "explicit, label, record_type, profile_id, " @@ -193,12 +194,33 @@ def do_upgrade(self): self.query("ALTER TABLE releases_tmp RENAME TO releases") self.query("INSERT OR REPLACE INTO 'deemon' ('property', 'value') VALUES ('version', '3.5.2')") self.commit() - logger.debug(f"Database upgraded to version 3.5.2") if current_ver < parse_version("3.6"): # album_release_ts REMOVED pass + if current_ver < parse_version("3.7"): + self.query("CREATE TABLE playlists_tmp (" + "'id' INTEGER UNIQUE," + "'title' TEXT," + "'url' TEXT," + "'bitrate' TEXT," + "'alerts' INTEGER," + "'profile_id' INTEGER DEFAULT 1," + "'download_path' TEXT," + "'refreshed' INTEGER DEFAULT 0," + "'trans_id' INTEGER," + "'monitor_artists' INTEGER DEFAULT 0)") + self.query("INSERT OR REPLACE INTO playlists_tmp (id, title, url, bitrate, " + "alerts, profile_id, download_path, refreshed, trans_id) SELECT " + "id, title, url, bitrate, alerts, profile_id, download_path, " + "refreshed, trans_id FROM playlists") + self.query("DROP TABLE playlists") + self.query("ALTER TABLE playlists_tmp RENAME TO playlists") + self.query("INSERT OR REPLACE INTO 'deemon' ('property', 'value') VALUES ('version', '3.7')") + self.commit() + logger.debug(f"Database upgraded to version 3.7") + def query(self, query, values=None): if values is None: values = {} @@ -530,7 +552,7 @@ def fast_monitor(self, values): def fast_monitor_playlist(self, values): self.cursor.executemany( - "INSERT OR REPLACE INTO playlists (id, title, url, bitrate, alerts, profile_id, download_path, trans_id) VALUES (:id, :title, :link, :bitrate, :alerts, :profile_id, :download_path, :trans_id)", + "INSERT OR REPLACE INTO playlists (id, title, url, bitrate, alerts, profile_id, download_path, trans_id, monitor_artists) VALUES (:id, :title, :link, :bitrate, :alerts, :profile_id, :download_path, :trans_id, :monitor_artists)", values) def insert_multiple(self, table, values): diff --git a/deemon/core/dmi.py b/deemon/core/dmi.py index 718e276..75c7416 100644 --- a/deemon/core/dmi.py +++ b/deemon/core/dmi.py @@ -7,7 +7,7 @@ from deemix.downloader import Downloader from deemix.settings import load as LoadSettings from deemix.types.DownloadObjects import Collection -from deemix.utils import formatListener +from deemix.utils import formatListener, pathtemplates from deezer import Deezer from deezer.api import APIError from deezer.gw import GWAPIError @@ -38,6 +38,9 @@ def __init__(self): self.db = Database() self.dz = Deezer() + # Override deemix code causing unhandled TypeError exception on line 158 + pathtemplates.generateTrackName = self.generateTrackName + if config.deemix_path() == "": self.config_dir = localpaths.getConfigFolder() else: @@ -198,6 +201,62 @@ def generatePlaylistItem(self, dz, link_id, bitrate, playlistAPI=None, playlistT } }) + + def generateTrackName(filename, track, settings): + from deemix.utils.pathtemplates import fixName + + logger.debug("[PATCH] Overriding deemix method generateTrackName with bug fix") + + c = settings['illegalCharacterReplacer'] + filename = filename.replace("%title%", fixName(track.title, c)) + filename = filename.replace("%artist%", fixName(track.mainArtist.name, c)) + filename = filename.replace("%artists%", fixName(", ".join(track.artists), c)) + filename = filename.replace("%allartists%", fixName(track.artistsString, c)) + filename = filename.replace("%mainartists%", fixName(track.mainArtistsString, c)) + if track.featArtistsString: + filename = filename.replace("%featartists%", fixName('('+track.featArtistsString+')', c)) + else: + filename = filename.replace("%featartists%", '') + filename = filename.replace("%album%", fixName(track.album.title, c)) + filename = filename.replace("%albumartist%", fixName(track.album.mainArtist.name, c)) + filename = filename.replace("%tracknumber%", pad(track.trackNumber, track.album.trackTotal, settings)) + filename = filename.replace("%tracktotal%", str(track.album.trackTotal)) + filename = filename.replace("%discnumber%", str(track.discNumber)) + filename = filename.replace("%disctotal%", str(track.album.discTotal)) + if len(track.album.genre) > 0: + filename = filename.replace("%genre%", fixName(track.album.genre[0], c)) + else: + filename = filename.replace("%genre%", "Unknown") + filename = filename.replace("%year%", str(track.date.year)) + filename = filename.replace("%date%", track.dateString) + filename = filename.replace("%bpm%", str(track.bpm)) + filename = filename.replace("%label%", fixName(track.album.label, c)) + filename = filename.replace("%isrc%", track.ISRC) + + """ BEGIN DEEMON PATCH """ + """ Catches exception when track.album.barcode == None """ + try: + filename = filename.replace("%upc%", track.album.barcode) + except TypeError: + pass + """ END DEEMON PATCH """ + + filename = filename.replace("%explicit%", "(Explicit)" if track.explicit else "") + + filename = filename.replace("%track_id%", str(track.id)) + filename = filename.replace("%album_id%", str(track.album.id)) + filename = filename.replace("%artist_id%", str(track.mainArtist.id)) + if track.playlist: + filename = filename.replace("%playlist_id%", str(track.playlist.playlistID)) + filename = filename.replace("%position%", pad(track.position, track.playlist.trackTotal, settings)) + else: + filename = filename.replace("%playlist_id%", '') + filename = filename.replace("%position%", pad(track.position, track.album.trackTotal, settings)) + filename = filename.replace('\\', pathSep).replace('/', pathSep) + return antiDot(fixLongName(filename)) + + + class GenerationError(Exception): def __init__(self, link, message, errid=None): super().__init__() diff --git a/deemon/utils/dataprocessor.py b/deemon/utils/dataprocessor.py index dcf5757..b065060 100644 --- a/deemon/utils/dataprocessor.py +++ b/deemon/utils/dataprocessor.py @@ -4,11 +4,15 @@ logger = logging.getLogger(__name__) -def read_file_as_csv(file): +def read_file_as_csv(file, split_new_line=True): with open(file, 'r', encoding="utf-8-sig", errors="replace") as f: make_csv = f.read() - csv_to_list = make_csv.split('\n') + if split_new_line: + csv_to_list = make_csv.split('\n') + else: + csv_to_list = make_csv.split(', ') sorted_list = sorted(list(filter(None, csv_to_list))) + sorted_list = list(set(sorted_list)) return sorted_list diff --git a/requirements.txt b/requirements.txt index 7ec9ebb..845e2f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ deemix~=3.6 -packaging -requests -click~=8.0 -setuptools -PlexAPI~=4.5 -tqdm~=4.61 +packaging~=23.0 +requests~=2.28.0 +click~=8.1.0 +setuptools~=65.6.3 +PlexAPI~=4.5.2 +tqdm~=4.61.0 +mutagen~=1.46 +Unidecode~=1.3.6