Merge branch 'beta'

deemon/__init__.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 from deemon.utils import startup
 
-__version__ = '2.17.2'
-__dbversion__ = '3.6'
+__version__ = '2.18'
+__dbversion__ = '3.7'
 
 appdata = startup.get_appdata_dir()
 startup.init_appdata_dir(appdata)

deemon/cli.py

@@ -8,7 +8,7 @@
 from packaging.version import parse as parse_version
 
 from deemon import __version__
-from deemon.cmd import download, rollback, backup, extra, tests
+from deemon.cmd import download, rollback, backup, extra, tests, upgradelib
 from deemon.cmd.artistconfig import artist_lookup
 from deemon.cmd.monitor import Monitor
 from deemon.cmd.profile import ProfileConfig
@@ -127,19 +127,24 @@ def test(email, exclusions):
 
 @run.command(name='download', no_args_is_help=True)
 @click.argument('artist', nargs=-1, required=False)
+@click.option('-m', '--monitored', is_flag=True, help='Download all currently monitored artists')
+@click.option('-i', '--artist-id', multiple=True, metavar='ID', type=int, help='Download by artist ID')
 @click.option('-A', '--album-id', multiple=True, metavar='ID', type=int, help='Download by album ID')
+@click.option('-T', '--track-id', multiple=True, metavar='ID', type=int, help='Download by track ID')
+@click.option('-u', '--url', metavar='URL', multiple=True, help='Download by URL of artist/album/track/playlist')
+@click.option('-f', '--file', metavar='FILE', help='Download batch of artists or artist IDs from file', hidden=True)
+@click.option('--artist-file', metavar='FILE', help='Download batch of artists or artist IDs from file')
+@click.option('--album-file', metavar='FILE', help='Download batch of album IDs from file')
+@click.option('--track-file', metavar='FILE', help='Download batch of track IDs from file')
 @click.option('-a', '--after', 'from_date', metavar="YYYY-MM-DD", type=str, help='Grab releases released after this date')
 @click.option('-B', '--before', 'to_date', metavar="YYYY-MM-DD", type=str, help='Grab releases released before this date')
 @click.option('-b', '--bitrate', metavar="BITRATE", help='Set custom bitrate for this operation')
-@click.option('-f', '--file', metavar='FILE', help='Download batch of artists and/or artist IDs from file')
-@click.option('-i', '--artist-id', multiple=True, metavar='ID', type=int, help='Download by artist ID')
-@click.option('-m', '--monitored', is_flag=True, help='Download all currently monitored artists')
 @click.option('-o', '--download-path', metavar="PATH", type=str, help='Specify custom download directory')
 @click.option('-t', '--record-type', metavar="TYPE", type=str, help='Specify record types to download')
-@click.option('-u', '--url', metavar='URL', multiple=True, help='Download by URL of artist/album/track/playlist')
 def download_command(artist, artist_id, album_id, url, file, bitrate,
                      record_type, download_path, from_date, to_date,
-                     monitored):
+                     monitored, track_id, track_file, artist_file,
+                     album_file):
     """
     Download specific artist, album ID or by URL
 
@@ -148,20 +153,23 @@ def download_command(artist, artist_id, album_id, url, file, bitrate,
         download Mozart
         download -i 100 -t album -b 9
     """
+
     if bitrate:
         config.set('bitrate', bitrate)
     if download_path:
         config.set('download_path', download_path)
     if record_type:
         config.set('record_type', record_type)
 
-    if monitored:
-        artists, artist_ids, album_ids, urls = None, None, None, None
-    else:
-        artists = dataprocessor.csv_to_list(artist) if artist else None
-        artist_ids = [x for x in artist_id] if artist_id else None
-        album_ids = [x for x in album_id] if album_id else None
-        urls = [x for x in url] if url else None
+    artists = dataprocessor.csv_to_list(artist) if artist else None
+    artist_ids = [x for x in artist_id] if artist_id else None
+    album_ids = [x for x in album_id] if album_id else None
+    track_ids = [x for x in track_id] if track_id else None
+    urls = [x for x in url] if url else None
+
+    if file:
+        logger.info("WARNING: -f/--file has been replaced with --artist-file and will be removed in future versions.")
+        artist_file = file
 
     if download_path and download_path != "":
         if Path(download_path).exists:
@@ -172,7 +180,7 @@ def download_command(artist, artist_id, album_id, url, file, bitrate,
 
     dl = download.Download()
     dl.set_dates(from_date, to_date)
-    dl.download(artists, artist_ids, album_ids, urls, file)
+    dl.download(artists, artist_ids, album_ids, urls, artist_file, track_file, album_file, track_ids, monitored=monitored)
 
 
 @run.command(name='monitor', context_settings={"ignore_unknown_options": False}, no_args_is_help=True)
@@ -184,12 +192,13 @@ def download_command(artist, artist_id, album_id, url, file, bitrate,
 @click.option('-I', '--import', 'im', metavar="PATH", help="Monitor artists/IDs from file or directory")
 @click.option('-i', '--artist-id', is_flag=True, help="Monitor artist by ID")
 @click.option('-p', '--playlist', is_flag=True, help='Monitor Deezer playlist by URL')
+@click.option('--include-artists', is_flag=True, help='Also monitor artists from playlist')
 @click.option('-u', '--url', is_flag=True, help='Monitor artist by URL')
 @click.option('-R', '--remove', is_flag=True, help='Stop monitoring an artist')
 @click.option('-s', '--search', 'search_flag', is_flag=True, help='Show similar artist results to choose from')
 @click.option('-T', '--time-machine', type=str, metavar="YYYY-MM-DD", help="Refresh newly added artists on this date")
 @click.option('-t', '--record-type', metavar="TYPE", type=str, help='Specify record types to download')
-def monitor_command(artist, im, playlist, bitrate, record_type, alerts, artist_id,
+def monitor_command(artist, im, playlist, include_artists, bitrate, record_type, alerts, artist_id,
                     dl, remove, url, download_path, search_flag, time_machine):
     """
     Monitor artist for new releases by ID, URL or name.
@@ -248,7 +257,7 @@ def monitor_command(artist, im, playlist, bitrate, record_type, alerts, artist_i
     if im:
         monitor.importer(im)
     elif playlist:
-        monitor.playlists(playlist_id)
+        monitor.playlists(playlist_id, include_artists)
     elif artist_id:
         monitor.artist_ids(dataprocessor.csv_to_list(artist))
     elif artist:
@@ -435,11 +444,13 @@ def profile_command(profile, add, clear, delete, edit):
     else:
         pc.show()
 
+
 @run.command(name="extra")
 def extra_command():
     """Fetch extra release info"""
     extra.main()
 
+
 @run.command(name="search")
 @click.argument('query', nargs=-1, required=False)
 def search(query):
@@ -469,3 +480,24 @@ def rollback_command(num, view):
     elif num:
         rollback.rollback_last(num)
 
+
+@click.group(name="library")
+def library_command():
+    """
+    Library options such as upgrading from MP3 to FLAC
+    """
+
+
+@library_command.command(name="upgrade")
+@click.argument('library', metavar='PATH')
+@click.option('-A', '--album-only', is_flag=True, help="Get album IDs instead of track IDs (Fastest)")
+@click.option('-E', '--allow-exclusions', is_flag=True, help="Allow exclusions to be applied")
+@click.option('-O', '--output', metavar='PATH', help="Output file to save IDs (default: current directory)")
+def library_upgrade_command(library, output, album_only, allow_exclusions):
+    """ (BETA) Scans MP3 files in PATH and generates a text file containing album/track IDs """
+    if not output:
+        output = Path.cwd()
+    upgradelib.upgrade(library, output, album_only, allow_exclusions)
+
+
+run.add_command(library_command)

deemon/cmd/download.py

@@ -1,5 +1,7 @@
 import logging
 import os
+import sys
+
 import requests
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
@@ -65,7 +67,7 @@ def __init__(self, artist=None, album=None, track=None, playlist=None,
             self.artist_name = track["artist"]["name"]
             self.track_id = track["id"]
             self.track_title = track["title"]
-            self.url = track["link"]
+            self.url = f"https://deezer.com/track/{self.track_id}"
 
         if playlist:
             self.url = playlist["url"]
@@ -224,7 +226,8 @@ def download_queue(self, queue_list: list = None):
                 refresh_plex(plex)
         return True
 
-    def download(self, artist, artist_id, album_id, url, input_file, auto=True, from_date: str = None):
+    def download(self, artist, artist_id, album_id, url,
+                 artist_file, track_file, album_file, track_id, auto=True, monitored=False):
 
         def filter_artist_by_record_type(artist):
             album_api = self.api.get_artist_albums(query={'artist_name': '', 'artist_id': artist['id']})
@@ -263,7 +266,7 @@ def get_api_result(artist=None, artist_id=None, album_id=None, track_id=None):
                     logger.error(f"Album ID {album_id} not found.")
             if track_id:
                 try:
-                    return self.dz.api.get_track(track_id)
+                    return self.api.get_track(track_id)
                 except (deezer.api.DataException, IndexError):
                     logger.error(f"Track ID {track_id} not found.")
 
@@ -305,7 +308,7 @@ def process_album_by_id(i):
             if not album_id_result:
                 logger.debug(f"Album ID {i} was not found")
                 return
-            logger.debug(f"Requested Album: {i}, "
+            logger.debug(f"Requested album: {i}, "
                          f"Found: {album_id_result['artist']['name']} - {album_id_result['title']}")
             if album_id_result and not queue_item_exists(album_id_result['id']):
                 self.queue_list.append(QueueItem(album=album_id_result))
@@ -320,6 +323,17 @@ def process_track_by_id(id):
             if track_id_result and not queue_item_exists(id):
                 self.queue_list.append(QueueItem(track=track_id_result))
 
+        def process_track_file(id):
+            if not queue_item_exists(id):
+                track_data = {
+                    "artist": {
+                        "name": "TRACK ID"
+                    },
+                    "id": id,
+                    "title": id
+                }
+                self.queue_list.append(QueueItem(track=track_data))
+
         def process_playlist_by_id(id):
             playlist_api = self.api.get_playlist(id)
             self.queue_list.append(QueueItem(playlist=playlist_api))
@@ -356,7 +370,7 @@ def extract_id_from_url(url):
                 logger.info(":: Getting releases that were released before "
                             f"{dates.ui_date(self.release_to)}")
 
-        if all(not x for x in [artist, artist_id, album_id, url, input_file]):
+        if monitored:
             artist_id = self.db.get_all_monitored_artist_ids()
 
         if artist:
@@ -368,11 +382,35 @@ def extract_id_from_url(url):
         if album_id:
             [process_album_by_id(i) for i in album_id]
 
-        if input_file:
-            logger.info(f":: Reading from file {input_file}")
-            if Path(input_file).exists():
-                artists_csv = utils.dataprocessor.read_file_as_csv(input_file)
-                artist_list = utils.dataprocessor.process_input_file(artists_csv)
+        if track_id:
+            [process_track_by_id(i) for i in track_id]
+
+        if album_file:
+            logger.info(f":: Reading from file {album_file}")
+            if Path(album_file).exists():
+                album_list = utils.dataprocessor.read_file_as_csv(album_file, split_new_line=False)
+                album_list = utils.dataprocessor.process_input_file(album_list)
+                if album_list:
+                    if isinstance(album_list[0], int):
+                        with ThreadPoolExecutor(max_workers=self.api.max_threads) as ex:
+                            _api_results = list(tqdm(ex.map(process_album_by_id, album_list),
+                                                     total=len(album_list),
+                                                     desc=f"Fetching album data for {len(album_list)} "
+                                                          f"album(s), please wait...", ascii=" #",
+                                                     bar_format=ui.TQDM_FORMAT))
+                    else:
+                        logger.debug(f"Invalid album ID: \"{album_list[0]}\"")
+                        logger.error(f"Invalid album ID file detected.")
+            else:
+                logger.error(f"The file {album_file} could not be found")
+                sys.exit()
+
+        if artist_file:
+            # TODO artist_file is in different format than album_file and track_file
+            # TODO is one continuous CSV line better than separate lines?
+            logger.info(f":: Reading from file {artist_file}")
+            if Path(artist_file).exists():
+                artist_list = utils.dataprocessor.read_file_as_csv(artist_file)
                 if artist_list:
                     if isinstance(artist_list[0], int):
                         with ThreadPoolExecutor(max_workers=self.api.max_threads) as ex:
@@ -381,15 +419,38 @@ def extract_id_from_url(url):
                                                      desc=f"Fetching artist release data for {len(artist_list)} "
                                                           f"artist(s), please wait...", ascii=" #",
                                                      bar_format=ui.TQDM_FORMAT))
-                    else:
-                        if isinstance(artist_list[0], int):
-                            with ThreadPoolExecutor(max_workers=self.api.max_threads) as ex:
-                                _api_results = list(tqdm(ex.map(process_artist_by_name, artist_list),
-                                                         total=len(artist_list),
-                                                         desc=f"Fetching artist release data for {len(artist_list)} "
-                                                              f"artist(s), please wait...",
-                                                         ascii=" #",
-                                                         bar_format=ui.TQDM_FORMAT))
+                    elif isinstance(artist_list[0], str):
+                        with ThreadPoolExecutor(max_workers=self.api.max_threads) as ex:
+                            _api_results = list(tqdm(ex.map(process_artist_by_name, artist_list),
+                                                     total=len(artist_list),
+                                                     desc=f"Fetching artist release data for {len(artist_list)} "
+                                                          f"artist(s), please wait...",
+                                                     ascii=" #",
+                                                     bar_format=ui.TQDM_FORMAT))
+            else:
+                logger.error(f"The file {artist_file} could not be found")
+                sys.exit()
+
+        if track_file:
+            logger.info(f":: Reading from file {track_file}")
+            if Path(track_file).exists():
+                track_list = utils.dataprocessor.read_file_as_csv(track_file, split_new_line=False)
+                try:
+                    track_list = [int(x) for x in track_list]
+                except TypeError:
+                    logger.info("Track file must only contain track IDs")
+                    return
+
+                if track_list:
+                    with ThreadPoolExecutor(max_workers=self.api.max_threads) as ex:
+                        _api_results = list(tqdm(ex.map(process_track_file, track_list),
+                                                 total=len(track_list),
+                                                 desc=f"Fetching track release data for {len(track_list)} "
+                                                      f"track(s), please wait...", ascii=" #",
+                                                 bar_format=ui.TQDM_FORMAT))
+            else:
+                logger.error(f"The file {track_file} could not be found")
+                sys.exit()
 
         if url:
             logger.debug("Processing URLs")

deemon/cmd/monitor.py

@@ -123,7 +123,11 @@ def build_artist_query(self, api_result: list):
             self.db.commit()
             return True
 
-    def build_playlist_query(self, api_result: list):
+    def build_playlist_query(self, api_result: list, include_artists: bool):
+
+        if include_artists:
+            include_artists = '1'
+
         existing = self.db.get_all_monitored_playlist_ids() or []
         playlists_to_add = []
         pbar = tqdm(api_result, total=len(api_result), desc="Setting up playlists for monitoring...", ascii=" #",
@@ -134,8 +138,16 @@ def build_playlist_query(self, api_result: list):
             if playlist['id'] in existing:
                 logger.info(f"   Already monitoring {playlist['title']}, skipping...")
             else:
-                playlist.update({'bitrate': self.bitrate, 'alerts': self.alerts, 'download_path': self.download_path,
-                                 'profile_id': config.profile_id(), 'trans_id': config.transaction_id()})
+                playlist.update(
+                    {
+                        'bitrate': self.bitrate,
+                        'alerts': self.alerts,
+                        'download_path': self.download_path,
+                        'profile_id': config.profile_id(),
+                        'trans_id': config.transaction_id(),
+                        'monitor_artists': include_artists
+                    }
+                )
                 playlists_to_add.append(playlist)
         if len(playlists_to_add):
             logger.debug("New playlists have been monitored. Saving changes to the database...")
@@ -214,7 +226,7 @@ def importer(self, import_path: str):
             return
 
     # @performance.timeit
-    def playlists(self, playlists: list):
+    def playlists(self, playlists: list, include_artists: bool):
         if self.remove:
             return self.purge_playlists(ids=playlists)
         ids = [int(x) for x in playlists]
@@ -225,7 +237,7 @@ def playlists(self, playlists: list):
                      desc=f"Fetching playlist data for {len(ids):,} playlist(s), please wait...",
                      ascii=" #", bar_format=ui.TQDM_FORMAT))
 
-        if self.build_playlist_query(api_result):
+        if self.build_playlist_query(api_result, include_artists):
             self.call_refresh()
         else:
             print("")