Merge branch 'develop' into 'master'

Develop

See merge request namibsun/python/xdcc-dl!6

.gitlab-ci.yml

@@ -5,7 +5,7 @@ stages:
   - release
 
 default:
-  image: namboy94/ci-docker-environment:0.8.0
+  image: namboy94/ci-docker-environment:0.14.0
   before_script:
     - echo "$SERVER_ACCESS_KEY" > ~/.ssh/id_rsa
     - chmod 0600 ~/.ssh/id_rsa

CHANGELOG

@@ -1,3 +1,11 @@
+V 5.0.0:
+  - Replaced horriblesubs search engine with subsplease
+  - Fixed issues with the bot being banned due to bot detection mechanisms
+  - Avoid overlap if terminal window is too small during download
+  - Added wait_time parameter
+  - Added file size display to xdcc-search and xdcc-browse
+  - Added xdcc.eu search engine
+  - Added --username and --channel
 V 4.0.0:
   - Went over to completely rely on python's built-in logging
   - Throttle value is now passed as parameter

bin/xdcc-browse

@@ -56,7 +56,10 @@ def main(args: argparse.Namespace, logger: logging.Logger):
             packs,
             timeout=args.timeout,
             fallback_channel=args.fallback_channel,
-            throttle=args.throttle
+            throttle=args.throttle,
+            wait_time=args.wait_time,
+            username=args.username,
+            channel_join_delay=args.channel_join_delay
         )
 
     except ConnectionError:
@@ -72,7 +75,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("search_term", help="The term to search for")
     parser.add_argument("--search-engine",
-                        default=SearchEngineType.HORRIBLESUBS.name.lower(),
+                        default=SearchEngineType.SUBSPLEASE.name.lower(),
                         choices=SearchEngineType.choices(True),
                         help="The Search Engine to use")
     add_xdcc_argparse_arguments(parser)

bin/xdcc-dl

@@ -46,7 +46,10 @@ def main(args: argparse.Namespace, logger: logging.Logger):
             packs,
             timeout=args.timeout,
             fallback_channel=args.fallback_channel,
-            throttle=args.throttle
+            throttle=args.throttle,
+            wait_time=args.wait_time,
+            username=args.username,
+            channel_join_delay=args.channel_join_delay
         )
 
     except DownloadIncomplete:

bin/xdcc-search

@@ -23,6 +23,7 @@ from puffotter.init import cli_start
 from requests.exceptions import ConnectionError
 from xdcc_dl import sentry_dsn
 from xdcc_dl.pack_search.SearchEngine import SearchEngineType
+from puffotter.units import human_readable_bytes
 
 
 def main(args: argparse.Namespace):
@@ -35,9 +36,10 @@ def main(args: argparse.Namespace):
         search_engine = SearchEngineType.resolve(args.search_engine)
         results = search_engine.search(args.search_term)
         for result in results:
-            message = "{} (xdcc-dl \"{}\")".format(
+            message = "{} [{}] (xdcc-dl \"{}\")".format(
                 result.filename,
-                result.get_request_message(True)
+                human_readable_bytes(result.get_size()),
+                result.get_request_message(True),
             )
             if result.server.address != "irc.rizon.net":
                 message = message[0:-1]

setup.py

@@ -48,7 +48,8 @@
             "colorama",
             "irc",
             "puffotter",
-            "sentry-sdk"
+            "sentry-sdk",
+            "names"
         ],
         include_package_data=True,
         zip_safe=False

version

@@ -1 +1 @@
-4.0.0
+5.0.0

xdcc_dl/entities/User.py

@@ -18,8 +18,8 @@
 LICENSE"""
 
 # imports
+import names
 import random
-import string
 
 
 class User(object):
@@ -32,12 +32,15 @@ def __init__(self, username: str = "random"):
         Initializes the User
 
         :param username: the user's username. If left empty, or the string
-                         'random' is passed, a random username consisting only
-                         of ASCII characters will be generated as the username.
+                         'random' is passed, a random username is generated
+                         using the names package.
                          An empty string will also result in a random username
         """
         if username == "random" or username == "":
-            self.username = self.generate_random_username()
+            self.username = \
+                names.get_first_name() + \
+                names.get_last_name() + \
+                str(random.randint(10, 100))
         else:
             self.username = username
 
@@ -46,14 +49,3 @@ def get_name(self) -> str:
         :return: The user's username
         """
         return self.username
-
-    @staticmethod
-    def generate_random_username(length: int = 10) -> str:
-        """
-        Generates a random username of given length
-
-        :param length: The length of the username
-        :return:       The random username
-        """
-        return "".join(random.choice(string.ascii_uppercase)
-                       for _ in range(length))

xdcc_dl/entities/XDCCPack.py

@@ -21,6 +21,7 @@
 import os
 import re
 from xdcc_dl.entities.IrcServer import IrcServer
+from puffotter.units import human_readable_bytes
 
 
 class XDCCPack(object):
@@ -107,7 +108,7 @@ def set_directory(self, directory: str):
 
     def set_size(self, size: int):
         """
-        Sets the file size of the XDCC pack
+        Sets the file size of the XDCC pack in Bytes
 
         :param size: the size of the pack
         :return:     None
@@ -168,8 +169,9 @@ def __str__(self) -> str:
         """
         :return: A string representation of the pack
         """
-        return self.filename + " (/msg " + self.bot + " " + \
-            self.get_request_message() + ")"
+        return f"{self.filename} (/msg {self.bot} " \
+               f"{self.get_request_message()}) " \
+               f"[{human_readable_bytes(self.size)}]"
 
     def __eq__(self, other) -> bool:
         """

xdcc_dl/helper.py

@@ -57,7 +57,17 @@ def add_xdcc_argparse_arguments(parser: ArgumentParser):
                         help="Limits the download speed of xdcc-dl. "
                              "Append K,M or G for more convenient units")
     parser.add_argument("--timeout", default=120, type=int,
-                        help="Sets a timeout for starting the download")
+                        help="If the download didn't start during the "
+                             "specified timeout, the program will stop")
     parser.add_argument("--fallback-channel",
                         help="Fallback channel in case a channel could not"
                              "be joined automatically using WHOIS commands")
+    parser.add_argument("--wait-time", default=0, type=int,
+                        help="Waits for the specified amount of time before "
+                             "sending the xdcc send request")
+    parser.add_argument("--username",
+                        help="Specifies a user name for the downloader bot")
+    parser.add_argument("--channel-join-delay", type=int,
+                        help="Specifies a delay in seconds for how long the"
+                             "downloader should wait before connecting to"
+                             "channels")

xdcc_dl/pack_search/SearchEngine.py

@@ -22,7 +22,8 @@
 from xdcc_dl.entities.XDCCPack import XDCCPack
 from xdcc_dl.pack_search.procedures.nibl import find_nibl_packs
 from xdcc_dl.pack_search.procedures.ixirc import find_ixirc_packs
-from xdcc_dl.pack_search.procedures.horriblesubs import find_horriblesubs_packs
+from xdcc_dl.pack_search.procedures.subsplease import find_subsplease_packs
+from xdcc_dl.pack_search.procedures.xdcc_eu import find_xdcc_eu_packs
 
 
 class SearchEngine:
@@ -53,9 +54,10 @@ class SearchEngineType(Enum):
     The different implemented search engines
     """
 
-    HORRIBLESUBS = SearchEngine("Horriblesubs", find_horriblesubs_packs)
+    SUBSPLEASE = SearchEngine("SubsPlease", find_subsplease_packs)
     NIBL = SearchEngine("Nibl", find_nibl_packs)
     IXIRC = SearchEngine("iXirc", find_ixirc_packs)
+    XDCC_EU = SearchEngine("xdcc-eu", find_xdcc_eu_packs)
 
     @classmethod
     def choices(cls, lower: bool = True) -> Set[str]:

xdcc_dl/pack_search/procedures/ixirc.py

@@ -172,7 +172,10 @@ def get_page_results(page_content: BeautifulSoup) -> List[XDCCPack]:
             elif column_count == 5:
                 pass  # This is the 'gets' section, we don't need that
             elif column_count == 6:
-                size = line_part.text.replace("\xa0", " ").replace(" ", "")
+                size = line_part.text\
+                    .replace("\xa0", " ")\
+                    .replace(" ", "")\
+                    .replace("B", "")
 
         # Resets state after a pack was successfully parsed,
         # and adds xdcc pack to results

xdcc_dl/pack_search/procedures/nibl.py

@@ -23,6 +23,7 @@
 from bs4 import BeautifulSoup
 from xdcc_dl.entities.XDCCPack import XDCCPack
 from xdcc_dl.entities.IrcServer import IrcServer
+from puffotter.units import byte_string_to_byte_count
 
 
 def find_nibl_packs(search_phrase: str) -> List[XDCCPack]:
@@ -65,10 +66,11 @@ def find_nibl_packs(search_phrase: str) -> List[XDCCPack]:
 
         server = "irc.rizon.net"
         packnumber = int(pack_numbers[i].text)
-        size = file_sizes[i].text
+        size = file_sizes[i].text.lower()
 
         result = XDCCPack(IrcServer(server), bot, packnumber)
-        result.set_size(size)
+
+        result.set_size(byte_string_to_byte_count(size))
         result.set_filename(filename)
         results.append(result)
         i += 1

xdcc_dl/pack_search/procedures/horriblesubs.py → xdcc_dl/pack_search/procedures/subsplease.py

@@ -24,9 +24,9 @@
 from xdcc_dl.entities.IrcServer import IrcServer
 
 
-def find_horriblesubs_packs(search_phrase: str) -> List[XDCCPack]:
+def find_subsplease_packs(search_phrase: str) -> List[XDCCPack]:
     """
-    Method that conducts the xdcc pack search for xdcc.horriblesubs.info
+    Method that conducts the xdcc pack search for subsplease.org
 
     :return: the search results as a list of XDCCPack objects
     """
@@ -36,7 +36,7 @@ def find_horriblesubs_packs(search_phrase: str) -> List[XDCCPack]:
     search_query = search_phrase.replace(" ", "%20")
     search_query = search_query.replace("!", "%21")
 
-    url = "http://xdcc.horriblesubs.info/search.php?t=" + search_query
+    url = "https://subsplease.org/xdcc/search.php?t=" + search_query
     scraper = cfscrape.create_scraper()
     results = scraper.get(url).text.split(";")
 
@@ -51,7 +51,7 @@ def find_horriblesubs_packs(search_phrase: str) -> List[XDCCPack]:
             packnumber = int(result["n"])
             pack = XDCCPack(IrcServer("irc.rizon.net"), botname, packnumber)
             pack.set_filename(filename)
-            pack.set_size(filesize)
+            pack.set_size(filesize * 1000 * 1000)
             packs.append(pack)
 
         except IndexError:  # In case the line is not parseable
@@ -62,7 +62,7 @@ def find_horriblesubs_packs(search_phrase: str) -> List[XDCCPack]:
 
 def parse_result(result: str) -> Dict[str, str]:
     """
-    Turns the weird horriblesubs response syntax into a useable dictionary
+    Turns the weird subsplease response syntax into a useable dictionary
     :param result: The result to parse
     :return: The result as a dictionary
     """

xdcc_dl/pack_search/procedures/xdcc_eu.py

@@ -0,0 +1,57 @@
+"""LICENSE
+Copyright 2016 Hermann Krumrey <hermann@krumreyh.com>
+
+This file is part of xdcc-dl.
+
+xdcc-dl is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+xdcc-dl is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with xdcc-dl.  If not, see <http://www.gnu.org/licenses/>.
+LICENSE"""
+
+import requests
+from typing import List
+from bs4 import BeautifulSoup
+from xdcc_dl.entities.XDCCPack import XDCCPack
+from xdcc_dl.entities.IrcServer import IrcServer
+from puffotter.units import byte_string_to_byte_count
+
+
+def find_xdcc_eu_packs(search_phrase: str) -> List[XDCCPack]:
+    """
+    Method that conducts the xdcc pack search for xdcc.eu
+
+    :return: the search results as a list of XDCCPack objects
+    """
+    url = "https://www.xdcc.eu/search.php?searchkey=" + search_phrase
+    response = requests.get(url)
+    soup = BeautifulSoup(response.text, "html.parser")
+    entries = soup.select("tr")
+    entries.pop(0)
+
+    packs = []
+    for entry in entries:
+        parts = entry.select("td")
+        info = parts[1].select("a")[1]
+        server = IrcServer(info["data-s"])
+        pack_message = info["data-p"]
+        bot, pack_number = pack_message.split(" xdcc send #")
+
+        size = byte_string_to_byte_count(parts[5].text)
+        filename = parts[6].text
+
+        pack = XDCCPack(server, bot, int(pack_number))
+        pack.set_size(size)
+        pack.set_filename(filename)
+
+        packs.append(pack)
+
+    return packs