From fa97ca6e4ec480d46854ef5e071d1bc1428ddccb Mon Sep 17 00:00:00 2001 From: Hermann Krumrey Date: Mon, 28 Oct 2019 10:24:58 +0100 Subject: [PATCH 1/9] Avoid overlap if terminal window is too small during download --- CHANGELOG | 2 ++ version | 2 +- xdcc_dl/xdcc/XDCCClient.py | 9 +++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index c961469..fe85554 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,5 @@ +V 4.1.0: + - Avoid overlap if terminal window is too small during download V 4.0.0: - Went over to completely rely on python's built-in logging - Throttle value is now passed as parameter diff --git a/version b/version index 0c89fc9..99eba4d 100644 --- a/version +++ b/version @@ -1 +1 @@ -4.0.0 \ No newline at end of file +4.1.0 \ No newline at end of file diff --git a/xdcc_dl/xdcc/XDCCClient.py b/xdcc_dl/xdcc/XDCCClient.py index 60a0a4e..605548a 100644 --- a/xdcc_dl/xdcc/XDCCClient.py +++ b/xdcc_dl/xdcc/XDCCClient.py @@ -27,6 +27,7 @@ import irc.client from colorama import Fore, Back from threading import Thread, Lock +from subprocess import check_output, CalledProcessError from typing import Optional, IO, Any, List, Union from puffotter.units import human_readable_bytes, byte_string_to_byte_count from puffotter.print import pprint @@ -601,6 +602,14 @@ def progress_printer(self): human_readable_bytes(self.filesize), speed ) + + try: + rows, _columns = check_output(['stty', 'size']).split() + columns = int(_columns) + except (ValueError, CalledProcessError): + columns = 80 + log_message = log_message[0:columns] + pprint(log_message, end="\r", bg="lyellow", fg="black") time.sleep(0.1) self.logger.info("Progress Printer stopped") From 15404c8d2b99caddf59b10dad2214c0dfe16b7c7 Mon Sep 17 00:00:00 2001 From: Hermann Krumrey Date: Mon, 28 Oct 2019 10:35:34 +0100 Subject: [PATCH 2/9] Add wait time parameter --- CHANGELOG | 1 + bin/xdcc-browse | 3 ++- bin/xdcc-dl | 3 ++- xdcc_dl/helper.py | 6 +++++- xdcc_dl/xdcc/XDCCClient.py | 10 +++++++++- xdcc_dl/xdcc/__init__.py | 8 ++++++-- 6 files changed, 25 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index fe85554..e576908 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,6 @@ V 4.1.0: - Avoid overlap if terminal window is too small during download + - Added wait_time parameter V 4.0.0: - Went over to completely rely on python's built-in logging - Throttle value is now passed as parameter diff --git a/bin/xdcc-browse b/bin/xdcc-browse index 53f43af..e0bce58 100644 --- a/bin/xdcc-browse +++ b/bin/xdcc-browse @@ -56,7 +56,8 @@ def main(args: argparse.Namespace, logger: logging.Logger): packs, timeout=args.timeout, fallback_channel=args.fallback_channel, - throttle=args.throttle + throttle=args.throttle, + wait_time=args.wait_time ) except ConnectionError: diff --git a/bin/xdcc-dl b/bin/xdcc-dl index 8055270..bd9fb99 100644 --- a/bin/xdcc-dl +++ b/bin/xdcc-dl @@ -46,7 +46,8 @@ def main(args: argparse.Namespace, logger: logging.Logger): packs, timeout=args.timeout, fallback_channel=args.fallback_channel, - throttle=args.throttle + throttle=args.throttle, + wait_time=args.wait_time ) except DownloadIncomplete: diff --git a/xdcc_dl/helper.py b/xdcc_dl/helper.py index 8f4fccf..6bbc93b 100644 --- a/xdcc_dl/helper.py +++ b/xdcc_dl/helper.py @@ -57,7 +57,11 @@ def add_xdcc_argparse_arguments(parser: ArgumentParser): help="Limits the download speed of xdcc-dl. " "Append K,M or G for more convenient units") parser.add_argument("--timeout", default=120, type=int, - help="Sets a timeout for starting the download") + help="If the download didn't start during the " + "specified timeout, the program will stop") parser.add_argument("--fallback-channel", help="Fallback channel in case a channel could not" "be joined automatically using WHOIS commands") + parser.add_argument("--wait-time", default=0, type=int, + help="Waits for the specified amount of time before " + "sending the xdcc send request") diff --git a/xdcc_dl/xdcc/XDCCClient.py b/xdcc_dl/xdcc/XDCCClient.py index 605548a..cf28435 100644 --- a/xdcc_dl/xdcc/XDCCClient.py +++ b/xdcc_dl/xdcc/XDCCClient.py @@ -51,7 +51,8 @@ def __init__( retry: bool = False, timeout: int = 120, fallback_channel: Optional[str] = None, - throttle: Union[int, str] = -1 + throttle: Union[int, str] = -1, + wait_time: int = 0 ): """ Initializes the XDCC IRC client @@ -63,6 +64,8 @@ def __init__( :param throttle: Throttles the download to n bytes per second. If this value is <= 0, the download speed will be unlimited + :param wait_time: Waits for the specified amount of time before sending + a message """ self.logger = ColorLogger( logging.getLogger(self.__class__.__name__), @@ -92,6 +95,7 @@ def __init__( self.timeout = timeout self.timed_out = False self.fallback_channel = fallback_channel + self.wait_time = wait_time self.connected = True self.disconnected = False @@ -481,6 +485,10 @@ def _send_xdcc_request_message(self, conn: ServerConnection): :param conn: The connection to use :return: None """ + self.logger.info("Waiting for {}s before sending message" + .format(self.wait_time)) + time.sleep(self.wait_time) + msg = self.pack.get_request_message() self.logger.info("Send XDCC Message: " + msg) self.message_sent = True diff --git a/xdcc_dl/xdcc/__init__.py b/xdcc_dl/xdcc/__init__.py index aa5c51e..cb524f2 100644 --- a/xdcc_dl/xdcc/__init__.py +++ b/xdcc_dl/xdcc/__init__.py @@ -26,7 +26,8 @@ def download_packs( packs: List[XDCCPack], timeout: int = 120, fallback_channel: Optional[str] = None, - throttle: Union[int, str] = -1 + throttle: Union[int, str] = -1, + wait_time: int = 0 ): """ Downloads a list of XDCC Packs @@ -36,6 +37,8 @@ def download_packs( :param throttle: Throttles the download to n bytes per second. If this value is <= 0, the download speed will be unlimited + :param wait_time: Waits for the specified amount of time before sending + a message :return: None """ for pack in packs: @@ -43,6 +46,7 @@ def download_packs( pack, timeout=timeout, fallback_channel=fallback_channel, - throttle=throttle + throttle=throttle, + wait_time=wait_time ) client.download() From 828a897f0028afe7a1aa2ba59e9145af21a1023c Mon Sep 17 00:00:00 2001 From: Hermann Krumrey Date: Wed, 14 Oct 2020 17:43:28 +0200 Subject: [PATCH 3/9] Replace horriblesubs with subsplease --- .gitlab-ci.yml | 2 +- CHANGELOG | 2 ++ bin/xdcc-browse | 2 +- version | 2 +- xdcc_dl/pack_search/SearchEngine.py | 4 ++-- .../procedures/{horriblesubs.py => subsplease.py} | 8 ++++---- 6 files changed, 11 insertions(+), 9 deletions(-) rename xdcc_dl/pack_search/procedures/{horriblesubs.py => subsplease.py} (90%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1f91445..28b9373 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ stages: - release default: - image: namboy94/ci-docker-environment:0.8.0 + image: namboy94/ci-docker-environment:0.14.0 before_script: - echo "$SERVER_ACCESS_KEY" > ~/.ssh/id_rsa - chmod 0600 ~/.ssh/id_rsa diff --git a/CHANGELOG b/CHANGELOG index e576908..115b685 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,5 @@ +V 4.2.0: + - Replaced horriblesubs search engine with subsplease V 4.1.0: - Avoid overlap if terminal window is too small during download - Added wait_time parameter diff --git a/bin/xdcc-browse b/bin/xdcc-browse index e0bce58..113873f 100644 --- a/bin/xdcc-browse +++ b/bin/xdcc-browse @@ -73,7 +73,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("search_term", help="The term to search for") parser.add_argument("--search-engine", - default=SearchEngineType.HORRIBLESUBS.name.lower(), + default=SearchEngineType.SUBSPLEASE.name.lower(), choices=SearchEngineType.choices(True), help="The Search Engine to use") add_xdcc_argparse_arguments(parser) diff --git a/version b/version index 99eba4d..ef8d756 100644 --- a/version +++ b/version @@ -1 +1 @@ -4.1.0 \ No newline at end of file +4.2.0 \ No newline at end of file diff --git a/xdcc_dl/pack_search/SearchEngine.py b/xdcc_dl/pack_search/SearchEngine.py index 9fa24d3..0dcd918 100644 --- a/xdcc_dl/pack_search/SearchEngine.py +++ b/xdcc_dl/pack_search/SearchEngine.py @@ -22,7 +22,7 @@ from xdcc_dl.entities.XDCCPack import XDCCPack from xdcc_dl.pack_search.procedures.nibl import find_nibl_packs from xdcc_dl.pack_search.procedures.ixirc import find_ixirc_packs -from xdcc_dl.pack_search.procedures.horriblesubs import find_horriblesubs_packs +from xdcc_dl.pack_search.procedures.subsplease import find_subsplease_packs class SearchEngine: @@ -53,7 +53,7 @@ class SearchEngineType(Enum): The different implemented search engines """ - HORRIBLESUBS = SearchEngine("Horriblesubs", find_horriblesubs_packs) + SUBSPLEASE = SearchEngine("SubsPlease", find_subsplease_packs) NIBL = SearchEngine("Nibl", find_nibl_packs) IXIRC = SearchEngine("iXirc", find_ixirc_packs) diff --git a/xdcc_dl/pack_search/procedures/horriblesubs.py b/xdcc_dl/pack_search/procedures/subsplease.py similarity index 90% rename from xdcc_dl/pack_search/procedures/horriblesubs.py rename to xdcc_dl/pack_search/procedures/subsplease.py index 910e151..fd5dc3f 100644 --- a/xdcc_dl/pack_search/procedures/horriblesubs.py +++ b/xdcc_dl/pack_search/procedures/subsplease.py @@ -24,9 +24,9 @@ from xdcc_dl.entities.IrcServer import IrcServer -def find_horriblesubs_packs(search_phrase: str) -> List[XDCCPack]: +def find_subsplease_packs(search_phrase: str) -> List[XDCCPack]: """ - Method that conducts the xdcc pack search for xdcc.horriblesubs.info + Method that conducts the xdcc pack search for subsplease.org :return: the search results as a list of XDCCPack objects """ @@ -36,7 +36,7 @@ def find_horriblesubs_packs(search_phrase: str) -> List[XDCCPack]: search_query = search_phrase.replace(" ", "%20") search_query = search_query.replace("!", "%21") - url = "http://xdcc.horriblesubs.info/search.php?t=" + search_query + url = "https://subsplease.org/xdcc/search.php?t=" + search_query scraper = cfscrape.create_scraper() results = scraper.get(url).text.split(";") @@ -62,7 +62,7 @@ def find_horriblesubs_packs(search_phrase: str) -> List[XDCCPack]: def parse_result(result: str) -> Dict[str, str]: """ - Turns the weird horriblesubs response syntax into a useable dictionary + Turns the weird subsplease response syntax into a useable dictionary :param result: The result to parse :return: The result as a dictionary """ From d3465a3a6af704132a1a790f45fe3f9d7ac59864 Mon Sep 17 00:00:00 2001 From: Hermann Krumrey Date: Wed, 4 Nov 2020 22:56:25 +0100 Subject: [PATCH 4/9] Fix issues with being banned due to bot detection mechanisms --- CHANGELOG | 1 + setup.py | 3 ++- xdcc_dl/entities/User.py | 20 ++++---------------- xdcc_dl/xdcc/XDCCClient.py | 9 +++++++++ 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 115b685..ce2a41c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,6 @@ V 4.2.0: - Replaced horriblesubs search engine with subsplease + - Fixed issues with the bot being banned due to bot detection mechanisms V 4.1.0: - Avoid overlap if terminal window is too small during download - Added wait_time parameter diff --git a/setup.py b/setup.py index 1b0a25f..4a87be1 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,8 @@ "colorama", "irc", "puffotter", - "sentry-sdk" + "sentry-sdk", + "names" ], include_package_data=True, zip_safe=False diff --git a/xdcc_dl/entities/User.py b/xdcc_dl/entities/User.py index 7b230a8..fc5b95f 100644 --- a/xdcc_dl/entities/User.py +++ b/xdcc_dl/entities/User.py @@ -18,8 +18,7 @@ LICENSE""" # imports -import random -import string +import names class User(object): @@ -32,12 +31,12 @@ def __init__(self, username: str = "random"): Initializes the User :param username: the user's username. If left empty, or the string - 'random' is passed, a random username consisting only - of ASCII characters will be generated as the username. + 'random' is passed, a random username is generated + using the names package. An empty string will also result in a random username """ if username == "random" or username == "": - self.username = self.generate_random_username() + self.username = names.get_first_name() else: self.username = username @@ -46,14 +45,3 @@ def get_name(self) -> str: :return: The user's username """ return self.username - - @staticmethod - def generate_random_username(length: int = 10) -> str: - """ - Generates a random username of given length - - :param length: The length of the username - :return: The random username - """ - return "".join(random.choice(string.ascii_uppercase) - for _ in range(length)) diff --git a/xdcc_dl/xdcc/XDCCClient.py b/xdcc_dl/xdcc/XDCCClient.py index cf28435..b380ed7 100644 --- a/xdcc_dl/xdcc/XDCCClient.py +++ b/xdcc_dl/xdcc/XDCCClient.py @@ -22,6 +22,7 @@ import struct import shlex import socket +import random import logging import irc.events import irc.client @@ -165,12 +166,18 @@ def download(self) -> str: try: self.logger.info("Connecting to " + self.server.address + ":" + str(self.server.port)) + self.user = User("Toni") self.connect( self.server.address, self.server.port, self.user.username ) + delay = random.randint(5, 10) + self.logger.info(f"Delaying download initialization by {delay}s") + time.sleep(delay) + self.connected = True + self.connect_start_time = time.time() self.timeout_watcher_thread.start() @@ -280,6 +287,8 @@ def on_whoischannels(self, conn: ServerConnection, event: Event): for channel in channels: # Join all channels to avoid only joining a members-only channel + time.sleep(random.randint(1, 3)) + self.logger.info(f"Joining channel {channel}") conn.join(channel) def on_endofwhois(self, conn: ServerConnection, _: Event): From 150984b93381c1b651ff247b99a7c33182abe4ce Mon Sep 17 00:00:00 2001 From: Hermann Krumrey Date: Wed, 4 Nov 2020 23:31:00 +0100 Subject: [PATCH 5/9] Add file size display to browse and search --- CHANGELOG | 4 ++-- bin/xdcc-search | 6 ++++-- version | 2 +- xdcc_dl/entities/XDCCPack.py | 8 +++++--- xdcc_dl/pack_search/procedures/ixirc.py | 17 +++++++++++++++-- xdcc_dl/pack_search/procedures/nibl.py | 15 +++++++++++++-- xdcc_dl/pack_search/procedures/subsplease.py | 2 +- 7 files changed, 41 insertions(+), 13 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index ce2a41c..0cdde2c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,9 +1,9 @@ -V 4.2.0: +V 4.1.0: - Replaced horriblesubs search engine with subsplease - Fixed issues with the bot being banned due to bot detection mechanisms -V 4.1.0: - Avoid overlap if terminal window is too small during download - Added wait_time parameter + - Added file size display to xdcc-search and xdcc-browse V 4.0.0: - Went over to completely rely on python's built-in logging - Throttle value is now passed as parameter diff --git a/bin/xdcc-search b/bin/xdcc-search index 42546b0..1c04b3b 100644 --- a/bin/xdcc-search +++ b/bin/xdcc-search @@ -23,6 +23,7 @@ from puffotter.init import cli_start from requests.exceptions import ConnectionError from xdcc_dl import sentry_dsn from xdcc_dl.pack_search.SearchEngine import SearchEngineType +from puffotter.units import human_readable_bytes def main(args: argparse.Namespace): @@ -35,9 +36,10 @@ def main(args: argparse.Namespace): search_engine = SearchEngineType.resolve(args.search_engine) results = search_engine.search(args.search_term) for result in results: - message = "{} (xdcc-dl \"{}\")".format( + message = "{} [{}] (xdcc-dl \"{}\")".format( result.filename, - result.get_request_message(True) + human_readable_bytes(result.get_size()), + result.get_request_message(True), ) if result.server.address != "irc.rizon.net": message = message[0:-1] diff --git a/version b/version index ef8d756..99eba4d 100644 --- a/version +++ b/version @@ -1 +1 @@ -4.2.0 \ No newline at end of file +4.1.0 \ No newline at end of file diff --git a/xdcc_dl/entities/XDCCPack.py b/xdcc_dl/entities/XDCCPack.py index 05380af..8aacd0d 100644 --- a/xdcc_dl/entities/XDCCPack.py +++ b/xdcc_dl/entities/XDCCPack.py @@ -21,6 +21,7 @@ import os import re from xdcc_dl.entities.IrcServer import IrcServer +from puffotter.units import human_readable_bytes class XDCCPack(object): @@ -107,7 +108,7 @@ def set_directory(self, directory: str): def set_size(self, size: int): """ - Sets the file size of the XDCC pack + Sets the file size of the XDCC pack in Bytes :param size: the size of the pack :return: None @@ -168,8 +169,9 @@ def __str__(self) -> str: """ :return: A string representation of the pack """ - return self.filename + " (/msg " + self.bot + " " + \ - self.get_request_message() + ")" + return f"{self.filename} (/msg {self.bot} " \ + f"{self.get_request_message()}) " \ + f"[{human_readable_bytes(self.size)}]" def __eq__(self, other) -> bool: """ diff --git a/xdcc_dl/pack_search/procedures/ixirc.py b/xdcc_dl/pack_search/procedures/ixirc.py index f5bd2af..ee0d66d 100644 --- a/xdcc_dl/pack_search/procedures/ixirc.py +++ b/xdcc_dl/pack_search/procedures/ixirc.py @@ -172,7 +172,10 @@ def get_page_results(page_content: BeautifulSoup) -> List[XDCCPack]: elif column_count == 5: pass # This is the 'gets' section, we don't need that elif column_count == 6: - size = line_part.text.replace("\xa0", " ").replace(" ", "") + size = line_part.text\ + .replace("\xa0", " ")\ + .replace(" ", "")\ + .replace("B", "") # Resets state after a pack was successfully parsed, # and adds xdcc pack to results @@ -184,7 +187,17 @@ def get_page_results(page_content: BeautifulSoup) -> List[XDCCPack]: # Generate XDCCPack and append it to the list result = XDCCPack(IrcServer(server), bot, pack_number) result.set_filename(file_name) - result.set_size(byte_string_to_byte_count(size)) + + # TODO Make this nicer + try: + result.set_size(byte_string_to_byte_count(size)) + except ValueError: + size_parts = size.split(".", 1) + for char in size_parts[1]: + if not char.isdigit(): + size_parts[0] += char + result.set_size(byte_string_to_byte_count(size_parts[0])) + results.append(result) # Resets state after invalid pack diff --git a/xdcc_dl/pack_search/procedures/nibl.py b/xdcc_dl/pack_search/procedures/nibl.py index 3929aae..533e2d5 100644 --- a/xdcc_dl/pack_search/procedures/nibl.py +++ b/xdcc_dl/pack_search/procedures/nibl.py @@ -23,6 +23,7 @@ from bs4 import BeautifulSoup from xdcc_dl.entities.XDCCPack import XDCCPack from xdcc_dl.entities.IrcServer import IrcServer +from puffotter.units import byte_string_to_byte_count def find_nibl_packs(search_phrase: str) -> List[XDCCPack]: @@ -65,10 +66,20 @@ def find_nibl_packs(search_phrase: str) -> List[XDCCPack]: server = "irc.rizon.net" packnumber = int(pack_numbers[i].text) - size = file_sizes[i].text + size = file_sizes[i].text.lower() result = XDCCPack(IrcServer(server), bot, packnumber) - result.set_size(size) + + # TODO Make this nicer + try: + result.set_size(byte_string_to_byte_count(size)) + except ValueError: + size_parts = size.split(".", 1) + for char in size_parts[1]: + if not char.isdigit(): + size_parts[0] += char + result.set_size(byte_string_to_byte_count(size_parts[0])) + result.set_filename(filename) results.append(result) i += 1 diff --git a/xdcc_dl/pack_search/procedures/subsplease.py b/xdcc_dl/pack_search/procedures/subsplease.py index fd5dc3f..5c6561b 100644 --- a/xdcc_dl/pack_search/procedures/subsplease.py +++ b/xdcc_dl/pack_search/procedures/subsplease.py @@ -51,7 +51,7 @@ def find_subsplease_packs(search_phrase: str) -> List[XDCCPack]: packnumber = int(result["n"]) pack = XDCCPack(IrcServer("irc.rizon.net"), botname, packnumber) pack.set_filename(filename) - pack.set_size(filesize) + pack.set_size(filesize * 1000 * 1000) packs.append(pack) except IndexError: # In case the line is not parseable From 53300edd41202cb22a7381ec46c95c26c0e6651e Mon Sep 17 00:00:00 2001 From: Hermann Krumrey Date: Thu, 5 Nov 2020 15:52:09 +0100 Subject: [PATCH 6/9] Rely on fixed puffotter implementation to avoid byte count conversion issues --- xdcc_dl/pack_search/procedures/ixirc.py | 12 +----------- xdcc_dl/pack_search/procedures/nibl.py | 11 +---------- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/xdcc_dl/pack_search/procedures/ixirc.py b/xdcc_dl/pack_search/procedures/ixirc.py index ee0d66d..2a3fa50 100644 --- a/xdcc_dl/pack_search/procedures/ixirc.py +++ b/xdcc_dl/pack_search/procedures/ixirc.py @@ -187,17 +187,7 @@ def get_page_results(page_content: BeautifulSoup) -> List[XDCCPack]: # Generate XDCCPack and append it to the list result = XDCCPack(IrcServer(server), bot, pack_number) result.set_filename(file_name) - - # TODO Make this nicer - try: - result.set_size(byte_string_to_byte_count(size)) - except ValueError: - size_parts = size.split(".", 1) - for char in size_parts[1]: - if not char.isdigit(): - size_parts[0] += char - result.set_size(byte_string_to_byte_count(size_parts[0])) - + result.set_size(byte_string_to_byte_count(size)) results.append(result) # Resets state after invalid pack diff --git a/xdcc_dl/pack_search/procedures/nibl.py b/xdcc_dl/pack_search/procedures/nibl.py index 533e2d5..e61a8b1 100644 --- a/xdcc_dl/pack_search/procedures/nibl.py +++ b/xdcc_dl/pack_search/procedures/nibl.py @@ -70,16 +70,7 @@ def find_nibl_packs(search_phrase: str) -> List[XDCCPack]: result = XDCCPack(IrcServer(server), bot, packnumber) - # TODO Make this nicer - try: - result.set_size(byte_string_to_byte_count(size)) - except ValueError: - size_parts = size.split(".", 1) - for char in size_parts[1]: - if not char.isdigit(): - size_parts[0] += char - result.set_size(byte_string_to_byte_count(size_parts[0])) - + result.set_size(byte_string_to_byte_count(size)) result.set_filename(filename) results.append(result) i += 1 From 2586f8c8f12071554783bca34824b7ecdee25330 Mon Sep 17 00:00:00 2001 From: Hermann Krumrey Date: Thu, 5 Nov 2020 16:16:06 +0100 Subject: [PATCH 7/9] Add xdcc.eu search engine --- CHANGELOG | 3 +- version | 2 +- xdcc_dl/pack_search/SearchEngine.py | 2 + xdcc_dl/pack_search/procedures/xdcc_eu.py | 57 +++++++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 xdcc_dl/pack_search/procedures/xdcc_eu.py diff --git a/CHANGELOG b/CHANGELOG index 0cdde2c..6ca8b65 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,9 +1,10 @@ -V 4.1.0: +V 5.0.0: - Replaced horriblesubs search engine with subsplease - Fixed issues with the bot being banned due to bot detection mechanisms - Avoid overlap if terminal window is too small during download - Added wait_time parameter - Added file size display to xdcc-search and xdcc-browse + - Added xdcc.eu search engine V 4.0.0: - Went over to completely rely on python's built-in logging - Throttle value is now passed as parameter diff --git a/version b/version index 99eba4d..28cbf7c 100644 --- a/version +++ b/version @@ -1 +1 @@ -4.1.0 \ No newline at end of file +5.0.0 \ No newline at end of file diff --git a/xdcc_dl/pack_search/SearchEngine.py b/xdcc_dl/pack_search/SearchEngine.py index 0dcd918..10e36a0 100644 --- a/xdcc_dl/pack_search/SearchEngine.py +++ b/xdcc_dl/pack_search/SearchEngine.py @@ -23,6 +23,7 @@ from xdcc_dl.pack_search.procedures.nibl import find_nibl_packs from xdcc_dl.pack_search.procedures.ixirc import find_ixirc_packs from xdcc_dl.pack_search.procedures.subsplease import find_subsplease_packs +from xdcc_dl.pack_search.procedures.xdcc_eu import find_xdcc_eu_packs class SearchEngine: @@ -56,6 +57,7 @@ class SearchEngineType(Enum): SUBSPLEASE = SearchEngine("SubsPlease", find_subsplease_packs) NIBL = SearchEngine("Nibl", find_nibl_packs) IXIRC = SearchEngine("iXirc", find_ixirc_packs) + XDCC_EU = SearchEngine("xdcc-eu", find_xdcc_eu_packs) @classmethod def choices(cls, lower: bool = True) -> Set[str]: diff --git a/xdcc_dl/pack_search/procedures/xdcc_eu.py b/xdcc_dl/pack_search/procedures/xdcc_eu.py new file mode 100644 index 0000000..3299b4f --- /dev/null +++ b/xdcc_dl/pack_search/procedures/xdcc_eu.py @@ -0,0 +1,57 @@ +"""LICENSE +Copyright 2016 Hermann Krumrey + +This file is part of xdcc-dl. + +xdcc-dl is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +xdcc-dl is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with xdcc-dl. If not, see . +LICENSE""" + +import requests +from typing import List +from bs4 import BeautifulSoup +from xdcc_dl.entities.XDCCPack import XDCCPack +from xdcc_dl.entities.IrcServer import IrcServer +from puffotter.units import byte_string_to_byte_count + + +def find_xdcc_eu_packs(search_phrase: str) -> List[XDCCPack]: + """ + Method that conducts the xdcc pack search for xdcc.eu + + :return: the search results as a list of XDCCPack objects + """ + url = "https://www.xdcc.eu/search.php?searchkey=" + search_phrase + response = requests.get(url) + soup = BeautifulSoup(response.text, "html.parser") + entries = soup.select("tr") + entries.pop(0) + + packs = [] + for entry in entries: + parts = entry.select("td") + info = parts[1].select("a")[1] + server = IrcServer(info["data-s"]) + pack_message = info["data-p"] + bot, pack_number = pack_message.split(" xdcc send #") + + size = byte_string_to_byte_count(parts[5].text) + filename = parts[6].text + + pack = XDCCPack(server, bot, int(pack_number)) + pack.set_size(size) + pack.set_filename(filename) + + packs.append(pack) + + return packs From 112c245ea1fc174f5e686d91b4eae8c78833625c Mon Sep 17 00:00:00 2001 From: Hermann Krumrey Date: Thu, 5 Nov 2020 16:29:47 +0100 Subject: [PATCH 8/9] Add more command line options for username and channel join delay --- bin/xdcc-browse | 4 +++- bin/xdcc-dl | 4 +++- xdcc_dl/helper.py | 6 ++++++ xdcc_dl/xdcc/XDCCClient.py | 21 +++++++++++++++------ xdcc_dl/xdcc/__init__.py | 13 +++++++++++-- 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/bin/xdcc-browse b/bin/xdcc-browse index 113873f..bf0d3cc 100644 --- a/bin/xdcc-browse +++ b/bin/xdcc-browse @@ -57,7 +57,9 @@ def main(args: argparse.Namespace, logger: logging.Logger): timeout=args.timeout, fallback_channel=args.fallback_channel, throttle=args.throttle, - wait_time=args.wait_time + wait_time=args.wait_time, + username=args.username, + channel_join_delay=args.channel_join_delay ) except ConnectionError: diff --git a/bin/xdcc-dl b/bin/xdcc-dl index bd9fb99..9b4dab5 100644 --- a/bin/xdcc-dl +++ b/bin/xdcc-dl @@ -47,7 +47,9 @@ def main(args: argparse.Namespace, logger: logging.Logger): timeout=args.timeout, fallback_channel=args.fallback_channel, throttle=args.throttle, - wait_time=args.wait_time + wait_time=args.wait_time, + username=args.username, + channel_join_delay=args.channel_join_delay ) except DownloadIncomplete: diff --git a/xdcc_dl/helper.py b/xdcc_dl/helper.py index 6bbc93b..b45b8e9 100644 --- a/xdcc_dl/helper.py +++ b/xdcc_dl/helper.py @@ -65,3 +65,9 @@ def add_xdcc_argparse_arguments(parser: ArgumentParser): parser.add_argument("--wait-time", default=0, type=int, help="Waits for the specified amount of time before " "sending the xdcc send request") + parser.add_argument("--username", + help="Specifies a user name for the downloader bot") + parser.add_argument("--channel-join-delay", + help="Specifies a delay in seconds for how long the" + "downloader should wait before connecting to" + "channels") diff --git a/xdcc_dl/xdcc/XDCCClient.py b/xdcc_dl/xdcc/XDCCClient.py index b380ed7..5cf9cd2 100644 --- a/xdcc_dl/xdcc/XDCCClient.py +++ b/xdcc_dl/xdcc/XDCCClient.py @@ -53,7 +53,9 @@ def __init__( timeout: int = 120, fallback_channel: Optional[str] = None, throttle: Union[int, str] = -1, - wait_time: int = 0 + wait_time: int = 0, + username: str = "", + channel_join_delay: Optional[int] = None ): """ Initializes the XDCC IRC client @@ -67,6 +69,8 @@ def __init__( unlimited :param wait_time: Waits for the specified amount of time before sending a message + :param username: If specified sets the username to log on with + :param channel_join_delay: If specifies sets the channel join delay """ self.logger = ColorLogger( logging.getLogger(self.__class__.__name__), @@ -85,7 +89,7 @@ def __init__( if self.download_limit <= 0: self.download_limit = -1 - self.user = User() + self.user = User(username) self.pack = pack self.server = pack.server self.downloading = False @@ -100,6 +104,11 @@ def __init__( self.connected = True self.disconnected = False + if channel_join_delay is None: + self.channel_join_delay = random.randint(5, 10) + else: + self.channel_join_delay = channel_join_delay + # XDCC state variables self.peer_address = "" self.peer_port = -1 @@ -166,15 +175,15 @@ def download(self) -> str: try: self.logger.info("Connecting to " + self.server.address + ":" + str(self.server.port)) - self.user = User("Toni") self.connect( self.server.address, self.server.port, self.user.username ) - delay = random.randint(5, 10) - self.logger.info(f"Delaying download initialization by {delay}s") - time.sleep(delay) + + self.logger.info(f"Delaying download initialization by " + f"{self.channel_join_delay}s") + time.sleep(self.channel_join_delay) self.connected = True diff --git a/xdcc_dl/xdcc/__init__.py b/xdcc_dl/xdcc/__init__.py index cb524f2..ea8b926 100644 --- a/xdcc_dl/xdcc/__init__.py +++ b/xdcc_dl/xdcc/__init__.py @@ -27,7 +27,9 @@ def download_packs( timeout: int = 120, fallback_channel: Optional[str] = None, throttle: Union[int, str] = -1, - wait_time: int = 0 + wait_time: int = 0, + username: Optional[str] = None, + channel_join_delay: Optional[int] = None ): """ Downloads a list of XDCC Packs @@ -39,6 +41,11 @@ def download_packs( unlimited :param wait_time: Waits for the specified amount of time before sending a message + :param username: The username to use. If not specified, will use a random + one. + :param channel_join_delay: Delays the joining of channels by a set amount + of seconds. If not specified, the bot will wait + a random amount of time :return: None """ for pack in packs: @@ -47,6 +54,8 @@ def download_packs( timeout=timeout, fallback_channel=fallback_channel, throttle=throttle, - wait_time=wait_time + wait_time=wait_time, + username="" if username is None else username, + channel_join_delay=channel_join_delay ) client.download() From 955df03e7981094e8f685c07b5380f7e0795d6a4 Mon Sep 17 00:00:00 2001 From: Hermann Krumrey Date: Fri, 6 Nov 2020 11:22:53 +0100 Subject: [PATCH 9/9] Improve random username generation and timeouts with wait times --- CHANGELOG | 1 + xdcc_dl/entities/User.py | 6 +++++- xdcc_dl/helper.py | 2 +- xdcc_dl/xdcc/XDCCClient.py | 12 +++++++----- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 6ca8b65..b1fee3e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,7 @@ V 5.0.0: - Added wait_time parameter - Added file size display to xdcc-search and xdcc-browse - Added xdcc.eu search engine + - Added --username and --channel V 4.0.0: - Went over to completely rely on python's built-in logging - Throttle value is now passed as parameter diff --git a/xdcc_dl/entities/User.py b/xdcc_dl/entities/User.py index fc5b95f..8f61b81 100644 --- a/xdcc_dl/entities/User.py +++ b/xdcc_dl/entities/User.py @@ -19,6 +19,7 @@ # imports import names +import random class User(object): @@ -36,7 +37,10 @@ def __init__(self, username: str = "random"): An empty string will also result in a random username """ if username == "random" or username == "": - self.username = names.get_first_name() + self.username = \ + names.get_first_name() + \ + names.get_last_name() + \ + str(random.randint(10, 100)) else: self.username = username diff --git a/xdcc_dl/helper.py b/xdcc_dl/helper.py index b45b8e9..96f1c51 100644 --- a/xdcc_dl/helper.py +++ b/xdcc_dl/helper.py @@ -67,7 +67,7 @@ def add_xdcc_argparse_arguments(parser: ArgumentParser): "sending the xdcc send request") parser.add_argument("--username", help="Specifies a user name for the downloader bot") - parser.add_argument("--channel-join-delay", + parser.add_argument("--channel-join-delay", type=int, help="Specifies a delay in seconds for how long the" "downloader should wait before connecting to" "channels") diff --git a/xdcc_dl/xdcc/XDCCClient.py b/xdcc_dl/xdcc/XDCCClient.py index 5cf9cd2..30bc54c 100644 --- a/xdcc_dl/xdcc/XDCCClient.py +++ b/xdcc_dl/xdcc/XDCCClient.py @@ -97,7 +97,7 @@ def __init__( self.channels = None # type: Optional[List[str]] self.message_sent = False self.connect_start_time = 0.0 - self.timeout = timeout + self.timeout = timeout + wait_time self.timed_out = False self.fallback_channel = fallback_channel self.wait_time = wait_time @@ -173,8 +173,9 @@ def download(self) -> str: message = "" try: - self.logger.info("Connecting to " + self.server.address + ":" + - str(self.server.port)) + self.logger.info(f"Connecting to " + f"{self.server.address}:{self.server.port} " + f"as user '{self.user.username}'") self.connect( self.server.address, self.server.port, @@ -572,7 +573,8 @@ def timeout_watcher(self): timeout time, a ping will be sent and handled by the on_ping method :return: None """ - while not self.connected: + while not self.connected \ + or self.connect_start_time + self.wait_time > time.time(): pass self.logger.info("Timeout watcher started") while not self.message_sent and not self.disconnected: @@ -581,7 +583,7 @@ def timeout_watcher(self): if self.timeout < (time.time() - self.connect_start_time): self.logger.info("Timeout detected") self.connection.ping(self.server.address) - time.sleep(2) + break self.logger.info("Message sent without timeout") def progress_printer(self):