diff --git a/.editorconfig b/.editorconfig index 05dc0a91d..dbf1990df 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,7 +9,7 @@ trim_trailing_whitespace = true [**.py] indent_size = 4 indent_style = space -max_line_length = 79 +max_line_length = 100 [Makefile] indent_style = tab diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c88dde73e..0a6310522 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,22 +11,17 @@ on: jobs: test: - runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: python-version: - - "3.5" - - "3.6" - - "3.7" - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - - "pypy3.9" + - "pypy3.10" steps: - uses: actions/checkout@v4 @@ -45,7 +40,7 @@ jobs: - name: Install dependencies run: | pip install -r requirements.txt - pip install flake8 youtube-dl + pip install ruff youtube-dl - name: Install yt-dlp run: | @@ -63,16 +58,13 @@ jobs: ;; esac - - name: Lint with flake8 + - name: Lint with Ruff run: | - case "${{ matrix.python-version }}" in - 3.4|3.5|3.6|3.7) - flake8 --extend-exclude scripts/export_tests.py,scripts/pyprint.py . - ;; - *) - flake8 . - ;; - esac + ruff check + + - name: Check formatting with Ruff + run: | + ruff format --check - name: Run tests run: | diff --git a/.gitignore b/.gitignore index af2643ae9..e1039685b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.devcontainer/ archive/ # Byte-compiled / optimized / DLL files diff --git a/README.rst b/README.rst index 335101cb2..76b7402f1 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,7 @@ and powerful `filenaming capabilities 1: import itertools + extractor._module_iter = itertools.chain(*modules) elif not modules: extractor._module_iter = () @@ -201,54 +220,57 @@ def main(): if args.update: from . import update + extr = update.UpdateExtractor.from_url("update:" + args.update) ujob = update.UpdateJob(extr) return ujob.run() - elif args.list_modules: + if args.list_modules: extractor.modules.append("") sys.stdout.write("\n".join(extractor.modules)) elif args.list_extractors is not None: write = sys.stdout.write - fmt = ("{}{}\nCategory: {} - Subcategory: {}" - "\nExample : {}\n\n").format + fmt = ("{}{}\nCategory: {} - Subcategory: {}" "\nExample : {}\n\n").format extractors = extractor.extractors() if args.list_extractors: - fltr = util.build_extractor_filter( - args.list_extractors, negate=False) + fltr = util.build_extractor_filter(args.list_extractors, negate=False) extractors = filter(fltr, extractors) for extr in extractors: - write(fmt( - extr.__name__, - "\n" + extr.__doc__ if extr.__doc__ else "", - extr.category, extr.subcategory, - extr.example, - )) + write( + fmt( + extr.__name__, + "\n" + extr.__doc__ if extr.__doc__ else "", + extr.category, + extr.subcategory, + extr.example, + ) + ) elif args.clear_cache: from . import cache + log = logging.getLogger("cache") cnt = cache.clear(args.clear_cache) if cnt is None: log.error("Database file not available") return 1 - else: - log.info( - "Deleted %d %s from '%s'", - cnt, "entry" if cnt == 1 else "entries", cache._path(), - ) + log.info( + "Deleted %d %s from '%s'", + cnt, + "entry" if cnt == 1 else "entries", + cache._path(), + ) elif args.config: if args.config == "init": return config.initialize() - elif args.config == "status": + if args.config == "status": return config.status() - else: - return config.open_extern() + return config.open_extern() else: input_files = config.get((), "input-files") @@ -259,20 +281,19 @@ def main(): args.input_files.append(input_file) if not args.urls and not args.input_files: - if args.cookies_from_browser or config.interpolate( - ("extractor",), "cookies"): + if args.cookies_from_browser or config.interpolate(("extractor",), "cookies"): args.urls.append("noop") else: parser.error( "The following arguments are required: URL\nUse " - "'gallery-dl --help' to get a list of all options.") + "'gallery-dl --help' to get a list of all options." + ) if args.list_urls: jobtype = job.UrlJob jobtype.maxdepth = args.list_urls if config.get(("output",), "fallback", True): - jobtype.handle_url = \ - staticmethod(jobtype.handle_url_fallback) + jobtype.handle_url = staticmethod(jobtype.handle_url_fallback) elif args.dump_json: jobtype = job.DataJob jobtype.resolve = args.dump_json - 1 @@ -283,16 +304,14 @@ def main(): input_manager.log = input_log = logging.getLogger("inputfile") # unsupported file logging handler - handler = output.setup_logging_handler( - "unsupportedfile", fmt="{message}") + handler = output.setup_logging_handler("unsupportedfile", fmt="{message}") if handler: ulog = job.Job.ulog = logging.getLogger("unsupported") ulog.addHandler(handler) ulog.propagate = False # error file logging handler - handler = output.setup_logging_handler( - "errorfile", fmt="{message}", mode="a") + handler = output.setup_logging_handler("errorfile", fmt="{message}", mode="a") if handler: elog = input_manager.err = logging.getLogger("errorfile") elog.addHandler(handler) @@ -311,8 +330,7 @@ def main(): return getattr(exc, "code", 128) pformat = config.get(("output",), "progress", True) - if pformat and len(input_manager.urls) > 1 and \ - args.loglevel < logging.ERROR: + if pformat and len(input_manager.urls) > 1 and args.loglevel < logging.ERROR: input_manager.progress(pformat) # process input URLs @@ -357,13 +375,13 @@ def main(): pass except OSError as exc: import errno + if exc.errno != errno.EPIPE: raise return 1 -class InputManager(): - +class InputManager: def __init__(self): self.urls = [] self.files = () @@ -448,7 +466,7 @@ def add_file(self, path, action=None): # empty line or comment continue - elif line[0] == "-": + if line[0] == "-": # config spec if len(line) >= 2 and line[1] == "G": conf = gconf @@ -463,15 +481,18 @@ def add_file(self, path, action=None): if not sep: raise exception.InputFileError( "Invalid KEY=VALUE pair '%s' on line %s in %s", - line, n+1, path) + line, + n + 1, + path, + ) try: value = util.json_loads(value.strip()) except ValueError as exc: self.log.debug("%s: %s", exc.__class__.__name__, exc) raise exception.InputFileError( - "Unable to parse '%s' on line %s in %s", - value, n+1, path) + "Unable to parse '%s' on line %s in %s", value, n + 1, path + ) key = key.strip().split(".") conf.append((key[:-1], key[-1], value)) @@ -481,6 +502,7 @@ def add_file(self, path, action=None): if " #" in line or "\t#" in line: if strip_comment is None: import re + strip_comment = re.compile(r"\s+#.*").sub line = strip_comment("", line) if gconf or lconf: @@ -532,9 +554,7 @@ def _rewrite(self): with open(path, "w", encoding="utf-8") as fp: fp.writelines(lines) except Exception as exc: - self.log.warning( - "Unable to update '%s' (%s: %s)", - path, exc.__class__.__name__, exc) + self.log.warning("Unable to update '%s' (%s: %s)", path, exc.__class__.__name__, exc) @staticmethod def _action_comment(lines, indicies): @@ -564,16 +584,21 @@ def __next__(self): self._url = url if self._pformat: - output.stderr_write(self._pformat({ - "total" : len(self.urls), - "current": self._index + 1, - "url" : url, - })) + output.stderr_write( + self._pformat( + { + "total": len(self.urls), + "current": self._index + 1, + "url": url, + } + ) + ) return url -class ExtendedUrl(): +class ExtendedUrl: """URL with attached config key-value pairs""" + __slots__ = ("value", "gconfig", "lconfig") def __init__(self, url, gconf, lconf): diff --git a/gallery_dl/__main__.py b/gallery_dl/__main__.py index db3a34e81..3601adc85 100644 --- a/gallery_dl/__main__.py +++ b/gallery_dl/__main__.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2017-2023 Mike Fährmann # @@ -11,6 +10,7 @@ if not __package__ and not hasattr(sys, "frozen"): import os.path + path = os.path.realpath(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(os.path.dirname(path))) diff --git a/gallery_dl/actions.py b/gallery_dl/actions.py index 668032d5d..2a6de0e34 100644 --- a/gallery_dl/actions.py +++ b/gallery_dl/actions.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,15 @@ """ """ -import re -import time +import functools import logging import operator -import functools -from . import util, exception +import re +import time +from contextlib import suppress + +from . import exception +from . import util def parse(actionspec): @@ -74,8 +75,7 @@ def parse(actionspec): return actions -class LoggerAdapter(): - +class LoggerAdapter: def __init__(self, logger, job): self.logger = logger self.extra = job._logger_extra @@ -126,14 +126,17 @@ def _chain_actions(actions): def _chain(args): for action in actions: action(args) + return _chain # -------------------------------------------------------------------- + def action_print(opts): def _print(_): print(opts) + return None, _print @@ -151,6 +154,7 @@ def action_status(opts): def _status(args): args["job"].status = op(args["job"].status, value) + return _status, None @@ -159,12 +163,14 @@ def action_level(opts): def _level(args): args["level"] = level + return _level, None def action_exec(opts): def _exec(_): util.Popen(opts, shell=True).wait() + return None, _exec @@ -175,6 +181,7 @@ def action_wait(opts): def _wait(args): time.sleep(seconds()) else: + def _wait(args): input("Press Enter to continue") @@ -194,24 +201,23 @@ def action_restart(opts): def action_exit(opts): - try: + with suppress(ValueError): opts = int(opts) - except ValueError: - pass def _exit(args): raise SystemExit(opts) + return None, _exit ACTIONS = { - "abort" : action_abort, - "exec" : action_exec, - "exit" : action_exit, - "level" : action_level, - "print" : action_print, - "restart" : action_restart, - "status" : action_status, + "abort": action_abort, + "exec": action_exec, + "exit": action_exit, + "level": action_level, + "print": action_print, + "restart": action_restart, + "status": action_status, "terminate": action_terminate, - "wait" : action_wait, + "wait": action_wait, } diff --git a/gallery_dl/aes.py b/gallery_dl/aes.py index 891104abf..0b55894da 100644 --- a/gallery_dl/aes.py +++ b/gallery_dl/aes.py @@ -1,10 +1,8 @@ -# -*- coding: utf-8 -*- - # This is a slightly modified version of yt-dlp's aes module. # https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/aes.py -import struct import binascii +import struct from math import ceil try: @@ -17,32 +15,36 @@ if Cryptodome_AES: + def aes_cbc_decrypt_bytes(data, key, iv): """Decrypt bytes with AES-CBC using pycryptodome""" - return Cryptodome_AES.new( - key, Cryptodome_AES.MODE_CBC, iv).decrypt(data) + return Cryptodome_AES.new(key, Cryptodome_AES.MODE_CBC, iv).decrypt(data) def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce): """Decrypt bytes with AES-GCM using pycryptodome""" - return Cryptodome_AES.new( - key, Cryptodome_AES.MODE_GCM, nonce).decrypt_and_verify(data, tag) + return Cryptodome_AES.new(key, Cryptodome_AES.MODE_GCM, nonce).decrypt_and_verify(data, tag) else: + def aes_cbc_decrypt_bytes(data, key, iv): """Decrypt bytes with AES-CBC using native implementation""" - return intlist_to_bytes(aes_cbc_decrypt( - bytes_to_intlist(data), - bytes_to_intlist(key), - bytes_to_intlist(iv), - )) + return intlist_to_bytes( + aes_cbc_decrypt( + bytes_to_intlist(data), + bytes_to_intlist(key), + bytes_to_intlist(iv), + ) + ) def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce): """Decrypt bytes with AES-GCM using native implementation""" - return intlist_to_bytes(aes_gcm_decrypt_and_verify( - bytes_to_intlist(data), - bytes_to_intlist(key), - bytes_to_intlist(tag), - bytes_to_intlist(nonce), - )) + return intlist_to_bytes( + aes_gcm_decrypt_and_verify( + bytes_to_intlist(data), + bytes_to_intlist(key), + bytes_to_intlist(tag), + bytes_to_intlist(nonce), + ) + ) bytes_to_intlist = list @@ -55,7 +57,7 @@ def intlist_to_bytes(xs): def unpad_pkcs7(data): - return data[:-data[-1]] + return data[: -data[-1]] BLOCK_SIZE_BYTES = 16 @@ -75,9 +77,9 @@ def aes_ecb_encrypt(data, key, iv=None): encrypted_data = [] for i in range(block_count): - block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES] + block = data[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES] encrypted_data += aes_encrypt(block, expanded_key) - encrypted_data = encrypted_data[:len(data)] + encrypted_data = encrypted_data[: len(data)] return encrypted_data @@ -96,9 +98,9 @@ def aes_ecb_decrypt(data, key, iv=None): encrypted_data = [] for i in range(block_count): - block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES] + block = data[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES] encrypted_data += aes_decrypt(block, expanded_key) - encrypted_data = encrypted_data[:len(data)] + encrypted_data = encrypted_data[: len(data)] return encrypted_data @@ -131,12 +133,12 @@ def aes_ctr_encrypt(data, key, iv): encrypted_data = [] for i in range(block_count): counter_block = next(counter) - block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES] + block = data[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES] block += [0] * (BLOCK_SIZE_BYTES - len(block)) cipher_counter_block = aes_encrypt(counter_block, expanded_key) encrypted_data += xor(block, cipher_counter_block) - encrypted_data = encrypted_data[:len(data)] + encrypted_data = encrypted_data[: len(data)] return encrypted_data @@ -156,13 +158,13 @@ def aes_cbc_decrypt(data, key, iv): decrypted_data = [] previous_cipher_block = iv for i in range(block_count): - block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES] + block = data[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES] block += [0] * (BLOCK_SIZE_BYTES - len(block)) decrypted_block = aes_decrypt(block, expanded_key) decrypted_data += xor(decrypted_block, previous_cipher_block) previous_cipher_block = block - decrypted_data = decrypted_data[:len(data)] + decrypted_data = decrypted_data[: len(data)] return decrypted_data @@ -182,7 +184,7 @@ def aes_cbc_encrypt(data, key, iv): encrypted_data = [] previous_cipher_block = iv for i in range(block_count): - block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES] + block = data[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES] remaining_length = BLOCK_SIZE_BYTES - len(block) block += [remaining_length] * remaining_length mixed_block = xor(block, previous_cipher_block) @@ -213,10 +215,8 @@ def aes_gcm_decrypt_and_verify(data, key, tag, nonce): if len(nonce) == 12: j0 = nonce + [0, 0, 0, 1] else: - fill = (BLOCK_SIZE_BYTES - (len(nonce) % BLOCK_SIZE_BYTES)) % \ - BLOCK_SIZE_BYTES + 8 - ghash_in = nonce + [0] * fill + bytes_to_intlist( - (8 * len(nonce)).to_bytes(8, "big")) + fill = (BLOCK_SIZE_BYTES - (len(nonce) % BLOCK_SIZE_BYTES)) % BLOCK_SIZE_BYTES + 8 + ghash_in = nonce + [0] * fill + bytes_to_intlist((8 * len(nonce)).to_bytes(8, "big")) j0 = ghash(hash_subkey, ghash_in) # TODO: add nonce support to aes_ctr_decrypt @@ -224,19 +224,17 @@ def aes_gcm_decrypt_and_verify(data, key, tag, nonce): # nonce_ctr = j0[:12] iv_ctr = inc(j0) - decrypted_data = aes_ctr_decrypt( - data, key, iv_ctr + [0] * (BLOCK_SIZE_BYTES - len(iv_ctr))) + decrypted_data = aes_ctr_decrypt(data, key, iv_ctr + [0] * (BLOCK_SIZE_BYTES - len(iv_ctr))) - pad_len = ( - (BLOCK_SIZE_BYTES - (len(data) % BLOCK_SIZE_BYTES)) % BLOCK_SIZE_BYTES) + pad_len = (BLOCK_SIZE_BYTES - (len(data) % BLOCK_SIZE_BYTES)) % BLOCK_SIZE_BYTES s_tag = ghash( hash_subkey, - data + - [0] * pad_len + # pad - bytes_to_intlist( - (0 * 8).to_bytes(8, "big") + # length of associated data - ((len(data) * 8).to_bytes(8, "big")) # length of data - ) + data + + [0] * pad_len # pad + + bytes_to_intlist( + (0 * 8).to_bytes(8, "big") # length of associated data + + ((len(data) * 8).to_bytes(8, "big")) # length of data + ), ) if tag != aes_ctr_encrypt(s_tag, key, j0): @@ -261,8 +259,7 @@ def aes_encrypt(data, expanded_key): data = shift_rows(data) if i != rounds: data = list(iter_mix_columns(data, MIX_COLUMN_MATRIX)) - data = xor(data, expanded_key[ - i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]) + data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES]) return data @@ -278,8 +275,7 @@ def aes_decrypt(data, expanded_key): rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1 for i in range(rounds, 0, -1): - data = xor(data, expanded_key[ - i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]) + data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES]) if i != rounds: data = list(iter_mix_columns(data, MIX_COLUMN_MATRIX_INV)) data = shift_rows_inv(data) @@ -311,89 +307,548 @@ def aes_decrypt_text(data, password, key_size_bytes): password = bytes_to_intlist(password.encode("utf-8")) key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password)) - key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * \ - (key_size_bytes // BLOCK_SIZE_BYTES) + key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * ( + key_size_bytes // BLOCK_SIZE_BYTES + ) nonce = data[:NONCE_LENGTH_BYTES] cipher = data[NONCE_LENGTH_BYTES:] - return intlist_to_bytes(aes_ctr_decrypt( - cipher, key, nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES) - )) + return intlist_to_bytes( + aes_ctr_decrypt(cipher, key, nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)) + ) RCON = ( - 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, + 0x8D, + 0x01, + 0x02, + 0x04, + 0x08, + 0x10, + 0x20, + 0x40, + 0x80, + 0x1B, + 0x36, ) SBOX = ( - 0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, - 0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76, - 0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0, - 0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0, - 0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC, - 0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15, - 0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A, - 0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75, - 0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0, - 0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84, - 0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B, - 0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF, - 0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85, - 0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8, - 0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5, - 0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2, - 0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17, - 0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73, - 0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88, - 0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB, - 0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C, - 0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79, - 0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9, - 0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08, - 0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6, - 0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A, - 0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, - 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E, - 0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, - 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF, - 0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, - 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16, + 0x63, + 0x7C, + 0x77, + 0x7B, + 0xF2, + 0x6B, + 0x6F, + 0xC5, + 0x30, + 0x01, + 0x67, + 0x2B, + 0xFE, + 0xD7, + 0xAB, + 0x76, + 0xCA, + 0x82, + 0xC9, + 0x7D, + 0xFA, + 0x59, + 0x47, + 0xF0, + 0xAD, + 0xD4, + 0xA2, + 0xAF, + 0x9C, + 0xA4, + 0x72, + 0xC0, + 0xB7, + 0xFD, + 0x93, + 0x26, + 0x36, + 0x3F, + 0xF7, + 0xCC, + 0x34, + 0xA5, + 0xE5, + 0xF1, + 0x71, + 0xD8, + 0x31, + 0x15, + 0x04, + 0xC7, + 0x23, + 0xC3, + 0x18, + 0x96, + 0x05, + 0x9A, + 0x07, + 0x12, + 0x80, + 0xE2, + 0xEB, + 0x27, + 0xB2, + 0x75, + 0x09, + 0x83, + 0x2C, + 0x1A, + 0x1B, + 0x6E, + 0x5A, + 0xA0, + 0x52, + 0x3B, + 0xD6, + 0xB3, + 0x29, + 0xE3, + 0x2F, + 0x84, + 0x53, + 0xD1, + 0x00, + 0xED, + 0x20, + 0xFC, + 0xB1, + 0x5B, + 0x6A, + 0xCB, + 0xBE, + 0x39, + 0x4A, + 0x4C, + 0x58, + 0xCF, + 0xD0, + 0xEF, + 0xAA, + 0xFB, + 0x43, + 0x4D, + 0x33, + 0x85, + 0x45, + 0xF9, + 0x02, + 0x7F, + 0x50, + 0x3C, + 0x9F, + 0xA8, + 0x51, + 0xA3, + 0x40, + 0x8F, + 0x92, + 0x9D, + 0x38, + 0xF5, + 0xBC, + 0xB6, + 0xDA, + 0x21, + 0x10, + 0xFF, + 0xF3, + 0xD2, + 0xCD, + 0x0C, + 0x13, + 0xEC, + 0x5F, + 0x97, + 0x44, + 0x17, + 0xC4, + 0xA7, + 0x7E, + 0x3D, + 0x64, + 0x5D, + 0x19, + 0x73, + 0x60, + 0x81, + 0x4F, + 0xDC, + 0x22, + 0x2A, + 0x90, + 0x88, + 0x46, + 0xEE, + 0xB8, + 0x14, + 0xDE, + 0x5E, + 0x0B, + 0xDB, + 0xE0, + 0x32, + 0x3A, + 0x0A, + 0x49, + 0x06, + 0x24, + 0x5C, + 0xC2, + 0xD3, + 0xAC, + 0x62, + 0x91, + 0x95, + 0xE4, + 0x79, + 0xE7, + 0xC8, + 0x37, + 0x6D, + 0x8D, + 0xD5, + 0x4E, + 0xA9, + 0x6C, + 0x56, + 0xF4, + 0xEA, + 0x65, + 0x7A, + 0xAE, + 0x08, + 0xBA, + 0x78, + 0x25, + 0x2E, + 0x1C, + 0xA6, + 0xB4, + 0xC6, + 0xE8, + 0xDD, + 0x74, + 0x1F, + 0x4B, + 0xBD, + 0x8B, + 0x8A, + 0x70, + 0x3E, + 0xB5, + 0x66, + 0x48, + 0x03, + 0xF6, + 0x0E, + 0x61, + 0x35, + 0x57, + 0xB9, + 0x86, + 0xC1, + 0x1D, + 0x9E, + 0xE1, + 0xF8, + 0x98, + 0x11, + 0x69, + 0xD9, + 0x8E, + 0x94, + 0x9B, + 0x1E, + 0x87, + 0xE9, + 0xCE, + 0x55, + 0x28, + 0xDF, + 0x8C, + 0xA1, + 0x89, + 0x0D, + 0xBF, + 0xE6, + 0x42, + 0x68, + 0x41, + 0x99, + 0x2D, + 0x0F, + 0xB0, + 0x54, + 0xBB, + 0x16, ) SBOX_INV = ( - 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, - 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, - 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, - 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, - 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, - 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, - 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, - 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, - 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, - 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, - 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, - 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, - 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, - 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, - 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, - 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, - 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, - 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, - 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, - 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, - 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, - 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, - 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, - 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, - 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, - 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, - 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, - 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, - 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, - 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, - 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, - 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d + 0x52, + 0x09, + 0x6A, + 0xD5, + 0x30, + 0x36, + 0xA5, + 0x38, + 0xBF, + 0x40, + 0xA3, + 0x9E, + 0x81, + 0xF3, + 0xD7, + 0xFB, + 0x7C, + 0xE3, + 0x39, + 0x82, + 0x9B, + 0x2F, + 0xFF, + 0x87, + 0x34, + 0x8E, + 0x43, + 0x44, + 0xC4, + 0xDE, + 0xE9, + 0xCB, + 0x54, + 0x7B, + 0x94, + 0x32, + 0xA6, + 0xC2, + 0x23, + 0x3D, + 0xEE, + 0x4C, + 0x95, + 0x0B, + 0x42, + 0xFA, + 0xC3, + 0x4E, + 0x08, + 0x2E, + 0xA1, + 0x66, + 0x28, + 0xD9, + 0x24, + 0xB2, + 0x76, + 0x5B, + 0xA2, + 0x49, + 0x6D, + 0x8B, + 0xD1, + 0x25, + 0x72, + 0xF8, + 0xF6, + 0x64, + 0x86, + 0x68, + 0x98, + 0x16, + 0xD4, + 0xA4, + 0x5C, + 0xCC, + 0x5D, + 0x65, + 0xB6, + 0x92, + 0x6C, + 0x70, + 0x48, + 0x50, + 0xFD, + 0xED, + 0xB9, + 0xDA, + 0x5E, + 0x15, + 0x46, + 0x57, + 0xA7, + 0x8D, + 0x9D, + 0x84, + 0x90, + 0xD8, + 0xAB, + 0x00, + 0x8C, + 0xBC, + 0xD3, + 0x0A, + 0xF7, + 0xE4, + 0x58, + 0x05, + 0xB8, + 0xB3, + 0x45, + 0x06, + 0xD0, + 0x2C, + 0x1E, + 0x8F, + 0xCA, + 0x3F, + 0x0F, + 0x02, + 0xC1, + 0xAF, + 0xBD, + 0x03, + 0x01, + 0x13, + 0x8A, + 0x6B, + 0x3A, + 0x91, + 0x11, + 0x41, + 0x4F, + 0x67, + 0xDC, + 0xEA, + 0x97, + 0xF2, + 0xCF, + 0xCE, + 0xF0, + 0xB4, + 0xE6, + 0x73, + 0x96, + 0xAC, + 0x74, + 0x22, + 0xE7, + 0xAD, + 0x35, + 0x85, + 0xE2, + 0xF9, + 0x37, + 0xE8, + 0x1C, + 0x75, + 0xDF, + 0x6E, + 0x47, + 0xF1, + 0x1A, + 0x71, + 0x1D, + 0x29, + 0xC5, + 0x89, + 0x6F, + 0xB7, + 0x62, + 0x0E, + 0xAA, + 0x18, + 0xBE, + 0x1B, + 0xFC, + 0x56, + 0x3E, + 0x4B, + 0xC6, + 0xD2, + 0x79, + 0x20, + 0x9A, + 0xDB, + 0xC0, + 0xFE, + 0x78, + 0xCD, + 0x5A, + 0xF4, + 0x1F, + 0xDD, + 0xA8, + 0x33, + 0x88, + 0x07, + 0xC7, + 0x31, + 0xB1, + 0x12, + 0x10, + 0x59, + 0x27, + 0x80, + 0xEC, + 0x5F, + 0x60, + 0x51, + 0x7F, + 0xA9, + 0x19, + 0xB5, + 0x4A, + 0x0D, + 0x2D, + 0xE5, + 0x7A, + 0x9F, + 0x93, + 0xC9, + 0x9C, + 0xEF, + 0xA0, + 0xE0, + 0x3B, + 0x4D, + 0xAE, + 0x2A, + 0xF5, + 0xB0, + 0xC8, + 0xEB, + 0xBB, + 0x3C, + 0x83, + 0x53, + 0x99, + 0x61, + 0x17, + 0x2B, + 0x04, + 0x7E, + 0xBA, + 0x77, + 0xD6, + 0x26, + 0xE1, + 0x69, + 0x14, + 0x63, + 0x55, + 0x21, + 0x0C, + 0x7D, ) MIX_COLUMN_MATRIX = ( @@ -411,73 +866,521 @@ def aes_decrypt_text(data, password, key_size_bytes): ) RIJNDAEL_EXP_TABLE = ( - 0x01, 0x03, 0x05, 0x0F, 0x11, 0x33, 0x55, 0xFF, - 0x1A, 0x2E, 0x72, 0x96, 0xA1, 0xF8, 0x13, 0x35, - 0x5F, 0xE1, 0x38, 0x48, 0xD8, 0x73, 0x95, 0xA4, - 0xF7, 0x02, 0x06, 0x0A, 0x1E, 0x22, 0x66, 0xAA, - 0xE5, 0x34, 0x5C, 0xE4, 0x37, 0x59, 0xEB, 0x26, - 0x6A, 0xBE, 0xD9, 0x70, 0x90, 0xAB, 0xE6, 0x31, - 0x53, 0xF5, 0x04, 0x0C, 0x14, 0x3C, 0x44, 0xCC, - 0x4F, 0xD1, 0x68, 0xB8, 0xD3, 0x6E, 0xB2, 0xCD, - 0x4C, 0xD4, 0x67, 0xA9, 0xE0, 0x3B, 0x4D, 0xD7, - 0x62, 0xA6, 0xF1, 0x08, 0x18, 0x28, 0x78, 0x88, - 0x83, 0x9E, 0xB9, 0xD0, 0x6B, 0xBD, 0xDC, 0x7F, - 0x81, 0x98, 0xB3, 0xCE, 0x49, 0xDB, 0x76, 0x9A, - 0xB5, 0xC4, 0x57, 0xF9, 0x10, 0x30, 0x50, 0xF0, - 0x0B, 0x1D, 0x27, 0x69, 0xBB, 0xD6, 0x61, 0xA3, - 0xFE, 0x19, 0x2B, 0x7D, 0x87, 0x92, 0xAD, 0xEC, - 0x2F, 0x71, 0x93, 0xAE, 0xE9, 0x20, 0x60, 0xA0, - 0xFB, 0x16, 0x3A, 0x4E, 0xD2, 0x6D, 0xB7, 0xC2, - 0x5D, 0xE7, 0x32, 0x56, 0xFA, 0x15, 0x3F, 0x41, - 0xC3, 0x5E, 0xE2, 0x3D, 0x47, 0xC9, 0x40, 0xC0, - 0x5B, 0xED, 0x2C, 0x74, 0x9C, 0xBF, 0xDA, 0x75, - 0x9F, 0xBA, 0xD5, 0x64, 0xAC, 0xEF, 0x2A, 0x7E, - 0x82, 0x9D, 0xBC, 0xDF, 0x7A, 0x8E, 0x89, 0x80, - 0x9B, 0xB6, 0xC1, 0x58, 0xE8, 0x23, 0x65, 0xAF, - 0xEA, 0x25, 0x6F, 0xB1, 0xC8, 0x43, 0xC5, 0x54, - 0xFC, 0x1F, 0x21, 0x63, 0xA5, 0xF4, 0x07, 0x09, - 0x1B, 0x2D, 0x77, 0x99, 0xB0, 0xCB, 0x46, 0xCA, - 0x45, 0xCF, 0x4A, 0xDE, 0x79, 0x8B, 0x86, 0x91, - 0xA8, 0xE3, 0x3E, 0x42, 0xC6, 0x51, 0xF3, 0x0E, - 0x12, 0x36, 0x5A, 0xEE, 0x29, 0x7B, 0x8D, 0x8C, - 0x8F, 0x8A, 0x85, 0x94, 0xA7, 0xF2, 0x0D, 0x17, - 0x39, 0x4B, 0xDD, 0x7C, 0x84, 0x97, 0xA2, 0xFD, - 0x1C, 0x24, 0x6C, 0xB4, 0xC7, 0x52, 0xF6, 0x01, + 0x01, + 0x03, + 0x05, + 0x0F, + 0x11, + 0x33, + 0x55, + 0xFF, + 0x1A, + 0x2E, + 0x72, + 0x96, + 0xA1, + 0xF8, + 0x13, + 0x35, + 0x5F, + 0xE1, + 0x38, + 0x48, + 0xD8, + 0x73, + 0x95, + 0xA4, + 0xF7, + 0x02, + 0x06, + 0x0A, + 0x1E, + 0x22, + 0x66, + 0xAA, + 0xE5, + 0x34, + 0x5C, + 0xE4, + 0x37, + 0x59, + 0xEB, + 0x26, + 0x6A, + 0xBE, + 0xD9, + 0x70, + 0x90, + 0xAB, + 0xE6, + 0x31, + 0x53, + 0xF5, + 0x04, + 0x0C, + 0x14, + 0x3C, + 0x44, + 0xCC, + 0x4F, + 0xD1, + 0x68, + 0xB8, + 0xD3, + 0x6E, + 0xB2, + 0xCD, + 0x4C, + 0xD4, + 0x67, + 0xA9, + 0xE0, + 0x3B, + 0x4D, + 0xD7, + 0x62, + 0xA6, + 0xF1, + 0x08, + 0x18, + 0x28, + 0x78, + 0x88, + 0x83, + 0x9E, + 0xB9, + 0xD0, + 0x6B, + 0xBD, + 0xDC, + 0x7F, + 0x81, + 0x98, + 0xB3, + 0xCE, + 0x49, + 0xDB, + 0x76, + 0x9A, + 0xB5, + 0xC4, + 0x57, + 0xF9, + 0x10, + 0x30, + 0x50, + 0xF0, + 0x0B, + 0x1D, + 0x27, + 0x69, + 0xBB, + 0xD6, + 0x61, + 0xA3, + 0xFE, + 0x19, + 0x2B, + 0x7D, + 0x87, + 0x92, + 0xAD, + 0xEC, + 0x2F, + 0x71, + 0x93, + 0xAE, + 0xE9, + 0x20, + 0x60, + 0xA0, + 0xFB, + 0x16, + 0x3A, + 0x4E, + 0xD2, + 0x6D, + 0xB7, + 0xC2, + 0x5D, + 0xE7, + 0x32, + 0x56, + 0xFA, + 0x15, + 0x3F, + 0x41, + 0xC3, + 0x5E, + 0xE2, + 0x3D, + 0x47, + 0xC9, + 0x40, + 0xC0, + 0x5B, + 0xED, + 0x2C, + 0x74, + 0x9C, + 0xBF, + 0xDA, + 0x75, + 0x9F, + 0xBA, + 0xD5, + 0x64, + 0xAC, + 0xEF, + 0x2A, + 0x7E, + 0x82, + 0x9D, + 0xBC, + 0xDF, + 0x7A, + 0x8E, + 0x89, + 0x80, + 0x9B, + 0xB6, + 0xC1, + 0x58, + 0xE8, + 0x23, + 0x65, + 0xAF, + 0xEA, + 0x25, + 0x6F, + 0xB1, + 0xC8, + 0x43, + 0xC5, + 0x54, + 0xFC, + 0x1F, + 0x21, + 0x63, + 0xA5, + 0xF4, + 0x07, + 0x09, + 0x1B, + 0x2D, + 0x77, + 0x99, + 0xB0, + 0xCB, + 0x46, + 0xCA, + 0x45, + 0xCF, + 0x4A, + 0xDE, + 0x79, + 0x8B, + 0x86, + 0x91, + 0xA8, + 0xE3, + 0x3E, + 0x42, + 0xC6, + 0x51, + 0xF3, + 0x0E, + 0x12, + 0x36, + 0x5A, + 0xEE, + 0x29, + 0x7B, + 0x8D, + 0x8C, + 0x8F, + 0x8A, + 0x85, + 0x94, + 0xA7, + 0xF2, + 0x0D, + 0x17, + 0x39, + 0x4B, + 0xDD, + 0x7C, + 0x84, + 0x97, + 0xA2, + 0xFD, + 0x1C, + 0x24, + 0x6C, + 0xB4, + 0xC7, + 0x52, + 0xF6, + 0x01, ) RIJNDAEL_LOG_TABLE = ( - 0x00, 0x00, 0x19, 0x01, 0x32, 0x02, 0x1a, 0xc6, - 0x4b, 0xc7, 0x1b, 0x68, 0x33, 0xee, 0xdf, 0x03, - 0x64, 0x04, 0xe0, 0x0e, 0x34, 0x8d, 0x81, 0xef, - 0x4c, 0x71, 0x08, 0xc8, 0xf8, 0x69, 0x1c, 0xc1, - 0x7d, 0xc2, 0x1d, 0xb5, 0xf9, 0xb9, 0x27, 0x6a, - 0x4d, 0xe4, 0xa6, 0x72, 0x9a, 0xc9, 0x09, 0x78, - 0x65, 0x2f, 0x8a, 0x05, 0x21, 0x0f, 0xe1, 0x24, - 0x12, 0xf0, 0x82, 0x45, 0x35, 0x93, 0xda, 0x8e, - 0x96, 0x8f, 0xdb, 0xbd, 0x36, 0xd0, 0xce, 0x94, - 0x13, 0x5c, 0xd2, 0xf1, 0x40, 0x46, 0x83, 0x38, - 0x66, 0xdd, 0xfd, 0x30, 0xbf, 0x06, 0x8b, 0x62, - 0xb3, 0x25, 0xe2, 0x98, 0x22, 0x88, 0x91, 0x10, - 0x7e, 0x6e, 0x48, 0xc3, 0xa3, 0xb6, 0x1e, 0x42, - 0x3a, 0x6b, 0x28, 0x54, 0xfa, 0x85, 0x3d, 0xba, - 0x2b, 0x79, 0x0a, 0x15, 0x9b, 0x9f, 0x5e, 0xca, - 0x4e, 0xd4, 0xac, 0xe5, 0xf3, 0x73, 0xa7, 0x57, - 0xaf, 0x58, 0xa8, 0x50, 0xf4, 0xea, 0xd6, 0x74, - 0x4f, 0xae, 0xe9, 0xd5, 0xe7, 0xe6, 0xad, 0xe8, - 0x2c, 0xd7, 0x75, 0x7a, 0xeb, 0x16, 0x0b, 0xf5, - 0x59, 0xcb, 0x5f, 0xb0, 0x9c, 0xa9, 0x51, 0xa0, - 0x7f, 0x0c, 0xf6, 0x6f, 0x17, 0xc4, 0x49, 0xec, - 0xd8, 0x43, 0x1f, 0x2d, 0xa4, 0x76, 0x7b, 0xb7, - 0xcc, 0xbb, 0x3e, 0x5a, 0xfb, 0x60, 0xb1, 0x86, - 0x3b, 0x52, 0xa1, 0x6c, 0xaa, 0x55, 0x29, 0x9d, - 0x97, 0xb2, 0x87, 0x90, 0x61, 0xbe, 0xdc, 0xfc, - 0xbc, 0x95, 0xcf, 0xcd, 0x37, 0x3f, 0x5b, 0xd1, - 0x53, 0x39, 0x84, 0x3c, 0x41, 0xa2, 0x6d, 0x47, - 0x14, 0x2a, 0x9e, 0x5d, 0x56, 0xf2, 0xd3, 0xab, - 0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, - 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5, - 0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, - 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07, + 0x00, + 0x00, + 0x19, + 0x01, + 0x32, + 0x02, + 0x1A, + 0xC6, + 0x4B, + 0xC7, + 0x1B, + 0x68, + 0x33, + 0xEE, + 0xDF, + 0x03, + 0x64, + 0x04, + 0xE0, + 0x0E, + 0x34, + 0x8D, + 0x81, + 0xEF, + 0x4C, + 0x71, + 0x08, + 0xC8, + 0xF8, + 0x69, + 0x1C, + 0xC1, + 0x7D, + 0xC2, + 0x1D, + 0xB5, + 0xF9, + 0xB9, + 0x27, + 0x6A, + 0x4D, + 0xE4, + 0xA6, + 0x72, + 0x9A, + 0xC9, + 0x09, + 0x78, + 0x65, + 0x2F, + 0x8A, + 0x05, + 0x21, + 0x0F, + 0xE1, + 0x24, + 0x12, + 0xF0, + 0x82, + 0x45, + 0x35, + 0x93, + 0xDA, + 0x8E, + 0x96, + 0x8F, + 0xDB, + 0xBD, + 0x36, + 0xD0, + 0xCE, + 0x94, + 0x13, + 0x5C, + 0xD2, + 0xF1, + 0x40, + 0x46, + 0x83, + 0x38, + 0x66, + 0xDD, + 0xFD, + 0x30, + 0xBF, + 0x06, + 0x8B, + 0x62, + 0xB3, + 0x25, + 0xE2, + 0x98, + 0x22, + 0x88, + 0x91, + 0x10, + 0x7E, + 0x6E, + 0x48, + 0xC3, + 0xA3, + 0xB6, + 0x1E, + 0x42, + 0x3A, + 0x6B, + 0x28, + 0x54, + 0xFA, + 0x85, + 0x3D, + 0xBA, + 0x2B, + 0x79, + 0x0A, + 0x15, + 0x9B, + 0x9F, + 0x5E, + 0xCA, + 0x4E, + 0xD4, + 0xAC, + 0xE5, + 0xF3, + 0x73, + 0xA7, + 0x57, + 0xAF, + 0x58, + 0xA8, + 0x50, + 0xF4, + 0xEA, + 0xD6, + 0x74, + 0x4F, + 0xAE, + 0xE9, + 0xD5, + 0xE7, + 0xE6, + 0xAD, + 0xE8, + 0x2C, + 0xD7, + 0x75, + 0x7A, + 0xEB, + 0x16, + 0x0B, + 0xF5, + 0x59, + 0xCB, + 0x5F, + 0xB0, + 0x9C, + 0xA9, + 0x51, + 0xA0, + 0x7F, + 0x0C, + 0xF6, + 0x6F, + 0x17, + 0xC4, + 0x49, + 0xEC, + 0xD8, + 0x43, + 0x1F, + 0x2D, + 0xA4, + 0x76, + 0x7B, + 0xB7, + 0xCC, + 0xBB, + 0x3E, + 0x5A, + 0xFB, + 0x60, + 0xB1, + 0x86, + 0x3B, + 0x52, + 0xA1, + 0x6C, + 0xAA, + 0x55, + 0x29, + 0x9D, + 0x97, + 0xB2, + 0x87, + 0x90, + 0x61, + 0xBE, + 0xDC, + 0xFC, + 0xBC, + 0x95, + 0xCF, + 0xCD, + 0x37, + 0x3F, + 0x5B, + 0xD1, + 0x53, + 0x39, + 0x84, + 0x3C, + 0x41, + 0xA2, + 0x6D, + 0x47, + 0x14, + 0x2A, + 0x9E, + 0x5D, + 0x56, + 0xF2, + 0xD3, + 0xAB, + 0x44, + 0x11, + 0x92, + 0xD9, + 0x23, + 0x20, + 0x2E, + 0x89, + 0xB4, + 0x7C, + 0xB8, + 0x26, + 0x77, + 0x99, + 0xE3, + 0xA5, + 0x67, + 0x4A, + 0xED, + 0xDE, + 0xC5, + 0x31, + 0xFE, + 0x18, + 0x0D, + 0x63, + 0x8C, + 0x80, + 0xC0, + 0xF7, + 0x70, + 0x07, ) @@ -497,21 +1400,20 @@ def key_expansion(data): temp = data[-4:] temp = key_schedule_core(temp, rcon_iteration) rcon_iteration += 1 - data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes]) + data += xor(temp, data[-key_size_bytes : 4 - key_size_bytes]) for _ in range(3): temp = data[-4:] - data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes]) + data += xor(temp, data[-key_size_bytes : 4 - key_size_bytes]) if key_size_bytes == 32: temp = data[-4:] temp = sub_bytes(temp) - data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes]) + data += xor(temp, data[-key_size_bytes : 4 - key_size_bytes]) - for _ in range(3 if key_size_bytes == 32 else - 2 if key_size_bytes == 24 else 0): + for _ in range(3 if key_size_bytes == 32 else 2 if key_size_bytes == 24 else 0): temp = data[-4:] - data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes]) + data += xor(temp, data[-key_size_bytes : 4 - key_size_bytes]) data = data[:expanded_key_size_bytes] return data @@ -552,30 +1454,21 @@ def iter_mix_columns(data, matrix): for row in matrix: mixed = 0 for j in range(4): - if data[i:i + 4][j] == 0 or row[j] == 0: + if data[i : i + 4][j] == 0 or row[j] == 0: mixed ^= 0 else: mixed ^= RIJNDAEL_EXP_TABLE[ - (RIJNDAEL_LOG_TABLE[data[i + j]] + - RIJNDAEL_LOG_TABLE[row[j]]) % 0xFF + (RIJNDAEL_LOG_TABLE[data[i + j]] + RIJNDAEL_LOG_TABLE[row[j]]) % 0xFF ] yield mixed def shift_rows(data): - return [ - data[((column + row) & 0b11) * 4 + row] - for column in range(4) - for row in range(4) - ] + return [data[((column + row) & 0b11) * 4 + row] for column in range(4) for row in range(4)] def shift_rows_inv(data): - return [ - data[((column - row) & 0b11) * 4 + row] - for column in range(4) - for row in range(4) - ] + return [data[((column - row) & 0b11) * 4 + row] for column in range(4) for row in range(4)] def shift_block(data): @@ -607,8 +1500,7 @@ def block_product(block_x, block_y): # NIST SP 800-38D, Algorithm 1 if len(block_x) != BLOCK_SIZE_BYTES or len(block_y) != BLOCK_SIZE_BYTES: - raise ValueError( - "Length of blocks need to be %d bytes" % BLOCK_SIZE_BYTES) + raise ValueError("Length of blocks need to be %d bytes" % BLOCK_SIZE_BYTES) block_r = [0xE1] + [0] * (BLOCK_SIZE_BYTES - 1) block_v = block_y[:] @@ -631,12 +1523,11 @@ def ghash(subkey, data): # NIST SP 800-38D, Algorithm 2 if len(data) % BLOCK_SIZE_BYTES: - raise ValueError( - "Length of data should be %d bytes" % BLOCK_SIZE_BYTES) + raise ValueError("Length of data should be %d bytes" % BLOCK_SIZE_BYTES) last_y = [0] * BLOCK_SIZE_BYTES for i in range(0, len(data), BLOCK_SIZE_BYTES): - block = data[i: i + BLOCK_SIZE_BYTES] + block = data[i : i + BLOCK_SIZE_BYTES] last_y = block_product(xor(last_y, block), subkey) return last_y diff --git a/gallery_dl/archive.py b/gallery_dl/archive.py index 5f05bbfd8..607f39e93 100644 --- a/gallery_dl/archive.py +++ b/gallery_dl/archive.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -10,13 +8,13 @@ import os import sqlite3 -from . import formatter +from contextlib import suppress +from . import formatter -class DownloadArchive(): - def __init__(self, path, format_string, pragma=None, - cache_key="_archive_key"): +class DownloadArchive: + def __init__(self, path, format_string, pragma=None, cache_key="_archive_key"): try: con = sqlite3.connect(path, timeout=60, check_same_thread=False) except sqlite3.OperationalError: @@ -35,24 +33,22 @@ def __init__(self, path, format_string, pragma=None, cursor.execute("PRAGMA " + stmt) try: - cursor.execute("CREATE TABLE IF NOT EXISTS archive " - "(entry TEXT PRIMARY KEY) WITHOUT ROWID") + cursor.execute( + "CREATE TABLE IF NOT EXISTS archive " "(entry TEXT PRIMARY KEY) WITHOUT ROWID" + ) except sqlite3.OperationalError: # fallback for missing WITHOUT ROWID support (#553) - cursor.execute("CREATE TABLE IF NOT EXISTS archive " - "(entry TEXT PRIMARY KEY)") + cursor.execute("CREATE TABLE IF NOT EXISTS archive " "(entry TEXT PRIMARY KEY)") def add(self, kwdict): """Add item described by 'kwdict' to archive""" key = kwdict.get(self._cache_key) or self.keygen(kwdict) - self.cursor.execute( - "INSERT OR IGNORE INTO archive (entry) VALUES (?)", (key,)) + self.cursor.execute("INSERT OR IGNORE INTO archive (entry) VALUES (?)", (key,)) def check(self, kwdict): """Return True if the item described by 'kwdict' exists in archive""" key = kwdict[self._cache_key] = self.keygen(kwdict) - self.cursor.execute( - "SELECT 1 FROM archive WHERE entry=? LIMIT 1", (key,)) + self.cursor.execute("SELECT 1 FROM archive WHERE entry=? LIMIT 1", (key,)) return self.cursor.fetchone() def finalize(self): @@ -60,23 +56,18 @@ def finalize(self): class DownloadArchiveMemory(DownloadArchive): - - def __init__(self, path, format_string, pragma=None, - cache_key="_archive_key"): + def __init__(self, path, format_string, pragma=None, cache_key="_archive_key"): DownloadArchive.__init__(self, path, format_string, pragma, cache_key) self.keys = set() def add(self, kwdict): - self.keys.add( - kwdict.get(self._cache_key) or - self.keygen(kwdict)) + self.keys.add(kwdict.get(self._cache_key) or self.keygen(kwdict)) def check(self, kwdict): key = kwdict[self._cache_key] = self.keygen(kwdict) if key in self.keys: return True - self.cursor.execute( - "SELECT 1 FROM archive WHERE entry=? LIMIT 1", (key,)) + self.cursor.execute("SELECT 1 FROM archive WHERE entry=? LIMIT 1", (key,)) return self.cursor.fetchone() def finalize(self): @@ -85,10 +76,8 @@ def finalize(self): cursor = self.cursor with self.connection: - try: + with suppress(sqlite3.OperationalError): cursor.execute("BEGIN") - except sqlite3.OperationalError: - pass stmt = "INSERT OR IGNORE INTO archive (entry) VALUES (?)" if len(self.keys) < 100: diff --git a/gallery_dl/cache.py b/gallery_dl/cache.py index 923ed32c9..f45997d4e 100644 --- a/gallery_dl/cache.py +++ b/gallery_dl/cache.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2016-2021 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,16 +6,20 @@ """Decorators to keep function results in an in-memory and database cache""" -import sqlite3 +import functools +import os import pickle +import sqlite3 import time -import os -import functools -from . import config, util +from contextlib import suppress + +from . import config +from . import util -class CacheDecorator(): +class CacheDecorator: """Simplified in-memory cache""" + def __init__(self, func, keyarg): self.func = func self.cache = {} @@ -38,14 +40,13 @@ def update(self, key, value): self.cache[key] = value def invalidate(self, key=""): - try: + with suppress(KeyError): del self.cache[key] - except KeyError: - pass class MemoryCacheDecorator(CacheDecorator): """In-memory cache""" + def __init__(self, func, keyarg, maxage): CacheDecorator.__init__(self, func, keyarg) self.maxage = maxage @@ -67,13 +68,14 @@ def update(self, key, value): self.cache[key] = value, int(time.time()) + self.maxage -class DatabaseCacheDecorator(): +class DatabaseCacheDecorator: """Database cache""" + db = None _init = True def __init__(self, func, keyarg, maxage): - self.key = "%s.%s" % (func.__module__, func.__name__) + self.key = f"{func.__module__}.{func.__name__}" self.func = func self.cache = {} self.keyarg = keyarg @@ -87,21 +89,19 @@ def __call__(self, *args, **kwargs): timestamp = int(time.time()) # in-memory cache lookup - try: + with suppress(KeyError): value, expires = self.cache[key] + if expires > timestamp: return value - except KeyError: - pass # database lookup - fullkey = "%s-%s" % (self.key, key) + fullkey = f"{self.key}-{key}" with self.database() as db: cursor = db.cursor() - try: + # Silently swallow exception - workaround for Python 3.6 + with suppress(sqlite3.OperationalError): cursor.execute("BEGIN EXCLUSIVE") - except sqlite3.OperationalError: - pass # Silently swallow exception - workaround for Python 3.6 cursor.execute( "SELECT value, expires FROM data WHERE key=? LIMIT 1", (fullkey,), @@ -128,18 +128,16 @@ def update(self, key, value): with self.database() as db: db.execute( "INSERT OR REPLACE INTO data VALUES (?,?,?)", - ("%s-%s" % (self.key, key), pickle.dumps(value), expires), + (f"{self.key}-{key}", pickle.dumps(value), expires), ) def invalidate(self, key): - try: + with suppress(KeyError): del self.cache[key] - except KeyError: - pass with self.database() as db: db.execute( "DELETE FROM data WHERE key=?", - ("%s-%s" % (self.key, key),), + (f"{self.key}-{key}",), ) def database(self): @@ -154,17 +152,21 @@ def database(self): def memcache(maxage=None, keyarg=None): if maxage: + def wrap(func): return MemoryCacheDecorator(func, keyarg, maxage) else: + def wrap(func): return CacheDecorator(func, keyarg) + return wrap def cache(maxage=3600, keyarg=None): def wrap(func): return DatabaseCacheDecorator(func, keyarg, maxage) + return wrap @@ -182,9 +184,8 @@ def clear(module): cursor.execute("DELETE FROM data") else: cursor.execute( - "DELETE FROM data " - "WHERE key LIKE 'gallery_dl.extractor.' || ? || '.%'", - (module.lower(),) + "DELETE FROM data " "WHERE key LIKE 'gallery_dl.extractor.' || ? || '.%'", + (module.lower(),), ) except sqlite3.OperationalError: pass # database not initialized, cannot be modified, etc. @@ -218,8 +219,7 @@ def _init(): # restrict access permissions for new db files os.close(os.open(dbfile, os.O_CREAT | os.O_RDONLY, 0o600)) - DatabaseCacheDecorator.db = sqlite3.connect( - dbfile, timeout=60, check_same_thread=False) + DatabaseCacheDecorator.db = sqlite3.connect(dbfile, timeout=60, check_same_thread=False) except (OSError, TypeError, sqlite3.OperationalError): global cache cache = memcache diff --git a/gallery_dl/config.py b/gallery_dl/config.py index 855fb4f7b..ff906a98d 100644 --- a/gallery_dl/config.py +++ b/gallery_dl/config.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,9 +6,10 @@ """Global configuration module""" -import sys -import os.path import logging +import os.path +import sys + from . import util log = logging.getLogger("config") @@ -32,18 +31,20 @@ _default_configs = [ "/etc/gallery-dl.conf", "${XDG_CONFIG_HOME}/gallery-dl/config.json" - if os.environ.get("XDG_CONFIG_HOME") else - "${HOME}/.config/gallery-dl/config.json", + if os.environ.get("XDG_CONFIG_HOME") + else "${HOME}/.config/gallery-dl/config.json", "${HOME}/.gallery-dl.conf", ] if util.EXECUTABLE: # look for config file in PyInstaller executable directory (#682) - _default_configs.append(os.path.join( - os.path.dirname(sys.executable), - "gallery-dl.conf", - )) + _default_configs.append( + os.path.join( + os.path.dirname(sys.executable), + "gallery-dl.conf", + ) + ) # -------------------------------------------------------------------- @@ -82,8 +83,7 @@ def initialize(): except OSError as exc: log.debug("%s: %s", exc.__class__.__name__, exc) else: - log.error("Unable to create a new configuration file " - "at any of the default paths") + log.error("Unable to create a new configuration file " "at any of the default paths") return 1 log.info("Created a basic configuration file at '%s'", path) @@ -108,6 +108,7 @@ def open_extern(): openers = (editor,) + openers import shutil + for opener in openers: opener = shutil.which(opener) if opener: @@ -124,8 +125,7 @@ def open_extern(): with open(path, encoding="utf-8") as fp: util.json_loads(fp.read()) except Exception as exc: - log.warning("%s when parsing '%s': %s", - exc.__class__.__name__, path, exc) + log.warning("%s when parsing '%s': %s", exc.__class__.__name__, path, exc) return 2 return retcode @@ -155,8 +155,7 @@ def status(): paths.append((path, status)) - fmt = "{{:<{}}} : {{}}\n".format( - max(len(p[0]) for p in paths)).format + fmt = f"{{:<{max(len(p[0]) for p in paths)}}} : {{}}\n".format for path, status in paths: stdout_write(fmt(path, status)) @@ -174,8 +173,7 @@ def load(files=None, strict=False, loads=util.json_loads): log.error(exc) raise SystemExit(1) except Exception as exc: - log.error("%s when loading '%s': %s", - exc.__class__.__name__, path, exc) + log.error("%s when loading '%s': %s", exc.__class__.__name__, path, exc) if strict: raise SystemExit(2) else: @@ -303,7 +301,7 @@ def unset(path, key, conf=_config): pass -class apply(): +class apply: """Context Manager: apply a collection of key-value pairs""" def __init__(self, kvlist): diff --git a/gallery_dl/cookies.py b/gallery_dl/cookies.py index 71b0b6b0a..f3cb2c908 100644 --- a/gallery_dl/cookies.py +++ b/gallery_dl/cookies.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -21,41 +19,41 @@ import tempfile from hashlib import pbkdf2_hmac from http.cookiejar import Cookie -from . import aes, text, util +from . import aes +from . import text +from . import util -SUPPORTED_BROWSERS_CHROMIUM = { - "brave", "chrome", "chromium", "edge", "opera", "thorium", "vivaldi"} +SUPPORTED_BROWSERS_CHROMIUM = {"brave", "chrome", "chromium", "edge", "opera", "thorium", "vivaldi"} SUPPORTED_BROWSERS = SUPPORTED_BROWSERS_CHROMIUM | {"firefox", "safari"} logger = logging.getLogger("cookies") def load_cookies(browser_specification): - browser_name, profile, keyring, container, domain = \ - _parse_browser_specification(*browser_specification) + browser_name, profile, keyring, container, domain = _parse_browser_specification( + *browser_specification + ) if browser_name == "firefox": return load_cookies_firefox(profile, container, domain) - elif browser_name == "safari": + if browser_name == "safari": return load_cookies_safari(profile, domain) - elif browser_name in SUPPORTED_BROWSERS_CHROMIUM: + if browser_name in SUPPORTED_BROWSERS_CHROMIUM: return load_cookies_chromium(browser_name, profile, keyring, domain) - else: - raise ValueError("unknown browser '{}'".format(browser_name)) + raise ValueError(f"unknown browser '{browser_name}'") def load_cookies_firefox(profile=None, container=None, domain=None): path, container_id = _firefox_cookies_database(profile, container) - sql = ("SELECT name, value, host, path, isSecure, expiry " - "FROM moz_cookies") + sql = "SELECT name, value, host, path, isSecure, expiry " "FROM moz_cookies" conditions = [] parameters = [] if container_id is False: conditions.append("NOT INSTR(originAttributes,'userContextId=')") elif container_id: - uid = "%userContextId={}".format(container_id) + uid = f"%userContextId={container_id}" conditions.append("originAttributes LIKE ? OR originAttributes LIKE ?") parameters += (uid, uid + "&%") @@ -73,14 +71,24 @@ def load_cookies_firefox(profile=None, container=None, domain=None): with DatabaseConnection(path) as db: cookies = [ Cookie( - 0, name, value, None, False, - domain, True if domain else False, + 0, + name, + value, + None, + False, + domain, + bool(domain), domain[0] == "." if domain else False, - path, True if path else False, secure, expires, - False, None, None, {}, + path, + bool(path), + secure, + expires, + False, + None, + None, + {}, ) - for name, value, domain, path, secure, expires in db.execute( - sql, parameters) + for name, value, domain, path, secure, expires in db.execute(sql, parameters) ] _log_info("Extracted %s cookies from Firefox", len(cookies)) @@ -107,8 +115,7 @@ def load_cookies_safari(profile=None, domain=None): return cookies -def load_cookies_chromium(browser_name, profile=None, - keyring=None, domain=None): +def load_cookies_chromium(browser_name, profile=None, keyring=None, domain=None): config = _chromium_browser_settings(browser_name) path = _chromium_cookies_database(profile, config) _log_debug("Extracting cookies from %s", path) @@ -129,30 +136,36 @@ def load_cookies_chromium(browser_name, profile=None, cursor = db.cursor() try: - meta_version = int(cursor.execute( - "SELECT value FROM meta WHERE key = 'version'").fetchone()[0]) + meta_version = int( + cursor.execute("SELECT value FROM meta WHERE key = 'version'").fetchone()[0] + ) except Exception as exc: - _log_warning("Failed to get cookie database meta version (%s: %s)", - exc.__class__.__name__, exc) + _log_warning( + "Failed to get cookie database meta version (%s: %s)", exc.__class__.__name__, exc + ) meta_version = 0 try: rows = cursor.execute( "SELECT host_key, name, value, encrypted_value, path, " - "expires_utc, is_secure FROM cookies" + condition, parameters) + "expires_utc, is_secure FROM cookies" + condition, + parameters, + ) except sqlite3.OperationalError: rows = cursor.execute( "SELECT host_key, name, value, encrypted_value, path, " - "expires_utc, secure FROM cookies" + condition, parameters) + "expires_utc, secure FROM cookies" + condition, + parameters, + ) failed_cookies = 0 unencrypted_cookies = 0 decryptor = _chromium_cookie_decryptor( - config["directory"], config["keyring"], keyring, meta_version) + config["directory"], config["keyring"], keyring, meta_version + ) cookies = [] for domain, name, value, enc_value, path, expires, secure in rows: - if not value and enc_value: # encrypted value = decryptor.decrypt(enc_value) if value is None: @@ -162,31 +175,39 @@ def load_cookies_chromium(browser_name, profile=None, value = value.decode() unencrypted_cookies += 1 - if expires: - # https://stackoverflow.com/a/43520042 - expires = int(expires) // 1000000 - 11644473600 - else: - expires = None + # https://stackoverflow.com/a/43520042 + expires = int(expires) // 1000000 - 11644473600 if expires else None domain = domain.decode() path = path.decode() name = name.decode() - cookies.append(Cookie( - 0, name, value, None, False, - domain, True if domain else False, - domain[0] == "." if domain else False, - path, True if path else False, secure, expires, - False, None, None, {}, - )) + cookies.append( + Cookie( + 0, + name, + value, + None, + False, + domain, + bool(domain), + domain[0] == "." if domain else False, + path, + bool(path), + secure, + expires, + False, + None, + None, + {}, + ) + ) - if failed_cookies > 0: - failed_message = " ({} could not be decrypted)".format(failed_cookies) - else: - failed_message = "" + failed_message = f" ({failed_cookies} could not be decrypted)" if failed_cookies > 0 else "" - _log_info("Extracted %s cookies from %s%s", - len(cookies), browser_name.capitalize(), failed_message) + _log_info( + "Extracted %s cookies from %s%s", len(cookies), browser_name.capitalize(), failed_message + ) counts = decryptor.cookie_counts counts["unencrypted"] = unencrypted_cookies _log_debug("version breakdown: %s", counts) @@ -196,6 +217,7 @@ def load_cookies_chromium(browser_name, profile=None, # -------------------------------------------------------------------- # firefox + def _firefox_cookies_database(profile=None, container=None): if not profile: search_root = _firefox_browser_directory() @@ -206,8 +228,7 @@ def _firefox_cookies_database(profile=None, container=None): path = _find_most_recently_used_file(search_root, "cookies.sqlite") if path is None: - raise FileNotFoundError("Unable to find Firefox cookies database in " - "{}".format(search_root)) + raise FileNotFoundError("Unable to find Firefox cookies database in " f"{search_root}") _log_debug("Extracting cookies from %s", path) if not container or container == "none": @@ -218,54 +239,51 @@ def _firefox_cookies_database(profile=None, container=None): container_id = None else: - containers_path = os.path.join( - os.path.dirname(path), "containers.json") + containers_path = os.path.join(os.path.dirname(path), "containers.json") try: with open(containers_path) as fp: identities = util.json_loads(fp.read())["identities"] except OSError: - _log_error("Unable to read Firefox container database at '%s'", - containers_path) + _log_error("Unable to read Firefox container database at '%s'", containers_path) raise except KeyError: identities = () for context in identities: if container == context.get("name") or container == text.extr( - context.get("l10nID", ""), "userContext", ".label"): + context.get("l10nID", ""), "userContext", ".label" + ): container_id = context["userContextId"] break else: - raise ValueError("Unable to find Firefox container '{}'".format( - container)) - _log_debug("Only loading cookies from container '%s' (ID %s)", - container, container_id) + raise ValueError(f"Unable to find Firefox container '{container}'") + _log_debug("Only loading cookies from container '%s' (ID %s)", container, container_id) return path, container_id def _firefox_browser_directory(): if sys.platform in ("win32", "cygwin"): - return os.path.expandvars( - r"%APPDATA%\Mozilla\Firefox\Profiles") + return os.path.expandvars(r"%APPDATA%\Mozilla\Firefox\Profiles") if sys.platform == "darwin": - return os.path.expanduser( - "~/Library/Application Support/Firefox/Profiles") + return os.path.expanduser("~/Library/Application Support/Firefox/Profiles") return os.path.expanduser("~/.mozilla/firefox") # -------------------------------------------------------------------- # safari + def _safari_cookies_database(): try: path = os.path.expanduser("~/Library/Cookies/Cookies.binarycookies") return open(path, "rb") except FileNotFoundError: _log_debug("Trying secondary cookie location") - path = os.path.expanduser("~/Library/Containers/com.apple.Safari/Data" - "/Library/Cookies/Cookies.binarycookies") + path = os.path.expanduser( + "~/Library/Containers/com.apple.Safari/Data" "/Library/Cookies/Cookies.binarycookies" + ) return open(path, "rb") @@ -273,8 +291,7 @@ def _safari_parse_cookies_header(data): p = DataParser(data) p.expect_bytes(b"cook", "database signature") number_of_pages = p.read_uint(big_endian=True) - page_sizes = [p.read_uint(big_endian=True) - for _ in range(number_of_pages)] + page_sizes = [p.read_uint(big_endian=True) for _ in range(number_of_pages)] return page_sizes, p.cursor @@ -291,8 +308,7 @@ def _safari_parse_cookies_page(data, cookies, domain=None): for i, record_offset in enumerate(record_offsets): p.skip_to(record_offset, "space between records") - record_length = _safari_parse_cookies_record( - data[record_offset:], cookies, domain) + record_length = _safari_parse_cookies_record(data[record_offset:], cookies, domain) p.read_bytes(record_length) p.skip_to_end("space in between pages") @@ -302,7 +318,7 @@ def _safari_parse_cookies_record(data, cookies, host=None): record_size = p.read_uint() p.skip(4, "unknown record field 1") flags = p.read_uint() - is_secure = True if (flags & 0x0001) else False + is_secure = bool(flags & 0x0001) p.skip(4, "unknown record field 2") domain_offset = p.read_uint() name_offset = p.read_uint() @@ -310,7 +326,7 @@ def _safari_parse_cookies_record(data, cookies, host=None): value_offset = p.read_uint() p.skip(8, "unknown record field 3") expiration_date = _mac_absolute_time_to_posix(p.read_double()) - _creation_date = _mac_absolute_time_to_posix(p.read_double()) # noqa: F841 + _creation_date = _mac_absolute_time_to_posix(p.read_double()) try: p.skip_to(domain_offset) @@ -320,9 +336,8 @@ def _safari_parse_cookies_record(data, cookies, host=None): if host[0] == ".": if host[1:] != domain and not domain.endswith(host): return record_size - else: - if host != domain and ("." + host) != domain: - return record_size + elif host != domain and ("." + host) != domain: + return record_size p.skip_to(name_offset) name = p.read_cstring() @@ -338,13 +353,26 @@ def _safari_parse_cookies_record(data, cookies, host=None): p.skip_to(record_size, "space at the end of the record") - cookies.append(Cookie( - 0, name, value, None, False, - domain, True if domain else False, - domain[0] == "." if domain else False, - path, True if path else False, is_secure, expiration_date, - False, None, None, {}, - )) + cookies.append( + Cookie( + 0, + name, + value, + None, + False, + domain, + bool(domain), + domain[0] == "." if domain else False, + path, + bool(path), + is_secure, + expiration_date, + False, + None, + None, + {}, + ) + ) return record_size @@ -352,13 +380,13 @@ def _safari_parse_cookies_record(data, cookies, host=None): # -------------------------------------------------------------------- # chromium + def _chromium_cookies_database(profile, config): if profile is None: search_root = config["directory"] elif _is_path(profile): search_root = profile - config["directory"] = (os.path.dirname(profile) - if config["profiles"] else profile) + config["directory"] = os.path.dirname(profile) if config["profiles"] else profile elif config["profiles"]: search_root = os.path.join(config["directory"], profile) else: @@ -367,8 +395,9 @@ def _chromium_cookies_database(profile, config): path = _find_most_recently_used_file(search_root, "Cookies") if path is None: - raise FileNotFoundError("Unable to find {} cookies database in " - "'{}'".format(config["browser"], search_root)) + raise FileNotFoundError( + "Unable to find {} cookies database in " "'{}'".format(config["browser"], search_root) + ) return path @@ -381,76 +410,69 @@ def _chromium_browser_settings(browser_name): appdata_local = os.path.expandvars("%LOCALAPPDATA%") appdata_roaming = os.path.expandvars("%APPDATA%") browser_dir = { - "brave" : join(appdata_local, - R"BraveSoftware\Brave-Browser\User Data"), - "chrome" : join(appdata_local, R"Google\Chrome\User Data"), + "brave": join(appdata_local, R"BraveSoftware\Brave-Browser\User Data"), + "chrome": join(appdata_local, R"Google\Chrome\User Data"), "chromium": join(appdata_local, R"Chromium\User Data"), - "edge" : join(appdata_local, R"Microsoft\Edge\User Data"), - "opera" : join(appdata_roaming, R"Opera Software\Opera Stable"), - "thorium" : join(appdata_local, R"Thorium\User Data"), - "vivaldi" : join(appdata_local, R"Vivaldi\User Data"), + "edge": join(appdata_local, R"Microsoft\Edge\User Data"), + "opera": join(appdata_roaming, R"Opera Software\Opera Stable"), + "thorium": join(appdata_local, R"Thorium\User Data"), + "vivaldi": join(appdata_local, R"Vivaldi\User Data"), }[browser_name] elif sys.platform == "darwin": appdata = os.path.expanduser("~/Library/Application Support") browser_dir = { - "brave" : join(appdata, "BraveSoftware/Brave-Browser"), - "chrome" : join(appdata, "Google/Chrome"), + "brave": join(appdata, "BraveSoftware/Brave-Browser"), + "chrome": join(appdata, "Google/Chrome"), "chromium": join(appdata, "Chromium"), - "edge" : join(appdata, "Microsoft Edge"), - "opera" : join(appdata, "com.operasoftware.Opera"), - "thorium" : join(appdata, "Thorium"), - "vivaldi" : join(appdata, "Vivaldi"), + "edge": join(appdata, "Microsoft Edge"), + "opera": join(appdata, "com.operasoftware.Opera"), + "thorium": join(appdata, "Thorium"), + "vivaldi": join(appdata, "Vivaldi"), }[browser_name] else: - config = (os.environ.get("XDG_CONFIG_HOME") or - os.path.expanduser("~/.config")) + config = os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config") browser_dir = { - "brave" : join(config, "BraveSoftware/Brave-Browser"), - "chrome" : join(config, "google-chrome"), + "brave": join(config, "BraveSoftware/Brave-Browser"), + "chrome": join(config, "google-chrome"), "chromium": join(config, "chromium"), - "edge" : join(config, "microsoft-edge"), - "opera" : join(config, "opera"), - "thorium" : join(config, "Thorium"), - "vivaldi" : join(config, "vivaldi"), + "edge": join(config, "microsoft-edge"), + "opera": join(config, "opera"), + "thorium": join(config, "Thorium"), + "vivaldi": join(config, "vivaldi"), }[browser_name] # Linux keyring names can be determined by snooping on dbus # while opening the browser in KDE: # dbus-monitor "interface="org.kde.KWallet"" "type=method_return" keyring_name = { - "brave" : "Brave", - "chrome" : "Chrome", + "brave": "Brave", + "chrome": "Chrome", "chromium": "Chromium", - "edge" : "Microsoft Edge" if sys.platform == "darwin" else - "Chromium", - "opera" : "Opera" if sys.platform == "darwin" else "Chromium", - "thorium" : "Thorium", - "vivaldi" : "Vivaldi" if sys.platform == "darwin" else "Chrome", + "edge": "Microsoft Edge" if sys.platform == "darwin" else "Chromium", + "opera": "Opera" if sys.platform == "darwin" else "Chromium", + "thorium": "Thorium", + "vivaldi": "Vivaldi" if sys.platform == "darwin" else "Chrome", }[browser_name] browsers_without_profiles = {"opera"} return { - "browser" : browser_name, + "browser": browser_name, "directory": browser_dir, - "keyring" : keyring_name, - "profiles" : browser_name not in browsers_without_profiles + "keyring": keyring_name, + "profiles": browser_name not in browsers_without_profiles, } -def _chromium_cookie_decryptor( - browser_root, browser_keyring_name, keyring=None, meta_version=0): +def _chromium_cookie_decryptor(browser_root, browser_keyring_name, keyring=None, meta_version=0): if sys.platform in ("win32", "cygwin"): - return WindowsChromiumCookieDecryptor( - browser_root, meta_version) + return WindowsChromiumCookieDecryptor(browser_root, meta_version) elif sys.platform == "darwin": - return MacChromiumCookieDecryptor( - browser_keyring_name, meta_version) + return MacChromiumCookieDecryptor(browser_keyring_name, meta_version) else: - return LinuxChromiumCookieDecryptor( - browser_keyring_name, keyring, meta_version) + return LinuxChromiumCookieDecryptor(browser_keyring_name, keyring, meta_version) class ChromiumCookieDecryptor: @@ -498,15 +520,14 @@ def __init__(self, browser_keyring_name, keyring=None, meta_version=0): self._v10_key = self.derive_key(b"peanuts") self._v11_key = None if password is None else self.derive_key(password) self._cookie_counts = {"v10": 0, "v11": 0, "other": 0} - self._offset = (32 if meta_version >= 24 else 0) + self._offset = 32 if meta_version >= 24 else 0 @staticmethod def derive_key(password): # values from # https://chromium.googlesource.com/chromium/src/+/refs/heads # /main/components/os_crypt/os_crypt_linux.cc - return pbkdf2_sha1(password, salt=b"saltysalt", - iterations=1, key_length=16) + return pbkdf2_sha1(password, salt=b"saltysalt", iterations=1, key_length=16) @property def cookie_counts(self): @@ -520,16 +541,15 @@ def decrypt(self, encrypted_value): self._cookie_counts["v10"] += 1 value = _decrypt_aes_cbc(ciphertext, self._v10_key, self._offset) - elif version == b"v11": + if version == b"v11": self._cookie_counts["v11"] += 1 if self._v11_key is None: _log_warning("Unable to decrypt v11 cookies: no key found") return None value = _decrypt_aes_cbc(ciphertext, self._v11_key, self._offset) - else: - self._cookie_counts["other"] += 1 - return None + self._cookie_counts["other"] += 1 + return None if value is None: value = _decrypt_aes_cbc(ciphertext, self._empty_key, self._offset) @@ -543,15 +563,14 @@ def __init__(self, browser_keyring_name, meta_version=0): password = _get_mac_keyring_password(browser_keyring_name) self._v10_key = None if password is None else self.derive_key(password) self._cookie_counts = {"v10": 0, "other": 0} - self._offset = (32 if meta_version >= 24 else 0) + self._offset = 32 if meta_version >= 24 else 0 @staticmethod def derive_key(password): # values from # https://chromium.googlesource.com/chromium/src/+/refs/heads # /main/components/os_crypt/os_crypt_mac.mm - return pbkdf2_sha1(password, salt=b"saltysalt", - iterations=1003, key_length=16) + return pbkdf2_sha1(password, salt=b"saltysalt", iterations=1003, key_length=16) @property def cookie_counts(self): @@ -568,20 +587,19 @@ def decrypt(self, encrypted_value): return None return _decrypt_aes_cbc(ciphertext, self._v10_key, self._offset) - else: - self._cookie_counts["other"] += 1 - # other prefixes are considered "old data", - # which were stored as plaintext - # https://chromium.googlesource.com/chromium/src/+/refs/heads - # /main/components/os_crypt/os_crypt_mac.mm - return encrypted_value + self._cookie_counts["other"] += 1 + # other prefixes are considered "old data", + # which were stored as plaintext + # https://chromium.googlesource.com/chromium/src/+/refs/heads + # /main/components/os_crypt/os_crypt_mac.mm + return encrypted_value class WindowsChromiumCookieDecryptor(ChromiumCookieDecryptor): def __init__(self, browser_root, meta_version=0): self._v10_key = _get_windows_v10_key(browser_root) self._cookie_counts = {"v10": 0, "other": 0} - self._offset = (32 if meta_version >= 24 else 0) + self._offset = 32 if meta_version >= 24 else 0 @property def cookie_counts(self): @@ -607,25 +625,24 @@ def decrypt(self, encrypted_value): raw_ciphertext = ciphertext nonce = raw_ciphertext[:nonce_length] - ciphertext = raw_ciphertext[ - nonce_length:-authentication_tag_length] + ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length] authentication_tag = raw_ciphertext[-authentication_tag_length:] return _decrypt_aes_gcm( - ciphertext, self._v10_key, nonce, authentication_tag, - self._offset) + ciphertext, self._v10_key, nonce, authentication_tag, self._offset + ) - else: - self._cookie_counts["other"] += 1 - # any other prefix means the data is DPAPI encrypted - # https://chromium.googlesource.com/chromium/src/+/refs/heads - # /main/components/os_crypt/os_crypt_win.cc - return _decrypt_windows_dpapi(encrypted_value).decode() + self._cookie_counts["other"] += 1 + # any other prefix means the data is DPAPI encrypted + # https://chromium.googlesource.com/chromium/src/+/refs/heads + # /main/components/os_crypt/os_crypt_win.cc + return _decrypt_windows_dpapi(encrypted_value).decode() # -------------------------------------------------------------------- # keyring + def _choose_linux_keyring(): """ https://chromium.googlesource.com/chromium/src/+/refs/heads @@ -642,7 +659,7 @@ def _choose_linux_keyring(): def _get_kwallet_network_wallet(): - """ The name of the wallet used to store network passwords. + """The name of the wallet used to store network passwords. https://chromium.googlesource.com/chromium/src/+/refs/heads /main/components/os_crypt/kwallet_dbus.cc @@ -654,22 +671,22 @@ def _get_kwallet_network_wallet(): default_wallet = "kdewallet" try: proc, stdout = Popen_communicate( - "dbus-send", "--session", "--print-reply=literal", + "dbus-send", + "--session", + "--print-reply=literal", "--dest=org.kde.kwalletd5", "/modules/kwalletd5", - "org.kde.KWallet.networkWallet" + "org.kde.KWallet.networkWallet", ) if proc.returncode != 0: _log_warning("Failed to read NetworkWallet") return default_wallet - else: - network_wallet = stdout.decode().strip() - _log_debug("NetworkWallet = '%s'", network_wallet) - return network_wallet + network_wallet = stdout.decode().strip() + _log_debug("NetworkWallet = '%s'", network_wallet) + return network_wallet except Exception as exc: - _log_warning("Error while obtaining NetworkWallet (%s: %s)", - exc.__class__.__name__, exc) + _log_warning("Error while obtaining NetworkWallet (%s: %s)", exc.__class__.__name__, exc) return default_wallet @@ -680,7 +697,8 @@ def _get_kwallet_password(browser_keyring_name): _log_error( "kwallet-query command not found. KWallet and kwallet-query " "must be installed to read from KWallet. kwallet-query should be " - "included in the kwallet package for your distribution") + "included in the kwallet package for your distribution" + ) return b"" network_wallet = _get_kwallet_network_wallet() @@ -688,20 +706,23 @@ def _get_kwallet_password(browser_keyring_name): try: proc, stdout = Popen_communicate( "kwallet-query", - "--read-password", browser_keyring_name + " Safe Storage", - "--folder", browser_keyring_name + " Keys", + "--read-password", + browser_keyring_name + " Safe Storage", + "--folder", + browser_keyring_name + " Keys", network_wallet, ) if proc.returncode != 0: - _log_error("kwallet-query failed with return code {}. " - "Please consult the kwallet-query man page " - "for details".format(proc.returncode)) + _log_error( + f"kwallet-query failed with return code {proc.returncode}. " + "Please consult the kwallet-query man page " + "for details" + ) return b"" if stdout.lower().startswith(b"failed to read"): - _log_debug("Failed to read password from kwallet. " - "Using empty string instead") + _log_debug("Failed to read password from kwallet. " "Using empty string instead") # This sometimes occurs in KDE because chrome does not check # hasEntry and instead just tries to read the value (which # kwallet returns "") whereas kwallet-query checks hasEntry. @@ -711,13 +732,11 @@ def _get_kwallet_password(browser_keyring_name): # This may be a bug, as the intended behaviour is to generate a # random password and store it, but that doesn't matter here. return b"" - else: - if stdout[-1:] == b"\n": - stdout = stdout[:-1] - return stdout + if stdout[-1:] == b"\n": + stdout = stdout[:-1] + return stdout except Exception as exc: - _log_warning("Error when running kwallet-query (%s: %s)", - exc.__class__.__name__, exc) + _log_warning("Error when running kwallet-query (%s: %s)", exc.__class__.__name__, exc) return b"" @@ -740,9 +759,8 @@ def _get_gnome_keyring_password(browser_keyring_name): for item in col.get_all_items(): if item.get_label() == label: return item.get_secret() - else: - _log_error("Failed to read from GNOME keyring") - return b"" + _log_error("Failed to read from GNOME keyring") + return b"" finally: con.close() @@ -761,9 +779,9 @@ def _get_linux_keyring_password(browser_keyring_name, keyring): if keyring == KEYRING_KWALLET: return _get_kwallet_password(browser_keyring_name) - elif keyring == KEYRING_GNOMEKEYRING: + if keyring == KEYRING_GNOMEKEYRING: return _get_gnome_keyring_password(browser_keyring_name) - elif keyring == KEYRING_BASICTEXT: + if keyring == KEYRING_BASICTEXT: # when basic text is chosen, all cookies are stored as v10 # so no keyring password is required return None @@ -771,22 +789,23 @@ def _get_linux_keyring_password(browser_keyring_name, keyring): def _get_mac_keyring_password(browser_keyring_name): - _log_debug("Using find-generic-password to obtain " - "password from OSX keychain") + _log_debug("Using find-generic-password to obtain " "password from OSX keychain") try: proc, stdout = Popen_communicate( - "security", "find-generic-password", + "security", + "find-generic-password", "-w", # write password to stdout - "-a", browser_keyring_name, # match "account" - "-s", browser_keyring_name + " Safe Storage", # match "service" + "-a", + browser_keyring_name, # match "account" + "-s", + browser_keyring_name + " Safe Storage", # match "service" ) if stdout[-1:] == b"\n": stdout = stdout[:-1] return stdout except Exception as exc: - _log_warning("Error when using find-generic-password (%s: %s)", - exc.__class__.__name__, exc) + _log_warning("Error when using find-generic-password (%s: %s)", exc.__class__.__name__, exc) return None @@ -808,12 +827,13 @@ def _get_windows_v10_key(browser_root): if not encrypted_key.startswith(prefix): _log_error("Invalid Local State key") return None - return _decrypt_windows_dpapi(encrypted_key[len(prefix):]) + return _decrypt_windows_dpapi(encrypted_key[len(prefix) :]) # -------------------------------------------------------------------- # utility + class ParserError(Exception): pass @@ -825,19 +845,18 @@ def __init__(self, data): def read_bytes(self, num_bytes): if num_bytes < 0: - raise ParserError("invalid read of {} bytes".format(num_bytes)) + raise ParserError(f"invalid read of {num_bytes} bytes") end = self.cursor + num_bytes if end > len(self._data): raise ParserError("reached end of input") - data = self._data[self.cursor:end] + data = self._data[self.cursor : end] self.cursor = end return data def expect_bytes(self, expected_value, message): value = self.read_bytes(len(expected_value)) if value != expected_value: - raise ParserError("unexpected value: {} != {} ({})".format( - value, expected_value, message)) + raise ParserError(f"unexpected value: {value} != {expected_value} ({message})") def read_uint(self, big_endian=False): data_format = ">I" if big_endian else " 0: - _log_debug("Skipping {} bytes ({}): {!r}".format( - num_bytes, description, self.read_bytes(num_bytes))) + _log_debug( + f"Skipping {num_bytes} bytes ({description}): {self.read_bytes(num_bytes)!r}" + ) elif num_bytes < 0: - raise ParserError("Invalid skip of {} bytes".format(num_bytes)) + raise ParserError(f"Invalid skip of {num_bytes} bytes") def skip_to(self, offset, description="unknown"): self.skip(offset - self.cursor, description) @@ -870,8 +889,7 @@ def skip_to_end(self, description="unknown"): self.skip_to(len(self._data), description) -class DatabaseConnection(): - +class DatabaseConnection: def __init__(self, path): self.path = path self.database = None @@ -884,20 +902,23 @@ def __enter__(self): if util.WINDOWS: path = "/" + os.path.abspath(path) - uri = "file:{}?mode=ro&immutable=1".format(path) + uri = f"file:{path}?mode=ro&immutable=1" self.database = sqlite3.connect( - uri, uri=True, isolation_level=None, check_same_thread=False) + uri, uri=True, isolation_level=None, check_same_thread=False + ) return self.database except Exception as exc: - _log_debug("Falling back to temporary database copy (%s: %s)", - exc.__class__.__name__, exc) + _log_debug( + "Falling back to temporary database copy (%s: %s)", exc.__class__.__name__, exc + ) try: self.directory = tempfile.TemporaryDirectory(prefix="gallery-dl-") path_copy = os.path.join(self.directory.name, "copy.sqlite") shutil.copyfile(self.path, path_copy) self.database = sqlite3.connect( - path_copy, isolation_level=None, check_same_thread=False) + path_copy, isolation_level=None, check_same_thread=False + ) return self.database except BaseException: if self.directory: @@ -911,8 +932,7 @@ def __exit__(self, exc_type, exc_value, traceback): def Popen_communicate(*args): - proc = util.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + proc = util.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) try: stdout, stderr = proc.communicate() except BaseException: # Including KeyboardInterrupt @@ -954,23 +974,21 @@ def _get_linux_desktop_environment(env): desktop_session = env.get("DESKTOP_SESSION") if xdg_current_desktop: - xdg_current_desktop = (xdg_current_desktop.partition(":")[0] - .strip().lower()) + xdg_current_desktop = xdg_current_desktop.partition(":")[0].strip().lower() if xdg_current_desktop == "unity": if desktop_session and "gnome-fallback" in desktop_session: return DE_GNOME - else: - return DE_UNITY - elif xdg_current_desktop == "gnome": + return DE_UNITY + if xdg_current_desktop == "gnome": return DE_GNOME - elif xdg_current_desktop == "x-cinnamon": + if xdg_current_desktop == "x-cinnamon": return DE_CINNAMON - elif xdg_current_desktop == "kde": + if xdg_current_desktop == "kde": return DE_KDE - elif xdg_current_desktop == "pantheon": + if xdg_current_desktop == "pantheon": return DE_PANTHEON - elif xdg_current_desktop == "xfce": + if xdg_current_desktop == "xfce": return DE_XFCE if desktop_session: @@ -997,10 +1015,8 @@ def pbkdf2_sha1(password, salt, iterations, key_length): return pbkdf2_hmac("sha1", password, salt, iterations, key_length) -def _decrypt_aes_cbc(ciphertext, key, offset=0, - initialization_vector=b" " * 16): - plaintext = aes.unpad_pkcs7(aes.aes_cbc_decrypt_bytes( - ciphertext, key, initialization_vector)) +def _decrypt_aes_cbc(ciphertext, key, offset=0, initialization_vector=b" " * 16): + plaintext = aes.unpad_pkcs7(aes.aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector)) if offset: plaintext = plaintext[offset:] try: @@ -1011,8 +1027,7 @@ def _decrypt_aes_cbc(ciphertext, key, offset=0, def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, offset=0): try: - plaintext = aes.aes_gcm_decrypt_and_verify_bytes( - ciphertext, key, authentication_tag, nonce) + plaintext = aes.aes_gcm_decrypt_and_verify_bytes(ciphertext, key, authentication_tag, nonce) if offset: plaintext = plaintext[offset:] return plaintext.decode() @@ -1032,8 +1047,7 @@ def _decrypt_windows_dpapi(ciphertext): from ctypes.wintypes import DWORD class DATA_BLOB(ctypes.Structure): - _fields_ = [("cbData", DWORD), - ("pbData", ctypes.POINTER(ctypes.c_char))] + _fields_ = [("cbData", DWORD), ("pbData", ctypes.POINTER(ctypes.c_char))] buffer = ctypes.create_string_buffer(ciphertext) blob_in = DATA_BLOB(ctypes.sizeof(buffer), buffer) @@ -1045,7 +1059,7 @@ class DATA_BLOB(ctypes.Structure): None, # pvReserved: must be NULL None, # pPromptStruct: information about prompts to display 0, # dwFlags - ctypes.byref(blob_out) # pDataOut + ctypes.byref(blob_out), # pDataOut ) if not ret: _log_warning("Failed to decrypt cookie (DPAPI)") @@ -1078,13 +1092,12 @@ def _is_path(value): return os.path.sep in value -def _parse_browser_specification( - browser, profile=None, keyring=None, container=None, domain=None): +def _parse_browser_specification(browser, profile=None, keyring=None, container=None, domain=None): browser = browser.lower() if browser not in SUPPORTED_BROWSERS: - raise ValueError("Unsupported browser '{}'".format(browser)) + raise ValueError(f"Unsupported browser '{browser}'") if keyring and keyring not in SUPPORTED_KEYRINGS: - raise ValueError("Unsupported keyring '{}'".format(keyring)) + raise ValueError(f"Unsupported keyring '{keyring}'") if profile and _is_path(profile): profile = os.path.expanduser(profile) return browser, profile, keyring, container, domain diff --git a/gallery_dl/downloader/__init__.py b/gallery_dl/downloader/__init__.py index e1b936e46..4f0b42485 100644 --- a/gallery_dl/downloader/__init__.py +++ b/gallery_dl/downloader/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2021 Mike Fährmann # # This program is free software; you can redistribute it and/or modify diff --git a/gallery_dl/downloader/common.py b/gallery_dl/downloader/common.py index 1168d83df..46c8977f5 100644 --- a/gallery_dl/downloader/common.py +++ b/gallery_dl/downloader/common.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -9,11 +7,14 @@ """Common classes and constants used by downloader modules.""" import os -from .. import config, util + +from .. import config +from .. import util -class DownloaderBase(): +class DownloaderBase: """Base class for downloaders""" + scheme = "" def __init__(self, job): diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index 54750ac73..53af50d75 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,18 @@ """Downloader module for http:// and https:// URLs""" -import time import mimetypes -from requests.exceptions import RequestException, ConnectionError, Timeout -from .common import DownloaderBase -from .. import text, util +import time from ssl import SSLError +from requests.exceptions import ConnectionError +from requests.exceptions import RequestException +from requests.exceptions import Timeout + +from .. import text +from .. import util +from .common import DownloaderBase + class HttpDownloader(DownloaderBase): scheme = "http" @@ -50,35 +53,30 @@ def __init__(self, job): if self.minsize: minsize = text.parse_bytes(self.minsize) if not minsize: - self.log.warning( - "Invalid minimum file size (%r)", self.minsize) + self.log.warning("Invalid minimum file size (%r)", self.minsize) self.minsize = minsize if self.maxsize: maxsize = text.parse_bytes(self.maxsize) if not maxsize: - self.log.warning( - "Invalid maximum file size (%r)", self.maxsize) + self.log.warning("Invalid maximum file size (%r)", self.maxsize) self.maxsize = maxsize if isinstance(self.chunk_size, str): chunk_size = text.parse_bytes(self.chunk_size) if not chunk_size: - self.log.warning( - "Invalid chunk size (%r)", self.chunk_size) + self.log.warning("Invalid chunk size (%r)", self.chunk_size) chunk_size = 32768 self.chunk_size = chunk_size if self.rate: rate = text.parse_bytes(self.rate) if rate: - if rate < self.chunk_size: - self.chunk_size = rate + self.chunk_size = min(rate, self.chunk_size) self.rate = rate self.receive = self._receive_rate else: self.log.warning("Invalid rate limit (%r)", self.rate) if self.progress is not None: self.receive = self._receive_rate - if self.progress < 0.0: - self.progress = 0.0 + self.progress = max(self.progress, 0.0) def download(self, url, pathfmt): try: @@ -98,10 +96,8 @@ def _download_impl(self, url, pathfmt): metadata = self.metadata kwdict = pathfmt.kwdict - expected_status = kwdict.get( - "_http_expected_status", ()) - adjust_extension = kwdict.get( - "_http_adjust_extension", self.adjust_extension) + expected_status = kwdict.get("_http_expected_status", ()) + adjust_extension = kwdict.get("_http_adjust_extension", self.adjust_extension) if self.part and not metadata: pathfmt.part_enable(self.partdir) @@ -111,7 +107,7 @@ def _download_impl(self, url, pathfmt): if response: self.release_conn(response) response = None - self.log.warning("%s (%s/%s)", msg, tries, self.retries+1) + self.log.warning("%s (%s/%s)", msg, tries, self.retries + 1) if tries > self.retries: return False time.sleep(tries) @@ -131,12 +127,13 @@ def _download_impl(self, url, pathfmt): # partial content file_size = pathfmt.part_size() if file_size: - headers["Range"] = "bytes={}-".format(file_size) + headers["Range"] = f"bytes={file_size}-" # connect to (remote) source try: response = self.session.request( - kwdict.get("_http_method", "GET"), url, + kwdict.get("_http_method", "GET"), + url, stream=True, headers=headers, data=kwdict.get("_http_data"), @@ -162,7 +159,7 @@ def _download_impl(self, url, pathfmt): elif code == 416 and file_size: # Requested Range Not Satisfiable break else: - msg = "'{} {}' for '{}'".format(code, response.reason, url) + msg = f"'{code} {response.reason}' for '{url}'" if code in self.retry_codes or 500 <= code < 600: continue retry = kwdict.get("_http_retry") @@ -195,15 +192,15 @@ def _download_impl(self, url, pathfmt): if self.minsize and size < self.minsize: self.release_conn(response) self.log.warning( - "File size smaller than allowed minimum (%s < %s)", - size, self.minsize) + "File size smaller than allowed minimum (%s < %s)", size, self.minsize + ) pathfmt.temppath = "" return True if self.maxsize and size > self.maxsize: self.release_conn(response) self.log.warning( - "File size larger than allowed maximum (%s > %s)", - size, self.maxsize) + "File size larger than allowed maximum (%s > %s)", size, self.maxsize + ) pathfmt.temppath = "" return True @@ -240,18 +237,16 @@ def _download_impl(self, url, pathfmt): content = response.iter_content(self.chunk_size) # check filename extension against file header - if adjust_extension and not offset and \ - pathfmt.extension in SIGNATURE_CHECKS: + if adjust_extension and not offset and pathfmt.extension in SIGNATURE_CHECKS: try: file_header = next( - content if response.raw.chunked - else response.iter_content(16), b"") + content if response.raw.chunked else response.iter_content(16), b"" + ) except (RequestException, SSLError) as exc: msg = str(exc) print() continue - if self._adjust_extension(pathfmt, file_header) and \ - pathfmt.exists(): + if self._adjust_extension(pathfmt, file_header) and pathfmt.exists(): pathfmt.temppath = "" response.close() return True @@ -272,8 +267,7 @@ def _download_impl(self, url, pathfmt): fp.write(file_header) offset += len(file_header) elif offset: - if adjust_extension and \ - pathfmt.extension in SIGNATURE_CHECKS: + if adjust_extension and pathfmt.extension in SIGNATURE_CHECKS: self._adjust_extension(pathfmt, fp.read(16)) fp.seek(offset) @@ -287,8 +281,7 @@ def _download_impl(self, url, pathfmt): # check file size if size and fp.tell() < size: - msg = "file size mismatch ({} < {})".format( - fp.tell(), size) + msg = f"file size mismatch ({fp.tell()} < {size})" print() continue @@ -310,8 +303,10 @@ def release_conn(self, response): except (RequestException, SSLError) as exc: print() self.log.debug( - "Unable to consume response body (%s: %s); " - "closing the connection anyway", exc.__class__.__name__, exc) + "Unable to consume response body (%s: %s); " "closing the connection anyway", + exc.__class__.__name__, + exc, + ) response.close() @staticmethod @@ -334,13 +329,12 @@ def _receive_rate(self, fp, content, bytes_total, bytes_start): write(data) - if progress is not None: - if time_elapsed > progress: - self.out.progress( - bytes_total, - bytes_start + bytes_downloaded, - int(bytes_downloaded / time_elapsed), - ) + if progress is not None and time_elapsed > progress: + self.out.progress( + bytes_total, + bytes_start + bytes_downloaded, + int(bytes_downloaded / time_elapsed), + ) if rate: time_expected = bytes_downloaded / rate @@ -378,51 +372,46 @@ def _adjust_extension(pathfmt, file_header): MIME_TYPES = { - "image/jpeg" : "jpg", - "image/jpg" : "jpg", - "image/png" : "png", - "image/gif" : "gif", - "image/bmp" : "bmp", - "image/x-bmp" : "bmp", + "image/jpeg": "jpg", + "image/jpg": "jpg", + "image/png": "png", + "image/gif": "gif", + "image/bmp": "bmp", + "image/x-bmp": "bmp", "image/x-ms-bmp": "bmp", - "image/webp" : "webp", - "image/avif" : "avif", - "image/heic" : "heic", - "image/heif" : "heif", - "image/svg+xml" : "svg", - "image/ico" : "ico", - "image/icon" : "ico", - "image/x-icon" : "ico", - "image/vnd.microsoft.icon" : "ico", - "image/x-photoshop" : "psd", - "application/x-photoshop" : "psd", + "image/webp": "webp", + "image/avif": "avif", + "image/heic": "heic", + "image/heif": "heif", + "image/svg+xml": "svg", + "image/ico": "ico", + "image/icon": "ico", + "image/x-icon": "ico", + "image/vnd.microsoft.icon": "ico", + "image/x-photoshop": "psd", + "application/x-photoshop": "psd", "image/vnd.adobe.photoshop": "psd", - "video/webm": "webm", - "video/ogg" : "ogg", - "video/mp4" : "mp4", - "video/m4v" : "m4v", + "video/ogg": "ogg", + "video/mp4": "mp4", + "video/m4v": "m4v", "video/x-m4v": "m4v", "video/quicktime": "mov", - - "audio/wav" : "wav", + "audio/wav": "wav", "audio/x-wav": "wav", - "audio/webm" : "webm", - "audio/ogg" : "ogg", - "audio/mpeg" : "mp3", - - "application/zip" : "zip", + "audio/webm": "webm", + "audio/ogg": "ogg", + "audio/mpeg": "mp3", + "application/zip": "zip", "application/x-zip": "zip", "application/x-zip-compressed": "zip", - "application/rar" : "rar", + "application/rar": "rar", "application/x-rar": "rar", "application/x-rar-compressed": "rar", - "application/x-7z-compressed" : "7z", - - "application/pdf" : "pdf", + "application/x-7z-compressed": "7z", + "application/pdf": "pdf", "application/x-pdf": "pdf", "application/x-shockwave-flash": "swf", - "application/ogg": "ogg", # https://www.iana.org/assignments/media-types/model/obj "model/obj": "obj", @@ -431,43 +420,40 @@ def _adjust_extension(pathfmt, file_header): # https://en.wikipedia.org/wiki/List_of_file_signatures SIGNATURE_CHECKS = { - "jpg" : lambda s: s[0:3] == b"\xFF\xD8\xFF", - "png" : lambda s: s[0:8] == b"\x89PNG\r\n\x1A\n", - "gif" : lambda s: s[0:6] in (b"GIF87a", b"GIF89a"), - "bmp" : lambda s: s[0:2] == b"BM", - "webp": lambda s: (s[0:4] == b"RIFF" and - s[8:12] == b"WEBP"), + "jpg": lambda s: s[0:3] == b"\xff\xd8\xff", + "png": lambda s: s[0:8] == b"\x89PNG\r\n\x1a\n", + "gif": lambda s: s[0:6] in (b"GIF87a", b"GIF89a"), + "bmp": lambda s: s[0:2] == b"BM", + "webp": lambda s: (s[0:4] == b"RIFF" and s[8:12] == b"WEBP"), "avif": lambda s: s[4:11] == b"ftypavi" and s[11] in b"fs", - "heic": lambda s: (s[4:10] == b"ftyphe" and s[10:12] in ( - b"ic", b"im", b"is", b"ix", b"vc", b"vm", b"vs")), - "svg" : lambda s: s[0:5] == b"= 0 else float("inf"), + "retries": retries + 1 if retries >= 0 else float("inf"), "socket_timeout": self.config("timeout", extractor._timeout), "nocheckcertificate": not self.config("verify", extractor._verify), "proxy": self.proxies.get("http") if self.proxies else None, @@ -43,18 +43,17 @@ def download(self, url, pathfmt): try: module = ytdl.import_module(self.config("module")) except (ImportError, SyntaxError) as exc: - self.log.error("Cannot import module '%s'", - getattr(exc, "name", "")) + self.log.error("Cannot import module '%s'", getattr(exc, "name", "")) self.log.debug("", exc_info=exc) self.download = lambda u, p: False return False self.ytdl_instance = ytdl_instance = ytdl.construct_YoutubeDL( - module, self, self.ytdl_opts) + module, self, self.ytdl_opts + ) if self.outtmpl == "default": self.outtmpl = module.DEFAULT_OUTTMPL if self.forward_cookies: - self.log.debug("Forwarding cookies to %s", - ytdl_instance.__module__) + self.log.debug("Forwarding cookies to %s", ytdl_instance.__module__) set_cookie = ytdl_instance.cookiejar.set_cookie for cookie in self.session.cookies: set_cookie(cookie) @@ -68,8 +67,7 @@ def download(self, url, pathfmt): try: manifest = kwdict.pop("_ytdl_manifest", None) if manifest: - info_dict = self._extract_manifest( - ytdl_instance, url, manifest) + info_dict = self._extract_manifest(ytdl_instance, url, manifest) else: info_dict = self._extract_info(ytdl_instance, url) except Exception as exc: @@ -82,10 +80,8 @@ def download(self, url, pathfmt): if "entries" in info_dict: index = kwdict.get("_ytdl_index") if index is None: - return self._download_playlist( - ytdl_instance, pathfmt, info_dict) - else: - info_dict = info_dict["entries"][index] + return self._download_playlist(ytdl_instance, pathfmt, info_dict) + info_dict = info_dict["entries"][index] extra = kwdict.get("_ytdl_extra") if extra: @@ -108,12 +104,10 @@ def _download_video(self, ytdl_instance, pathfmt, info_dict): if self.outtmpl: self._set_outtmpl(ytdl_instance, self.outtmpl) - pathfmt.filename = filename = \ - ytdl_instance.prepare_filename(info_dict) + pathfmt.filename = filename = ytdl_instance.prepare_filename(info_dict) pathfmt.extension = info_dict["ext"] pathfmt.path = pathfmt.directory + filename - pathfmt.realpath = pathfmt.temppath = ( - pathfmt.realdirectory + filename) + pathfmt.realpath = pathfmt.temppath = pathfmt.realdirectory + filename else: pathfmt.set_extension(info_dict["ext"]) pathfmt.build_path() @@ -122,8 +116,7 @@ def _download_video(self, ytdl_instance, pathfmt, info_dict): pathfmt.temppath = "" return True if self.part and self.partdir: - pathfmt.temppath = os.path.join( - self.partdir, pathfmt.filename) + pathfmt.temppath = os.path.join(self.partdir, pathfmt.filename) self._set_outtmpl(ytdl_instance, pathfmt.temppath.replace("%", "%%")) @@ -153,16 +146,14 @@ def _extract_manifest(self, ytdl, url, manifest): if manifest == "hls": try: - formats, subtitles = extr._extract_m3u8_formats_and_subtitles( - url, video_id, "mp4") + formats, subtitles = extr._extract_m3u8_formats_and_subtitles(url, video_id, "mp4") except AttributeError: formats = extr._extract_m3u8_formats(url, video_id, "mp4") subtitles = None elif manifest == "dash": try: - formats, subtitles = extr._extract_mpd_formats_and_subtitles( - url, video_id) + formats, subtitles = extr._extract_mpd_formats_and_subtitles(url, video_id) except AttributeError: formats = extr._extract_mpd_formats(url, video_id) subtitles = None @@ -172,17 +163,16 @@ def _extract_manifest(self, ytdl, url, manifest): return None info_dict = { - "id" : video_id, - "title" : video_id, - "formats" : formats, + "id": video_id, + "title": video_id, + "formats": formats, "subtitles": subtitles, } # extr._extra_manifest_info(info_dict, url) return ytdl.process_ie_result(info_dict, download=False) def _progress_hook(self, info): - if info["status"] == "downloading" and \ - info["elapsed"] >= self.progress: + if info["status"] == "downloading" and info["elapsed"] >= self.progress: total = info.get("total_bytes") or info.get("total_bytes_estimate") speed = info.get("speed") self.out.progress( diff --git a/gallery_dl/exception.py b/gallery_dl/exception.py index 6b2ce3a8e..328fb6b4c 100644 --- a/gallery_dl/exception.py +++ b/gallery_dl/exception.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -31,6 +29,7 @@ class GalleryDLException(Exception): """Base class for GalleryDL exceptions""" + default = None msgfmt = None code = 1 @@ -39,7 +38,7 @@ def __init__(self, message=None, fmt=True): if not message: message = self.default elif isinstance(message, Exception): - message = "{}: {}".format(message.__class__.__name__, message) + message = f"{message.__class__.__name__}: {message}" if self.msgfmt and fmt: message = self.msgfmt.format(message) Exception.__init__(self, message) @@ -51,6 +50,7 @@ class ExtractionError(GalleryDLException): class HttpError(ExtractionError): """HTTP request during data extraction failed""" + default = "HTTP request failed" code = 4 @@ -61,13 +61,13 @@ def __init__(self, message="", response=None): else: self.status = response.status_code if not message: - message = "'{} {}' for '{}'".format( - response.status_code, response.reason, response.url) + message = f"'{response.status_code} {response.reason}' for '{response.url}'" ExtractionError.__init__(self, message) class NotFoundError(ExtractionError): """Requested resource (gallery/image) could not be found""" + msgfmt = "Requested {} could not be found" default = "resource (gallery/image)" code = 8 @@ -75,48 +75,55 @@ class NotFoundError(ExtractionError): class AuthenticationError(ExtractionError): """Invalid or missing login credentials""" + default = "Invalid or missing login credentials" code = 16 class AuthorizationError(ExtractionError): """Insufficient privileges to access a resource""" + default = "Insufficient privileges to access the specified resource" code = 16 class FormatError(GalleryDLException): """Error while building output paths""" + code = 32 class FilenameFormatError(FormatError): """Error while building output filenames""" + msgfmt = "Applying filename format string failed ({})" class DirectoryFormatError(FormatError): """Error while building output directory paths""" + msgfmt = "Applying directory format string failed ({})" class FilterError(GalleryDLException): """Error while evaluating a filter expression""" + msgfmt = "Evaluating filter expression failed ({})" code = 32 class InputFileError(GalleryDLException): """Error when parsing input file""" + code = 32 def __init__(self, message, *args): - GalleryDLException.__init__( - self, message % args if args else message) + GalleryDLException.__init__(self, message % args if args else message) class NoExtractorError(GalleryDLException): """No extractor can handle the given URL""" + code = 64 @@ -131,9 +138,11 @@ def __init__(self, message=None, *args): class TerminateExtraction(GalleryDLException): """Terminate data extraction""" + code = 0 class RestartExtraction(GalleryDLException): """Restart data extraction""" + code = 0 diff --git a/gallery_dl/extractor/2ch.py b/gallery_dl/extractor/2ch.py index dbbf21b63..2dc76c222 100644 --- a/gallery_dl/extractor/2ch.py +++ b/gallery_dl/extractor/2ch.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://2ch.hk/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message class _2chThreadExtractor(Extractor): """Extractor for 2ch threads""" + category = "2ch" subcategory = "thread" root = "https://2ch.hk" @@ -26,16 +27,16 @@ def __init__(self, match): self.board, self.thread = match.groups() def items(self): - url = "{}/{}/res/{}.json".format(self.root, self.board, self.thread) + url = f"{self.root}/{self.board}/res/{self.thread}.json" posts = self.request(url).json()["threads"][0]["posts"] op = posts[0] title = op.get("subject") or text.remove_html(op["comment"]) thread = { - "board" : self.board, + "board": self.board, "thread": self.thread, - "title" : text.unescape(title)[:50], + "title": text.unescape(title)[:50], } yield Message.Directory, thread @@ -52,14 +53,14 @@ def items(self): file.update(post) file["filename"] = file["fullname"].rpartition(".")[0] - file["tim"], _, file["extension"] = \ - file["name"].rpartition(".") + file["tim"], _, file["extension"] = file["name"].rpartition(".") yield Message.Url, self.root + file["path"], file class _2chBoardExtractor(Extractor): """Extractor for 2ch boards""" + category = "2ch" subcategory = "board" root = "https://2ch.hk" @@ -72,20 +73,18 @@ def __init__(self, match): def items(self): # index page - url = "{}/{}/index.json".format(self.root, self.board) + url = f"{self.root}/{self.board}/index.json" index = self.request(url).json() index["_extractor"] = _2chThreadExtractor for thread in index["threads"]: - url = "{}/{}/res/{}.html".format( - self.root, self.board, thread["thread_num"]) + url = "{}/{}/res/{}.html".format(self.root, self.board, thread["thread_num"]) yield Message.Queue, url, index # pages 1..n for n in util.advance(index["pages"], 1): - url = "{}/{}/{}.json".format(self.root, self.board, n) + url = f"{self.root}/{self.board}/{n}.json" page = self.request(url).json() page["_extractor"] = _2chThreadExtractor for thread in page["threads"]: - url = "{}/{}/res/{}.html".format( - self.root, self.board, thread["thread_num"]) + url = "{}/{}/res/{}.html".format(self.root, self.board, thread["thread_num"]) yield Message.Queue, url, page diff --git a/gallery_dl/extractor/2chan.py b/gallery_dl/extractor/2chan.py index 337ba4804..fec036ccd 100644 --- a/gallery_dl/extractor/2chan.py +++ b/gallery_dl/extractor/2chan.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,14 @@ """Extractors for https://www.2chan.net/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class _2chanThreadExtractor(Extractor): """Extractor for 2chan threads""" + category = "2chan" subcategory = "thread" directory_fmt = ("{category}", "{board_name}", "{thread}") @@ -28,8 +28,7 @@ def __init__(self, match): self.server, self.board, self.thread = match.groups() def items(self): - url = "https://{}.2chan.net/{}/res/{}.htm".format( - self.server, self.board, self.thread) + url = f"https://{self.server}.2chan.net/{self.board}/res/{self.thread}.htm" page = self.request(url).text data = self.metadata(page) yield Message.Directory, data @@ -42,8 +41,7 @@ def items(self): def metadata(self, page): """Collect metadata for extractor-job""" - title, _, boardname = text.extr( - page, "", "").rpartition(" - ") + title, _, boardname = text.extr(page, "", "").rpartition(" - ") return { "server": self.server, "title": title, @@ -54,12 +52,8 @@ def metadata(self, page): def posts(self, page): """Build a list of all post-objects""" - page = text.extr( - page, '
') - return [ - self.parse(post) - for post in page.split('') - ] + page = text.extr(page, '
') + return [self.parse(post) for post in page.split("
")] def parse(self, post): """Build post-object by extracting data from an HTML post""" @@ -76,19 +70,27 @@ def parse(self, post): @staticmethod def _extract_post(post): - return text.extract_all(post, ( - ("post", 'class="csb">' , '<'), - ("name", 'class="cnm">' , '<'), - ("now" , 'class="cnw">' , '<'), - ("no" , 'class="cno">No.', '<'), - (None , '', ''), - ))[0] + return text.extract_all( + post, + ( + ("post", 'class="csb">', "<"), + ("name", 'class="cnm">', "<"), + ("now", 'class="cnw">', "<"), + ("no", 'class="cno">No.', "<"), + (None, "", ""), + ), + )[0] @staticmethod def _extract_image(post, data): - text.extract_all(post, ( - (None , '_blank', ''), - ("filename", '>', '<'), - ("fsize" , '(', ' '), - ), 0, data) + text.extract_all( + post, + ( + (None, "_blank", ""), + ("filename", ">", "<"), + ("fsize", "(", " "), + ), + 0, + data, + ) diff --git a/gallery_dl/extractor/2chen.py b/gallery_dl/extractor/2chen.py index 0c978897a..002f5ea95 100644 --- a/gallery_dl/extractor/2chen.py +++ b/gallery_dl/extractor/2chen.py @@ -1,19 +1,19 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://sturdychan.help/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:sturdychan.help|2chen\.(?:moe|club))" class _2chenThreadExtractor(Extractor): """Extractor for 2chen threads""" + category = "2chen" subcategory = "thread" root = "https://sturdychan.help" @@ -28,13 +28,12 @@ def __init__(self, match): self.board, self.thread = match.groups() def items(self): - url = "{}/{}/{}".format(self.root, self.board, self.thread) + url = f"{self.root}/{self.board}/{self.thread}" page = self.request(url, encoding="utf-8", notfound="thread").text data = self.metadata(page) yield Message.Directory, data for post in self.posts(page): - url = post["url"] if not url: continue @@ -44,40 +43,38 @@ def items(self): post.update(data) post["time"] = text.parse_int(post["date"].timestamp()) - yield Message.Url, url, text.nameext_from_url( - post["filename"], post) + yield Message.Url, url, text.nameext_from_url(post["filename"], post) def metadata(self, page): - board, pos = text.extract(page, 'class="board">/', '/<') + board, pos = text.extract(page, 'class="board">/', "/<") title = text.extract(page, "

", "

", pos)[0] return { - "board" : board, + "board": board, "thread": self.thread, - "title" : text.unescape(title), + "title": text.unescape(title), } def posts(self, page): """Return iterable with relevant posts""" - return map(self.parse, text.extract_iter( - page, 'class="glass media', '')) + return map(self.parse, text.extract_iter(page, 'class="glass media', "")) def parse(self, post): extr = text.extract_from(post) return { - "name" : text.unescape(extr("", "")), - "date" : text.parse_datetime( - extr("")[2], - "%d %b %Y (%a) %H:%M:%S" + "name": text.unescape(extr("", "")), + "date": text.parse_datetime( + extr("")[2], "%d %b %Y (%a) %H:%M:%S" ), - "no" : extr('href="#p', '"'), - "url" : extr('[^&#]+)") + + pattern = ( + r"(?:https?://)?(?:www\.)?behoimi\.org/post" r"(?:/(?:index)?)?\?tags=(?P[^&#]+)" + ) example = "http://behoimi.org/post?tags=TAG" def posts(self): @@ -36,6 +37,7 @@ def posts(self): class _3dbooruPoolExtractor(_3dbooruBase, moebooru.MoebooruPoolExtractor): """Extractor for image-pools from behoimi.org""" + pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/pool/show/(?P\d+)" example = "http://behoimi.org/pool/show/12345" @@ -46,6 +48,7 @@ def posts(self): class _3dbooruPostExtractor(_3dbooruBase, moebooru.MoebooruPostExtractor): """Extractor for single images from behoimi.org""" + pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/post/show/(?P\d+)" example = "http://behoimi.org/post/show/12345" @@ -54,10 +57,12 @@ def posts(self): return self._pagination(self.root + "/post/index.json", params) -class _3dbooruPopularExtractor( - _3dbooruBase, moebooru.MoebooruPopularExtractor): +class _3dbooruPopularExtractor(_3dbooruBase, moebooru.MoebooruPopularExtractor): """Extractor for popular images from behoimi.org""" - pattern = (r"(?:https?://)?(?:www\.)?behoimi\.org" - r"/post/popular_(?Pby_(?:day|week|month)|recent)" - r"(?:\?(?P[^#]*))?") + + pattern = ( + r"(?:https?://)?(?:www\.)?behoimi\.org" + r"/post/popular_(?Pby_(?:day|week|month)|recent)" + r"(?:\?(?P[^#]*))?" + ) example = "http://behoimi.org/post/popular_by_month" diff --git a/gallery_dl/extractor/4archive.py b/gallery_dl/extractor/4archive.py index 948a60546..3af8af62d 100644 --- a/gallery_dl/extractor/4archive.py +++ b/gallery_dl/extractor/4archive.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://4archive.org/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message class _4archiveThreadExtractor(Extractor): """Extractor for 4archive threads""" + category = "4archive" subcategory = "thread" directory_fmt = ("{category}", "{board}", "{thread} {title}") @@ -27,8 +28,7 @@ def __init__(self, match): self.board, self.thread = match.groups() def items(self): - url = "{}/board/{}/thread/{}".format( - self.root, self.board, self.thread) + url = f"{self.root}/board/{self.board}/thread/{self.thread}" page = self.request(url).text data = self.metadata(page) posts = self.posts(page) @@ -41,22 +41,17 @@ def items(self): post["time"] = int(util.datetime_to_timestamp(post["date"])) yield Message.Directory, post if "url" in post: - yield Message.Url, post["url"], text.nameext_from_url( - post["filename"], post) + yield Message.Url, post["url"], text.nameext_from_url(post["filename"], post) def metadata(self, page): return { - "board" : self.board, + "board": self.board, "thread": text.parse_int(self.thread), - "title" : text.unescape(text.extr( - page, 'class="subject">', "")) + "title": text.unescape(text.extr(page, 'class="subject">', "")), } def posts(self, page): - return [ - self.parse(post) - for post in page.split('class="postContainer')[1:] - ] + return [self.parse(post) for post in page.split('class="postContainer')[1:]] @staticmethod def parse(post): @@ -64,28 +59,29 @@ def parse(post): data = { "name": extr('class="name">', ""), "date": text.parse_datetime( - extr('class="dateTime postNum">', "<").strip(), - "%Y-%m-%d %H:%M:%S"), - "no" : text.parse_int(extr('href="#p', '"')), + extr('class="dateTime postNum">', "<").strip(), "%Y-%m-%d %H:%M:%S" + ), + "no": text.parse_int(extr('href="#p', '"')), } if 'class="file"' in post: extr('class="fileText"', ">File: ").strip()[1:], - "size" : text.parse_bytes(extr(" (", ", ")[:-1]), - "width" : text.parse_int(extr("", "x")), - "height" : text.parse_int(extr("", "px")), - }) + data.update( + { + "url": extr('href="', '"'), + "filename": extr('rel="noreferrer noopener"', "").strip()[1:], + "size": text.parse_bytes(extr(" (", ", ")[:-1]), + "width": text.parse_int(extr("", "x")), + "height": text.parse_int(extr("", "px")), + } + ) extr("
", "
"))) + data["com"] = text.unescape(text.remove_html(extr(">", ""))) return data class _4archiveBoardExtractor(Extractor): """Extractor for 4archive boards""" + category = "4archive" subcategory = "board" root = "https://4archive.org" @@ -100,12 +96,11 @@ def __init__(self, match): def items(self): data = {"_extractor": _4archiveThreadExtractor} while True: - url = "{}/board/{}/{}".format(self.root, self.board, self.num) + url = f"{self.root}/board/{self.board}/{self.num}" page = self.request(url).text if 'class="thread"' not in page: return for thread in text.extract_iter(page, 'class="thread" id="t', '"'): - url = "{}/board/{}/thread/{}".format( - self.root, self.board, thread) + url = f"{self.root}/board/{self.board}/thread/{thread}" yield Message.Queue, url, data self.num += 1 diff --git a/gallery_dl/extractor/4chan.py b/gallery_dl/extractor/4chan.py index 2db60422e..73631ab41 100644 --- a/gallery_dl/extractor/4chan.py +++ b/gallery_dl/extractor/4chan.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,19 +6,20 @@ """Extractors for https://www.4chan.org/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class _4chanThreadExtractor(Extractor): """Extractor for 4chan threads""" + category = "4chan" subcategory = "thread" directory_fmt = ("{category}", "{board}", "{thread} {title}") filename_fmt = "{tim} {filename}.{extension}" archive_fmt = "{board}_{thread}_{tim}" - pattern = (r"(?:https?://)?boards\.4chan(?:nel)?\.org" - r"/([^/]+)/thread/(\d+)") + pattern = r"(?:https?://)?boards\.4chan(?:nel)?\.org" r"/([^/]+)/thread/(\d+)" example = "https://boards.4channel.org/a/thread/12345/" def __init__(self, match): @@ -28,15 +27,14 @@ def __init__(self, match): self.board, self.thread = match.groups() def items(self): - url = "https://a.4cdn.org/{}/thread/{}.json".format( - self.board, self.thread) + url = f"https://a.4cdn.org/{self.board}/thread/{self.thread}.json" posts = self.request(url).json()["posts"] title = posts[0].get("sub") or text.remove_html(posts[0]["com"]) data = { - "board" : self.board, + "board": self.board, "thread": self.thread, - "title" : text.unescape(title)[:50], + "title": text.unescape(title)[:50], } yield Message.Directory, data @@ -45,13 +43,13 @@ def items(self): post.update(data) post["extension"] = post["ext"][1:] post["filename"] = text.unescape(post["filename"]) - url = "https://i.4cdn.org/{}/{}{}".format( - post["board"], post["tim"], post["ext"]) + url = "https://i.4cdn.org/{}/{}{}".format(post["board"], post["tim"], post["ext"]) yield Message.Url, url, post class _4chanBoardExtractor(Extractor): """Extractor for 4chan boards""" + category = "4chan" subcategory = "board" pattern = r"(?:https?://)?boards\.4chan(?:nel)?\.org/([^/?#]+)/\d*$" @@ -62,13 +60,12 @@ def __init__(self, match): self.board = match.group(1) def items(self): - url = "https://a.4cdn.org/{}/threads.json".format(self.board) + url = f"https://a.4cdn.org/{self.board}/threads.json" threads = self.request(url).json() for page in threads: for thread in page["threads"]: - url = "https://boards.4chan.org/{}/thread/{}/".format( - self.board, thread["no"]) + url = "https://boards.4chan.org/{}/thread/{}/".format(self.board, thread["no"]) thread["page"] = page["page"] thread["_extractor"] = _4chanThreadExtractor yield Message.Queue, url, thread diff --git a/gallery_dl/extractor/4chanarchives.py b/gallery_dl/extractor/4chanarchives.py index 27ac7c556..ec188bc51 100644 --- a/gallery_dl/extractor/4chanarchives.py +++ b/gallery_dl/extractor/4chanarchives.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,14 @@ """Extractors for https://4chanarchives.com/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class _4chanarchivesThreadExtractor(Extractor): """Extractor for threads on 4chanarchives.com""" + category = "4chanarchives" subcategory = "thread" root = "https://4chanarchives.com" @@ -29,15 +29,13 @@ def __init__(self, match): self.board, self.thread = match.groups() def items(self): - url = "{}/board/{}/thread/{}".format( - self.root, self.board, self.thread) + url = f"{self.root}/board/{self.board}/thread/{self.thread}" page = self.request(url).text data = self.metadata(page) posts = self.posts(page) if not data["title"]: - data["title"] = text.unescape(text.remove_html( - posts[0]["com"]))[:50] + data["title"] = text.unescape(text.remove_html(posts[0]["com"]))[:50] for post in posts: post.update(data) @@ -47,16 +45,14 @@ def items(self): def metadata(self, page): return { - "board" : self.board, - "thread" : self.thread, - "title" : text.unescape(text.extr( - page, 'property="og:title" content="', '"')), + "board": self.board, + "thread": self.thread, + "title": text.unescape(text.extr(page, 'property="og:title" content="', '"')), } def posts(self, page): """Build a list of all post objects""" - return [self.parse(html) for html in text.extract_iter( - page, 'id="pc', '')] + return [self.parse(html) for html in text.extract_iter(page, 'id="pc', "")] def parse(self, html): """Build post object by extracting data from an HTML post""" @@ -70,11 +66,10 @@ def parse(self, html): def _extract_post(html): extr = text.extract_from(html) return { - "no" : text.parse_int(extr('', '"')), - "name": extr('class="name">', '<'), - "time": extr('class="dateTime postNum" >', '<').rstrip(), - "com" : text.unescape( - html[html.find('")[2]), + "no": text.parse_int(extr("", '"')), + "name": extr('class="name">', "<"), + "time": extr('class="dateTime postNum" >', "<").rstrip(), + "com": text.unescape(html[html.find("")[2]), } @staticmethod @@ -89,6 +84,7 @@ def _extract_file(html, post): class _4chanarchivesBoardExtractor(Extractor): """Extractor for boards on 4chanarchives.com""" + category = "4chanarchives" subcategory = "board" root = "https://4chanarchives.com" @@ -106,7 +102,7 @@ def items(self): data["pageCount"]: return - url = "{}/{}/{}.json".format(self.root, board, pnum) + url = f"{self.root}/{board}/{pnum}.json" threads = self.request(url).json()["threads"] diff --git a/gallery_dl/extractor/8muses.py b/gallery_dl/extractor/8muses.py index f88a0c674..1e4004ef4 100644 --- a/gallery_dl/extractor/8muses.py +++ b/gallery_dl/extractor/8muses.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,20 +6,22 @@ """Extractors for https://comics.8muses.com/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message class _8musesAlbumExtractor(Extractor): """Extractor for image albums on comics.8muses.com""" + category = "8muses" subcategory = "album" directory_fmt = ("{category}", "{album[path]}") filename_fmt = "{page:>03}.{extension}" archive_fmt = "{hash}" root = "https://comics.8muses.com" - pattern = (r"(?:https?://)?(?:comics\.|www\.)?8muses\.com" - r"(/comics/album/[^?#]+)(\?[^#]+)?") + pattern = r"(?:https?://)?(?:comics\.|www\.)?8muses\.com" r"(/comics/album/[^?#]+)(\?[^#]+)?" example = "https://comics.8muses.com/comics/album/PATH/TITLE" def __init__(self, match): @@ -33,9 +33,11 @@ def items(self): url = self.root + self.path + self.params while True: - data = self._unobfuscate(text.extr( - self.request(url).text, - 'id="ractive-public" type="text/plain">', '')) + data = self._unobfuscate( + text.extr( + self.request(url).text, 'id="ractive-public" type="text/plain">', "" + ) + ) images = data.get("pictures") if images: @@ -45,11 +47,11 @@ def items(self): for num, image in enumerate(images, 1): url = self.root + "/image/fl/" + image["publicUri"] img = { - "url" : url, - "page" : num, - "hash" : image["publicUri"], - "count" : count, - "album" : album, + "url": url, + "page": num, + "hash": image["publicUri"], + "count": count, + "album": album, "extension": "jpg", } yield Message.Url, url, img @@ -58,38 +60,44 @@ def items(self): if albums: for album in albums: url = self.root + "/comics/album/" + album["permalink"] - yield Message.Queue, url, { - "url" : url, - "name" : album["name"], - "private" : album["isPrivate"], - "_extractor": _8musesAlbumExtractor, - } + yield ( + Message.Queue, + url, + { + "url": url, + "name": album["name"], + "private": album["isPrivate"], + "_extractor": _8musesAlbumExtractor, + }, + ) if data["page"] >= data["pages"]: return path, _, num = self.path.rstrip("/").rpartition("/") path = path if num.isdecimal() else self.path - url = "{}{}/{}{}".format( - self.root, path, data["page"] + 1, self.params) + url = "{}{}/{}{}".format(self.root, path, data["page"] + 1, self.params) def _make_album(self, album): return { - "id" : album["id"], - "path" : album["path"], - "parts" : album["path"].split("/"), - "title" : album["name"], + "id": album["id"], + "path": album["path"], + "parts": album["path"].split("/"), + "title": album["name"], "private": album["isPrivate"], - "url" : self.root + "/comics/album/" + album["permalink"], - "parent" : text.parse_int(album["parentId"]), - "views" : text.parse_int(album["numberViews"]), - "likes" : text.parse_int(album["numberLikes"]), - "date" : text.parse_datetime( - album["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ"), + "url": self.root + "/comics/album/" + album["permalink"], + "parent": text.parse_int(album["parentId"]), + "views": text.parse_int(album["numberViews"]), + "likes": text.parse_int(album["numberLikes"]), + "date": text.parse_datetime(album["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ"), } @staticmethod def _unobfuscate(data): - return util.json_loads("".join([ - chr(33 + (ord(c) + 14) % 94) if "!" <= c <= "~" else c - for c in text.unescape(data.strip("\t\n\r !")) - ])) + return util.json_loads( + "".join( + [ + chr(33 + (ord(c) + 14) % 94) if "!" <= c <= "~" else c + for c in text.unescape(data.strip("\t\n\r !")) + ] + ) + ) diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index 903370213..330116fe8 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -1,13 +1,11 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import sys import re +import sys modules = [ "2ch", @@ -238,10 +236,7 @@ def add_module(module): def extractors(): """Yield all available extractor classes""" - return sorted( - _list_classes(), - key=lambda x: x.__name__ - ) + return sorted(_list_classes(), key=lambda x: x.__name__) # -------------------------------------------------------------------- @@ -255,7 +250,7 @@ def _list_classes(): for module in _module_iter: yield from add_module(module) - globals()["_list_classes"] = lambda : _cache + globals()["_list_classes"] = lambda: _cache def _modules_internal(): @@ -267,11 +262,7 @@ def _modules_internal(): def _modules_path(path, files): sys.path.insert(0, path) try: - return [ - __import__(name[:-3]) - for name in files - if name.endswith(".py") - ] + return [__import__(name[:-3]) for name in files if name.endswith(".py")] finally: del sys.path[0] @@ -279,9 +270,9 @@ def _modules_path(path, files): def _get_classes(module): """Return a list of all extractor classes in a module""" return [ - cls for cls in module.__dict__.values() if ( - hasattr(cls, "pattern") and cls.__module__ == module.__name__ - ) + cls + for cls in module.__dict__.values() + if (hasattr(cls, "pattern") and cls.__module__ == module.__name__) ] diff --git a/gallery_dl/extractor/adultempire.py b/gallery_dl/extractor/adultempire.py index 1617414a7..d309b46e2 100644 --- a/gallery_dl/extractor/adultempire.py +++ b/gallery_dl/extractor/adultempire.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,16 +6,16 @@ """Extractors for https://www.adultempire.com/""" -from .common import GalleryExtractor from .. import text +from .common import GalleryExtractor class AdultempireGalleryExtractor(GalleryExtractor): """Extractor for image galleries from www.adultempire.com""" + category = "adultempire" root = "https://www.adultempire.com" - pattern = (r"(?:https?://)?(?:www\.)?adult(?:dvd)?empire\.com" - r"(/(\d+)/gallery\.html)") + pattern = r"(?:https?://)?(?:www\.)?adult(?:dvd)?empire\.com" r"(/(\d+)/gallery\.html)" example = "https://www.adultempire.com/12345/gallery.html" def __init__(self, match): @@ -28,12 +26,12 @@ def metadata(self, page): extr = text.extract_from(page, page.index('
')) return { "gallery_id": text.parse_int(self.gallery_id), - "title" : text.unescape(extr('title="', '"')), - "studio" : extr(">studio", "<").strip(), - "date" : text.parse_datetime(extr( - ">released", "<").strip(), "%m/%d/%Y"), - "actors" : sorted(text.split_html(extr( - '
    studio", "<").strip(), + "date": text.parse_datetime(extr(">released", "<").strip(), "%m/%d/%Y"), + "actors": sorted( + text.split_html(extr('
      "))[1:] + ), } def images(self, page): diff --git a/gallery_dl/extractor/agnph.py b/gallery_dl/extractor/agnph.py index 653b73f1b..29b526892 100644 --- a/gallery_dl/extractor/agnph.py +++ b/gallery_dl/extractor/agnph.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,12 @@ """Extractors for https://agn.ph/""" -from . import booru -from .. import text - -from xml.etree import ElementTree import collections import re +from xml.etree import ElementTree as ET + +from .. import text +from . import booru BASE_PATTERN = r"(?:https?://)?agn\.ph" @@ -38,7 +36,7 @@ def _init(self): def _prepare(self, post): post["date"] = text.parse_timestamp(post["created_at"]) post["status"] = post["status"].strip() - post["has_children"] = ("true" in post["has_children"]) + post["has_children"] = "true" in post["has_children"] def _xml_to_dict(self, xml): return {element.tag: element.text for element in xml} @@ -46,14 +44,13 @@ def _xml_to_dict(self, xml): def _pagination(self, url, params): params["api"] = "xml" if "page" in params: - params["page"] = \ - self.page_start + text.parse_int(params["page"]) - 1 + params["page"] = self.page_start + text.parse_int(params["page"]) - 1 else: params["page"] = self.page_start while True: data = self.request(url, params=params).text - root = ElementTree.fromstring(data) + root = ET.fromstring(data) yield from map(self._xml_to_dict, root) @@ -68,8 +65,7 @@ def _html(self, post): return self.request(url).text def _tags(self, post, page): - tag_container = text.extr( - page, '
        ', '

        Statistics

        ') + tag_container = text.extr(page, '
          ', "

          Statistics

          ") if not tag_container: return @@ -107,7 +103,6 @@ class AgnphPostExtractor(AgnphExtractor): example = "https://agn.ph/gallery/post/show/12345/" def posts(self): - url = "{}/gallery/post/show/{}/?api=xml".format( - self.root, self.groups[0]) - post = ElementTree.fromstring(self.request(url).text) + url = f"{self.root}/gallery/post/show/{self.groups[0]}/?api=xml" + post = ET.fromstring(self.request(url).text) return (self._xml_to_dict(post),) diff --git a/gallery_dl/extractor/ao3.py b/gallery_dl/extractor/ao3.py index d3ab84688..1445839c2 100644 --- a/gallery_dl/extractor/ao3.py +++ b/gallery_dl/extractor/ao3.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,16 +6,19 @@ """Extractors for https://archiveofourown.org/""" -from .common import Extractor, Message -from .. import text, util, exception +from .. import exception +from .. import text +from .. import util from ..cache import cache +from .common import Extractor +from .common import Message -BASE_PATTERN = (r"(?:https?://)?(?:www\.)?" - r"a(?:rchiveofourown|o3)\.(?:org|com|net)") +BASE_PATTERN = r"(?:https?://)?(?:www\.)?" r"a(?:rchiveofourown|o3)\.(?:org|com|net)" class Ao3Extractor(Extractor): """Base class for ao3 extractors""" + category = "ao3" root = "https://archiveofourown.org" categorytransfer = True @@ -59,13 +60,13 @@ def works(self): def login(self): if self.cookies_check(self.cookies_names): - return + return None username, password = self._get_auth_info() if username: return self.cookies_update(self._login_impl(username, password)) - @cache(maxage=90*86400, keyarg=1) + @cache(maxage=90 * 86400, keyarg=1) def _login_impl(self, username, password): self.log.info("Logging in as %s", username) @@ -73,17 +74,16 @@ def _login_impl(self, username, password): page = self.request(url).text pos = page.find('id="loginform"') - token = text.extract( - page, ' name="authenticity_token" value="', '"', pos)[0] + token = text.extract(page, ' name="authenticity_token" value="', '"', pos)[0] if not token: self.log.error("Unable to extract 'authenticity_token'") data = { "authenticity_token": text.unescape(token), - "user[login]" : username, - "user[password]" : password, - "user[remember_me]" : "1", - "commit" : "Log In", + "user[login]": username, + "user[password]": password, + "user[remember_me]": "1", + "commit": "Log In", } response = self.request(url, method="POST", data=data) @@ -96,7 +96,7 @@ def _login_impl(self, username, password): return { "remember_user_token": remember, - "user_credentials" : "1", + "user_credentials": "1", } def _pagination(self, path, needle='
        • Adult Content WarningAdult Content Warning', "")), - "warnings" : text.split_html( - extr('
          ', "
          ")), - "categories" : text.split_html( - extr('
          ', "
          ")), - "fandom" : text.split_html( - extr('
          ', "
          ")), - "relationships": text.split_html( - extr('
          ', "
          ")), - "characters" : text.split_html( - extr('
          ', "
          ")), - "tags" : text.split_html( - extr('
          ', "
          ")), - "lang" : extr('
          ', "
          "), - "date" : text.parse_datetime( - extr('
          ', "<"), "%Y-%m-%d"), + "id": text.parse_int(work_id), + "rating": text.split_html(extr('
          ', "
          ")), + "warnings": text.split_html(extr('
          ', "
          ")), + "categories": text.split_html(extr('
          ', "
          ")), + "fandom": text.split_html(extr('
          ', "
          ")), + "relationships": text.split_html(extr('
          ', "
          ")), + "characters": text.split_html(extr('
          ', "
          ")), + "tags": text.split_html(extr('
          ', "
          ")), + "lang": extr('
          ', "
          "), + "date": text.parse_datetime(extr('
          ', "<"), "%Y-%m-%d"), "date_completed": text.parse_datetime( - extr('>Completed:
          ', "<"), "%Y-%m-%d"), - "date_updated" : text.parse_timestamp( - path.rpartition("updated_at=")[2]), - "words" : text.parse_int( - extr('
          ', "<").replace(",", "")), - "chapters" : chapters, - "comments" : text.parse_int( - extr('
          ', "<").replace(",", "")), - "likes" : text.parse_int( - extr('
          ', "<").replace(",", "")), - "bookmarks" : text.parse_int(text.remove_html( - extr('
          ', "
          ")).replace(",", "")), - "views" : text.parse_int( - extr('
          ', "<").replace(",", "")), - "title" : text.unescape(text.remove_html( - extr(' class="title heading">', "")).strip()), - "author" : text.unescape(text.remove_html( - extr(' class="byline heading">', ""))), - "summary" : text.split_html( - extr(' class="heading">Summary:', "
")), + extr('>Completed:
', "<"), "%Y-%m-%d" + ), + "date_updated": text.parse_timestamp(path.rpartition("updated_at=")[2]), + "words": text.parse_int(extr('
', "<").replace(",", "")), + "chapters": chapters, + "comments": text.parse_int(extr('
', "<").replace(",", "")), + "likes": text.parse_int(extr('
', "<").replace(",", "")), + "bookmarks": text.parse_int( + text.remove_html(extr('
', "
")).replace(",", "") + ), + "views": text.parse_int(extr('
', "<").replace(",", "")), + "title": text.unescape( + text.remove_html(extr(' class="title heading">', "")).strip() + ), + "author": text.unescape(text.remove_html(extr(' class="byline heading">', ""))), + "summary": text.split_html(extr(' class="heading">Summary:', "")), } data["language"] = util.code_to_language(data["lang"]) @@ -209,11 +196,11 @@ def items(self): if series: extr = text.extract_from(series) data["series"] = { - "prev" : extr(' class="previous" href="/works/', '"'), + "prev": extr(' class="previous" href="/works/', '"'), "index": extr(' class="position">Part ', " "), - "id" : extr(' href="/series/', '"'), - "name" : text.unescape(extr(">", "<")), - "next" : extr(' class="next" href="/works/', '"'), + "id": extr(' href="/series/', '"'), + "name": text.unescape(extr(">", "<")), + "next": extr(' class="next" href="/works/', '"'), } else: data["series"] = None @@ -230,6 +217,7 @@ def items(self): class Ao3SeriesExtractor(Ao3Extractor): """Extractor for AO3 works of a series""" + subcategory = "series" pattern = BASE_PATTERN + r"(/series/(\d+))" example = "https://archiveofourown.org/series/12345" @@ -237,6 +225,7 @@ class Ao3SeriesExtractor(Ao3Extractor): class Ao3TagExtractor(Ao3Extractor): """Extractor for AO3 works by tag""" + subcategory = "tag" pattern = BASE_PATTERN + r"(/tags/([^/?#]+)/works(?:/?\?.+)?)" example = "https://archiveofourown.org/tags/TAG/works" @@ -244,6 +233,7 @@ class Ao3TagExtractor(Ao3Extractor): class Ao3SearchExtractor(Ao3Extractor): """Extractor for AO3 search results""" + subcategory = "search" pattern = BASE_PATTERN + r"(/works/search/?\?.+)" example = "https://archiveofourown.org/works/search?work_search[query]=air" @@ -251,36 +241,39 @@ class Ao3SearchExtractor(Ao3Extractor): class Ao3UserExtractor(Ao3Extractor): """Extractor for an AO3 user profile""" + subcategory = "user" - pattern = (BASE_PATTERN + r"/users/([^/?#]+(?:/pseuds/[^/?#]+)?)" - r"(?:/profile)?/?(?:$|\?|#)") + pattern = BASE_PATTERN + r"/users/([^/?#]+(?:/pseuds/[^/?#]+)?)" r"(?:/profile)?/?(?:$|\?|#)" example = "https://archiveofourown.org/users/USER" def initialize(self): pass def items(self): - base = "{}/users/{}/".format(self.root, self.groups[0]) - return self._dispatch_extractors(( - (Ao3UserWorksExtractor , base + "works"), - (Ao3UserSeriesExtractor , base + "series"), - (Ao3UserBookmarkExtractor, base + "bookmarks"), - ), ("user-works", "user-series")) + base = f"{self.root}/users/{self.groups[0]}/" + return self._dispatch_extractors( + ( + (Ao3UserWorksExtractor, base + "works"), + (Ao3UserSeriesExtractor, base + "series"), + (Ao3UserBookmarkExtractor, base + "bookmarks"), + ), + ("user-works", "user-series"), + ) class Ao3UserWorksExtractor(Ao3Extractor): """Extractor for works of an AO3 user""" + subcategory = "user-works" - pattern = (BASE_PATTERN + r"(/users/([^/?#]+)/(?:pseuds/([^/?#]+)/)?" - r"works(?:/?\?.+)?)") + pattern = BASE_PATTERN + r"(/users/([^/?#]+)/(?:pseuds/([^/?#]+)/)?" r"works(?:/?\?.+)?)" example = "https://archiveofourown.org/users/USER/works" class Ao3UserSeriesExtractor(Ao3Extractor): """Extractor for series of an AO3 user""" + subcategory = "user-series" - pattern = (BASE_PATTERN + r"(/users/([^/?#]+)/(?:pseuds/([^/?#]+)/)?" - r"series(?:/?\?.+)?)") + pattern = BASE_PATTERN + r"(/users/([^/?#]+)/(?:pseuds/([^/?#]+)/)?" r"series(?:/?\?.+)?)" example = "https://archiveofourown.org/users/USER/series" def items(self): @@ -298,9 +291,9 @@ def series(self): class Ao3UserBookmarkExtractor(Ao3Extractor): """Extractor for bookmarked works of an AO3 user""" + subcategory = "user-bookmark" - pattern = (BASE_PATTERN + r"(/users/([^/?#]+)/(?:pseuds/([^/?#]+)/)?" - r"bookmarks(?:/?\?.+)?)") + pattern = BASE_PATTERN + r"(/users/([^/?#]+)/(?:pseuds/([^/?#]+)/)?" r"bookmarks(?:/?\?.+)?)" example = "https://archiveofourown.org/users/USER/bookmarks" def items(self): @@ -309,6 +302,7 @@ def items(self): class Ao3SubscriptionsExtractor(Ao3Extractor): """Extractor for your AO3 account's subscriptions""" + subcategory = "subscriptions" pattern = BASE_PATTERN + r"(/users/([^/?#]+)/subscriptions(?:/?\?.+)?)" example = "https://archiveofourown.org/users/USER/subscriptions" diff --git a/gallery_dl/extractor/architizer.py b/gallery_dl/extractor/architizer.py index 8064e7898..720f4b0f0 100644 --- a/gallery_dl/extractor/architizer.py +++ b/gallery_dl/extractor/architizer.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,15 @@ """Extractors for https://architizer.com/""" -from .common import GalleryExtractor, Extractor, Message from .. import text +from .common import Extractor +from .common import GalleryExtractor +from .common import Message class ArchitizerProjectExtractor(GalleryExtractor): """Extractor for project pages on architizer.com""" + category = "architizer" subcategory = "project" root = "https://architizer.com" @@ -24,7 +25,7 @@ class ArchitizerProjectExtractor(GalleryExtractor): example = "https://architizer.com/projects/NAME/" def __init__(self, match): - url = "{}/projects/{}/".format(self.root, match.group(1)) + url = f"{self.root}/projects/{match.group(1)}/" GalleryExtractor.__init__(self, match, url) def metadata(self, page): @@ -32,34 +33,30 @@ def metadata(self, page): extr('id="Pages"', "") return { - "title" : extr('data-name="', '"'), - "slug" : extr('data-slug="', '"'), - "gid" : extr('data-gid="', '"').rpartition(".")[2], - "firm" : extr('data-firm-leaders-str="', '"'), - "location" : extr("

", "<").strip(), - "type" : text.unescape(text.remove_html(extr( - '
Type
', 'STATUS', 'YEAR', 'SIZE', '', '') - .replace("
", "\n")), + "title": extr('data-name="', '"'), + "slug": extr('data-slug="', '"'), + "gid": extr('data-gid="', '"').rpartition(".")[2], + "firm": extr('data-firm-leaders-str="', '"'), + "location": extr("

", "<").strip(), + "type": text.unescape(text.remove_html(extr('
Type
', "STATUS', "YEAR', "SIZE', "', "").replace("
", "\n") + ), } def images(self, page): return [ (url, None) - for url in text.extract_iter( - page, 'property="og:image:secure_url" content="', "?") + for url in text.extract_iter(page, 'property="og:image:secure_url" content="', "?") ] class ArchitizerFirmExtractor(Extractor): """Extractor for all projects of a firm""" + category = "architizer" subcategory = "firm" root = "https://architizer.com" @@ -71,12 +68,11 @@ def __init__(self, match): self.firm = match.group(1) def items(self): - url = url = "{}/firms/{}/?requesting_merlin=pages".format( - self.root, self.firm) + url = url = f"{self.root}/firms/{self.firm}/?requesting_merlin=pages" page = self.request(url).text data = {"_extractor": ArchitizerProjectExtractor} for project in text.extract_iter(page, '= 400: self.log.warning( "Unable to fetch post %s ('%s %s')", - post_id, response.status_code, response.reason) + post_id, + response.status_code, + response.reason, + ) return None headers = response.headers @@ -141,40 +146,38 @@ def _parse_post(self, post_id): # fix 'Last-Modified' header lmod = headers["last-modified"] if lmod[22] != ":": - lmod = "{}:{} GMT".format(lmod[:22], lmod[22:24]) + lmod = f"{lmod[:22]}:{lmod[22:24]} GMT" - post_url = "{}/g4/view/{}".format(self.root, post_id) + post_url = f"{self.root}/g4/view/{post_id}" extr = text.extract_from(self.request(post_url).text) - title, _, artist = text.unescape(extr( - "g4 :: ", "<")).rpartition(" by ") + title, _, artist = text.unescape(extr("<title>g4 :: ", "<")).rpartition(" by ") return { - "id" : text.parse_int(post_id), - "url" : url, - "user" : self.user or artist, - "title" : title, + "id": text.parse_int(post_id), + "url": url, + "user": self.user or artist, + "title": title, "artist": artist, - "path" : text.split_html(extr( - "cookiecrumb'>", '</span'))[4:-1:2], - "date" : datetime(*parsedate_tz(lmod)[:6]), - "size" : text.parse_int(clen), - "views" : text.parse_int(extr("Views</b>:", "<").replace(",", "")), - "width" : text.parse_int(extr("Resolution</b>:", "x")), + "path": text.split_html(extr("cookiecrumb'>", "</span"))[4:-1:2], + "date": datetime(*parsedate_tz(lmod)[:6]), + "size": text.parse_int(clen), + "views": text.parse_int(extr("Views</b>:", "<").replace(",", "")), + "width": text.parse_int(extr("Resolution</b>:", "x")), "height": text.parse_int(extr("", "<")), - "comments" : text.parse_int(extr("Comments</b>:", "<")), + "comments": text.parse_int(extr("Comments</b>:", "<")), "favorites": text.parse_int(extr("Favorites</b>:", "<")), - "tags" : text.split_html(extr("class='taglist'>", "</span>")), - "description": text.unescape(text.remove_html(extr( - "<p>", "</p>"), "", "")), - "filename" : fname, + "tags": text.split_html(extr("class='taglist'>", "</span>")), + "description": text.unescape(text.remove_html(extr("<p>", "</p>"), "", "")), + "filename": fname, "extension": ext, - "_mtime" : lmod, + "_mtime": lmod, } class AryionGalleryExtractor(AryionExtractor): """Extractor for a user's gallery on eka's portal""" + subcategory = "gallery" categorytransfer = True pattern = BASE_PATTERN + r"/(?:gallery/|user/|latest.php\?name=)([^/?#]+)" @@ -195,15 +198,15 @@ def skip(self, num): def posts(self): if self.recursive: - url = "{}/g4/gallery/{}".format(self.root, self.user) + url = f"{self.root}/g4/gallery/{self.user}" return self._pagination_params(url) - else: - url = "{}/g4/latest.php?name={}".format(self.root, self.user) - return util.advance(self._pagination_next(url), self.offset) + url = f"{self.root}/g4/latest.php?name={self.user}" + return util.advance(self._pagination_next(url), self.offset) class AryionFavoriteExtractor(AryionExtractor): """Extractor for a user's favorites gallery""" + subcategory = "favorite" directory_fmt = ("{category}", "{user!l}", "favorites") archive_fmt = "f_{user}_{id}" @@ -212,13 +215,13 @@ class AryionFavoriteExtractor(AryionExtractor): example = "https://aryion.com/g4/favorites/USER" def posts(self): - url = "{}/g4/favorites/{}".format(self.root, self.user) - return self._pagination_params( - url, None, "class='gallery-item favorite' id='") + url = f"{self.root}/g4/favorites/{self.user}" + return self._pagination_params(url, None, "class='gallery-item favorite' id='") class AryionTagExtractor(AryionExtractor): """Extractor for tag searches on eka's portal""" + subcategory = "tag" directory_fmt = ("{category}", "tags", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" @@ -239,6 +242,7 @@ def posts(self): class AryionPostExtractor(AryionExtractor): """Extractor for individual posts on eka's portal""" + subcategory = "post" pattern = BASE_PATTERN + r"/view/(\d+)" example = "https://aryion.com/g4/view/12345" diff --git a/gallery_dl/extractor/batoto.py b/gallery_dl/extractor/batoto.py index 786acd9fe..00dea8371 100644 --- a/gallery_dl/extractor/batoto.py +++ b/gallery_dl/extractor/batoto.py @@ -1,24 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://bato.to/""" -from .common import Extractor, ChapterExtractor, MangaExtractor -from .. import text, exception import re -BASE_PATTERN = (r"(?:https?://)?(?:" - r"(?:ba|d|h|m|w)to\.to|" - r"(?:(?:manga|read)toto|batocomic|[xz]bato)\.(?:com|net|org)|" - r"comiko\.(?:net|org)|" - r"bat(?:otoo|o?two)\.com)") +from .. import exception +from .. import text +from .common import ChapterExtractor +from .common import Extractor +from .common import MangaExtractor + +BASE_PATTERN = ( + r"(?:https?://)?(?:" + r"(?:ba|d|h|m|w)to\.to|" + r"(?:(?:manga|read)toto|batocomic|[xz]bato)\.(?:com|net|org)|" + r"comiko\.(?:net|org)|" + r"bat(?:otoo|o?two)\.com)" +) -class BatotoBase(): +class BatotoBase: """Base class for batoto extractors""" + category = "batoto" root = "https://bato.to" @@ -29,13 +34,14 @@ def request(self, url, **kwargs): class BatotoChapterExtractor(BatotoBase, ChapterExtractor): """Extractor for bato.to manga chapters""" + pattern = BASE_PATTERN + r"/(?:title/[^/?#]+|chapter)/(\d+)" example = "https://bato.to/title/12345-MANGA/54321" def __init__(self, match): self.root = text.root_from_url(match.group(0)) self.chapter_id = match.group(1) - url = "{}/title/0/{}".format(self.root, self.chapter_id) + url = f"{self.root}/title/0/{self.chapter_id}" ChapterExtractor.__init__(self, match, url) def metadata(self, page): @@ -45,8 +51,7 @@ def metadata(self, page): except ValueError: manga = info = None - manga_id = text.extr( - extr('rel="canonical" href="', '"'), "/title/", "/") + manga_id = text.extr(extr('rel="canonical" href="', '"'), "/title/", "/") if not manga: manga = extr('link-hover">', "<") @@ -54,8 +59,9 @@ def metadata(self, page): info = text.unescape(info) match = re.match( - r"(?i)(?:(?:Volume|S(?:eason)?)\s*(\d+)\s+)?" - r"(?:Chapter|Episode)\s*(\d+)([\w.]*)", info) + r"(?i)(?:(?:Volume|S(?:eason)?)\s*(\d+)\s+)?" r"(?:Chapter|Episode)\s*(\d+)([\w.]*)", + info, + ) if match: volume, chapter, minor = match.groups() else: @@ -63,40 +69,38 @@ def metadata(self, page): minor = "" return { - "manga" : text.unescape(manga), - "manga_id" : text.parse_int(manga_id), - "chapter_url" : extr(self.chapter_id + "-ch_", '"'), - "title" : text.unescape(text.remove_html(extr( - "selected>", "</option")).partition(" : ")[2]), - "volume" : text.parse_int(volume), - "chapter" : text.parse_int(chapter), - "chapter_minor" : minor, + "manga": text.unescape(manga), + "manga_id": text.parse_int(manga_id), + "chapter_url": extr(self.chapter_id + "-ch_", '"'), + "title": text.unescape( + text.remove_html(extr("selected>", "</option")).partition(" : ")[2] + ), + "volume": text.parse_int(volume), + "chapter": text.parse_int(chapter), + "chapter_minor": minor, "chapter_string": info, - "chapter_id" : text.parse_int(self.chapter_id), - "date" : text.parse_timestamp(extr(' time="', '"')[:-3]), + "chapter_id": text.parse_int(self.chapter_id), + "date": text.parse_timestamp(extr(' time="', '"')[:-3]), } def images(self, page): - images_container = text.extr(page, 'pageOpts', ':[0,0]}"') + images_container = text.extr(page, "pageOpts", ':[0,0]}"') images_container = text.unescape(images_container) - return [ - (url, None) - for url in text.extract_iter(images_container, r"\"", r"\"") - ] + return [(url, None) for url in text.extract_iter(images_container, r"\"", r"\"")] class BatotoMangaExtractor(BatotoBase, MangaExtractor): """Extractor for bato.to manga""" + reverse = False chapterclass = BatotoChapterExtractor - pattern = (BASE_PATTERN + - r"/(?:title/(\d+)[^/?#]*|series/(\d+)(?:/[^/?#]*)?)/?$") + pattern = BASE_PATTERN + r"/(?:title/(\d+)[^/?#]*|series/(\d+)(?:/[^/?#]*)?)/?$" example = "https://bato.to/title/12345-MANGA/" def __init__(self, match): self.root = text.root_from_url(match.group(0)) self.manga_id = match.group(1) or match.group(2) - url = "{}/title/{}".format(self.root, self.manga_id) + url = f"{self.root}/title/{self.manga_id}" MangaExtractor.__init__(self, match, url) def chapters(self, page): @@ -108,8 +112,7 @@ def chapters(self, page): data = { "manga_id": text.parse_int(self.manga_id), - "manga" : text.unescape(extr( - "<title>", "<").rpartition(" - ")[0]), + "manga": text.unescape(extr("<title>", "<").rpartition(" - ")[0]), } extr('<div data-hk="0-0-0-0"', "") @@ -124,9 +127,8 @@ def chapters(self, page): data["chapter"] = text.parse_int(chapter) data["chapter_minor"] = sep + minor - data["date"] = text.parse_datetime( - extr('time="', '"'), "%Y-%m-%dT%H:%M:%S.%fZ") + data["date"] = text.parse_datetime(extr('time="', '"'), "%Y-%m-%dT%H:%M:%S.%fZ") - url = "{}/title/{}".format(self.root, href) + url = f"{self.root}/title/{href}" results.append((url, data.copy())) return results diff --git a/gallery_dl/extractor/bbc.py b/gallery_dl/extractor/bbc.py index 54aaac4e3..07d9a695d 100644 --- a/gallery_dl/extractor/bbc.py +++ b/gallery_dl/extractor/bbc.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,42 +6,42 @@ """Extractors for https://bbc.co.uk/""" -from .common import GalleryExtractor, Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import GalleryExtractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?bbc\.co\.uk(/programmes/" class BbcGalleryExtractor(GalleryExtractor): """Extractor for a programme gallery on bbc.co.uk""" + category = "bbc" root = "https://www.bbc.co.uk" - directory_fmt = ("{category}", "{path[0]}", "{path[1]}", "{path[2]}", - "{path[3:]:J - /}") + directory_fmt = ("{category}", "{path[0]}", "{path[1]}", "{path[2]}", "{path[3:]:J - /}") filename_fmt = "{num:>02}.{extension}" archive_fmt = "{programme}_{num}" pattern = BASE_PATTERN + r"[^/?#]+(?!/galleries)(?:/[^/?#]+)?)$" example = "https://www.bbc.co.uk/programmes/PATH" def metadata(self, page): - data = util.json_loads(text.extr( - page, '<script type="application/ld+json">', '</script>')) + data = util.json_loads(text.extr(page, '<script type="application/ld+json">', "</script>")) return { "programme": self.gallery_url.split("/")[4], - "path": list(util.unique_sequence( - element["name"] - for element in data["itemListElement"] - )), + "path": list( + util.unique_sequence(element["name"] for element in data["itemListElement"]) + ), } def images(self, page): width = self.config("width") width = width - width % 16 if width else 1920 - dimensions = "/{}xn/".format(width) + dimensions = f"/{width}xn/" return [ - (src.replace("/320x180_b/", dimensions), - {"_fallback": self._fallback_urls(src, width)}) + (src.replace("/320x180_b/", dimensions), {"_fallback": self._fallback_urls(src, width)}) for src in text.extract_iter(page, 'data-image-src="', '"') ] @@ -52,11 +50,12 @@ def _fallback_urls(src, max_width): front, _, back = src.partition("/320x180_b/") for width in (1920, 1600, 1280, 976): if width < max_width: - yield "{}/{}xn/{}".format(front, width, back) + yield f"{front}/{width}xn/{back}" class BbcProgrammeExtractor(Extractor): """Extractor for all galleries of a bbc programme""" + category = "bbc" subcategory = "programme" root = "https://www.bbc.co.uk" @@ -75,7 +74,8 @@ def items(self): while True: page = self.request(galleries_url, params=params).text for programme_id in text.extract_iter( - page, '<a href="https://www.bbc.co.uk/programmes/', '"'): + page, '<a href="https://www.bbc.co.uk/programmes/', '"' + ): url = "https://www.bbc.co.uk/programmes/" + programme_id yield Message.Queue, url, data if 'rel="next"' not in page: diff --git a/gallery_dl/extractor/behance.py b/gallery_dl/extractor/behance.py index 14598b7a0..3384e1a32 100644 --- a/gallery_dl/extractor/behance.py +++ b/gallery_dl/extractor/behance.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,16 @@ """Extractors for https://www.behance.net/""" -from .common import Extractor, Message -from .. import text, util, exception +from .. import exception +from .. import text +from .. import util +from .common import Extractor +from .common import Message class BehanceExtractor(Extractor): """Base class for behance extractors""" + category = "behance" root = "https://www.behance.net" request_interval = (2.0, 4.0) @@ -36,28 +38,23 @@ def _request_graphql(self, endpoint, variables): url = self.root + "/v3/graphql" headers = { "Origin": self.root, - "X-BCP" : self._bcp, + "X-BCP": self._bcp, "X-Requested-With": "XMLHttpRequest", } data = { - "query" : GRAPHQL_QUERIES[endpoint], + "query": GRAPHQL_QUERIES[endpoint], "variables": variables, } - return self.request(url, method="POST", headers=headers, - json=data).json()["data"] + return self.request(url, method="POST", headers=headers, json=data).json()["data"] def _update(self, data): # compress data to simple lists if data.get("fields") and isinstance(data["fields"][0], dict): - data["fields"] = [ - field.get("name") or field.get("label") - for field in data["fields"] - ] + data["fields"] = [field.get("name") or field.get("label") for field in data["fields"]] data["owners"] = [ - owner.get("display_name") or owner.get("displayName") - for owner in data["owners"] + owner.get("display_name") or owner.get("displayName") for owner in data["owners"] ] tags = data.get("tags") or () @@ -66,7 +63,8 @@ def _update(self, data): data["tags"] = tags data["date"] = text.parse_timestamp( - data.get("publishedOn") or data.get("conceived_on") or 0) + data.get("publishedOn") or data.get("conceived_on") or 0 + ) # backwards compatibility data["gallery_id"] = data["id"] @@ -78,6 +76,7 @@ def _update(self, data): class BehanceGalleryExtractor(BehanceExtractor): """Extractor for image galleries from www.behance.net""" + subcategory = "gallery" directory_fmt = ("{category}", "{owners:J, }", "{id} {name}") filename_fmt = "{category}_{id}_{num:>02}.{extension}" @@ -108,23 +107,21 @@ def items(self): yield Message.Directory, data for data["num"], (url, module) in enumerate(imgs, 1): data["module"] = module - data["extension"] = (module.get("extension") or - text.ext_from_url(url)) + data["extension"] = module.get("extension") or text.ext_from_url(url) yield Message.Url, url, data def get_gallery_data(self): """Collect gallery info dict""" - url = "{}/gallery/{}/a".format(self.root, self.gallery_id) + url = f"{self.root}/gallery/{self.gallery_id}/a" cookies = { "gki": '{"feature_project_view":false,' - '"feature_discover_login_prompt":false,' - '"feature_project_login_prompt":false}', + '"feature_discover_login_prompt":false,' + '"feature_project_login_prompt":false}', "ilo0": "true", } page = self.request(url, cookies=cookies).text - data = util.json_loads(text.extr( - page, 'id="beconfig-store_state">', '</script>')) + data = util.json_loads(text.extr(page, 'id="beconfig-store_state">', "</script>")) return self._update(data["project"]["project"]) def get_images(self, data): @@ -133,10 +130,10 @@ def get_images(self, data): access = data.get("matureAccess") if access == "logged-out": raise exception.AuthorizationError( - "Mature content galleries require logged-in cookies") + "Mature content galleries require logged-in cookies" + ) if access == "restricted-safe": - raise exception.AuthorizationError( - "Mature content blocked in account settings") + raise exception.AuthorizationError("Mature content blocked in account settings") if access and access != "allowed": raise exception.AuthorizationError() return () @@ -156,11 +153,13 @@ def get_images(self, data): size["url"].rsplit("/", 2)[1]: size for size in module["imageSizes"]["allAvailable"] } - size = (sizes.get("source") or - sizes.get("max_3840") or - sizes.get("fs") or - sizes.get("hd") or - sizes.get("disp")) + size = ( + sizes.get("source") + or sizes.get("max_3840") + or sizes.get("fs") + or sizes.get("hd") + or sizes.get("disp") + ) append((size["url"], module)) elif mtype == "video": @@ -181,15 +180,13 @@ def get_images(self, data): try: renditions = module["videoData"]["renditions"] except Exception: - self.log.warning("No download URLs for video %s", - module.get("id") or "???") + self.log.warning("No download URLs for video %s", module.get("id") or "???") continue try: - url = [ - r["url"] for r in renditions - if text.ext_from_url(r["url"]) != "m3u8" - ][-1] + url = [r["url"] for r in renditions if text.ext_from_url(r["url"]) != "m3u8"][ + -1 + ] except Exception as exc: self.log.debug("%s: %s", exc.__class__.__name__, exc) url = "ytdl:" + renditions[-1]["url"] @@ -221,6 +218,7 @@ def get_images(self, data): class BehanceUserExtractor(BehanceExtractor): """Extractor for a user's galleries from www.behance.net""" + subcategory = "user" categorytransfer = True pattern = r"(?:https?://)?(?:www\.)?behance\.net/([^/?#]+)/?$" @@ -234,7 +232,7 @@ def galleries(self): endpoint = "GetProfileProjects" variables = { "username": self.user, - "after" : "MAo=", # "0" in base64 + "after": "MAo=", # "0" in base64 } while True: @@ -249,6 +247,7 @@ def galleries(self): class BehanceCollectionExtractor(BehanceExtractor): """Extractor for a collection's galleries from www.behance.net""" + subcategory = "collection" categorytransfer = True pattern = r"(?:https?://)?(?:www\.)?behance\.net/collection/(\d+)" @@ -263,8 +262,8 @@ def galleries(self): variables = { "afterItem": "MAo=", # "0" in base64 "firstItem": 40, - "id" : int(self.collection_id), - "shouldGetItems" : True, + "id": int(self.collection_id), + "shouldGetItems": True, "shouldGetMoodboardFields": False, "shouldGetRecommendations": False, } @@ -437,7 +436,6 @@ def galleries(self): } } """, - "GetMoodboardItemsAndRecommendations": """\ query GetMoodboardItemsAndRecommendations( $id: Int! @@ -689,5 +687,4 @@ def galleries(self): } } """, - } diff --git a/gallery_dl/extractor/bilibili.py b/gallery_dl/extractor/bilibili.py index d5c419eb5..3594b2e41 100644 --- a/gallery_dl/extractor/bilibili.py +++ b/gallery_dl/extractor/bilibili.py @@ -1,17 +1,21 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://www.bilibili.com/""" -from .common import Extractor, Message -from .. import text, util, exception +from contextlib import suppress + +from .. import exception +from .. import text +from .. import util +from .common import Extractor +from .common import Message class BilibiliExtractor(Extractor): """Base class for bilibili extractors""" + category = "bilibili" root = "https://www.bilibili.com" request_interval = (3.0, 6.0) @@ -22,6 +26,7 @@ def _init(self): class BilibiliUserArticlesExtractor(BilibiliExtractor): """Extractor for a bilibili user's articles""" + subcategory = "user-articles" pattern = r"(?:https?://)?space\.bilibili\.com/(\d+)/article" example = "https://space.bilibili.com/12345/article" @@ -35,9 +40,9 @@ def items(self): class BilibiliArticleExtractor(BilibiliExtractor): """Extractor for a bilibili article""" + subcategory = "article" - pattern = (r"(?:https?://)?" - r"(?:t\.bilibili\.com|(?:www\.)?bilibili.com/opus)/(\d+)") + pattern = r"(?:https?://)?" r"(?:t\.bilibili\.com|(?:www\.)?bilibili.com/opus)/(\d+)" example = "https://www.bilibili.com/opus/12345" directory_fmt = ("{category}", "{username}") filename_fmt = "{id}_{num}.{extension}" @@ -49,21 +54,19 @@ def items(self): # Flatten modules list modules = {} for module in article["detail"]["modules"]: - del module['module_type'] + del module["module_type"] modules.update(module) article["detail"]["modules"] = modules article["username"] = modules["module_author"]["name"] pics = [] - for paragraph in modules['module_content']['paragraphs']: + for paragraph in modules["module_content"]["paragraphs"]: if "pic" not in paragraph: continue - try: + with suppress(Exception): pics.extend(paragraph["pic"]["pics"]) - except Exception: - pass article["count"] = len(pics) yield Message.Directory, article @@ -73,7 +76,7 @@ def items(self): yield Message.Url, url, text.nameext_from_url(url, article) -class BilibiliAPI(): +class BilibiliAPI: def __init__(self, extractor): self.extractor = extractor @@ -107,10 +110,10 @@ def article(self, article_id): while True: page = self.extractor.request(url).text try: - return util.json_loads(text.extr( - page, "window.__INITIAL_STATE__=", "};") + "}") + return util.json_loads(text.extr(page, "window.__INITIAL_STATE__=", "};") + "}") except Exception: if "window._riskdata_" not in page: raise exception.StopExtraction( - "%s: Unable to extract INITIAL_STATE data", article_id) + "%s: Unable to extract INITIAL_STATE data", article_id + ) self.extractor.wait(seconds=300) diff --git a/gallery_dl/extractor/blogger.py b/gallery_dl/extractor/blogger.py index 37075eaa3..980ca772a 100644 --- a/gallery_dl/extractor/blogger.py +++ b/gallery_dl/extractor/blogger.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,16 +6,19 @@ """Extractors for Blogger blogs""" -from .common import BaseExtractor, Message -from .. import text, util import re +from .. import text +from .. import util +from .common import BaseExtractor +from .common import Message + class BloggerExtractor(BaseExtractor): """Base class for blogger extractors""" + basecategory = "blogger" - directory_fmt = ("blogger", "{blog[name]}", - "{post[date]:%Y-%m-%d} {post[title]}") + directory_fmt = ("blogger", "{blog[name]}", "{post[date]:%Y-%m-%d} {post[title]}") filename_fmt = "{num:>03}.{extension}" archive_fmt = "{post[id]}_{num}" @@ -36,11 +37,13 @@ def items(self): sub = re.compile(r"(/|=)(?:[sw]\d+|w\d+-h\d+)(?=/|$)").sub findall_image = re.compile( r'src="(https?://(?:' - r'blogger\.googleusercontent\.com/img|' - r'lh\d+(?:-\w+)?\.googleusercontent\.com|' - r'\d+\.bp\.blogspot\.com)/[^"]+)').findall + r"blogger\.googleusercontent\.com/img|" + r"lh\d+(?:-\w+)?\.googleusercontent\.com|" + r'\d+\.bp\.blogspot\.com)/[^"]+)' + ).findall findall_video = re.compile( - r'src="(https?://www\.blogger\.com/video\.g\?token=[^"]+)').findall + r'src="(https?://www\.blogger\.com/video\.g\?token=[^"]+)' + ).findall metadata = self.metadata() for post in self.posts(blog): @@ -54,12 +57,13 @@ def items(self): page = self.request(post["url"]).text for url in findall_video(page): page = self.request(url).text - video_config = util.json_loads(text.extr( - page, 'var VIDEO_CONFIG =', '\n')) - files.append(max( - video_config["streams"], - key=lambda x: x["format_id"], - )["play_url"]) + video_config = util.json_loads(text.extr(page, "var VIDEO_CONFIG =", "\n")) + files.append( + max( + video_config["streams"], + key=lambda x: x["format_id"], + )["play_url"] + ) post["author"] = post["author"]["displayName"] post["replies"] = post["replies"]["totalItems"] @@ -84,16 +88,19 @@ def metadata(self): """Return additional metadata""" -BASE_PATTERN = BloggerExtractor.update({ - "blogspot": { - "root": None, - "pattern": r"[\w-]+\.blogspot\.com", - }, -}) +BASE_PATTERN = BloggerExtractor.update( + { + "blogspot": { + "root": None, + "pattern": r"[\w-]+\.blogspot\.com", + }, + } +) class BloggerPostExtractor(BloggerExtractor): """Extractor for a single blog post""" + subcategory = "post" pattern = BASE_PATTERN + r"(/\d\d\d\d/\d\d/[^/?#]+\.html)" example = "https://BLOG.blogspot.com/1970/01/TITLE.html" @@ -108,6 +115,7 @@ def posts(self, blog): class BloggerBlogExtractor(BloggerExtractor): """Extractor for an entire Blogger blog""" + subcategory = "blog" pattern = BASE_PATTERN + r"/?$" example = "https://BLOG.blogspot.com/" @@ -118,6 +126,7 @@ def posts(self, blog): class BloggerSearchExtractor(BloggerExtractor): """Extractor for Blogger search resuls""" + subcategory = "search" pattern = BASE_PATTERN + r"/search/?\?q=([^&#]+)" example = "https://BLOG.blogspot.com/search?q=QUERY" @@ -135,6 +144,7 @@ def metadata(self): class BloggerLabelExtractor(BloggerExtractor): """Extractor for Blogger posts by label""" + subcategory = "label" pattern = BASE_PATTERN + r"/search/label/([^/?#]+)" example = "https://BLOG.blogspot.com/search/label/LABEL" @@ -150,11 +160,12 @@ def metadata(self): return {"label": self.label} -class BloggerAPI(): +class BloggerAPI: """Minimal interface for the Blogger v3 API Ref: https://developers.google.com/blogger """ + API_KEY = "AIzaSyCN9ax34oMMyM07g_M-5pjeDp_312eITK8" def __init__(self, extractor): @@ -165,24 +176,23 @@ def blog_by_url(self, url): return self._call("blogs/byurl", {"url": url}, "blog") def blog_posts(self, blog_id, label=None): - endpoint = "blogs/{}/posts".format(blog_id) + endpoint = f"blogs/{blog_id}/posts" params = {"labels": label} return self._pagination(endpoint, params) def blog_search(self, blog_id, query): - endpoint = "blogs/{}/posts/search".format(blog_id) + endpoint = f"blogs/{blog_id}/posts/search" params = {"q": query} return self._pagination(endpoint, params) def post_by_path(self, blog_id, path): - endpoint = "blogs/{}/posts/bypath".format(blog_id) + endpoint = f"blogs/{blog_id}/posts/bypath" return self._call(endpoint, {"path": path}, "post") def _call(self, endpoint, params, notfound=None): url = "https://www.googleapis.com/blogger/v3/" + endpoint params["key"] = self.api_key - return self.extractor.request( - url, params=params, notfound=notfound).json() + return self.extractor.request(url, params=params, notfound=notfound).json() def _pagination(self, endpoint, params): while True: diff --git a/gallery_dl/extractor/bluesky.py b/gallery_dl/extractor/bluesky.py index 18a504732..2f977911d 100644 --- a/gallery_dl/extractor/bluesky.py +++ b/gallery_dl/extractor/bluesky.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,17 +6,21 @@ """Extractors for https://bsky.app/""" -from .common import Extractor, Message -from .. import text, util, exception -from ..cache import cache, memcache +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from ..cache import memcache +from .common import Extractor +from .common import Message -BASE_PATTERN = (r"(?:https?://)?" - r"(?:(?:www\.)?(?:c|[fv]x)?bs[ky]y[ex]?\.app|main\.bsky\.dev)") +BASE_PATTERN = r"(?:https?://)?" r"(?:(?:www\.)?(?:c|[fv]x)?bs[ky]y[ex]?\.app|main\.bsky\.dev)" USER_PATTERN = BASE_PATTERN + r"/profile/([^/?#]+)" class BlueskyExtractor(Extractor): """Base class for bluesky extractors""" + category = "bluesky" directory_fmt = ("{category}", "{author[handle]}") filename_fmt = "{createdAt[:19]}_{post_id}_{num}.{extension}" @@ -36,8 +38,8 @@ def _init(self): meta = meta.replace(" ", "").split(",") elif not isinstance(meta, (list, tuple)): meta = ("user", "facets") - self._metadata_user = ("user" in meta) - self._metadata_facets = ("facets" in meta) + self._metadata_user = "user" in meta + self._metadata_facets = "facets" in meta self.api = BlueskyAPI(self) self._user = self._user_did = None @@ -62,9 +64,7 @@ def items(self): yield Message.Directory, post if files: did = post["author"]["did"] - base = ( - "{}/xrpc/com.atproto.sync.getBlob?did={}&cid=".format( - self.api.get_service_endpoint(did), did)) + base = f"{self.api.get_service_endpoint(did)}/xrpc/com.atproto.sync.getBlob?did={did}&cid=" # noqa: E501 for post["num"], file in enumerate(files, 1): post.update(file) yield Message.Url, base + file["filename"], post @@ -116,8 +116,7 @@ def _prepare(self, post): post["instance"] = self.instance post["post_id"] = self._pid(post) - post["date"] = text.parse_datetime( - post["createdAt"][:19], "%Y-%m-%dT%H:%M:%S") + post["date"] = text.parse_datetime(post["createdAt"][:19], "%Y-%m-%dT%H:%M:%S") def _extract_files(self, post): if "embed" not in post: @@ -154,10 +153,10 @@ def _extract_media(self, media, key): return { "description": media.get("alt") or "", - "width" : width, - "height" : height, - "filename" : cid, - "extension" : data["mimeType"].rpartition("/")[2], + "width": width, + "height": height, + "filename": cid, + "extension": data["mimeType"].rpartition("/")[2], } def _make_post(self, actor, kind): @@ -168,27 +167,33 @@ def _make_post(self, actor, kind): return () cid = profile[kind].rpartition("/")[2].partition("@")[0] - return ({ - "post": { - "embed": {"images": [{ - "alt": kind, - "image": { - "$type" : "blob", - "ref" : {"$link": cid}, - "mimeType": "image/jpeg", - "size" : 0, + return ( + { + "post": { + "embed": { + "images": [ + { + "alt": kind, + "image": { + "$type": "blob", + "ref": {"$link": cid}, + "mimeType": "image/jpeg", + "size": 0, + }, + "aspectRatio": { + "width": 1000, + "height": 1000, + }, + } + ] }, - "aspectRatio": { - "width" : 1000, - "height": 1000, - }, - }]}, - "author" : profile, - "record" : (), - "createdAt": "", - "uri" : cid, + "author": profile, + "record": (), + "createdAt": "", + "uri": cid, + }, }, - },) + ) class BlueskyUserExtractor(BlueskyExtractor): @@ -200,15 +205,18 @@ def initialize(self): pass def items(self): - base = "{}/profile/{}/".format(self.root, self.user) - return self._dispatch_extractors(( - (BlueskyAvatarExtractor , base + "avatar"), - (BlueskyBackgroundExtractor, base + "banner"), - (BlueskyPostsExtractor , base + "posts"), - (BlueskyRepliesExtractor , base + "replies"), - (BlueskyMediaExtractor , base + "media"), - (BlueskyLikesExtractor , base + "likes"), - ), ("media",)) + base = f"{self.root}/profile/{self.user}/" + return self._dispatch_extractors( + ( + (BlueskyAvatarExtractor, base + "avatar"), + (BlueskyBackgroundExtractor, base + "banner"), + (BlueskyPostsExtractor, base + "posts"), + (BlueskyRepliesExtractor, base + "replies"), + (BlueskyMediaExtractor, base + "media"), + (BlueskyLikesExtractor, base + "likes"), + ), + ("media",), + ) class BlueskyPostsExtractor(BlueskyExtractor): @@ -333,10 +341,10 @@ class BlueskyHashtagExtractor(BlueskyExtractor): example = "https://bsky.app/hashtag/NAME" def posts(self): - return self.api.search_posts("#"+self.user, self.groups[1]) + return self.api.search_posts("#" + self.user, self.groups[1]) -class BlueskyAPI(): +class BlueskyAPI: """Interface for the Bluesky API https://www.docs.bsky.app/docs/category/http-reference @@ -365,17 +373,16 @@ def get_actor_likes(self, actor): def get_author_feed(self, actor, filter="posts_and_author_threads"): endpoint = "app.bsky.feed.getAuthorFeed" params = { - "actor" : self._did_from_actor(actor, True), + "actor": self._did_from_actor(actor, True), "filter": filter, - "limit" : "100", + "limit": "100", } return self._pagination(endpoint, params) def get_feed(self, actor, feed): endpoint = "app.bsky.feed.getFeed" params = { - "feed" : "at://{}/app.bsky.feed.generator/{}".format( - self._did_from_actor(actor), feed), + "feed": f"at://{self._did_from_actor(actor)}/app.bsky.feed.generator/{feed}", "limit": "100", } return self._pagination(endpoint, params) @@ -391,8 +398,7 @@ def get_follows(self, actor): def get_list_feed(self, actor, list): endpoint = "app.bsky.feed.getListFeed" params = { - "list" : "at://{}/app.bsky.graph.list/{}".format( - self._did_from_actor(actor), list), + "list": f"at://{self._did_from_actor(actor)}/app.bsky.graph.list/{list}", "limit": "100", } return self._pagination(endpoint, params) @@ -400,9 +406,8 @@ def get_list_feed(self, actor, list): def get_post_thread(self, actor, post_id): endpoint = "app.bsky.feed.getPostThread" params = { - "uri": "at://{}/app.bsky.feed.post/{}".format( - self._did_from_actor(actor), post_id), - "depth" : self.extractor.config("depth", "0"), + "uri": f"at://{self._did_from_actor(actor)}/app.bsky.feed.post/{post_id}", + "depth": self.extractor.config("depth", "0"), "parentHeight": "0", } @@ -433,9 +438,8 @@ def resolve_handle(self, handle): @memcache(keyarg=1) def get_service_endpoint(self, did): - if did.startswith('did:web:'): - url = "https://{}/.well-known/did.json".format( - did.rpartition(":")[2]) + if did.startswith("did:web:"): + url = "https://{}/.well-known/did.json".format(did.rpartition(":")[2]) else: url = "https://plc.directory/" + did @@ -451,21 +455,19 @@ def get_service_endpoint(self, did): def search_posts(self, query, sort=None): endpoint = "app.bsky.feed.searchPosts" params = { - "q" : query, + "q": query, "limit": "100", - "sort" : sort, + "sort": sort, } return self._pagination(endpoint, params, "posts") def _did_from_actor(self, actor, user_did=False): - if actor.startswith("did:"): - did = actor - else: - did = self.resolve_handle(actor) - + did = actor if actor.startswith("did:") else self.resolve_handle(actor) extr = self.extractor + if user_did and not extr.config("reposts", False): extr._user_did = did + if extr._metadata_user: extr._user = user = self.get_profile(did) user["instance"] = extr._instance(user["handle"]) @@ -490,29 +492,30 @@ def _authenticate_impl(self, username): headers = None data = { "identifier": username, - "password" : self.password, + "password": self.password, } - url = "{}/xrpc/{}".format(self.root, endpoint) + url = f"{self.root}/xrpc/{endpoint}" response = self.extractor.request( - url, method="POST", headers=headers, json=data, fatal=None) + url, method="POST", headers=headers, json=data, fatal=None + ) data = response.json() if response.status_code != 200: self.log.debug("Server response: %s", data) - raise exception.AuthenticationError('"{}: {}"'.format( - data.get("error"), data.get("message"))) + raise exception.AuthenticationError( + '"{}: {}"'.format(data.get("error"), data.get("message")) + ) _refresh_token_cache.update(self.username, data["refreshJwt"]) return "Bearer " + data["accessJwt"] def _call(self, endpoint, params): - url = "{}/xrpc/{}".format(self.root, endpoint) + url = f"{self.root}/xrpc/{endpoint}" while True: self.authenticate() - response = self.extractor.request( - url, params=params, headers=self.headers, fatal=None) + response = self.extractor.request(url, params=params, headers=self.headers, fatal=None) if response.status_code < 400: return response.json() @@ -523,11 +526,9 @@ def _call(self, endpoint, params): try: data = response.json() - msg = "API request failed ('{}: {}')".format( - data["error"], data["message"]) + msg = "API request failed ('{}: {}')".format(data["error"], data["message"]) except Exception: - msg = "API request failed ({} {})".format( - response.status_code, response.reason) + msg = f"API request failed ({response.status_code} {response.reason})" self.extractor.log.debug("Server response: %s", response.text) raise exception.StopExtraction(msg) @@ -543,6 +544,6 @@ def _pagination(self, endpoint, params, key="feed"): params["cursor"] = cursor -@cache(maxage=84*86400, keyarg=0) +@cache(maxage=84 * 86400, keyarg=0) def _refresh_token_cache(username): return None diff --git a/gallery_dl/extractor/booru.py b/gallery_dl/extractor/booru.py index 7e26f38b3..35a72327b 100644 --- a/gallery_dl/extractor/booru.py +++ b/gallery_dl/extractor/booru.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,16 @@ """Extractors for *booru sites""" -from .common import BaseExtractor, Message -from .. import text import operator +from .. import text +from .common import BaseExtractor +from .common import Message + class BooruExtractor(BaseExtractor): """Base class for *booru extractors""" + basecategory = "booru" filename_fmt = "{category}_{id}_{md5}.{extension}" page_start = 0 @@ -42,8 +43,11 @@ def items(self): url = self.root + url except Exception as exc: self.log.debug("%s: %s", exc.__class__.__name__, exc) - self.log.warning("Unable to fetch download URL for post %s " - "(md5: %s)", post.get("id"), post.get("md5")) + self.log.warning( + "Unable to fetch download URL for post %s " "(md5: %s)", + post.get("id"), + post.get("md5"), + ) continue if fetch_html: diff --git a/gallery_dl/extractor/boosty.py b/gallery_dl/extractor/boosty.py index 33823be15..1d80e2b42 100644 --- a/gallery_dl/extractor/boosty.py +++ b/gallery_dl/extractor/boosty.py @@ -1,23 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://www.boosty.to/""" -from .common import Extractor, Message -from .. import text, util, exception +from .. import exception +from .. import text +from .. import util +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?boosty\.to" class BoostyExtractor(Extractor): """Base class for boosty extractors""" + category = "boosty" root = "https://www.boosty.to" - directory_fmt = ("{category}", "{user[blogUrl]} ({user[id]})", - "{post[date]:%Y-%m-%d} {post[int_id]}") + directory_fmt = ( + "{category}", + "{user[blogUrl]} ({user[id]})", + "{post[date]:%Y-%m-%d} {post[int_id]}", + ) filename_fmt = "{num:>02} {file[id]}.{extension}" archive_fmt = "{file[id]}" cookies_domain = ".boosty.to" @@ -43,8 +48,16 @@ def _init(self): # low: 360p # lowest: 240p # tiny: 144p - videos = ("ultra_hd", "quad_hd", "full_hd", - "high", "medium", "low", "lowest", "tiny") + videos = ( + "ultra_hd", + "quad_hd", + "full_hd", + "high", + "medium", + "low", + "lowest", + "tiny", + ) self.videos = videos def items(self): @@ -55,8 +68,8 @@ def items(self): files = self._process_post(post) data = { - "post" : post, - "user" : post.pop("user", None), + "post": post, + "user": post.pop("user", None), "count": len(files), } @@ -93,19 +106,10 @@ def _process_post(self, post): elif type == "ok_video": if not self.videos: - self.log.debug("%s: Skipping video %s", - post["int_id"], block["id"]) + self.log.debug("%s: Skipping video %s", post["int_id"], block["id"]) continue - fmts = { - fmt["type"]: fmt["url"] - for fmt in block["playerUrls"] - if fmt["url"] - } - formats = [ - fmts[fmt] - for fmt in self.videos - if fmt in fmts - ] + fmts = {fmt["type"]: fmt["url"] for fmt in block["playerUrls"] if fmt["url"]} + formats = [fmts[fmt] for fmt in self.videos if fmt in fmts] if formats: formats = iter(formats) block["url"] = next(formats) @@ -113,8 +117,8 @@ def _process_post(self, post): files.append(block) else: self.log.warning( - "%s: Found no suitable video format for %s", - post["int_id"], block["id"]) + "%s: Found no suitable video format for %s", post["int_id"], block["id"] + ) elif type == "link": url = block["url"] @@ -125,8 +129,7 @@ def _process_post(self, post): files.append(self._update_url(post, block)) else: - self.log.debug("%s: Unsupported data type '%s'", - post["int_id"], type) + self.log.debug("%s: Unsupported data type '%s'", post["int_id"], type) except Exception as exc: self.log.debug("%s: %s", exc.__class__.__name__, exc) @@ -152,6 +155,7 @@ def _update_url(self, post, block): class BoostyUserExtractor(BoostyExtractor): """Extractor for boosty.to user profiles""" + subcategory = "user" pattern = BASE_PATTERN + r"/([^/?#]+)(?:\?([^#]+))?$" example = "https://boosty.to/USER" @@ -166,6 +170,7 @@ def posts(self): class BoostyMediaExtractor(BoostyExtractor): """Extractor for boosty.to user media""" + subcategory = "media" directory_fmt = "{category}", "{user[blogUrl]} ({user[id]})", "media" filename_fmt = "{post[id]}_{num}.{extension}" @@ -181,6 +186,7 @@ def posts(self): class BoostyFeedExtractor(BoostyExtractor): """Extractor for your boosty.to subscription feed""" + subcategory = "feed" pattern = BASE_PATTERN + r"/(?:\?([^#]+))?(?:$|#)" example = "https://boosty.to/" @@ -192,6 +198,7 @@ def posts(self): class BoostyPostExtractor(BoostyExtractor): """Extractor for boosty.to posts""" + subcategory = "post" pattern = BASE_PATTERN + r"/([^/?#]+)/posts/([0-9a-f-]+)" example = "https://boosty.to/USER/posts/01234567-89ab-cdef-0123-456789abcd" @@ -205,6 +212,7 @@ def posts(self): class BoostyFollowingExtractor(BoostyExtractor): """Extractor for your boosty.to subscribed users""" + subcategory = "following" pattern = BASE_PATTERN + r"/app/settings/subscriptions" example = "https://boosty.to/app/settings/subscriptions" @@ -216,8 +224,9 @@ def items(self): yield Message.Queue, url, user -class BoostyAPI(): +class BoostyAPI: """Interface for the Boosty API""" + root = "https://api.boosty.to" def __init__(self, extractor, access_token=None): @@ -230,29 +239,34 @@ def __init__(self, extractor, access_token=None): if not access_token: auth = self.extractor.cookies.get("auth", domain=".boosty.to") if auth: - access_token = text.extr( - auth, "%22accessToken%22%3A%22", "%22") + access_token = text.extr(auth, "%22accessToken%22%3A%22", "%22") if access_token: self.headers["Authorization"] = "Bearer " + access_token def blog_posts(self, username, params): - endpoint = "/v1/blog/{}/post/".format(username) - params = self._merge_params(params, { - "limit" : "5", - "offset" : None, - "comments_limit": "2", - "reply_limit" : "1", - }) + endpoint = f"/v1/blog/{username}/post/" + params = self._merge_params( + params, + { + "limit": "5", + "offset": None, + "comments_limit": "2", + "reply_limit": "1", + }, + ) return self._pagination(endpoint, params) def blog_media_album(self, username, type="all", params=()): - endpoint = "/v1/blog/{}/media_album/".format(username) - params = self._merge_params(params, { - "type" : type.rstrip("s"), - "limit" : "15", - "limit_by": "media", - "offset" : None, - }) + endpoint = f"/v1/blog/{username}/media_album/" + params = self._merge_params( + params, + { + "type": type.rstrip("s"), + "limit": "15", + "limit_by": "media", + "offset": None, + }, + ) return self._pagination(endpoint, params, self._transform_media_posts) def _transform_media_posts(self, data): @@ -266,16 +280,19 @@ def _transform_media_posts(self, data): return posts def post(self, username, post_id): - endpoint = "/v1/blog/{}/post/{}".format(username, post_id) + endpoint = f"/v1/blog/{username}/post/{post_id}" return self._call(endpoint) def feed_posts(self, params=None): endpoint = "/v1/feed/post/" - params = self._merge_params(params, { - "limit" : "5", - "offset" : None, - "comments_limit": "2", - }) + params = self._merge_params( + params, + { + "limit": "5", + "offset": None, + "comments_limit": "2", + }, + ) if "only_allowed" not in params and self.extractor.only_allowed: params["only_allowed"] = "true" if "only_bought" not in params and self.extractor.only_bought: @@ -290,20 +307,23 @@ def user(self, username): def user_subscriptions(self, params=None): endpoint = "/v1/user/subscriptions" - params = self._merge_params(params, { - "limit" : "30", - "with_follow": "true", - "offset" : None, - }) + params = self._merge_params( + params, + { + "limit": "30", + "with_follow": "true", + "offset": None, + }, + ) return self._pagination_users(endpoint, params) def _merge_params(self, params_web, params_api): if params_web: web_to_api = { "isOnlyAllowedPosts": "is_only_allowed", - "postsTagsIds" : "tags_ids", - "postsFrom" : "from_ts", - "postsTo" : "to_ts", + "postsTagsIds": "tags_ids", + "postsFrom": "from_ts", + "postsTo": "to_ts", } for name, value in params_web.items(): name = web_to_api.get(name, name) @@ -315,16 +335,16 @@ def _call(self, endpoint, params=None): while True: response = self.extractor.request( - url, params=params, headers=self.headers, - fatal=None, allow_redirects=False) + url, params=params, headers=self.headers, fatal=None, allow_redirects=False + ) if response.status_code < 300: return response.json() - elif response.status_code < 400: + if response.status_code < 400: raise exception.AuthenticationError("Invalid API access token") - elif response.status_code == 429: + if response.status_code == 429: self.extractor.wait(seconds=600) else: diff --git a/gallery_dl/extractor/bunkr.py b/gallery_dl/extractor/bunkr.py index 3e1245254..961f92075 100644 --- a/gallery_dl/extractor/bunkr.py +++ b/gallery_dl/extractor/bunkr.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,16 +6,16 @@ """Extractors for https://bunkr.si/""" +import random + +from .. import config +from .. import exception +from .. import text from .common import Extractor from .lolisafe import LolisafeAlbumExtractor -from .. import text, config, exception -import random if config.get(("extractor", "bunkr"), "tlds"): - BASE_PATTERN = ( - r"(?:bunkr:(?:https?://)?([^/?#]+)|" - r"(?:https?://)?(?:app\.)?(bunkr+\.\w+))" - ) + BASE_PATTERN = r"(?:bunkr:(?:https?://)?([^/?#]+)|" r"(?:https?://)?(?:app\.)?(bunkr+\.\w+))" else: BASE_PATTERN = ( r"(?:bunkr:(?:https?://)?([^/?#]+)|" @@ -58,6 +56,7 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): """Extractor for bunkr.si albums""" + category = "bunkr" root = "https://bunkr.si" pattern = BASE_PATTERN + r"/a/([^/?#]+)" @@ -83,8 +82,7 @@ def request(self, url, **kwargs): root, path = self._split(url) if root not in CF_DOMAINS: continue - self.log.debug("Redirect to known CF challenge domain '%s'", - root) + self.log.debug("Redirect to known CF challenge domain '%s'", root) except exception.HttpError as exc: if exc.status != 403: @@ -102,7 +100,8 @@ def request(self, url, **kwargs): else: if not DOMAINS: raise exception.StopExtraction( - "All Bunkr domains require solving a CF challenge") + "All Bunkr domains require solving a CF challenge" + ) # select alternative domain root = "https://" + random.choice(DOMAINS) @@ -112,19 +111,17 @@ def request(self, url, **kwargs): def fetch_album(self, album_id): # album metadata page = self.request(self.root + "/a/" + album_id).text - title, size = text.split_html(text.extr( - page, "<h1", "</span>").partition(">")[2]) + title, size = text.split_html(text.extr(page, "<h1", "</span>").partition(">")[2]) if "&" in title: - title = title.replace( - "<", "<").replace(">", ">").replace("&", "&") + title = title.replace("<", "<").replace(">", ">").replace("&", "&") # files items = list(text.extract_iter(page, "<!-- item -->", "<!-- -->")) return self._extract_files(items), { - "album_id" : album_id, - "album_name" : title, - "album_size" : text.extr(size, "(", ")"), - "count" : len(items), + "album_id": album_id, + "album_name": title, + "album_size": text.extr(size, "(", ")"), + "count": len(items), } def _extract_files(self, items): @@ -136,8 +133,7 @@ def _extract_files(self, items): info = text.split_html(item) file["name"] = info[0] file["size"] = info[2] - file["date"] = text.parse_datetime( - info[-1], "%H:%M:%S %d/%m/%Y") + file["date"] = text.parse_datetime(info[-1], "%H:%M:%S %d/%m/%Y") yield file except exception.StopExtraction: @@ -149,18 +145,18 @@ def _extract_files(self, items): def _extract_file(self, webpage_url): response = self.request(webpage_url) page = response.text - file_url = (text.extr(page, '<source src="', '"') or - text.extr(page, '<img src="', '"')) + file_url = text.extr(page, '<source src="', '"') or text.extr(page, '<img src="', '"') if not file_url: - webpage_url = text.unescape(text.rextract( - page, ' href="', '"', page.rindex("Download"))[0]) + webpage_url = text.unescape( + text.rextract(page, ' href="', '"', page.rindex("Download"))[0] + ) response = self.request(webpage_url) file_url = text.rextract(response.text, ' href="', '"')[0] return { - "file" : text.unescape(file_url), - "_http_headers" : {"Referer": response.url}, + "file": text.unescape(file_url), + "_http_headers": {"Referer": response.url}, "_http_validate": self._validate, } @@ -177,6 +173,7 @@ def _split(self, url): class BunkrMediaExtractor(BunkrAlbumExtractor): """Extractor for bunkr.si media links""" + subcategory = "media" directory_fmt = ("{category}",) pattern = BASE_PATTERN + r"(/[vid]/[^/?#]+)" @@ -190,9 +187,9 @@ def fetch_album(self, album_id): return (), {} return (file,), { - "album_id" : "", - "album_name" : "", - "album_size" : -1, + "album_id": "", + "album_name": "", + "album_size": -1, "description": "", - "count" : 1, + "count": 1, } diff --git a/gallery_dl/extractor/catbox.py b/gallery_dl/extractor/catbox.py index 6c81f5394..0f3890000 100644 --- a/gallery_dl/extractor/catbox.py +++ b/gallery_dl/extractor/catbox.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,15 @@ """Extractors for https://catbox.moe/""" -from .common import GalleryExtractor, Extractor, Message from .. import text +from .common import Extractor +from .common import GalleryExtractor +from .common import Message class CatboxAlbumExtractor(GalleryExtractor): """Extractor for catbox albums""" + category = "catbox" subcategory = "album" root = "https://catbox.moe" @@ -26,23 +27,22 @@ class CatboxAlbumExtractor(GalleryExtractor): def metadata(self, page): extr = text.extract_from(page) return { - "album_id" : self.gallery_url.rpartition("/")[2], - "album_name" : text.unescape(extr("<h1>", "<")), - "date" : text.parse_datetime(extr( - "<p>Created ", "<"), "%B %d %Y"), + "album_id": self.gallery_url.rpartition("/")[2], + "album_name": text.unescape(extr("<h1>", "<")), + "date": text.parse_datetime(extr("<p>Created ", "<"), "%B %d %Y"), "description": text.unescape(extr("<p>", "<")), } def images(self, page): return [ ("https://files.catbox.moe/" + path, None) - for path in text.extract_iter( - page, ">https://files.catbox.moe/", "<") + for path in text.extract_iter(page, ">https://files.catbox.moe/", "<") ] class CatboxFileExtractor(Extractor): """Extractor for catbox files""" + category = "catbox" subcategory = "file" archive_fmt = "{filename}" diff --git a/gallery_dl/extractor/chevereto.py b/gallery_dl/extractor/chevereto.py index aedcea457..67e193988 100644 --- a/gallery_dl/extractor/chevereto.py +++ b/gallery_dl/extractor/chevereto.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,14 +6,20 @@ """Extractors for Chevereto galleries""" -from .common import BaseExtractor, Message from .. import text +from .common import BaseExtractor +from .common import Message class CheveretoExtractor(BaseExtractor): """Base class for chevereto extractors""" + basecategory = "chevereto" - directory_fmt = ("{category}", "{user}", "{album}",) + directory_fmt = ( + "{category}", + "{user}", + "{album}", + ) archive_fmt = "{id}" def __init__(self, match): @@ -26,27 +30,29 @@ def _pagination(self, url): while url: page = self.request(url).text - for item in text.extract_iter( - page, '<div class="list-item-image ', 'image-container'): + for item in text.extract_iter(page, '<div class="list-item-image ', "image-container"): yield text.extr(item, '<a href="', '"') url = text.extr(page, '<a data-pagination="next" href="', '" ><') -BASE_PATTERN = CheveretoExtractor.update({ - "jpgfish": { - "root": "https://jpg5.su", - "pattern": r"jpe?g\d?\.(?:su|pet|fish(?:ing)?|church)", - }, - "imgkiwi": { - "root": "https://img.kiwi", - "pattern": r"img\.kiwi", - }, -}) +BASE_PATTERN = CheveretoExtractor.update( + { + "jpgfish": { + "root": "https://jpg5.su", + "pattern": r"jpe?g\d?\.(?:su|pet|fish(?:ing)?|church)", + }, + "imgkiwi": { + "root": "https://img.kiwi", + "pattern": r"img\.kiwi", + }, + } +) class CheveretoImageExtractor(CheveretoExtractor): """Extractor for chevereto Images""" + subcategory = "image" pattern = BASE_PATTERN + r"(/im(?:g|age)/[^/?#]+)" example = "https://jpg2.su/img/TITLE.ID" @@ -56,10 +62,10 @@ def items(self): extr = text.extract_from(self.request(url).text) image = { - "id" : self.path.rpartition(".")[2], - "url" : extr('<meta property="og:image" content="', '"'), + "id": self.path.rpartition(".")[2], + "url": extr('<meta property="og:image" content="', '"'), "album": text.extr(extr("Added to <a", "/a>"), ">", "<"), - "user" : extr('username: "', '"'), + "user": extr('username: "', '"'), } text.nameext_from_url(image["url"], image) @@ -69,6 +75,7 @@ def items(self): class CheveretoAlbumExtractor(CheveretoExtractor): """Extractor for chevereto Albums""" + subcategory = "album" pattern = BASE_PATTERN + r"(/a(?:lbum)?/[^/?#]+(?:/sub)?)" example = "https://jpg2.su/album/TITLE.ID" @@ -77,10 +84,7 @@ def items(self): url = self.root + self.path data = {"_extractor": CheveretoImageExtractor} - if self.path.endswith("/sub"): - albums = self._pagination(url) - else: - albums = (url,) + albums = self._pagination(url) if self.path.endswith("/sub") else (url,) for album in albums: for image in self._pagination(album): @@ -89,6 +93,7 @@ def items(self): class CheveretoUserExtractor(CheveretoExtractor): """Extractor for chevereto Users""" + subcategory = "user" pattern = BASE_PATTERN + r"(/(?!img|image|a(?:lbum)?)[^/?#]+(?:/albums)?)" example = "https://jpg2.su/USER" diff --git a/gallery_dl/extractor/cien.py b/gallery_dl/extractor/cien.py index 378365eb7..83e680611 100644 --- a/gallery_dl/extractor/cien.py +++ b/gallery_dl/extractor/cien.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,8 +6,10 @@ """Extractors for https://ci-en.net/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?ci-en\.(?:net|dlsite\.com)" @@ -33,8 +33,7 @@ def _pagination_articles(self, url, params): while True: page = self.request(url, params=params).text - for card in text.extract_iter( - page, ' class="c-cardCase-item', '</div>'): + for card in text.extract_iter(page, ' class="c-cardCase-item', "</div>"): article_url = text.extr(card, ' href="', '"') yield Message.Queue, article_url, data @@ -52,12 +51,12 @@ class CienArticleExtractor(CienExtractor): example = "https://ci-en.net/creator/123/article/12345" def items(self): - url = "{}/creator/{}/article/{}".format( - self.root, self.groups[0], self.groups[1]) + url = f"{self.root}/creator/{self.groups[0]}/article/{self.groups[1]}" page = self.request(url, notfound="article").text - post = util.json_loads(text.extr( - page, '<script type="application/ld+json">', '</script>'))[0] + post = util.json_loads(text.extr(page, '<script type="application/ld+json">', "</script>"))[ + 0 + ] files = self._extract_files(page) @@ -90,10 +89,10 @@ def _extract_files(self, page): self._extract_files_gallery(page, files) else: generators = { - "image" : self._extract_files_image, - "video" : self._extract_files_video, + "image": self._extract_files_image, + "video": self._extract_files_video, "download": self._extract_files_download, - "gallery" : self._extract_files_gallery, + "gallery": self._extract_files_gallery, "gallerie": self._extract_files_gallery, } if isinstance(filetypes, str): @@ -104,33 +103,32 @@ def _extract_files(self, page): return files def _extract_files_image(self, page, files): - for image in text.extract_iter( - page, 'class="file-player-image"', "</figure>"): + for image in text.extract_iter(page, 'class="file-player-image"', "</figure>"): size = text.extr(image, ' data-size="', '"') w, _, h = size.partition("x") - files.append({ - "url" : text.extr(image, ' data-raw="', '"'), - "width" : text.parse_int(w), - "height": text.parse_int(h), - "type" : "image", - }) + files.append( + { + "url": text.extr(image, ' data-raw="', '"'), + "width": text.parse_int(w), + "height": text.parse_int(h), + "type": "image", + } + ) def _extract_files_video(self, page, files): - for video in text.extract_iter( - page, "<vue-file-player", "</vue-file-player>"): + for video in text.extract_iter(page, "<vue-file-player", "</vue-file-player>"): path = text.extr(video, ' base-path="', '"') name = text.extr(video, ' file-name="', '"') auth = text.extr(video, ' auth-key="', '"') file = text.nameext_from_url(name) - file["url"] = "{}video-web.mp4?{}".format(path, auth) + file["url"] = f"{path}video-web.mp4?{auth}" file["type"] = "video" files.append(file) def _extract_files_download(self, page, files): - for download in text.extract_iter( - page, 'class="downloadBlock', "</div>"): + for download in text.extract_iter(page, 'class="downloadBlock', "</div>"): name = text.extr(download, "<p>", "<") file = text.nameext_from_url(name.rpartition(" ")[0]) @@ -139,20 +137,17 @@ def _extract_files_download(self, page, files): files.append(file) def _extract_files_gallery(self, page, files): - for gallery in text.extract_iter( - page, "<vue-image-gallery", "</vue-image-gallery>"): - + for gallery in text.extract_iter(page, "<vue-image-gallery", "</vue-image-gallery>"): url = self.root + "/api/creator/gallery/images" params = { - "hash" : text.extr(gallery, ' hash="', '"'), + "hash": text.extr(gallery, ' hash="', '"'), "gallery_id": text.extr(gallery, ' gallery-id="', '"'), - "time" : text.extr(gallery, ' time="', '"'), + "time": text.extr(gallery, ' time="', '"'), } data = self.request(url, params=params).json() url = self.root + "/api/creator/gallery/imagePath" - for params["page"], params["file_id"] in enumerate( - data["imgList"]): + for params["page"], params["file_id"] in enumerate(data["imgList"]): path = self.request(url, params=params).json()["path"] file = params.copy() @@ -166,7 +161,7 @@ class CienCreatorExtractor(CienExtractor): example = "https://ci-en.net/creator/123" def items(self): - url = "{}/creator/{}/article".format(self.root, self.groups[0]) + url = f"{self.root}/creator/{self.groups[0]}/article" params = text.parse_query(self.groups[1]) params["mode"] = "list" return self._pagination_articles(url, params) @@ -193,7 +188,6 @@ def items(self): page = self.request(url).text data = {"_extractor": CienCreatorExtractor} - for subscription in text.extract_iter( - page, 'class="c-grid-subscriptionInfo', '</figure>'): + for subscription in text.extract_iter(page, 'class="c-grid-subscriptionInfo', "</figure>"): url = text.extr(subscription, ' href="', '"') yield Message.Queue, url, data diff --git a/gallery_dl/extractor/civitai.py b/gallery_dl/extractor/civitai.py index 1e8cb4241..4da5964eb 100644 --- a/gallery_dl/extractor/civitai.py +++ b/gallery_dl/extractor/civitai.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,17 +6,22 @@ """Extractors for https://www.civitai.com/""" -from .common import Extractor, Message -from .. import text, util, exception import itertools import time +from .. import exception +from .. import text +from .. import util +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?civitai\.com" USER_PATTERN = BASE_PATTERN + r"/user/([^/?#]+)" class CivitaiExtractor(Extractor): """Base class for civitai extractors""" + category = "civitai" root = "https://civitai.com" directory_fmt = ("{category}", "{username|user[username]}", "images") @@ -39,7 +42,7 @@ def _init(self): if not isinstance(quality, str): quality = ",".join(quality) self._image_quality = quality - self._image_ext = ("png" if quality == "original=true" else "jpg") + self._image_ext = "png" if quality == "original=true" else "jpg" else: self._image_quality = "original=true" self._image_ext = "png" @@ -50,7 +53,7 @@ def _init(self): metadata = metadata.split(",") elif not isinstance(metadata, (list, tuple)): metadata = ("generation",) - self._meta_generation = ("generation" in metadata) + self._meta_generation = "generation" in metadata else: self._meta_generation = False @@ -66,15 +69,10 @@ def items(self): posts = self.posts() if posts: for post in posts: - - if "images" in post: - images = post["images"] - else: - images = self.api.images_post(post["id"]) + images = post.get("images", self.api.images_post(post["id"])) post = self.api.post(post["id"]) - post["date"] = text.parse_datetime( - post["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date"] = text.parse_datetime(post["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") data = { "post": post, "user": post["user"], @@ -92,10 +90,8 @@ def items(self): for image in images: url = self._url(image) if self._meta_generation: - image["generation"] = self.api.image_generationdata( - image["id"]) - image["date"] = text.parse_datetime( - image["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + image["generation"] = self.api.image_generationdata(image["id"]) + image["date"] = text.parse_datetime(image["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") text.nameext_from_url(url, image) image["extension"] = self._image_ext yield Message.Directory, image @@ -125,17 +121,19 @@ def _url(self, image): mime = image.get("mimeType") or self._image_ext name = "{}.{}".format(image.get("id"), mime.rpartition("/")[2]) return ( - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/{}/{}/{}".format( - url, self._image_quality, name) + f"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/{url}/{self._image_quality}/{name}" ) def _image_results(self, images): for num, file in enumerate(images, 1): - data = text.nameext_from_url(file["url"], { - "num" : num, - "file": file, - "url" : self._url(file), - }) + data = text.nameext_from_url( + file["url"], + { + "num": num, + "file": file, + "url": self._url(file), + }, + ) if not data["extension"]: data["extension"] = self._image_ext if "id" not in file and data["filename"].isdecimal(): @@ -147,9 +145,12 @@ def _image_results(self, images): class CivitaiModelExtractor(CivitaiExtractor): subcategory = "model" - directory_fmt = ("{category}", "{user[username]}", - "{model[id]}{model[name]:? //}", - "{version[id]}{version[name]:? //}") + directory_fmt = ( + "{category}", + "{user[username]}", + "{model[id]}{model[name]:? //}", + "{version[id]}{version[name]:? //}", + ) pattern = BASE_PATTERN + r"/models/(\d+)(?:/?\?modelVersionId=(\d+))?" example = "https://civitai.com/models/12345/TITLE" @@ -176,13 +177,12 @@ def items(self): versions = (version,) for version in versions: - version["date"] = text.parse_datetime( - version["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + version["date"] = text.parse_datetime(version["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") data = { - "model" : model, + "model": model, "version": version, - "user" : user, + "user": user, } yield Message.Directory, data @@ -196,37 +196,35 @@ def _extract_files(self, model, version, user): return self._extract_files_image(model, version, user) generators = { - "model" : self._extract_files_model, - "image" : self._extract_files_image, - "gallery" : self._extract_files_gallery, + "model": self._extract_files_model, + "image": self._extract_files_image, + "gallery": self._extract_files_gallery, "gallerie": self._extract_files_gallery, } if isinstance(filetypes, str): filetypes = filetypes.split(",") return itertools.chain.from_iterable( - generators[ft.rstrip("s")](model, version, user) - for ft in filetypes + generators[ft.rstrip("s")](model, version, user) for ft in filetypes ) def _extract_files_model(self, model, version, user): files = [] for num, file in enumerate(version["files"], 1): - file["uuid"] = "model-{}-{}-{}".format( - model["id"], version["id"], file["id"]) - files.append({ - "num" : num, - "file" : file, - "filename" : file["name"], - "extension": "bin", - "url" : file.get("downloadUrl") or - "{}/api/download/models/{}".format( - self.root, version["id"]), - "_http_headers" : { - "Authorization": self.api.headers.get("Authorization")}, - "_http_validate": self._validate_file_model, - }) + file["uuid"] = "model-{}-{}-{}".format(model["id"], version["id"], file["id"]) + files.append( + { + "num": num, + "file": file, + "filename": file["name"], + "extension": "bin", + "url": file.get("downloadUrl") + or "{}/api/download/models/{}".format(self.root, version["id"]), + "_http_headers": {"Authorization": self.api.headers.get("Authorization")}, + "_http_validate": self._validate_file_model, + } + ) return files @@ -252,11 +250,9 @@ def _extract_files_gallery(self, model, version, user): def _validate_file_model(self, response): if response.headers.get("Content-Type", "").startswith("text/html"): - alert = text.extr( - response.text, 'mantine-Alert-message">', "</div></div></div>") + alert = text.extr(response.text, 'mantine-Alert-message">', "</div></div></div>") if alert: - msg = "\"{}\" - 'api-key' required".format( - text.remove_html(alert)) + msg = f"\"{text.remove_html(alert)}\" - 'api-key' required" else: msg = "'api-key' required to download this file" self.log.warning(msg) @@ -275,8 +271,12 @@ def images(self): class CivitaiPostExtractor(CivitaiExtractor): subcategory = "post" - directory_fmt = ("{category}", "{username|user[username]}", "posts", - "{post[id]}{post[title]:? //}") + directory_fmt = ( + "{category}", + "{username|user[username]}", + "posts", + "{post[id]}{post[title]:? //}", + ) pattern = BASE_PATTERN + r"/posts/(\d+)" example = "https://civitai.com/posts/12345" @@ -333,12 +333,15 @@ def initialize(self): pass def items(self): - base = "{}/user/{}/".format(self.root, self.groups[0]) - return self._dispatch_extractors(( - (CivitaiUserModelsExtractor, base + "models"), - (CivitaiUserPostsExtractor , base + "posts"), - (CivitaiUserImagesExtractor, base + "images"), - ), ("user-models", "user-posts")) + base = f"{self.root}/user/{self.groups[0]}/" + return self._dispatch_extractors( + ( + (CivitaiUserModelsExtractor, base + "models"), + (CivitaiUserPostsExtractor, base + "posts"), + (CivitaiUserImagesExtractor, base + "images"), + ), + ("user-models", "user-posts"), + ) class CivitaiUserModelsExtractor(CivitaiExtractor): @@ -354,8 +357,12 @@ def models(self): class CivitaiUserPostsExtractor(CivitaiExtractor): subcategory = "user-posts" - directory_fmt = ("{category}", "{username|user[username]}", "posts", - "{post[id]}{post[title]:? //}") + directory_fmt = ( + "{category}", + "{username|user[username]}", + "posts", + "{post[id]}{post[title]:? //}", + ) pattern = USER_PATTERN + r"/posts/?(?:\?([^#]+))?" example = "https://civitai.com/user/USER/posts" @@ -383,9 +390,9 @@ def images(self): return self.api.images(params) def images_reactions(self): - if "Authorization" not in self.api.headers and \ - not self.cookies.get( - "__Secure-civitai-token", domain=".civitai.com"): + if "Authorization" not in self.api.headers and not self.cookies.get( + "__Secure-civitai-token", domain=".civitai.com" + ): raise exception.AuthorizationError("api-key or cookies required") params = self.params @@ -395,12 +402,11 @@ def images_reactions(self): if isinstance(params["reactions"], str): params["reactions"] = (params["reactions"],) else: - params["reactions"] = ( - "Like", "Dislike", "Heart", "Laugh", "Cry") + params["reactions"] = ("Like", "Dislike", "Heart", "Laugh", "Cry") return self.api.images(params) -class CivitaiRestAPI(): +class CivitaiRestAPI: """Interface for the Civitai Public REST API https://developer.civitai.com/docs/api/public-rest @@ -424,9 +430,11 @@ def __init__(self, extractor): self.nsfw = nsfw def image(self, image_id): - return self.images({ - "imageId": image_id, - }) + return self.images( + { + "imageId": image_id, + } + ) def images(self, params): endpoint = "/v1/images" @@ -435,17 +443,19 @@ def images(self, params): return self._pagination(endpoint, params) def images_gallery(self, model, version, user): - return self.images({ - "modelId" : model["id"], - "modelVersionId": version["id"], - }) + return self.images( + { + "modelId": model["id"], + "modelVersionId": version["id"], + } + ) def model(self, model_id): - endpoint = "/v1/models/{}".format(model_id) + endpoint = f"/v1/models/{model_id}" return self._call(endpoint) def model_version(self, model_version_id): - endpoint = "/v1/model-versions/{}".format(model_version_id) + endpoint = f"/v1/model-versions/{model_version_id}" return self._call(endpoint) def models(self, params): @@ -455,13 +465,9 @@ def models_tag(self, tag): return self.models({"tag": tag}) def _call(self, endpoint, params=None): - if endpoint[0] == "/": - url = self.root + endpoint - else: - url = endpoint + url = self.root + endpoint if endpoint[0] == "/" else endpoint - response = self.extractor.request( - url, params=params, headers=self.headers) + response = self.extractor.request(url, params=params, headers=self.headers) return response.json() def _pagination(self, endpoint, params): @@ -476,18 +482,18 @@ def _pagination(self, endpoint, params): params = None -class CivitaiTrpcAPI(): +class CivitaiTrpcAPI: """Interface for the Civitai tRPC API""" def __init__(self, extractor): self.extractor = extractor self.root = extractor.root + "/api/trpc/" self.headers = { - "content-type" : "application/json", + "content-type": "application/json", "x-client-version": "5.0.211", - "x-client-date" : "", - "x-client" : "web", - "x-fingerprint" : "undefined", + "x-client-date": "", + "x-client": "web", + "x-fingerprint": "undefined", } api_key = extractor.config("api-key") if api_key: @@ -515,16 +521,19 @@ def images(self, params, defaults=True): endpoint = "image.getInfinite" if defaults: - params = self._merge_params(params, { - "useIndex" : True, - "period" : "AllTime", - "sort" : "Newest", - "types" : ["image"], - "withMeta" : False, # Metadata Only - "fromPlatform" : False, # Made On-Site - "browsingLevel": self.nsfw, - "include" : ["cosmetics"], - }) + params = self._merge_params( + params, + { + "useIndex": True, + "period": "AllTime", + "sort": "Newest", + "types": ["image"], + "withMeta": False, # Metadata Only + "fromPlatform": False, # Made On-Site + "browsingLevel": self.nsfw, + "include": ["cosmetics"], + }, + ) params = self._type_params(params) return self._pagination(endpoint, params) @@ -532,13 +541,13 @@ def images(self, params, defaults=True): def images_gallery(self, model, version, user): endpoint = "image.getImagesAsPostsInfinite" params = { - "period" : "AllTime", - "sort" : "Newest", + "period": "AllTime", + "sort": "Newest", "modelVersionId": version["id"], - "modelId" : model["id"], - "hidden" : False, - "limit" : 50, - "browsingLevel" : self.nsfw, + "modelId": model["id"], + "hidden": False, + "limit": 50, + "browsingLevel": self.nsfw, } for post in self._pagination(endpoint, params): @@ -546,7 +555,7 @@ def images_gallery(self, model, version, user): def images_post(self, post_id): params = { - "postId" : int(post_id), + "postId": int(post_id), "pending": True, } return self.images(params) @@ -565,18 +574,21 @@ def models(self, params, defaults=True): endpoint = "model.getAll" if defaults: - params = self._merge_params(params, { - "period" : "AllTime", - "periodMode" : "published", - "sort" : "Newest", - "pending" : False, - "hidden" : False, - "followed" : False, - "earlyAccess" : False, - "fromPlatform" : False, - "supportsGeneration": False, - "browsingLevel": self.nsfw, - }) + params = self._merge_params( + params, + { + "period": "AllTime", + "periodMode": "published", + "sort": "Newest", + "pending": False, + "hidden": False, + "followed": False, + "earlyAccess": False, + "fromPlatform": False, + "supportsGeneration": False, + "browsingLevel": self.nsfw, + }, + ) return self._pagination(endpoint, params) @@ -593,16 +605,19 @@ def posts(self, params, defaults=True): meta = {"cursor": ("Date",)} if defaults: - params = self._merge_params(params, { - "browsingLevel": self.nsfw, - "period" : "AllTime", - "periodMode" : "published", - "sort" : "Newest", - "followed" : False, - "draftOnly" : False, - "pending" : True, - "include" : ["cosmetics"], - }) + params = self._merge_params( + params, + { + "browsingLevel": self.nsfw, + "period": "AllTime", + "periodMode": "published", + "sort": "Newest", + "followed": False, + "draftOnly": False, + "pending": True, + "include": ["cosmetics"], + }, + ) return self._pagination(endpoint, params, meta) @@ -615,12 +630,9 @@ def _call(self, endpoint, params, meta=None): url = self.root + endpoint headers = self.headers - if meta: - input = {"json": params, "meta": {"values": meta}} - else: - input = {"json": params} + input_ = {"json": params, "meta": {"values": meta}} if meta else {"json": params} - params = {"input": util.json_dumps(input)} + params = {"input": util.json_dumps(input_)} headers["x-client-date"] = str(int(time.time() * 1000)) response = self.extractor.request(url, params=params, headers=headers) @@ -650,8 +662,8 @@ def _merge_params(self, params_user, params_default): def _type_params(self, params): for key, type in ( - ("tags" , int), - ("modelId" , int), + ("tags", int), + ("modelId", int), ("modelVersionId", int), ): if key in params: diff --git a/gallery_dl/extractor/cohost.py b/gallery_dl/extractor/cohost.py index 0524239b4..69a223979 100644 --- a/gallery_dl/extractor/cohost.py +++ b/gallery_dl/extractor/cohost.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,18 +6,21 @@ """Extractors for https://cohost.org/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?cohost\.org" class CohostExtractor(Extractor): """Base class for cohost extractors""" + category = "cohost" root = "https://cohost.org" directory_fmt = ("{category}", "{postingProject[handle]}") - filename_fmt = ("{postId}_{headline:?/_/[b:200]}{num}.{extension}") + filename_fmt = "{postId}_{headline:?/_/[b:200]}{num}.{extension}" archive_fmt = "{postId}_{num}" def _init(self): @@ -33,14 +34,12 @@ def items(self): reason = post.get("limitedVisibilityReason") if reason and reason != "none": if reason == "log-in-first": - reason = ("This page's posts are visible only to users " - "who are logged in.") + reason = "This page's posts are visible only to users " "who are logged in." self.log.warning('%s: "%s"', post["postId"], reason) files = self._extract_files(post) post["count"] = len(files) - post["date"] = text.parse_datetime( - post["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date"] = text.parse_datetime(post["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") yield Message.Directory, post for post["num"], file in enumerate(files, 1): @@ -53,7 +52,7 @@ def posts(self): return () def _request_api(self, endpoint, input): - url = "{}/api/v1/trpc/{}".format(self.root, endpoint) + url = f"{self.root}/api/v1/trpc/{endpoint}" params = {"batch": "1", "input": util.json_dumps({"0": input})} headers = {"content-type": "application/json"} @@ -91,14 +90,14 @@ def _extract_blocks(self, post, files, shared=None): elif type == "ask": post["ask"] = block["ask"] else: - self.log.debug("%s: Unsupported block type '%s'", - post["postId"], type) + self.log.debug("%s: Unsupported block type '%s'", post["postId"], type) except Exception as exc: self.log.debug("%s: %s", exc.__class__.__name__, exc) class CohostUserExtractor(CohostExtractor): """Extractor for media from a cohost user""" + subcategory = "user" pattern = BASE_PATTERN + r"/([^/?#]+)/?(?:$|\?|#)" example = "https://cohost.org/USER" @@ -109,10 +108,10 @@ def posts(self): "projectHandle": self.groups[0], "page": 0, "options": { - "pinnedPostsAtTop" : True if self.pinned else False, - "hideReplies" : not self.replies, - "hideShares" : not self.shares, - "hideAsks" : not self.asks, + "pinnedPostsAtTop": bool(self.pinned), + "hideReplies": not self.replies, + "hideShares": not self.shares, + "hideAsks": not self.asks, "viewingOnProjectPage": True, }, } @@ -137,6 +136,7 @@ def posts(self): class CohostPostExtractor(CohostExtractor): """Extractor for media from a single cohost post""" + subcategory = "post" pattern = BASE_PATTERN + r"/([^/?#]+)/post/(\d+)" example = "https://cohost.org/USER/post/12345" @@ -161,21 +161,20 @@ def posts(self): class CohostTagExtractor(CohostExtractor): """Extractor for tagged posts""" + subcategory = "tag" pattern = BASE_PATTERN + r"/([^/?#]+)/tagged/([^/?#]+)(?:\?([^#]+))?" example = "https://cohost.org/USER/tagged/TAG" def posts(self): user, tag, query = self.groups - url = "{}/{}/tagged/{}".format(self.root, user, tag) + url = f"{self.root}/{user}/tagged/{tag}" params = text.parse_query(query) - post_feed_key = ("tagged-post-feed" if user == "rc" else - "project-tagged-post-feed") + post_feed_key = "tagged-post-feed" if user == "rc" else "project-tagged-post-feed" while True: page = self.request(url, params=params).text - data = util.json_loads(text.extr( - page, 'id="__COHOST_LOADER_STATE__">', '</script>')) + data = util.json_loads(text.extr(page, 'id="__COHOST_LOADER_STATE__">', "</script>")) try: feed = data[post_feed_key] @@ -188,24 +187,23 @@ def posts(self): if not pagination.get("morePagesForward"): return params["refTimestamp"] = pagination["refTimestamp"] - params["skipPosts"] = \ - pagination["currentSkip"] + pagination["idealPageStride"] + params["skipPosts"] = pagination["currentSkip"] + pagination["idealPageStride"] class CohostLikesExtractor(CohostExtractor): """Extractor for liked posts""" + subcategory = "likes" pattern = BASE_PATTERN + r"/rc/liked-posts" example = "https://cohost.org/rc/liked-posts" def posts(self): - url = "{}/rc/liked-posts".format(self.root) + url = f"{self.root}/rc/liked-posts" params = {} while True: page = self.request(url, params=params).text - data = util.json_loads(text.extr( - page, 'id="__COHOST_LOADER_STATE__">', '</script>')) + data = util.json_loads(text.extr(page, 'id="__COHOST_LOADER_STATE__">', "</script>")) try: feed = data["liked-posts-feed"] @@ -218,5 +216,4 @@ def posts(self): if not pagination.get("morePagesForward"): return params["refTimestamp"] = pagination["refTimestamp"] - params["skipPosts"] = \ - pagination["currentSkip"] + pagination["idealPageStride"] + params["skipPosts"] = pagination["currentSkip"] + pagination["idealPageStride"] diff --git a/gallery_dl/extractor/comicvine.py b/gallery_dl/extractor/comicvine.py index d076795c1..bbcc2bbcf 100644 --- a/gallery_dl/extractor/comicvine.py +++ b/gallery_dl/extractor/comicvine.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,15 @@ """Extractors for https://comicvine.gamespot.com/""" -from .booru import BooruExtractor -from .. import text import operator +from .. import text +from .booru import BooruExtractor + class ComicvineTagExtractor(BooruExtractor): """Extractor for a gallery on comicvine.gamespot.com""" + category = "comicvine" subcategory = "tag" basecategory = "" @@ -23,8 +23,7 @@ class ComicvineTagExtractor(BooruExtractor): directory_fmt = ("{category}", "{tag}") filename_fmt = "{filename}.{extension}" archive_fmt = "{id}" - pattern = (r"(?:https?://)?comicvine\.gamespot\.com" - r"(/([^/?#]+)/(\d+-\d+)/images/.*)") + pattern = r"(?:https?://)?comicvine\.gamespot\.com" r"(/([^/?#]+)/(\d+-\d+)/images/.*)" example = "https://comicvine.gamespot.com/TAG/123-45/images/" def __init__(self, match): @@ -38,10 +37,10 @@ def posts(self): url = self.root + "/js/image-data.json" params = { "images": text.extract( - self.request(self.root + self.path).text, - 'data-gallery-id="', '"')[0], - "start" : self.page_start, - "count" : self.per_page, + self.request(self.root + self.path).text, 'data-gallery-id="', '"' + )[0], + "start": self.page_start, + "count": self.per_page, "object": self.object_id, } @@ -61,6 +60,5 @@ def skip(self, num): @staticmethod def _prepare(post): - post["date"] = text.parse_datetime( - post["dateCreated"], "%a, %b %d %Y") + post["date"] = text.parse_datetime(post["dateCreated"], "%a, %b %d %Y") post["tags"] = [tag["name"] for tag in post["tags"] if tag["name"]] diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 090823e12..a324968d6 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,26 +6,43 @@ """Common classes and constants used by extractor modules.""" +from __future__ import annotations + +import datetime +import getpass +import logging +import netrc import os +import queue +import random import re import ssl +import threading import time -import netrc -import queue -import random -import getpass -import logging -import datetime +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Any +from typing import AnyStr +from typing import Literal + import requests -import threading from requests.adapters import HTTPAdapter + +from .. import cache +from .. import config +from .. import exception +from .. import output +from .. import text +from .. import util from .message import Message -from .. import config, output, text, util, cache, exception -urllib3 = requests.packages.urllib3 +if TYPE_CHECKING: + from collections.abc import Generator -class Extractor(): +urllib3 = requests.packages.urllib3 + +class Extractor: category = "" subcategory = "" basecategory = "" @@ -47,8 +62,8 @@ class Extractor(): request_interval_429 = 60.0 request_timestamp = 0.0 - def __init__(self, match): - self.log = logging.getLogger(self.category) + def __init__(self, match: re.Match[AnyStr]) -> None: + self.log: logging.Logger = logging.getLogger(self.category) self.url = match.string self.match = match self.groups = match.groups() @@ -66,20 +81,20 @@ def __iter__(self): self.initialize() return self.items() - def initialize(self): + def initialize(self) -> None: self._init_options() self._init_session() self._init_cookies() self._init() self.initialize = util.noop - def finalize(self): + def finalize(self) -> None: pass - def items(self): + def items(self) -> Generator[tuple[int, Literal[1]], Any, None]: yield Message.Version, 1 - def skip(self, num): + def skip(self, num) -> Literal[0]: return 0 def config(self, key, default=None): @@ -91,14 +106,14 @@ def config2(self, key, key2, default=None, sentinel=util.SENTINEL): return value return self.config(key2, default) - def config_deprecated(self, key, deprecated, default=None, - sentinel=util.SENTINEL, history=set()): + def config_deprecated( + self, key, deprecated, default=None, sentinel=util.SENTINEL, history=set() + ): value = self.config(deprecated, sentinel) if value is not sentinel: if deprecated not in history: history.add(deprecated) - self.log.warning("'%s' is deprecated. Use '%s' instead.", - deprecated, key) + self.log.warning("'%s' is deprecated. Use '%s' instead.", deprecated, key) default = value value = self.config(key, sentinel) @@ -113,8 +128,7 @@ def config_instance(self, key, default=None): return default def _config_shared(self, key, default=None): - return config.interpolate_common( - ("extractor",), self._cfgpath, key, default) + return config.interpolate_common(("extractor",), self._cfgpath, key, default) def _config_shared_accumulate(self, key): first = True @@ -127,13 +141,21 @@ def _config_shared_accumulate(self, key): else: conf = config.get(extr, path[0]) if conf: - values[:0] = config.accumulate( - (self.subcategory,), key, conf=conf) + values[:0] = config.accumulate((self.subcategory,), key, conf=conf) return values - def request(self, url, method="GET", session=None, - retries=None, retry_codes=None, encoding=None, - fatal=True, notfound=None, **kwargs): + def request( + self, + url, + method="GET", + session=None, + retries=None, + retry_codes=None, + encoding=None, + fatal=True, + notfound=None, + **kwargs, + ): if session is None: session = self.session if retries is None: @@ -162,31 +184,32 @@ def request(self, url, method="GET", session=None, tries = 1 if self._interval: - seconds = (self._interval() - - (time.time() - Extractor.request_timestamp)) + seconds = self._interval() - (time.time() - Extractor.request_timestamp) if seconds > 0.0: self.sleep(seconds, "request") while True: try: response = session.request(method, url, **kwargs) - except (requests.exceptions.ConnectionError, - requests.exceptions.Timeout, - requests.exceptions.ChunkedEncodingError, - requests.exceptions.ContentDecodingError) as exc: + except ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.ChunkedEncodingError, + requests.exceptions.ContentDecodingError, + ) as exc: msg = exc code = 0 - except (requests.exceptions.RequestException) as exc: + except requests.exceptions.RequestException as exc: raise exception.HttpError(exc) else: code = response.status_code if self._write_pages: self._dump_response(response) if ( - code < 400 or - code < 500 and ( - not fatal and code != 429 or fatal is None) or - fatal is ... + code < 400 + or code < 500 + and (not fatal and code != 429 or fatal is None) + or fatal is ... ): if encoding: response.encoding = encoding @@ -194,11 +217,9 @@ def request(self, url, method="GET", session=None, if notfound and code == 404: raise exception.NotFoundError(notfound) - msg = "'{} {}' for '{}'".format( - code, response.reason, response.url) + msg = f"'{code} {response.reason}' for '{response.url}'" server = response.headers.get("Server") - if server and server.startswith("cloudflare") and \ - code in (403, 503): + if server and server.startswith("cloudflare") and code in (403, 503): mitigated = response.headers.get("cf-mitigated") if mitigated and mitigated.lower() == "challenge": self.log.warning("Cloudflare challenge") @@ -213,7 +234,7 @@ def request(self, url, method="GET", session=None, if code == 429 and self._handle_429(response): continue - elif code == 429 and self._interval_429: + if code == 429 and self._interval_429: pass elif code not in retry_codes and code < 500: break @@ -221,19 +242,17 @@ def request(self, url, method="GET", session=None, finally: Extractor.request_timestamp = time.time() - self.log.debug("%s (%s/%s)", msg, tries, retries+1) + self.log.debug("%s (%s/%s)", msg, tries, retries + 1) if tries > retries: break seconds = tries if self._interval: s = self._interval() - if seconds < s: - seconds = s + seconds = max(seconds, s) if code == 429 and self._interval_429: s = self._interval_429() - if seconds < s: - seconds = s + seconds = max(seconds, s) self.wait(seconds=seconds, reason="429 Too Many Requests") else: self.sleep(seconds, "retry") @@ -243,8 +262,7 @@ def request(self, url, method="GET", session=None, _handle_429 = util.false - def wait(self, seconds=None, until=None, adjust=1.0, - reason="rate limit"): + def wait(self, seconds=None, until=None, adjust=1.0, reason="rate limit"): now = time.time() if seconds: @@ -266,13 +284,12 @@ def wait(self, seconds=None, until=None, adjust=1.0, if reason: t = datetime.datetime.fromtimestamp(until).time() - isotime = "{:02}:{:02}:{:02}".format(t.hour, t.minute, t.second) + isotime = f"{t.hour:02}:{t.minute:02}:{t.second:02}" self.log.info("Waiting until %s (%s)", isotime, reason) time.sleep(seconds) def sleep(self, seconds, reason): - self.log.debug("Sleeping %.2f seconds (%s)", - seconds, reason) + self.log.debug("Sleeping %.2f seconds (%s)", seconds, reason) time.sleep(seconds) def input(self, prompt, echo=True): @@ -291,8 +308,7 @@ def _check_input_allowed(self, prompt=""): if input is None: input = output.TTY_STDIN if not input: - raise exception.StopExtraction( - "User input required (%s)", prompt.strip(" :")) + raise exception.StopExtraction("User input required (%s)", prompt.strip(" :")) def _get_auth_info(self): """Return authentication information as (username, password) tuple""" @@ -346,7 +362,7 @@ def _init_session(self): ssl_options = ssl_ciphers = 0 # .netrc Authorization headers are alwsays disabled - session.trust_env = True if self.config("proxy-env", False) else False + session.trust_env = bool(self.config("proxy-env", False)) browser = self.config("browser") if browser is None: @@ -355,8 +371,7 @@ def _init_session(self): browser, _, platform = browser.lower().partition(":") if not platform or platform == "auto": - platform = ("Windows NT 10.0; Win64; x64" - if util.WINDOWS else "X11; Linux x86_64") + platform = "Windows NT 10.0; Win64; x64" if util.WINDOWS else "X11; Linux x86_64" elif platform == "windows": platform = "Windows NT 10.0; Win64; x64" elif platform == "linux": @@ -376,14 +391,15 @@ def _init_session(self): else: headers[key] = value - ssl_options |= (ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | - ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1) + ssl_options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 ssl_ciphers = SSL_CIPHERS[browser] else: useragent = self.config("user-agent") if useragent is None: - useragent = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64; " - "rv:128.0) Gecko/20100101 Firefox/128.0") + useragent = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; " + "rv:128.0) Gecko/20100101 Firefox/128.0" + ) elif useragent == "browser": useragent = _browser_useragent() headers["User-Agent"] = useragent @@ -430,8 +446,7 @@ def _init_session(self): ssl_options |= ssl.OP_NO_TLSv1_2 self.log.debug("TLS 1.2 disabled.") - adapter = _build_requests_adapter( - ssl_options, ssl_ciphers, source_address) + adapter = _build_requests_adapter(ssl_options, ssl_ciphers, source_address) session.mount("https://", adapter) session.mount("http://", adapter) @@ -477,6 +492,7 @@ def cookies_load(self, cookies_source): if cookies is None: from ..cookies import load_cookies + try: cookies = load_cookies(cookies_source) except Exception as exc: @@ -493,9 +509,10 @@ def cookies_load(self, cookies_source): else: self.log.warning( - "Expected 'dict', 'list', or 'str' value for 'cookies' " - "option, got '%s' (%s)", - cookies_source.__class__.__name__, cookies_source) + "Expected 'dict', 'list', or 'str' value for 'cookies' " "option, got '%s' (%s)", + cookies_source.__class__.__name__, + cookies_source, + ) def cookies_store(self): """Store the session's cookies in a cookies.txt file""" @@ -549,22 +566,22 @@ def cookies_check(self, cookies_names, domain=None): now = time.time() for cookie in self.cookies: - if cookie.name in names and ( - not domain or cookie.domain == domain): - + if cookie.name in names and (not domain or cookie.domain == domain): if cookie.expires: diff = int(cookie.expires - now) if diff <= 0: - self.log.warning( - "Cookie '%s' has expired", cookie.name) + self.log.warning("Cookie '%s' has expired", cookie.name) continue - elif diff <= 86400: + if diff <= 86400: hours = diff // 3600 self.log.warning( "Cookie '%s' will expire in less than %s hour%s", - cookie.name, hours + 1, "s" if hours else "") + cookie.name, + hours + 1, + "s" if hours else "", + ) names.discard(cookie.name) if not names: @@ -573,8 +590,7 @@ def cookies_check(self, cookies_names, domain=None): def _prepare_ddosguard_cookies(self): if not self.cookies.get("__ddg2", domain=self.cookies_domain): - self.cookies.set( - "__ddg2", util.generate_token(), domain=self.cookies_domain) + self.cookies.set("__ddg2", util.generate_token(), domain=self.cookies_domain) def _cache(self, func, maxage, keyarg=None): # return cache.DatabaseCacheDecorator(func, maxage, keyarg) @@ -585,6 +601,7 @@ def _cache_memory(self, func, maxage=None, keyarg=None): def _get_date_min_max(self, dmin=None, dmax=None): """Retrieve and parse 'date-min' and 'date-max' config values""" + def get(key, default): ts = self.config(key, default) if isinstance(ts, str): @@ -594,15 +611,13 @@ def get(key, default): self.log.warning("Unable to parse '%s': %s", key, exc) ts = default return ts + fmt = self.config("date-format", "%Y-%m-%dT%H:%M:%S") return get("date-min", dmin), get("date-max", dmax) def _dispatch_extractors(self, extractor_data, default=()): """ """ - extractors = { - data[0].subcategory: data - for data in extractor_data - } + extractors = {data[0].subcategory: data for data in extractor_data} include = self.config("include", default) or () if include == "all": @@ -642,30 +657,25 @@ def _dump_response(self, response, history=True): fname = "{:>02}_{}".format( Extractor._dump_index, - Extractor._dump_sanitize('_', response.url), + Extractor._dump_sanitize("_", response.url), ) - if util.WINDOWS: - path = os.path.abspath(fname)[:255] - else: - path = fname[:251] + path = os.path.abspath(fname)[:255] if util.WINDOWS else fname[:251] try: - with open(path + ".txt", 'wb') as fp: + with open(path + ".txt", "wb") as fp: util.dump_response( - response, fp, + response, + fp, headers=(self._write_pages in ("all", "ALL")), - hide_auth=(self._write_pages != "ALL") + hide_auth=(self._write_pages != "ALL"), ) - self.log.info("Writing '%s' response to '%s'", - response.url, path + ".txt") + self.log.info("Writing '%s' response to '%s'", response.url, path + ".txt") except Exception as e: - self.log.warning("Failed to dump HTTP request (%s: %s)", - e.__class__.__name__, e) + self.log.warning("Failed to dump HTTP request (%s: %s)", e.__class__.__name__, e) class GalleryExtractor(Extractor): - subcategory = "gallery" filename_fmt = "{category}_{gallery_id}_{num:>03}.{extension}" directory_fmt = ("{category}", "{gallery_id} {title}") @@ -680,8 +690,7 @@ def items(self): self.login() if self.gallery_url: - page = self.request( - self.gallery_url, notfound=self.subcategory).text + page = self.request(self.gallery_url, notfound=self.subcategory).text else: page = None @@ -693,7 +702,7 @@ def items(self): images = util.enumerate_reversed(imgs, 1, data["count"]) else: images = zip( - range(1, data["count"]+1), + range(1, data["count"] + 1), imgs, ) else: @@ -728,20 +737,18 @@ def images(self, page): class ChapterExtractor(GalleryExtractor): - subcategory = "chapter" directory_fmt = ( - "{category}", "{manga}", - "{volume:?v/ />02}c{chapter:>03}{chapter_minor:?//}{title:?: //}") - filename_fmt = ( - "{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}") - archive_fmt = ( - "{manga}_{chapter}{chapter_minor}_{page}") + "{category}", + "{manga}", + "{volume:?v/ />02}c{chapter:>03}{chapter_minor:?//}{title:?: //}", + ) + filename_fmt = "{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}" + archive_fmt = "{manga}_{chapter}{chapter_minor}_{page}" enum = "page" class MangaExtractor(Extractor): - subcategory = "manga" categorytransfer = True chapterclass = None @@ -773,7 +780,7 @@ def chapters(self, page): """Return a list of all (chapter-url, metadata)-tuples""" -class AsynchronousMixin(): +class AsynchronousMixin: """Run info extraction in a separate thread""" def __iter__(self): @@ -821,7 +828,7 @@ def _init_category(self): for index, group in enumerate(self.groups): if group is not None: if index: - self.category, self.root, info = self.instances[index-1] + self.category, self.root, info = self.instances[index - 1] if not self.root: self.root = text.root_from_url(self.match.group(0)) self.config_instance = info.get @@ -848,7 +855,7 @@ def update(cls, instances): pattern = info.get("pattern") if not pattern: - pattern = re.escape(root[root.index(":") + 3:]) + pattern = re.escape(root[root.index(":") + 3 :]) pattern_list.append(pattern + "()") return ( @@ -858,7 +865,6 @@ def update(cls, instances): class RequestsAdapter(HTTPAdapter): - def __init__(self, ssl_context=None, source_address=None): self.ssl_context = ssl_context self.source_address = source_address @@ -884,7 +890,8 @@ def _build_requests_adapter(ssl_options, ssl_ciphers, source_address): if ssl_options or ssl_ciphers: ssl_context = urllib3.connection.create_urllib3_context( - options=ssl_options or None, ciphers=ssl_ciphers) + options=ssl_options or None, ciphers=ssl_ciphers + ) if not requests.__version__ < "2.32": # https://github.com/psf/requests/pull/6731 ssl_context.load_default_certs() @@ -892,16 +899,15 @@ def _build_requests_adapter(ssl_options, ssl_ciphers, source_address): else: ssl_context = None - adapter = _adapter_cache[key] = RequestsAdapter( - ssl_context, source_address) + adapter = _adapter_cache[key] = RequestsAdapter(ssl_context, source_address) return adapter @cache.cache(maxage=86400) def _browser_useragent(): """Get User-Agent header from default browser""" - import webbrowser import socket + import webbrowser server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -933,10 +939,12 @@ def _browser_useragent(): HTTP_HEADERS = { "firefox": ( - ("User-Agent", "Mozilla/5.0 ({}; " - "rv:128.0) Gecko/20100101 Firefox/128.0"), - ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9," - "image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8"), + ("User-Agent", "Mozilla/5.0 ({}; " "rv:128.0) Gecko/20100101 Firefox/128.0"), + ( + "Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9," + "image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8", + ), ("Accept-Language", "en-US,en;q=0.5"), ("Accept-Encoding", None), ("Referer", None), @@ -951,11 +959,17 @@ def _browser_useragent(): "chrome": ( ("Connection", "keep-alive"), ("Upgrade-Insecure-Requests", "1"), - ("User-Agent", "Mozilla/5.0 ({}) AppleWebKit/537.36 (KHTML, " - "like Gecko) Chrome/111.0.0.0 Safari/537.36"), - ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9," - "image/avif,image/webp,image/apng,*/*;q=0.8," - "application/signed-exchange;v=b3;q=0.7"), + ( + "User-Agent", + "Mozilla/5.0 ({}) AppleWebKit/537.36 (KHTML, " + "like Gecko) Chrome/111.0.0.0 Safari/537.36", + ), + ( + "Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9," + "image/avif,image/webp,image/apng,*/*;q=0.8," + "application/signed-exchange;v=b3;q=0.7", + ), ("Referer", None), ("Sec-Fetch-Site", "same-origin"), ("Sec-Fetch-Mode", "no-cors"), @@ -1008,10 +1022,8 @@ def _browser_useragent(): # disable Basic Authorization header injection from .netrc data -try: +with suppress(Exception): requests.sessions.get_netrc_auth = lambda _: None -except Exception: - pass # detect brotli support try: @@ -1030,6 +1042,7 @@ def _browser_useragent(): if action: try: import warnings + warnings.simplefilter(action, urllib3.exceptions.HTTPWarning) except Exception: pass diff --git a/gallery_dl/extractor/cyberdrop.py b/gallery_dl/extractor/cyberdrop.py index a514696b0..f3126aefd 100644 --- a/gallery_dl/extractor/cyberdrop.py +++ b/gallery_dl/extractor/cyberdrop.py @@ -1,14 +1,12 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://cyberdrop.me/""" +from .. import text from . import lolisafe from .common import Message -from .. import text class CyberdropAlbumExtractor(lolisafe.LolisafeAlbumExtractor): @@ -29,25 +27,27 @@ def items(self): yield Message.Url, file["url"], file def fetch_album(self, album_id): - url = "{}/a/{}".format(self.root, album_id) + url = f"{self.root}/a/{album_id}" page = self.request(url).text extr = text.extract_from(page) desc = extr('property="og:description" content="', '"') - if desc.startswith("A privacy-focused censorship-resistant file " - "sharing platform free for everyone."): + if desc.startswith( + "A privacy-focused censorship-resistant file " "sharing platform free for everyone." + ): desc = "" extr('id="title"', "") album = { - "album_id" : self.album_id, - "album_name" : text.unescape(extr('title="', '"')), - "album_size" : text.parse_bytes(extr( - '<p class="title">', "B")), - "date" : text.parse_datetime(extr( - '<p class="title">', '<'), "%d.%m.%Y"), - "description": text.unescape(text.unescape( # double - desc.rpartition(" [R")[0])), + "album_id": self.album_id, + "album_name": text.unescape(extr('title="', '"')), + "album_size": text.parse_bytes(extr('<p class="title">', "B")), + "date": text.parse_datetime(extr('<p class="title">', "<"), "%d.%m.%Y"), + "description": text.unescape( + text.unescape( # double + desc.rpartition(" [R")[0] + ) + ), } file_ids = list(text.extract_iter(page, 'id="file" href="/f/', '"')) @@ -57,13 +57,12 @@ def fetch_album(self, album_id): def _extract_files(self, file_ids): for file_id in file_ids: try: - url = "{}/api/file/info/{}".format(self.root_api, file_id) + url = f"{self.root_api}/api/file/info/{file_id}" file = self.request(url).json() auth = self.request(file["auth_url"]).json() file["url"] = auth["url"] except Exception as exc: - self.log.warning("%s (%s: %s)", - file_id, exc.__class__.__name__, exc) + self.log.warning("%s (%s: %s)", file_id, exc.__class__.__name__, exc) continue yield file diff --git a/gallery_dl/extractor/danbooru.py b/gallery_dl/extractor/danbooru.py index 1746647e9..b8ae5f01e 100644 --- a/gallery_dl/extractor/danbooru.py +++ b/gallery_dl/extractor/danbooru.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,17 @@ """Extractors for https://danbooru.donmai.us/ and other Danbooru instances""" -from .common import BaseExtractor, Message -from .. import text, util import datetime +from .. import text +from .. import util +from .common import BaseExtractor +from .common import Message + class DanbooruExtractor(BaseExtractor): """Base class for danbooru extractors""" + basecategory = "Danbooru" filename_fmt = "{category}_{id}_{filename}.{extension}" page_limit = 1000 @@ -29,7 +31,7 @@ def _init(self): threshold = self.config("threshold") if isinstance(threshold, int): - self.threshold = 1 if threshold < 1 else threshold + self.threshold = max(threshold, 1) else: self.threshold = self.per_page @@ -58,7 +60,6 @@ def items(self): data = self.metadata() for post in self.posts(): - try: url = post["file_url"] except KeyError: @@ -69,33 +70,29 @@ def items(self): continue text.nameext_from_url(url, post) - post["date"] = text.parse_datetime( - post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + post["date"] = text.parse_datetime(post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") - post["tags"] = ( - post["tag_string"].split(" ") - if post["tag_string"] else ()) + post["tags"] = post["tag_string"].split(" ") if post["tag_string"] else () post["tags_artist"] = ( - post["tag_string_artist"].split(" ") - if post["tag_string_artist"] else ()) + post["tag_string_artist"].split(" ") if post["tag_string_artist"] else () + ) post["tags_character"] = ( - post["tag_string_character"].split(" ") - if post["tag_string_character"] else ()) + post["tag_string_character"].split(" ") if post["tag_string_character"] else () + ) post["tags_copyright"] = ( - post["tag_string_copyright"].split(" ") - if post["tag_string_copyright"] else ()) + post["tag_string_copyright"].split(" ") if post["tag_string_copyright"] else () + ) post["tags_general"] = ( - post["tag_string_general"].split(" ") - if post["tag_string_general"] else ()) + post["tag_string_general"].split(" ") if post["tag_string_general"] else () + ) post["tags_meta"] = ( - post["tag_string_meta"].split(" ") - if post["tag_string_meta"] else ()) + post["tag_string_meta"].split(" ") if post["tag_string_meta"] else () + ) if post["extension"] == "zip": if self.ugoira: post["_ugoira_original"] = False - post["_ugoira_frame_data"] = post["frames"] = \ - self._ugoira_frames(post) + post["_ugoira_frame_data"] = post["frames"] = self._ugoira_frames(post) post["_http_adjust_extension"] = False else: url = post["large_file_url"] @@ -128,14 +125,12 @@ def _pagination(self, endpoint, params, prefix=None): if posts: if self.includes: params_meta = { - "only" : self.includes, + "only": self.includes, "limit": len(posts), - "tags" : "id:" + ",".join(str(p["id"]) for p in posts), + "tags": "id:" + ",".join(str(p["id"]) for p in posts), } data = { - meta["id"]: meta - for meta in self.request( - url, params=params_meta).json() + meta["id"]: meta for meta in self.request(url, params=params_meta).json() } for post in posts: post.update(data[post["id"]]) @@ -157,40 +152,41 @@ def _pagination(self, endpoint, params, prefix=None): first = False def _ugoira_frames(self, post): - data = self.request("{}/posts/{}.json?only=media_metadata".format( - self.root, post["id"]) + data = self.request( + "{}/posts/{}.json?only=media_metadata".format(self.root, post["id"]) ).json()["media_metadata"]["metadata"] ext = data["ZIP:ZipFileName"].rpartition(".")[2] fmt = ("{:>06}." + ext).format delays = data["Ugoira:FrameDelays"] - return [{"file": fmt(index), "delay": delay} - for index, delay in enumerate(delays)] - - -BASE_PATTERN = DanbooruExtractor.update({ - "danbooru": { - "root": None, - "pattern": r"(?:(?:danbooru|hijiribe|sonohara|safebooru)\.donmai\.us" - r"|donmai\.moe)", - }, - "atfbooru": { - "root": "https://booru.allthefallen.moe", - "pattern": r"booru\.allthefallen\.moe", - }, - "aibooru": { - "root": None, - "pattern": r"(?:safe\.)?aibooru\.online", - }, - "booruvar": { - "root": "https://booru.borvar.art", - "pattern": r"booru\.borvar\.art", - }, -}) + return [{"file": fmt(index), "delay": delay} for index, delay in enumerate(delays)] + + +BASE_PATTERN = DanbooruExtractor.update( + { + "danbooru": { + "root": None, + "pattern": r"(?:(?:danbooru|hijiribe|sonohara|safebooru)\.donmai\.us" r"|donmai\.moe)", + }, + "atfbooru": { + "root": "https://booru.allthefallen.moe", + "pattern": r"booru\.allthefallen\.moe", + }, + "aibooru": { + "root": None, + "pattern": r"(?:safe\.)?aibooru\.online", + }, + "booruvar": { + "root": "https://booru.borvar.art", + "pattern": r"booru\.borvar\.art", + }, + } +) class DanbooruTagExtractor(DanbooruExtractor): """Extractor for danbooru posts from tag searches""" + subcategory = "tag" directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" @@ -215,8 +211,7 @@ def posts(self): prefix = "b" else: prefix = None - elif tag.startswith( - ("id:", "md5", "ordfav:", "ordfavgroup:", "ordpool:")): + elif tag.startswith(("id:", "md5", "ordfav:", "ordfavgroup:", "ordpool:")): prefix = None break @@ -225,6 +220,7 @@ def posts(self): class DanbooruPoolExtractor(DanbooruExtractor): """Extractor for posts from danbooru pools""" + subcategory = "pool" directory_fmt = ("{category}", "pool", "{pool[id]} {pool[name]}") archive_fmt = "p_{pool[id]}_{id}" @@ -236,7 +232,7 @@ def __init__(self, match): self.pool_id = match.group(match.lastindex) def metadata(self): - url = "{}/pools/{}.json".format(self.root, self.pool_id) + url = f"{self.root}/pools/{self.pool_id}.json" pool = self.request(url).json() pool["name"] = pool["name"].replace("_", " ") self.post_ids = pool.pop("post_ids", ()) @@ -249,6 +245,7 @@ def posts(self): class DanbooruPostExtractor(DanbooruExtractor): """Extractor for single danbooru posts""" + subcategory = "post" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)" @@ -259,7 +256,7 @@ def __init__(self, match): self.post_id = match.group(match.lastindex) def posts(self): - url = "{}/posts/{}.json".format(self.root, self.post_id) + url = f"{self.root}/posts/{self.post_id}.json" post = self.request(url).json() if self.includes: params = {"only": self.includes} @@ -269,6 +266,7 @@ def posts(self): class DanbooruPopularExtractor(DanbooruExtractor): """Extractor for popular images from danbooru""" + subcategory = "popular" directory_fmt = ("{category}", "popular", "{scale}", "{date}") archive_fmt = "P_{scale[0]}_{date}_{id}" diff --git a/gallery_dl/extractor/desktopography.py b/gallery_dl/extractor/desktopography.py index 35bb2992f..eb684a4d9 100644 --- a/gallery_dl/extractor/desktopography.py +++ b/gallery_dl/extractor/desktopography.py @@ -1,26 +1,27 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://desktopography.net/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?desktopography\.net" class DesktopographyExtractor(Extractor): """Base class for desktopography extractors""" + category = "desktopography" archive_fmt = "{filename}" root = "https://desktopography.net" class DesktopographySiteExtractor(DesktopographyExtractor): - """Extractor for all desktopography exhibitions """ + """Extractor for all desktopography exhibitions""" + subcategory = "site" pattern = BASE_PATTERN + r"/$" example = "https://desktopography.net/" @@ -30,16 +31,15 @@ def items(self): data = {"_extractor": DesktopographyExhibitionExtractor} for exhibition_year in text.extract_iter( - page, - '<a href="https://desktopography.net/exhibition-', - '/">'): - + page, '<a href="https://desktopography.net/exhibition-', '/">' + ): url = self.root + "/exhibition-" + exhibition_year + "/" yield Message.Queue, url, data class DesktopographyExhibitionExtractor(DesktopographyExtractor): """Extractor for a yearly desktopography exhibition""" + subcategory = "exhibition" pattern = BASE_PATTERN + r"/exhibition-([^/?#]+)/" example = "https://desktopography.net/exhibition-2020/" @@ -49,7 +49,7 @@ def __init__(self, match): self.year = match.group(1) def items(self): - url = "{}/exhibition-{}/".format(self.root, self.year) + url = f"{self.root}/exhibition-{self.year}/" base_entry_url = "https://desktopography.net/portfolios/" page = self.request(url).text @@ -59,16 +59,15 @@ def items(self): } for entry_url in text.extract_iter( - page, - '<a class="overlay-background" href="' + base_entry_url, - '">'): - + page, '<a class="overlay-background" href="' + base_entry_url, '">' + ): url = base_entry_url + entry_url yield Message.Queue, url, data class DesktopographyEntryExtractor(DesktopographyExtractor): """Extractor for all resolutions of a desktopography wallpaper""" + subcategory = "entry" pattern = BASE_PATTERN + r"/portfolios/([\w-]+)" example = "https://desktopography.net/portfolios/NAME/" @@ -78,18 +77,15 @@ def __init__(self, match): self.entry = match.group(1) def items(self): - url = "{}/portfolios/{}".format(self.root, self.entry) + url = f"{self.root}/portfolios/{self.entry}" page = self.request(url).text entry_data = {"entry": self.entry} yield Message.Directory, entry_data for image_data in text.extract_iter( - page, - '<a target="_blank" href="https://desktopography.net', - '">'): - - path, _, filename = image_data.partition( - '" class="wallpaper-button" download="') + page, '<a target="_blank" href="https://desktopography.net', '">' + ): + path, _, filename = image_data.partition('" class="wallpaper-button" download="') text.nameext_from_url(filename, entry_data) yield Message.Url, self.root + path, entry_data diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index ea3f13df3..4f4cc2753 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,14 +6,20 @@ """Extractors for https://www.deviantart.com/""" -from .common import Extractor, Message -from .. import text, util, exception -from ..cache import cache, memcache +import binascii import collections import mimetypes -import binascii -import time import re +import time +from typing import AnyStr + +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from ..cache import memcache +from .common import Extractor +from .common import Message BASE_PATTERN = ( r"(?:https?://)?(?:" @@ -27,6 +31,7 @@ class DeviantartExtractor(Extractor): """Base class for deviantart extractors""" + category = "deviantart" root = "https://www.deviantart.com" directory_fmt = ("{category}", "{username}") @@ -35,12 +40,12 @@ class DeviantartExtractor(Extractor): cookies_names = ("auth", "auth_secure", "userinfo") _last_request = 0 - def __init__(self, match): + def __init__(self, match: re.Match[AnyStr]) -> None: Extractor.__init__(self, match) - self.user = (match.group(1) or match.group(2) or "").lower() + self.user: str = str(match.group(1) or match.group(2) or "").lower() self.offset = 0 - def _init(self): + def _init(self) -> None: self.jwt = self.config("jwt", False) self.flat = self.config("flat", True) self.extra = self.config("extra", False) @@ -68,11 +73,10 @@ def _init(self): self.quality = "-fullview.png?" self.quality_sub = re.compile(r"-fullview\.[a-z0-9]+\?").sub else: - self.quality = ",q_{}".format(self.quality) + self.quality = f",q_{self.quality}" self.quality_sub = re.compile(r",q_\d+").sub - if isinstance(self.original, str) and \ - self.original.lower().startswith("image"): + if isinstance(self.original, str) and self.original.lower().startswith("image"): self.original = True self._update_content = self._update_content_image else: @@ -96,8 +100,7 @@ def request(self, url, **kwargs): kwargs["fatal"] = False while True: response = Extractor.request(self, url, **kwargs) - if response.status_code != 403 or \ - b"Request blocked." not in response.content: + if response.status_code != 403 or b"Request blocked." not in response.content: return response self.wait(seconds=300, reason="CloudFront block") @@ -138,14 +141,12 @@ def items(self): if deviation["is_deleted"]: # prevent crashing in case the deviation really is # deleted - self.log.debug( - "Skipping %s (deleted)", deviation["deviationid"]) + self.log.debug("Skipping %s (deleted)", deviation["deviationid"]) continue tier_access = deviation.get("tier_access") if tier_access == "locked": - self.log.debug( - "Skipping %s (access locked)", deviation["deviationid"]) + self.log.debug("Skipping %s (access locked)", deviation["deviationid"]) continue if "premium_folder_data" in deviation: @@ -166,9 +167,8 @@ def items(self): deviation["is_original"] = True yield self.commit(deviation, content) - if "videos" in deviation and deviation["videos"]: - video = max(deviation["videos"], - key=lambda x: text.parse_int(x["quality"][:-1])) + if deviation.get("videos"): + video = max(deviation["videos"], key=lambda x: text.parse_int(x["quality"][:-1])) deviation["is_original"] = False yield self.commit(deviation, video) @@ -189,12 +189,11 @@ def items(self): user = comment["user"] name = user["username"].lower() if user["usericon"] == DEFAULT_AVATAR: - self.log.debug( - "Skipping avatar of '%s' (default)", name) + self.log.debug("Skipping avatar of '%s' (default)", name) continue _user_details.update(name, user) - url = "{}/{}/avatar/".format(self.root, name) + url = f"{self.root}/{name}/avatar/" comment["_extractor"] = DeviantartAvatarExtractor yield Message.Queue, url, comment @@ -204,8 +203,7 @@ def items(self): if self.previews_images: yield self.commit(deviation, preview) else: - mtype = mimetypes.guess_type( - "a." + deviation["extension"], False)[0] + mtype = mimetypes.guess_type("a." + deviation["extension"], False)[0] if mtype and not mtype.startswith("image/"): yield self.commit(deviation, preview) del deviation["is_preview"] @@ -217,11 +215,16 @@ def items(self): # /developers/http/v1/20210526/object/editor_text # the value of "features" is a JSON string with forward # slashes escaped - text_content = \ - deviation["text_content"]["body"]["features"].replace( - "\\/", "/") if "text_content" in deviation else None - for txt in (text_content, deviation.get("description"), - deviation.get("_journal")): + text_content = ( + deviation["text_content"]["body"]["features"].replace("\\/", "/") + if "text_content" in deviation + else None + ) + for txt in ( + text_content, + deviation.get("description"), + deviation.get("_journal"), + ): if txt is None: continue for match in DeviantartStashExtractor.pattern.finditer(txt): @@ -236,16 +239,17 @@ def prepare(self, deviation): """Adjust the contents of a Deviation-object""" if "index" not in deviation: try: - if deviation["url"].startswith(( - "https://www.deviantart.com/stash/", "https://sta.sh", - )): + if deviation["url"].startswith( + ( + "https://www.deviantart.com/stash/", + "https://sta.sh", + ) + ): filename = deviation["content"]["src"].split("/")[5] deviation["index_base36"] = filename.partition("-")[0][1:] - deviation["index"] = id_from_base36( - deviation["index_base36"]) + deviation["index"] = id_from_base36(deviation["index_base36"]) else: - deviation["index"] = text.parse_int( - deviation["url"].rpartition("-")[2]) + deviation["index"] = text.parse_int(deviation["url"].rpartition("-")[2]) except KeyError: deviation["index"] = 0 deviation["index_base36"] = "0" @@ -259,24 +263,27 @@ def prepare(self, deviation): deviation["username"] = deviation["author"]["username"] deviation["_username"] = deviation["username"].lower() - deviation["published_time"] = text.parse_int( - deviation["published_time"]) - deviation["date"] = text.parse_timestamp( - deviation["published_time"]) + deviation["published_time"] = text.parse_int(deviation["published_time"]) + deviation["date"] = text.parse_timestamp(deviation["published_time"]) if self.comments: deviation["comments"] = ( self._extract_comments(deviation["deviationid"], "deviation") - if deviation["stats"]["comments"] else () + if deviation["stats"]["comments"] + else () ) # filename metadata sub = re.compile(r"\W").sub - deviation["filename"] = "".join(( - sub("_", deviation["title"].lower()), "_by_", - sub("_", deviation["author"]["username"].lower()), "-d", - deviation["index_base36"], - )) + deviation["filename"] = "".join( + ( + sub("_", deviation["title"].lower()), + "_by_", + sub("_", deviation["author"]["username"].lower()), + "-d", + deviation["index_base36"], + ) + ) @staticmethod def commit(deviation, target): @@ -287,7 +294,7 @@ def commit(deviation, target): deviation["target"] = target deviation["extension"] = target["extension"] = text.ext_from_url(name) if "is_original" not in deviation: - deviation["is_original"] = ("/v1/" not in url) + deviation["is_original"] = "/v1/" not in url return Message.Url, url, deviation def _commit_journal_html(self, deviation, journal): @@ -312,7 +319,9 @@ def _commit_journal_html(self, deviation, journal): if html.find('<div class="boxtop journaltop">', 0, 250) != -1: needle = '<div class="boxtop journaltop">' header = HEADER_CUSTOM_TEMPLATE.format( - title=title, url=url, date=deviation["date"], + title=title, + url=url, + date=deviation["date"], ) else: needle = '<div usr class="gr">' @@ -321,7 +330,7 @@ def _commit_journal_html(self, deviation, journal): header = HEADER_TEMPLATE.format( title=title, url=url, - userurl="{}/{}/".format(self.root, urlname), + userurl=f"{self.root}/{urlname}/", username=username, date=deviation["date"], ) @@ -331,8 +340,7 @@ def _commit_journal_html(self, deviation, journal): else: html = JOURNAL_TEMPLATE_HTML_EXTRA.format(header, html) - html = JOURNAL_TEMPLATE_HTML.format( - title=title, html=html, shadow=shadow, css=css, cls=cls) + html = JOURNAL_TEMPLATE_HTML.format(title=title, html=html, shadow=shadow, css=css, cls=cls) deviation["extension"] = "htm" return Message.Url, html, deviation @@ -345,8 +353,7 @@ def _commit_journal_text(self, deviation, journal): html = html.partition("</style>")[2] head, _, tail = html.rpartition("<script") content = "\n".join( - text.unescape(text.remove_html(txt)) - for txt in (head or tail).split("<br />") + text.unescape(text.remove_html(txt)) for txt in (head or tail).split("<br />") ) txt = JOURNAL_TEMPLATE_TEXT.format( title=deviation["title"], @@ -373,18 +380,24 @@ def _extract_journal(self, deviation): html = text.extr( page, "<h2>Literature Text</h2></span><div>", - "</div></section></div></div>") + "</div></section></div></div>", + ) if html: return {"html": html} - self.log.debug("%s: Failed to extract journal HTML from webpage. " - "Falling back to __INITIAL_STATE__ markup.", - deviation["index"]) + self.log.debug( + "%s: Failed to extract journal HTML from webpage. " + "Falling back to __INITIAL_STATE__ markup.", + deviation["index"], + ) # parse __INITIAL_STATE__ as fallback - state = util.json_loads(text.extr( - page, 'window.__INITIAL_STATE__ = JSON.parse("', '");') - .replace("\\\\", "\\").replace("\\'", "'").replace('\\"', '"')) + state = util.json_loads( + text.extr(page, 'window.__INITIAL_STATE__ = JSON.parse("', '");') + .replace("\\\\", "\\") + .replace("\\'", "'") + .replace('\\"', '"') + ) deviations = state["@@entities"]["deviation"] content = deviations.popitem()[1]["textContent"] @@ -409,17 +422,14 @@ def _textcontent_to_html(self, deviation, content): return self._tiptap_to_html(markup) except Exception as exc: self.log.debug("", exc_info=exc) - self.log.error("%s: '%s: %s'", deviation["index"], - exc.__class__.__name__, exc) + self.log.error("%s: '%s: %s'", deviation["index"], exc.__class__.__name__, exc) - self.log.warning("%s: Unsupported '%s' markup.", - deviation["index"], html["type"]) + self.log.warning("%s: Unsupported '%s' markup.", deviation["index"], html["type"]) def _tiptap_to_html(self, markup): html = [] - html.append('<div data-editor-viewer="1" ' - 'class="_83r8m _2CKTq _3NjDa mDnFl">') + html.append('<div data-editor-viewer="1" ' 'class="_83r8m _2CKTq _3NjDa mDnFl">') data = util.json_loads(markup) for block in data["document"]["content"]: self._tiptap_process_content(html, block) @@ -466,7 +476,7 @@ def _tiptap_process_content(self, html, content): html.append(user.lower()) html.append('" data-da-type="da-mention" data-user="">@<!-- -->') html.append(user) - html.append('</a>') + html.append("</a>") else: self.log.warning("Unsupported content type '%s'", type) @@ -506,9 +516,11 @@ def _tiptap_process_deviation(self, html, content): media = dev.get("media") or () html.append('<div class="jjNX2">') - html.append('<figure class="Qf-HY" data-da-type="da-deviation" ' - 'data-deviation="" ' - 'data-width="" data-link="" data-alignment="center">') + html.append( + '<figure class="Qf-HY" data-da-type="da-deviation" ' + 'data-deviation="" ' + 'data-width="" data-link="" data-alignment="center">' + ) if "baseUri" in media: url, formats = self._eclipse_media(media) @@ -516,9 +528,11 @@ def _tiptap_process_deviation(self, html, content): html.append('<a href="') html.append(text.escape(dev["url"])) - html.append('" class="_3ouD5" style="margin:0 auto;display:flex;' - 'align-items:center;justify-content:center;' - 'overflow:hidden;width:780px;height:') + html.append( + '" class="_3ouD5" style="margin:0 auto;display:flex;' + "align-items:center;justify-content:center;" + "overflow:hidden;width:780px;height:" + ) html.append(str(780 * full["h"] / full["w"])) html.append('px">') @@ -536,7 +550,7 @@ def _tiptap_process_deviation(self, html, content): html.append(text.escape(dev["url"])) html.append('" class="_3ouD5">') - html.append('''\ + html.append("""\ <section class="Q91qI aG7Yi" style="width:350px;height:313px">\ <div class="_16ECM _1xMkk" aria-hidden="true">\ <svg height="100%" viewBox="0 0 15 12" preserveAspectRatio="xMidYMin slice" \ @@ -548,13 +562,13 @@ def _tiptap_process_deviation(self, html, content): </linearGradient>\ <text class="_2uqbc" fill="url(#app-root-3)" text-anchor="end" x="15" y="11">J\ </text></svg></div><div class="_1xz9u">Literature</div><h3 class="_2WvKD">\ -''') +""") html.append(text.escape(dev["title"])) html.append('</h3><div class="_2CPLm">') html.append(text.escape(dev["textContent"]["excerpt"])) - html.append('</div></section></a></div>') + html.append("</div></section></a></div>") - html.append('</figure></div>') + html.append("</figure></div>") def _extract_content(self, deviation): content = deviation["content"] @@ -571,23 +585,21 @@ def _extract_content(self, deviation): if self.intermediary and deviation["index"] <= 790677560: # https://github.com/r888888888/danbooru/issues/4069 intermediary, count = re.subn( - r"(/f/[^/]+/[^/]+)/v\d+/.*", - r"/intermediary\1", content["src"], 1) + r"(/f/[^/]+/[^/]+)/v\d+/.*", r"/intermediary\1", content["src"], 1 + ) if count: deviation["is_original"] = False deviation["_fallback"] = (content["src"],) content["src"] = intermediary if self.quality: - content["src"] = self.quality_sub( - self.quality, content["src"], 1) + content["src"] = self.quality_sub(self.quality, content["src"], 1) return content @staticmethod def _find_folder(folders, name, uuid): if uuid.isdecimal(): - match = re.compile(name.replace( - "-", r"[^a-z0-9]+") + "$", re.IGNORECASE).match + match = re.compile(name.replace("-", r"[^a-z0-9]+") + "$", re.IGNORECASE).match for folder in folders: if match(folder["name"]): return folder @@ -598,17 +610,14 @@ def _find_folder(folders, name, uuid): raise exception.NotFoundError("folder") def _folder_urls(self, folders, category, extractor): - base = "{}/{}/{}/".format(self.root, self.user, category) + base = f"{self.root}/{self.user}/{category}/" for folder in folders: folder["_extractor"] = extractor url = "{}{}/{}".format(base, folder["folderid"], folder["name"]) yield url, folder def _update_content_default(self, deviation, content): - if "premium_folder_data" in deviation or deviation.get("is_mature"): - public = False - else: - public = None + public = False if "premium_folder_data" in deviation or deviation.get("is_mature") else None data = self.api.deviation_download(deviation["deviationid"], public) content.update(data) @@ -637,19 +646,18 @@ def _update_token(self, deviation, content): # header = b'{"typ":"JWT","alg":"none"}' payload = ( - b'{"sub":"urn:app:","iss":"urn:app:","obj":[[{"path":"/f/' + - url.partition("/f/")[2].encode() + - b'"}]],"aud":["urn:service:file.download"]}' + b'{"sub":"urn:app:","iss":"urn:app:","obj":[[{"path":"/f/' + + url.partition("/f/")[2].encode() + + b'"}]],"aud":["urn:service:file.download"]}' ) deviation["_fallback"] = (content["src"],) deviation["is_original"] = True - content["src"] = ( - "{}?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.{}.".format( - url, - # base64 of 'header' is precomputed as 'eyJ0eX...' - # binascii.b2a_base64(header).rstrip(b"=\n").decode(), - binascii.b2a_base64(payload).rstrip(b"=\n").decode()) + content["src"] = "{}?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.{}.".format( + url, + # base64 of 'header' is precomputed as 'eyJ0eX...' + # binascii.b2a_base64(header).rstrip(b"=\n").decode(), + binascii.b2a_base64(payload).rstrip(b"=\n").decode(), ) def _extract_comments(self, target_id, target_type="deviation"): @@ -657,8 +665,7 @@ def _extract_comments(self, target_id, target_type="deviation"): comment_ids = [None] while comment_ids: - comments = self.api.comments( - target_id, target_type, comment_ids.pop()) + comments = self.api.comments(target_id, target_type, comment_ids.pop()) if results: results.extend(comments) @@ -690,8 +697,7 @@ def _fetch_premium(self, deviation): pass if not self.api.refresh_token_key: - self.log.warning( - "Unable to access premium content (no refresh-token)") + self.log.warning("Unable to access premium content (no refresh-token)") self._fetch_premium = lambda _: None return None @@ -702,28 +708,25 @@ def _fetch_premium(self, deviation): # premium_folder_data is no longer present when user has access (#5063) has_access = ("premium_folder_data" not in dev) or folder["has_access"] - if not has_access and folder["type"] == "watchers" and \ - self.config("auto-watch"): + if not has_access and folder["type"] == "watchers" and self.config("auto-watch"): if self.unwatch is not None: self.unwatch.append(username) if self.api.user_friends_watch(username): has_access = True - self.log.info( - "Watching %s for premium folder access", username) + self.log.info("Watching %s for premium folder access", username) else: self.log.warning( - "Error when trying to watch %s. " - "Try again with a new refresh-token", username) + "Error when trying to watch %s. " "Try again with a new refresh-token", + username, + ) if has_access: self.log.info("Fetching premium folder data") else: - self.log.warning("Unable to access premium content (type: %s)", - folder["type"]) + self.log.warning("Unable to access premium content (type: %s)", folder["type"]) cache = self._premium_cache - for dev in self.api.gallery( - username, folder["gallery_id"], public=False): + for dev in self.api.gallery(username, folder["gallery_id"], public=False): cache[dev["deviationid"]] = dev if has_access else None return cache[deviation["deviationid"]] @@ -734,12 +737,11 @@ def _unwatch_premium(self): self.api.user_friends_unwatch(username) def _eclipse_media(self, media, format="preview"): - url = [media["baseUri"], ] + url = [ + media["baseUri"], + ] - formats = { - fmt["t"]: fmt - for fmt in media["types"] - } + formats = {fmt["t"]: fmt for fmt in media["types"]} tokens = media["token"] if len(tokens) == 1: @@ -752,7 +754,7 @@ def _eclipse_media(self, media, format="preview"): def _eclipse_to_oauth(self, eclipse_api, deviations): for obj in deviations: - deviation = obj["deviation"] if "deviation" in obj else obj + deviation = obj.get("deviation", obj) deviation_uuid = eclipse_api.deviation_extended_fetch( deviation["deviationId"], deviation["author"]["username"], @@ -763,6 +765,7 @@ def _eclipse_to_oauth(self, eclipse_api, deviations): class DeviantartUserExtractor(DeviantartExtractor): """Extractor for an artist's user profile""" + subcategory = "user" pattern = BASE_PATTERN + r"/?$" example = "https://www.deviantart.com/USER" @@ -773,23 +776,28 @@ def initialize(self): skip = Extractor.skip def items(self): - base = "{}/{}/".format(self.root, self.user) - return self._dispatch_extractors(( - (DeviantartAvatarExtractor , base + "avatar"), - (DeviantartBackgroundExtractor, base + "banner"), - (DeviantartGalleryExtractor , base + "gallery"), - (DeviantartScrapsExtractor , base + "gallery/scraps"), - (DeviantartJournalExtractor , base + "posts"), - (DeviantartStatusExtractor , base + "posts/statuses"), - (DeviantartFavoriteExtractor , base + "favourites"), - ), ("gallery",)) + base = f"{self.root}/{self.user}/" + return self._dispatch_extractors( + ( + (DeviantartAvatarExtractor, base + "avatar"), + (DeviantartBackgroundExtractor, base + "banner"), + (DeviantartGalleryExtractor, base + "gallery"), + (DeviantartScrapsExtractor, base + "gallery/scraps"), + (DeviantartJournalExtractor, base + "posts"), + (DeviantartStatusExtractor, base + "posts/statuses"), + (DeviantartFavoriteExtractor, base + "favourites"), + ), + ("gallery",), + ) ############################################################################### # OAuth ####################################################################### + class DeviantartGalleryExtractor(DeviantartExtractor): """Extractor for all deviations from an artist's gallery""" + subcategory = "gallery" archive_fmt = "g_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/gallery(?:/all|/?\?catpath=)?/?$" @@ -804,6 +812,7 @@ def deviations(self): class DeviantartAvatarExtractor(DeviantartExtractor): """Extractor for an artist's avatar""" + subcategory = "avatar" archive_fmt = "a_{_username}_{index}" pattern = BASE_PATTERN + r"/avatar" @@ -837,27 +846,27 @@ def deviations(self): fmt, _, ext = fmt.rpartition(".") if fmt: fmt = "-" + fmt - url = "https://a.deviantart.net/avatars{}/{}/{}/{}.{}?{}".format( - fmt, name[0], name[1], name, ext, index) + url = f"https://a.deviantart.net/avatars{fmt}/{name[0]}/{name[1]}/{name}.{ext}?{index}" results.append(self._make_deviation(url, user, index, fmt)) return results def _make_deviation(self, url, user, index, fmt): return { - "author" : user, - "da_category" : "avatar", - "index" : text.parse_int(index), - "is_deleted" : False, + "author": user, + "da_category": "avatar", + "index": text.parse_int(index), + "is_deleted": False, "is_downloadable": False, - "published_time" : 0, - "title" : "avatar" + fmt, - "stats" : {"comments": 0}, - "content" : {"src": url}, + "published_time": 0, + "title": "avatar" + fmt, + "stats": {"comments": 0}, + "content": {"src": url}, } class DeviantartBackgroundExtractor(DeviantartExtractor): """Extractor for an artist's banner""" + subcategory = "background" archive_fmt = "b_{index}" pattern = BASE_PATTERN + r"/ba(?:nner|ckground)" @@ -865,14 +874,14 @@ class DeviantartBackgroundExtractor(DeviantartExtractor): def deviations(self): try: - return (self.api.user_profile(self.user.lower()) - ["cover_deviation"]["cover_deviation"],) + return (self.api.user_profile(self.user.lower())["cover_deviation"]["cover_deviation"],) except Exception: return () class DeviantartFolderExtractor(DeviantartExtractor): """Extractor for deviations inside an artist's gallery folder""" + subcategory = "folder" directory_fmt = ("{category}", "{username}", "{folder[title]}") archive_fmt = "F_{folder[uuid]}_{index}.{extension}" @@ -890,7 +899,7 @@ def deviations(self): folder = self._find_folder(folders, self.folder_name, self.folder_id) self.folder = { "title": folder["name"], - "uuid" : folder["folderid"], + "uuid": folder["folderid"], "index": self.folder_id, "owner": self.user, } @@ -903,10 +912,10 @@ def prepare(self, deviation): class DeviantartStashExtractor(DeviantartExtractor): """Extractor for sta.sh-ed deviations""" + subcategory = "stash" archive_fmt = "{index}.{extension}" - pattern = (r"(?:https?://)?(?:(?:www\.)?deviantart\.com/stash|sta\.sh)" - r"/([a-z0-9]+)") + pattern = r"(?:https?://)?(?:(?:www\.)?deviantart\.com/stash|sta\.sh)" r"/([a-z0-9]+)" example = "https://www.deviantart.com/stash/abcde" skip = Extractor.skip @@ -922,17 +931,15 @@ def deviations(self, stash_id=None): page = self._limited_request(url).text if stash_id[0] == "0": - uuid = text.extr(page, '//deviation/', '"') + uuid = text.extr(page, "//deviation/", '"') if uuid: deviation = self.api.deviation(uuid) deviation["_page"] = page - deviation["index"] = text.parse_int(text.extr( - page, '\\"deviationId\\":', ',')) + deviation["index"] = text.parse_int(text.extr(page, '\\"deviationId\\":', ",")) yield deviation return - for sid in text.extract_iter( - page, 'href="https://www.deviantart.com/stash/', '"'): + for sid in text.extract_iter(page, 'href="https://www.deviantart.com/stash/', '"'): if sid == stash_id or sid.endswith("#comments"): continue yield from self.deviations(sid) @@ -940,6 +947,7 @@ def deviations(self, stash_id=None): class DeviantartFavoriteExtractor(DeviantartExtractor): """Extractor for an artist's favorites""" + subcategory = "favorite" directory_fmt = ("{category}", "{username}", "Favourites") archive_fmt = "f_{_username}_{index}.{extension}" @@ -950,15 +958,14 @@ def deviations(self): if self.flat: return self.api.collections_all(self.user, self.offset) folders = self.api.collections_folders(self.user) - return self._folder_urls( - folders, "favourites", DeviantartCollectionExtractor) + return self._folder_urls(folders, "favourites", DeviantartCollectionExtractor) class DeviantartCollectionExtractor(DeviantartExtractor): """Extractor for a single favorite collection""" + subcategory = "collection" - directory_fmt = ("{category}", "{username}", "Favourites", - "{collection[title]}") + directory_fmt = ("{category}", "{username}", "Favourites", "{collection[title]}") archive_fmt = "C_{collection[uuid]}_{index}.{extension}" pattern = BASE_PATTERN + r"/favourites/([^/?#]+)/([^/?#]+)" example = "https://www.deviantart.com/USER/favourites/12345/TITLE" @@ -971,11 +978,10 @@ def __init__(self, match): def deviations(self): folders = self.api.collections_folders(self.user) - folder = self._find_folder( - folders, self.collection_name, self.collection_id) + folder = self._find_folder(folders, self.collection_name, self.collection_id) self.collection = { "title": folder["name"], - "uuid" : folder["folderid"], + "uuid": folder["folderid"], "index": self.collection_id, "owner": self.user, } @@ -988,6 +994,7 @@ def prepare(self, deviation): class DeviantartJournalExtractor(DeviantartExtractor): """Extractor for an artist's journals""" + subcategory = "journal" directory_fmt = ("{category}", "{username}", "Journal") archive_fmt = "j_{_username}_{index}.{extension}" @@ -1000,6 +1007,7 @@ def deviations(self): class DeviantartStatusExtractor(DeviantartExtractor): """Extractor for an artist's status updates""" + subcategory = "status" directory_fmt = ("{category}", "{username}", "Status") filename_fmt = "{category}_{index}_{title}_{date}.{extension}" @@ -1020,8 +1028,7 @@ def status(self, status): yield from self.status(item["status"].copy()) # assume is_deleted == true means necessary fields are missing if status["is_deleted"]: - self.log.warning( - "Skipping status %s (deleted)", status.get("statusid")) + self.log.warning("Skipping status %s (deleted)", status.get("statusid")) return yield status @@ -1054,13 +1061,13 @@ def prepare(self, deviation): deviation["stats"] = {"comments": comments_count} if self.comments: deviation["comments"] = ( - self._extract_comments(deviation["statusid"], "status") - if comments_count else () + self._extract_comments(deviation["statusid"], "status") if comments_count else () ) class DeviantartTagExtractor(DeviantartExtractor): """Extractor for deviations from tag searches""" + subcategory = "tag" directory_fmt = ("{category}", "Tags", "{search_tags}") archive_fmt = "T_{search_tags}_{index}.{extension}" @@ -1081,9 +1088,11 @@ def prepare(self, deviation): class DeviantartWatchExtractor(DeviantartExtractor): """Extractor for Deviations from watched users""" + subcategory = "watch" - pattern = (r"(?:https?://)?(?:www\.)?deviantart\.com" - r"/(?:watch/deviations|notifications/watch)()()") + pattern = ( + r"(?:https?://)?(?:www\.)?deviantart\.com" r"/(?:watch/deviations|notifications/watch)()()" + ) example = "https://www.deviantart.com/watch/deviations" def deviations(self): @@ -1092,6 +1101,7 @@ def deviations(self): class DeviantartWatchPostsExtractor(DeviantartExtractor): """Extractor for Posts from watched users""" + subcategory = "watch-posts" pattern = r"(?:https?://)?(?:www\.)?deviantart\.com/watch/posts()()" example = "https://www.deviantart.com/watch/posts" @@ -1103,15 +1113,19 @@ def deviations(self): ############################################################################### # Eclipse ##################################################################### + class DeviantartDeviationExtractor(DeviantartExtractor): """Extractor for single deviations""" + subcategory = "deviation" archive_fmt = "g_{_username}_{index}.{extension}" - pattern = (BASE_PATTERN + r"/(art|journal)/(?:[^/?#]+-)?(\d+)" - r"|(?:https?://)?(?:www\.)?(?:fx)?deviantart\.com/" - r"(?:view/|deviation/|view(?:-full)?\.php/*\?(?:[^#]+&)?id=)" - r"(\d+)" # bare deviation ID without slug - r"|(?:https?://)?fav\.me/d([0-9a-z]+)") # base36 + pattern = ( + BASE_PATTERN + r"/(art|journal)/(?:[^/?#]+-)?(\d+)" + r"|(?:https?://)?(?:www\.)?(?:fx)?deviantart\.com/" + r"(?:view/|deviation/|view(?:-full)?\.php/*\?(?:[^#]+&)?id=)" + r"(\d+)" # bare deviation ID without slug + r"|(?:https?://)?fav\.me/d([0-9a-z]+)" + ) # base36 example = "https://www.deviantart.com/UsER/art/TITLE-12345" skip = Extractor.skip @@ -1119,18 +1133,16 @@ class DeviantartDeviationExtractor(DeviantartExtractor): def __init__(self, match): DeviantartExtractor.__init__(self, match) self.type = match.group(3) - self.deviation_id = \ - match.group(4) or match.group(5) or id_from_base36(match.group(6)) + self.deviation_id = match.group(4) or match.group(5) or id_from_base36(match.group(6)) def deviations(self): if self.user: - url = "{}/{}/{}/{}".format( - self.root, self.user, self.type or "art", self.deviation_id) + url = "{}/{}/{}/{}".format(self.root, self.user, self.type or "art", self.deviation_id) else: - url = "{}/view/{}/".format(self.root, self.deviation_id) + url = f"{self.root}/view/{self.deviation_id}/" page = self._limited_request(url, notfound="deviation").text - uuid = text.extr(page, '"deviationUuid\\":\\"', '\\') + uuid = text.extr(page, '"deviationUuid\\":\\"', "\\") if not uuid: raise exception.NotFoundError("deviation") @@ -1141,6 +1153,7 @@ def deviations(self): class DeviantartScrapsExtractor(DeviantartExtractor): """Extractor for an artist's scraps""" + subcategory = "scraps" directory_fmt = ("{category}", "{username}", "Scraps") archive_fmt = "s_{_username}_{index}.{extension}" @@ -1152,16 +1165,18 @@ def deviations(self): eclipse_api = DeviantartEclipseAPI(self) return self._eclipse_to_oauth( - eclipse_api, eclipse_api.gallery_scraps(self.user, self.offset)) + eclipse_api, eclipse_api.gallery_scraps(self.user, self.offset) + ) class DeviantartSearchExtractor(DeviantartExtractor): """Extractor for deviantart search results""" + subcategory = "search" directory_fmt = ("{category}", "Search", "{search_tags}") archive_fmt = "Q_{search_tags}_{index}.{extension}" - pattern = (r"(?:https?://)?www\.deviantart\.com" - r"/search(?:/deviations)?/?\?([^#]+)") + cookies_domain = ".deviantart.com" + pattern = r"(?:https?://)?www\.deviantart\.com" r"/search(?:/deviations)?/?\?([^#]+)" example = "https://www.deviantart.com/search?q=QUERY" skip = Extractor.skip @@ -1175,8 +1190,7 @@ def deviations(self): logged_in = self.login() eclipse_api = DeviantartEclipseAPI(self) - search = (eclipse_api.search_deviations - if logged_in else self._search_html) + search = eclipse_api.search_deviations if logged_in else self._search_html return self._eclipse_to_oauth(eclipse_api, search(self.query)) def prepare(self, deviation): @@ -1193,15 +1207,18 @@ def _search_html(self, params): raise exception.StopExtraction("HTTP redirect to login page") page = response.text - for dev in DeviantartDeviationExtractor.pattern.findall( - page)[2::3]: + for dev in DeviantartDeviationExtractor.pattern.findall(page)[2::3]: yield { "deviationId": dev[3], "author": {"username": dev[0]}, "isJournal": dev[2] == "journal", } - cursor = text.extr(page, r'\"cursor\":\"', '\\',) + cursor = text.extr( + page, + r"\"cursor\":\"", + "\\", + ) if not cursor: return params["cursor"] = cursor @@ -1209,6 +1226,7 @@ def _search_html(self, params): class DeviantartGallerySearchExtractor(DeviantartExtractor): """Extractor for deviantart gallery searches""" + subcategory = "gallery-search" archive_fmt = "g_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/gallery/?\?(q=[^#]+)" @@ -1226,12 +1244,14 @@ def deviations(self): self.search = query["q"] return self._eclipse_to_oauth( - eclipse_api, eclipse_api.galleries_search( + eclipse_api, + eclipse_api.galleries_search( self.user, self.search, self.offset, query.get("sort", "most-recent"), - )) + ), + ) def prepare(self, deviation): DeviantartExtractor.prepare(self, deviation) @@ -1240,6 +1260,7 @@ def prepare(self, deviation): class DeviantartFollowingExtractor(DeviantartExtractor): """Extractor for user's watched users""" + subcategory = "following" pattern = BASE_PATTERN + "/(?:about#)?watching" example = "https://www.deviantart.com/USER/about#watching" @@ -1256,11 +1277,13 @@ def items(self): ############################################################################### # API Interfaces ############################################################## -class DeviantartOAuthAPI(): + +class DeviantartOAuthAPI: """Interface for the DeviantArt OAuth API https://www.deviantart.com/developers/http/v1/20160316 """ + CLIENT_ID = "5388" CLIENT_SECRET = "76b08c69cfb27f26d6161f9ab6d061a1" @@ -1298,14 +1321,19 @@ def __init__(self, extractor): metadata = extractor.config("metadata", False) if not metadata: - metadata = True if extractor.extra else False + metadata = bool(extractor.extra) if metadata: self.metadata = True if isinstance(metadata, str): if metadata == "all": - metadata = ("submission", "camera", "stats", - "collection", "gallery") + metadata = ( + "submission", + "camera", + "stats", + "collection", + "gallery", + ) else: metadata = metadata.replace(" ", "").split(",") elif not isinstance(metadata, (list, tuple)): @@ -1318,13 +1346,16 @@ def __init__(self, extractor): self.limit = 10 for param in metadata: self._metadata_params["ext_" + param] = "1" - if "ext_collection" in self._metadata_params or \ - "ext_gallery" in self._metadata_params: + if ( + "ext_collection" in self._metadata_params + or "ext_gallery" in self._metadata_params + ): if token: self._metadata_public = False else: - self.log.error("'collection' and 'gallery' metadata " - "require a refresh token") + self.log.error( + "'collection' and 'gallery' metadata " "require a refresh token" + ) else: # base metadata self.limit = 50 @@ -1341,32 +1372,30 @@ def __init__(self, extractor): def browse_deviantsyouwatch(self, offset=0): """Yield deviations from users you watch""" endpoint = "/browse/deviantsyouwatch" - params = {"limit": 50, "offset": offset, - "mature_content": self.mature} + params = {"limit": 50, "offset": offset, "mature_content": self.mature} return self._pagination(endpoint, params, public=False) def browse_posts_deviantsyouwatch(self, offset=0): """Yield posts from users you watch""" endpoint = "/browse/posts/deviantsyouwatch" - params = {"limit": 50, "offset": offset, - "mature_content": self.mature} + params = {"limit": 50, "offset": offset, "mature_content": self.mature} return self._pagination(endpoint, params, public=False, unpack=True) def browse_tags(self, tag, offset=0): - """ Browse a tag """ + """Browse a tag""" endpoint = "/browse/tags" params = { - "tag" : tag, - "offset" : offset, - "limit" : 50, + "tag": tag, + "offset": offset, + "limit": 50, "mature_content": self.mature, } return self._pagination(endpoint, params) def browse_user_journals(self, username, offset=0): journals = filter( - lambda post: "/journal/" in post["url"], - self.user_profile_posts(username)) + lambda post: "/journal/" in post["url"], self.user_profile_posts(username) + ) if offset: journals = util.advance(journals, offset) return journals @@ -1374,34 +1403,45 @@ def browse_user_journals(self, username, offset=0): def collections(self, username, folder_id, offset=0): """Yield all Deviation-objects contained in a collection folder""" endpoint = "/collections/" + folder_id - params = {"username": username, "offset": offset, "limit": 24, - "mature_content": self.mature} + params = { + "username": username, + "offset": offset, + "limit": 24, + "mature_content": self.mature, + } return self._pagination(endpoint, params) def collections_all(self, username, offset=0): """Yield all deviations in a user's collection""" endpoint = "/collections/all" - params = {"username": username, "offset": offset, "limit": 24, - "mature_content": self.mature} + params = { + "username": username, + "offset": offset, + "limit": 24, + "mature_content": self.mature, + } return self._pagination(endpoint, params) @memcache(keyarg=1) def collections_folders(self, username, offset=0): """Yield all collection folders of a specific user""" endpoint = "/collections/folders" - params = {"username": username, "offset": offset, "limit": 50, - "mature_content": self.mature} + params = { + "username": username, + "offset": offset, + "limit": 50, + "mature_content": self.mature, + } return self._pagination_list(endpoint, params) - def comments(self, target_id, target_type="deviation", - comment_id=None, offset=0): + def comments(self, target_id, target_type="deviation", comment_id=None, offset=0): """Fetch comments posted on a target""" - endpoint = "/comments/{}/{}".format(target_type, target_id) + endpoint = f"/comments/{target_type}/{target_id}" params = { - "commentid" : comment_id, - "maxdepth" : "5", - "offset" : offset, - "limit" : 50, + "commentid": comment_id, + "maxdepth": "5", + "offset": offset, + "limit": 50, "mature_content": self.mature, } return self._pagination_list(endpoint, params=params, key="thread") @@ -1411,8 +1451,7 @@ def deviation(self, deviation_id, public=None): endpoint = "/deviation/" + deviation_id deviation = self._call(endpoint, public=public) - if deviation.get("is_mature") and public is None and \ - self.refresh_token_key: + if deviation.get("is_mature") and public is None and self.refresh_token_key: deviation = self._call(endpoint, public=False) if self.metadata: @@ -1426,8 +1465,7 @@ def deviation_content(self, deviation_id, public=None): endpoint = "/deviation/content" params = {"deviationid": deviation_id} content = self._call(endpoint, params=params, public=public) - if public and content["html"].startswith( - ' <span class=\"username-with-symbol'): + if public and content["html"].startswith(' <span class="username-with-symbol'): if self.refresh_token_key: content = self._call(endpoint, params=params, public=False) else: @@ -1440,15 +1478,14 @@ def deviation_download(self, deviation_id, public=None): params = {"mature_content": self.mature} try: - return self._call( - endpoint, params=params, public=public, log=False) + return self._call(endpoint, params=params, public=public, log=False) except Exception: if not self.refresh_token_key: raise return self._call(endpoint, params=params, public=False) def deviation_metadata(self, deviations): - """ Fetch deviation metadata for a set of deviations""" + """Fetch deviation metadata for a set of deviations""" endpoint = "/deviation/metadata?" + "&".join( "deviationids[{}]={}".format(num, deviation["deviationid"]) for num, deviation in enumerate(deviations) @@ -1462,23 +1499,36 @@ def deviation_metadata(self, deviations): def gallery(self, username, folder_id, offset=0, extend=True, public=None): """Yield all Deviation-objects contained in a gallery folder""" endpoint = "/gallery/" + folder_id - params = {"username": username, "offset": offset, "limit": 24, - "mature_content": self.mature, "mode": "newest"} + params = { + "username": username, + "offset": offset, + "limit": 24, + "mature_content": self.mature, + "mode": "newest", + } return self._pagination(endpoint, params, extend, public) def gallery_all(self, username, offset=0): """Yield all Deviation-objects of a specific user""" endpoint = "/gallery/all" - params = {"username": username, "offset": offset, "limit": 24, - "mature_content": self.mature} + params = { + "username": username, + "offset": offset, + "limit": 24, + "mature_content": self.mature, + } return self._pagination(endpoint, params) @memcache(keyarg=1) def gallery_folders(self, username, offset=0): """Yield all gallery folders of a specific user""" endpoint = "/gallery/folders" - params = {"username": username, "offset": offset, "limit": 50, - "mature_content": self.mature} + params = { + "username": username, + "offset": offset, + "limit": 50, + "mature_content": self.mature, + } return self._pagination_list(endpoint, params) def user_friends(self, username, offset=0): @@ -1491,25 +1541,32 @@ def user_friends_watch(self, username): """Watch a user""" endpoint = "/user/friends/watch/" + username data = { - "watch[friend]" : "0", - "watch[deviations]" : "0", - "watch[journals]" : "0", + "watch[friend]": "0", + "watch[deviations]": "0", + "watch[journals]": "0", "watch[forum_threads]": "0", - "watch[critiques]" : "0", - "watch[scraps]" : "0", - "watch[activity]" : "0", - "watch[collections]" : "0", - "mature_content" : self.mature, + "watch[critiques]": "0", + "watch[scraps]": "0", + "watch[activity]": "0", + "watch[collections]": "0", + "mature_content": self.mature, } return self._call( - endpoint, method="POST", data=data, public=False, fatal=False, + endpoint, + method="POST", + data=data, + public=False, + fatal=False, ).get("success") def user_friends_unwatch(self, username): """Unwatch a user""" endpoint = "/user/friends/unwatch/" + username return self._call( - endpoint, method="POST", public=False, fatal=False, + endpoint, + method="POST", + public=False, + fatal=False, ).get("success") @memcache(keyarg=1) @@ -1520,23 +1577,22 @@ def user_profile(self, username): def user_profile_posts(self, username): endpoint = "/user/profile/posts" - params = {"username": username, "limit": 50, - "mature_content": self.mature} + params = {"username": username, "limit": 50, "mature_content": self.mature} return self._pagination(endpoint, params) def user_statuses(self, username, offset=0): """Yield status updates of a specific user""" statuses = filter( lambda post: "/status-update/" in post["url"], - self.user_profile_posts(username)) + self.user_profile_posts(username), + ) if offset: statuses = util.advance(statuses, offset) return statuses def authenticate(self, refresh_token_key): """Authenticate the application by requesting an access token""" - self.headers["Authorization"] = \ - self._authenticate_impl(refresh_token_key) + self.headers["Authorization"] = self._authenticate_impl(refresh_token_key) @cache(maxage=3600, keyarg=1) def _authenticate_impl(self, refresh_token_key): @@ -1544,24 +1600,25 @@ def _authenticate_impl(self, refresh_token_key): url = "https://www.deviantart.com/oauth2/token" if refresh_token_key: self.log.info("Refreshing private access token") - data = {"grant_type": "refresh_token", - "refresh_token": _refresh_token_cache(refresh_token_key)} + data = { + "grant_type": "refresh_token", + "refresh_token": _refresh_token_cache(refresh_token_key), + } else: self.log.info("Requesting public access token") data = {"grant_type": "client_credentials"} auth = util.HTTPBasicAuth(self.client_id, self.client_secret) - response = self.extractor.request( - url, method="POST", data=data, auth=auth, fatal=False) + response = self.extractor.request(url, method="POST", data=data, auth=auth, fatal=False) data = response.json() if response.status_code != 200: self.log.debug("Server response: %s", data) - raise exception.AuthenticationError('"{}" ({})'.format( - data.get("error_description"), data.get("error"))) + raise exception.AuthenticationError( + '"{}" ({})'.format(data.get("error_description"), data.get("error")) + ) if refresh_token_key: - _refresh_token_cache.update( - refresh_token_key, data["refresh_token"]) + _refresh_token_cache.update(refresh_token_key, data["refresh_token"]) return "Bearer " + data["access_token"] def _call(self, endpoint, fatal=True, log=True, public=None, **kwargs): @@ -1601,8 +1658,7 @@ def _call(self, endpoint, fatal=True, log=True, public=None, **kwargs): raise exception.AuthorizationError() self.log.debug(response.text) - msg = "API responded with {} {}".format( - status, response.reason) + msg = f"API responded with {status} {response.reason}" if status == 429: if self.delay < 30: self.delay += 1 @@ -1615,7 +1671,8 @@ def _call(self, endpoint, fatal=True, log=True, public=None, **kwargs): "Register your own OAuth application and use its " "credentials to prevent this error: " "https://gdl-org.github.io/docs/configuration.html" - "#extractor-deviantart-client-id-client-secret") + "#extractor-deviantart-client-id-client-secret" + ) else: if log: self.log.error(msg) @@ -1632,8 +1689,7 @@ def _should_switch_tokens(self, results, params): return False - def _pagination(self, endpoint, params, - extend=True, public=None, unpack=False, key="results"): + def _pagination(self, endpoint, params, extend=True, public=None, unpack=False, key="results"): warn = True if public is None: public = self.public @@ -1650,8 +1706,7 @@ def _pagination(self, endpoint, params, return if unpack: - results = [item["journal"] for item in results - if "journal" in item] + results = [item["journal"] for item in results if "journal" in item] if extend: if public and self._should_switch_tokens(results, params): if self.refresh_token_key: @@ -1663,7 +1718,8 @@ def _pagination(self, endpoint, params, self.log.warning( "Private or mature deviations detected! " "Run 'gallery-dl oauth:deviantart' and follow the " - "instructions to be able to access them.") + "instructions to be able to access them." + ) # "statusid" cannot be used instead if results and "deviationid" in results[0]: @@ -1675,15 +1731,13 @@ def _pagination(self, endpoint, params, for dev in self._shared_content(results): if not dev["is_deleted"]: continue - patch = self._call( - "/deviation/" + dev["deviationid"], fatal=False) + patch = self._call("/deviation/" + dev["deviationid"], fatal=False) if patch: dev.update(patch) yield from results - if not data["has_more"] and ( - self.strategy != "manual" or not results or not extend): + if not data["has_more"] and (self.strategy != "manual" or not results or not extend): return if "next_cursor" in data: @@ -1719,20 +1773,20 @@ def _metadata(self, deviations): else: n = self.limit for index in range(0, len(deviations), n): - self._metadata_batch(deviations[index:index+n]) + self._metadata_batch(deviations[index : index + n]) def _metadata_batch(self, deviations): """Fetch extended metadata for a single batch of deviations""" - for deviation, metadata in zip( - deviations, self.deviation_metadata(deviations)): + for deviation, metadata in zip(deviations, self.deviation_metadata(deviations)): deviation.update(metadata) deviation["tags"] = [t["tag_name"] for t in deviation["tags"]] def _folders(self, deviations): """Add a list of all containing folders to each deviation object""" for deviation in deviations: - deviation["folders"] = self._folders_map( - deviation["author"]["username"])[deviation["deviationid"]] + deviation["folders"] = self._folders_map(deviation["author"]["username"])[ + deviation["deviationid"] + ] @memcache(keyarg=1) def _folders_map(self, username): @@ -1741,10 +1795,7 @@ def _folders_map(self, username): folders = self.gallery_folders(username) # create 'folderid'-to-'folder' mapping - fmap = { - folder["folderid"]: folder - for folder in folders - } + fmap = {folder["folderid"]: folder for folder in folders} # add parent names to folders, but ignore "Featured" as parent featured = folders[0]["folderid"] @@ -1769,13 +1820,12 @@ def _folders_map(self, username): # map deviationids to folder names dmap = collections.defaultdict(list) for folder in folders: - for deviation in self.gallery( - username, folder["folderid"], 0, False): + for deviation in self.gallery(username, folder["folderid"], 0, False): dmap[deviation["deviationid"]].append(folder["name"]) return dmap -class DeviantartEclipseAPI(): +class DeviantartEclipseAPI: """Interface to the DeviantArt Eclipse API""" def __init__(self, extractor): @@ -1787,11 +1837,11 @@ def __init__(self, extractor): def deviation_extended_fetch(self, deviation_id, user, kind=None): endpoint = "/_puppy/dadeviation/init" params = { - "deviationid" : deviation_id, - "username" : user, - "type" : kind, - "include_session" : "false", - "expand" : "deviation.related", + "deviationid": deviation_id, + "username": user, + "type": kind, + "include_session": "false", + "expand": "deviation.related", "da_minor_version": "20230710", } return self._call(endpoint, params) @@ -1799,10 +1849,10 @@ def deviation_extended_fetch(self, deviation_id, user, kind=None): def gallery_scraps(self, user, offset=0): endpoint = "/_puppy/dashared/gallection/contents" params = { - "username" : user, - "type" : "gallery", - "offset" : offset, - "limit" : 24, + "username": user, + "type": "gallery", + "offset": offset, + "limit": 24, "scraps_folder": "true", } return self._pagination(endpoint, params) @@ -1811,11 +1861,11 @@ def galleries_search(self, user, query, offset=0, order="most-recent"): endpoint = "/_puppy/dashared/gallection/search" params = { "username": user, - "type" : "gallery", - "order" : order, - "q" : query, - "offset" : offset, - "limit" : 24, + "type": "gallery", + "order": order, + "q": query, + "offset": offset, + "limit": 24, } return self._pagination(endpoint, params) @@ -1833,12 +1883,12 @@ def user_watching(self, user, offset=0): endpoint = "/_puppy/gruser/module/watching" params = { - "gruserid" : gruserid, + "gruserid": gruserid, "gruser_typeid": "4", - "username" : user, - "moduleid" : moduleid, - "offset" : offset, - "limit" : 24, + "username": user, + "moduleid": moduleid, + "offset": offset, + "limit": 24, } return self._pagination(endpoint, params) @@ -1868,7 +1918,8 @@ def _pagination(self, endpoint, params, key="results"): self.log.warning( "Private deviations detected! " "Provide login credentials or session cookies " - "to be able to access them.") + "to be able to access them." + ) yield from results if not data.get("hasMore"): @@ -1886,7 +1937,7 @@ def _pagination(self, endpoint, params, key="results"): params["offset"] = int(params["offset"]) + len(results) def _ids_watching(self, user): - url = "{}/{}/about".format(self.extractor.root, user) + url = f"{self.extractor.root}/{user}/about" page = self.request(url).text gruser_id = text.extr(page, ' data-userid="', '"') @@ -1894,8 +1945,7 @@ def _ids_watching(self, user): pos = page.find('\\"name\\":\\"watching\\"') if pos < 0: raise exception.NotFoundError("'watching' module ID") - module_id = text.rextract( - page, '\\"id\\":', ',', pos)[0].strip('" ') + module_id = text.rextract(page, '\\"id\\":', ",", pos)[0].strip('" ') self._fetch_csrf_token(page) return gruser_id, module_id @@ -1903,8 +1953,7 @@ def _ids_watching(self, user): def _fetch_csrf_token(self, page=None): if page is None: page = self.request(self.extractor.root + "/").text - self.csrf_token = token = text.extr( - page, "window.__CSRF_TOKEN__ = '", "'") + self.csrf_token = token = text.extr(page, "window.__CSRF_TOKEN__ = '", "'") return token @@ -1916,14 +1965,14 @@ def _user_details(extr, name): return None -@cache(maxage=36500*86400, keyarg=0) +@cache(maxage=36500 * 86400, keyarg=0) def _refresh_token_cache(token): if token and token[0] == "#": return None return token -@cache(maxage=28*86400, keyarg=1) +@cache(maxage=28 * 86400, keyarg=1) def _login_impl(extr, username, password): extr.log.info("Logging in as %s", username) @@ -1951,10 +2000,7 @@ def _login_impl(extr, username, password): if not response.history: raise exception.AuthenticationError() - return { - cookie.name: cookie.value - for cookie in extr.cookies - } + return {cookie.name: cookie.value for cookie in extr.cookies} def id_from_base36(base36): diff --git a/gallery_dl/extractor/directlink.py b/gallery_dl/extractor/directlink.py index 2f0230af7..9e8dd5e0b 100644 --- a/gallery_dl/extractor/directlink.py +++ b/gallery_dl/extractor/directlink.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,19 +6,23 @@ """Direct link handling""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class DirectlinkExtractor(Extractor): """Extractor for direct links to images and other media files""" + category = "directlink" filename_fmt = "{domain}/{path}/{filename}.{extension}" archive_fmt = filename_fmt - pattern = (r"(?i)https?://(?P<domain>[^/?#]+)/(?P<path>[^?#]+\." - r"(?:jpe?g|jpe|png|gif|bmp|svg|web[mp]|avif|heic|psd" - r"|mp4|m4v|mov|mkv|og[gmv]|wav|mp3|opus|zip|rar|7z|pdf|swf))" - r"(?:\?(?P<query>[^#]*))?(?:#(?P<fragment>.*))?$") + pattern = ( + r"(?i)https?://(?P<domain>[^/?#]+)/(?P<path>[^?#]+\." + r"(?:jpe?g|jpe|png|gif|bmp|svg|web[mp]|avif|heic|psd" + r"|mp4|m4v|mov|mkv|og[gmv]|wav|mp3|opus|zip|rar|7z|pdf|swf))" + r"(?:\?(?P<query>[^#]*))?(?:#(?P<fragment>.*))?$" + ) example = "https://en.wikipedia.org/static/images/project-logos/enwiki.png" def __init__(self, match): @@ -36,8 +38,7 @@ def items(self): data["path"], _, name = data["path"].rpartition("/") data["filename"], _, ext = name.rpartition(".") data["extension"] = ext.lower() - data["_http_headers"] = { - "Referer": self.url.encode("latin-1", "ignore")} + data["_http_headers"] = {"Referer": self.url.encode("latin-1", "ignore")} yield Message.Directory, data yield Message.Url, self.url, data diff --git a/gallery_dl/extractor/dynastyscans.py b/gallery_dl/extractor/dynastyscans.py index 583869f1a..97bf15c8a 100644 --- a/gallery_dl/extractor/dynastyscans.py +++ b/gallery_dl/extractor/dynastyscans.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,20 +6,26 @@ """Extractors for https://dynasty-scans.com/""" -from .common import ChapterExtractor, MangaExtractor, Extractor, Message -from .. import text, util import re +from .. import text +from .. import util +from .common import ChapterExtractor +from .common import Extractor +from .common import MangaExtractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?(?:www\.)?dynasty-scans\.com" -class DynastyscansBase(): +class DynastyscansBase: """Base class for dynastyscans extractors""" + category = "dynastyscans" root = "https://dynasty-scans.com" def _parse_image_page(self, image_id): - url = "{}/images/{}".format(self.root, image_id) + url = f"{self.root}/images/{image_id}" extr = text.extract_from(self.request(url).text) date = extr("class='create_at'>", "</span>") @@ -32,52 +36,49 @@ def _parse_image_page(self, image_id): src = text.extr(src, 'href="', '"') if "Source<" in src else "" return { - "url" : self.root + url, + "url": self.root + url, "image_id": text.parse_int(image_id), - "tags" : text.split_html(tags), - "date" : text.remove_html(date), - "source" : text.unescape(src), + "tags": text.split_html(tags), + "date": text.remove_html(date), + "source": text.unescape(src), } class DynastyscansChapterExtractor(DynastyscansBase, ChapterExtractor): """Extractor for manga-chapters from dynasty-scans.com""" + pattern = BASE_PATTERN + r"(/chapters/[^/?#]+)" example = "https://dynasty-scans.com/chapters/NAME" def metadata(self, page): extr = text.extract_from(page) match = re.match( - (r"(?:<a[^>]*>)?([^<]+)(?:</a>)?" # manga name - r"(?: ch(\d+)([^:<]*))?" # chapter info - r"(?:: (.+))?"), # title + ( + r"(?:<a[^>]*>)?([^<]+)(?:</a>)?" # manga name + r"(?: ch(\d+)([^:<]*))?" # chapter info + r"(?:: (.+))?" + ), # title extr("<h3 id='chapter-title'><b>", "</b>"), ) author = extr(" by ", "</a>") - group = extr('"icon-print"></i> ', '</span>') + group = extr('"icon-print"></i> ', "</span>") return { - "manga" : text.unescape(match.group(1)), - "chapter" : text.parse_int(match.group(2)), + "manga": text.unescape(match.group(1)), + "chapter": text.parse_int(match.group(2)), "chapter_minor": match.group(3) or "", - "title" : text.unescape(match.group(4) or ""), - "author" : text.remove_html(author), - "group" : (text.remove_html(group) or - text.extr(group, ' alt="', '"')), - "date" : text.parse_datetime(extr( - '"icon-calendar"></i> ', '<'), "%b %d, %Y"), - "tags" : text.split_html(extr( - "class='tags'>", "<div id='chapter-actions'")), - "lang" : "en", + "title": text.unescape(match.group(4) or ""), + "author": text.remove_html(author), + "group": (text.remove_html(group) or text.extr(group, ' alt="', '"')), + "date": text.parse_datetime(extr('"icon-calendar"></i> ', "<"), "%b %d, %Y"), + "tags": text.split_html(extr("class='tags'>", "<div id='chapter-actions'")), + "lang": "en", "language": "English", } def images(self, page): data = text.extr(page, "var pages = ", ";\n") - return [ - (self.root + img["image"], None) - for img in util.json_loads(data) - ] + return [(self.root + img["image"], None) for img in util.json_loads(data)] class DynastyscansMangaExtractor(DynastyscansBase, MangaExtractor): @@ -87,14 +88,12 @@ class DynastyscansMangaExtractor(DynastyscansBase, MangaExtractor): example = "https://dynasty-scans.com/series/NAME" def chapters(self, page): - return [ - (self.root + path, {}) - for path in text.extract_iter(page, '<dd>\n<a href="', '"') - ] + return [(self.root + path, {}) for path in text.extract_iter(page, '<dd>\n<a href="', '"')] class DynastyscansSearchExtractor(DynastyscansBase, Extractor): """Extrator for image search results on dynasty-scans.com""" + subcategory = "search" directory_fmt = ("{category}", "Images") filename_fmt = "{image_id}.{extension}" @@ -127,6 +126,7 @@ def images(self): class DynastyscansImageExtractor(DynastyscansSearchExtractor): """Extractor for individual images on dynasty-scans.com""" + subcategory = "image" pattern = BASE_PATTERN + r"/images/(\d+)" example = "https://dynasty-scans.com/images/12345" diff --git a/gallery_dl/extractor/e621.py b/gallery_dl/extractor/e621.py index 553ec22f9..98238d515 100644 --- a/gallery_dl/extractor/e621.py +++ b/gallery_dl/extractor/e621.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,14 +6,16 @@ """Extractors for https://e621.net/ and other e621 instances""" -from .common import Message -from . import danbooru +from .. import text +from .. import util from ..cache import memcache -from .. import text, util +from . import danbooru +from .common import Message class E621Extractor(danbooru.DanbooruExtractor): """Base class for e621 extractors""" + basecategory = "E621" page_limit = 750 page_start = None @@ -32,8 +32,8 @@ def items(self): elif not isinstance(includes, (list, tuple)): includes = ("notes", "pools") - notes = ("notes" in includes) - pools = ("pools" in includes) + notes = "notes" in includes + pools = "pools" in includes data = self.metadata() for post in self.posts(): @@ -42,61 +42,62 @@ def items(self): if not file["url"]: md5 = file["md5"] file["url"] = "https://static1.{}/data/{}/{}/{}.{}".format( - self.root[8:], md5[0:2], md5[2:4], md5, file["ext"]) + self.root[8:], md5[0:2], md5[2:4], md5, file["ext"] + ) if notes and post.get("has_notes"): post["notes"] = self._get_notes(post["id"]) if pools and post["pools"]: - post["pools"] = self._get_pools( - ",".join(map(str, post["pools"]))) + post["pools"] = self._get_pools(",".join(map(str, post["pools"]))) post["filename"] = file["md5"] post["extension"] = file["ext"] - post["date"] = text.parse_datetime( - post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + post["date"] = text.parse_datetime(post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") post.update(data) yield Message.Directory, post yield Message.Url, file["url"], post def _get_notes(self, id): - return self.request( - "{}/notes.json?search[post_id]={}".format(self.root, id)).json() + return self.request(f"{self.root}/notes.json?search[post_id]={id}").json() @memcache(keyarg=1) def _get_pools(self, ids): - pools = self.request( - "{}/pools.json?search[id]={}".format(self.root, ids)).json() + pools = self.request(f"{self.root}/pools.json?search[id]={ids}").json() for pool in pools: pool["name"] = pool["name"].replace("_", " ") return pools -BASE_PATTERN = E621Extractor.update({ - "e621": { - "root": "https://e621.net", - "pattern": r"e621\.net", - }, - "e926": { - "root": "https://e926.net", - "pattern": r"e926\.net", - }, - "e6ai": { - "root": "https://e6ai.net", - "pattern": r"e6ai\.net", - }, -}) +BASE_PATTERN = E621Extractor.update( + { + "e621": { + "root": "https://e621.net", + "pattern": r"e621\.net", + }, + "e926": { + "root": "https://e926.net", + "pattern": r"e926\.net", + }, + "e6ai": { + "root": "https://e6ai.net", + "pattern": r"e6ai\.net", + }, + } +) class E621TagExtractor(E621Extractor, danbooru.DanbooruTagExtractor): """Extractor for e621 posts from tag searches""" + pattern = BASE_PATTERN + r"/posts?(?:\?.*?tags=|/index/\d+/)([^&#]+)" example = "https://e621.net/posts?tags=TAG" class E621PoolExtractor(E621Extractor, danbooru.DanbooruPoolExtractor): """Extractor for e621 pools""" + pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)" example = "https://e621.net/pools/12345" @@ -105,8 +106,7 @@ def posts(self): id_to_post = { post["id"]: post - for post in self._pagination( - "/posts.json", {"tags": "pool:" + self.pool_id}) + for post in self._pagination("/posts.json", {"tags": "pool:" + self.pool_id}) } posts = [] @@ -123,16 +123,18 @@ def posts(self): class E621PostExtractor(E621Extractor, danbooru.DanbooruPostExtractor): """Extractor for single e621 posts""" + pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)" example = "https://e621.net/posts/12345" def posts(self): - url = "{}/posts/{}.json".format(self.root, self.post_id) + url = f"{self.root}/posts/{self.post_id}.json" return (self.request(url).json()["post"],) class E621PopularExtractor(E621Extractor, danbooru.DanbooruPopularExtractor): """Extractor for popular images from e621""" + pattern = BASE_PATTERN + r"/explore/posts/popular(?:\?([^#]*))?" example = "https://e621.net/explore/posts/popular" @@ -142,6 +144,7 @@ def posts(self): class E621FavoriteExtractor(E621Extractor): """Extractor for e621 favorites""" + subcategory = "favorite" directory_fmt = ("{category}", "Favorites", "{user_id}") archive_fmt = "f_{user_id}_{id}" diff --git a/gallery_dl/extractor/erome.py b/gallery_dl/extractor/erome.py index e6d136f6c..918402e89 100644 --- a/gallery_dl/extractor/erome.py +++ b/gallery_dl/extractor/erome.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,11 +6,15 @@ """Extractors for https://www.erome.com/""" -from .common import Extractor, Message -from .. import text, util, exception -from ..cache import cache import itertools +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?(?:www\.)?erome\.com" @@ -30,40 +32,36 @@ def __init__(self, match): def items(self): for album_id in self.albums(): - url = "{}/a/{}".format(self.root, album_id) + url = f"{self.root}/a/{album_id}" try: page = self.request(url).text except exception.HttpError as exc: - self.log.warning( - "Unable to fetch album '%s' (%s)", album_id, exc) + self.log.warning("Unable to fetch album '%s' (%s)", album_id, exc) continue - title, pos = text.extract( - page, 'property="og:title" content="', '"') + title, pos = text.extract(page, 'property="og:title" content="', '"') pos = page.index('<div class="user-profile', pos) - user, pos = text.extract( - page, 'href="https://www.erome.com/', '"', pos) + user, pos = text.extract(page, 'href="https://www.erome.com/', '"', pos) urls = [] date = None groups = page.split('<div class="media-group"') for group in util.advance(groups, 1): - url = (text.extr(group, '<source src="', '"') or - text.extr(group, 'data-src="', '"')) + url = text.extr(group, '<source src="', '"') or text.extr(group, 'data-src="', '"') if url: urls.append(url) if not date: - ts = text.extr(group, '?v=', '"') + ts = text.extr(group, "?v=", '"') if len(ts) > 1: date = text.parse_timestamp(ts) data = { - "album_id" : album_id, - "title" : text.unescape(title), - "user" : text.unquote(user), - "count" : len(urls), - "date" : date, + "album_id": album_id, + "title": text.unescape(title), + "user": text.unquote(user), + "count": len(urls), + "date": date, "_http_headers": {"Referer": url}, } @@ -83,8 +81,7 @@ def request(self, url, **kwargs): response = Extractor.request(self, url, **kwargs) if response.cookies: _cookie_cache.update("", response.cookies) - if response.content.find( - b"<title>Please wait a few moments", 0, 600) < 0: + if response.content.find(b"Please wait a few moments", 0, 600) < 0: return response self.sleep(5.0, "check") @@ -101,6 +98,7 @@ def _pagination(self, url, params): class EromeAlbumExtractor(EromeExtractor): """Extractor for albums on erome.com""" + subcategory = "album" pattern = BASE_PATTERN + r"/a/(\w+)" example = "https://www.erome.com/a/ID" @@ -115,7 +113,7 @@ class EromeUserExtractor(EromeExtractor): example = "https://www.erome.com/USER" def albums(self): - url = "{}/{}".format(self.root, self.item) + url = f"{self.root}/{self.item}" return self._pagination(url, {}) diff --git a/gallery_dl/extractor/everia.py b/gallery_dl/extractor/everia.py index 94444ffb8..f0074997a 100644 --- a/gallery_dl/extractor/everia.py +++ b/gallery_dl/extractor/everia.py @@ -1,15 +1,15 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://everia.club""" -from .common import Extractor, Message -from .. import text import re +from .. import text +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?everia\.club" @@ -29,10 +29,7 @@ def _pagination(self, path, params=None, pnum=1): find_posts = re.compile(r'thumbnail">\s*= 300: @@ -56,12 +53,10 @@ def items(self): urls = re.findall(r'img.*?src="([^"]+)', content) data = { - "title": text.unescape( - text.extr(page, 'itemprop="headline">', "

")), + "title": text.unescape(text.extr(page, 'itemprop="headline">', "

")), "tags": list(text.extract_iter(page, 'rel="tag">', "")), "post_url": url, - "post_category": text.extr( - page, "post-in-category-", " ").capitalize(), + "post_category": text.extr(page, "post-in-category-", " ").capitalize(), "count": len(urls), } @@ -84,8 +79,7 @@ class EveriaCategoryExtractor(EveriaExtractor): class EveriaDateExtractor(EveriaExtractor): subcategory = "date" - pattern = (BASE_PATTERN + - r"(/\d{4}(?:/\d{2})?(?:/\d{2})?)(?:/page/\d+)?/?$") + pattern = BASE_PATTERN + r"(/\d{4}(?:/\d{2})?(?:/\d{2})?)(?:/page/\d+)?/?$" example = "https://everia.club/0000/00/00" diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index e7ba78e7c..b31024d54 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,18 +6,23 @@ """Extractors for https://e-hentai.org/ and https://exhentai.org/""" -from .common import Extractor, Message -from .. import text, util, exception -from ..cache import cache import collections import itertools import math +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?(e[x-]|g\.e-)hentai\.org" class ExhentaiExtractor(Extractor): """Base class for exhentai extractors""" + category = "exhentai" directory_fmt = ("{category}", "{gid} {title[:247]}") filename_fmt = "{gid}_{num:>04}_{image_token}_{filename}.{extension}" @@ -62,7 +65,7 @@ def login(self): raise exception.StopExtraction("Image limit reached!") if self.cookies_check(self.cookies_names): - return + return None username, password = self._get_auth_info() if username: @@ -76,7 +79,7 @@ def login(self): self.original = False self.limits = False - @cache(maxage=90*86400, keyarg=1) + @cache(maxage=90 * 86400, keyarg=1) def _login_impl(self, username, password): self.log.info("Logging in as %s", username) @@ -99,8 +102,7 @@ def _login_impl(self, username, password): content = response.content if b"You are now logged in as:" not in content: if b"The captcha was not entered correctly" in content: - raise exception.AuthenticationError( - "CAPTCHA required. Use cookies instead.") + raise exception.AuthenticationError("CAPTCHA required. Use cookies instead.") raise exception.AuthenticationError() # collect more cookies @@ -114,10 +116,9 @@ def _login_impl(self, username, password): class ExhentaiGalleryExtractor(ExhentaiExtractor): """Extractor for image galleries from exhentai.org""" + subcategory = "gallery" - pattern = (BASE_PATTERN + - r"(?:/g/(\d+)/([\da-f]{10})" - r"|/s/([\da-f]{10})/(\d+)-(\d+))") + pattern = BASE_PATTERN + r"(?:/g/(\d+)/([\da-f]{10})" r"|/s/([\da-f]{10})/(\d+)-(\d+))" example = "https://e-hentai.org/g/12345/67890abcde/" def __init__(self, match): @@ -149,22 +150,26 @@ def _init(self): def finalize(self): if self.data: - self.log.info("Use '%s/s/%s/%s-%s' as input URL " - "to continue downloading from the current position", - self.root, self.data["image_token"], - self.gallery_id, self.data["num"]) + self.log.info( + "Use '%s/s/%s/%s-%s' as input URL " + "to continue downloading from the current position", + self.root, + self.data["image_token"], + self.gallery_id, + self.data["num"], + ) def favorite(self, slot="0"): url = self.root + "/gallerypopups.php" params = { "gid": self.gallery_id, - "t" : self.gallery_token, + "t": self.gallery_token, "act": "addfav", } data = { - "favcat" : slot, - "apply" : "Apply Changes", - "update" : "1", + "favcat": slot, + "apply": "Apply Changes", + "update": "1", } self.request(url, method="POST", params=params, data=data) @@ -173,19 +178,17 @@ def items(self): if self.gallery_token: gpage = self._gallery_page() - self.image_token = text.extr(gpage, 'hentai.org/s/', '"') + self.image_token = text.extr(gpage, "hentai.org/s/", '"') if not self.image_token: self.log.debug("Page content:\n%s", gpage) - raise exception.StopExtraction( - "Failed to extract initial image token") + raise exception.StopExtraction("Failed to extract initial image token") ipage = self._image_page() else: ipage = self._image_page() - part = text.extr(ipage, 'hentai.org/g/', '"') + part = text.extr(ipage, "hentai.org/g/", '"') if not part: self.log.debug("Page content:\n%s", ipage) - raise exception.StopExtraction( - "Failed to extract gallery token") + raise exception.StopExtraction("Failed to extract gallery token") self.gallery_token = part.split("/")[1] gpage = self._gallery_page() @@ -193,8 +196,7 @@ def items(self): self.count = text.parse_int(data["filecount"]) yield Message.Directory, data - images = itertools.chain( - (self.image_from_page(ipage),), self.images_from_api()) + images = itertools.chain((self.image_from_page(ipage),), self.images_from_api()) for url, image in images: data.update(image) if self.limits: @@ -218,7 +220,8 @@ def _items_hitomi(self): data = {} from .hitomi import HitomiGalleryExtractor - url = "https://hitomi.la/galleries/{}.html".format(self.gallery_id) + + url = f"https://hitomi.la/galleries/{self.gallery_id}.html" data["_extractor"] = HitomiGalleryExtractor yield Message.Queue, url, data @@ -245,27 +248,27 @@ def metadata_from_page(self, page): self.api_url = api_url data = { - "gid" : self.gallery_id, - "token" : self.gallery_token, - "thumb" : extr("background:transparent url(", ")"), - "title" : text.unescape(extr('

', '

')), - "title_jpn" : text.unescape(extr('

', '

')), - "_" : extr('
', '<'), - "uploader" : extr('
', '
'), - "date" : text.parse_datetime(extr( - '>Posted:
'), "%Y-%m-%d %H:%M"), - "parent" : extr( - '>Parent:"), "%Y-%m-%d %H:%M" + ), + "parent": extr('>Parent:
', 'Visible:', '<'), - "language" : extr('>Language:', ' '), - "filesize" : text.parse_bytes(extr( - '>File Size:', '<').rstrip("Bbi")), - "filecount" : extr('>Length:', ' '), - "favorites" : extr('id="favcount">', ' '), - "rating" : extr(">Average: ", "<"), - "torrentcount" : extr('>Torrent Download (', ')'), + "gid": self.gallery_id, + "token": self.gallery_token, + "thumb": extr("background:transparent url(", ")"), + "title": text.unescape(extr('

', "

")), + "title_jpn": text.unescape(extr('

', "

")), + "_": extr('
", "<"), + "uploader": extr('
', "
"), + "date": text.parse_datetime( + extr('>Posted:
', "Visible:', "<") != "Yes", + "language": extr('>Language:', " "), + "filesize": text.parse_bytes( + extr('>File Size:', "<").rstrip("Bbi") + ), + "filecount": extr('>Length:', " "), + "favorites": extr('id="favcount">', " "), + "rating": extr(">Average: ", "<"), + "torrentcount": extr(">Torrent Download (", ")"), } uploader = data["uploader"] @@ -281,15 +284,15 @@ def metadata_from_page(self, page): data["lang"] = util.language_to_code(data["language"]) data["tags"] = [ text.unquote(tag.replace("+", " ")) - for tag in text.extract_iter(page, 'hentai.org/tag/', '"') + for tag in text.extract_iter(page, "hentai.org/tag/", '"') ] return data def metadata_from_api(self): data = { - "method" : "gdata", - "gidlist" : ((self.gallery_id, self.gallery_token),), + "method": "gdata", + "gidlist": ((self.gallery_id, self.gallery_token),), "namespace": 1, } @@ -307,12 +310,12 @@ def image_from_page(self, page): self.key_next = extr("'", "'") iurl = extr('= 0: origurl, pos = text.rextract(i6, '"', '"', pos) url = text.unescape(origurl) - data = self._parse_original_info(text.extract( - i6, "ownload original", "<", pos)[0]) + data = self._parse_original_info( + text.extract(i6, "ownload original", "<", pos)[0] + ) data["_fallback"] = self._fallback_original(nl, url) else: url = imgurl data = self._parse_image_info(url) - data["_fallback"] = self._fallback_1280( - nl, request["page"], imgkey) + data["_fallback"] = self._fallback_1280(nl, request["page"], imgkey) except IndexError: self.log.debug("Page content:\n%s", page) - raise exception.StopExtraction( - "Unable to parse image info for '%s'", url) + raise exception.StopExtraction("Unable to parse image info for '%s'", url) data["num"] = request["page"] data["image_token"] = imgkey @@ -385,8 +386,9 @@ def images_from_api(self): request["imgkey"] = nextkey def _validate_response(self, response): - if not response.history and response.headers.get( - "content-type", "").startswith("text/html"): + if not response.history and response.headers.get("content-type", "").startswith( + "text/html" + ): page = response.text self.log.warning("'%s'", page) @@ -394,7 +396,7 @@ def _validate_response(self, response): gp = self.config("gp") if gp == "stop": raise exception.StopExtraction("Not enough GP") - elif gp == "wait": + if gp == "wait": input("Press ENTER to continue.") return response.url @@ -423,8 +425,7 @@ def _check_509(self, url): # full 509.gif URLs # - https://exhentai.org/img/509.gif # - https://ehgt.org/g/509.gif - if url.endswith(("hentai.org/img/509.gif", - "ehgt.org/g/509.gif")): + if url.endswith(("hentai.org/img/509.gif", "ehgt.org/g/509.gif")): self.log.debug(url) self._report_limits() @@ -433,8 +434,7 @@ def _update_limits(self): cookies = { cookie.name: cookie.value for cookie in self.cookies - if cookie.domain == self.cookies_domain and - cookie.name != "igneous" + if cookie.domain == self.cookies_domain and cookie.name != "igneous" } page = self.request(url, cookies=cookies).text @@ -443,8 +443,7 @@ def _update_limits(self): self._remaining = self.limits - text.parse_int(current) def _gallery_page(self): - url = "{}/g/{}/{}/".format( - self.root, self.gallery_id, self.gallery_token) + url = f"{self.root}/g/{self.gallery_id}/{self.gallery_token}/" response = self.request(url, fatal=False) page = response.text @@ -457,8 +456,7 @@ def _gallery_page(self): return page def _image_page(self): - url = "{}/s/{}/{}-{}".format( - self.root, self.image_token, self.gallery_id, self.image_num) + url = f"{self.root}/s/{self.image_token}/{self.gallery_id}-{self.image_num}" page = self.request(url, fatal=False).text if page.startswith(("Invalid page", "Keep trying")): @@ -466,7 +464,7 @@ def _image_page(self): return page def _fallback_original(self, nl, fullimg): - url = "{}?nl={}".format(fullimg, nl) + url = f"{fullimg}?nl={nl}" for _ in util.repeat(self.fallback_retries): yield url @@ -475,8 +473,7 @@ def _fallback_1280(self, nl, num, token=None): token = self.key_start for _ in util.repeat(self.fallback_retries): - url = "{}/s/{}/{}-{}?nl={}".format( - self.root, token, self.gallery_id, num, nl) + url = f"{self.root}/s/{token}/{self.gallery_id}-{num}?nl={nl}" page = self.request(url, fatal=False).text if page.startswith(("Invalid page", "Keep trying")): @@ -498,9 +495,9 @@ def _parse_image_info(url): size = width = height = 0 return { - "cost" : 1, - "size" : text.parse_int(size), - "width" : text.parse_int(width), + "cost": 1, + "size": text.parse_int(size), + "width": text.parse_int(width), "height": text.parse_int(height), } @@ -511,15 +508,16 @@ def _parse_original_info(info): return { # 1 initial point + 1 per 0.1 MB - "cost" : 1 + math.ceil(size / 100000), - "size" : size, - "width" : text.parse_int(parts[0]), + "cost": 1 + math.ceil(size / 100000), + "size": size, + "width": text.parse_int(parts[0]), "height": text.parse_int(parts[2]), } class ExhentaiSearchExtractor(ExhentaiExtractor): """Extractor for exhentai search results""" + subcategory = "search" pattern = BASE_PATTERN + r"/(?:\?([^#]*)|tag/([^/?#]+))" example = "https://e-hentai.org/?f_search=QUERY" @@ -577,6 +575,7 @@ def items(self): class ExhentaiFavoriteExtractor(ExhentaiSearchExtractor): """Extractor for favorited exhentai galleries""" + subcategory = "favorite" pattern = BASE_PATTERN + r"/favorites\.php(?:\?([^#]*)())?" example = "https://e-hentai.org/favorites.php" diff --git a/gallery_dl/extractor/fanbox.py b/gallery_dl/extractor/fanbox.py index 9bbfb4384..c14f65977 100644 --- a/gallery_dl/extractor/fanbox.py +++ b/gallery_dl/extractor/fanbox.py @@ -1,26 +1,25 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://www.fanbox.cc/""" -from .common import Extractor, Message +import re + from .. import text from ..cache import memcache -import re +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?fanbox\.cc" USER_PATTERN = ( - r"(?:https?://)?(?:" - r"(?!www\.)([\w-]+)\.fanbox\.cc|" - r"(?:www\.)?fanbox\.cc/@([\w-]+))" + r"(?:https?://)?(?:" r"(?!www\.)([\w-]+)\.fanbox\.cc|" r"(?:www\.)?fanbox\.cc/@([\w-]+))" ) class FanboxExtractor(Extractor): """Base class for Fanbox extractors""" + category = "fanbox" root = "https://www.fanbox.cc" directory_fmt = ("{category}", "{creatorId}") @@ -41,9 +40,9 @@ def _init(self): includes = includes.split(",") elif not isinstance(includes, (list, tuple)): includes = ("user", "plan") - self._meta_user = ("user" in includes) - self._meta_plan = ("plan" in includes) - self._meta_comments = ("comments" in includes) + self._meta_user = "user" in includes + self._meta_plan = "plan" in includes + self._meta_comments = "comments" in includes else: self._meta_user = self._meta_plan = self._meta_comments = False @@ -71,13 +70,14 @@ def _pagination(self, url): try: yield self._get_post_data(item["id"]) except Exception as exc: - self.log.warning("Skipping post %s (%s: %s)", - item["id"], exc.__class__.__name__, exc) + self.log.warning( + "Skipping post %s (%s: %s)", item["id"], exc.__class__.__name__, exc + ) url = body["nextUrl"] def _get_post_data(self, post_id): """Fetch and process post data""" - url = "https://api.fanbox.cc/post.info?postId="+post_id + url = "https://api.fanbox.cc/post.info?postId=" + post_id post = self.request(url, headers=self.headers).json()["body"] content_body = post.pop("body", None) @@ -88,7 +88,7 @@ def _get_post_data(self, post_id): post["articleBody"] = content_body.copy() if "blocks" in content_body: content = [] # text content - images = [] # image IDs in 'body' order + images = [] # image IDs in 'body' order append = content.append append_img = images.append @@ -156,16 +156,18 @@ def _get_plan_data(self, creator_id): params = {"creatorId": creator_id} data = self.request(url, params=params, headers=self.headers).json() - plans = {0: { - "id" : "", - "title" : "", - "fee" : 0, - "description" : "", - "coverImageUrl" : "", - "creatorId" : creator_id, - "hasAdultContent": None, - "paymentMethod" : None, - }} + plans = { + 0: { + "id": "", + "title": "", + "fee": 0, + "description": "", + "coverImageUrl": "", + "creatorId": creator_id, + "hasAdultContent": None, + "paymentMethod": None, + } + } for plan in data["body"]: del plan["user"] plans[plan["fee"]] = plan @@ -173,8 +175,7 @@ def _get_plan_data(self, creator_id): return plans def _get_comment_data(self, post_id): - url = ("https://api.fanbox.cc/post.listComments" - "?limit=10&postId=" + post_id) + url = "https://api.fanbox.cc/post.listComments" "?limit=10&postId=" + post_id comments = [] while url: @@ -204,12 +205,9 @@ def _get_urls_from_post(self, content_body, post): html_urls = [] for href in text.extract_iter(content_body["html"], 'href="', '"'): - if "fanbox.pixiv.net/images/entry" in href: + if "fanbox.pixiv.net/images/entry" in href or "downloads.fanbox.cc" in href: html_urls.append(href) - elif "downloads.fanbox.cc" in href: - html_urls.append(href) - for src in text.extract_iter(content_body["html"], - 'data-src-original="', '"'): + for src in text.extract_iter(content_body["html"], 'data-src-original="', '"'): html_urls.append(src) for url in html_urls: @@ -282,35 +280,38 @@ def _process_embed(self, post, embed): is_video = False if provider == "soundcloud": - url = prefix+"https://soundcloud.com/"+content_id + url = prefix + "https://soundcloud.com/" + content_id is_video = True elif provider == "youtube": - url = prefix+"https://youtube.com/watch?v="+content_id + url = prefix + "https://youtube.com/watch?v=" + content_id is_video = True elif provider == "vimeo": - url = prefix+"https://vimeo.com/"+content_id + url = prefix + "https://vimeo.com/" + content_id is_video = True elif provider == "fanbox": # this is an old URL format that redirects # to a proper Fanbox URL - url = "https://www.pixiv.net/fanbox/"+content_id + url = "https://www.pixiv.net/fanbox/" + content_id # resolve redirect try: - url = self.request(url, method="HEAD", - allow_redirects=False).headers["location"] + url = self.request(url, method="HEAD", allow_redirects=False).headers["location"] except Exception as exc: url = None - self.log.warning("Unable to extract fanbox embed %s (%s: %s)", - content_id, exc.__class__.__name__, exc) + self.log.warning( + "Unable to extract fanbox embed %s (%s: %s)", + content_id, + exc.__class__.__name__, + exc, + ) else: final_post["_extractor"] = FanboxPostExtractor elif provider == "twitter": - url = "https://twitter.com/_/status/"+content_id + url = "https://twitter.com/_/status/" + content_id elif provider == "google_forms": templ = "https://docs.google.com/forms/d/e/{}/viewform?usp=sf_link" url = templ.format(content_id) else: - self.log.warning("service not recognized: {}".format(provider)) + self.log.warning("service not recognized: %s", provider) if url: final_post["embed"] = embed @@ -324,6 +325,7 @@ def _process_embed(self, post, embed): class FanboxCreatorExtractor(FanboxExtractor): """Extractor for a Fanbox creator's works""" + subcategory = "creator" pattern = USER_PATTERN + r"(?:/posts)?/?$" example = "https://USER.fanbox.cc/" @@ -345,12 +347,14 @@ def _pagination_creator(self, url): try: yield self._get_post_data(item["id"]) except Exception as exc: - self.log.warning("Skipping post %s (%s: %s)", - item["id"], exc.__class__.__name__, exc) + self.log.warning( + "Skipping post %s (%s: %s)", item["id"], exc.__class__.__name__, exc + ) class FanboxPostExtractor(FanboxExtractor): """Extractor for media from a single Fanbox post""" + subcategory = "post" pattern = USER_PATTERN + r"/posts/(\d+)" example = "https://USER.fanbox.cc/posts/12345" @@ -365,6 +369,7 @@ def posts(self): class FanboxHomeExtractor(FanboxExtractor): """Extractor for your Fanbox home feed""" + subcategory = "home" pattern = BASE_PATTERN + r"/?$" example = "https://fanbox.cc/" @@ -376,6 +381,7 @@ def posts(self): class FanboxSupportingExtractor(FanboxExtractor): """Extractor for your supported Fanbox users feed""" + subcategory = "supporting" pattern = BASE_PATTERN + r"/home/supporting" example = "https://fanbox.cc/home/supporting" @@ -387,6 +393,7 @@ def posts(self): class FanboxRedirectExtractor(Extractor): """Extractor for pixiv redirects to fanbox.cc""" + category = "fanbox" subcategory = "redirect" pattern = r"(?:https?://)?(?:www\.)?pixiv\.net/fanbox/creator/(\d+)" @@ -399,6 +406,5 @@ def __init__(self, match): def items(self): url = "https://www.pixiv.net/fanbox/creator/" + self.user_id data = {"_extractor": FanboxCreatorExtractor} - response = self.request( - url, method="HEAD", allow_redirects=False, notfound="user") + response = self.request(url, method="HEAD", allow_redirects=False, notfound="user") yield Message.Queue, response.headers["Location"], data diff --git a/gallery_dl/extractor/fanleaks.py b/gallery_dl/extractor/fanleaks.py index 886e89348..d3fb56c5c 100644 --- a/gallery_dl/extractor/fanleaks.py +++ b/gallery_dl/extractor/fanleaks.py @@ -1,17 +1,17 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://fanleaks.club/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class FanleaksExtractor(Extractor): """Base class for Fanleaks extractors""" + category = "fanleaks" directory_fmt = ("{category}", "{model}") filename_fmt = "{model_id}_{id}.{extension}" @@ -26,9 +26,9 @@ def extract_post(self, url): extr = text.extract_from(self.request(url, notfound="post").text) data = { "model_id": self.model_id, - "model" : text.unescape(extr('text-lg">', "")), - "id" : text.parse_int(self.id), - "type" : extr('type="', '"')[:5] or "photo", + "model": text.unescape(extr('text-lg">', "")), + "id": text.parse_int(self.id), + "type": extr('type="', '"')[:5] or "photo", } url = extr('src="', '"') yield Message.Directory, data @@ -37,6 +37,7 @@ def extract_post(self, url): class FanleaksPostExtractor(FanleaksExtractor): """Extractor for individual posts on fanleaks.club""" + subcategory = "post" pattern = r"(?:https?://)?(?:www\.)?fanleaks\.club/([^/?#]+)/(\d+)" example = "https://fanleaks.club/MODEL/12345" @@ -46,42 +47,40 @@ def __init__(self, match): self.id = match.group(2) def items(self): - url = "{}/{}/{}".format(self.root, self.model_id, self.id) + url = f"{self.root}/{self.model_id}/{self.id}" return self.extract_post(url) class FanleaksModelExtractor(FanleaksExtractor): """Extractor for all posts from a fanleaks model""" + subcategory = "model" - pattern = (r"(?:https?://)?(?:www\.)?fanleaks\.club" - r"/(?!latest/?$)([^/?#]+)/?$") + pattern = r"(?:https?://)?(?:www\.)?fanleaks\.club" r"/(?!latest/?$)([^/?#]+)/?$" example = "https://fanleaks.club/MODEL" def items(self): page_num = 1 - page = self.request( - self.root + "/" + self.model_id, notfound="model").text + page = self.request(self.root + "/" + self.model_id, notfound="model").text data = { "model_id": self.model_id, - "model" : text.unescape(text.extr(page, 'mt-4">', "")), - "type" : "photo", + "model": text.unescape(text.extr(page, 'mt-4">', "")), + "type": "photo", } page_url = text.extr(page, "url: '", "'") while True: - page = self.request("{}{}".format(page_url, page_num)).text + page = self.request(f"{page_url}{page_num}").text if not page: return for item in text.extract_iter(page, '"): self.id = id = text.extr(item, "/", '"') if "/icon-play.svg" in item: - url = "{}/{}/{}".format(self.root, self.model_id, id) + url = f"{self.root}/{self.model_id}/{id}" yield from self.extract_post(url) continue data["id"] = text.parse_int(id) - url = text.extr(item, 'src="', '"').replace( - "/thumbs/", "/", 1) + url = text.extr(item, 'src="', '"').replace("/thumbs/", "/", 1) yield Message.Directory, data yield Message.Url, url, text.nameext_from_url(url, data) page_num += 1 diff --git a/gallery_dl/extractor/fantia.py b/gallery_dl/extractor/fantia.py index 6218f1989..171c5866a 100644 --- a/gallery_dl/extractor/fantia.py +++ b/gallery_dl/extractor/fantia.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://fantia.jp/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message class FantiaExtractor(Extractor): """Base class for Fantia extractors""" + category = "fantia" root = "https://fantia.jp" directory_fmt = ("{category}", "{fanclub_id}") @@ -21,14 +22,14 @@ class FantiaExtractor(Extractor): def _init(self): self.headers = { - "Accept" : "application/json, text/plain, */*", + "Accept": "application/json, text/plain, */*", "X-Requested-With": "XMLHttpRequest", } self._empty_plan = { - "id" : 0, + "id": 0, "price": 0, "limit": 0, - "name" : "", + "name": "", "description": "", "thumb": self.root + "/images/fallback/plan/thumb_default.png", } @@ -52,15 +53,16 @@ def items(self): if content["visible_status"] != "visible": self.log.warning( - "Unable to download '%s' files from " - "%s#post-content-id-%s", content["visible_status"], - post["post_url"], content["id"]) + "Unable to download '%s' files from " "%s#post-content-id-%s", + content["visible_status"], + post["post_url"], + content["id"], + ) for file in files: post.update(file) post["num"] += 1 - text.nameext_from_url( - post["content_filename"] or file["file_url"], post) + text.nameext_from_url(post["content_filename"] or file["file_url"], post) yield Message.Url, file["file_url"], post post["content_num"] += 1 @@ -76,8 +78,7 @@ def _pagination(self, url): self._csrf_token(page) post_id = None - for post_id in text.extract_iter( - page, 'class="link-block" href="/posts/', '"'): + for post_id in text.extract_iter(page, 'class="link-block" href="/posts/', '"'): yield post_id if not post_id: @@ -87,12 +88,11 @@ def _pagination(self, url): def _csrf_token(self, page=None): if not page: page = self.request(self.root + "/").text - self.headers["X-CSRF-Token"] = text.extr( - page, 'name="csrf-token" content="', '"') + self.headers["X-CSRF-Token"] = text.extr(page, 'name="csrf-token" content="', '"') def _get_post_data(self, post_id): """Fetch and process post data""" - url = self.root+"/api/v1/posts/"+post_id + url = self.root + "/api/v1/posts/" + post_id resp = self.request(url, headers=self.headers).json()["post"] return { "post_id": resp["id"], @@ -101,13 +101,12 @@ def _get_post_data(self, post_id): "comment": resp["comment"], "rating": resp["rating"], "posted_at": resp["posted_at"], - "date": text.parse_datetime( - resp["posted_at"], "%a, %d %b %Y %H:%M:%S %z"), + "date": text.parse_datetime(resp["posted_at"], "%a, %d %b %Y %H:%M:%S %z"), "fanclub_id": resp["fanclub"]["id"], "fanclub_user_id": resp["fanclub"]["user"]["id"], "fanclub_user_name": resp["fanclub"]["user"]["name"], "fanclub_name": resp["fanclub"]["name"], - "fanclub_url": self.root+"/fanclubs/"+str(resp["fanclub"]["id"]), + "fanclub_url": self.root + "/fanclubs/" + str(resp["fanclub"]["id"]), "tags": [t["name"] for t in resp["tags"]], "_data": resp, } @@ -120,14 +119,17 @@ def _get_post_contents(self, post): except Exception: pass else: - contents.insert(0, { - "id": "thumb", - "title": "thumb", - "category": "thumb", - "download_uri": url, - "visible_status": "visible", - "plan": None, - }) + contents.insert( + 0, + { + "id": "thumb", + "title": "thumb", + "category": "thumb", + "download_uri": url, + "visible_status": "visible", + "plan": None, + }, + ) return contents @@ -144,15 +146,13 @@ def _process_content(self, post, content): if "post_content_photos" in content: for photo in content["post_content_photos"]: - files.append({"file_id" : photo["id"], - "file_url": photo["url"]["original"]}) + files.append({"file_id": photo["id"], "file_url": photo["url"]["original"]}) if "download_uri" in content: url = content["download_uri"] if url[0] == "/": url = self.root + url - files.append({"file_id" : content["id"], - "file_url": url}) + files.append({"file_id": content["id"], "file_url": url}) if content["category"] == "blog" and "comment" in content: comment_json = util.json_loads(content["comment"]) @@ -164,8 +164,9 @@ def _process_content(self, post, content): blog_text += insert elif isinstance(insert, dict) and "fantiaImage" in insert: img = insert["fantiaImage"] - files.append({"file_id" : img["id"], - "file_url": self.root + img["original_url"]}) + files.append( + {"file_id": img["id"], "file_url": self.root + img["original_url"]} + ) post["blogpost_text"] = blog_text else: post["blogpost_text"] = "" @@ -175,6 +176,7 @@ def _process_content(self, post, content): class FantiaCreatorExtractor(FantiaExtractor): """Extractor for a Fantia creator's works""" + subcategory = "creator" pattern = r"(?:https?://)?(?:www\.)?fantia\.jp/fanclubs/(\d+)" example = "https://fantia.jp/fanclubs/12345" @@ -184,12 +186,13 @@ def __init__(self, match): self.creator_id = match.group(1) def posts(self): - url = "{}/fanclubs/{}/posts".format(self.root, self.creator_id) + url = f"{self.root}/fanclubs/{self.creator_id}/posts" return self._pagination(url) class FantiaPostExtractor(FantiaExtractor): """Extractor for media from a single Fantia post""" + subcategory = "post" pattern = r"(?:https?://)?(?:www\.)?fantia\.jp/posts/(\d+)" example = "https://fantia.jp/posts/12345" diff --git a/gallery_dl/extractor/fapachi.py b/gallery_dl/extractor/fapachi.py index 80478caa3..e97af8a96 100644 --- a/gallery_dl/extractor/fapachi.py +++ b/gallery_dl/extractor/fapachi.py @@ -1,25 +1,24 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://fapachi.com/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class FapachiPostExtractor(Extractor): """Extractor for individual posts on fapachi.com""" + category = "fapachi" subcategory = "post" root = "https://fapachi.com" directory_fmt = ("{category}", "{user}") filename_fmt = "{user}_{id}.{extension}" archive_fmt = "{user}_{id}" - pattern = (r"(?:https?://)?(?:www\.)?fapachi\.com" - r"/(?!search/)([^/?#]+)/media/(\d+)") + pattern = r"(?:https?://)?(?:www\.)?fapachi\.com" r"/(?!search/)([^/?#]+)/media/(\d+)" example = "https://fapachi.com/MODEL/media/12345" def __init__(self, match): @@ -29,10 +28,9 @@ def __init__(self, match): def items(self): data = { "user": self.user, - "id" : self.id, + "id": self.id, } - page = self.request("{}/{}/media/{}".format( - self.root, self.user, self.id)).text + page = self.request(f"{self.root}/{self.user}/media/{self.id}").text url = self.root + text.extr(page, 'd-block" src="', '"') yield Message.Directory, data yield Message.Url, url, text.nameext_from_url(url, data) @@ -40,11 +38,13 @@ def items(self): class FapachiUserExtractor(Extractor): """Extractor for all posts from a fapachi user""" + category = "fapachi" subcategory = "user" root = "https://fapachi.com" - pattern = (r"(?:https?://)?(?:www\.)?fapachi\.com" - r"/(?!search(?:/|$))([^/?#]+)(?:/page/(\d+))?$") + pattern = ( + r"(?:https?://)?(?:www\.)?fapachi\.com" r"/(?!search(?:/|$))([^/?#]+)(?:/page/(\d+))?$" + ) example = "https://fapachi.com/MODEL" def __init__(self, match): @@ -55,8 +55,7 @@ def __init__(self, match): def items(self): data = {"_extractor": FapachiPostExtractor} while True: - page = self.request("{}/{}/page/{}".format( - self.root, self.user, self.num)).text + page = self.request(f"{self.root}/{self.user}/page/{self.num}").text for post in text.extract_iter(page, 'model-media-prew">', ">"): path = text.extr(post, '", None) + self.request(url, allow_redirects=False).text, 'class="uk-align-center"', "", None + ) if page is None: raise exception.NotFoundError("post") data = { "model": self.model, - "id" : text.parse_int(self.id), - "type" : "video" if 'type="video' in page else "photo", + "id": text.parse_int(self.id), + "type": "video" if 'type="video' in page else "photo", "thumbnail": text.extr(page, 'poster="', '"'), } - url = text.extr(page, 'src="', '"').replace( - ".md", "").replace(".th", "") + url = text.extr(page, 'src="', '"').replace(".md", "").replace(".th", "") yield Message.Directory, data yield Message.Url, url, text.nameext_from_url(url, data) class FapelloModelExtractor(Extractor): """Extractor for all posts from a fapello model""" + category = "fapello" subcategory = "model" - pattern = (BASE_PATTERN + r"/(?!top-(?:likes|followers)|popular_videos" - r"|videos|trending|search/?$)" - r"([^/?#]+)/?$") + pattern = ( + BASE_PATTERN + r"/(?!top-(?:likes|followers)|popular_videos" + r"|videos|trending|search/?$)" + r"([^/?#]+)/?$" + ) example = "https://fapello.com/model/" def __init__(self, match): @@ -66,8 +68,7 @@ def items(self): num = 1 data = {"_extractor": FapelloPostExtractor} while True: - url = "{}/ajax/model/{}/page-{}/".format( - self.root, self.model, num) + url = f"{self.root}/ajax/model/{self.model}/page-{num}/" page = self.request(url).text if not page: return @@ -81,11 +82,13 @@ def items(self): class FapelloPathExtractor(Extractor): """Extractor for models and posts from fapello.com paths""" + category = "fapello" subcategory = "path" - pattern = (BASE_PATTERN + - r"/(?!search/?$)(top-(?:likes|followers)|videos|trending" - r"|popular_videos/[^/?#]+)/?$") + pattern = ( + BASE_PATTERN + r"/(?!search/?$)(top-(?:likes|followers)|videos|trending" + r"|popular_videos/[^/?#]+)/?$" + ) example = "https://fapello.com/trending/" def __init__(self, match): @@ -106,12 +109,10 @@ def items(self): data = {"_extractor": FapelloModelExtractor} while True: - page = self.request("{}/ajax/{}/page-{}/".format( - self.root, self.path, num)).text + page = self.request(f"{self.root}/ajax/{self.path}/page-{num}/").text if not page: return - for item in text.extract_iter( - page, 'uk-transition-toggle">', ""): + for item in text.extract_iter(page, 'uk-transition-toggle">', ""): yield Message.Queue, text.extr(item, ' 0 and (int(size["width"]) > self.maxsize or - int(size["height"]) > self.maxsize): + if index > 0 and ( + int(size["width"]) > self.maxsize or int(size["height"]) > self.maxsize + ): del sizes[index:] break return sizes @@ -379,17 +391,19 @@ def urls_lookupGroup(self, groupname): """Returns a group NSID, given the url to a group's page.""" params = {"url": "https://www.flickr.com/groups/" + groupname} group = self._call("urls.lookupGroup", params)["group"] - return {"nsid": group["id"], - "path_alias": groupname, - "groupname": group["groupname"]["_content"]} + return { + "nsid": group["id"], + "path_alias": groupname, + "groupname": group["groupname"]["_content"], + } def urls_lookupUser(self, username): """Returns a user NSID, given the url to a user's photos or profile.""" params = {"url": "https://www.flickr.com/photos/" + username} user = self._call("urls.lookupUser", params)["user"] return { - "nsid" : user["id"], - "username" : user["username"]["_content"], + "nsid": user["id"], + "username": user["username"]["_content"], "path_alias": username, } @@ -418,25 +432,25 @@ def _call(self, method, params): self.log.debug("Server response: %s", data) if data["code"] == 1: raise exception.NotFoundError(self.extractor.subcategory) - elif data["code"] == 2: + if data["code"] == 2: raise exception.AuthorizationError(msg) - elif data["code"] == 98: + if data["code"] == 98: raise exception.AuthenticationError(msg) - elif data["code"] == 99: + if data["code"] == 99: raise exception.AuthorizationError(msg) raise exception.StopExtraction("API request failed: %s", msg) return data def _pagination(self, method, params, key="photos"): - extras = ("description,date_upload,tags,views,media," - "path_alias,owner_name,") + extras = "description,date_upload,tags,views,media," "path_alias,owner_name," includes = self.extractor.config("metadata") if includes: if isinstance(includes, (list, tuple)): includes = ",".join(includes) elif not isinstance(includes, str): - includes = ("license,date_taken,original_format,last_update," - "geo,machine_tags,o_dims") + includes = ( + "license,date_taken,original_format,last_update," "geo,machine_tags,o_dims" + ) extras = extras + includes + "," extras += ",".join("url_" + fmt[0] for fmt in self.formats) @@ -471,8 +485,8 @@ def _extract_format(self, photo): if "owner" in photo: photo["owner"] = { - "nsid" : photo["owner"], - "username" : photo["ownername"], + "nsid": photo["owner"], + "username": photo["ownername"], "path_alias": photo["pathalias"], } else: @@ -488,17 +502,15 @@ def _extract_format(self, photo): if key in photo: photo["width"] = text.parse_int(photo["width_" + fmt]) photo["height"] = text.parse_int(photo["height_" + fmt]) - if self.maxsize and (photo["width"] > self.maxsize or - photo["height"] > self.maxsize): + if self.maxsize and ( + photo["width"] > self.maxsize or photo["height"] > self.maxsize + ): continue photo["url"] = photo[key] photo["label"] = fmtname # remove excess data - keys = [ - key for key in photo - if key.startswith(("url_", "width_", "height_")) - ] + keys = [key for key in photo if key.startswith(("url_", "width_", "height_"))] for key in keys: del photo[key] break @@ -529,7 +541,10 @@ def _extract_metadata(self, photo): except Exception as exc: self.log.warning( "Unable to retrieve 'exif' data for %s (%s: %s)", - photo["id"], exc.__class__.__name__, exc) + photo["id"], + exc.__class__.__name__, + exc, + ) if self.contexts: try: @@ -537,7 +552,10 @@ def _extract_metadata(self, photo): except Exception as exc: self.log.warning( "Unable to retrieve 'contexts' data for %s (%s: %s)", - photo["id"], exc.__class__.__name__, exc) + photo["id"], + exc.__class__.__name__, + exc, + ) @staticmethod def _clean_info(info): diff --git a/gallery_dl/extractor/foolfuuka.py b/gallery_dl/extractor/foolfuuka.py index 44c4542b5..e99c37611 100644 --- a/gallery_dl/extractor/foolfuuka.py +++ b/gallery_dl/extractor/foolfuuka.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,16 @@ """Extractors for FoolFuuka 4chan archives""" -from .common import BaseExtractor, Message -from .. import text import itertools +from .. import text +from .common import BaseExtractor +from .common import Message + class FoolfuukaExtractor(BaseExtractor): """Base extractor for FoolFuuka based boards/archives""" + basecategory = "foolfuuka" filename_fmt = "{timestamp_ms} {filename_media}.{extension}" archive_fmt = "{board[shortname]}_{num}_{timestamp}" @@ -40,11 +41,9 @@ def items(self): if url and url[0] == "/": url = self.root + url - post["filename"], _, post["extension"] = \ - media["media"].rpartition(".") + post["filename"], _, post["extension"] = media["media"].rpartition(".") post["filename_media"] = media["media_filename"].rpartition(".")[0] - post["timestamp_ms"] = text.parse_int( - media["media_orig"].rpartition(".")[0]) + post["timestamp_ms"] = text.parse_int(media["media_orig"].rpartition(".")[0]) yield Message.Url, url, post def metadata(self): @@ -57,8 +56,7 @@ def remote(self, media): """Resolve a remote media link""" page = self.request(media["remote_media_link"]).text url = text.extr(page, 'http-equiv="Refresh" content="0; url=', '"') - if url.endswith(".webm") and \ - url.startswith("https://thebarchive.com/"): + if url.endswith(".webm") and url.startswith("https://thebarchive.com/"): return url[:-1] return url @@ -67,51 +65,53 @@ def _remote_direct(media): return media["remote_media_link"] -BASE_PATTERN = FoolfuukaExtractor.update({ - "4plebs": { - "root": "https://archive.4plebs.org", - "pattern": r"(?:archive\.)?4plebs\.org", - }, - "archivedmoe": { - "root": "https://archived.moe", - "pattern": r"archived\.moe", - }, - "archiveofsins": { - "root": "https://archiveofsins.com", - "pattern": r"(?:www\.)?archiveofsins\.com", - }, - "b4k": { - "root": "https://arch.b4k.co", - "pattern": r"arch\.b4k\.co", - }, - "desuarchive": { - "root": "https://desuarchive.org", - "pattern": r"desuarchive\.org", - }, - "fireden": { - "root": "https://boards.fireden.net", - "pattern": r"boards\.fireden\.net", - }, - "palanq": { - "root": "https://archive.palanq.win", - "pattern": r"archive\.palanq\.win", - }, - "rbt": { - "root": "https://rbt.asia", - "pattern": r"(?:rbt\.asia|(?:archive\.)?rebeccablacktech\.com)", - }, - "thebarchive": { - "root": "https://thebarchive.com", - "pattern": r"thebarchive\.com", - }, -}) +BASE_PATTERN = FoolfuukaExtractor.update( + { + "4plebs": { + "root": "https://archive.4plebs.org", + "pattern": r"(?:archive\.)?4plebs\.org", + }, + "archivedmoe": { + "root": "https://archived.moe", + "pattern": r"archived\.moe", + }, + "archiveofsins": { + "root": "https://archiveofsins.com", + "pattern": r"(?:www\.)?archiveofsins\.com", + }, + "b4k": { + "root": "https://arch.b4k.co", + "pattern": r"arch\.b4k\.co", + }, + "desuarchive": { + "root": "https://desuarchive.org", + "pattern": r"desuarchive\.org", + }, + "fireden": { + "root": "https://boards.fireden.net", + "pattern": r"boards\.fireden\.net", + }, + "palanq": { + "root": "https://archive.palanq.win", + "pattern": r"archive\.palanq\.win", + }, + "rbt": { + "root": "https://rbt.asia", + "pattern": r"(?:rbt\.asia|(?:archive\.)?rebeccablacktech\.com)", + }, + "thebarchive": { + "root": "https://thebarchive.com", + "pattern": r"thebarchive\.com", + }, + } +) class FoolfuukaThreadExtractor(FoolfuukaExtractor): """Base extractor for threads on FoolFuuka based boards/archives""" + subcategory = "thread" - directory_fmt = ("{category}", "{board[shortname]}", - "{thread_num} {title|comment[:50]}") + directory_fmt = ("{category}", "{board[shortname]}", "{thread_num} {title|comment[:50]}") pattern = BASE_PATTERN + r"/([^/?#]+)/thread/(\d+)" example = "https://archived.moe/a/thread/12345/" @@ -139,6 +139,7 @@ def posts(self): class FoolfuukaBoardExtractor(FoolfuukaExtractor): """Base extractor for FoolFuuka based boards/archives""" + subcategory = "board" pattern = BASE_PATTERN + r"/([^/?#]+)(?:/(?:page/)?(\d*))?$" example = "https://archived.moe/a/" @@ -149,9 +150,8 @@ def __init__(self, match): self.page = self.groups[-1] def items(self): - index_base = "{}/_/api/chan/index/?board={}&page=".format( - self.root, self.board) - thread_base = "{}/{}/thread/".format(self.root, self.board) + index_base = f"{self.root}/_/api/chan/index/?board={self.board}&page=" + thread_base = f"{self.root}/{self.board}/thread/" page = self.page for pnum in itertools.count(text.parse_int(page, 1)): @@ -175,6 +175,7 @@ def items(self): class FoolfuukaSearchExtractor(FoolfuukaExtractor): """Base extractor for search results on FoolFuuka based boards/archives""" + subcategory = "search" directory_fmt = ("{category}", "search", "{search}") pattern = BASE_PATTERN + r"/([^/?#]+)/search((?:/[^/?#]+/[^/?#]+)+)" @@ -230,6 +231,7 @@ def posts(self): class FoolfuukaGalleryExtractor(FoolfuukaExtractor): """Base extractor for FoolFuuka galleries""" + subcategory = "gallery" directory_fmt = ("{category}", "{board}", "gallery") pattern = BASE_PATTERN + r"/([^/?#]+)/gallery(?:/(\d+))?" @@ -240,7 +242,7 @@ def __init__(self, match): board = match.group(match.lastindex) if board.isdecimal(): - self.board = match.group(match.lastindex-1) + self.board = match.group(match.lastindex - 1) self.pages = (board,) else: self.board = board @@ -250,8 +252,7 @@ def metadata(self): return {"board": self.board} def posts(self): - base = "{}/_/api/chan/gallery/?board={}&page=".format( - self.root, self.board) + base = f"{self.root}/_/api/chan/gallery/?board={self.board}&page=" for page in self.pages: with self.request(base + page) as response: diff --git a/gallery_dl/extractor/foolslide.py b/gallery_dl/extractor/foolslide.py index bb684c2d6..8c8633ce0 100644 --- a/gallery_dl/extractor/foolslide.py +++ b/gallery_dl/extractor/foolslide.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,15 @@ """Extractors for FoOlSlide based sites""" -from .common import BaseExtractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import BaseExtractor +from .common import Message class FoolslideExtractor(BaseExtractor): """Base class for FoOlSlide extractors""" + basecategory = "foolslide" def __init__(self, match): @@ -22,7 +23,8 @@ def __init__(self, match): def request(self, url): return BaseExtractor.request( - self, url, encoding="utf-8", method="POST", data={"adult": "true"}) + self, url, encoding="utf-8", method="POST", data={"adult": "true"} + ) @staticmethod def parse_chapter_url(url, data): @@ -37,16 +39,15 @@ def parse_chapter_url(url, data): return data -BASE_PATTERN = FoolslideExtractor.update({ -}) +BASE_PATTERN = FoolslideExtractor.update({}) class FoolslideChapterExtractor(FoolslideExtractor): """Base class for chapter extractors for FoOlSlide based sites""" + subcategory = "chapter" directory_fmt = ("{category}", "{manga}", "{chapter_string}") - filename_fmt = ( - "{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}") + filename_fmt = "{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"(/read/[^/?#]+/[a-z-]+/\d+/\d+(?:/\d+)?)" example = "https://read.powermanga.org/read/MANGA/en/0/123/" @@ -60,8 +61,7 @@ def items(self): data["chapter_id"] = text.parse_int(imgs[0]["chapter_id"]) yield Message.Directory, data - enum = util.enumerate_reversed if self.config( - "page-reverse") else enumerate + enum = util.enumerate_reversed if self.config("page-reverse") else enumerate for data["page"], image in enum(imgs, 1): try: url = image["url"] @@ -78,11 +78,14 @@ def items(self): def metadata(self, page): extr = text.extract_from(page) - extr('

', '') - return self.parse_chapter_url(self.gallery_url, { - "manga" : text.unescape(extr('title="', '"')).strip(), - "chapter_string": text.unescape(extr('title="', '"')), - }) + extr('

', "") + return self.parse_chapter_url( + self.gallery_url, + { + "manga": text.unescape(extr('title="', '"')).strip(), + "chapter_string": text.unescape(extr('title="', '"')), + }, + ) def images(self, page): return util.json_loads(text.extr(page, "var pages = ", ";")) @@ -90,6 +93,7 @@ def images(self, page): class FoolslideMangaExtractor(FoolslideExtractor): """Base class for manga extractors for FoOlSlide based sites""" + subcategory = "manga" categorytransfer = True pattern = BASE_PATTERN + r"(/series/[^/?#]+)" @@ -108,17 +112,27 @@ def items(self): def chapters(self, page): extr = text.extract_from(page) - manga = text.unescape(extr('

', '

')).strip() - author = extr('Author: ', 'Artist: ', '', "")).strip() + author = extr("Author: ", "Artist: ", "")) + self._new_layout = "http-equiv=" not in extr("") path = extr('href="//d', '"') if not path: msg = text.remove_html( - extr('System Message', '') or - extr('System Message', '
') + extr("System Message", "") or extr("System Message", "") ).partition(" . Continue ")[0] - return self.log.warning( - "Unable to download post %s (\"%s\")", post_id, msg) + return self.log.warning('Unable to download post %s ("%s")', post_id, msg) pi = text.parse_int rh = text.remove_html - data = text.nameext_from_url(path, { - "id" : pi(post_id), - "url": "https://d" + path, - }) + data = text.nameext_from_url( + path, + { + "id": pi(post_id), + "url": "https://d" + path, + }, + ) if self._new_layout: - data["tags"] = text.split_html(extr( - 'class="tags-row">', '')) + data["tags"] = text.split_html(extr('class="tags-row">', "")) data["title"] = text.unescape(extr("

", "

")) data["artist"] = extr("", "<") data["_description"] = extr( 'class="submission-description user-submitted-links">', - ' ') - data["views"] = pi(rh(extr('class="views">', ''))) - data["favorites"] = pi(rh(extr('class="favorites">', ''))) - data["comments"] = pi(rh(extr('class="comments">', ''))) - data["rating"] = rh(extr('class="rating">', '')) - data["fa_category"] = rh(extr('>Category', '')) - data["theme"] = rh(extr('>', '<')) - data["species"] = rh(extr('>Species', '')) - data["gender"] = rh(extr('>Gender', '')) + " ", + ) + data["views"] = pi(rh(extr('class="views">', ""))) + data["favorites"] = pi(rh(extr('class="favorites">', ""))) + data["comments"] = pi(rh(extr('class="comments">', ""))) + data["rating"] = rh(extr('class="rating">', "")) + data["fa_category"] = rh(extr(">Category", "")) + data["theme"] = rh(extr(">", "<")) + data["species"] = rh(extr(">Species", "")) + data["gender"] = rh(extr(">Gender", "")) data["width"] = pi(extr("", "x")) data["height"] = pi(extr("", "p")) data["folders"] = folders = [] - for folder in extr( - "

Listed in Folders

", "").split(""): + for folder in extr("

Listed in Folders

", "").split(""): folder = rh(folder) if folder: folders.append(folder) @@ -130,12 +130,12 @@ def _parse_post(self, post_id): data["views"] = pi(extr("Views:", "<")) data["width"] = pi(extr("Resolution:", "x")) data["height"] = pi(extr("", "<")) - data["tags"] = text.split_html(extr( - 'id="keywords">', ''))[::2] - data["rating"] = extr('', ' ') + data[', ""))[::2] + data["rating"] = extr('', ', ' ') + '', + " ", + ) data["folders"] = () # folders not present in old layout data["artist_url"] = data["artist"].replace("_", "").lower() @@ -143,7 +143,8 @@ def _parse_post(self, post_id): data["date"] = text.parse_timestamp(data["filename"].partition(".")[0]) data["description"] = self._process_description(data["_description"]) data["thumbnail"] = "https://t.furaffinity.net/{}@600-{}.jpg".format( - post_id, path.rsplit("/", 2)[1]) + post_id, path.rsplit("/", 2)[1] + ) return data @@ -155,8 +156,7 @@ def _pagination(self, path): num = 1 while True: - url = "{}/{}/{}/{}/".format( - self.root, path, self.user, num) + url = f"{self.root}/{path}/{self.user}/{num}/" page = self.request(url).text post_id = None @@ -168,7 +168,7 @@ def _pagination(self, path): num += 1 def _pagination_favorites(self): - path = "/favorites/{}/".format(self.user) + path = f"/favorites/{self.user}/" while path: page = self.request(self.root + path).text @@ -189,22 +189,22 @@ def _pagination_favorites(self): def _pagination_search(self, query): url = self.root + "/search/" data = { - "page" : 1, - "order-by" : "relevancy", + "page": 1, + "order-by": "relevancy", "order-direction": "desc", - "range" : "all", - "range_from" : "", - "range_to" : "", - "rating-general" : "1", - "rating-mature" : "1", - "rating-adult" : "1", - "type-art" : "1", - "type-music" : "1", - "type-flash" : "1", - "type-story" : "1", - "type-photo" : "1", - "type-poetry" : "1", - "mode" : "extended", + "range": "all", + "range_from": "", + "range_to": "", + "rating-general": "1", + "rating-mature": "1", + "rating-adult": "1", + "type-art": "1", + "type-music": "1", + "type-flash": "1", + "type-story": "1", + "type-photo": "1", + "type-poetry": "1", + "mode": "extended", } data.update(query) @@ -229,6 +229,7 @@ def _pagination_search(self, query): class FuraffinityGalleryExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's gallery""" + subcategory = "gallery" pattern = BASE_PATTERN + r"/gallery/([^/?#]+)" example = "https://www.furaffinity.net/gallery/USER/" @@ -239,6 +240,7 @@ def posts(self): class FuraffinityScrapsExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's scraps""" + subcategory = "scraps" directory_fmt = ("{category}", "{user!l}", "Scraps") pattern = BASE_PATTERN + r"/scraps/([^/?#]+)" @@ -250,6 +252,7 @@ def posts(self): class FuraffinityFavoriteExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's favorites""" + subcategory = "favorite" directory_fmt = ("{category}", "{user!l}", "Favorites") pattern = BASE_PATTERN + r"/favorites/([^/?#]+)" @@ -267,6 +270,7 @@ def _parse_post(self, post_id): class FuraffinitySearchExtractor(FuraffinityExtractor): """Extractor for furaffinity search results""" + subcategory = "search" directory_fmt = ("{category}", "Search", "{search}") pattern = BASE_PATTERN + r"/search(?:/([^/?#]+))?/?[?&]([^#]+)" @@ -287,6 +291,7 @@ def posts(self): class FuraffinityPostExtractor(FuraffinityExtractor): """Extractor for individual posts on furaffinity""" + subcategory = "post" pattern = BASE_PATTERN + r"/(?:view|full)/(\d+)" example = "https://www.furaffinity.net/view/12345/" @@ -299,6 +304,7 @@ def posts(self): class FuraffinityUserExtractor(FuraffinityExtractor): """Extractor for furaffinity user profiles""" + subcategory = "user" cookies_domain = None pattern = BASE_PATTERN + r"/user/([^/?#]+)" @@ -310,22 +316,26 @@ def initialize(self): skip = Extractor.skip def items(self): - base = "{}/{{}}/{}/".format(self.root, self.user) - return self._dispatch_extractors(( - (FuraffinityGalleryExtractor , base.format("gallery")), - (FuraffinityScrapsExtractor , base.format("scraps")), - (FuraffinityFavoriteExtractor, base.format("favorites")), - ), ("gallery",)) + base = f"{self.root}/{{}}/{self.user}/" + return self._dispatch_extractors( + ( + (FuraffinityGalleryExtractor, base.format("gallery")), + (FuraffinityScrapsExtractor, base.format("scraps")), + (FuraffinityFavoriteExtractor, base.format("favorites")), + ), + ("gallery",), + ) class FuraffinityFollowingExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's watched users""" + subcategory = "following" pattern = BASE_PATTERN + "/watchlist/by/([^/?#]+)" example = "https://www.furaffinity.net/watchlist/by/USER/" def items(self): - url = "{}/watchlist/by/{}/".format(self.root, self.user) + url = f"{self.root}/watchlist/by/{self.user}/" data = {"_extractor": FuraffinityUserExtractor} while True: @@ -342,6 +352,7 @@ def items(self): class FuraffinitySubmissionsExtractor(FuraffinityExtractor): """Extractor for new furaffinity submissions""" + subcategory = "submissions" pattern = BASE_PATTERN + r"(/msg/submissions(?:/[^/?#]+)?)" example = "https://www.furaffinity.net/msg/submissions" @@ -355,12 +366,13 @@ def _pagination_submissions(self, url): while True: page = self.request(url).text - for post_id in text.extract_iter(page, 'id="sid-', '"'): - yield post_id + yield from text.extract_iter(page, 'id="sid-', '"') - path = (text.extr(page, '", "").strip() title, _, gallery_id = title.rpartition("#") return { - "gallery_id" : text.parse_int(gallery_id), + "gallery_id": text.parse_int(gallery_id), "gallery_hash": self.gallery_hash, - "title" : text.unescape(title[:-15]), - "views" : data.get("hits"), - "score" : data.get("rating"), - "tags" : (data.get("tags") or "").split(","), + "title": text.unescape(title[:-15]), + "views": data.get("hits"), + "score": data.get("rating"), + "tags": (data.get("tags") or "").split(","), } def images(self, page): - return [ - ("https:" + image["imageUrl"], image) - for image in self.data["images"] - ] + return [("https:" + image["imageUrl"], image) for image in self.data["images"]] class FuskatorSearchExtractor(Extractor): """Extractor for search results on fuskator.com""" + category = "fuskator" subcategory = "search" root = "https://fuskator.com" @@ -80,11 +84,10 @@ def items(self): while True: page = self.request(url).text - for path in text.extract_iter( - page, 'class="pic_pad">', '>>><') + pages = text.extr(page, 'class="pages">', ">>><") if not pages: return url = self.root + text.rextract(pages, 'href="', '"')[0] diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py index 37c776e6a..dbb457f07 100644 --- a/gallery_dl/extractor/gelbooru.py +++ b/gallery_dl/extractor/gelbooru.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,16 +6,20 @@ """Extractors for https://gelbooru.com/""" -from .common import Extractor, Message -from . import gelbooru_v02 -from .. import text, exception import binascii +from .. import exception +from .. import text +from . import gelbooru_v02 +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?(?:www\.)?gelbooru\.com/(?:index\.php)?\?" -class GelbooruBase(): +class GelbooruBase: """Base class for gelbooru extractors""" + category = "gelbooru" basecategory = "booru" root = "https://gelbooru.com" @@ -112,7 +114,7 @@ def _file_url(post): url = post["file_url"] if url.endswith((".webm", ".mp4")): md5 = post["md5"] - path = "/images/{}/{}/{}.webm".format(md5[0:2], md5[2:4], md5) + path = f"/images/{md5[0:2]}/{md5[2:4]}/{md5}.webm" post["_fallback"] = GelbooruBase._video_fallback(path) url = "https://img3.gelbooru.com" + path return url @@ -123,36 +125,38 @@ def _video_fallback(path): yield "https://img1.gelbooru.com" + path def _notes(self, post, page): - notes_data = text.extr(page, '
') + notes_data = text.extr(page, '
") if not notes_data: return post["notes"] = notes = [] extr = text.extract - for note in text.extract_iter(notes_data, ''): - notes.append({ - "width" : int(extr(note, 'data-width="', '"')[0]), - "height": int(extr(note, 'data-height="', '"')[0]), - "x" : int(extr(note, 'data-x="', '"')[0]), - "y" : int(extr(note, 'data-y="', '"')[0]), - "body" : extr(note, 'data-body="', '"')[0], - }) + for note in text.extract_iter(notes_data, ""): + notes.append( + { + "width": int(extr(note, 'data-width="', '"')[0]), + "height": int(extr(note, 'data-height="', '"')[0]), + "x": int(extr(note, 'data-x="', '"')[0]), + "y": int(extr(note, 'data-y="', '"')[0]), + "body": extr(note, 'data-body="', '"')[0], + } + ) def _skip_offset(self, num): self.offset += num return num -class GelbooruTagExtractor(GelbooruBase, - gelbooru_v02.GelbooruV02TagExtractor): +class GelbooruTagExtractor(GelbooruBase, gelbooru_v02.GelbooruV02TagExtractor): """Extractor for images from gelbooru.com based on search-tags""" + pattern = BASE_PATTERN + r"page=post&s=list&tags=([^&#]*)" example = "https://gelbooru.com/index.php?page=post&s=list&tags=TAG" -class GelbooruPoolExtractor(GelbooruBase, - gelbooru_v02.GelbooruV02PoolExtractor): +class GelbooruPoolExtractor(GelbooruBase, gelbooru_v02.GelbooruV02PoolExtractor): """Extractor for gelbooru pools""" + per_page = 45 pattern = BASE_PATTERN + r"page=pool&s=show&id=(\d+)" example = "https://gelbooru.com/index.php?page=pool&s=show&id=12345" @@ -163,8 +167,8 @@ def metadata(self): url = self.root + "/index.php" self._params = { "page": "pool", - "s" : "show", - "id" : self.pool_id, + "s": "show", + "id": self.pool_id, } page = self.request(url, params=self._params).text @@ -181,9 +185,9 @@ def posts(self): return self._pagination_html(self._params) -class GelbooruFavoriteExtractor(GelbooruBase, - gelbooru_v02.GelbooruV02FavoriteExtractor): +class GelbooruFavoriteExtractor(GelbooruBase, gelbooru_v02.GelbooruV02FavoriteExtractor): """Extractor for gelbooru favorites""" + per_page = 100 pattern = BASE_PATTERN + r"page=favorites&s=view&id=(\d+)" example = "https://gelbooru.com/index.php?page=favorites&s=view&id=12345" @@ -193,8 +197,8 @@ class GelbooruFavoriteExtractor(GelbooruBase, def posts(self): # get number of favorites params = { - "s" : "favorite", - "id" : self.favorite_id, + "s": "favorite", + "id": self.favorite_id, "limit": "2", } data = self._api_request(params, None, True) @@ -207,12 +211,11 @@ def posts(self): order = 1 if favs[0]["id"] < favs[1]["id"] else -1 except LookupError as exc: self.log.debug( - "Error when determining API favorite order (%s: %s)", - exc.__class__.__name__, exc) + "Error when determining API favorite order (%s: %s)", exc.__class__.__name__, exc + ) order = -1 else: - self.log.debug("API yields favorites in %sscending order", - "a" if order > 0 else "de") + self.log.debug("API yields favorites in %sscending order", "a" if order > 0 else "de") order_favs = self.config("order-posts") if order_favs and order_favs[0] in ("r", "a"): @@ -250,11 +253,11 @@ def _pagination(self, params, count): params["pid"] += 1 def _pagination_reverse(self, params, count): - pnum, last = divmod(count-1, self.per_page) + pnum, last = divmod(count - 1, self.per_page) if self.offset > last: # page number change self.offset -= last - diff, self.offset = divmod(self.offset-1, self.per_page) + diff, self.offset = divmod(self.offset - 1, self.per_page) pnum -= diff + 1 skip = self.offset @@ -279,20 +282,20 @@ def _pagination_reverse(self, params, count): return -class GelbooruPostExtractor(GelbooruBase, - gelbooru_v02.GelbooruV02PostExtractor): +class GelbooruPostExtractor(GelbooruBase, gelbooru_v02.GelbooruV02PostExtractor): """Extractor for single images from gelbooru.com""" - pattern = (BASE_PATTERN + - r"(?=(?:[^#]+&)?page=post(?:&|#|$))" - r"(?=(?:[^#]+&)?s=view(?:&|#|$))" - r"(?:[^#]+&)?id=(\d+)") + + pattern = ( + BASE_PATTERN + r"(?=(?:[^#]+&)?page=post(?:&|#|$))" + r"(?=(?:[^#]+&)?s=view(?:&|#|$))" + r"(?:[^#]+&)?id=(\d+)" + ) example = "https://gelbooru.com/index.php?page=post&s=view&id=12345" class GelbooruRedirectExtractor(GelbooruBase, Extractor): subcategory = "redirect" - pattern = (r"(?:https?://)?(?:www\.)?gelbooru\.com" - r"/redirect\.php\?s=([^&#]+)") + pattern = r"(?:https?://)?(?:www\.)?gelbooru\.com" r"/redirect\.php\?s=([^&#]+)" example = "https://gelbooru.com/redirect.php?s=BASE64" def __init__(self, match): @@ -300,7 +303,6 @@ def __init__(self, match): self.url_base64 = match.group(1) def items(self): - url = text.ensure_http_scheme(binascii.a2b_base64( - self.url_base64).decode()) + url = text.ensure_http_scheme(binascii.a2b_base64(self.url_base64).decode()) data = {"_extractor": GelbooruPostExtractor} yield Message.Queue, url, data diff --git a/gallery_dl/extractor/gelbooru_v01.py b/gallery_dl/extractor/gelbooru_v01.py index 0b96048b6..1d260a69a 100644 --- a/gallery_dl/extractor/gelbooru_v01.py +++ b/gallery_dl/extractor/gelbooru_v01.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,8 +6,8 @@ """Extractors for Gelbooru Beta 0.1.11 sites""" -from . import booru from .. import text +from . import booru class GelbooruV01Extractor(booru.BooruExtractor): @@ -17,27 +15,24 @@ class GelbooruV01Extractor(booru.BooruExtractor): per_page = 20 def _parse_post(self, post_id): - url = "{}/index.php?page=post&s=view&id={}".format( - self.root, post_id) + url = f"{self.root}/index.php?page=post&s=view&id={post_id}" extr = text.extract_from(self.request(url).text) post = { - "id" : post_id, - "created_at": extr('Posted: ', ' <'), - "uploader" : extr('By: ', ' <'), - "width" : extr('Size: ', 'x'), - "height" : extr('', ' <'), - "source" : extr('Source: ', ' <'), - "rating" : (extr('Rating: ', '<') or "?")[0].lower(), - "score" : extr('Score: ', ' <'), - "file_url" : extr('img', '<')), + "id": post_id, + "created_at": extr("Posted: ", " <"), + "uploader": extr("By: ", " <"), + "width": extr("Size: ", "x"), + "height": extr("", " <"), + "source": extr("Source: ", " <"), + "rating": (extr("Rating: ", "<") or "?")[0].lower(), + "score": extr("Score: ", " <"), + "file_url": extr('img', "<")), } post["md5"] = post["file_url"].rpartition("/")[2].partition(".")[0] - post["date"] = text.parse_datetime( - post["created_at"], "%Y-%m-%d %H:%M:%S") + post["date"] = text.parse_datetime(post["created_at"], "%Y-%m-%d %H:%M:%S") return post @@ -61,28 +56,30 @@ def _pagination(self, url, begin, end): pid += self.per_page -BASE_PATTERN = GelbooruV01Extractor.update({ - "thecollection": { - "root": "https://the-collection.booru.org", - "pattern": r"the-collection\.booru\.org", - }, - "illusioncardsbooru": { - "root": "https://illusioncards.booru.org", - "pattern": r"illusioncards\.booru\.org", - }, - "allgirlbooru": { - "root": "https://allgirl.booru.org", - "pattern": r"allgirl\.booru\.org", - }, - "drawfriends": { - "root": "https://drawfriends.booru.org", - "pattern": r"drawfriends\.booru\.org", - }, - "vidyart2": { - "root": "https://vidyart2.booru.org", - "pattern": r"vidyart2\.booru\.org", - }, -}) +BASE_PATTERN = GelbooruV01Extractor.update( + { + "thecollection": { + "root": "https://the-collection.booru.org", + "pattern": r"the-collection\.booru\.org", + }, + "illusioncardsbooru": { + "root": "https://illusioncards.booru.org", + "pattern": r"illusioncards\.booru\.org", + }, + "allgirlbooru": { + "root": "https://allgirl.booru.org", + "pattern": r"allgirl\.booru\.org", + }, + "drawfriends": { + "root": "https://drawfriends.booru.org", + "pattern": r"drawfriends\.booru\.org", + }, + "vidyart2": { + "root": "https://vidyart2.booru.org", + "pattern": r"vidyart2\.booru\.org", + }, + } +) class GelbooruV01TagExtractor(GelbooruV01Extractor): @@ -100,8 +97,7 @@ def metadata(self): return {"search_tags": text.unquote(self.tags.replace("+", " "))} def posts(self): - url = "{}/index.php?page=post&s=list&tags={}&pid=".format( - self.root, self.tags) + url = f"{self.root}/index.php?page=post&s=list&tags={self.tags}&pid=" return self._pagination(url, 'class="thumb">')) + tag_container = text.extr(page, '
").replace("\r\n", "\n"), "", "" + ) + ), + "ratings": [ + text.unescape(r) + for r in text.extract_iter(extr("class='ratings_box'", ""), "title='", "'") + ], + "date": text.parse_datetime(extr("datetime='", "'")), + "views": text.parse_int(extr(">Views", "<")), + "score": text.parse_int(extr(">Vote Score", "<")), + "media": text.unescape(extr(">Media", "<").strip()), + "tags": text.split_html(extr(">Tags ", "")), } body = data["_body"] if "", "").rpartition(">")[2]), - "author" : text.unescape(extr('alt="', '"')), - "date" : text.parse_datetime(extr( - ">Updated<", "").rpartition(">")[2], "%B %d, %Y"), - "status" : extr("class='indent'>", "<"), + "user": self.user, + "title": text.unescape(extr("
", "").rpartition(">")[2]), + "author": text.unescape(extr('alt="', '"')), + "date": text.parse_datetime( + extr(">Updated<", "").rpartition(">")[2], "%B %d, %Y" + ), + "status": extr("class='indent'>", "<"), } for c in ("Chapters", "Words", "Comments", "Views", "Rating"): - data[c.lower()] = text.parse_int(extr( - ">" + c + ":", "<").replace(",", "")) + data[c.lower()] = text.parse_int(extr(">" + c + ":", "<").replace(",", "")) - data["description"] = text.unescape(extr( - "class='storyDescript'>", "", ""), "title='", "'")] + data["ratings"] = [ + text.unescape(r) + for r in text.extract_iter(extr("class='ratings_box'", "
"), "title='", "'") + ] return text.nameext_from_url(data["src"], data) @@ -140,16 +141,14 @@ def _request_check(self, url, **kwargs): # and update PHPSESSID and content filters if necessary response = self.request(url, **kwargs) content = response.content - if len(content) < 5000 and \ - b'
', '') + url = f"{self.root}/stories/user/{self.user}" + return self._pagination(url, '
', "") class HentaifoundryStoryExtractor(HentaifoundryExtractor): """Extractor for a hentaifoundry story""" + subcategory = "story" archive_fmt = "s_{index}" pattern = BASE_PATTERN + r"/stories/user/([^/?#]+)/(\d+)" @@ -354,8 +359,7 @@ def __init__(self, match): self.index = match.group(3) def items(self): - story_url = "{}/stories/user/{}/{}/x?enterAgree=1".format( - self.root, self.user, self.index) + story_url = f"{self.root}/stories/user/{self.user}/{self.index}/x?enterAgree=1" story = self._parse_story(self.request(story_url).text) yield Message.Directory, story yield Message.Url, story["src"], story diff --git a/gallery_dl/extractor/hentaifox.py b/gallery_dl/extractor/hentaifox.py index 31a302d16..e77c123d4 100644 --- a/gallery_dl/extractor/hentaifox.py +++ b/gallery_dl/extractor/hentaifox.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,18 +6,23 @@ """Extractors for https://hentaifox.com/""" -from .common import GalleryExtractor, Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import GalleryExtractor +from .common import Message -class HentaifoxBase(): +class HentaifoxBase: """Base class for hentaifox extractors""" + category = "hentaifox" root = "https://hentaifox.com" class HentaifoxGalleryExtractor(HentaifoxBase, GalleryExtractor): """Extractor for image galleries on hentaifox.com""" + pattern = r"(?:https?://)?(?:www\.)?hentaifox\.com(/gallery/(\d+))" example = "https://hentaifox.com/gallery/12345/" @@ -31,8 +34,7 @@ def __init__(self, match): def _split(txt): return [ text.remove_html(tag.partition(">")[2], "", "") - for tag in text.extract_iter( - txt, "class='tag_btn", "Tags:", "")), + "artist": split(extr(">Artists:", "")), + "group": split(extr(">Groups:", "")), + "type": text.remove_html(extr(">Category:", ""): url, _, title = info.partition('">') yield { - "url" : text.urljoin(self.root, url), - "gallery_id": text.parse_int( - url.strip("/").rpartition("/")[2]), - "title" : text.unescape(title), + "url": text.urljoin(self.root, url), + "gallery_id": text.parse_int(url.strip("/").rpartition("/")[2]), + "title": text.unescape(title), "_extractor": HentaifoxGalleryExtractor, } diff --git a/gallery_dl/extractor/hentaihand.py b/gallery_dl/extractor/hentaihand.py index f3f43c473..c5c2dd963 100644 --- a/gallery_dl/extractor/hentaihand.py +++ b/gallery_dl/extractor/hentaihand.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,16 @@ """Extractors for https://hentaihand.com/""" -from .common import GalleryExtractor, Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import GalleryExtractor +from .common import Message class HentaihandGalleryExtractor(GalleryExtractor): """Extractor for image galleries on hentaihand.com""" + category = "hentaihand" root = "https://hentaihand.com" pattern = r"(?:https?://)?(?:www\.)?hentaihand\.com/\w+/comic/([\w-]+)" @@ -21,25 +23,23 @@ class HentaihandGalleryExtractor(GalleryExtractor): def __init__(self, match): self.slug = match.group(1) - url = "{}/api/comics/{}".format(self.root, self.slug) + url = f"{self.root}/api/comics/{self.slug}" GalleryExtractor.__init__(self, match, url) def metadata(self, page): info = util.json_loads(page) data = { - "gallery_id" : text.parse_int(info["id"]), - "title" : info["title"], - "title_alt" : info["alternative_title"], - "slug" : self.slug, - "type" : info["category"]["name"], - "language" : info["language"]["name"], - "lang" : util.language_to_code(info["language"]["name"]), - "tags" : [t["slug"] for t in info["tags"]], - "date" : text.parse_datetime( - info["uploaded_at"], "%Y-%m-%d"), + "gallery_id": text.parse_int(info["id"]), + "title": info["title"], + "title_alt": info["alternative_title"], + "slug": self.slug, + "type": info["category"]["name"], + "language": info["language"]["name"], + "lang": util.language_to_code(info["language"]["name"]), + "tags": [t["slug"] for t in info["tags"]], + "date": text.parse_datetime(info["uploaded_at"], "%Y-%m-%d"), } - for key in ("artists", "authors", "groups", "characters", - "relationships", "parodies"): + for key in ("artists", "authors", "groups", "characters", "relationships", "parodies"): data[key] = [v["name"] for v in info[key]] return data @@ -50,12 +50,15 @@ def images(self, _): class HentaihandTagExtractor(Extractor): """Extractor for tag searches on hentaihand.com""" + category = "hentaihand" subcategory = "tag" root = "https://hentaihand.com" - pattern = (r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com" - r"/\w+/(parody|character|tag|artist|group|language" - r"|category|relationship)/([^/?#]+)") + pattern = ( + r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com" + r"/\w+/(parody|character|tag|artist|group|language" + r"|category|relationship)/([^/?#]+)" + ) example = "https://hentaihand.com/en/tag/TAG" def __init__(self, match): @@ -63,22 +66,19 @@ def __init__(self, match): self.type, self.key = match.groups() def items(self): - if self.type[-1] == "y": - tpl = self.type[:-1] + "ies" - else: - tpl = self.type + "s" + tpl = self.type[:-1] + "ies" if self.type[-1] == "y" else self.type + "s" - url = "{}/api/{}/{}".format(self.root, tpl, self.key) + url = f"{self.root}/api/{tpl}/{self.key}" tid = self.request(url, notfound=self.type).json()["id"] url = self.root + "/api/comics" params = { "per_page": "18", - tpl : tid, - "page" : 1, - "q" : "", - "sort" : "uploaded_at", - "order" : "desc", + tpl: tid, + "page": 1, + "q": "", + "sort": "uploaded_at", + "order": "desc", "duration": "day", } while True: diff --git a/gallery_dl/extractor/hentaihere.py b/gallery_dl/extractor/hentaihere.py index ba9558c0d..0f4f17680 100644 --- a/gallery_dl/extractor/hentaihere.py +++ b/gallery_dl/extractor/hentaihere.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,31 +6,36 @@ """Extractors for https://hentaihere.com/""" -from .common import ChapterExtractor, MangaExtractor -from .. import text, util import re +from .. import text +from .. import util +from .common import ChapterExtractor +from .common import MangaExtractor + -class HentaihereBase(): +class HentaihereBase: """Base class for hentaihere extractors""" + category = "hentaihere" root = "https://hentaihere.com" class HentaihereChapterExtractor(HentaihereBase, ChapterExtractor): """Extractor for a single manga chapter from hentaihere.com""" + archive_fmt = "{chapter_id}_{page}" pattern = r"(?:https?://)?(?:www\.)?hentaihere\.com/m/S(\d+)/([^/?#]+)" example = "https://hentaihere.com/m/S12345/1/1/" def __init__(self, match): self.manga_id, self.chapter = match.groups() - url = "{}/m/S{}/{}/1".format(self.root, self.manga_id, self.chapter) + url = f"{self.root}/m/S{self.manga_id}/{self.chapter}/1" ChapterExtractor.__init__(self, match, url) def metadata(self, page): title = text.extr(page, "", "") - chapter_id = text.extr(page, 'report/C', '"') + chapter_id = text.extr(page, "report/C", '"') chapter, sep, minor = self.chapter.partition(".") pattern = r"Page 1 \| (.+) \(([^)]+)\) - Chapter \d+: (.+) by (.+) at " match = re.match(pattern, title) @@ -52,14 +55,12 @@ def metadata(self, page): @staticmethod def images(page): images = text.extr(page, "var rff_imageList = ", ";") - return [ - ("https://hentaicdn.com/hentai" + part, None) - for part in util.json_loads(images) - ] + return [("https://hentaicdn.com/hentai" + part, None) for part in util.json_loads(images)] class HentaihereMangaExtractor(HentaihereBase, MangaExtractor): """Extractor for hmanga from hentaihere.com""" + chapterclass = HentaihereChapterExtractor pattern = r"(?:https?://)?(?:www\.)?hentaihere\.com(/m/S\d+)/?$" example = "https://hentaihere.com/m/S12345" @@ -68,33 +69,34 @@ def chapters(self, page): results = [] pos = page.find('itemscope itemtype="http://schema.org/Book') + 1 - manga, pos = text.extract( - page, '', '', pos) - mtype, pos = text.extract( - page, '[', ']', pos) - manga_id = text.parse_int( - self.manga_url.rstrip("/").rpartition("/")[2][1:]) + manga, pos = text.extract(page, '', "", pos) + mtype, pos = text.extract(page, '[', "]", pos) + manga_id = text.parse_int(self.manga_url.rstrip("/").rpartition("/")[2][1:]) while True: - marker, pos = text.extract( - page, '
  • ', '', pos) + marker, pos = text.extract(page, '
  • ', "", pos) if marker is None: return results url, pos = text.extract(page, '\n', '<', pos) - chapter_id, pos = text.extract(page, '/C', '"', pos) + chapter, pos = text.extract(page, 'title="Tagged: -">\n', "<", pos) + chapter_id, pos = text.extract(page, "/C", '"', pos) chapter, _, title = text.unescape(chapter).strip().partition(" - ") chapter, sep, minor = chapter.partition(".") - results.append((url, { - "manga_id": manga_id, - "manga": manga, - "chapter": text.parse_int(chapter), - "chapter_minor": sep + minor, - "chapter_id": text.parse_int(chapter_id), - "type": mtype, - "title": title, - "lang": "en", - "language": "English", - })) + results.append( + ( + url, + { + "manga_id": manga_id, + "manga": manga, + "chapter": text.parse_int(chapter), + "chapter_minor": sep + minor, + "chapter_id": text.parse_int(chapter_id), + "type": mtype, + "title": title, + "lang": "en", + "language": "English", + }, + ) + ) diff --git a/gallery_dl/extractor/hentainexus.py b/gallery_dl/extractor/hentainexus.py index 286ee381a..aa29ee5a4 100644 --- a/gallery_dl/extractor/hentainexus.py +++ b/gallery_dl/extractor/hentainexus.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,22 +6,27 @@ """Extractors for https://hentainexus.com/""" -from .common import GalleryExtractor, Extractor, Message -from .. import text, util import binascii +from contextlib import suppress + +from .. import text +from .. import util +from .common import Extractor +from .common import GalleryExtractor +from .common import Message class HentainexusGalleryExtractor(GalleryExtractor): """Extractor for hentainexus galleries""" + category = "hentainexus" root = "https://hentainexus.com" - pattern = (r"(?i)(?:https?://)?(?:www\.)?hentainexus\.com" - r"/(?:view|read)/(\d+)") + pattern = r"(?i)(?:https?://)?(?:www\.)?hentainexus\.com" r"/(?:view|read)/(\d+)" example = "https://hentainexus.com/view/12345" def __init__(self, match): self.gallery_id = match.group(1) - url = "{}/view/{}".format(self.root, self.gallery_id) + url = f"{self.root}/view/{self.gallery_id}" GalleryExtractor.__init__(self, match, url) def metadata(self, page): @@ -31,13 +34,22 @@ def metadata(self, page): extr = text.extract_from(page) data = { "gallery_id": text.parse_int(self.gallery_id), - "cover" : extr('"og:image" content="', '"'), - "title" : extr('

    ', '

    '), + "cover": extr('"og:image" content="', '"'), + "title": extr('

    ', "

    "), } - for key in ("Artist", "Book", "Circle", "Event", "Language", - "Magazine", "Parody", "Publisher", "Description"): - value = rmve(extr('viewcolumn">' + key + '', '')) + for key in ( + "Artist", + "Book", + "Circle", + "Event", + "Language", + "Magazine", + "Parody", + "Publisher", + "Description", + ): + value = rmve(extr('viewcolumn">' + key + "", "")) value, sep, rest = value.rpartition(" (") data[key.lower()] = value if sep else rest @@ -59,10 +71,9 @@ def metadata(self, page): return data def images(self, _): - url = "{}/read/{}".format(self.root, self.gallery_id) + url = f"{self.root}/read/{self.gallery_id}" page = self.request(url).text - imgs = util.json_loads(self._decode(text.extr( - page, 'initReader("', '"'))) + imgs = util.json_loads(self._decode(text.extr(page, 'initReader("', '"'))) headers = None if not self.config("original", True): @@ -72,10 +83,8 @@ def images(self, _): results = [] for img in imgs: - try: + with suppress(KeyError): results.append((img["image"], img)) - except KeyError: - pass return results @staticmethod @@ -84,7 +93,7 @@ def _decode(data): hostname = "hentainexus.com" primes = (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53) blob = list(binascii.a2b_base64(data)) - for i in range(0, len(hostname)): + for i in range(len(hostname)): blob[i] = blob[i] ^ ord(hostname[i]) key = blob[0:64] @@ -93,10 +102,7 @@ def _decode(data): for k in key: C = C ^ k for _ in range(8): - if C & 1: - C = C >> 1 ^ 0xc - else: - C = C >> 1 + C = C >> 1 ^ 12 if C & 1 else C >> 1 k = primes[C & 0x7] x = 0 @@ -120,45 +126,45 @@ def _decode(data): @staticmethod def _join_title(data): - event = data['event'] - artist = data['artist'] - circle = data['circle'] - title = data['title'] - parody = data['parody'] - book = data['book'] - magazine = data['magazine'] + event = data["event"] + artist = data["artist"] + circle = data["circle"] + title = data["title"] + parody = data["parody"] + book = data["book"] + magazine = data["magazine"] # a few galleries have a large number of artists or parodies, # which get replaced with "Various" in the title string - if artist.count(',') >= 3: - artist = 'Various' - if parody.count(',') >= 3: - parody = 'Various' + if artist.count(",") >= 3: + artist = "Various" + if parody.count(",") >= 3: + parody = "Various" - jt = '' + jt = "" if event: - jt += '({}) '.format(event) + jt += f"({event}) " if circle: - jt += '[{} ({})] '.format(circle, artist) + jt += f"[{circle} ({artist})] " else: - jt += '[{}] '.format(artist) + jt += f"[{artist}] " jt += title - if parody.lower() != 'original work': - jt += ' ({})'.format(parody) + if parody.lower() != "original work": + jt += f" ({parody})" if book: - jt += ' ({})'.format(book) + jt += f" ({book})" if magazine: - jt += ' ({})'.format(magazine) + jt += f" ({magazine})" return jt class HentainexusSearchExtractor(Extractor): """Extractor for hentainexus search results""" + category = "hentainexus" subcategory = "search" root = "https://hentainexus.com" - pattern = (r"(?i)(?:https?://)?(?:www\.)?hentainexus\.com" - r"(?:/page/\d+)?/?(?:\?(q=[^/?#]+))?$") + pattern = r"(?i)(?:https?://)?(?:www\.)?hentainexus\.com" r"(?:/page/\d+)?/?(?:\?(q=[^/?#]+))?$" example = "https://hentainexus.com/?q=QUERY" def items(self): diff --git a/gallery_dl/extractor/hiperdex.py b/gallery_dl/extractor/hiperdex.py index c939a3c55..bbad5f1a2 100644 --- a/gallery_dl/extractor/hiperdex.py +++ b/gallery_dl/extractor/hiperdex.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,50 +6,44 @@ """Extractors for https://hipertoon.com/""" -from .common import ChapterExtractor, MangaExtractor +import re + from .. import text from ..cache import memcache -import re +from .common import ChapterExtractor +from .common import MangaExtractor -BASE_PATTERN = (r"((?:https?://)?(?:www\.)?" - r"(?:1st)?hiper(?:dex|toon)\d?\.(?:com|net|info|top))") +BASE_PATTERN = r"((?:https?://)?(?:www\.)?" r"(?:1st)?hiper(?:dex|toon)\d?\.(?:com|net|info|top))" -class HiperdexBase(): +class HiperdexBase: """Base class for hiperdex extractors""" + category = "hiperdex" root = "https://hipertoon.com" @memcache(keyarg=1) def manga_data(self, manga, page=None): if not page: - url = "{}/manga/{}/".format(self.root, manga) + url = f"{self.root}/manga/{manga}/" page = self.request(url).text extr = text.extract_from(page) return { - "url" : text.unescape(extr( - 'property="og:url" content="', '"')), - "manga" : text.unescape(extr( - ' property="name" title="', '"')), - "score" : text.parse_float(extr( - 'id="averagerate">', '<')), - "author" : text.remove_html(extr( - 'class="author-content">', '
    • ')), - "artist" : text.remove_html(extr( - 'class="artist-content">', '
    ')), - "genre" : text.split_html(extr( - 'class="genres-content">', ''))[::2], - "type" : extr( - 'class="summary-content">', '<').strip(), - "release": text.parse_int(text.remove_html(extr( - 'class="summary-content">', ''))), - "status" : extr( - 'class="summary-content">', '<').strip(), - "description": text.remove_html(text.unescape(extr( - "Summary ", ""))), + "url": text.unescape(extr('property="og:url" content="', '"')), + "manga": text.unescape(extr(' property="name" title="', '"')), + "score": text.parse_float(extr('id="averagerate">', "<")), + "author": text.remove_html(extr('class="author-content">', "")), + "artist": text.remove_html(extr('class="artist-content">', "")), + "genre": text.split_html(extr('class="genres-content">', ""))[::2], + "type": extr('class="summary-content">', "<").strip(), + "release": text.parse_int(text.remove_html(extr('class="summary-content">', ""))), + "status": extr('class="summary-content">', "<").strip(), + "description": text.remove_html( + text.unescape(extr("Summary ", "")) + ), "language": "English", - "lang" : "en", + "lang": "en", } def chapter_data(self, chapter): @@ -59,7 +51,7 @@ def chapter_data(self, chapter): chapter = chapter[8:] chapter, _, minor = chapter.partition("-") data = { - "chapter" : text.parse_int(chapter), + "chapter": text.parse_int(chapter), "chapter_minor": "." + minor if minor and minor != "end" else "", } data.update(self.manga_data(self.manga.lower())) @@ -68,6 +60,7 @@ def chapter_data(self, chapter): class HiperdexChapterExtractor(HiperdexBase, ChapterExtractor): """Extractor for hiperdex manga chapters""" + pattern = BASE_PATTERN + r"(/mangas?/([^/?#]+)/([^/?#]+))" example = "https://hipertoon.com/manga/MANGA/CHAPTER/" @@ -82,13 +75,13 @@ def metadata(self, _): def images(self, page): return [ (url.strip(), None) - for url in re.findall( - r'id="image-\d+"\s+(?:data-)?src="([^"]+)', page) + for url in re.findall(r'id="image-\d+"\s+(?:data-)?src="([^"]+)', page) ] class HiperdexMangaExtractor(HiperdexBase, MangaExtractor): """Extractor for hiperdex manga""" + chapterclass = HiperdexChapterExtractor pattern = BASE_PATTERN + r"(/mangas?/([^/?#]+))/?$" example = "https://hipertoon.com/manga/MANGA/" @@ -112,8 +105,7 @@ def chapters(self, page): html = self.request(url, method="POST", headers=headers).text results = [] - for item in text.extract_iter( - html, '
  • "): url = text.extr(item, 'href="', '"') chapter = url.rstrip("/").rpartition("/")[2] results.append((url, self.chapter_data(chapter))) @@ -122,6 +114,7 @@ def chapters(self, page): class HiperdexArtistExtractor(HiperdexBase, MangaExtractor): """Extractor for an artists's manga on hiperdex""" + subcategory = "artist" categorytransfer = False chapterclass = HiperdexMangaExtractor @@ -135,7 +128,7 @@ def __init__(self, match): def chapters(self, page): results = [] - for info in text.extract_iter(page, 'id="manga-item-', '= total: return class HitomiIndexExtractor(HitomiTagExtractor): """Extractor for galleries from index searches on hitomi.la""" + subcategory = "index" pattern = r"(?:https?://)?hitomi\.la/(\w+)-(\w+)\.html" example = "https://hitomi.la/index-LANG.html" @@ -163,8 +170,7 @@ def __init__(self, match): def items(self): data = {"_extractor": HitomiGalleryExtractor} - nozomi_url = "https://ltn.hitomi.la/{}-{}.nozomi".format( - self.tag, self.language) + nozomi_url = f"https://ltn.hitomi.la/{self.tag}-{self.language}.nozomi" headers = { "Origin": self.root, "Cache-Control": "max-age=0", @@ -173,26 +179,26 @@ def items(self): offset = 0 total = None while True: - headers["Referer"] = "{}/{}-{}.html?page={}".format( - self.root, self.tag, self.language, offset // 100 + 1) - headers["Range"] = "bytes={}-{}".format(offset, offset+99) + headers["Referer"] = ( + f"{self.root}/{self.tag}-{self.language}.html?page={offset // 100 + 1}" + ) + headers["Range"] = f"bytes={offset}-{offset + 99}" response = self.request(nozomi_url, headers=headers) for gallery_id in decode_nozomi(response.content): - gallery_url = "{}/galleries/{}.html".format( - self.root, gallery_id) + gallery_url = f"{self.root}/galleries/{gallery_id}.html" yield Message.Queue, gallery_url, data offset += 100 if total is None: - total = text.parse_int( - response.headers["content-range"].rpartition("/")[2]) + total = text.parse_int(response.headers["content-range"].rpartition("/")[2]) if offset >= total: return class HitomiSearchExtractor(Extractor): """Extractor for galleries from multiple tag searches on hitomi.la""" + category = "hitomi" subcategory = "search" root = "https://hitomi.la" @@ -211,28 +217,23 @@ def items(self): intersects = set.intersection(*results) for gallery_id in sorted(intersects, reverse=True): - gallery_url = "{}/galleries/{}.html".format( - self.root, gallery_id) + gallery_url = f"{self.root}/galleries/{gallery_id}.html" yield Message.Queue, gallery_url, data def get_nozomi_items(self, full_tag): area, tag, language = self.get_nozomi_args(full_tag) if area: - referer_base = "{}/n/{}/{}-{}.html".format( - self.root, area, tag, language) - nozomi_url = "https://ltn.hitomi.la/{}/{}-{}.nozomi".format( - area, tag, language) + referer_base = f"{self.root}/n/{area}/{tag}-{language}.html" + nozomi_url = f"https://ltn.hitomi.la/{area}/{tag}-{language}.nozomi" else: - referer_base = "{}/n/{}-{}.html".format( - self.root, tag, language) - nozomi_url = "https://ltn.hitomi.la/{}-{}.nozomi".format( - tag, language) + referer_base = f"{self.root}/n/{tag}-{language}.html" + nozomi_url = f"https://ltn.hitomi.la/{tag}-{language}.nozomi" headers = { "Origin": self.root, "Cache-Control": "max-age=0", - "Referer": "{}/search.html?{}".format(referer_base, self.query), + "Referer": f"{referer_base}/search.html?{self.query}", } response = self.request(nozomi_url, headers=headers) @@ -261,8 +262,7 @@ def _parse_gg(extr): m = {} keys = [] - for match in re.finditer( - r"case\s+(\d+):(?:\s*o\s*=\s*(\d+))?", page): + for match in re.finditer(r"case\s+(\d+):(?:\s*o\s*=\s*(\d+))?", page): key, value = match.groups() keys.append(int(key)) @@ -272,8 +272,7 @@ def _parse_gg(extr): m[key] = value keys.clear() - for match in re.finditer( - r"if\s+\(g\s*===?\s*(\d+)\)[\s{]*o\s*=\s*(\d+)", page): + for match in re.finditer(r"if\s+\(g\s*===?\s*(\d+)\)[\s{]*o\s*=\s*(\d+)", page): m[int(match.group(1))] = int(match.group(2)) d = re.search(r"(?:var\s|default:)\s*o\s*=\s*(\d+)", page) diff --git a/gallery_dl/extractor/hotleak.py b/gallery_dl/extractor/hotleak.py index ddfc54b33..14d565927 100644 --- a/gallery_dl/extractor/hotleak.py +++ b/gallery_dl/extractor/hotleak.py @@ -1,22 +1,27 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://hotleak.vip/""" -from .common import Extractor, Message -from .. import text, exception import binascii +from .. import exception +from .. import text +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?(?:www\.)?hotleak\.vip" class HotleakExtractor(Extractor): """Base class for hotleak extractors""" + category = "hotleak" - directory_fmt = ("{category}", "{creator}",) + directory_fmt = ( + "{category}", + "{creator}", + ) filename_fmt = "{creator}_{id}.{extension}" archive_fmt = "{type}_{creator}_{id}" root = "https://hotleak.vip" @@ -25,9 +30,7 @@ def items(self): for post in self.posts(): if not post["url"].startswith("ytdl:"): post["url"] = ( - post["url"] - .replace("/storage/storage/", "/storage/") - .replace("_thumb.", ".") + post["url"].replace("/storage/storage/", "/storage/").replace("_thumb.", ".") ) post["_http_expected_status"] = (404,) yield Message.Directory, post @@ -46,8 +49,7 @@ def _pagination(self, url, params): if "" not in page: return - for item in text.extract_iter( - page, '') + page = text.extr(page, '
    ', "") data = { - "id" : text.parse_int(self.id), + "id": text.parse_int(self.id), "creator": self.creator, - "type" : self.type, + "type": self.type, } if self.type == "photo": @@ -86,8 +88,7 @@ def posts(self): text.nameext_from_url(data["url"], data) elif self.type == "video": - data["url"] = "ytdl:" + decode_video_url(text.extr( - text.unescape(page), '"src":"', '"')) + data["url"] = "ytdl:" + decode_video_url(text.extr(text.unescape(page), '"src":"', '"')) text.nameext_from_url(data["url"], data) data["extension"] = "mp4" @@ -96,9 +97,9 @@ def posts(self): class HotleakCreatorExtractor(HotleakExtractor): """Extractor for all posts from a hotleak creator""" + subcategory = "creator" - pattern = (BASE_PATTERN + r"/(?!(?:hot|creators|videos|photos)(?:$|/))" - r"([^/?#]+)/?$") + pattern = BASE_PATTERN + r"/(?!(?:hot|creators|videos|photos)(?:$|/))" r"([^/?#]+)/?$" example = "https://hotleak.vip/MODEL" def __init__(self, match): @@ -106,7 +107,7 @@ def __init__(self, match): self.creator = match.group(1) def posts(self): - url = "{}/{}".format(self.root, self.creator) + url = f"{self.root}/{self.creator}" return self._pagination(url) def _pagination(self, url): @@ -115,12 +116,10 @@ def _pagination(self, url): while True: try: - response = self.request( - url, headers=headers, params=params, notfound="creator") + response = self.request(url, headers=headers, params=params, notfound="creator") except exception.HttpError as exc: if exc.response.status_code == 429: - self.wait( - until=exc.response.headers.get("X-RateLimit-Reset")) + self.wait(until=exc.response.headers.get("X-RateLimit-Reset")) continue raise @@ -139,8 +138,7 @@ def _pagination(self, url): elif post["type"] == 1: data["type"] = "video" - data["url"] = "ytdl:" + decode_video_url( - post["stream_url_play"]) + data["url"] = "ytdl:" + decode_video_url(post["stream_url_play"]) text.nameext_from_url(data["url"], data) data["extension"] = "mp4" @@ -150,6 +148,7 @@ def _pagination(self, url): class HotleakCategoryExtractor(HotleakExtractor): """Extractor for hotleak categories""" + subcategory = "category" pattern = BASE_PATTERN + r"/(hot|creators|videos|photos)(?:/?\?([^#]+))?" example = "https://hotleak.vip/photos" @@ -159,7 +158,7 @@ def __init__(self, match): self._category, self.params = match.groups() def items(self): - url = "{}/{}".format(self.root, self._category) + url = f"{self.root}/{self._category}" if self._category in ("hot", "creators"): data = {"_extractor": HotleakCreatorExtractor} @@ -172,6 +171,7 @@ def items(self): class HotleakSearchExtractor(HotleakExtractor): """Extractor for hotleak search results""" + subcategory = "search" pattern = BASE_PATTERN + r"/search(?:/?\?([^#]+))" example = "https://hotleak.vip/search?search=QUERY" diff --git a/gallery_dl/extractor/idolcomplex.py b/gallery_dl/extractor/idolcomplex.py index dfd9a3171..fba4fc839 100644 --- a/gallery_dl/extractor/idolcomplex.py +++ b/gallery_dl/extractor/idolcomplex.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,18 +6,22 @@ """Extractors for https://idol.sankakucomplex.com/""" -from .sankaku import SankakuExtractor -from .common import Message -from ..cache import cache -from .. import text, util, exception import collections import re +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from .common import Message +from .sankaku import SankakuExtractor + BASE_PATTERN = r"(?:https?://)?idol\.sankakucomplex\.com(?:/[a-z]{2})?" class IdolcomplexExtractor(SankakuExtractor): """Base class for idolcomplex extractors""" + category = "idolcomplex" root = "https://idol.sankakucomplex.com" cookies_domain = "idol.sankakucomplex.com" @@ -34,9 +36,7 @@ def __init__(self, match): self.start_post = 0 def _init(self): - self.find_pids = re.compile( - r" href=[\"#]/\w\w/posts/(\w+)" - ).findall + self.find_pids = re.compile(r" href=[\"#]/\w\w/posts/(\w+)").findall self.find_tags = re.compile( r'tag-type-([^"]+)">\s*]*?href="/[^?]*\?tags=([^"]+)' ).findall @@ -62,7 +62,7 @@ def post_ids(self): def login(self): if self.cookies_check(self.cookies_names): - return + return None username, password = self._get_auth_info() if username: @@ -70,7 +70,7 @@ def login(self): self.logged_in = False - @cache(maxage=90*86400, keyarg=1) + @cache(maxage=90 * 86400, keyarg=1) def _login_impl(self, username, password): self.log.info("Logging in as %s", username) @@ -80,15 +80,15 @@ def _login_impl(self, username, password): headers = { "Referer": url, } - url = self.root + (text.extr(page, '
    ") - vcnt = extr('>Votes:', "<") + vcnt = extr(">Votes:", "<") pid = extr(">Post ID:", "<") created = extr(' title="', '"') - file_url = extr('>Original:', 'id=') + file_url = extr(">Original:", "id=") if file_url: file_url = extr(' href="', '"') width = extr(">", "x") height = extr("", " ") else: - width = extr('') + width = extr("") file_url = extr('Rating:", "') + tags_html = text.extr(page, '")), - "genres" : text.split_html(extr( - "", "")), - "tags" : text.split_html(extr( - "", "")), - "uploader" : text.remove_html(extr( - "", "")), - "language" : extr(" ", "\n"), + "title": text.unescape(extr("

    ", "<").strip()), + "artists": text.split_html(extr("", "")), + "genres": text.split_html(extr("", "")), + "tags": text.split_html(extr("", "")), + "uploader": text.remove_html(extr("", "")), + "language": extr(" ", "\n"), } diff --git a/gallery_dl/extractor/toyhouse.py b/gallery_dl/extractor/toyhouse.py index 44d87ee83..21a270482 100644 --- a/gallery_dl/extractor/toyhouse.py +++ b/gallery_dl/extractor/toyhouse.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,14 +6,17 @@ """Extractors for https://toyhou.se/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?toyhou\.se" class ToyhouseExtractor(Extractor): """Base class for toyhouse extractors""" + category = "toyhouse" root = "https://toyhou.se" directory_fmt = ("{category}", "{user|artists!S}") @@ -51,17 +52,18 @@ def _parse_post(self, post, needle='\n
    ', '<'), - "%d %b %Y, %I:%M:%S %p"), + "date": text.parse_datetime( + extr('Credits\n

    \n
    ', "<"), "%d %b %Y, %I:%M:%S %p" + ), "artists": [ text.remove_html(artist) - for artist in extr( - '
    ', '
    \n
    ').split( - '
    ') + for artist in extr('
    ', "
    \n
    ").split( + '
    ' + ) + ], + "characters": text.split_html(extr('
    \n
    "))[ + 2: ], - "characters": text.split_html(extr( - '
    ', ''): + for post in text.extract_iter(page, '
  • ', "
    • "): cnt += 1 yield self._parse_post(post) @@ -89,24 +90,26 @@ def _pagination(self, path): def _accept_content_warning(self, page): pos = page.find(' name="_token"') + 1 token, pos = text.extract(page, ' value="', '"', pos) - user , pos = text.extract(page, ' value="', '"', pos) + user, pos = text.extract(page, ' value="', '"', pos) if not token or not user: return False data = {"_token": token, "user": user} - self.request(self.root + "/~account/warnings/accept", - method="POST", data=data, allow_redirects=False) + self.request( + self.root + "/~account/warnings/accept", method="POST", data=data, allow_redirects=False + ) return True class ToyhouseArtExtractor(ToyhouseExtractor): """Extractor for artworks of a toyhouse user""" + subcategory = "art" pattern = BASE_PATTERN + r"/([^/?#]+)/art" example = "https://www.toyhou.se/USER/art" def posts(self): - return self._pagination("/{}/art".format(self.user)) + return self._pagination(f"/{self.user}/art") def metadata(self): return {"user": self.user} @@ -114,14 +117,16 @@ def metadata(self): class ToyhouseImageExtractor(ToyhouseExtractor): """Extractor for individual toyhouse images""" + subcategory = "image" - pattern = (r"(?:https?://)?(?:" - r"(?:www\.)?toyhou\.se/~images|" - r"f\d+\.toyhou\.se/file/[^/?#]+/(?:image|watermark)s" - r")/(\d+)") + pattern = ( + r"(?:https?://)?(?:" + r"(?:www\.)?toyhou\.se/~images|" + r"f\d+\.toyhou\.se/file/[^/?#]+/(?:image|watermark)s" + r")/(\d+)" + ) example = "https://toyhou.se/~images/12345" def posts(self): - url = "{}/~images/{}".format(self.root, self.user) - return (self._parse_post( - self.request(url).text, '', '
    ')), - "date" : text.parse_datetime( - extr('id="Uploaded">', '
    ').strip(), "%Y %B %d"), - "rating" : text.parse_float(extr( - 'id="Rating">', '').partition(" ")[0]), - "type" : text.remove_html(extr('id="Category">' , '')), - "collection": text.remove_html(extr('id="Collection">', '')), - "group" : text.split_html(extr('id="Group">' , '')), - "artist" : text.split_html(extr('id="Artist">' , '')), - "parody" : text.split_html(extr('id="Parody">' , '')), - "characters": text.split_html(extr('id="Character">' , '')), - "tags" : text.split_html(extr('id="Tag">' , '')), - "language" : "English", - "lang" : "en", + "title": title_en or title_jp, + "title_en": title_en, + "title_jp": title_jp, + "thumbnail": extr('"og:image" content="', '"'), + "uploader": text.remove_html(extr('id="Uploader">', "")), + "date": text.parse_datetime(extr('id="Uploaded">', "").strip(), "%Y %B %d"), + "rating": text.parse_float(extr('id="Rating">', "").partition(" ")[0]), + "type": text.remove_html(extr('id="Category">', "")), + "collection": text.remove_html(extr('id="Collection">', "")), + "group": text.split_html(extr('id="Group">', "")), + "artist": text.split_html(extr('id="Artist">', "")), + "parody": text.split_html(extr('id="Parody">', "")), + "characters": text.split_html(extr('id="Character">', "")), + "tags": text.split_html(extr('id="Tag">', "")), + "language": "English", + "lang": "en", } def images(self, page): - url = "{}/Read/Index/{}?page=1".format(self.root, self.gallery_id) + url = f"{self.root}/Read/Index/{self.gallery_id}?page=1" headers = {"Referer": self.gallery_url} response = self.request(url, headers=headers, fatal=False) if "/Auth/" in response.url: raise exception.StopExtraction( "Failed to get gallery JSON data. Visit '%s' in a browser " - "and solve the CAPTCHA to continue.", response.url) + "and solve the CAPTCHA to continue.", + response.url, + ) page = response.text tpl, pos = text.extract(page, 'data-cdn="', '"') - cnt, pos = text.extract(page, '> of ', '<', pos) + cnt, pos = text.extract(page, "> of ", "<", pos) base, _, params = text.unescape(tpl).partition("[PAGE]") - return [ - (base + str(i) + params, None) - for i in range(1, text.parse_int(cnt)+1) - ] + return [(base + str(i) + params, None) for i in range(1, text.parse_int(cnt) + 1)] class TsuminoSearchExtractor(TsuminoBase, Extractor): """Extractor for search results on tsumino.com""" + subcategory = "search" pattern = r"(?i)(?:https?://)?(?:www\.)?tsumino\.com/(?:Books/?)?#(.+)" example = "https://www.tsumino.com/Books#QUERY" @@ -119,9 +121,9 @@ def items(self): def galleries(self): """Return all gallery results matching 'self.query'""" - url = "{}/Search/Operate?type=Book".format(self.root) + url = f"{self.root}/Search/Operate?type=Book" headers = { - "Referer": "{}/".format(self.root), + "Referer": f"{self.root}/", "X-Requested-With": "XMLHttpRequest", } data = { @@ -137,8 +139,7 @@ def galleries(self): data.update(self._parse(self.query)) while True: - info = self.request( - url, method="POST", headers=headers, data=data).json() + info = self.request(url, method="POST", headers=headers, data=data).json() for gallery in info["data"]: yield gallery["entry"] @@ -155,8 +156,7 @@ def _parse(self, query): return self._parse_simple(query) return self._parse_jsurl(query) except Exception as exc: - raise exception.StopExtraction( - "Invalid search query '%s' (%s)", query, exc) + raise exception.StopExtraction("Invalid search query '%s' (%s)", query, exc) @staticmethod def _parse_simple(query): @@ -196,8 +196,7 @@ def eat(expected): nonlocal i if data[i] != expected: - error = "bad JSURL syntax: expected '{}', got {}".format( - expected, data[i]) + error = f"bad JSURL syntax: expected '{expected}', got {data[i]}" raise ValueError(error) i += 1 @@ -217,10 +216,10 @@ def decode(): if beg < i: result += data[beg:i] if data[i + 1] == "*": - result += chr(int(data[i+2:i+6], 16)) + result += chr(int(data[i + 2 : i + 6], 16)) i += 6 else: - result += chr(int(data[i+1:i+3], 16)) + result += chr(int(data[i + 1 : i + 3], 16)) i += 3 beg = i @@ -239,7 +238,7 @@ def decode(): def parse_one(): nonlocal i - eat('~') + eat("~") result = "" ch = data[i] @@ -248,7 +247,7 @@ def parse_one(): if data[i] == "~": result = [] - if data[i+1] == ")": + if data[i + 1] == ")": i += 1 else: result.append(parse_one()) @@ -295,11 +294,11 @@ def parse_one(): def expand(key, value): if isinstance(value, list): for index, cvalue in enumerate(value): - ckey = "{}[{}]".format(key, index) + ckey = f"{key}[{index}]" yield from expand(ckey, cvalue) elif isinstance(value, dict): for ckey, cvalue in value.items(): - ckey = "{}[{}]".format(key, ckey) + ckey = f"{key}[{ckey}]" yield from expand(ckey, cvalue) else: yield key, value diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py index 8d1fcde54..7e3084add 100644 --- a/gallery_dl/extractor/tumblr.py +++ b/gallery_dl/extractor/tumblr.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,11 +6,17 @@ """Extractors for https://www.tumblr.com/""" -from .common import Extractor, Message -from .. import text, util, oauth, exception -from datetime import datetime, date, timedelta import re +from datetime import date +from datetime import datetime +from datetime import timedelta +from .. import exception +from .. import oauth +from .. import text +from .. import util +from .common import Extractor +from .common import Message BASE_PATTERN = ( r"(?:tumblr:(?:https?://)?([^/]+)|" @@ -21,12 +25,14 @@ r"([\w-]+\.tumblr\.com)))" ) -POST_TYPES = frozenset(("text", "quote", "link", "answer", "video", - "audio", "photo", "chat", "search")) +POST_TYPES = frozenset( + ("text", "quote", "link", "answer", "video", "audio", "photo", "chat", "search") +) class TumblrExtractor(Extractor): """Base class for tumblr extractors""" + category = "tumblr" directory_fmt = ("{category}", "{blog_name}") filename_fmt = "{category}_{blog_name}_{id}_{num:>02}.{extension}" @@ -67,12 +73,13 @@ def items(self): # pre-compile regular expressions self._sub_video = re.compile( - r"https?://((?:vt|vtt|ve)(?:\.media)?\.tumblr\.com" - r"/tumblr_[^_]+)_\d+\.([0-9a-z]+)").sub + r"https?://((?:vt|vtt|ve)(?:\.media)?\.tumblr\.com" r"/tumblr_[^_]+)_\d+\.([0-9a-z]+)" + ).sub if self.inline: self._sub_image = re.compile( r"https?://(\d+\.media\.tumblr\.com(?:/[0-9a-f]+)?" - r"/tumblr(?:_inline)?_[^_]+)_\d+\.([0-9a-z]+)").sub + r"/tumblr(?:_inline)?_[^_]+)_\d+\.([0-9a-z]+)" + ).sub self._subn_orig_image = re.compile(r"/s\d+x\d+/").subn _findall_image = re.compile(' best_photo["height"] or - alt_photo["width"] > best_photo["width"]): + if ( + alt_photo["height"] > best_photo["height"] + or alt_photo["width"] > best_photo["width"] + ): best_photo = alt_photo photo.update(best_photo) - if self.original and "/s2048x3072/" in photo["url"] and ( - photo["width"] == 2048 or photo["height"] == 3072): + if ( + self.original + and "/s2048x3072/" in photo["url"] + and (photo["width"] == 2048 or photo["height"] == 3072) + ): photo["url"], fb = self._original_photo(photo["url"]) if fb: post["_fallback"] = self._original_image_fallback( - photo["url"], post["id"]) + photo["url"], post["id"] + ) del photo["original_size"] del photo["alt_sizes"] - posts.append( - self._prepare_image(photo["url"], post.copy())) + posts.append(self._prepare_image(photo["url"], post.copy())) del post["photo"] post.pop("_fallback", None) @@ -142,8 +154,7 @@ def items(self): url = post.get("video_url") # type "video" if url: - posts.append(self._prepare( - self._original_video(url), post.copy())) + posts.append(self._prepare(self._original_video(url), post.copy())) if self.inline and "reblog" in post: # inline media # only "chat" posts are missing a "reblog" key in their @@ -152,8 +163,7 @@ def items(self): for url in _findall_image(body): url, fb = self._original_inline_image(url) if fb: - post["_fallback"] = self._original_image_fallback( - url, post["id"]) + post["_fallback"] = self._original_image_fallback(url, post["id"]) posts.append(self._prepare_image(url, post.copy())) post.pop("_fallback", None) for url in _findall_video(body): @@ -184,20 +194,18 @@ def _setup_posttypes(self): if types == "all": return POST_TYPES - elif not types: + if not types: return frozenset() - else: - if isinstance(types, str): - types = types.split(",") - types = frozenset(types) + if isinstance(types, str): + types = types.split(",") + types = frozenset(types) - invalid = types - POST_TYPES - if invalid: - types = types & POST_TYPES - self.log.warning("Invalid post types: '%s'", - "', '".join(sorted(invalid))) - return types + invalid = types - POST_TYPES + if invalid: + types = types & POST_TYPES + self.log.warning("Invalid post types: '%s'", "', '".join(sorted(invalid))) + return types @staticmethod def _prepare(url, post): @@ -214,9 +222,10 @@ def _prepare_image(url, post): # incorrect extensions will be corrected by 'adjust-extensions' if post["extension"] == "gif": post["_fallback"] = (url + "v",) - post["_http_headers"] = {"Accept": # copied from chrome 106 - "image/avif,image/webp,image/apng," - "image/svg+xml,image/*,*/*;q=0.8"} + post["_http_headers"] = { + "Accept": # copied from chrome 106 + "image/avif,image/webp,image/apng," "image/svg+xml,image/*,*/*;q=0.8" + } parts = post["filename"].split("_") try: @@ -270,12 +279,12 @@ def _original_image_fallback(self, url, post_id): for _ in util.repeat(self.fallback_retries): self.sleep(self.fallback_delay, "image token") yield self._update_image_token(url)[0] - self.log.warning("Unable to fetch higher-resolution " - "version of %s (%s)", url, post_id) + self.log.warning("Unable to fetch higher-resolution " "version of %s (%s)", url, post_id) class TumblrUserExtractor(TumblrExtractor): """Extractor for a Tumblr user's posts""" + subcategory = "user" pattern = BASE_PATTERN + r"(?:/page/\d+|/archive)?/?$" example = "https://www.tumblr.com/BLOG" @@ -286,6 +295,7 @@ def posts(self): class TumblrPostExtractor(TumblrExtractor): """Extractor for a single Tumblr post""" + subcategory = "post" pattern = BASE_PATTERN + r"/(?:post/|image/)?(\d+)" example = "https://www.tumblr.com/BLOG/12345" @@ -306,6 +316,7 @@ def _setup_posttypes(): class TumblrTagExtractor(TumblrExtractor): """Extractor for Tumblr user's posts by tag""" + subcategory = "tag" pattern = BASE_PATTERN + r"/tagged/([^/?#]+)" example = "https://www.tumblr.com/BLOG/tagged/TAG" @@ -320,6 +331,7 @@ def posts(self): class TumblrDayExtractor(TumblrExtractor): """Extractor for Tumblr user's posts by day""" + subcategory = "day" pattern = BASE_PATTERN + r"/day/(\d\d\d\d/\d\d/\d\d)" example = "https://www.tumblr.com/BLOG/day/1970/01/01" @@ -334,7 +346,8 @@ def _init(self): self.date_min = ( # 719163 == date(1970, 1, 1).toordinal() - (self.ordinal - 719163) * 86400) + (self.ordinal - 719163) * 86400 + ) self.api.before = self.date_min + 86400 @@ -344,6 +357,7 @@ def posts(self): class TumblrLikesExtractor(TumblrExtractor): """Extractor for a Tumblr user's liked posts""" + subcategory = "likes" directory_fmt = ("{category}", "{blog_name}", "likes") archive_fmt = "f_{blog[name]}_{id}_{num}" @@ -356,9 +370,9 @@ def posts(self): class TumblrSearchExtractor(TumblrExtractor): """Extractor for a Tumblr search""" + subcategory = "search" - pattern = (BASE_PATTERN + r"/search/([^/?#]+)" - r"(?:/([^/?#]+)(?:/([^/?#]+))?)?(?:/?\?([^#]+))?") + pattern = BASE_PATTERN + r"/search/([^/?#]+)" r"(?:/([^/?#]+)(?:/([^/?#]+))?)?(?:/?\?([^#]+))?" example = "https://www.tumblr.com/search/QUERY" def posts(self): @@ -372,6 +386,7 @@ class TumblrAPI(oauth.OAuth1API): https://github.com/tumblr/docs/blob/master/api.md """ + ROOT = "https://api.tumblr.com" API_KEY = "O3hU2tMi5e4Qs5t3vezEi6L0qRORJ5y9oUpSGsrWu8iA3UCc3B" API_SECRET = "sFdsK3PDdP2QpYMRAoq0oDnw0sFS24XigXmdfnaeNZpJpqAn03" @@ -386,7 +401,7 @@ def info(self, blog): try: return self.BLOG_CACHE[blog] except KeyError: - endpoint = "/v2/blog/{}/info".format(blog) + endpoint = f"/v2/blog/{blog}/info" params = {"api_key": self.api_key} if self.api_key else None self.BLOG_CACHE[blog] = blog = self._call(endpoint, params)["blog"] return blog @@ -394,12 +409,10 @@ def info(self, blog): def avatar(self, blog, size="512"): """Retrieve a blog avatar""" if self.api_key: - return "{}/v2/blog/{}/avatar/{}?api_key={}".format( - self.ROOT, blog, size, self.api_key) - endpoint = "/v2/blog/{}/avatar".format(blog) + return f"{self.ROOT}/v2/blog/{blog}/avatar/{size}?api_key={self.api_key}" + endpoint = f"/v2/blog/{blog}/avatar" params = {"size": size} - return self._call( - endpoint, params, allow_redirects=False)["avatar_url"] + return self._call(endpoint, params, allow_redirects=False)["avatar_url"] def posts(self, blog, params): """Retrieve published posts""" @@ -412,12 +425,12 @@ def posts(self, blog, params): if self.before and params["offset"]: self.log.warning("'offset' and 'date-max' cannot be used together") - endpoint = "/v2/blog/{}/posts".format(blog) + endpoint = f"/v2/blog/{blog}/posts" return self._pagination(endpoint, params, blog=blog, cache=True) def likes(self, blog): """Retrieve liked posts""" - endpoint = "/v2/blog/{}/likes".format(blog) + endpoint = f"/v2/blog/{blog}/likes" params = {"limit": "50", "before": self.before} if self.api_key: params["api_key"] = self.api_key @@ -465,21 +478,19 @@ def _call(self, endpoint, params, **kwargs): if status == 403: raise exception.AuthorizationError() - elif status == 404: + if status == 404: try: error = data["errors"][0]["detail"] - board = ("only viewable within the Tumblr dashboard" - in error) + board = "only viewable within the Tumblr dashboard" in error except Exception: board = False if board: - self.log.info("Run 'gallery-dl oauth:tumblr' " - "to access dashboard-only blogs") + self.log.info("Run 'gallery-dl oauth:tumblr' " "to access dashboard-only blogs") raise exception.AuthorizationError(error) raise exception.NotFoundError("user or post") - elif status == 429: + if status == 429: # daily rate limit if response.headers.get("x-ratelimit-perday-remaining") == "0": self.log.info("Daily API rate limit exceeded") @@ -491,7 +502,8 @@ def _call(self, endpoint, params, **kwargs): "Register your own OAuth application and use its " "credentials to prevent this error: " "https://gdl-org.github.io/docs/configuration.html" - "#extractor-tumblr-api-key-api-secret") + "#extractor-tumblr-api-key-api-secret" + ) if self.extractor.config("ratelimit") == "wait": self.extractor.wait(seconds=reset) @@ -500,7 +512,8 @@ def _call(self, endpoint, params, **kwargs): t = (datetime.now() + timedelta(0, float(reset))).time() raise exception.StopExtraction( "Aborting - Rate limit will reset at %s", - "{:02}:{:02}:{:02}".format(t.hour, t.minute, t.second)) + f"{t.hour:02}:{t.minute:02}:{t.second:02}", + ) # hourly rate limit reset = response.headers.get("x-ratelimit-perhour-reset") @@ -511,8 +524,7 @@ def _call(self, endpoint, params, **kwargs): raise exception.StopExtraction(data) - def _pagination(self, endpoint, params, - blog=None, key="posts", cache=False): + def _pagination(self, endpoint, params, blog=None, key="posts", cache=False): if self.api_key: params["api_key"] = self.api_key @@ -555,8 +567,7 @@ def _pagination(self, endpoint, params, params["offset"] = None else: # offset - params["offset"] = \ - text.parse_int(params["offset"]) + params["limit"] + params["offset"] = text.parse_int(params["offset"]) + params["limit"] params["before"] = None if params["offset"] >= data["total_posts"]: return diff --git a/gallery_dl/extractor/tumblrgallery.py b/gallery_dl/extractor/tumblrgallery.py index 448625ede..49fb87cf3 100644 --- a/gallery_dl/extractor/tumblrgallery.py +++ b/gallery_dl/extractor/tumblrgallery.py @@ -1,19 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://tumblrgallery.xyz/""" -from .common import GalleryExtractor from .. import text +from .common import GalleryExtractor BASE_PATTERN = r"(?:https?://)?tumblrgallery\.xyz" class TumblrgalleryExtractor(GalleryExtractor): """Base class for tumblrgallery extractors""" + category = "tumblrgallery" filename_fmt = "{category}_{gallery_id}_{num:>03}_{id}.{extension}" directory_fmt = ("{category}", "{gallery_id} {title}") @@ -22,8 +21,7 @@ class TumblrgalleryExtractor(GalleryExtractor): @staticmethod def _urls_from_page(page): - return text.extract_iter( - page, '
    ", "")), + "title": text.unescape(text.extr(page, "

    ", "

    ")), "gallery_id": self.gallery_id, } def images(self, _): page_num = 1 while True: - url = "{}/tumblrblog/gallery/{}/{}.html".format( - self.root, self.gallery_id, page_num) + url = f"{self.root}/tumblrblog/gallery/{self.gallery_id}/{page_num}.html" response = self.request(url, allow_redirects=False, fatal=False) if response.status_code >= 300: @@ -68,6 +66,7 @@ def images(self, _): class TumblrgalleryPostExtractor(TumblrgalleryExtractor): """Extractor for Posts on tumblrgallery.xyz""" + subcategory = "post" pattern = BASE_PATTERN + r"(/post/(\d+)\.html)" example = "https://tumblrgallery.xyz/post/12345.html" @@ -78,7 +77,7 @@ def __init__(self, match): def metadata(self, page): return { - "title" : text.remove_html( + "title": text.remove_html( text.unescape(text.extr(page, "", "")) ).replace("_", "-"), "gallery_id": self.gallery_id, @@ -91,6 +90,7 @@ def images(self, page): class TumblrgallerySearchExtractor(TumblrgalleryExtractor): """Extractor for Search result on tumblrgallery.xyz""" + subcategory = "search" filename_fmt = "{category}_{num:>03}_{gallery_id}_{id}_{title}.{extension}" directory_fmt = ("{category}", "{search_term}") @@ -111,22 +111,19 @@ def images(self, _): while True: page = self.request(self.root + "/" + page_url).text - for gallery_id in text.extract_iter( - page, '
    ", "") - )).replace("_", "-") + data["title"] = text.remove_html( + text.unescape(text.extr(post_page, "", "")) + ).replace("_", "-") yield url, data - next_url = text.extr( - page, ' = 300000000000000: - date = text.parse_timestamp( - ((tweet_id >> 22) + 1288834974657) // 1000) + date = text.parse_timestamp(((tweet_id >> 22) + 1288834974657) // 1000) else: try: - date = text.parse_datetime( - legacy["created_at"], "%a %b %d %H:%M:%S %z %Y") + date = text.parse_datetime(legacy["created_at"], "%a %b %d %H:%M:%S %z %Y") except Exception: date = util.NONE source = tweet.get("source") tdata = { - "tweet_id" : tweet_id, - "retweet_id" : text.parse_int( - tget("retweeted_status_id_str")), - "quote_id" : text.parse_int( - tget("quoted_by_id_str")), - "reply_id" : text.parse_int( - tget("in_reply_to_status_id_str")), - "conversation_id": text.parse_int( - tget("conversation_id_str")), - "date" : date, - "author" : author, - "user" : self._user or author, - "lang" : legacy["lang"], - "source" : text.extr(source, ">", "<") if source else "", - "sensitive" : tget("possibly_sensitive"), + "tweet_id": tweet_id, + "retweet_id": text.parse_int(tget("retweeted_status_id_str")), + "quote_id": text.parse_int(tget("quoted_by_id_str")), + "reply_id": text.parse_int(tget("in_reply_to_status_id_str")), + "conversation_id": text.parse_int(tget("conversation_id_str")), + "date": date, + "author": author, + "user": self._user or author, + "lang": legacy["lang"], + "source": text.extr(source, ">", "<") if source else "", + "sensitive": tget("possibly_sensitive"), "favorite_count": tget("favorite_count"), - "quote_count" : tget("quote_count"), - "reply_count" : tget("reply_count"), - "retweet_count" : tget("retweet_count"), + "quote_count": tget("quote_count"), + "reply_count": tget("reply_count"), + "retweet_count": tget("retweet_count"), "bookmark_count": tget("bookmark_count"), } @@ -352,39 +342,40 @@ def _transform_tweet(self, tweet): mentions = entities.get("user_mentions") if mentions: - tdata["mentions"] = [{ - "id": text.parse_int(u["id_str"]), - "name": u["screen_name"], - "nick": u["name"], - } for u in mentions] + tdata["mentions"] = [ + { + "id": text.parse_int(u["id_str"]), + "name": u["screen_name"], + "nick": u["name"], + } + for u in mentions + ] content = text.unescape(content) urls = entities.get("urls") if urls: - for url in urls: - try: + with suppress(KeyError): + for url in urls: content = content.replace(url["url"], url["expanded_url"]) - except KeyError: - pass txt, _, tco = content.rpartition(" ") tdata["content"] = txt if tco.startswith("https://t.co/") else content if "birdwatch_pivot" in tweet: try: - tdata["birdwatch"] = \ - tweet["birdwatch_pivot"]["subtitle"]["text"] + tdata["birdwatch"] = tweet["birdwatch_pivot"]["subtitle"]["text"] except KeyError: - self.log.debug("Unable to extract 'birdwatch' note from %s", - tweet["birdwatch_pivot"]) + self.log.debug( + "Unable to extract 'birdwatch' note from %s", tweet["birdwatch_pivot"] + ) if "in_reply_to_screen_name" in legacy: tdata["reply_to"] = legacy["in_reply_to_screen_name"] if "quoted_by" in legacy: tdata["quote_by"] = legacy["quoted_by"] if tdata["retweet_id"]: - tdata["content"] = "RT @{}: {}".format( - author["name"], tdata["content"]) + tdata["content"] = "RT @{}: {}".format(author["name"], tdata["content"]) tdata["date_original"] = text.parse_timestamp( - ((tdata["retweet_id"] >> 22) + 1288834974657) // 1000) + ((tdata["retweet_id"] >> 22) + 1288834974657) // 1000 + ) return tdata @@ -409,33 +400,29 @@ def _transform_user(self, user): entities = user["entities"] self._user_cache[uid] = udata = { - "id" : text.parse_int(uid), - "name" : user["screen_name"], - "nick" : user["name"], - "location" : uget("location"), - "date" : text.parse_datetime( - uget("created_at"), "%a %b %d %H:%M:%S %z %Y"), - "verified" : uget("verified", False), - "protected" : uget("protected", False), - "profile_banner" : uget("profile_banner_url", ""), - "profile_image" : uget( - "profile_image_url_https", "").replace("_normal.", "."), + "id": text.parse_int(uid), + "name": user["screen_name"], + "nick": user["name"], + "location": uget("location"), + "date": text.parse_datetime(uget("created_at"), "%a %b %d %H:%M:%S %z %Y"), + "verified": uget("verified", False), + "protected": uget("protected", False), + "profile_banner": uget("profile_banner_url", ""), + "profile_image": uget("profile_image_url_https", "").replace("_normal.", "."), "favourites_count": uget("favourites_count"), - "followers_count" : uget("followers_count"), - "friends_count" : uget("friends_count"), - "listed_count" : uget("listed_count"), - "media_count" : uget("media_count"), - "statuses_count" : uget("statuses_count"), + "followers_count": uget("followers_count"), + "friends_count": uget("friends_count"), + "listed_count": uget("listed_count"), + "media_count": uget("media_count"), + "statuses_count": uget("statuses_count"), } descr = user["description"] urls = entities["description"].get("urls") if urls: - for url in urls: - try: + with suppress(KeyError): + for url in urls: descr = descr.replace(url["url"], url["expanded_url"]) - except KeyError: - pass udata["description"] = descr if "url" in entities: @@ -473,16 +460,14 @@ def _users_result(self, users): def _expand_tweets(self, tweets): seen = set() for tweet in tweets: - obj = tweet["legacy"] if "legacy" in tweet else tweet + obj = tweet.get("legacy", tweet) cid = obj.get("conversation_id_str") if not cid: tid = obj["id_str"] - self.log.warning( - "Unable to expand %s (no 'conversation_id')", tid) + self.log.warning("Unable to expand %s (no 'conversation_id')", tid) continue if cid in seen: - self.log.debug( - "Skipping expansion of %s (previously seen)", cid) + self.log.debug("Skipping expansion of %s (previously seen)", cid) continue seen.add(cid) try: @@ -528,12 +513,14 @@ def tweets(self): def finalize(self): if self._cursor: - self.log.info("Use '-o cursor=%s' to continue downloading " - "from the current position", self._cursor) + self.log.info( + "Use '-o cursor=%s' to continue downloading " "from the current position", + self._cursor, + ) def login(self): if self.cookies_check(self.cookies_names): - return + return None username, password = self._get_auth_info() if username: @@ -542,9 +529,12 @@ def login(self): class TwitterUserExtractor(TwitterExtractor): """Extractor for a Twitter user""" + subcategory = "user" - pattern = (BASE_PATTERN + r"/(?!search)(?:([^/?#]+)/?(?:$|[?#])" - r"|i(?:/user/|ntent/user\?user_id=)(\d+))") + pattern = ( + BASE_PATTERN + r"/(?!search)(?:([^/?#]+)/?(?:$|[?#])" + r"|i(?:/user/|ntent/user\?user_id=)(\d+))" + ) example = "https://x.com/USER" def __init__(self, match): @@ -560,21 +550,25 @@ def finalize(self): pass def items(self): - base = "{}/{}/".format(self.root, self.user) - return self._dispatch_extractors(( - (TwitterInfoExtractor , base + "info"), - (TwitterAvatarExtractor , base + "photo"), - (TwitterBackgroundExtractor, base + "header_photo"), - (TwitterTimelineExtractor , base + "timeline"), - (TwitterTweetsExtractor , base + "tweets"), - (TwitterMediaExtractor , base + "media"), - (TwitterRepliesExtractor , base + "with_replies"), - (TwitterLikesExtractor , base + "likes"), - ), ("timeline",)) + base = f"{self.root}/{self.user}/" + return self._dispatch_extractors( + ( + (TwitterInfoExtractor, base + "info"), + (TwitterAvatarExtractor, base + "photo"), + (TwitterBackgroundExtractor, base + "header_photo"), + (TwitterTimelineExtractor, base + "timeline"), + (TwitterTweetsExtractor, base + "tweets"), + (TwitterMediaExtractor, base + "media"), + (TwitterRepliesExtractor, base + "with_replies"), + (TwitterLikesExtractor, base + "likes"), + ), + ("timeline",), + ) class TwitterTimelineExtractor(TwitterExtractor): """Extractor for a Twitter user timeline""" + subcategory = "timeline" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/timeline(?!\w)" example = "https://x.com/USER/timeline" @@ -631,7 +625,7 @@ def tweets(self): query += " include:retweets include:nativeretweets" if state <= 2: - self._cursor_prefix = "2_{}/".format(tweet_id) + self._cursor_prefix = f"2_{tweet_id}/" if reset: self._cursor = self._cursor_prefix @@ -647,7 +641,7 @@ def tweets(self): if state <= 3: # yield unfiltered search results - self._cursor_prefix = "3_{}/".format(tweet_id) + self._cursor_prefix = f"3_{tweet_id}/" if reset: self._cursor = self._cursor_prefix @@ -659,8 +653,7 @@ def _select_tweet_source(self): if strategy is None or strategy == "auto": if self.retweets or self.textonly: return self.api.user_tweets - else: - return self.api.user_media + return self.api.user_media if strategy == "tweets": return self.api.user_tweets if strategy == "media": @@ -672,6 +665,7 @@ def _select_tweet_source(self): class TwitterTweetsExtractor(TwitterExtractor): """Extractor for Tweets from a user's Tweets timeline""" + subcategory = "tweets" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/tweets(?!\w)" example = "https://x.com/USER/tweets" @@ -682,6 +676,7 @@ def tweets(self): class TwitterRepliesExtractor(TwitterExtractor): """Extractor for Tweets from a user's timeline including replies""" + subcategory = "replies" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/with_replies(?!\w)" example = "https://x.com/USER/with_replies" @@ -692,6 +687,7 @@ def tweets(self): class TwitterMediaExtractor(TwitterExtractor): """Extractor for Tweets from a user's Media timeline""" + subcategory = "media" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/media(?!\w)" example = "https://x.com/USER/media" @@ -702,6 +698,7 @@ def tweets(self): class TwitterLikesExtractor(TwitterExtractor): """Extractor for liked tweets""" + subcategory = "likes" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/likes(?!\w)" example = "https://x.com/USER/likes" @@ -715,6 +712,7 @@ def tweets(self): class TwitterBookmarkExtractor(TwitterExtractor): """Extractor for bookmarked tweets""" + subcategory = "bookmark" pattern = BASE_PATTERN + r"/i/bookmarks()" example = "https://x.com/i/bookmarks" @@ -725,12 +723,14 @@ def tweets(self): def _transform_tweet(self, tweet): tdata = TwitterExtractor._transform_tweet(self, tweet) tdata["date_bookmarked"] = text.parse_timestamp( - (int(tweet["sortIndex"] or 0) >> 20) // 1000) + (int(tweet["sortIndex"] or 0) >> 20) // 1000 + ) return tdata class TwitterListExtractor(TwitterExtractor): """Extractor for Twitter lists""" + subcategory = "list" pattern = BASE_PATTERN + r"/i/lists/(\d+)/?$" example = "https://x.com/i/lists/12345" @@ -741,6 +741,7 @@ def tweets(self): class TwitterListMembersExtractor(TwitterExtractor): """Extractor for members of a Twitter list""" + subcategory = "list-members" pattern = BASE_PATTERN + r"/i/lists/(\d+)/members" example = "https://x.com/i/lists/12345/members" @@ -752,6 +753,7 @@ def items(self): class TwitterFollowingExtractor(TwitterExtractor): """Extractor for followed users""" + subcategory = "following" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/following(?!\w)" example = "https://x.com/USER/following" @@ -763,6 +765,7 @@ def items(self): class TwitterSearchExtractor(TwitterExtractor): """Extractor for Twitter search results""" + subcategory = "search" pattern = BASE_PATTERN + r"/search/?\?(?:[^&#]+&)*q=([^&#]+)" example = "https://x.com/search?q=QUERY" @@ -780,32 +783,31 @@ def tweets(self): if user: user = None break - else: - user = item[5:] + user = item[5:] if user is not None: - try: + with suppress(KeyError): self._assign_user(self.api.user_by_screen_name(user)) - except KeyError: - pass return self.api.search_timeline(query) class TwitterHashtagExtractor(TwitterExtractor): """Extractor for Twitter hashtags""" + subcategory = "hashtag" pattern = BASE_PATTERN + r"/hashtag/([^/?#]+)" example = "https://x.com/hashtag/NAME" def items(self): - url = "{}/search?q=%23{}".format(self.root, self.user) + url = f"{self.root}/search?q=%23{self.user}" data = {"_extractor": TwitterSearchExtractor} yield Message.Queue, url, data class TwitterCommunityExtractor(TwitterExtractor): """Extractor for a Twitter community""" + subcategory = "community" pattern = BASE_PATTERN + r"/i/communities/(\d+)" example = "https://x.com/i/communities/12345" @@ -818,6 +820,7 @@ def tweets(self): class TwitterCommunitiesExtractor(TwitterExtractor): """Extractor for followed Twitter communities""" + subcategory = "communities" pattern = BASE_PATTERN + r"/([^/?#]+)/communities/?$" example = "https://x.com/i/communities" @@ -828,9 +831,9 @@ def tweets(self): class TwitterEventExtractor(TwitterExtractor): """Extractor for Tweets from a Twitter Event""" + subcategory = "event" - directory_fmt = ("{category}", "Events", - "{event[id]} {event[short_title]}") + directory_fmt = ("{category}", "Events", "{event[id]} {event[short_title]}") pattern = BASE_PATTERN + r"/i/events/(\d+)" example = "https://x.com/i/events/12345" @@ -843,9 +846,9 @@ def tweets(self): class TwitterTweetExtractor(TwitterExtractor): """Extractor for individual tweets""" + subcategory = "tweet" - pattern = (BASE_PATTERN + r"/([^/?#]+|i/web)/status/(\d+)" - r"/?(?:$|\?|#|photo/|video/)") + pattern = BASE_PATTERN + r"/([^/?#]+|i/web)/status/(\d+)" r"/?(?:$|\?|#|photo/|video/)" example = "https://x.com/USER/status/12345" def __init__(self, match): @@ -855,12 +858,15 @@ def __init__(self, match): def tweets(self): conversations = self.config("conversations") if conversations: - self._accessible = (conversations == "accessible") + self._accessible = conversations == "accessible" return self._tweets_conversation(self.tweet_id) endpoint = self.config("tweet-endpoint") - if endpoint == "detail" or endpoint in (None, "auto") and \ - self.api.headers["x-twitter-auth-type"]: + if ( + endpoint == "detail" + or endpoint in (None, "auto") + and self.api.headers["x-twitter-auth-type"] + ): return self._tweets_detail(self.tweet_id) return self._tweets_single(self.tweet_id) @@ -871,8 +877,7 @@ def _tweets_single(self, tweet_id): try: self._assign_user(tweet["core"]["user_results"]["result"]) except KeyError: - raise exception.StopExtraction( - "'%s'", tweet.get("reason") or "Unavailable") + raise exception.StopExtraction("'%s'", tweet.get("reason") or "Unavailable") yield tweet @@ -891,8 +896,7 @@ def _tweets_detail(self, tweet_id): tweets = [] for tweet in self.api.tweet_detail(tweet_id): - if tweet["rest_id"] == tweet_id or \ - tweet.get("_retweet_id_str") == tweet_id: + if tweet["rest_id"] == tweet_id or tweet.get("_retweet_id_str") == tweet_id: if self._user_obj is None: self._assign_user(tweet["core"]["user_results"]["result"]) tweets.append(tweet) @@ -909,8 +913,7 @@ def _tweets_conversation(self, tweet_id): for tweet in tweets: buffer.append(tweet) - if tweet["rest_id"] == tweet_id or \ - tweet.get("_retweet_id_str") == tweet_id: + if tweet["rest_id"] == tweet_id or tweet.get("_retweet_id_str") == tweet_id: self._assign_user(tweet["core"]["user_results"]["result"]) break else: @@ -924,18 +927,20 @@ def _tweets_conversation(self, tweet_id): class TwitterQuotesExtractor(TwitterExtractor): """Extractor for quotes of a Tweet""" + subcategory = "quotes" pattern = BASE_PATTERN + r"/(?:[^/?#]+|i/web)/status/(\d+)/quotes" example = "https://x.com/USER/status/12345/quotes" def items(self): - url = "{}/search?q=quoted_tweet_id:{}".format(self.root, self.user) + url = f"{self.root}/search?q=quoted_tweet_id:{self.user}" data = {"_extractor": TwitterSearchExtractor} yield Message.Queue, url, data class TwitterInfoExtractor(TwitterExtractor): """Extractor for a user's profile data""" + subcategory = "info" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/info" example = "https://x.com/USER/info" @@ -964,8 +969,9 @@ def tweets(self): user = self._user_obj url = user["legacy"]["profile_image_url_https"] - if url == ("https://abs.twimg.com/sticky" - "/default_profile_images/default_profile_normal.png"): + if url == ( + "https://abs.twimg.com/sticky" "/default_profile_images/default_profile_normal.png" + ): return () url = url.replace("_normal.", ".") @@ -1007,8 +1013,7 @@ def __init__(self, match): TwitterExtractor._init_sizes(self) def items(self): - base = "https://pbs.twimg.com/media/{}?format={}&name=".format( - self.id, self.fmt) + base = f"https://pbs.twimg.com/media/{self.id}?format={self.fmt}&name=" data = { "filename": self.id, @@ -1020,8 +1025,7 @@ def items(self): yield Message.Url, base + self._size_image, data -class TwitterAPI(): - +class TwitterAPI: def __init__(self, extractor): self.extractor = extractor self.log = extractor.log @@ -1057,8 +1061,8 @@ def __init__(self, extractor): "Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "same-origin", "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejR" - "COuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu" - "4FA33AGWWjCpTnA", + "COuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu" + "4FA33AGWWjCpTnA", } self.params = { "include_profile_interstitial_type": "1", @@ -1098,8 +1102,8 @@ def __init__(self, extractor): "spelling_corrections": None, "include_ext_edit_control": "true", "ext": "mediaStats,highlightedLabel,hasNftAvatar,voiceInfo," - "enrichments,superFollowMetadata,unmentionInfo,editControl," - "collab_control,vibe", + "enrichments,superFollowMetadata,unmentionInfo,editControl," + "collab_control,vibe", } self.features = { "hidden_profile_likes_enabled": True, @@ -1109,8 +1113,7 @@ def __init__(self, extractor): "highlights_tweets_tab_ui_enabled": True, "responsive_web_twitter_article_notes_tab_enabled": True, "creator_subscriptions_tweet_preview_api_enabled": True, - "responsive_web_graphql_" - "skip_user_profile_image_extensions_enabled": False, + "responsive_web_graphql_" "skip_user_profile_image_extensions_enabled": False, "responsive_web_graphql_timeline_navigation_enabled": True, } self.features_pagination = { @@ -1118,8 +1121,7 @@ def __init__(self, extractor): "verified_phone_label_enabled": False, "creator_subscriptions_tweet_preview_api_enabled": True, "responsive_web_graphql_timeline_navigation_enabled": True, - "responsive_web_graphql_skip_user_profile_" - "image_extensions_enabled": False, + "responsive_web_graphql_skip_user_profile_" "image_extensions_enabled": False, "c9s_tweet_anatomy_moderator_badge_enabled": True, "tweetypie_unmention_optimization_enabled": True, "responsive_web_edit_tweet_api_enabled": True, @@ -1130,8 +1132,7 @@ def __init__(self, extractor): "tweet_awards_web_tipping_enabled": False, "freedom_of_speech_not_reach_fetch_enabled": True, "standardized_nudges_misinfo": True, - "tweet_with_visibility_results_prefer_gql_" - "limited_actions_policy_enabled": True, + "tweet_with_visibility_results_prefer_gql_" "limited_actions_policy_enabled": True, "rweb_video_timestamps_enabled": True, "longform_notetweets_rich_text_read_enabled": True, "longform_notetweets_inline_media_enabled": True, @@ -1149,7 +1150,7 @@ def tweet_result_by_rest_id(self, tweet_id): } params = { "variables": self._json_dumps(variables), - "features" : self._json_dumps(self.features_pagination), + "features": self._json_dumps(self.features_pagination), } tweet = self._call(endpoint, params)["data"]["tweetResult"]["result"] if "tweet" in tweet: @@ -1179,7 +1180,8 @@ def tweet_detail(self, tweet_id): "withV2Timeline": True, } return self._pagination_tweets( - endpoint, variables, ("threaded_conversation_with_injections_v2",)) + endpoint, variables, ("threaded_conversation_with_injections_v2",) + ) def user_tweets(self, screen_name): endpoint = "/graphql/5ICa5d9-AitXZrIA3H-4MQ/UserTweets" @@ -1240,8 +1242,8 @@ def user_bookmarks(self): features = self.features_pagination.copy() features["graphql_timeline_v2_bookmark_timeline"] = True return self._pagination_tweets( - endpoint, variables, ("bookmark_timeline_v2", "timeline"), False, - features=features) + endpoint, variables, ("bookmark_timeline_v2", "timeline"), False, features=features + ) def list_latest_tweets_timeline(self, list_id): endpoint = "/graphql/HjsWc-nwwHKYwHenbHm-tw/ListLatestTweetsTimeline" @@ -1249,8 +1251,7 @@ def list_latest_tweets_timeline(self, list_id): "listId": list_id, "count": 100, } - return self._pagination_tweets( - endpoint, variables, ("list", "tweets_timeline", "timeline")) + return self._pagination_tweets(endpoint, variables, ("list", "tweets_timeline", "timeline")) def search_timeline(self, query): endpoint = "/graphql/fZK7JipRHWtiZsTodhsTfQ/SearchTimeline" @@ -1262,8 +1263,8 @@ def search_timeline(self, query): } return self._pagination_tweets( - endpoint, variables, - ("search_by_raw_query", "search_timeline", "timeline")) + endpoint, variables, ("search_by_raw_query", "search_timeline", "timeline") + ) def community_tweets_timeline(self, community_id): endpoint = "/graphql/7B2AdxSuC-Er8qUr3Plm_w/CommunityTweetsTimeline" @@ -1275,9 +1276,10 @@ def community_tweets_timeline(self, community_id): "withCommunity": True, } return self._pagination_tweets( - endpoint, variables, - ("communityResults", "result", "ranked_community_timeline", - "timeline")) + endpoint, + variables, + ("communityResults", "result", "ranked_community_timeline", "timeline"), + ) def community_media_timeline(self, community_id): endpoint = "/graphql/qAGUldfcIoMv5KyAyVLYog/CommunityMediaTimeline" @@ -1287,23 +1289,23 @@ def community_media_timeline(self, community_id): "withCommunity": True, } return self._pagination_tweets( - endpoint, variables, - ("communityResults", "result", "community_media_timeline", - "timeline")) + endpoint, + variables, + ("communityResults", "result", "community_media_timeline", "timeline"), + ) def communities_main_page_timeline(self, screen_name): - endpoint = ("/graphql/GtOhw2mstITBepTRppL6Uw" - "/CommunitiesMainPageTimeline") + endpoint = "/graphql/GtOhw2mstITBepTRppL6Uw" "/CommunitiesMainPageTimeline" variables = { "count": 100, "withCommunity": True, } return self._pagination_tweets( - endpoint, variables, - ("viewer", "communities_timeline", "timeline")) + endpoint, variables, ("viewer", "communities_timeline", "timeline") + ) def live_event_timeline(self, event_id): - endpoint = "/2/live_event/timeline/{}.json".format(event_id) + endpoint = f"/2/live_event/timeline/{event_id}.json" params = self.params.copy() params["timeline_id"] = "recap" params["urt"] = "true" @@ -1311,12 +1313,11 @@ def live_event_timeline(self, event_id): return self._pagination_legacy(endpoint, params) def live_event(self, event_id): - endpoint = "/1.1/live_event/1/{}/timeline.json".format(event_id) + endpoint = f"/1.1/live_event/1/{event_id}/timeline.json" params = self.params.copy() params["count"] = "0" params["urt"] = "true" - return (self._call(endpoint, params) - ["twitter_objects"]["live_events"][event_id]) + return self._call(endpoint, params)["twitter_objects"]["live_events"][event_id] def list_members(self, list_id): endpoint = "/graphql/BQp2IEYkgxuSxqbTAr1e1g/ListMembers" @@ -1325,8 +1326,7 @@ def list_members(self, list_id): "count": 100, "withSafetyModeUserFields": True, } - return self._pagination_users( - endpoint, variables, ("list", "members_timeline", "timeline")) + return self._pagination_users(endpoint, variables, ("list", "members_timeline", "timeline")) def user_following(self, screen_name): endpoint = "/graphql/PAnE9toEjRfE-4tozRcsfw/Following" @@ -1342,10 +1342,12 @@ def user_by_rest_id(self, rest_id): endpoint = "/graphql/tD8zKvQzwY3kdx5yz6YmOw/UserByRestId" features = self.features params = { - "variables": self._json_dumps({ - "userId": rest_id, - "withSafetyModeUserFields": True, - }), + "variables": self._json_dumps( + { + "userId": rest_id, + "withSafetyModeUserFields": True, + } + ), "features": self._json_dumps(features), } return self._call(endpoint, params)["data"]["user"]["result"] @@ -1354,15 +1356,15 @@ def user_by_rest_id(self, rest_id): def user_by_screen_name(self, screen_name): endpoint = "/graphql/k5XapwcSikNsEsILW5FvgA/UserByScreenName" features = self.features.copy() - features["subscriptions_verification_info_" - "is_identity_verified_enabled"] = True - features["subscriptions_verification_info_" - "verified_since_enabled"] = True + features["subscriptions_verification_info_" "is_identity_verified_enabled"] = True + features["subscriptions_verification_info_" "verified_since_enabled"] = True params = { - "variables": self._json_dumps({ - "screen_name": screen_name, - "withSafetyModeUserFields": True, - }), + "variables": self._json_dumps( + { + "screen_name": screen_name, + "withSafetyModeUserFields": True, + } + ), "features": self._json_dumps(features), } return self._call(endpoint, params)["data"]["user"]["result"] @@ -1378,26 +1380,31 @@ def _user_id_by_screen_name(self, screen_name): return user["rest_id"] except KeyError: if "unavailable_message" in user: - raise exception.NotFoundError("{} ({})".format( - user["unavailable_message"].get("text"), - user.get("reason")), False) - else: - raise exception.NotFoundError("user") + raise exception.NotFoundError( + "{} ({})".format(user["unavailable_message"].get("text"), user.get("reason")), + False, + ) + raise exception.NotFoundError("user") @cache(maxage=3600) def _guest_token(self): endpoint = "/1.1/guest/activate.json" self.log.info("Requesting guest token") - return str(self._call( - endpoint, None, "POST", False, "https://api.x.com", - )["guest_token"]) + return str( + self._call( + endpoint, + None, + "POST", + False, + "https://api.x.com", + )["guest_token"] + ) def _authenticate_guest(self): guest_token = self._guest_token() if guest_token != self.headers["x-guest-token"]: self.headers["x-guest-token"] = guest_token - self.extractor.cookies.set( - "gt", guest_token, domain=self.extractor.cookies_domain) + self.extractor.cookies.set("gt", guest_token, domain=self.extractor.cookies_domain) def _call(self, endpoint, params, method="GET", auth=True, root=None): url = (root or self.root) + endpoint @@ -1407,8 +1414,8 @@ def _call(self, endpoint, params, method="GET", auth=True, root=None): self._authenticate_guest() response = self.extractor.request( - url, method=method, params=params, - headers=self.headers, fatal=None) + url, method=method, params=params, headers=self.headers, fatal=None + ) # update 'x-csrf-token' header (#1170) csrf_token = response.cookies.get("ct0") @@ -1451,8 +1458,7 @@ def _call(self, endpoint, params, method="GET", auth=True, root=None): continue _login_impl.invalidate(username) - self.extractor.cookies_update( - _login_impl(self.extractor, username, password)) + self.extractor.cookies_update(_login_impl(self.extractor, username, password)) self.__init__(self.extractor) retry = True @@ -1463,34 +1469,31 @@ def _call(self, endpoint, params, method="GET", auth=True, root=None): if self.headers["x-twitter-auth-type"]: self.log.debug("Retrying API request") continue - else: - # fall through to "Login Required" - response.status_code = 404 + # fall through to "Login Required" + response.status_code = 404 if response.status_code < 400: return data - elif response.status_code in (403, 404) and \ - not self.headers["x-twitter-auth-type"]: + if response.status_code in (403, 404) and not self.headers["x-twitter-auth-type"]: raise exception.AuthorizationError("Login required") - elif response.status_code == 429: + if response.status_code == 429: self._handle_ratelimit(response) continue # error - try: + with suppress(Exception): errors = ", ".join(e["message"] for e in errors) - except Exception: - pass raise exception.StopExtraction( - "%s %s (%s)", response.status_code, response.reason, errors) + "%s %s (%s)", response.status_code, response.reason, errors + ) def _pagination_legacy(self, endpoint, params): extr = self.extractor cursor = extr._init_cursor() if cursor: params["cursor"] = cursor - original_retweets = (extr.retweets == "original") + original_retweets = extr.retweets == "original" bottom = ("cursor-bottom-", "sq-cursor-bottom") while True: @@ -1513,21 +1516,21 @@ def _pagination_legacy(self, endpoint, params): elif "replaceEntry" in instr: entry = instr["replaceEntry"]["entry"] if entry["entryId"].startswith(bottom): - cursor = (entry["content"]["operation"] - ["cursor"]["value"]) + cursor = entry["content"]["operation"]["cursor"]["value"] # collect tweet IDs and cursor value for entry in entries: entry_startswith = entry["entryId"].startswith if entry_startswith(("tweet-", "sq-I-t-")): - tweet_ids.append( - entry["content"]["item"]["content"]["tweet"]["id"]) + tweet_ids.append(entry["content"]["item"]["content"]["tweet"]["id"]) elif entry_startswith("homeConversation-"): tweet_ids.extend( - entry["content"]["timelineModule"]["metadata"] - ["conversationMetadata"]["allTweetIds"][::-1]) + entry["content"]["timelineModule"]["metadata"]["conversationMetadata"][ + "allTweetIds" + ][::-1] + ) elif entry_startswith(bottom): cursor = entry["content"]["operation"]["cursor"] @@ -1561,10 +1564,8 @@ def _pagination_legacy(self, endpoint, params): tweet = retweet elif retweet: tweet["author"] = users[retweet["user_id_str"]] - if "extended_entities" in retweet and \ - "extended_entities" not in tweet: - tweet["extended_entities"] = \ - retweet["extended_entities"] + if "extended_entities" in retweet and "extended_entities" not in tweet: + tweet["extended_entities"] = retweet["extended_entities"] tweet["user"] = users[tweet["user_id_str"]] yield tweet @@ -1582,10 +1583,9 @@ def _pagination_legacy(self, endpoint, params): return extr._update_cursor(None) params["cursor"] = extr._update_cursor(cursor) - def _pagination_tweets(self, endpoint, variables, - path=None, stop_tweets=True, features=None): + def _pagination_tweets(self, endpoint, variables, path=None, stop_tweets=True, features=None): extr = self.extractor - original_retweets = (extr.retweets == "original") + original_retweets = extr.retweets == "original" pinned_tweet = extr.pinned params = {"variables": None} @@ -1603,8 +1603,7 @@ def _pagination_tweets(self, endpoint, variables, try: if path is None: - instructions = (data["user"]["result"]["timeline_v2"] - ["timeline"]["instructions"]) + instructions = data["user"]["result"]["timeline_v2"]["timeline"]["instructions"] else: instructions = data for key in path: @@ -1641,23 +1640,21 @@ def _pagination_tweets(self, endpoint, variables, if user: user = user["legacy"] if user.get("blocked_by"): - if self.headers["x-twitter-auth-type"] and \ - extr.config("logout"): + if self.headers["x-twitter-auth-type"] and extr.config("logout"): extr.cookies_file = None del extr.cookies["auth_token"] self.headers["x-twitter-auth-type"] = None extr.log.info("Retrying API request as guest") continue raise exception.AuthorizationError( - "{} blocked your account".format( - user["screen_name"])) + "{} blocked your account".format(user["screen_name"]) + ) elif user.get("protected"): raise exception.AuthorizationError( - "{}'s Tweets are protected".format( - user["screen_name"])) + "{}'s Tweets are protected".format(user["screen_name"]) + ) - raise exception.StopExtraction( - "Unable to retrieve Tweets from this timeline") + raise exception.StopExtraction("Unable to retrieve Tweets from this timeline") tweets = [] tweet = None @@ -1674,20 +1671,16 @@ def _pagination_tweets(self, endpoint, variables, if esw("tweet-"): tweets.append(entry) - elif esw(("profile-grid-", - "communities-grid-")): + elif esw(("profile-grid-", "communities-grid-")): if "content" in entry: tweets.extend(entry["content"]["items"]) else: tweets.append(entry) - elif esw(("homeConversation-", - "profile-conversation-", - "conversationthread-")): + elif esw(("homeConversation-", "profile-conversation-", "conversationthread-")): tweets.extend(entry["content"]["items"]) elif esw("tombstone-"): item = entry["content"]["itemContent"] - item["tweet_results"] = \ - {"result": {"tombstone": item["tombstoneInfo"]}} + item["tweet_results"] = {"result": {"tombstone": item["tombstoneInfo"]}} tweets.append(entry) elif esw("cursor-bottom-"): cursor = entry["content"] @@ -1700,18 +1693,16 @@ def _pagination_tweets(self, endpoint, variables, for entry in tweets: try: - item = ((entry.get("content") or entry["item"]) - ["itemContent"]) + item = (entry.get("content") or entry["item"])["itemContent"] if "promotedMetadata" in item and not extr.ads: extr.log.debug( - "Skipping %s (ad)", - (entry.get("entryId") or "").rpartition("-")[2]) + "Skipping %s (ad)", (entry.get("entryId") or "").rpartition("-")[2] + ) continue tweet = item["tweet_results"]["result"] if "tombstone" in tweet: - tweet = self._process_tombstone( - entry, tweet["tombstone"]) + tweet = self._process_tombstone(entry, tweet["tombstone"]) if not tweet: continue @@ -1721,8 +1712,8 @@ def _pagination_tweets(self, endpoint, variables, tweet["sortIndex"] = entry.get("sortIndex") except KeyError: extr.log.debug( - "Skipping %s (deleted)", - (entry.get("entryId") or "").rpartition("-")[2]) + "Skipping %s (deleted)", (entry.get("entryId") or "").rpartition("-")[2] + ) continue if "retweeted_status_result" in legacy: @@ -1731,33 +1722,29 @@ def _pagination_tweets(self, endpoint, variables, retweet = retweet["tweet"] if original_retweets: try: - retweet["legacy"]["retweeted_status_id_str"] = \ - retweet["rest_id"] + retweet["legacy"]["retweeted_status_id_str"] = retweet["rest_id"] retweet["_retweet_id_str"] = tweet["rest_id"] tweet = retweet except KeyError: continue else: try: - legacy["retweeted_status_id_str"] = \ - retweet["rest_id"] - tweet["author"] = \ - retweet["core"]["user_results"]["result"] + legacy["retweeted_status_id_str"] = retweet["rest_id"] + tweet["author"] = retweet["core"]["user_results"]["result"] rtlegacy = retweet["legacy"] if "note_tweet" in retweet: tweet["note_tweet"] = retweet["note_tweet"] - if "extended_entities" in rtlegacy and \ - "extended_entities" not in legacy: - legacy["extended_entities"] = \ - rtlegacy["extended_entities"] + if ( + "extended_entities" in rtlegacy + and "extended_entities" not in legacy + ): + legacy["extended_entities"] = rtlegacy["extended_entities"] - if "withheld_scope" in rtlegacy and \ - "withheld_scope" not in legacy: - legacy["withheld_scope"] = \ - rtlegacy["withheld_scope"] + if "withheld_scope" in rtlegacy and "withheld_scope" not in legacy: + legacy["withheld_scope"] = rtlegacy["withheld_scope"] legacy["full_text"] = rtlegacy["full_text"] except KeyError: @@ -1768,17 +1755,15 @@ def _pagination_tweets(self, endpoint, variables, if "quoted_status_result" in tweet: try: quoted = tweet["quoted_status_result"]["result"] - quoted["legacy"]["quoted_by"] = ( - tweet["core"]["user_results"]["result"] - ["legacy"]["screen_name"]) + quoted["legacy"]["quoted_by"] = tweet["core"]["user_results"]["result"][ + "legacy" + ]["screen_name"] quoted["legacy"]["quoted_by_id_str"] = tweet["rest_id"] quoted["sortIndex"] = entry.get("sortIndex") yield quoted except KeyError: - extr.log.debug( - "Skipping quote of %s (deleted)", - tweet.get("rest_id")) + extr.log.debug("Skipping quote of %s (deleted)", tweet.get("rest_id")) continue if stop_tweets and not tweet: @@ -1794,7 +1779,7 @@ def _pagination_users(self, endpoint, variables, path=None): variables["cursor"] = cursor params = { "variables": None, - "features" : self._json_dumps(self.features_pagination), + "features": self._json_dumps(self.features_pagination), } while True: @@ -1804,8 +1789,7 @@ def _pagination_users(self, endpoint, variables, path=None): try: if path is None: - instructions = (data["user"]["result"]["timeline"] - ["timeline"]["instructions"]) + instructions = data["user"]["result"]["timeline"]["timeline"]["instructions"] else: for key in path: data = data[key] @@ -1818,8 +1802,7 @@ def _pagination_users(self, endpoint, variables, path=None): for entry in instr["entries"]: if entry["entryId"].startswith("user-"): try: - user = (entry["content"]["itemContent"] - ["user_results"]["result"]) + user = entry["content"]["itemContent"]["user_results"]["result"] except KeyError: pass else: @@ -1836,7 +1819,7 @@ def _handle_ratelimit(self, response): rl = self.extractor.config("ratelimit") if rl == "abort": raise exception.StopExtraction("Rate limit exceeded") - elif rl and isinstance(rl, str) and rl.startswith("wait:"): + if rl and isinstance(rl, str) and rl.startswith("wait:"): until = None seconds = text.parse_float(rl.partition(":")[2]) or 60.0 else: @@ -1848,21 +1831,19 @@ def _process_tombstone(self, entry, tombstone): text = (tombstone.get("richText") or tombstone["text"])["text"] tweet_id = entry["entryId"].rpartition("-")[2] - if text.startswith("Age-restricted"): - if self._nsfw_warning: - self._nsfw_warning = False - self.log.warning('"%s"', text) + if text.startswith("Age-restricted") and self._nsfw_warning: + self._nsfw_warning = False + self.log.warning('"%s"', text) self.log.debug("Skipping %s ('%s')", tweet_id, text) -@cache(maxage=365*86400, keyarg=1) +@cache(maxage=365 * 86400, keyarg=1) def _login_impl(extr, username, password): - def process(data, params=None): response = extr.request( - url, params=params, headers=headers, json=data, - method="POST", fatal=None) + url, params=params, headers=headers, json=data, method="POST", fatal=None + ) # update 'x-csrf-token' header (#5945) csrf_token = response.cookies.get("ct0") @@ -1876,15 +1857,14 @@ def process(data, params=None): else: if response.status_code < 400: try: - return (data["flow_token"], - data["subtasks"][0]["subtask_id"]) + return (data["flow_token"], data["subtasks"][0]["subtask_id"]) except LookupError: pass errors = [] for error in data.get("errors") or (): msg = error.get("message") - errors.append('"{}"'.format(msg) if msg else "Unknown error") + errors.append(f'"{msg}"' if msg else "Unknown error") extr.log.debug(response.text) raise exception.AuthenticationError(", ".join(errors)) @@ -1984,7 +1964,8 @@ def process(data, params=None): } elif subtask == "LoginEnterAlternateIdentifierSubtask": alt = extr.config("username-alt") or extr.input( - "Alternate Identifier (username, email, phone number): ") + "Alternate Identifier (username, email, phone number): " + ) data = { "enter_text": { "text": alt, @@ -2016,8 +1997,7 @@ def process(data, params=None): elif subtask == "DenyLoginSubtask": raise exception.AuthenticationError("Login rejected as suspicious") elif subtask == "LoginSuccessSubtask": - raise exception.AuthenticationError( - "No 'auth_token' cookie received") + raise exception.AuthenticationError("No 'auth_token' cookie received") else: raise exception.StopExtraction("Unrecognized subtask %s", subtask) @@ -2028,10 +2008,7 @@ def process(data, params=None): "subtask_inputs": [inputs], } - extr.sleep(random.uniform(1.0, 3.0), "login ({})".format(subtask)) + extr.sleep(random.uniform(1.0, 3.0), f"login ({subtask})") flow_token, subtask = process(data) - return { - cookie.name: cookie.value - for cookie in extr.cookies - } + return {cookie.name: cookie.value for cookie in extr.cookies} diff --git a/gallery_dl/extractor/unsplash.py b/gallery_dl/extractor/unsplash.py index a1b87b9c3..86a6b04da 100644 --- a/gallery_dl/extractor/unsplash.py +++ b/gallery_dl/extractor/unsplash.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,14 +6,17 @@ """Extractors for https://unsplash.com/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?unsplash\.com" class UnsplashExtractor(Extractor): """Base class for unsplash extractors""" + category = "unsplash" directory_fmt = ("{category}", "{user[username]}") filename_fmt = "{id}.{extension}" @@ -33,8 +34,7 @@ def items(self): metadata = self.metadata() for photo in self.photos(): - util.delete_items( - photo, ("current_user_collections", "related_collections")) + util.delete_items(photo, ("current_user_collections", "related_collections")) url = photo["urls"][fmt] text.nameext_from_url(url, photo) @@ -74,41 +74,45 @@ def _pagination(self, url, params, results=False): class UnsplashImageExtractor(UnsplashExtractor): """Extractor for a single unsplash photo""" + subcategory = "image" pattern = BASE_PATTERN + r"/photos/([^/?#]+)" example = "https://unsplash.com/photos/ID" def photos(self): - url = "{}/napi/photos/{}".format(self.root, self.item) + url = f"{self.root}/napi/photos/{self.item}" return (self.request(url).json(),) class UnsplashUserExtractor(UnsplashExtractor): """Extractor for all photos of an unsplash user""" + subcategory = "user" pattern = BASE_PATTERN + r"/@(\w+)/?$" example = "https://unsplash.com/@USER" def photos(self): - url = "{}/napi/users/{}/photos".format(self.root, self.item) + url = f"{self.root}/napi/users/{self.item}/photos" params = {"order_by": "latest"} return self._pagination(url, params) class UnsplashFavoriteExtractor(UnsplashExtractor): """Extractor for all likes of an unsplash user""" + subcategory = "favorite" pattern = BASE_PATTERN + r"/@(\w+)/likes" example = "https://unsplash.com/@USER/likes" def photos(self): - url = "{}/napi/users/{}/likes".format(self.root, self.item) + url = f"{self.root}/napi/users/{self.item}/likes" params = {"order_by": "latest"} return self._pagination(url, params) class UnsplashCollectionExtractor(UnsplashExtractor): """Extractor for an unsplash collection""" + subcategory = "collection" pattern = BASE_PATTERN + r"/collections/([^/?#]+)(?:/([^/?#]+))?" example = "https://unsplash.com/collections/12345/TITLE" @@ -121,13 +125,14 @@ def metadata(self): return {"collection_id": self.item, "collection_title": self.title} def photos(self): - url = "{}/napi/collections/{}/photos".format(self.root, self.item) + url = f"{self.root}/napi/collections/{self.item}/photos" params = {"order_by": "latest"} return self._pagination(url, params) class UnsplashSearchExtractor(UnsplashExtractor): """Extractor for unsplash search results""" + subcategory = "search" pattern = BASE_PATTERN + r"/s/photos/([^/?#]+)(?:\?([^#]+))?" example = "https://unsplash.com/s/photos/QUERY" @@ -138,7 +143,7 @@ def __init__(self, match): def photos(self): url = self.root + "/napi/search/photos" - params = {"query": text.unquote(self.item.replace('-', ' '))} + params = {"query": text.unquote(self.item.replace("-", " "))} if self.query: params.update(text.parse_query(self.query)) return self._pagination(url, params, True) diff --git a/gallery_dl/extractor/uploadir.py b/gallery_dl/extractor/uploadir.py index ce34e7de8..96e49d182 100644 --- a/gallery_dl/extractor/uploadir.py +++ b/gallery_dl/extractor/uploadir.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,14 @@ """Extractors for https://uploadir.com/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class UploadirFileExtractor(Extractor): """Extractor for uploadir files""" + category = "uploadir" subcategory = "file" root = "https://uploadir.com" @@ -27,7 +27,7 @@ def __init__(self, match): self.file_id = match.group(1) def items(self): - url = "{}/u/{}".format(self.root, self.file_id) + url = f"{self.root}/u/{self.file_id}" response = self.request(url, method="HEAD", allow_redirects=False) if 300 <= response.status_code < 400: @@ -38,18 +38,20 @@ def items(self): url = self.root + extr('class="form" action="', '"') token = extr('name="authenticity_token" value="', '"') - data = text.nameext_from_url(name, { - "_http_method": "POST", - "_http_data" : { - "authenticity_token": token, - "upload_id": self.file_id, + data = text.nameext_from_url( + name, + { + "_http_method": "POST", + "_http_data": { + "authenticity_token": token, + "upload_id": self.file_id, + }, }, - }) + ) else: hcd = response.headers.get("Content-Disposition") - name = (hcd.partition("filename*=UTF-8''")[2] or - text.extr(hcd, 'filename="', '"')) + name = hcd.partition("filename*=UTF-8''")[2] or text.extr(hcd, 'filename="', '"') data = text.nameext_from_url(name) data["id"] = self.file_id diff --git a/gallery_dl/extractor/urlgalleries.py b/gallery_dl/extractor/urlgalleries.py index bb8005584..3ee79faf2 100644 --- a/gallery_dl/extractor/urlgalleries.py +++ b/gallery_dl/extractor/urlgalleries.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://urlgalleries.net/""" -from .common import GalleryExtractor, Message -from .. import text, exception +from .. import exception +from .. import text +from .common import GalleryExtractor +from .common import Message class UrlgalleriesGalleryExtractor(GalleryExtractor): """Base class for Urlgalleries extractors""" + category = "urlgalleries" root = "https://urlgalleries.net" request_interval = (0.5, 1.5) @@ -20,13 +21,11 @@ class UrlgalleriesGalleryExtractor(GalleryExtractor): def items(self): blog, self.gallery_id = self.groups - url = "https://{}.urlgalleries.net/porn-gallery-{}/?a=10000".format( - blog, self.gallery_id) + url = f"https://{blog}.urlgalleries.net/porn-gallery-{self.gallery_id}/?a=10000" with self.request(url, allow_redirects=False, fatal=...) as response: if 300 <= response.status_code < 500: - if response.headers.get("location", "").endswith( - "/not_found_adult.php"): + if response.headers.get("location", "").endswith("/not_found_adult.php"): raise exception.NotFoundError("gallery") raise exception.HttpError(None, response) page = response.text @@ -35,7 +34,7 @@ def items(self): data = self.metadata(page) data["count"] = len(imgs) - root = "https://{}.urlgalleries.net".format(blog) + root = f"https://{blog}.urlgalleries.net" yield Message.Directory, data for data["num"], img in enumerate(imgs, 1): page = self.request(root + img).text @@ -47,11 +46,10 @@ def metadata(self, page): return { "gallery_id": self.gallery_id, "_site": extr(' title="', '"'), # site name - "blog" : text.unescape(extr(' title="', '"')), + "blog": text.unescape(extr(' title="', '"')), "_rprt": extr(' title="', '"'), # report button "title": text.unescape(extr(' title="', '"').strip()), - "date" : text.parse_datetime( - extr(" images in gallery | ", "<"), "%B %d, %Y %H:%M"), + "date": text.parse_datetime(extr(" images in gallery | ", "<"), "%B %d, %Y %H:%M"), } def images(self, page): diff --git a/gallery_dl/extractor/urlshortener.py b/gallery_dl/extractor/urlshortener.py index 49a3debdf..7b6f92228 100644 --- a/gallery_dl/extractor/urlshortener.py +++ b/gallery_dl/extractor/urlshortener.py @@ -1,36 +1,39 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for general-purpose URL shorteners""" -from .common import BaseExtractor, Message from .. import exception +from .common import BaseExtractor +from .common import Message class UrlshortenerExtractor(BaseExtractor): """Base class for URL shortener extractors""" + basecategory = "urlshortener" -BASE_PATTERN = UrlshortenerExtractor.update({ - "bitly": { - "root": "https://bit.ly", - "pattern": r"bit\.ly", - }, - "tco": { - # t.co sends 'http-equiv="refresh"' (200) when using browser UA - "headers": {"User-Agent": None}, - "root": "https://t.co", - "pattern": r"t\.co", - }, -}) +BASE_PATTERN = UrlshortenerExtractor.update( + { + "bitly": { + "root": "https://bit.ly", + "pattern": r"bit\.ly", + }, + "tco": { + # t.co sends 'http-equiv="refresh"' (200) when using browser UA + "headers": {"User-Agent": None}, + "root": "https://t.co", + "pattern": r"t\.co", + }, + } +) class UrlshortenerLinkExtractor(UrlshortenerExtractor): """Extractor for general-purpose URL shorteners""" + subcategory = "link" pattern = BASE_PATTERN + r"/([^/?#]+)" example = "https://bit.ly/abcde" @@ -44,8 +47,12 @@ def _init(self): def items(self): response = self.request( - "{}/{}".format(self.root, self.id), headers=self.headers, - method="HEAD", allow_redirects=False, notfound="URL") + f"{self.root}/{self.id}", + headers=self.headers, + method="HEAD", + allow_redirects=False, + notfound="URL", + ) try: yield Message.Queue, response.headers["location"], {} except KeyError: diff --git a/gallery_dl/extractor/vanillarock.py b/gallery_dl/extractor/vanillarock.py index 1ce969f7e..40a2effb4 100644 --- a/gallery_dl/extractor/vanillarock.py +++ b/gallery_dl/extractor/vanillarock.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,14 @@ """Extractors for https://vanilla-rock.com/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class VanillarockExtractor(Extractor): """Base class for vanillarock extractors""" + category = "vanillarock" root = "https://vanilla-rock.com" @@ -24,12 +24,12 @@ def __init__(self, match): class VanillarockPostExtractor(VanillarockExtractor): """Extractor for blogposts on vanilla-rock.com""" + subcategory = "post" directory_fmt = ("{category}", "{path}") filename_fmt = "{num:>02}.{extension}" archive_fmt = "{filename}" - pattern = (r"(?:https?://)?(?:www\.)?vanilla-rock\.com" - r"(/(?!category/|tag/)[^/?#]+)/?$") + pattern = r"(?:https?://)?(?:www\.)?vanilla-rock\.com" r"(/(?!category/|tag/)[^/?#]+)/?$" example = "https://vanilla-rock.com/TITLE" def items(self): @@ -38,7 +38,7 @@ def items(self): imgs = [] while True: - img = extr('
    ', '
    ') + img = extr('
    ', "
    ") if not img: break imgs.append(text.extr(img, 'href="', '"')) @@ -46,11 +46,9 @@ def items(self): data = { "count": len(imgs), "title": text.unescape(name), - "path" : self.path.strip("/"), - "date" : text.parse_datetime(extr( - '
    ', '
    '), "%Y-%m-%d %H:%M"), - "tags" : text.split_html(extr( - '
    ', '
    '))[::2], + "path": self.path.strip("/"), + "date": text.parse_datetime(extr('
    ', "
    "), "%Y-%m-%d %H:%M"), + "tags": text.split_html(extr('
    ', "
    "))[::2], } yield Message.Directory, data @@ -60,9 +58,9 @@ def items(self): class VanillarockTagExtractor(VanillarockExtractor): """Extractor for vanillarock blog posts by tag or category""" + subcategory = "tag" - pattern = (r"(?:https?://)?(?:www\.)?vanilla-rock\.com" - r"(/(?:tag|category)/[^?#]+)") + pattern = r"(?:https?://)?(?:www\.)?vanilla-rock\.com" r"(/(?:tag|category)/[^?#]+)" example = "https://vanilla-rock.com/tag/TAG" def items(self): @@ -72,7 +70,7 @@ def items(self): while url: extr = text.extract_from(self.request(url).text) while True: - post = extr('

    ', '

    ') + post = extr('

    ', "

    ") if not post: break yield Message.Queue, text.extr(post, 'href="', '"'), data diff --git a/gallery_dl/extractor/vichan.py b/gallery_dl/extractor/vichan.py index 654c4512c..c1e100b3f 100644 --- a/gallery_dl/extractor/vichan.py +++ b/gallery_dl/extractor/vichan.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,29 +6,34 @@ """Extractors for vichan imageboards""" -from .common import BaseExtractor, Message from .. import text +from .common import BaseExtractor +from .common import Message class VichanExtractor(BaseExtractor): """Base class for vichan extractors""" + basecategory = "vichan" -BASE_PATTERN = VichanExtractor.update({ - "8kun": { - "root": "https://8kun.top", - "pattern": r"8kun\.top", - }, - "smugloli": { - "root": None, - "pattern": r"smuglo(?:\.li|li\.net)", - }, -}) +BASE_PATTERN = VichanExtractor.update( + { + "8kun": { + "root": "https://8kun.top", + "pattern": r"8kun\.top", + }, + "smugloli": { + "root": None, + "pattern": r"smuglo(?:\.li|li\.net)", + }, + } +) class VichanThreadExtractor(VichanExtractor): """Extractor for vichan threads""" + subcategory = "thread" directory_fmt = ("{category}", "{board}", "{thread} {title}") filename_fmt = "{time}{num:?-//} {filename}.{extension}" @@ -41,20 +44,19 @@ class VichanThreadExtractor(VichanExtractor): def __init__(self, match): VichanExtractor.__init__(self, match) index = match.lastindex - self.board = match.group(index-1) + self.board = match.group(index - 1) self.thread = match.group(index) def items(self): - url = "{}/{}/res/{}.json".format(self.root, self.board, self.thread) + url = f"{self.root}/{self.board}/res/{self.thread}.json" posts = self.request(url).json()["posts"] title = posts[0].get("sub") or text.remove_html(posts[0]["com"]) - process = (self._process_8kun if self.category == "8kun" else - self._process) + process = self._process_8kun if self.category == "8kun" else self._process data = { - "board" : self.board, + "board": self.board, "thread": self.thread, - "title" : text.unescape(title)[:50], - "num" : 0, + "title": text.unescape(title)[:50], + "num": 0, } yield Message.Directory, data @@ -62,15 +64,13 @@ def items(self): if "filename" in post: yield process(post, data) if "extra_files" in post: - for post["num"], filedata in enumerate( - post["extra_files"], 1): + for post["num"], filedata in enumerate(post["extra_files"], 1): yield process(post, filedata) def _process(self, post, data): post.update(data) post["extension"] = post["ext"][1:] - post["url"] = "{}/{}/src/{}{}".format( - self.root, post["board"], post["tim"], post["ext"]) + post["url"] = "{}/{}/src/{}{}".format(self.root, post["board"], post["tim"], post["ext"]) return Message.Url, post["url"], post @staticmethod @@ -80,17 +80,18 @@ def _process_8kun(post, data): tim = post["tim"] if len(tim) > 16: - post["url"] = "https://media.128ducks.com/file_store/{}{}".format( - tim, post["ext"]) + post["url"] = "https://media.128ducks.com/file_store/{}{}".format(tim, post["ext"]) else: post["url"] = "https://media.128ducks.com/{}/src/{}{}".format( - post["board"], tim, post["ext"]) + post["board"], tim, post["ext"] + ) return Message.Url, post["url"], post class VichanBoardExtractor(VichanExtractor): """Extractor for vichan boards""" + subcategory = "board" pattern = BASE_PATTERN + r"/([^/?#]+)(?:/index|/catalog|/\d+|/?$)" example = "https://8kun.top/a/" @@ -100,13 +101,12 @@ def __init__(self, match): self.board = match.group(match.lastindex) def items(self): - url = "{}/{}/threads.json".format(self.root, self.board) + url = f"{self.root}/{self.board}/threads.json" threads = self.request(url).json() for page in threads: for thread in page["threads"]: - url = "{}/{}/res/{}.html".format( - self.root, self.board, thread["no"]) + url = "{}/{}/res/{}.html".format(self.root, self.board, thread["no"]) thread["page"] = page["page"] thread["_extractor"] = VichanThreadExtractor yield Message.Queue, url, thread diff --git a/gallery_dl/extractor/vipergirls.py b/gallery_dl/extractor/vipergirls.py index 5cde0d6c7..149f348f2 100644 --- a/gallery_dl/extractor/vipergirls.py +++ b/gallery_dl/extractor/vipergirls.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,17 +6,21 @@ """Extractors for https://vipergirls.to/""" -from .common import Extractor, Message -from .. import text, util, exception -from ..cache import cache +from xml.etree import ElementTree as ET -from xml.etree import ElementTree +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?vipergirls\.to" class VipergirlsExtractor(Extractor): """Base class for vipergirls extractors""" + category = "vipergirls" root = "https://vipergirls.to" request_interval = 0.5 @@ -67,30 +69,29 @@ def login(self): if username: self.cookies_update(self._login_impl(username, password)) - @cache(maxage=90*86400, keyarg=1) + @cache(maxage=90 * 86400, keyarg=1) def _login_impl(self, username, password): self.log.info("Logging in as %s", username) - url = "{}/login.php?do=login".format(self.root) + url = f"{self.root}/login.php?do=login" data = { "vb_login_username": username, "vb_login_password": password, - "do" : "login", - "cookieuser" : "1", + "do": "login", + "cookieuser": "1", } response = self.request(url, method="POST", data=data) if not response.cookies.get("vg_password"): raise exception.AuthenticationError() - return {cookie.name: cookie.value - for cookie in response.cookies} + return {cookie.name: cookie.value for cookie in response.cookies} def like(self, post, user_hash): url = self.root + "/post_thanks.php" params = { - "do" : "post_thanks_add", - "p" : post.get("id"), + "do": "post_thanks_add", + "p": post.get("id"), "securitytoken": user_hash, } @@ -100,9 +101,9 @@ def like(self, post, user_hash): class VipergirlsThreadExtractor(VipergirlsExtractor): """Extractor for vipergirls threads""" + subcategory = "thread" - pattern = (BASE_PATTERN + - r"/threads/(\d+)(?:-[^/?#]+)?(/page\d+)?(?:$|#|\?(?!p=))") + pattern = BASE_PATTERN + r"/threads/(\d+)(?:-[^/?#]+)?(/page\d+)?(?:$|#|\?(?!p=))" example = "https://vipergirls.to/threads/12345-TITLE" def __init__(self, match): @@ -110,15 +111,15 @@ def __init__(self, match): self.thread_id, self.page = match.groups() def posts(self): - url = "{}/vr.php?t={}".format(self.root, self.thread_id) - return ElementTree.fromstring(self.request(url).text) + url = f"{self.root}/vr.php?t={self.thread_id}" + return ET.fromstring(self.request(url).text) class VipergirlsPostExtractor(VipergirlsExtractor): """Extractor for vipergirls posts""" + subcategory = "post" - pattern = (BASE_PATTERN + - r"/threads/(\d+)(?:-[^/?#]+)?\?p=\d+[^#]*#post(\d+)") + pattern = BASE_PATTERN + r"/threads/(\d+)(?:-[^/?#]+)?\?p=\d+[^#]*#post(\d+)" example = "https://vipergirls.to/threads/12345-TITLE?p=23456#post23456" def __init__(self, match): @@ -127,5 +128,5 @@ def __init__(self, match): self.page = 0 def posts(self): - url = "{}/vr.php?p={}".format(self.root, self.post_id) - return ElementTree.fromstring(self.request(url).text) + url = f"{self.root}/vr.php?p={self.post_id}" + return ET.fromstring(self.request(url).text) diff --git a/gallery_dl/extractor/vk.py b/gallery_dl/extractor/vk.py index ea034a79a..bf9400e5a 100644 --- a/gallery_dl/extractor/vk.py +++ b/gallery_dl/extractor/vk.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,15 +6,19 @@ """Extractors for https://vk.com/""" -from .common import Extractor, Message -from .. import text, exception import re +from .. import exception +from .. import text +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https://)?(?:www\.|m\.)?vk\.com" class VkExtractor(Extractor): """Base class for vk extractors""" + category = "vk" directory_fmt = ("{category}", "{user[name]|user[id]}") filename_fmt = "{id}.{extension}" @@ -39,7 +41,6 @@ def items(self): yield Message.Directory, data for photo in self.photos(): - for size in sizes: size += "_" if size in photo: @@ -74,26 +75,30 @@ def _pagination(self, photos_id): url = self.root + "/al_photos.php" headers = { "X-Requested-With": "XMLHttpRequest", - "Origin" : self.root, - "Referer" : self.root + "/" + photos_id, + "Origin": self.root, + "Referer": self.root + "/" + photos_id, } data = { - "act" : "show", - "al" : "1", + "act": "show", + "al": "1", "direction": "1", - "list" : photos_id, - "offset" : self.offset, + "list": photos_id, + "offset": self.offset, } while True: payload = self.request( - url, method="POST", headers=headers, data=data, + url, + method="POST", + headers=headers, + data=data, ).json()["payload"][1] if len(payload) < 4: self.log.debug(payload) raise exception.AuthorizationError( - text.unescape(payload[0]) if payload[0] else None) + text.unescape(payload[0]) if payload[0] else None + ) total = payload[1] photos = payload[3] @@ -114,10 +119,11 @@ def _pagination(self, photos_id): class VkPhotosExtractor(VkExtractor): """Extractor for photos from a vk user""" + subcategory = "photos" - pattern = (BASE_PATTERN + r"/(?:" - r"(?:albums|photos|id)(-?\d+)" - r"|(?!(?:album|tag)-?\d+_?)([^/?#]+))") + pattern = ( + BASE_PATTERN + r"/(?:" r"(?:albums|photos|id)(-?\d+)" r"|(?!(?:album|tag)-?\d+_?)([^/?#]+))" + ) example = "https://vk.com/id12345" def __init__(self, match): @@ -134,27 +140,28 @@ def metadata(self): url = "{}/{}{}".format(self.root, prefix, user_id.lstrip("-")) data = self._extract_profile(url) else: - url = "{}/{}".format(self.root, self.user_name) + url = f"{self.root}/{self.user_name}" data = self._extract_profile(url) self.user_id = data["user"]["id"] return data def _extract_profile(self, url): extr = text.extract_from(self.request(url).text) - return {"user": { - "name": text.unescape(extr( - 'rel="canonical" href="https://vk.com/', '"')), - "nick": text.unescape(extr( - '

    ', "<")).replace(" ", " "), - "info": text.unescape(text.remove_html(extr( - '', '', "<")).replace(" ", " "), + "info": text.unescape( + text.remove_html(extr('', ""): path = text.rextract(wp, ' src="', '"')[0] if path: image = text.nameext_from_url(path) diff --git a/gallery_dl/extractor/warosu.py b/gallery_dl/extractor/warosu.py index 61a36d5a8..7a6e6df4c 100644 --- a/gallery_dl/extractor/warosu.py +++ b/gallery_dl/extractor/warosu.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,14 @@ """Extractors for https://warosu.org/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class WarosuThreadExtractor(Extractor): """Extractor for threads on warosu.org""" + category = "warosu" subcategory = "thread" root = "https://warosu.org" @@ -28,14 +28,13 @@ def __init__(self, match): self.board, self.thread = match.groups() def items(self): - url = "{}/{}/thread/{}".format(self.root, self.board, self.thread) + url = f"{self.root}/{self.board}/thread/{self.thread}" page = self.request(url).text data = self.metadata(page) posts = self.posts(page) if not data["title"]: - data["title"] = text.unescape(text.remove_html( - posts[0]["com"]))[:50] + data["title"] = text.unescape(text.remove_html(posts[0]["com"]))[:50] yield Message.Directory, data for post in posts: @@ -49,10 +48,10 @@ def metadata(self, page): boardname = text.extr(page, "", "") title = text.unescape(text.extr(page, "class=filetitle>", "<")) return { - "board" : self.board, + "board": self.board, "board_name": boardname.split(" - ")[1], - "thread" : self.thread, - "title" : title, + "thread": self.thread, + "title": title, } def posts(self, page): @@ -73,12 +72,11 @@ def parse(self, post): def _extract_post(self, post): extr = text.extract_from(post) return { - "no" : extr("id=p", ">"), + "no": extr("id=p", ">"), "name": extr("class=postername>", "<").strip(), "time": extr("class=posttime title=", "000>"), - "now" : extr("", "<").strip(), - "com" : text.unescape(text.remove_html(extr( - "
    ", "
    ").strip())), + "now": extr("", "<").strip(), + "com": text.unescape(text.remove_html(extr("
    ", "
    ").strip())), } def _extract_image(self, post, data): @@ -86,8 +84,7 @@ def _extract_image(self, post, data): data["fsize"] = extr(" File: ", ", ") data["w"] = extr("", "x") data["h"] = extr("", ", ") - data["filename"] = text.unquote(extr( - "", "<").rstrip().rpartition(".")[0]) + data["filename"] = text.unquote(extr("", "<").rstrip().rpartition(".")[0]) extr("
    ", "") url = extr("    ") diff --git a/gallery_dl/extractor/weasyl.py b/gallery_dl/extractor/weasyl.py index 13b052069..adda29a8c 100644 --- a/gallery_dl/extractor/weasyl.py +++ b/gallery_dl/extractor/weasyl.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://www.weasyl.com/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https://)?(?:www\.)?weasyl.com/" @@ -24,22 +23,19 @@ def populate_submission(data): # Some submissions don't have content and can be skipped if "submission" in data["media"]: data["url"] = data["media"]["submission"][0]["url"] - data["date"] = text.parse_datetime( - data["posted_at"][:19], "%Y-%m-%dT%H:%M:%S") + data["date"] = text.parse_datetime(data["posted_at"][:19], "%Y-%m-%dT%H:%M:%S") text.nameext_from_url(data["url"], data) return True return False def _init(self): - self.session.headers['X-Weasyl-API-Key'] = self.config("api-key") + self.session.headers["X-Weasyl-API-Key"] = self.config("api-key") def request_submission(self, submitid): - return self.request( - "{}/api/submissions/{}/view".format(self.root, submitid)).json() + return self.request(f"{self.root}/api/submissions/{submitid}/view").json() def retrieve_journal(self, journalid): - data = self.request( - "{}/api/journals/{}/view".format(self.root, journalid)).json() + data = self.request(f"{self.root}/api/journals/{journalid}/view").json() data["extension"] = "html" data["html"] = "text:" + data["content"] data["date"] = text.parse_datetime(data["posted_at"]) @@ -47,9 +43,9 @@ def retrieve_journal(self, journalid): def submissions(self, owner_login, folderid=None): metadata = self.config("metadata") - url = "{}/api/users/{}/gallery".format(self.root, owner_login) + url = f"{self.root}/api/users/{owner_login}/gallery" params = { - "nextid" : None, + "nextid": None, "folderid": folderid, } @@ -57,8 +53,7 @@ def submissions(self, owner_login, folderid=None): data = self.request(url, params=params).json() for submission in data["submissions"]: if metadata: - submission = self.request_submission( - submission["submitid"]) + submission = self.request_submission(submission["submitid"]) if self.populate_submission(submission): submission["folderid"] = folderid # Do any submissions have more than one url? If so @@ -150,9 +145,9 @@ def __init__(self, match): def items(self): yield Message.Directory, {"owner_login": self.owner_login} - url = "{}/journals/{}".format(self.root, self.owner_login) + url = f"{self.root}/journals/{self.owner_login}" page = self.request(url).text - for journalid in text.extract_iter(page, 'href="/journal/', '/'): + for journalid in text.extract_iter(page, 'href="/journal/', "/"): data = self.retrieve_journal(journalid) yield Message.Url, data["html"], data @@ -173,7 +168,7 @@ def items(self): else: path = "/favorites" params = { - "userid" : userid, + "userid": userid, "feature": "submit", } diff --git a/gallery_dl/extractor/webmshare.py b/gallery_dl/extractor/webmshare.py index 7e2b5ea32..2c9e93673 100644 --- a/gallery_dl/extractor/webmshare.py +++ b/gallery_dl/extractor/webmshare.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,19 +6,20 @@ """Extractors for https://webmshare.com/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class WebmshareVideoExtractor(Extractor): """Extractor for webmshare videos""" + category = "webmshare" subcategory = "video" root = "https://webmshare.com" filename_fmt = "{id}{title:? //}.{extension}" archive_fmt = "{id}" - pattern = (r"(?:https?://)?(?:s\d+\.)?webmshare\.com" - r"/(?:play/|download-webm/)?(\w{3,})") + pattern = r"(?:https?://)?(?:s\d+\.)?webmshare\.com" r"/(?:play/|download-webm/)?(\w{3,})" example = "https://webmshare.com/_ID_" def __init__(self, match): @@ -28,23 +27,19 @@ def __init__(self, match): self.video_id = match.group(1) def items(self): - url = "{}/{}".format(self.root, self.video_id) + url = f"{self.root}/{self.video_id}" extr = text.extract_from(self.request(url).text) data = { - "title": text.unescape(extr( - 'property="og:title" content="', '"').rpartition(" — ")[0]), + "title": text.unescape(extr('property="og:title" content="', '"').rpartition(" — ")[0]), "thumb": "https:" + extr('property="og:image" content="', '"'), - "url" : "https:" + extr('property="og:video" content="', '"'), - "width": text.parse_int(extr( - 'property="og:video:width" content="', '"')), - "height": text.parse_int(extr( - 'property="og:video:height" content="', '"')), - "date" : text.parse_datetime(extr( - "Added ", "<"), "%B %d, %Y"), - "views": text.parse_int(extr('glyphicon-eye-open">', '<')), - "id" : self.video_id, - "filename" : self.video_id, + "url": "https:" + extr('property="og:video" content="', '"'), + "width": text.parse_int(extr('property="og:video:width" content="', '"')), + "height": text.parse_int(extr('property="og:video:height" content="', '"')), + "date": text.parse_datetime(extr("Added ", "<"), "%B %d, %Y"), + "views": text.parse_int(extr('glyphicon-eye-open">', "<")), + "id": self.video_id, + "filename": self.video_id, "extension": "webm", } diff --git a/gallery_dl/extractor/webtoons.py b/gallery_dl/extractor/webtoons.py index 70ab259d9..2c366b541 100644 --- a/gallery_dl/extractor/webtoons.py +++ b/gallery_dl/extractor/webtoons.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2020 Leonardo Taccari # Copyright 2021-2023 Mike Fährmann # @@ -9,74 +7,91 @@ """Extractors for https://www.webtoons.com/""" -from .common import GalleryExtractor, Extractor, Message -from .. import exception, text, util +from .. import exception +from .. import text +from .. import util +from .common import Extractor +from .common import GalleryExtractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?webtoons\.com/(([^/?#]+)" -class WebtoonsBase(): +class WebtoonsBase: category = "webtoons" root = "https://www.webtoons.com" cookies_domain = ".webtoons.com" def setup_agegate_cookies(self): - self.cookies_update({ - "atGDPR" : "AD_CONSENT", - "needCCPA" : "false", - "needCOPPA" : "false", - "needGDPR" : "false", - "pagGDPR" : "true", - "ageGatePass": "true", - }) + self.cookies_update( + { + "atGDPR": "AD_CONSENT", + "needCCPA": "false", + "needCOPPA": "false", + "needGDPR": "false", + "pagGDPR": "true", + "ageGatePass": "true", + } + ) def request(self, url, **kwargs): response = Extractor.request(self, url, **kwargs) if response.history and "/ageGate" in response.url: raise exception.StopExtraction( - "HTTP redirect to age gate check ('%s')", response.request.url) + "HTTP redirect to age gate check ('%s')", response.request.url + ) return response class WebtoonsEpisodeExtractor(WebtoonsBase, GalleryExtractor): """Extractor for an episode on webtoons.com""" + subcategory = "episode" directory_fmt = ("{category}", "{comic}") filename_fmt = "{episode_no}-{num:>02}.{extension}" archive_fmt = "{title_no}_{episode_no}_{num}" - pattern = (BASE_PATTERN + r"/([^/?#]+)/([^/?#]+)/(?:[^/?#]+))" - r"/viewer(?:\?([^#'\"]+))") - example = ("https://www.webtoons.com/en/GENRE/TITLE/NAME/viewer" - "?title_no=123&episode_no=12345") + pattern = BASE_PATTERN + r"/([^/?#]+)/([^/?#]+)/(?:[^/?#]+))" r"/viewer(?:\?([^#'\"]+))" + example = "https://www.webtoons.com/en/GENRE/TITLE/NAME/viewer" "?title_no=123&episode_no=12345" test = ( - (("https://www.webtoons.com/en/comedy/safely-endangered" - "/ep-572-earth/viewer?title_no=352&episode_no=572"), { - "url": "55bec5d7c42aba19e3d0d56db25fdf0b0b13be38", - "content": ("1748c7e82b6db910fa179f6dc7c4281b0f680fa7", - "42055e44659f6ffc410b3fb6557346dfbb993df3", - "49e1f2def04c6f7a6a3dacf245a1cd9abe77a6a9"), - "count": 5, - }), - (("https://www.webtoons.com/en/challenge/punderworld" - "/happy-earth-day-/viewer?title_no=312584&episode_no=40"), { - "exception": exception.NotFoundError, - "keyword": { - "comic": "punderworld", - "description": str, - "episode": "36", - "episode_no": "40", - "genre": "challenge", - "title": r"re:^Punderworld - .+", - "title_no": "312584", + ( + ( + "https://www.webtoons.com/en/comedy/safely-endangered" + "/ep-572-earth/viewer?title_no=352&episode_no=572" + ), + { + "url": "55bec5d7c42aba19e3d0d56db25fdf0b0b13be38", + "content": ( + "1748c7e82b6db910fa179f6dc7c4281b0f680fa7", + "42055e44659f6ffc410b3fb6557346dfbb993df3", + "49e1f2def04c6f7a6a3dacf245a1cd9abe77a6a9", + ), + "count": 5, }, - }), + ), + ( + ( + "https://www.webtoons.com/en/challenge/punderworld" + "/happy-earth-day-/viewer?title_no=312584&episode_no=40" + ), + { + "exception": exception.NotFoundError, + "keyword": { + "comic": "punderworld", + "description": str, + "episode": "36", + "episode_no": "40", + "genre": "challenge", + "title": r"re:^Punderworld - .+", + "title_no": "312584", + }, + }, + ), ) def __init__(self, match): - self.path, self.lang, self.genre, self.comic, self.query = \ - match.groups() + self.path, self.lang, self.genre, self.comic, self.query = match.groups() - url = "{}/{}/viewer?{}".format(self.root, self.path, self.query) + url = f"{self.root}/{self.path}/viewer?{self.query}" GalleryExtractor.__init__(self, match, url) def _init(self): @@ -91,60 +106,55 @@ def metadata(self, page): title = extr('', '<') + if extr('
    ", "<") episode_name = extr('

    #', '<') - else: - episode = "" + episode = extr(">#", "<") if extr('', '') + if extr('
    ", "") else: username = author_name = "" return { - "genre" : self.genre, - "comic" : self.comic, - "title_no" : self.title_no, - "episode_no" : self.episode_no, - "title" : text.unescape(title), - "episode" : episode, - "comic_name" : text.unescape(comic_name), + "genre": self.genre, + "comic": self.comic, + "title_no": self.title_no, + "episode_no": self.episode_no, + "title": text.unescape(title), + "episode": episode, + "comic_name": text.unescape(comic_name), "episode_name": text.unescape(episode_name), - "username" : username, - "author_name" : text.unescape(author_name), - "description" : text.unescape(descr), - "lang" : self.lang, - "language" : util.code_to_language(self.lang), + "username": username, + "author_name": text.unescape(author_name), + "description": text.unescape(descr), + "lang": self.lang, + "language": util.code_to_language(self.lang), } @staticmethod def images(page): return [ (url.replace("://webtoon-phinf.", "://swebtoon-phinf."), None) - for url in text.extract_iter( - page, 'class="_images" data-url="', '"') + for url in text.extract_iter(page, 'class="_images" data-url="', '"') ] class WebtoonsComicExtractor(WebtoonsBase, Extractor): """Extractor for an entire comic on webtoons.com""" + subcategory = "comic" categorytransfer = True - pattern = (BASE_PATTERN + r"/([^/?#]+)/([^/?#]+))" - r"/list(?:\?([^#]+))") + pattern = BASE_PATTERN + r"/([^/?#]+)/([^/?#]+))" r"/list(?:\?([^#]+))" example = "https://www.webtoons.com/en/GENRE/TITLE/list?title_no=123" def __init__(self, match): Extractor.__init__(self, match) - self.path, self.lang, self.genre, self.comic, self.query = \ - match.groups() + self.path, self.lang, self.genre, self.comic, self.query = match.groups() def _init(self): self.setup_agegate_cookies() @@ -157,12 +167,11 @@ def items(self): page = None data = { "_extractor": WebtoonsEpisodeExtractor, - "title_no" : text.parse_int(self.title_no), + "title_no": text.parse_int(self.title_no), } while True: - path = "/{}/list?title_no={}&page={}".format( - self.path, self.title_no, self.page_no) + path = f"/{self.path}/list?title_no={self.title_no}&page={self.page_no}" if page and path not in page: return @@ -185,8 +194,5 @@ def items(self): @staticmethod def get_episode_urls(page): """Extract and return all episode urls in 'page'""" - page = text.extr(page, 'id="_listUl"', '') - return [ - match.group(0) - for match in WebtoonsEpisodeExtractor.pattern.finditer(page) - ] + page = text.extr(page, 'id="_listUl"', "") + return [match.group(0) for match in WebtoonsEpisodeExtractor.pattern.finditer(page)] diff --git a/gallery_dl/extractor/weibo.py b/gallery_dl/extractor/weibo.py index 9885d79f7..cd7c03a32 100644 --- a/gallery_dl/extractor/weibo.py +++ b/gallery_dl/extractor/weibo.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,11 +6,15 @@ """Extractors for https://www.weibo.com/""" -from .common import Extractor, Message -from .. import text, util, exception -from ..cache import cache import random +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?(?:www\.|m\.)?weibo\.c(?:om|n)" USER_PATTERN = BASE_PATTERN + r"/(?:(u|n|p(?:rofile)?)/)?([^/?#]+)(?:/home)?" @@ -34,7 +36,7 @@ def _init(self): self.retweets = self.config("retweets", False) self.videos = self.config("videos", True) self.gifs = self.config("gifs", True) - self.gifs_video = (self.gifs == "video") + self.gifs_video = self.gifs == "video" cookies = _cookie_cache() if cookies is not None: @@ -46,8 +48,8 @@ def request(self, url, **kwargs): if response.history: if "login.sina.com" in response.url: raise exception.StopExtraction( - "HTTP redirect to login page (%s)", - response.url.partition("?")[0]) + "HTTP redirect to login page (%s)", response.url.partition("?")[0] + ) if "passport.weibo.com" in response.url: self._sina_visitor_system(response) response = Extractor.request(self, url, **kwargs) @@ -55,10 +57,9 @@ def request(self, url, **kwargs): return response def items(self): - original_retweets = (self.retweets == "original") + original_retweets = self.retweets == "original" for status in self.statuses(): - if "ori_mid" in status and not self.retweets: self.log.debug("Skipping %s (快转 retweet)", status["id"]) continue @@ -80,8 +81,7 @@ def items(self): files = [] self._extract_status(status, files) - status["date"] = text.parse_datetime( - status["created_at"], "%a %b %d %H:%M:%S %z %Y") + status["date"] = text.parse_datetime(status["created_at"], "%a %b %d %H:%M:%S %z %Y") status["count"] = len(files) yield Message.Directory, status @@ -127,7 +127,6 @@ def _extract_status(self, status, files): elif pic_type == "livephoto" and self.livephoto: append(pic["largest"].copy()) append({"url": pic["video"]}) - else: append(pic["largest"].copy()) @@ -138,27 +137,23 @@ def _extract_status(self, status, files): def _extract_video(self, info): try: - media = max(info["playback_list"], - key=lambda m: m["meta"]["quality_index"]) + media = max(info["playback_list"], key=lambda m: m["meta"]["quality_index"]) except Exception: - return {"url": (info.get("stream_url_hd") or - info.get("stream_url") or "")} + return {"url": (info.get("stream_url_hd") or info.get("stream_url") or "")} else: return media["play_info"].copy() def _status_by_id(self, status_id): - url = "{}/ajax/statuses/show?id={}".format(self.root, status_id) + url = f"{self.root}/ajax/statuses/show?id={status_id}" return self.request(url).json() def _user_id(self): if len(self.user) >= 10 and self.user.isdecimal(): return self.user[-10:] - else: - url = "{}/ajax/profile/info?{}={}".format( - self.root, - "screen_name" if self._prefix == "n" else "custom", - self.user) - return self.request(url).json()["data"]["user"]["idstr"] + url = "{}/ajax/profile/info?{}={}".format( + self.root, "screen_name" if self._prefix == "n" else "custom", self.user + ) + return self.request(url).json()["data"]["user"]["idstr"] def _pagination(self, endpoint, params): url = self.root + "/ajax" + endpoint @@ -177,8 +172,7 @@ def _pagination(self, endpoint, params): if not data.get("ok"): self.log.debug(response.content) if "since_id" not in params: # first iteration - raise exception.StopExtraction( - '"%s"', data.get("msg") or "unknown error") + raise exception.StopExtraction('"%s"', data.get("msg") or "unknown error") data = data["data"] statuses = data["list"] @@ -219,22 +213,21 @@ def _sina_visitor_system(self, response): data = { "cb": "gen_callback", "fp": '{"os":"1","browser":"Gecko109,0,0,0","fonts":"undefined",' - '"screenInfo":"1920*1080*24","plugins":""}', + '"screenInfo":"1920*1080*24","plugins":""}', } - page = Extractor.request( - self, passport_url, method="POST", headers=headers, data=data).text + page = Extractor.request(self, passport_url, method="POST", headers=headers, data=data).text data = util.json_loads(text.extr(page, "(", ");"))["data"] passport_url = "https://passport.weibo.com/visitor/visitor" params = { - "a" : "incarnate", - "t" : data["tid"], - "w" : "3" if data.get("new_tid") else "2", - "c" : "{:>03}".format(data.get("confidence") or 100), - "gc" : "", - "cb" : "cross_domain", - "from" : "weibo", + "a": "incarnate", + "t": data["tid"], + "w": "3" if data.get("new_tid") else "2", + "c": "{:>03}".format(data.get("confidence") or 100), + "gc": "", + "cb": "cross_domain", + "from": "weibo", "_rand": random.random(), } response = Extractor.request(self, passport_url, params=params) @@ -243,6 +236,7 @@ def _sina_visitor_system(self, response): class WeiboUserExtractor(WeiboExtractor): """Extractor for weibo user profiles""" + subcategory = "user" pattern = USER_PATTERN + r"(?:$|#)" example = "https://weibo.com/USER" @@ -253,18 +247,22 @@ class WeiboUserExtractor(WeiboExtractor): # pass def items(self): - base = "{}/u/{}?tabtype=".format(self.root, self._user_id()) - return self._dispatch_extractors(( - (WeiboHomeExtractor , base + "home"), - (WeiboFeedExtractor , base + "feed"), - (WeiboVideosExtractor , base + "video"), - (WeiboNewvideoExtractor, base + "newVideo"), - (WeiboAlbumExtractor , base + "album"), - ), ("feed",)) + base = f"{self.root}/u/{self._user_id()}?tabtype=" + return self._dispatch_extractors( + ( + (WeiboHomeExtractor, base + "home"), + (WeiboFeedExtractor, base + "feed"), + (WeiboVideosExtractor, base + "video"), + (WeiboNewvideoExtractor, base + "newVideo"), + (WeiboAlbumExtractor, base + "album"), + ), + ("feed",), + ) class WeiboHomeExtractor(WeiboExtractor): """Extractor for weibo 'home' listings""" + subcategory = "home" pattern = USER_PATTERN + r"\?tabtype=home" example = "https://weibo.com/USER?tabtype=home" @@ -277,6 +275,7 @@ def statuses(self): class WeiboFeedExtractor(WeiboExtractor): """Extractor for weibo user feeds""" + subcategory = "feed" pattern = USER_PATTERN + r"\?tabtype=feed" example = "https://weibo.com/USER?tabtype=feed" @@ -289,6 +288,7 @@ def statuses(self): class WeiboVideosExtractor(WeiboExtractor): """Extractor for weibo 'video' listings""" + subcategory = "videos" pattern = USER_PATTERN + r"\?tabtype=video" example = "https://weibo.com/USER?tabtype=video" @@ -303,6 +303,7 @@ def statuses(self): class WeiboNewvideoExtractor(WeiboExtractor): """Extractor for weibo 'newVideo' listings""" + subcategory = "newvideo" pattern = USER_PATTERN + r"\?tabtype=newVideo" example = "https://weibo.com/USER?tabtype=newVideo" @@ -315,6 +316,7 @@ def statuses(self): class WeiboArticleExtractor(WeiboExtractor): """Extractor for weibo 'article' listings""" + subcategory = "article" pattern = USER_PATTERN + r"\?tabtype=article" example = "https://weibo.com/USER?tabtype=article" @@ -327,6 +329,7 @@ def statuses(self): class WeiboAlbumExtractor(WeiboExtractor): """Extractor for weibo 'album' listings""" + subcategory = "album" pattern = USER_PATTERN + r"\?tabtype=album" example = "https://weibo.com/USER?tabtype=album" @@ -349,6 +352,7 @@ def statuses(self): class WeiboStatusExtractor(WeiboExtractor): """Extractor for images from a status on weibo.cn""" + subcategory = "status" pattern = BASE_PATTERN + r"/(detail|status|\d+)/(\w+)" example = "https://weibo.com/detail/12345" @@ -361,6 +365,6 @@ def statuses(self): return (status,) -@cache(maxage=365*86400) +@cache(maxage=365 * 86400) def _cookie_cache(): return None diff --git a/gallery_dl/extractor/wikiart.py b/gallery_dl/extractor/wikiart.py index 938c0485a..d0afe3c65 100644 --- a/gallery_dl/extractor/wikiart.py +++ b/gallery_dl/extractor/wikiart.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,14 +6,16 @@ """Extractors for https://www.wikiart.org/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?wikiart\.org/([a-z]+)" class WikiartExtractor(Extractor): """Base class for wikiart extractors""" + category = "wikiart" filename_fmt = "{id}_{title}.{extension}" archive_fmt = "{id}" @@ -66,6 +66,7 @@ def _pagination(self, url, extra_params=None, key="Paintings", stop=False): class WikiartArtistExtractor(WikiartExtractor): """Extractor for an artist's paintings on wikiart.org""" + subcategory = "artist" directory_fmt = ("{category}", "{artist[artistName]}") pattern = BASE_PATTERN + r"/(?!\w+-by-)([\w-]+)/?$" @@ -77,18 +78,18 @@ def __init__(self, match): self.artist = None def metadata(self): - url = "{}/{}/{}?json=2".format(self.root, self.lang, self.artist_name) + url = f"{self.root}/{self.lang}/{self.artist_name}?json=2" self.artist = self.request(url).json() return {"artist": self.artist} def paintings(self): - url = "{}/{}/{}/mode/all-paintings".format( - self.root, self.lang, self.artist_name) + url = f"{self.root}/{self.lang}/{self.artist_name}/mode/all-paintings" return self._pagination(url) class WikiartImageExtractor(WikiartArtistExtractor): """Extractor for individual paintings on wikiart.org""" + subcategory = "image" pattern = BASE_PATTERN + r"/(?!(?:paintings|artists)-by-)([\w-]+)/([\w-]+)" example = "https://www.wikiart.org/en/ARTIST/TITLE" @@ -102,14 +103,17 @@ def paintings(self): if not sep or not year.isdecimal(): title = self.title url = "{}/{}/Search/{} {}".format( - self.root, self.lang, - self.artist.get("artistName") or self.artist_name, title, + self.root, + self.lang, + self.artist.get("artistName") or self.artist_name, + title, ) return self._pagination(url, stop=True) class WikiartArtworksExtractor(WikiartExtractor): """Extractor for artwork collections on wikiart.org""" + subcategory = "artworks" directory_fmt = ("{category}", "Artworks by {group!c}", "{type}") pattern = BASE_PATTERN + r"/paintings-by-([\w-]+)/([\w-]+)" @@ -124,15 +128,15 @@ def metadata(self): return {"group": self.group, "type": self.type} def paintings(self): - url = "{}/{}/paintings-by-{}/{}".format( - self.root, self.lang, self.group, self.type) + url = f"{self.root}/{self.lang}/paintings-by-{self.group}/{self.type}" return self._pagination(url) class WikiartArtistsExtractor(WikiartExtractor): """Extractor for artist collections on wikiart.org""" + subcategory = "artists" - pattern = (BASE_PATTERN + r"/artists-by-([\w-]+)/([\w-]+)") + pattern = BASE_PATTERN + r"/artists-by-([\w-]+)/([\w-]+)" example = "https://www.wikiart.org/en/artists-by-GROUP/TYPE" def __init__(self, match): @@ -141,8 +145,7 @@ def __init__(self, match): self.type = match.group(3) def items(self): - url = "{}/{}/App/Search/Artists-by-{}".format( - self.root, self.lang, self.group) + url = f"{self.root}/{self.lang}/App/Search/Artists-by-{self.group}" params = {"json": "3", "searchterm": self.type} for artist in self._pagination(url, params, "Artists"): diff --git a/gallery_dl/extractor/wikifeet.py b/gallery_dl/extractor/wikifeet.py index d3586c017..ba9467e9e 100644 --- a/gallery_dl/extractor/wikifeet.py +++ b/gallery_dl/extractor/wikifeet.py @@ -1,23 +1,22 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://www.wikifeet.com/""" +from .. import text +from .. import util from .common import GalleryExtractor -from .. import text, util class WikifeetGalleryExtractor(GalleryExtractor): """Extractor for image galleries from wikifeet.com""" + category = "wikifeet" directory_fmt = ("{category}", "{celebrity}") filename_fmt = "{category}_{celeb}_{pid}.{extension}" archive_fmt = "{type}_{celeb}_{pid}" - pattern = (r"(?:https?://)(?:(?:www\.)?wikifeetx?|" - r"men\.wikifeet)\.com/([^/?#]+)") + pattern = r"(?:https?://)(?:(?:www\.)?wikifeetx?|" r"men\.wikifeet)\.com/([^/?#]+)" example = "https://www.wikifeet.com/CELEB" def __init__(self, match): @@ -31,14 +30,15 @@ def __init__(self, match): def metadata(self, page): extr = text.extract_from(page) return { - "celeb" : self.celeb, - "type" : self.type, - "rating" : text.parse_float(extr('"ratingValue": "', '"')), - "celebrity" : text.unescape(extr("times'>", "

    ")), - "shoesize" : text.remove_html(extr("Shoe Size:", "edit")), + "celeb": self.celeb, + "type": self.type, + "rating": text.parse_float(extr('"ratingValue": "', '"')), + "celebrity": text.unescape(extr("times'>", "

    ")), + "shoesize": text.remove_html(extr("Shoe Size:", "edit")), "birthplace": text.remove_html(extr("Birthplace:", "edit")), - "birthday" : text.parse_datetime(text.remove_html( - extr("Birth Date:", "edit")), "%Y-%m-%d"), + "birthday": text.parse_datetime( + text.remove_html(extr("Birth Date:", "edit")), "%Y-%m-%d" + ), } def images(self, page): @@ -52,14 +52,14 @@ def images(self, page): } ufmt = "https://pics.wikifeet.com/" + self.celeb + "-Feet-{}.jpg" return [ - (ufmt.format(data["pid"]), { - "pid" : data["pid"], - "width" : data["pw"], - "height": data["ph"], - "tags" : [ - tagmap[tag] - for tag in data["tags"] if tag in tagmap - ], - }) + ( + ufmt.format(data["pid"]), + { + "pid": data["pid"], + "width": data["pw"], + "height": data["ph"], + "tags": [tagmap[tag] for tag in data["tags"] if tag in tagmap], + }, + ) for data in util.json_loads(text.extr(page, "['gdata'] = ", ";")) ] diff --git a/gallery_dl/extractor/wikimedia.py b/gallery_dl/extractor/wikimedia.py index 4eae53750..8c85c34ee 100644 --- a/gallery_dl/extractor/wikimedia.py +++ b/gallery_dl/extractor/wikimedia.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022 Ailothaen # Copyright 2024 Mike Fährmann # @@ -9,13 +7,16 @@ """Extractors for Wikimedia sites""" -from .common import BaseExtractor, Message -from .. import text, exception +from .. import exception +from .. import text from ..cache import cache +from .common import BaseExtractor +from .common import Message class WikimediaExtractor(BaseExtractor): """Base class for wikimedia extractors""" + basecategory = "wikimedia" filename_fmt = "{filename} ({sha1[:8]}).{extension}" archive_fmt = "{sha1}" @@ -28,7 +29,8 @@ def __init__(self, match): self.category = self.root.split(".")[-2] elif self.category in ("fandom", "wikigg"): self.category = "{}-{}".format( - self.category, self.root.partition(".")[0].rpartition("/")[2]) + self.category, self.root.partition(".")[0].rpartition("/")[2] + ) self.per_page = self.config("limit", 50) @@ -42,7 +44,7 @@ def _init(self): else: self.api_url = None - @cache(maxage=36500*86400, keyarg=1) + @cache(maxage=36500 * 86400, keyarg=1) def _search_api_path(self, root): self.log.debug("Probing possible API endpoints") for path in ("/api.php", "/w/api.php", "/wiki/api.php"): @@ -55,18 +57,12 @@ def _search_api_path(self, root): @staticmethod def prepare(image): """Adjust the content of a image object""" - image["metadata"] = { - m["name"]: m["value"] - for m in image["metadata"] or ()} - image["commonmetadata"] = { - m["name"]: m["value"] - for m in image["commonmetadata"] or ()} + image["metadata"] = {m["name"]: m["value"] for m in image["metadata"] or ()} + image["commonmetadata"] = {m["name"]: m["value"] for m in image["commonmetadata"] or ()} filename = image["canonicaltitle"] - image["filename"], _, image["extension"] = \ - filename.partition(":")[2].rpartition(".") - image["date"] = text.parse_datetime( - image["timestamp"], "%Y-%m-%dT%H:%M:%SZ") + image["filename"], _, image["extension"] = filename.partition(":")[2].rpartition(".") + image["date"] = text.parse_datetime(image["timestamp"], "%Y-%m-%dT%H:%M:%SZ") def items(self): for info in self._pagination(self.params): @@ -126,65 +122,68 @@ def _pagination(self, params): params.update(continuation) -BASE_PATTERN = WikimediaExtractor.update({ - "wikimedia": { - "root": None, - "pattern": r"[a-z]{2,}\." - r"wik(?:i(?:pedia|quote|books|source|news|versity|data" - r"|voyage)|tionary)" - r"\.org", - "api-path": "/w/api.php", - }, - "wikispecies": { - "root": "https://species.wikimedia.org", - "pattern": r"species\.wikimedia\.org", - "api-path": "/w/api.php", - }, - "wikimediacommons": { - "root": "https://commons.wikimedia.org", - "pattern": r"commons\.wikimedia\.org", - "api-path": "/w/api.php", - }, - "mediawiki": { - "root": "https://www.mediawiki.org", - "pattern": r"(?:www\.)?mediawiki\.org", - "api-path": "/w/api.php", - }, - "fandom": { - "root": None, - "pattern": r"[\w-]+\.fandom\.com", - "api-path": "/api.php", - }, - "wikigg": { - "root": None, - "pattern": r"\w+\.wiki\.gg", - "api-path": "/api.php", - }, - "mariowiki": { - "root": "https://www.mariowiki.com", - "pattern": r"(?:www\.)?mariowiki\.com", - "api-path": "/api.php", - }, - "bulbapedia": { - "root": "https://bulbapedia.bulbagarden.net", - "pattern": r"(?:bulbapedia|archives)\.bulbagarden\.net", - "api-path": "/w/api.php", - }, - "pidgiwiki": { - "root": "https://www.pidgi.net", - "pattern": r"(?:www\.)?pidgi\.net", - "api-path": "/wiki/api.php", - }, - "azurlanewiki": { - "root": "https://azurlane.koumakan.jp", - "pattern": r"azurlane\.koumakan\.jp", - "api-path": "/w/api.php", - }, -}) +BASE_PATTERN = WikimediaExtractor.update( + { + "wikimedia": { + "root": None, + "pattern": r"[a-z]{2,}\." + r"wik(?:i(?:pedia|quote|books|source|news|versity|data" + r"|voyage)|tionary)" + r"\.org", + "api-path": "/w/api.php", + }, + "wikispecies": { + "root": "https://species.wikimedia.org", + "pattern": r"species\.wikimedia\.org", + "api-path": "/w/api.php", + }, + "wikimediacommons": { + "root": "https://commons.wikimedia.org", + "pattern": r"commons\.wikimedia\.org", + "api-path": "/w/api.php", + }, + "mediawiki": { + "root": "https://www.mediawiki.org", + "pattern": r"(?:www\.)?mediawiki\.org", + "api-path": "/w/api.php", + }, + "fandom": { + "root": None, + "pattern": r"[\w-]+\.fandom\.com", + "api-path": "/api.php", + }, + "wikigg": { + "root": None, + "pattern": r"\w+\.wiki\.gg", + "api-path": "/api.php", + }, + "mariowiki": { + "root": "https://www.mariowiki.com", + "pattern": r"(?:www\.)?mariowiki\.com", + "api-path": "/api.php", + }, + "bulbapedia": { + "root": "https://bulbapedia.bulbagarden.net", + "pattern": r"(?:bulbapedia|archives)\.bulbagarden\.net", + "api-path": "/w/api.php", + }, + "pidgiwiki": { + "root": "https://www.pidgi.net", + "pattern": r"(?:www\.)?pidgi\.net", + "api-path": "/wiki/api.php", + }, + "azurlanewiki": { + "root": "https://azurlane.koumakan.jp", + "pattern": r"azurlane\.koumakan\.jp", + "api-path": "/w/api.php", + }, + } +) class WikimediaArticleExtractor(WikimediaExtractor): """Extractor for wikimedia articles""" + subcategory = "article" directory_fmt = ("{category}", "{page}") pattern = BASE_PATTERN + r"/(?!static/)([^?#]+)" @@ -210,19 +209,19 @@ def __init__(self, match): if prefix == "category": self.params = { "generator": "categorymembers", - "gcmtitle" : path, - "gcmtype" : "file", - "gcmlimit" : self.per_page, + "gcmtitle": path, + "gcmtype": "file", + "gcmlimit": self.per_page, } elif prefix == "file": self.params = { - "titles" : path, + "titles": path, } else: self.params = { "generator": "images", - "gimlimit" : self.per_page, - "titles" : path, + "gimlimit": self.per_page, + "titles": path, } def prepare(self, image): @@ -232,6 +231,7 @@ def prepare(self, image): class WikimediaWikiExtractor(WikimediaExtractor): """Extractor for all files on a MediaWiki instance""" + subcategory = "wiki" pattern = BASE_PATTERN + r"/?$" example = "https://en.wikipedia.org/" @@ -241,7 +241,7 @@ def __init__(self, match): # ref: https://www.mediawiki.org/wiki/API:Allpages self.params = { - "generator" : "allpages", + "generator": "allpages", "gapnamespace": 6, # "File" namespace - "gaplimit" : self.per_page, + "gaplimit": self.per_page, } diff --git a/gallery_dl/extractor/xhamster.py b/gallery_dl/extractor/xhamster.py index 6dc9362a5..1f197713d 100644 --- a/gallery_dl/extractor/xhamster.py +++ b/gallery_dl/extractor/xhamster.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,15 +6,19 @@ """Extractors for https://xhamster.com/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message -BASE_PATTERN = (r"(?:https?://)?((?:[\w-]+\.)?xhamster" - r"(?:\d?\.(?:com|one|desi)|\.porncache\.net))") +BASE_PATTERN = ( + r"(?:https?://)?((?:[\w-]+\.)?xhamster" r"(?:\d?\.(?:com|one|desi)|\.porncache\.net))" +) class XhamsterExtractor(Extractor): """Base class for xhamster extractors""" + category = "xhamster" def __init__(self, match): @@ -26,9 +28,9 @@ def __init__(self, match): class XhamsterGalleryExtractor(XhamsterExtractor): """Extractor for image galleries on xhamster.com""" + subcategory = "gallery" - directory_fmt = ("{category}", "{user[name]}", - "{gallery[id]} {gallery[title]}") + directory_fmt = ("{category}", "{user[name]}", "{gallery[id]} {gallery[title]}") filename_fmt = "{num:>03}_{id}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"(/photos/gallery/[^/?#]+)" @@ -54,26 +56,24 @@ def metadata(self): imgs = self.data["photosGalleryModel"] return { - "user": - { - "id" : text.parse_int(user["id"]), - "url" : user["pageURL"], - "name" : user["name"], - "retired" : user["retired"], - "verified" : user["verified"], + "user": { + "id": text.parse_int(user["id"]), + "url": user["pageURL"], + "name": user["name"], + "retired": user["retired"], + "verified": user["verified"], "subscribers": user["subscribers"], }, - "gallery": - { - "id" : text.parse_int(imgs["id"]), - "tags" : [c["name"] for c in imgs["categories"]], - "date" : text.parse_timestamp(imgs["created"]), - "views" : text.parse_int(imgs["views"]), - "likes" : text.parse_int(imgs["rating"]["likes"]), - "dislikes" : text.parse_int(imgs["rating"]["dislikes"]), - "title" : text.unescape(imgs["title"]), + "gallery": { + "id": text.parse_int(imgs["id"]), + "tags": [c["name"] for c in imgs["categories"]], + "date": text.parse_timestamp(imgs["created"]), + "views": text.parse_int(imgs["views"]), + "likes": text.parse_int(imgs["rating"]["likes"]), + "dislikes": text.parse_int(imgs["rating"]["dislikes"]), + "title": text.unescape(imgs["title"]), "description": text.unescape(imgs["description"]), - "thumbnail" : imgs["thumbURL"], + "thumbnail": imgs["thumbURL"], }, "count": text.parse_int(imgs["quantity"]), } @@ -95,12 +95,12 @@ def images(self): def _data(self, url): page = self.request(url).text - return util.json_loads(text.extr( - page, "window.initials=", "").rstrip("\n\r;")) + return util.json_loads(text.extr(page, "window.initials=", "").rstrip("\n\r;")) class XhamsterUserExtractor(XhamsterExtractor): """Extractor for all galleries of an xhamster user""" + subcategory = "user" pattern = BASE_PATTERN + r"/users/([^/?#]+)(?:/photos)?/?(?:$|[?#])" example = "https://xhamster.com/users/USER/photos" @@ -110,7 +110,7 @@ def __init__(self, match): self.user = match.group(2) def items(self): - url = "{}/users/{}/photos".format(self.root, self.user) + url = f"{self.root}/users/{self.user}/photos" data = {"_extractor": XhamsterGalleryExtractor} while url: diff --git a/gallery_dl/extractor/xvideos.py b/gallery_dl/extractor/xvideos.py index da9d6b07a..139cd876e 100644 --- a/gallery_dl/extractor/xvideos.py +++ b/gallery_dl/extractor/xvideos.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,24 +6,27 @@ """Extractors for https://www.xvideos.com/""" -from .common import GalleryExtractor, Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import GalleryExtractor +from .common import Message -BASE_PATTERN = (r"(?:https?://)?(?:www\.)?xvideos\.com" - r"/(?:profiles|(?:amateur-|model-)?channels)") +BASE_PATTERN = r"(?:https?://)?(?:www\.)?xvideos\.com" r"/(?:profiles|(?:amateur-|model-)?channels)" -class XvideosBase(): +class XvideosBase: """Base class for xvideos extractors""" + category = "xvideos" root = "https://www.xvideos.com" class XvideosGalleryExtractor(XvideosBase, GalleryExtractor): """Extractor for user profile galleries on xvideos.com""" + subcategory = "gallery" - directory_fmt = ("{category}", "{user[name]}", - "{gallery[id]} {gallery[title]}") + directory_fmt = ("{category}", "{user[name]}", "{gallery[id]} {gallery[title]}") filename_fmt = "{category}_{gallery[id]}_{num:>03}.{extension}" archive_fmt = "{gallery[id]}_{num}" pattern = BASE_PATTERN + r"/([^/?#]+)/photos/(\d+)" @@ -33,41 +34,38 @@ class XvideosGalleryExtractor(XvideosBase, GalleryExtractor): def __init__(self, match): self.user, self.gallery_id = match.groups() - url = "{}/profiles/{}/photos/{}".format( - self.root, self.user, self.gallery_id) + url = f"{self.root}/profiles/{self.user}/photos/{self.gallery_id}" GalleryExtractor.__init__(self, match, url) def metadata(self, page): extr = text.extract_from(page) user = { - "id" : text.parse_int(extr('"id_user":', ',')), + "id": text.parse_int(extr('"id_user":', ",")), "display": extr('"display":"', '"'), - "sex" : extr('"sex":"', '"'), - "name" : self.user, + "sex": extr('"sex":"', '"'), + "name": self.user, } title = extr('"title":"', '"') - user["description"] = extr( - '', '').strip() - tags = extr('Tagged:', '<').strip() + user["description"] = extr('', "").strip() + tags = extr("Tagged:", "<").strip() return { "user": user, "gallery": { - "id" : text.parse_int(self.gallery_id), + "id": text.parse_int(self.gallery_id), "title": text.unescape(title), - "tags" : text.unescape(tags).split(", ") if tags else [], + "tags": text.unescape(tags).split(", ") if tags else [], }, } def images(self, page): results = [ (url, None) - for url in text.extract_iter( - page, 'Next"))["data"] + data = util.json_loads(text.extr(page, "xv.conf=", ";"))["data"] if not isinstance(data["galleries"], dict): return @@ -107,7 +104,7 @@ def items(self): galleries = [ { - "id" : text.parse_int(gid), + "id": text.parse_int(gid), "title": text.unescape(gdata["title"]), "count": gdata["nb_pics"], "_extractor": XvideosGalleryExtractor, @@ -117,6 +114,5 @@ def items(self): galleries.sort(key=lambda x: x["id"]) for gallery in galleries: - url = "https://www.xvideos.com/profiles/{}/photos/{}".format( - self.user, gallery["id"]) + url = "https://www.xvideos.com/profiles/{}/photos/{}".format(self.user, gallery["id"]) yield Message.Queue, url, gallery diff --git a/gallery_dl/extractor/ytdl.py b/gallery_dl/extractor/ytdl.py index 168845ea7..b7303f683 100644 --- a/gallery_dl/extractor/ytdl.py +++ b/gallery_dl/extractor/ytdl.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,16 @@ """Extractors for sites supported by youtube-dl""" -from .common import Extractor, Message -from .. import ytdl, config, exception +from .. import config +from .. import exception +from .. import ytdl +from .common import Extractor +from .common import Message class YoutubeDLExtractor(Extractor): """Generic extractor for youtube-dl supported URLs""" + category = "ytdl" directory_fmt = ("{category}", "{subcategory}") filename_fmt = "{title}-{id}.{extension}" @@ -23,8 +25,7 @@ class YoutubeDLExtractor(Extractor): def __init__(self, match): # import main youtube_dl module - ytdl_module = ytdl.import_module(config.get( - ("extractor", "ytdl"), "module")) + ytdl_module = ytdl.import_module(config.get(("extractor", "ytdl"), "module")) self.ytdl_module_name = ytdl_module.__name__ # find suitable youtube_dl extractor @@ -49,19 +50,19 @@ def __init__(self, match): def items(self): # import subcategory module ytdl_module = ytdl.import_module( - config.get(("extractor", "ytdl", self.subcategory), "module") or - self.ytdl_module_name) + config.get(("extractor", "ytdl", self.subcategory), "module") or self.ytdl_module_name + ) self.log.debug("Using %s", ytdl_module) # construct YoutubeDL object extr_opts = { - "extract_flat" : "in_playlist", + "extract_flat": "in_playlist", "force_generic_extractor": self.force_generic_extractor, } user_opts = { - "retries" : self._retries, - "socket_timeout" : self._timeout, - "nocheckcertificate" : not self._verify, + "retries": self._retries, + "socket_timeout": self._timeout, + "nocheckcertificate": not self._verify, } if self._proxies: @@ -72,8 +73,7 @@ def items(self): user_opts["username"], user_opts["password"] = username, password del username, password - ytdl_instance = ytdl.construct_YoutubeDL( - ytdl_module, self, user_opts, extr_opts) + ytdl_instance = ytdl.construct_YoutubeDL(ytdl_module, self, user_opts, extr_opts) # transfer cookies to ytdl cookies = self.cookies @@ -85,17 +85,15 @@ def items(self): # extract youtube_dl info_dict try: info_dict = ytdl_instance._YoutubeDL__extract_info( - self.ytdl_url, - ytdl_instance.get_info_extractor(self.ytdl_ie_key), - False, {}, True) + self.ytdl_url, ytdl_instance.get_info_extractor(self.ytdl_ie_key), False, {}, True + ) except ytdl_module.utils.YoutubeDLError: raise exception.StopExtraction("Failed to extract video data") if not info_dict: return elif "entries" in info_dict: - results = self._process_entries( - ytdl_module, ytdl_instance, info_dict["entries"]) + results = self._process_entries(ytdl_module, ytdl_instance, info_dict["entries"]) else: results = (info_dict,) @@ -105,9 +103,7 @@ def items(self): info_dict["_ytdl_info_dict"] = info_dict info_dict["_ytdl_instance"] = ytdl_instance - url = "ytdl:" + (info_dict.get("url") or - info_dict.get("webpage_url") or - self.ytdl_url) + url = "ytdl:" + (info_dict.get("url") or info_dict.get("webpage_url") or self.ytdl_url) yield Message.Directory, info_dict yield Message.Url, url, info_dict @@ -120,16 +116,15 @@ def _process_entries(self, ytdl_module, ytdl_instance, entries): if entry.get("_type") in ("url", "url_transparent"): try: entry = ytdl_instance.extract_info( - entry["url"], False, - ie_key=entry.get("ie_key")) + entry["url"], False, ie_key=entry.get("ie_key") + ) except ytdl_module.utils.YoutubeDLError: continue if not entry: continue if "entries" in entry: - yield from self._process_entries( - ytdl_module, ytdl_instance, entry["entries"]) + yield from self._process_entries(ytdl_module, ytdl_instance, entry["entries"]) else: yield entry diff --git a/gallery_dl/extractor/zerochan.py b/gallery_dl/extractor/zerochan.py index f9b1a7f4c..b6dae44f4 100644 --- a/gallery_dl/extractor/zerochan.py +++ b/gallery_dl/extractor/zerochan.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,17 +6,21 @@ """Extractors for https://www.zerochan.net/""" -from .booru import BooruExtractor -from ..cache import cache -from .. import text, util, exception import collections import re +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from .booru import BooruExtractor + BASE_PATTERN = r"(?:https?://)?(?:www\.)?zerochan\.net" class ZerochanExtractor(BooruExtractor): """Base class for zerochan extractors""" + category = "zerochan" root = "https://www.zerochan.net" filename_fmt = "{id}.{extension}" @@ -32,7 +34,7 @@ class ZerochanExtractor(BooruExtractor): def login(self): self._logged_in = True if self.cookies_check(self.cookies_names): - return + return None username, password = self._get_auth_info() if username: @@ -40,20 +42,20 @@ def login(self): self._logged_in = False - @cache(maxage=90*86400, keyarg=1) + @cache(maxage=90 * 86400, keyarg=1) def _login_impl(self, username, password): self.log.info("Logging in as %s", username) url = self.root + "/login" headers = { - "Origin" : self.root, - "Referer" : url, + "Origin": self.root, + "Referer": url, } data = { - "ref" : "/", - "name" : username, + "ref": "/", + "name": username, "password": password, - "login" : "Login", + "login": "Login", } response = self.request(url, method="POST", headers=headers, data=data) @@ -63,23 +65,21 @@ def _login_impl(self, username, password): return response.cookies def _parse_entry_html(self, entry_id): - url = "{}/{}".format(self.root, entry_id) + url = f"{self.root}/{entry_id}" extr = text.extract_from(self.request(url).text) data = { - "id" : text.parse_int(entry_id), - "author" : text.parse_unicode_escapes(extr(' "name": "', '"')), + "id": text.parse_int(entry_id), + "author": text.parse_unicode_escapes(extr(' "name": "', '"')), "file_url": extr('"contentUrl": "', '"'), - "date" : text.parse_datetime(extr('"datePublished": "', '"')), - "width" : text.parse_int(extr('"width": "', ' ')), - "height" : text.parse_int(extr('"height": "', ' ')), - "size" : text.parse_bytes(extr('"contentSize": "', 'B')), - "path" : text.split_html(extr( - 'class="breadcrumbs', ''))[2:], + "date": text.parse_datetime(extr('"datePublished": "', '"')), + "width": text.parse_int(extr('"width": "', " ")), + "height": text.parse_int(extr('"height": "', " ")), + "size": text.parse_bytes(extr('"contentSize": "', "B")), + "path": text.split_html(extr('class="breadcrumbs', ""))[2:], "uploader": extr('href="/user/', '"'), - "tags" : extr('
      '), - "source" : text.unescape(text.extr( - extr('id="source-url"', ''), 'href="', '"')), + "tags": extr('
        "), + "source": text.unescape(text.extr(extr('id="source-url"', ""), 'href="', '"')), } html = data["tags"] @@ -92,7 +92,7 @@ def _parse_entry_html(self, entry_id): return data def _parse_entry_api(self, entry_id): - url = "{}/{}?json".format(self.root, entry_id) + url = f"{self.root}/{entry_id}?json" text = self.request(url).text try: item = util.json_loads(text) @@ -103,14 +103,14 @@ def _parse_entry_api(self, entry_id): item = util.json_loads(text) data = { - "id" : item["id"], + "id": item["id"], "file_url": item["full"], - "width" : item["width"], - "height" : item["height"], - "size" : item["size"], - "name" : item["primary"], - "md5" : item["hash"], - "source" : item.get("source"), + "width": item["width"], + "height": item["height"], + "size": item["size"], + "name": item["primary"], + "md5": item["hash"], + "source": item.get("source"), } if not self._logged_in: @@ -146,8 +146,7 @@ def _init(self): self.session.headers["User-Agent"] = util.USERAGENT def metadata(self): - return {"search_tags": text.unquote( - self.search_tag.replace("+", " "))} + return {"search_tags": text.unquote(self.search_tag.replace("+", " "))} def posts_html(self): url = self.root + "/" + self.search_tag @@ -157,11 +156,11 @@ def posts_html(self): while True: page = self.request(url, params=params).text - thumbs = text.extr(page, '
          ") extr = text.extract_from(thumbs) while True: - post = extr('
        • ") if not post: break @@ -172,13 +171,12 @@ def posts_html(self): yield post else: yield { - "id" : extr('href="/', '"'), - "name" : extr('alt="', '"'), - "width" : extr('title="', '✕'), - "height": extr('', ' '), - "size" : extr('', 'b'), - "file_url": "https://static." + extr( - '", ""))[:-11], + "slug": self.slug, + "title": text.unescape(text.extr(page, "", ""))[:-11], } def images(self, page): @@ -48,10 +47,15 @@ def images_v2(self, page): results = [] while True: - for path in text.extract_iter( - page, ' class="picbox"> limit: obj = obj[:limit_hint] + hint return fmt(obj) + return apply_limit def _default_format(format_spec, default): def wrap(obj): return format(obj, format_spec) + return wrap -class Literal(): +class Literal: # __getattr__, __getattribute__, and __class_getitem__ # are all slower than regular __getitem__ diff --git a/gallery_dl/job.py b/gallery_dl/job.py index be244279b..5ccd87be6 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -1,37 +1,35 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import sys +import collections import errno -import logging import functools -import collections +import logging +import sys -from . import ( - extractor, - downloader, - postprocessor, - archive, - config, - exception, - formatter, - output, - path, - text, - util, - version, -) +from . import archive +from . import config +from . import downloader +from . import exception +from . import extractor +from . import formatter +from . import output +from . import path +from . import postprocessor +from . import text +from . import util +from . import version from .extractor.message import Message + stdout_write = output.stdout_write -class Job(): +class Job: """Base class for Job types""" + ulog = None _logger_adapter = output.LoggerAdapter @@ -49,15 +47,14 @@ def __init__(self, extr, parent=None): cfgpath = [] if parent: - if extr.category == parent.extractor.category or \ - extr.category in parent.parents: + if extr.category == parent.extractor.category or extr.category in parent.parents: parents = parent.parents else: parents = parent.parents + (parent.extractor.category,) if parents: for category in parents: - cat = "{}>{}".format(category, extr.category) + cat = f"{category}>{extr.category}" cfgpath.append((cat, extr.subcategory)) cfgpath.append((extr.category, extr.subcategory)) self.parents = parents @@ -78,16 +75,18 @@ def __init__(self, extr, parent=None): actions = extr.config("actions") if actions: - from .actions import LoggerAdapter, parse + from .actions import LoggerAdapter + from .actions import parse + self._logger_adapter = LoggerAdapter self._logger_actions = parse(actions) path_proxy = output.PathfmtProxy(self) self._logger_extra = { - "job" : self, + "job": self, "extractor": extr, - "path" : path_proxy, - "keywords" : output.KwdictProxy(self), + "path": path_proxy, + "keywords": output.KwdictProxy(self), } extr.log = self._wrap_logger(extr.log) extr.log.debug("Using %s for '%s'", extr.__class__.__name__, extr.url) @@ -106,8 +105,7 @@ def __init__(self, extr, parent=None): self.metadata_http = extr.config2("metadata-http", "http-metadata") metadata_path = extr.config2("metadata-path", "path-metadata") metadata_version = extr.config2("metadata-version", "version-metadata") - metadata_extractor = extr.config2( - "metadata-extractor", "extractor-metadata") + metadata_extractor = extr.config2("metadata-extractor", "extractor-metadata") if metadata_path: self.kwdict[metadata_path] = path_proxy @@ -115,9 +113,9 @@ def __init__(self, extr, parent=None): self.kwdict[metadata_extractor] = extr if metadata_version: self.kwdict[metadata_version] = { - "version" : version.__version__, - "is_executable" : util.EXECUTABLE, - "current_git_head": util.git_head() + "version": version.__version__, + "is_executable": util.EXECUTABLE, + "current_git_head": util.git_head(), } # user-supplied metadata kwdict = extr.config("keywords") @@ -142,8 +140,7 @@ def run(self): self._init() # sleep before extractor start - sleep = util.build_duration_func( - extractor.config("sleep-extractor")) + sleep = util.build_duration_func(extractor.config("sleep-extractor")) if sleep: extractor.sleep(sleep(), "extractor") @@ -161,16 +158,20 @@ def run(self): log.debug("", exc_info=exc) self.status |= exc.code except OSError as exc: - log.error("Unable to download data: %s: %s", - exc.__class__.__name__, exc) + log.error("Unable to download data: %s: %s", exc.__class__.__name__, exc) log.debug("", exc_info=exc) self.status |= 128 except Exception as exc: - log.error(("An unexpected error occurred: %s - %s. " - "Please run gallery-dl again with the --verbose flag, " - "copy its output and report this issue on " - "https://github.com/mikf/gallery-dl/issues ."), - exc.__class__.__name__, exc) + log.error( + ( + "An unexpected error occurred: %s - %s. " + "Please run gallery-dl again with the --verbose flag, " + "copy its output and report this issue on " + "https://github.com/mikf/gallery-dl/issues ." + ), + exc.__class__.__name__, + exc, + ) log.debug("", exc_info=exc) self.status |= 1 except BaseException: @@ -256,8 +257,7 @@ def _prepare_predicates(self, target, skip=True): try: pred = util.RangePredicate(prange) except ValueError as exc: - self.extractor.log.warning( - "invalid %s range: %s", target, exc) + self.extractor.log.warning("invalid %s range: %s", target, exc) else: if skip and pred.lower > 1 and not pfilter: pred.index += self.extractor.skip(pred.lower - 1) @@ -334,7 +334,6 @@ def handle_url(self, url, kwdict): # download from URL if not self.download(url): - # use fallback URLs if available/enabled fallback = kwdict.get("_fallback", ()) if self.fallback else () for num, url in enumerate(fallback, 1): @@ -345,8 +344,7 @@ def handle_url(self, url, kwdict): else: # download failed self.status |= 4 - self.log.error("Failed to download %s", - pathfmt.filename or url) + self.log.error("Failed to download %s", pathfmt.filename or url) if "error" in hooks: for callback in hooks["error"]: callback(pathfmt) @@ -436,10 +434,12 @@ def handle_queue(self, url, kwdict): if status: self.status |= status - if (status & 95 and # not FormatError or OSError - "_fallback" in kwdict and self.fallback): - fallback = kwdict["_fallback"] = \ - iter(kwdict["_fallback"]) + if ( + status & 95 # not FormatError or OSError + and "_fallback" in kwdict + and self.fallback + ): + fallback = kwdict["_fallback"] = iter(kwdict["_fallback"]) try: url = next(fallback) except StopIteration: @@ -478,10 +478,9 @@ def handle_finalize(self): if "finalize-error" in hooks: for callback in hooks["finalize-error"]: callback(pathfmt) - else: - if "finalize-success" in hooks: - for callback in hooks["finalize-success"]: - callback(pathfmt) + elif "finalize-success" in hooks: + for callback in hooks["finalize-success"]: + callback(pathfmt) def handle_skip(self): pathfmt = self.pathfmt @@ -551,23 +550,25 @@ def initialize(self, kwdict=None): archive_path = cfg("archive") if archive_path: archive_path = util.expand_path(archive_path) - archive_format = (cfg("archive-prefix", extr.category) + - cfg("archive-format", extr.archive_fmt)) - archive_pragma = (cfg("archive-pragma")) + archive_format = cfg("archive-prefix", extr.category) + cfg( + "archive-format", extr.archive_fmt + ) + archive_pragma = cfg("archive-pragma") try: if "{" in archive_path: - archive_path = formatter.parse( - archive_path).format_map(kwdict) + archive_path = formatter.parse(archive_path).format_map(kwdict) if cfg("archive-mode") == "memory": archive_cls = archive.DownloadArchiveMemory else: archive_cls = archive.DownloadArchive - self.archive = archive_cls( - archive_path, archive_format, archive_pragma) + self.archive = archive_cls(archive_path, archive_format, archive_pragma) except Exception as exc: extr.log.warning( "Failed to open download archive at '%s' (%s: %s)", - archive_path, exc.__class__.__name__, exc) + archive_path, + exc.__class__.__name__, + exc, + ) else: extr.log.debug("Using download archive '%s'", archive_path) @@ -578,8 +579,8 @@ def initialize(self, kwdict=None): else: if isinstance(events, str): events = events.split(",") - self._archive_write_file = ("file" in events) - self._archive_write_skip = ("skip" in events) + self._archive_write_file = "file" in events + self._archive_write_skip = "skip" in events skip = cfg("skip", True) if skip: @@ -634,8 +635,7 @@ def initialize(self, kwdict=None): else: clist = pp_dict.get("blacklist") negate = True - if clist and not util.build_extractor_filter( - clist, negate)(extr): + if clist and not util.build_extractor_filter(clist, negate)(extr): continue name = pp_dict.get("name") @@ -646,8 +646,9 @@ def initialize(self, kwdict=None): try: pp_obj = pp_cls(self, pp_dict) except Exception as exc: - pp_log.error("'%s' initialization failed: %s: %s", - name, exc.__class__.__name__, exc) + pp_log.error( + "'%s' initialization failed: %s: %s", name, exc.__class__.__name__, exc + ) pp_log.debug("", exc_info=exc) else: pp_list.append(pp_obj) @@ -664,8 +665,7 @@ def register_hooks(self, hooks, options=None): if expr: condition = util.compile_filter(expr) for hook, callback in hooks.items(): - self.hooks[hook].append(functools.partial( - self._call_hook, callback, condition)) + self.hooks[hook].append(functools.partial(self._call_hook, callback, condition)) else: for hook, callback in hooks.items(): self.hooks[hook].append(callback) @@ -715,19 +715,20 @@ def __init__(self, url, parent=None): self.private = config.get(("output",), "private") def handle_url(self, url, kwdict): - stdout_write("\nKeywords for filenames and --filter:\n" - "------------------------------------\n") + stdout_write( + "\nKeywords for filenames and --filter:\n" "------------------------------------\n" + ) if self.metadata_http and url.startswith("http"): kwdict[self.metadata_http] = util.extract_headers( - self.extractor.request(url, method="HEAD")) + self.extractor.request(url, method="HEAD") + ) self.print_kwdict(kwdict) raise exception.StopExtraction() def handle_directory(self, kwdict): - stdout_write("Keywords for directory names:\n" - "-----------------------------\n") + stdout_write("Keywords for directory names:\n" "-----------------------------\n") self.print_kwdict(kwdict) def handle_queue(self, url, kwdict): @@ -738,18 +739,16 @@ def handle_queue(self, url, kwdict): if not util.filter_dict(kwdict): self.extractor.log.info( "This extractor only spawns other extractors " - "and does not provide any metadata on its own.") + "and does not provide any metadata on its own." + ) if extr: - self.extractor.log.info( - "Showing results for '%s' instead:\n", url) + self.extractor.log.info("Showing results for '%s' instead:\n", url) KeywordJob(extr, self).run() else: - self.extractor.log.info( - "Try 'gallery-dl -K \"%s\"' instead.", url) + self.extractor.log.info("Try 'gallery-dl -K \"%s\"' instead.", url) else: - stdout_write("Keywords for --chapter-filter:\n" - "------------------------------\n") + stdout_write("Keywords for --chapter-filter:\n" "------------------------------\n") self.print_kwdict(kwdict) if extr or self.extractor.categorytransfer: stdout_write("\n") @@ -765,7 +764,7 @@ def print_kwdict(self, kwdict, prefix="", markers=None): if markers is None: markers = {markerid} elif markerid in markers: - write("{}\n \n".format(prefix[:-2])) + write(f"{prefix[:-2]}\n \n") return # ignore circular reference else: markers.add(markerid) @@ -784,20 +783,21 @@ def print_kwdict(self, kwdict, prefix="", markers=None): elif isinstance(value[0], dict): self.print_kwdict(value[0], key + "[N]['", markers) else: - fmt = (" {:>%s} {}\n" % len(str(len(value)))).format + fmt = (" {:>%s} {}\n" % len(str(len(value)))).format # noqa: UP031 write(key + "[N]\n") for idx, val in enumerate(value, 0): write(fmt(idx, val)) else: # string or number - write("{}\n {}\n".format(key, value)) + write(f"{key}\n {value}\n") markers.remove(markerid) class UrlJob(Job): """Print download urls""" + maxdepth = 1 def __init__(self, url, parent=None, depth=1): @@ -819,10 +819,7 @@ def handle_url_fallback(url, kwdict): def handle_queue(self, url, kwdict): cls = kwdict.get("_extractor") - if cls: - extr = cls.from_url(url) - else: - extr = extractor.find(url) + extr = cls.from_url(url) if cls else extractor.find(url) if extr: self.status |= self.__class__(extr, self, self.depth + 1).run() @@ -839,8 +836,12 @@ def run(self): pc = self._print_config if ex.basecategory: - pm("Category / Subcategory / Basecategory", - ex.category, ex.subcategory, ex.basecategory) + pm( + "Category / Subcategory / Basecategory", + ex.category, + ex.subcategory, + ex.basecategory, + ) else: pm("Category / Subcategory", ex.category, ex.subcategory) @@ -852,28 +853,26 @@ def run(self): return 0 def _print_multi(self, title, *values): - stdout_write("{}\n {}\n\n".format( - title, " / ".join(map(util.json_dumps, values)))) + stdout_write("{}\n {}\n\n".format(title, " / ".join(map(util.json_dumps, values)))) def _print_config(self, title, optname, value): optval = self.extractor.config(optname, util.SENTINEL) + if optval is not util.SENTINEL: stdout_write( - "{} (custom):\n {}\n{} (default):\n {}\n\n".format( - title, util.json_dumps(optval), - title, util.json_dumps(value))) + f"{title} (custom):\n {util.json_dumps(optval)}\n{title} (default):\n " + f"{util.json_dumps(value)}\n\n" + ) elif value: - stdout_write( - "{} (default):\n {}\n\n".format( - title, util.json_dumps(value))) + stdout_write(f"{title} (default):\n {util.json_dumps(value)}\n\n") class DataJob(Job): """Collect extractor results and dump them""" + resolve = False - def __init__(self, url, parent=None, file=sys.stdout, ensure_ascii=True, - resolve=False): + def __init__(self, url, parent=None, file=sys.stdout, ensure_ascii=True, resolve=False): Job.__init__(self, url, parent) self.file = file self.data = [] @@ -890,8 +889,7 @@ def run(self): self._init() extractor = self.extractor - sleep = util.build_duration_func( - extractor.config("sleep-extractor")) + sleep = util.build_duration_func(extractor.config("sleep-extractor")) if sleep: extractor.sleep(sleep(), "extractor") @@ -932,14 +930,11 @@ def handle_queue(self, url, kwdict): def handle_queue_resolve(self, url, kwdict): cls = kwdict.get("_extractor") - if cls: - extr = cls.from_url(url) - else: - extr = extractor.find(url) + extr = cls.from_url(url) if cls else extractor.find(url) if not extr: return self.data.append((Message.Queue, url, self.filter(kwdict))) - job = self.__class__(extr, self, None, self.ascii, self.resolve-1) + job = self.__class__(extr, self, None, self.ascii, self.resolve - 1) job.data = self.data job.run() diff --git a/gallery_dl/oauth.py b/gallery_dl/oauth.py index 8508ee1e0..8aab27707 100644 --- a/gallery_dl/oauth.py +++ b/gallery_dl/oauth.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2020 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,12 @@ """OAuth helper functions and classes""" +import binascii +import hashlib import hmac -import time import random import string -import hashlib -import binascii +import time import urllib.parse import requests @@ -41,13 +39,13 @@ def concat(*args): class OAuth1Session(requests.Session): """Extension to requests.Session to support OAuth 1.0""" - def __init__(self, consumer_key, consumer_secret, - token=None, token_secret=None): - + def __init__(self, consumer_key, consumer_secret, token=None, token_secret=None): requests.Session.__init__(self) self.auth = OAuth1Client( - consumer_key, consumer_secret, - token, token_secret, + consumer_key, + consumer_secret, + token, + token_secret, ) def rebuild_auth(self, prepared_request, response): @@ -59,9 +57,7 @@ def rebuild_auth(self, prepared_request, response): class OAuth1Client(requests.auth.AuthBase): """OAuth1.0a authentication""" - def __init__(self, consumer_key, consumer_secret, - token=None, token_secret=None): - + def __init__(self, consumer_key, consumer_secret, token=None, token_secret=None): self.consumer_key = consumer_key self.consumer_secret = consumer_secret self.token = token @@ -82,7 +78,8 @@ def __call__(self, request): oauth_params.append(("oauth_signature", signature)) request.headers["Authorization"] = "OAuth " + ",".join( - key + '="' + value + '"' for key, value in oauth_params) + key + '="' + value + '"' for key, value in oauth_params + ) return request @@ -103,8 +100,9 @@ def generate_signature(self, request, params): return quote(binascii.b2a_base64(signature)[:-1].decode()) -class OAuth1API(): +class OAuth1API: """Base class for OAuth1.0 based API interfaces""" + API_KEY = None API_SECRET = None @@ -124,8 +122,7 @@ def __init__(self, extractor): if api_key and api_secret and token and token_secret: self.log.debug("Using %s OAuth1.0 authentication", key_type) - self.session = OAuth1Session( - api_key, api_secret, token, token_secret) + self.session = OAuth1Session(api_key, api_secret, token, token_secret) self.api_key = None else: self.log.debug("Using %s api_key authentication", key_type) @@ -138,6 +135,6 @@ def request(self, url, **kwargs): return self.extractor.request(url, **kwargs) -@cache(maxage=36500*86400, keyarg=0) +@cache(maxage=36500 * 86400, keyarg=0) def _token_cache(key): return None, None diff --git a/gallery_dl/option.py b/gallery_dl/option.py index a3f78e5b8..7396ed61f 100644 --- a/gallery_dl/option.py +++ b/gallery_dl/option.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -12,17 +10,23 @@ import logging import os.path import sys -from . import job, util, version +from contextlib import suppress + +from . import job +from . import util +from . import version class ConfigAction(argparse.Action): """Set argparse results as config values""" + def __call__(self, parser, namespace, values, option_string=None): namespace.options.append(((), self.dest, values)) class ConfigConstAction(argparse.Action): """Set argparse const values as config values""" + def __call__(self, parser, namespace, values, option_string=None): namespace.options.append(((), self.dest, self.const)) @@ -38,15 +42,19 @@ def __call__(self, parser, namespace, values, option_string=None): class DeprecatedConfigConstAction(argparse.Action): """Set argparse const values as config values + deprecation warning""" + def __call__(self, parser, namespace, values, option_string=None): sys.stderr.write( "warning: {} is deprecated. Use {} instead.\n".format( - "/".join(self.option_strings), self.choices)) + "/".join(self.option_strings), self.choices + ) + ) namespace.options.append(((), self.dest, self.const)) class ConfigParseAction(argparse.Action): """Parse KEY=VALUE config options""" + def __call__(self, parser, namespace, values, option_string=None): key, value = _parse_option(values) key = key.split(".") # splitting an empty string becomes [""] @@ -55,6 +63,7 @@ def __call__(self, parser, namespace, values, option_string=None): class PPParseAction(argparse.Action): """Parse KEY=VALUE post processor options""" + def __call__(self, parser, namespace, values, option_string=None): key, value = _parse_option(values) namespace.options_pp[key] = value @@ -62,89 +71,100 @@ def __call__(self, parser, namespace, values, option_string=None): class InputfileAction(argparse.Action): """Collect input files""" + def __call__(self, parser, namespace, value, option_string=None): namespace.input_files.append((value, self.const)) class MtimeAction(argparse.Action): """Configure mtime post processors""" + def __call__(self, parser, namespace, value, option_string=None): - namespace.postprocessors.append({ - "name": "mtime", - "value": "{" + (self.const or value) + "}", - }) + namespace.postprocessors.append( + { + "name": "mtime", + "value": "{" + (self.const or value) + "}", + } + ) class RenameAction(argparse.Action): """Configure rename post processors""" + def __call__(self, parser, namespace, value, option_string=None): if self.const: - namespace.postprocessors.append({ - "name": "rename", - "to" : value, - }) + namespace.postprocessors.append( + { + "name": "rename", + "to": value, + } + ) else: - namespace.postprocessors.append({ - "name": "rename", - "from": value, - }) + namespace.postprocessors.append( + { + "name": "rename", + "from": value, + } + ) class UgoiraAction(argparse.Action): """Configure ugoira post processors""" + def __call__(self, parser, namespace, value, option_string=None): - if self.const: - value = self.const - else: - value = value.strip().lower() + value = self.const if self.const else value.strip().lower() if value in ("webm", "vp9"): pp = { - "extension" : "webm", - "ffmpeg-args" : ("-c:v", "libvpx-vp9", - "-crf", "12", - "-b:v", "0", "-an"), + "extension": "webm", + "ffmpeg-args": ("-c:v", "libvpx-vp9", "-crf", "12", "-b:v", "0", "-an"), } elif value == "vp9-lossless": pp = { - "extension" : "webm", - "ffmpeg-args" : ("-c:v", "libvpx-vp9", - "-lossless", "1", - "-pix_fmt", "yuv420p", "-an"), + "extension": "webm", + "ffmpeg-args": ( + "-c:v", + "libvpx-vp9", + "-lossless", + "1", + "-pix_fmt", + "yuv420p", + "-an", + ), } elif value == "vp8": pp = { - "extension" : "webm", - "ffmpeg-args" : ("-c:v", "libvpx", - "-crf", "4", - "-b:v", "5000k", "-an"), + "extension": "webm", + "ffmpeg-args": ("-c:v", "libvpx", "-crf", "4", "-b:v", "5000k", "-an"), } elif value == "mp4": pp = { - "extension" : "mp4", - "ffmpeg-args" : ("-c:v", "libx264", "-an", "-b:v", "5M"), + "extension": "mp4", + "ffmpeg-args": ("-c:v", "libx264", "-an", "-b:v", "5M"), "libx264-prevent-odd": True, } elif value == "gif": pp = { - "extension" : "gif", - "ffmpeg-args" : ("-filter_complex", "[0:v] split [a][b];" - "[a] palettegen [p];[b][p] paletteuse"), + "extension": "gif", + "ffmpeg-args": ( + "-filter_complex", + "[0:v] split [a][b];" "[a] palettegen [p];[b][p] paletteuse", + ), "repeat-last-frame": False, } elif value == "mkv" or value == "copy": pp = { - "extension" : "mkv", - "ffmpeg-args" : ("-c:v", "copy"), + "extension": "mkv", + "ffmpeg-args": ("-c:v", "copy"), "repeat-last-frame": False, } elif value == "zip" or value == "archive": pp = { - "mode" : "archive", + "mode": "archive", } namespace.options.append(((), "ugoira", "original")) else: - parser.error("Unsupported Ugoira format '{}'".format(value)) + parser.error(f"Unsupported Ugoira format '{value}'") pp["name"] = "ugoira" pp["whitelist"] = ("pixiv", "danbooru") @@ -170,34 +190,46 @@ def __call__(self, parser, namespace, value, option_string=None): event = ("prepare",) else: event = event.strip().lower() - if event not in {"init", "file", "after", "skip", "error", - "prepare", "prepare-after", "post", "post-after", - "finalize", "finalize-success", "finalize-error"}: + if event not in { + "init", + "file", + "after", + "skip", + "error", + "prepare", + "prepare-after", + "post", + "post-after", + "finalize", + "finalize-success", + "finalize-error", + }: format_string = value event = ("prepare",) if not format_string: return - if "{" not in format_string and \ - " " not in format_string and \ - format_string[0] != "\f": + if "{" not in format_string and " " not in format_string and format_string[0] != "\f": format_string = "{" + format_string + "}" if format_string[-1] != "\n": format_string += "\n" - namespace.postprocessors.append({ - "name" : "metadata", - "event" : event, - "filename" : filename, - "base-directory": base or ".", - "content-format": format_string, - "open" : mode, - }) + namespace.postprocessors.append( + { + "name": "metadata", + "event": event, + "filename": filename, + "base-directory": base or ".", + "content-format": format_string, + "open": mode, + } + ) class Formatter(argparse.HelpFormatter): """Custom HelpFormatter class to customize help output""" + def __init__(self, prog): argparse.HelpFormatter.__init__(self, prog, max_help_position=30) @@ -211,10 +243,9 @@ def _format_action_invocation(self, action, join=", ".join): def _parse_option(opt): key, _, value = opt.partition("=") - try: + with suppress(ValueError): value = util.json_loads(value) - except ValueError: - pass + return key, value @@ -228,585 +259,829 @@ def build_parser(): general = parser.add_argument_group("General Options") general.add_argument( - "-h", "--help", + "-h", + "--help", action="help", help="Print this help message and exit", ) general.add_argument( "--version", - action="version", version=version.__version__, + action="version", + version=version.__version__, help="Print program version and exit", ) general.add_argument( - "-f", "--filename", - dest="filename", metavar="FORMAT", - help=("Filename format string for downloaded files " - "('/O' for \"original\" filenames)"), + "-f", + "--filename", + dest="filename", + metavar="FORMAT", + help=("Filename format string for downloaded files " "('/O' for \"original\" filenames)"), ) general.add_argument( - "-d", "--destination", - dest="base-directory", metavar="PATH", action=ConfigAction, + "-d", + "--destination", + dest="base-directory", + metavar="PATH", + action=ConfigAction, help="Target location for file downloads", ) general.add_argument( - "-D", "--directory", - dest="directory", metavar="PATH", + "-D", + "--directory", + dest="directory", + metavar="PATH", help="Exact location for file downloads", ) general.add_argument( - "-X", "--extractors", - dest="extractor_sources", metavar="PATH", action="append", + "-X", + "--extractors", + dest="extractor_sources", + metavar="PATH", + action="append", help="Load external extractors from PATH", ) general.add_argument( "--user-agent", - dest="user-agent", metavar="UA", action=ConfigAction, + dest="user-agent", + metavar="UA", + action=ConfigAction, help="User-Agent request header", ) general.add_argument( "--clear-cache", - dest="clear_cache", metavar="MODULE", - help="Delete cached login sessions, cookies, etc. for MODULE " - "(ALL to delete everything)", + dest="clear_cache", + metavar="MODULE", + help="Delete cached login sessions, cookies, etc. for MODULE " "(ALL to delete everything)", ) update = parser.add_argument_group("Update Options") if util.EXECUTABLE: update.add_argument( - "-U", "--update", - dest="update", action="store_const", const="latest", + "-U", + "--update", + dest="update", + action="store_const", + const="latest", help="Update to the latest version", ) update.add_argument( "--update-to", - dest="update", metavar="CHANNEL[@TAG]", - help=("Switch to a dfferent release channel (stable or dev) " - "or upgrade/downgrade to a specific version"), + dest="update", + metavar="CHANNEL[@TAG]", + help=( + "Switch to a dfferent release channel (stable or dev) " + "or upgrade/downgrade to a specific version" + ), ) update.add_argument( "--update-check", - dest="update", action="store_const", const="check", + dest="update", + action="store_const", + const="check", help="Check if a newer version is available", ) else: update.add_argument( - "-U", "--update-check", - dest="update", action="store_const", const="check", + "-U", + "--update-check", + dest="update", + action="store_const", + const="check", help="Check if a newer version is available", ) input = parser.add_argument_group("Input Options") input.add_argument( "urls", - metavar="URL", nargs="*", + metavar="URL", + nargs="*", help=argparse.SUPPRESS, ) input.add_argument( - "-i", "--input-file", - dest="input_files", metavar="FILE", action=InputfileAction, const=None, + "-i", + "--input-file", + dest="input_files", + metavar="FILE", + action=InputfileAction, + const=None, default=[], - help=("Download URLs found in FILE ('-' for stdin). " - "More than one --input-file can be specified"), + help=( + "Download URLs found in FILE ('-' for stdin). " + "More than one --input-file can be specified" + ), ) input.add_argument( - "-I", "--input-file-comment", - dest="input_files", metavar="FILE", action=InputfileAction, const="c", - help=("Download URLs found in FILE. " - "Comment them out after they were downloaded successfully."), + "-I", + "--input-file-comment", + dest="input_files", + metavar="FILE", + action=InputfileAction, + const="c", + help=( + "Download URLs found in FILE. " + "Comment them out after they were downloaded successfully." + ), ) input.add_argument( - "-x", "--input-file-delete", - dest="input_files", metavar="FILE", action=InputfileAction, const="d", - help=("Download URLs found in FILE. " - "Delete them after they were downloaded successfully."), + "-x", + "--input-file-delete", + dest="input_files", + metavar="FILE", + action=InputfileAction, + const="d", + help=( + "Download URLs found in FILE. " "Delete them after they were downloaded successfully." + ), ) input.add_argument( "--no-input", - dest="input", nargs=0, action=ConfigConstAction, const=False, + dest="input", + nargs=0, + action=ConfigConstAction, + const=False, help=("Do not prompt for passwords/tokens"), ) output = parser.add_argument_group("Output Options") output.add_argument( - "-q", "--quiet", - dest="loglevel", default=logging.INFO, - action="store_const", const=logging.ERROR, + "-q", + "--quiet", + dest="loglevel", + default=logging.INFO, + action="store_const", + const=logging.ERROR, help="Activate quiet mode", ) output.add_argument( - "-w", "--warning", + "-w", + "--warning", dest="loglevel", - action="store_const", const=logging.WARNING, + action="store_const", + const=logging.WARNING, help="Print only warnings and errors", ) output.add_argument( - "-v", "--verbose", + "-v", + "--verbose", dest="loglevel", - action="store_const", const=logging.DEBUG, + action="store_const", + const=logging.DEBUG, help="Print various debugging information", ) output.add_argument( - "-g", "--get-urls", - dest="list_urls", action="count", + "-g", + "--get-urls", + dest="list_urls", + action="count", help="Print URLs instead of downloading", ) output.add_argument( - "-G", "--resolve-urls", - dest="list_urls", action="store_const", const=128, + "-G", + "--resolve-urls", + dest="list_urls", + action="store_const", + const=128, help="Print URLs instead of downloading; resolve intermediary URLs", ) output.add_argument( - "-j", "--dump-json", - dest="dump_json", action="count", + "-j", + "--dump-json", + dest="dump_json", + action="count", help="Print JSON information", ) output.add_argument( - "-J", "--resolve-json", - dest="dump_json", action="store_const", const=128, + "-J", + "--resolve-json", + dest="dump_json", + action="store_const", + const=128, help="Print JSON information; resolve intermediary URLs", ) output.add_argument( - "-s", "--simulate", - dest="jobtype", action="store_const", const=job.SimulationJob, + "-s", + "--simulate", + dest="jobtype", + action="store_const", + const=job.SimulationJob, help="Simulate data extraction; do not download anything", ) output.add_argument( - "-E", "--extractor-info", - dest="jobtype", action="store_const", const=job.InfoJob, + "-E", + "--extractor-info", + dest="jobtype", + action="store_const", + const=job.InfoJob, help="Print extractor defaults and settings", ) output.add_argument( - "-K", "--list-keywords", - dest="jobtype", action="store_const", const=job.KeywordJob, - help=("Print a list of available keywords and example values " - "for the given URLs"), + "-K", + "--list-keywords", + dest="jobtype", + action="store_const", + const=job.KeywordJob, + help=("Print a list of available keywords and example values " "for the given URLs"), ) output.add_argument( - "-e", "--error-file", - dest="errorfile", metavar="FILE", action=ConfigAction, + "-e", + "--error-file", + dest="errorfile", + metavar="FILE", + action=ConfigAction, help="Add input URLs which returned an error to FILE", ) output.add_argument( - "-N", "--print", - dest="postprocessors", metavar="[EVENT:]FORMAT", - action=PrintAction, const="-", default=[], - help=("Write FORMAT during EVENT (default 'prepare') to standard " - "output. Examples: 'id' or 'post:{md5[:8]}'"), + "-N", + "--print", + dest="postprocessors", + metavar="[EVENT:]FORMAT", + action=PrintAction, + const="-", + default=[], + help=( + "Write FORMAT during EVENT (default 'prepare') to standard " + "output. Examples: 'id' or 'post:{md5[:8]}'" + ), ) output.add_argument( "--print-to-file", - dest="postprocessors", metavar="[EVENT:]FORMAT FILE", - action=PrintAction, nargs=2, + dest="postprocessors", + metavar="[EVENT:]FORMAT FILE", + action=PrintAction, + nargs=2, help="Append FORMAT during EVENT to FILE", ) output.add_argument( "--list-modules", - dest="list_modules", action="store_true", + dest="list_modules", + action="store_true", help="Print a list of available extractor modules", ) output.add_argument( "--list-extractors", - dest="list_extractors", metavar="CATEGORIES", nargs="*", - help=("Print a list of extractor classes " - "with description, (sub)category and example URL"), + dest="list_extractors", + metavar="CATEGORIES", + nargs="*", + help=( + "Print a list of extractor classes " "with description, (sub)category and example URL" + ), ) output.add_argument( "--write-log", - dest="logfile", metavar="FILE", action=ConfigAction, + dest="logfile", + metavar="FILE", + action=ConfigAction, help="Write logging output to FILE", ) output.add_argument( "--write-unsupported", - dest="unsupportedfile", metavar="FILE", action=ConfigAction, - help=("Write URLs, which get emitted by other extractors but cannot " - "be handled, to FILE"), + dest="unsupportedfile", + metavar="FILE", + action=ConfigAction, + help=( + "Write URLs, which get emitted by other extractors but cannot " "be handled, to FILE" + ), ) output.add_argument( "--write-pages", - dest="write-pages", nargs=0, action=ConfigConstAction, const=True, - help=("Write downloaded intermediary pages to files " - "in the current directory to debug problems"), + dest="write-pages", + nargs=0, + action=ConfigConstAction, + const=True, + help=( + "Write downloaded intermediary pages to files " + "in the current directory to debug problems" + ), ) output.add_argument( "--print-traffic", - dest="print_traffic", action="store_true", + dest="print_traffic", + action="store_true", help=("Display sent and read HTTP traffic"), ) output.add_argument( "--no-colors", - dest="colors", action="store_false", + dest="colors", + action="store_false", help=("Do not emit ANSI color codes in output"), ) networking = parser.add_argument_group("Networking Options") networking.add_argument( - "-R", "--retries", - dest="retries", metavar="N", type=int, action=ConfigAction, - help=("Maximum number of retries for failed HTTP requests " - "or -1 for infinite retries (default: 4)"), + "-R", + "--retries", + dest="retries", + metavar="N", + type=int, + action=ConfigAction, + help=( + "Maximum number of retries for failed HTTP requests " + "or -1 for infinite retries (default: 4)" + ), ) networking.add_argument( "--http-timeout", - dest="timeout", metavar="SECONDS", type=float, action=ConfigAction, + dest="timeout", + metavar="SECONDS", + type=float, + action=ConfigAction, help="Timeout for HTTP connections (default: 30.0)", ) networking.add_argument( "--proxy", - dest="proxy", metavar="URL", action=ConfigAction, + dest="proxy", + metavar="URL", + action=ConfigAction, help="Use the specified proxy", ) networking.add_argument( "--source-address", - dest="source-address", metavar="IP", action=ConfigAction, + dest="source-address", + metavar="IP", + action=ConfigAction, help="Client-side IP address to bind to", ) networking.add_argument( - "-4", "--force-ipv4", - dest="source-address", nargs=0, action=ConfigConstAction, + "-4", + "--force-ipv4", + dest="source-address", + nargs=0, + action=ConfigConstAction, const="0.0.0.0", help="Make all connections via IPv4", ) networking.add_argument( - "-6", "--force-ipv6", - dest="source-address", nargs=0, action=ConfigConstAction, const="::", + "-6", + "--force-ipv6", + dest="source-address", + nargs=0, + action=ConfigConstAction, + const="::", help="Make all connections via IPv6", ) networking.add_argument( "--no-check-certificate", - dest="verify", nargs=0, action=ConfigConstAction, const=False, + dest="verify", + nargs=0, + action=ConfigConstAction, + const=False, help="Disable HTTPS certificate validation", ) downloader = parser.add_argument_group("Downloader Options") downloader.add_argument( - "-r", "--limit-rate", - dest="rate", metavar="RATE", action=ConfigAction, + "-r", + "--limit-rate", + dest="rate", + metavar="RATE", + action=ConfigAction, help="Maximum download rate (e.g. 500k or 2.5M)", ) downloader.add_argument( "--chunk-size", - dest="chunk-size", metavar="SIZE", action=ConfigAction, + dest="chunk-size", + metavar="SIZE", + action=ConfigAction, help="Size of in-memory data chunks (default: 32k)", ) downloader.add_argument( "--sleep", - dest="sleep", metavar="SECONDS", action=ConfigAction, - help=("Number of seconds to wait before each download. " - "This can be either a constant value or a range " - "(e.g. 2.7 or 2.0-3.5)"), + dest="sleep", + metavar="SECONDS", + action=ConfigAction, + help=( + "Number of seconds to wait before each download. " + "This can be either a constant value or a range " + "(e.g. 2.7 or 2.0-3.5)" + ), ) downloader.add_argument( "--sleep-request", - dest="sleep-request", metavar="SECONDS", action=ConfigAction, - help=("Number of seconds to wait between HTTP requests " - "during data extraction"), + dest="sleep-request", + metavar="SECONDS", + action=ConfigAction, + help=("Number of seconds to wait between HTTP requests " "during data extraction"), ) downloader.add_argument( "--sleep-extractor", - dest="sleep-extractor", metavar="SECONDS", action=ConfigAction, - help=("Number of seconds to wait before starting data extraction " - "for an input URL"), + dest="sleep-extractor", + metavar="SECONDS", + action=ConfigAction, + help=("Number of seconds to wait before starting data extraction " "for an input URL"), ) downloader.add_argument( "--no-part", - dest="part", nargs=0, action=ConfigConstAction, const=False, + dest="part", + nargs=0, + action=ConfigConstAction, + const=False, help="Do not use .part files", ) downloader.add_argument( "--no-skip", - dest="skip", nargs=0, action=ConfigConstAction, const=False, + dest="skip", + nargs=0, + action=ConfigConstAction, + const=False, help="Do not skip downloads; overwrite existing files", ) downloader.add_argument( "--no-mtime", - dest="mtime", nargs=0, action=ConfigConstAction, const=False, - help=("Do not set file modification times according to " - "Last-Modified HTTP response headers") + dest="mtime", + nargs=0, + action=ConfigConstAction, + const=False, + help=( + "Do not set file modification times according to " "Last-Modified HTTP response headers" + ), ) downloader.add_argument( "--no-download", - dest="download", nargs=0, action=ConfigConstAction, const=False, - help=("Do not download any files") + dest="download", + nargs=0, + action=ConfigConstAction, + const=False, + help=("Do not download any files"), ) configuration = parser.add_argument_group("Configuration Options") configuration.add_argument( - "-o", "--option", - dest="options", metavar="KEY=VALUE", - action=ConfigParseAction, default=[], - help=("Additional options. " - "Example: -o browser=firefox") , + "-o", + "--option", + dest="options", + metavar="KEY=VALUE", + action=ConfigParseAction, + default=[], + help=("Additional options. " "Example: -o browser=firefox"), ) configuration.add_argument( - "-c", "--config", - dest="configs_json", metavar="FILE", action="append", + "-c", + "--config", + dest="configs_json", + metavar="FILE", + action="append", help="Additional configuration files", ) configuration.add_argument( "--config-yaml", - dest="configs_yaml", metavar="FILE", action="append", + dest="configs_yaml", + metavar="FILE", + action="append", help="Additional configuration files in YAML format", ) configuration.add_argument( "--config-toml", - dest="configs_toml", metavar="FILE", action="append", + dest="configs_toml", + metavar="FILE", + action="append", help="Additional configuration files in TOML format", ) configuration.add_argument( "--config-create", - dest="config", action="store_const", const="init", + dest="config", + action="store_const", + const="init", help="Create a basic configuration file", ) configuration.add_argument( "--config-status", - dest="config", action="store_const", const="status", + dest="config", + action="store_const", + const="status", help="Show configuration file status", ) configuration.add_argument( "--config-open", - dest="config", action="store_const", const="open", + dest="config", + action="store_const", + const="open", help="Open configuration file in external application", ) configuration.add_argument( "--config-ignore", - dest="config_load", action="store_false", + dest="config_load", + action="store_false", help="Do not read default configuration files", ) configuration.add_argument( "--ignore-config", - dest="config_load", action="store_false", + dest="config_load", + action="store_false", help=argparse.SUPPRESS, ) authentication = parser.add_argument_group("Authentication Options") authentication.add_argument( - "-u", "--username", - dest="username", metavar="USER", action=ConfigAction, + "-u", + "--username", + dest="username", + metavar="USER", + action=ConfigAction, help="Username to login with", ) authentication.add_argument( - "-p", "--password", - dest="password", metavar="PASS", action=ConfigAction, + "-p", + "--password", + dest="password", + metavar="PASS", + action=ConfigAction, help="Password belonging to the given username", ) authentication.add_argument( "--netrc", - dest="netrc", nargs=0, action=ConfigConstAction, const=True, + dest="netrc", + nargs=0, + action=ConfigConstAction, + const=True, help="Enable .netrc authentication data", ) cookies = parser.add_argument_group("Cookie Options") cookies.add_argument( - "-C", "--cookies", - dest="cookies", metavar="FILE", action=ConfigAction, + "-C", + "--cookies", + dest="cookies", + metavar="FILE", + action=ConfigAction, help="File to load additional cookies from", ) cookies.add_argument( "--cookies-export", - dest="cookies-update", metavar="FILE", action=ConfigAction, + dest="cookies-update", + metavar="FILE", + action=ConfigAction, help="Export session cookies to FILE", ) cookies.add_argument( "--cookies-from-browser", dest="cookies_from_browser", metavar="BROWSER[/DOMAIN][+KEYRING][:PROFILE][::CONTAINER]", - help=("Name of the browser to load cookies from, with optional " - "domain prefixed with '/', " - "keyring name prefixed with '+', " - "profile prefixed with ':', and " - "container prefixed with '::' " - "('none' for no container (default), 'all' for all containers)"), + help=( + "Name of the browser to load cookies from, with optional " + "domain prefixed with '/', " + "keyring name prefixed with '+', " + "profile prefixed with ':', and " + "container prefixed with '::' " + "('none' for no container (default), 'all' for all containers)" + ), ) selection = parser.add_argument_group("Selection Options") selection.add_argument( - "-A", "--abort", - dest="abort", metavar="N", type=int, - help=("Stop current extractor run " - "after N consecutive file downloads were skipped"), + "-A", + "--abort", + dest="abort", + metavar="N", + type=int, + help=("Stop current extractor run " "after N consecutive file downloads were skipped"), ) selection.add_argument( - "-T", "--terminate", - dest="terminate", metavar="N", type=int, - help=("Stop current and parent extractor runs " - "after N consecutive file downloads were skipped"), + "-T", + "--terminate", + dest="terminate", + metavar="N", + type=int, + help=( + "Stop current and parent extractor runs " + "after N consecutive file downloads were skipped" + ), ) selection.add_argument( "--filesize-min", - dest="filesize-min", metavar="SIZE", action=ConfigAction, + dest="filesize-min", + metavar="SIZE", + action=ConfigAction, help="Do not download files smaller than SIZE (e.g. 500k or 2.5M)", ) selection.add_argument( "--filesize-max", - dest="filesize-max", metavar="SIZE", action=ConfigAction, + dest="filesize-max", + metavar="SIZE", + action=ConfigAction, help="Do not download files larger than SIZE (e.g. 500k or 2.5M)", ) selection.add_argument( "--download-archive", - dest="archive", metavar="FILE", action=ConfigAction, - help=("Record all downloaded or skipped files in FILE and " - "skip downloading any file already in it"), + dest="archive", + metavar="FILE", + action=ConfigAction, + help=( + "Record all downloaded or skipped files in FILE and " + "skip downloading any file already in it" + ), ) selection.add_argument( "--range", - dest="image-range", metavar="RANGE", action=ConfigAction, - help=("Index range(s) specifying which files to download. " - "These can be either a constant value, range, or slice " - "(e.g. '5', '8-20', or '1:24:3')"), + dest="image-range", + metavar="RANGE", + action=ConfigAction, + help=( + "Index range(s) specifying which files to download. " + "These can be either a constant value, range, or slice " + "(e.g. '5', '8-20', or '1:24:3')" + ), ) selection.add_argument( "--chapter-range", - dest="chapter-range", metavar="RANGE", action=ConfigAction, - help=("Like '--range', but applies to manga chapters " - "and other delegated URLs"), + dest="chapter-range", + metavar="RANGE", + action=ConfigAction, + help=("Like '--range', but applies to manga chapters " "and other delegated URLs"), ) selection.add_argument( "--filter", - dest="image-filter", metavar="EXPR", action=ConfigAction, - help=("Python expression controlling which files to download. " - "Files for which the expression evaluates to False are ignored. " - "Available keys are the filename-specific ones listed by '-K'. " - "Example: --filter \"image_width >= 1000 and " - "rating in ('s', 'q')\""), + dest="image-filter", + metavar="EXPR", + action=ConfigAction, + help=( + "Python expression controlling which files to download. " + "Files for which the expression evaluates to False are ignored. " + "Available keys are the filename-specific ones listed by '-K'. " + 'Example: --filter "image_width >= 1000 and ' + "rating in ('s', 'q')\"" + ), ) selection.add_argument( "--chapter-filter", - dest="chapter-filter", metavar="EXPR", action=ConfigAction, - help=("Like '--filter', but applies to manga chapters " - "and other delegated URLs"), + dest="chapter-filter", + metavar="EXPR", + action=ConfigAction, + help=("Like '--filter', but applies to manga chapters " "and other delegated URLs"), ) infojson = { - "name" : "metadata", - "event" : "init", + "name": "metadata", + "event": "init", "filename": "info.json", } postprocessor = parser.add_argument_group("Post-processing Options") postprocessor.add_argument( - "-P", "--postprocessor", - dest="postprocessors", metavar="NAME", action="append", + "-P", + "--postprocessor", + dest="postprocessors", + metavar="NAME", + action="append", help="Activate the specified post processor", ) postprocessor.add_argument( "--no-postprocessors", - dest="postprocess", nargs=0, action=ConfigConstAction, const=False, - help=("Do not run any post processors") + dest="postprocess", + nargs=0, + action=ConfigConstAction, + const=False, + help=("Do not run any post processors"), ) postprocessor.add_argument( - "-O", "--postprocessor-option", - dest="options_pp", metavar="KEY=VALUE", - action=PPParseAction, default={}, + "-O", + "--postprocessor-option", + dest="options_pp", + metavar="KEY=VALUE", + action=PPParseAction, + default={}, help="Additional post processor options", ) postprocessor.add_argument( "--write-metadata", dest="postprocessors", - action="append_const", const="metadata", + action="append_const", + const="metadata", help="Write metadata to separate JSON files", ) postprocessor.add_argument( "--write-info-json", dest="postprocessors", - action="append_const", const=infojson, + action="append_const", + const=infojson, help="Write gallery metadata to a info.json file", ) postprocessor.add_argument( "--write-infojson", dest="postprocessors", - action="append_const", const=infojson, + action="append_const", + const=infojson, help=argparse.SUPPRESS, ) postprocessor.add_argument( "--write-tags", dest="postprocessors", - action="append_const", const={"name": "metadata", "mode": "tags"}, + action="append_const", + const={"name": "metadata", "mode": "tags"}, help="Write image tags to separate text files", ) postprocessor.add_argument( "--zip", dest="postprocessors", - action="append_const", const="zip", + action="append_const", + const="zip", help="Store downloaded files in a ZIP archive", ) postprocessor.add_argument( "--cbz", dest="postprocessors", - action="append_const", const={ - "name" : "zip", + action="append_const", + const={ + "name": "zip", "extension": "cbz", }, help="Store downloaded files in a CBZ archive", ) postprocessor.add_argument( "--mtime", - dest="postprocessors", metavar="NAME", action=MtimeAction, - help=("Set file modification times according to metadata " - "selected by NAME. Examples: 'date' or 'status[date]'"), + dest="postprocessors", + metavar="NAME", + action=MtimeAction, + help=( + "Set file modification times according to metadata " + "selected by NAME. Examples: 'date' or 'status[date]'" + ), ) postprocessor.add_argument( "--mtime-from-date", - dest="postprocessors", nargs=0, action=MtimeAction, + dest="postprocessors", + nargs=0, + action=MtimeAction, const="date|status[date]", help=argparse.SUPPRESS, ) postprocessor.add_argument( "--rename", - dest="postprocessors", metavar="FORMAT", action=RenameAction, const=0, - help=("Rename previously downloaded files from FORMAT " - "to the current filename format"), + dest="postprocessors", + metavar="FORMAT", + action=RenameAction, + const=0, + help=("Rename previously downloaded files from FORMAT " "to the current filename format"), ) postprocessor.add_argument( "--rename-to", - dest="postprocessors", metavar="FORMAT", action=RenameAction, const=1, - help=("Rename previously downloaded files from the current filename " - "format to FORMAT"), + dest="postprocessors", + metavar="FORMAT", + action=RenameAction, + const=1, + help=("Rename previously downloaded files from the current filename " "format to FORMAT"), ) postprocessor.add_argument( "--ugoira", - dest="postprocessors", metavar="FMT", action=UgoiraAction, - help=("Convert Pixiv Ugoira to FMT using FFmpeg. " - "Supported formats are 'webm', 'mp4', 'gif', " - "'vp8', 'vp9', 'vp9-lossless', 'copy', 'zip'."), + dest="postprocessors", + metavar="FMT", + action=UgoiraAction, + help=( + "Convert Pixiv Ugoira to FMT using FFmpeg. " + "Supported formats are 'webm', 'mp4', 'gif', " + "'vp8', 'vp9', 'vp9-lossless', 'copy', 'zip'." + ), ) postprocessor.add_argument( "--ugoira-conv", - dest="postprocessors", nargs=0, action=UgoiraAction, const="vp8", + dest="postprocessors", + nargs=0, + action=UgoiraAction, + const="vp8", help=argparse.SUPPRESS, ) postprocessor.add_argument( "--ugoira-conv-lossless", - dest="postprocessors", nargs=0, action=UgoiraAction, + dest="postprocessors", + nargs=0, + action=UgoiraAction, const="vp9-lossless", help=argparse.SUPPRESS, ) postprocessor.add_argument( "--ugoira-conv-copy", - dest="postprocessors", nargs=0, action=UgoiraAction, const="copy", + dest="postprocessors", + nargs=0, + action=UgoiraAction, + const="copy", help=argparse.SUPPRESS, ) postprocessor.add_argument( "--exec", - dest="postprocessors", metavar="CMD", - action=AppendCommandAction, const={"name": "exec"}, - help=("Execute CMD for each downloaded file. " - "Supported replacement fields are " - "{} or {_path}, {_directory}, {_filename}. " - "Example: --exec \"convert {} {}.png && rm {}\""), + dest="postprocessors", + metavar="CMD", + action=AppendCommandAction, + const={"name": "exec"}, + help=( + "Execute CMD for each downloaded file. " + "Supported replacement fields are " + "{} or {_path}, {_directory}, {_filename}. " + 'Example: --exec "convert {} {}.png && rm {}"' + ), ) postprocessor.add_argument( "--exec-after", - dest="postprocessors", metavar="CMD", - action=AppendCommandAction, const={ - "name": "exec", "event": "finalize"}, - help=("Execute CMD after all files were downloaded. " - "Example: --exec-after \"cd {_directory} " - "&& convert * ../doc.pdf\""), + dest="postprocessors", + metavar="CMD", + action=AppendCommandAction, + const={"name": "exec", "event": "finalize"}, + help=( + "Execute CMD after all files were downloaded. " + 'Example: --exec-after "cd {_directory} ' + '&& convert * ../doc.pdf"' + ), ) - try: + with suppress(Exception): # restore normal behavior when adding '-4' or '-6' as arguments parser._has_negative_number_optionals.clear() - except Exception: - pass return parser diff --git a/gallery_dl/output.py b/gallery_dl/output.py index 1649487b2..e36484833 100644 --- a/gallery_dl/output.py +++ b/gallery_dl/output.py @@ -1,18 +1,18 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import logging import os -import sys import shutil -import logging +import sys import unicodedata -from . import config, util, formatter +from . import config +from . import formatter +from . import util # -------------------------------------------------------------------- # Globals @@ -72,17 +72,29 @@ class Logger(logging.Logger): """Custom Logger that includes extra info in log records""" - def makeRecord(self, name, level, fn, lno, msg, args, exc_info, - func=None, extra=None, sinfo=None, - factory=logging._logRecordFactory): + def makeRecord( + self, + name, + level, + fn, + lno, + msg, + args, + exc_info, + func=None, + extra=None, + sinfo=None, + factory=logging._logRecordFactory, + ): rv = factory(name, level, fn, lno, msg, args, exc_info, func, sinfo) if extra: rv.__dict__.update(extra) return rv -class LoggerAdapter(): +class LoggerAdapter: """Trimmed-down version of logging.LoggingAdapter""" + __slots__ = ("logger", "extra") def __init__(self, logger, job): @@ -110,7 +122,7 @@ def error(self, msg, *args, **kwargs): self.logger._log(logging.ERROR, msg, args, **kwargs) -class PathfmtProxy(): +class PathfmtProxy: __slots__ = ("job",) def __init__(self, job): @@ -127,7 +139,7 @@ def __str__(self): return "" -class KwdictProxy(): +class KwdictProxy: __slots__ = ("job",) def __init__(self, job): @@ -144,9 +156,8 @@ class Formatter(logging.Formatter): def __init__(self, fmt, datefmt): if isinstance(fmt, dict): for key in LOG_LEVELS: - value = fmt[key] if key in fmt else LOG_FORMAT - fmt[key] = (formatter.parse(value).format_map, - "{asctime" in value) + value = fmt.get(key, LOG_FORMAT) + fmt[key] = (formatter.parse(value).format_map, "{asctime" in value) else: if fmt == LOG_FORMAT: fmt = (fmt.format_map, False) @@ -225,21 +236,18 @@ def configure_logging(loglevel): lf[level] = ansifmt(c, logfmt) if c else logfmt logfmt = lf - handler.setFormatter(Formatter( - logfmt, opts.get("format-date", LOG_FORMAT_DATE))) + handler.setFormatter(Formatter(logfmt, opts.get("format-date", LOG_FORMAT_DATE))) if "level" in opts and handler.level == LOG_LEVEL: handler.setLevel(opts["level"]) - if minlevel > handler.level: - minlevel = handler.level + minlevel = min(minlevel, handler.level) # file logging handler handler = setup_logging_handler("logfile", lvl=loglevel) if handler: root.addHandler(handler) - if minlevel > handler.level: - minlevel = handler.level + minlevel = min(minlevel, handler.level) root.setLevel(minlevel) @@ -262,25 +270,26 @@ def setup_logging_handler(key, fmt=LOG_FORMAT, lvl=LOG_LEVEL, mode="w"): os.makedirs(os.path.dirname(path)) handler = logging.FileHandler(path, mode, encoding) except (OSError, ValueError) as exc: - logging.getLogger("gallery-dl").warning( - "%s: %s", key, exc) + logging.getLogger("gallery-dl").warning("%s: %s", key, exc) return None except TypeError as exc: - logging.getLogger("gallery-dl").warning( - "%s: missing or invalid path (%s)", key, exc) + logging.getLogger("gallery-dl").warning("%s: missing or invalid path (%s)", key, exc) return None handler.setLevel(opts.get("level", lvl)) - handler.setFormatter(Formatter( - opts.get("format", fmt), - opts.get("format-date", LOG_FORMAT_DATE), - )) + handler.setFormatter( + Formatter( + opts.get("format", fmt), + opts.get("format-date", LOG_FORMAT_DATE), + ) + ) return handler # -------------------------------------------------------------------- # Utility functions + def stdout_write_flush(s): sys.stdout.write(s) sys.stdout.flush() @@ -292,6 +301,7 @@ def stderr_write_flush(s): if getattr(sys.stdout, "line_buffering", None): + def stdout_write(s): sys.stdout.write(s) else: @@ -299,6 +309,7 @@ def stdout_write(s): if getattr(sys.stderr, "line_buffering", None): + def stderr_write(s): sys.stderr.write(s) else: @@ -324,28 +335,30 @@ def configure_standard_streams(): except AttributeError: # no 'reconfigure' support oget = options.get - setattr(sys, name, stream.__class__( - stream.buffer, - encoding=oget("encoding", stream.encoding), - errors=oget("errors", "replace"), - newline=oget("newline", stream.newlines), - line_buffering=oget("line_buffering", stream.line_buffering), - )) + setattr( + sys, + name, + stream.__class__( + stream.buffer, + encoding=oget("encoding", stream.encoding), + errors=oget("errors", "replace"), + newline=oget("newline", stream.newlines), + line_buffering=oget("line_buffering", stream.line_buffering), + ), + ) # -------------------------------------------------------------------- # Downloader output + def select(): """Select a suitable output class""" mode = config.get(("output",), "mode") if mode is None or mode == "auto": try: - if TTY_STDOUT: - output = ColorOutput() if ANSI else TerminalOutput() - else: - output = PipeOutput() + output = (ColorOutput() if ANSI else TerminalOutput()) if TTY_STDOUT else PipeOutput() except Exception: output = PipeOutput() elif isinstance(mode, dict): @@ -354,12 +367,12 @@ def select(): output = NullOutput() else: output = { - "default" : PipeOutput, - "pipe" : PipeOutput, - "term" : TerminalOutput, + "default": PipeOutput, + "pipe": PipeOutput, + "term": TerminalOutput, "terminal": TerminalOutput, - "color" : ColorOutput, - "null" : NullOutput, + "color": ColorOutput, + "null": NullOutput, }[mode.lower()]() if not config.get(("output",), "skip", True): @@ -367,8 +380,7 @@ def select(): return output -class NullOutput(): - +class NullOutput: def start(self, path): """Print a message indicating the start of a download""" @@ -383,7 +395,6 @@ def progress(self, bytes_total, bytes_downloaded, bytes_per_second): class PipeOutput(NullOutput): - def skip(self, path): stdout_write(CHAR_SKIP + path + "\n") @@ -391,8 +402,7 @@ def success(self, path): stdout_write(path + "\n") -class TerminalOutput(): - +class TerminalOutput: def __init__(self): shorten = config.get(("output",), "shorten", True) if shorten: @@ -416,14 +426,12 @@ def progress(self, bytes_total, bytes_downloaded, bytes_per_second): bdl = util.format_value(bytes_downloaded) bps = util.format_value(bytes_per_second) if bytes_total is None: - stderr_write("\r{:>7}B {:>7}B/s ".format(bdl, bps)) + stderr_write(f"\r{bdl:>7}B {bps:>7}B/s ") else: - stderr_write("\r{:>3}% {:>7}B {:>7}B/s ".format( - bytes_downloaded * 100 // bytes_total, bdl, bps)) + stderr_write(f"\r{bytes_downloaded * 100 // bytes_total:>3}% {bdl:>7}B {bps:>7}B/s ") class ColorOutput(TerminalOutput): - def __init__(self): TerminalOutput.__init__(self) @@ -431,10 +439,8 @@ def __init__(self): if colors is None: colors = COLORS_DEFAULT - self.color_skip = "\033[{}m".format( - colors.get("skip", "2")) - self.color_success = "\r\033[{}m".format( - colors.get("success", "1;32")) + self.color_skip = "\033[{}m".format(colors.get("skip", "2")) + self.color_success = "\r\033[{}m".format(colors.get("success", "1;32")) def start(self, path): stdout_write_flush(self.shorten(path)) @@ -446,10 +452,8 @@ def success(self, path): stdout_write(self.color_success + self.shorten(path) + "\033[0m\n") -class CustomOutput(): - +class CustomOutput: def __init__(self, options): - fmt_skip = options.get("skip") fmt_start = options.get("start") fmt_success = options.get("success") @@ -467,21 +471,18 @@ def __init__(self, options): func = shorten_string_eaw if shorten == "eaw" else shorten_string width = shutil.get_terminal_size().columns - self._fmt_skip = self._make_func( - func, fmt_skip, width - off_skip) - self._fmt_start = self._make_func( - func, fmt_start, width - off_start) - self._fmt_success = self._make_func( - func, fmt_success, width - off_success) + self._fmt_skip = self._make_func(func, fmt_skip, width - off_skip) + self._fmt_start = self._make_func(func, fmt_start, width - off_start) + self._fmt_success = self._make_func(func, fmt_success, width - off_success) else: self._fmt_skip = fmt_skip.format self._fmt_start = fmt_start.format self._fmt_success = fmt_success.format - self._fmt_progress = (options.get("progress") or - "\r{0:>7}B {1:>7}B/s ").format - self._fmt_progress_total = (options.get("progress-total") or - "\r{3:>3}% {0:>7}B {1:>7}B/s ").format + self._fmt_progress = (options.get("progress") or "\r{0:>7}B {1:>7}B/s ").format + self._fmt_progress_total = ( + options.get("progress-total") or "\r{3:>3}% {0:>7}B {1:>7}B/s " + ).format @staticmethod def _make_func(shorten, format_string, limit): @@ -503,16 +504,16 @@ def progress(self, bytes_total, bytes_downloaded, bytes_per_second): if bytes_total is None: stderr_write(self._fmt_progress(bdl, bps)) else: - stderr_write(self._fmt_progress_total( - bdl, bps, util.format_value(bytes_total), - bytes_downloaded * 100 // bytes_total)) + stderr_write( + self._fmt_progress_total( + bdl, bps, util.format_value(bytes_total), bytes_downloaded * 100 // bytes_total + ) + ) class EAWCache(dict): - def __missing__(self, key): - width = self[key] = \ - 2 if unicodedata.east_asian_width(key) in "WF" else 1 + width = self[key] = 2 if unicodedata.east_asian_width(key) in "WF" else 1 return width @@ -521,7 +522,7 @@ def shorten_string(txt, limit, sep="…"): if len(txt) <= limit: return txt limit -= len(sep) - return txt[:limit // 2] + sep + txt[-((limit+1) // 2):] + return txt[: limit // 2] + sep + txt[-((limit + 1) // 2) :] def shorten_string_eaw(txt, limit, sep="…", cache=EAWCache()): @@ -536,7 +537,7 @@ def shorten_string_eaw(txt, limit, sep="…", cache=EAWCache()): limit -= len(sep) if text_width == len(txt): # all characters have a width of 1 - return txt[:limit // 2] + sep + txt[-((limit+1) // 2):] + return txt[: limit // 2] + sep + txt[-((limit + 1) // 2) :] # wide characters left = 0 @@ -548,11 +549,11 @@ def shorten_string_eaw(txt, limit, sep="…", cache=EAWCache()): left += 1 right = -1 - rwidth = (limit+1) // 2 + (lwidth + char_widths[left]) + rwidth = (limit + 1) // 2 + (lwidth + char_widths[left]) while True: rwidth -= char_widths[right] if rwidth < 0: break right -= 1 - return txt[:left] + sep + txt[right+1:] + return txt[:left] + sep + txt[right + 1 :] diff --git a/gallery_dl/path.py b/gallery_dl/path.py index f57b02e17..26ef92f3b 100644 --- a/gallery_dl/path.py +++ b/gallery_dl/path.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,24 +6,26 @@ """Filesystem path handling""" +import functools import os import re import shutil -import functools -from . import util, formatter, exception + +from . import exception +from . import formatter +from . import util WINDOWS = util.WINDOWS EXTENSION_MAP = { "jpeg": "jpg", - "jpe" : "jpg", + "jpe": "jpg", "jfif": "jpg", - "jif" : "jpg", - "jfi" : "jpg", + "jif": "jpg", + "jfi": "jpg", } -class PathFormat(): - +class PathFormat: def __init__(self, extractor): config = extractor.config kwdefault = config("keywords-default") @@ -38,15 +38,14 @@ def __init__(self, extractor): filename_fmt = extractor.filename_fmt elif isinstance(filename_fmt, dict): self.filename_conditions = [ - (util.compile_filter(expr), - formatter.parse(fmt, kwdefault).format_map) - for expr, fmt in filename_fmt.items() if expr + (util.compile_filter(expr), formatter.parse(fmt, kwdefault).format_map) + for expr, fmt in filename_fmt.items() + if expr ] self.build_filename = self.build_filename_conditional filename_fmt = filename_fmt.get("", extractor.filename_fmt) - self.filename_formatter = formatter.parse( - filename_fmt, kwdefault).format_map + self.filename_formatter = formatter.parse(filename_fmt, kwdefault).format_map except Exception as exc: raise exception.FilenameFormatError(exc) @@ -57,18 +56,18 @@ def __init__(self, extractor): directory_fmt = extractor.directory_fmt elif isinstance(directory_fmt, dict): self.directory_conditions = [ - (util.compile_filter(expr), [ - formatter.parse(fmt, kwdefault).format_map - for fmt in fmts - ]) - for expr, fmts in directory_fmt.items() if expr + ( + util.compile_filter(expr), + [formatter.parse(fmt, kwdefault).format_map for fmt in fmts], + ) + for expr, fmts in directory_fmt.items() + if expr ] self.build_directory = self.build_directory_conditional directory_fmt = directory_fmt.get("", extractor.directory_fmt) self.directory_formatters = [ - formatter.parse(dirfmt, kwdefault).format_map - for dirfmt in directory_fmt + formatter.parse(dirfmt, kwdefault).format_map for dirfmt in directory_fmt ] except Exception as exc: raise exception.DirectoryFormatError(exc) @@ -92,11 +91,11 @@ def __init__(self, extractor): restrict = config("path-restrict", "auto") replace = config("path-replace", "_") if restrict == "auto": - restrict = "\\\\|/<>:\"?*" if WINDOWS else "/" + restrict = '\\\\|/<>:"?*' if WINDOWS else "/" elif restrict == "unix": restrict = "/" elif restrict == "windows": - restrict = "\\\\|/<>:\"?*" + restrict = '\\\\|/<>:"?*' elif restrict == "ascii": restrict = "^0-9A-Za-z_." elif restrict == "ascii+": @@ -138,15 +137,16 @@ def __init__(self, extractor): def _build_cleanfunc(chars, repl): if not chars: return util.identity - elif isinstance(chars, dict): + if isinstance(chars, dict): + def func(x, table=str.maketrans(chars)): return x.translate(table) elif len(chars) == 1: + def func(x, c=chars, r=repl): return x.replace(c, r) else: - return functools.partial( - re.compile("[" + chars + "]").sub, repl) + return functools.partial(re.compile("[" + chars + "]").sub, repl) return func def open(self, mode="wb"): @@ -188,7 +188,8 @@ def set_directory(self, kwdict): segments = self.build_directory(kwdict) if segments: self.directory = directory = self.basedirectory + self.clean_path( - os.sep.join(segments) + os.sep) + os.sep.join(segments) + os.sep + ) else: self.directory = directory = self.basedirectory @@ -226,8 +227,7 @@ def fix_extension(self, _=None): """Fix filenames without a given filename extension""" try: if not self.extension: - self.kwdict["extension"] = \ - self.prefix + self.extension_map("", "") + self.kwdict["extension"] = self.prefix + self.extension_map("", "") self.build_path() if self.path[-1] == ".": self.path = self.path[:-1] @@ -244,8 +244,7 @@ def fix_extension(self, _=None): def build_filename(self, kwdict): """Apply 'kwdict' to filename format string""" try: - return self.clean_path(self.clean_segment( - self.filename_formatter(kwdict))) + return self.clean_path(self.clean_segment(self.filename_formatter(kwdict))) except Exception as exc: raise exception.FilenameFormatError(exc) @@ -312,8 +311,7 @@ def part_enable(self, part_directory=None): if self.extension: self.temppath += ".part" else: - self.kwdict["extension"] = self.prefix + self.extension_map( - "part", "part") + self.kwdict["extension"] = self.prefix + self.extension_map("part", "part") self.build_path() if part_directory: self.temppath = os.path.join( diff --git a/gallery_dl/postprocessor/__init__.py b/gallery_dl/postprocessor/__init__.py index 7837b063d..0e0cf7f39 100644 --- a/gallery_dl/postprocessor/__init__.py +++ b/gallery_dl/postprocessor/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify diff --git a/gallery_dl/postprocessor/classify.py b/gallery_dl/postprocessor/classify.py index 34af1d940..9830b86f3 100644 --- a/gallery_dl/postprocessor/classify.py +++ b/gallery_dl/postprocessor/classify.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2021 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,31 +6,25 @@ """Categorize files by file extension""" -from .common import PostProcessor import os +from .common import PostProcessor -class ClassifyPP(PostProcessor): +class ClassifyPP(PostProcessor): DEFAULT_MAPPING = { - "Music" : ("mp3", "aac", "flac", "ogg", "wma", "m4a", "wav"), - "Video" : ("flv", "ogv", "avi", "mp4", "mpg", "mpeg", "3gp", "mkv", - "webm", "vob", "wmv"), - "Pictures" : ("jpg", "jpeg", "png", "gif", "bmp", "svg", "webp"), - "Archives" : ("zip", "rar", "7z", "tar", "gz", "bz2"), + "Music": ("mp3", "aac", "flac", "ogg", "wma", "m4a", "wav"), + "Video": ("flv", "ogv", "avi", "mp4", "mpg", "mpeg", "3gp", "mkv", "webm", "vob", "wmv"), + "Pictures": ("jpg", "jpeg", "png", "gif", "bmp", "svg", "webp"), + "Archives": ("zip", "rar", "7z", "tar", "gz", "bz2"), } def __init__(self, job, options): PostProcessor.__init__(self, job) mapping = options.get("mapping", self.DEFAULT_MAPPING) - self.mapping = { - ext: directory - for directory, exts in mapping.items() - for ext in exts - } - job.register_hooks( - {"prepare": self.prepare, "file": self.move}, options) + self.mapping = {ext: directory for directory, exts in mapping.items() for ext in exts} + job.register_hooks({"prepare": self.prepare, "file": self.move}, options) def prepare(self, pathfmt): ext = pathfmt.extension diff --git a/gallery_dl/postprocessor/common.py b/gallery_dl/postprocessor/common.py index d4e160341..790eaad5e 100644 --- a/gallery_dl/postprocessor/common.py +++ b/gallery_dl/postprocessor/common.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,10 +6,12 @@ """Common classes and constants used by postprocessor modules.""" -from .. import util, formatter, archive +from .. import archive +from .. import formatter +from .. import util -class PostProcessor(): +class PostProcessor: """Base class for postprocessors""" def __init__(self, job): @@ -28,24 +28,28 @@ def _init_archive(self, job, options, prefix=None): archive_path = util.expand_path(archive_path) if not prefix: prefix = "_" + self.name.upper() + "_" - archive_format = ( - options.get("archive-prefix", extr.category) + - options.get("archive-format", prefix + extr.archive_fmt)) + archive_format = options.get("archive-prefix", extr.category) + options.get( + "archive-format", prefix + extr.archive_fmt + ) try: if "{" in archive_path: - archive_path = formatter.parse(archive_path).format_map( - job.pathfmt.kwdict) + archive_path = formatter.parse(archive_path).format_map(job.pathfmt.kwdict) self.archive = archive.DownloadArchive( - archive_path, archive_format, + archive_path, + archive_format, options.get("archive-pragma"), - "_archive_" + self.name) + "_archive_" + self.name, + ) except Exception as exc: self.log.warning( "Failed to open %s archive at '%s' (%s: %s)", - self.name, archive_path, exc.__class__.__name__, exc) + self.name, + archive_path, + exc.__class__.__name__, + exc, + ) else: - self.log.debug( - "Using %s archive '%s'", self.name, archive_path) + self.log.debug("Using %s archive '%s'", self.name, archive_path) return True self.archive = None diff --git a/gallery_dl/postprocessor/compare.py b/gallery_dl/postprocessor/compare.py index 3bb63c80e..1481e2bb0 100644 --- a/gallery_dl/postprocessor/compare.py +++ b/gallery_dl/postprocessor/compare.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,15 @@ """Compare versions of the same file and replace/enumerate them on mismatch""" -from .common import PostProcessor -from .. import text, util, exception import os +from .. import exception +from .. import text +from .. import util +from .common import PostProcessor -class ComparePP(PostProcessor): +class ComparePP(PostProcessor): def __init__(self, job, options): PostProcessor.__init__(self, job) if options.get("shallow"): @@ -32,11 +32,10 @@ def __init__(self, job, options): elif equal == "exit": self._equal_exc = SystemExit - job.register_hooks({"file": ( - self.enumerate - if options.get("action") == "enumerate" else - self.replace - )}, options) + job.register_hooks( + {"file": (self.enumerate if options.get("action") == "enumerate" else self.replace)}, + options, + ) def replace(self, pathfmt): try: diff --git a/gallery_dl/postprocessor/exec.py b/gallery_dl/postprocessor/exec.py index 7d2be2b95..11352f640 100644 --- a/gallery_dl/postprocessor/exec.py +++ b/gallery_dl/postprocessor/exec.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,15 @@ """Execute processes""" -from .common import PostProcessor -from .. import util, formatter import os import re +from .. import formatter +from .. import util +from .common import PostProcessor if util.WINDOWS: + def quote(s): return '"' + s.replace('"', '\\"') + '"' else: @@ -22,7 +22,6 @@ def quote(s): class ExecPP(PostProcessor): - def __init__(self, job, options): PostProcessor.__init__(self, job) @@ -81,8 +80,7 @@ def _exec(self, args, shell): self.log.debug("Running '%s'", args) retcode = util.Popen(args, shell=shell).wait() if retcode: - self.log.warning("'%s' returned with non-zero exit status (%d)", - args, retcode) + self.log.warning("'%s' returned with non-zero exit status (%d)", args, retcode) def _exec_async(self, args, shell): self.log.debug("Running '%s'", args) diff --git a/gallery_dl/postprocessor/hash.py b/gallery_dl/postprocessor/hash.py index 92a747799..c9617903b 100644 --- a/gallery_dl/postprocessor/hash.py +++ b/gallery_dl/postprocessor/hash.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,12 @@ """Compute file hash digests""" -from .common import PostProcessor import hashlib +from .common import PostProcessor -class HashPP(PostProcessor): +class HashPP(PostProcessor): def __init__(self, job, options): PostProcessor.__init__(self, job) @@ -41,10 +39,7 @@ def __init__(self, job, options): job.register_hooks({event: self.run for event in events}, options) def run(self, pathfmt): - hashes = [ - (key, hashlib.new(name)) - for key, name in self.hashes - ] + hashes = [(key, hashlib.new(name)) for key, name in self.hashes] size = self.chunk_size with self._open(pathfmt) as fp: diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py index 3ef9fbce1..9bf330f09 100644 --- a/gallery_dl/postprocessor/metadata.py +++ b/gallery_dl/postprocessor/metadata.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,15 +6,16 @@ """Write metadata to external files""" -from .common import PostProcessor -from .. import util, formatter import json -import sys import os +import sys +from .. import formatter +from .. import util +from .common import PostProcessor -class MetadataPP(PostProcessor): +class MetadataPP(PostProcessor): def __init__(self, job, options): PostProcessor.__init__(self, job) @@ -73,8 +72,7 @@ def __init__(self, job, options): if isinstance(directory, list): self._directory = self._directory_format self._directory_formatters = [ - formatter.parse(dirfmt, util.NONE).format_map - for dirfmt in directory + formatter.parse(dirfmt, util.NONE).format_map for dirfmt in directory ] elif directory: self._directory = self._directory_custom @@ -190,8 +188,7 @@ def _filename(self, pathfmt): return (pathfmt.filename or "metadata") + "." + self.extension def _filename_custom(self, pathfmt): - return pathfmt.clean_path(pathfmt.clean_segment( - self._filename_fmt(pathfmt.kwdict))) + return pathfmt.clean_path(pathfmt.clean_segment(self._filename_fmt(pathfmt.kwdict))) def _filename_extfmt(self, pathfmt): kwdict = pathfmt.kwdict @@ -253,10 +250,8 @@ def _make_filter(self, options): exclude = set(exclude) if private: - return lambda d: {k: v for k, v in d.items() - if k not in exclude} - return lambda d: {k: v for k, v in util.filter_dict(d).items() - if k not in exclude} + return lambda d: {k: v for k, v in d.items() if k not in exclude} + return lambda d: {k: v for k, v in util.filter_dict(d).items() if k not in exclude} if not private: return util.filter_dict diff --git a/gallery_dl/postprocessor/mtime.py b/gallery_dl/postprocessor/mtime.py index 6ded1e29d..514893175 100644 --- a/gallery_dl/postprocessor/mtime.py +++ b/gallery_dl/postprocessor/mtime.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,15 @@ """Use metadata as file modification time""" -from .common import PostProcessor -from .. import text, util, formatter from datetime import datetime +from .. import formatter +from .. import text +from .. import util +from .common import PostProcessor -class MtimePP(PostProcessor): +class MtimePP(PostProcessor): def __init__(self, job, options): PostProcessor.__init__(self, job) value = options.get("value") @@ -38,8 +38,8 @@ def run(self, pathfmt): pathfmt.kwdict["_mtime"] = ( util.datetime_to_timestamp(mtime) - if isinstance(mtime, datetime) else - text.parse_int(mtime) + if isinstance(mtime, datetime) + else text.parse_int(mtime) ) diff --git a/gallery_dl/postprocessor/python.py b/gallery_dl/postprocessor/python.py index db71da250..d461d081b 100644 --- a/gallery_dl/postprocessor/python.py +++ b/gallery_dl/postprocessor/python.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,11 @@ """Run Python functions""" -from .common import PostProcessor from .. import util +from .common import PostProcessor class PythonPP(PostProcessor): - def __init__(self, job, options): PostProcessor.__init__(self, job) diff --git a/gallery_dl/postprocessor/rename.py b/gallery_dl/postprocessor/rename.py index f71738d2a..081baf8c9 100644 --- a/gallery_dl/postprocessor/rename.py +++ b/gallery_dl/postprocessor/rename.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,13 @@ """Rename files""" -from .common import PostProcessor -from .. import formatter import os +from .. import formatter +from .common import PostProcessor -class RenamePP(PostProcessor): +class RenamePP(PostProcessor): def __init__(self, job, options): PostProcessor.__init__(self, job) @@ -24,19 +22,24 @@ def __init__(self, job, options): if old: self._old = self._apply_format(old) - self._new = (self._apply_format(new) if new else - self._apply_pathfmt) - job.register_hooks({ - "prepare": self.rename_from, - }, options) + self._new = self._apply_format(new) if new else self._apply_pathfmt + job.register_hooks( + { + "prepare": self.rename_from, + }, + options, + ) elif new: self._old = self._apply_pathfmt self._new = self._apply_format(new) - job.register_hooks({ - "skip" : self.rename_to_skip, - "prepare-after": self.rename_to_pafter, - }, options) + job.register_hooks( + { + "skip": self.rename_to_skip, + "prepare-after": self.rename_to_pafter, + }, + options, + ) else: raise ValueError("Option 'from' or 'to' is required") @@ -69,8 +72,10 @@ def rename_to_pafter(self, pathfmt): def _rename(self, path_old, name_old, path_new, name_new): if self.skip and os.path.exists(path_new): return self.log.warning( - "Not renaming '%s' to '%s' since another file with the " - "same name exists", name_old, name_new) + "Not renaming '%s' to '%s' since another file with the " "same name exists", + name_old, + name_new, + ) self.log.info("'%s' -> '%s'", name_old, name_new) os.replace(path_old, path_new) @@ -82,8 +87,7 @@ def _apply_format(self, format_string): fmt = formatter.parse(format_string).format_map def apply(pathfmt): - return pathfmt.clean_path(pathfmt.clean_segment(fmt( - pathfmt.kwdict))) + return pathfmt.clean_path(pathfmt.clean_segment(fmt(pathfmt.kwdict))) return apply diff --git a/gallery_dl/postprocessor/ugoira.py b/gallery_dl/postprocessor/ugoira.py index fec4ab025..184976c6d 100644 --- a/gallery_dl/postprocessor/ugoira.py +++ b/gallery_dl/postprocessor/ugoira.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,17 +6,19 @@ """Convert Pixiv Ugoira to WebM""" -from .common import PostProcessor -from .. import util +import os +import shutil import subprocess import tempfile import zipfile -import shutil -import os + +from .. import util +from .common import PostProcessor try: from math import gcd except ImportError: + def gcd(a, b): while b: a, b = b, a % b @@ -26,7 +26,6 @@ def gcd(a, b): class UgoiraPP(PostProcessor): - def __init__(self, job, options): PostProcessor.__init__(self, job) self.args = options.get("ffmpeg-args") or () @@ -48,8 +47,7 @@ def __init__(self, job, options): ext = options.get("extension") mode = options.get("mode") or options.get("ffmpeg-demuxer") if mode is None or mode == "auto": - if ext in (None, "webm", "mkv") and ( - mkvmerge or shutil.which("mkvmerge")): + if ext in (None, "webm", "mkv") and (mkvmerge or shutil.which("mkvmerge")): mode = "mkvmerge" else: mode = "concat" @@ -82,13 +80,18 @@ def __init__(self, job, options): vcodec = None for index, arg in enumerate(self.args): arg, _, stream = arg.partition(":") - if arg == "-vcodec" or arg in ("-c", "-codec") and ( - not stream or stream.partition(":")[0] in ("v", "V")): + if ( + arg == "-vcodec" + or arg in ("-c", "-codec") + and (not stream or stream.partition(":")[0] in ("v", "V")) + ): vcodec = self.args[index + 1] # use filter when using libx264/5 self.prevent_odd = ( - vcodec in ("libx264", "libx265") or - not vcodec and self.extension.lower() in ("mp4", "mkv")) + vcodec in ("libx264", "libx265") + or not vcodec + and self.extension.lower() in ("mp4", "mkv") + ) else: self.prevent_odd = False @@ -98,11 +101,14 @@ def __init__(self, job, options): if self.prevent_odd: args += ("-vf", "crop=iw-mod(iw\\,2):ih-mod(ih\\,2)") - job.register_hooks({ - "prepare": self.prepare, - "file" : self.convert_from_zip, - "after" : self.convert_from_files, - }, options) + job.register_hooks( + { + "prepare": self.prepare, + "file": self.convert_from_zip, + "after": self.convert_from_files, + }, + options, + ) def prepare(self, pathfmt): self._convert_zip = self._convert_files = False @@ -136,7 +142,7 @@ def prepare(self, pathfmt): def convert_from_zip(self, pathfmt): if not self._convert_zip: - return + return None self._zip_source = True with self._tempdir() as tempdir: @@ -146,13 +152,16 @@ def convert_from_zip(self, pathfmt): zfile.extractall(tempdir) except FileNotFoundError: pathfmt.realpath = pathfmt.temppath - return + return None except Exception as exc: pathfmt.realpath = pathfmt.temppath self.log.error( "%s: Unable to extract frames from %s (%s: %s)", - pathfmt.kwdict.get("id"), pathfmt.filename, - exc.__class__.__name__, exc) + pathfmt.kwdict.get("id"), + pathfmt.filename, + exc.__class__.__name__, + exc, + ) return self.log.debug("", exc_info=exc) if self.convert(pathfmt, tempdir): @@ -170,18 +179,17 @@ def convert_from_files(self, pathfmt): with tempfile.TemporaryDirectory() as tempdir: for frame in self._files: - # update frame filename extension - frame["file"] = name = "{}.{}".format( - frame["file"].partition(".")[0], frame["ext"]) + frame["file"] = name = "{}.{}".format(frame["file"].partition(".")[0], frame["ext"]) if tempdir: # move frame into tempdir try: self._copy_file(frame["path"], tempdir + "/" + name) except OSError as exc: - self.log.debug("Unable to copy frame %s (%s: %s)", - name, exc.__class__.__name__, exc) + self.log.debug( + "Unable to copy frame %s (%s: %s)", name, exc.__class__.__name__, exc + ) return pathfmt.kwdict["num"] = 0 @@ -227,8 +235,7 @@ def convert_to_animation(self, pathfmt, tempdir): self._finalize(pathfmt, tempdir) except OSError as exc: print() - self.log.error("Unable to invoke FFmpeg (%s: %s)", - exc.__class__.__name__, exc) + self.log.error("Unable to invoke FFmpeg (%s: %s)", exc.__class__.__name__, exc) self.log.debug("", exc_info=exc) pathfmt.realpath = pathfmt.temppath except Exception as exc: @@ -247,14 +254,10 @@ def convert_to_archive(self, pathfmt, tempdir): frames = self._frames if self.metadata: - if isinstance(self.metadata, str): - metaname = self.metadata - else: - metaname = "animation.json" - framedata = util.json_dumps([ - {"file": frame["file"], "delay": frame["delay"]} - for frame in frames - ]).encode() + metaname = self.metadata if isinstance(self.metadata, str) else "animation.json" + framedata = util.json_dumps( + [{"file": frame["file"], "delay": frame["delay"]} for frame in frames] + ).encode() if self._zip_source: self.delete = False @@ -268,17 +271,14 @@ def convert_to_archive(self, pathfmt, tempdir): else: if self.mtime: dt = pathfmt.kwdict["date_url"] or pathfmt.kwdict["date"] - mtime = (dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second) + mtime = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second) with zipfile.ZipFile(pathfmt.realpath, "w") as zfile: for frame in frames: - zinfo = zipfile.ZipInfo.from_file( - frame["path"], frame["file"]) + zinfo = zipfile.ZipInfo.from_file(frame["path"], frame["file"]) if self.mtime: zinfo.date_time = mtime - with open(frame["path"], "rb") as src, \ - zfile.open(zinfo, "w") as dst: - shutil.copyfileobj(src, dst, 1024*8) + with open(frame["path"], "rb") as src, zfile.open(zinfo, "w") as dst: + shutil.copyfileobj(src, dst, 1024 * 8) if self.metadata: zinfo = zipfile.ZipInfo(metaname) if self.mtime: @@ -297,8 +297,7 @@ def _exec(self, args): retcode = util.Popen(args, stdout=out, stderr=out).wait() if retcode: print() - self.log.error("Non-zero exit status when running %s (%s)", - args, retcode) + self.log.error("Non-zero exit status when running %s (%s)", args, retcode) raise ValueError() return retcode @@ -330,9 +329,8 @@ def _process_image2(self, pathfmt, tempdir): last_copy = last.copy() frames.append(last_copy) name, _, ext = last_copy["file"].rpartition(".") - last_copy["file"] = "{:>06}.{}".format(int(name)+1, ext) - shutil.copyfile(tempdir + last["file"], - tempdir + last_copy["file"]) + last_copy["file"] = f"{int(name) + 1:>06}.{ext}" + shutil.copyfile(tempdir + last["file"], tempdir + last_copy["file"]) # adjust frame mtime values ts = 0 @@ -342,13 +340,14 @@ def _process_image2(self, pathfmt, tempdir): return [ self.ffmpeg, - "-f", "image2", - "-ts_from_file", "2", - "-pattern_type", "sequence", - "-i", "{}%06d.{}".format( - tempdir.replace("%", "%%"), - frame["file"].rpartition(".")[2] - ), + "-f", + "image2", + "-ts_from_file", + "2", + "-pattern_type", + "sequence", + "-i", + "{}%06d.{}".format(tempdir.replace("%", "%%"), frame["file"].rpartition(".")[2]), ] def _process_mkvmerge(self, pathfmt, tempdir): @@ -357,19 +356,23 @@ def _process_mkvmerge(self, pathfmt, tempdir): return [ self.ffmpeg, - "-f", "image2", - "-pattern_type", "sequence", - "-i", "{}/%06d.{}".format( - tempdir.replace("%", "%%"), - self._frames[0]["file"].rpartition(".")[2] + "-f", + "image2", + "-pattern_type", + "sequence", + "-i", + "{}/%06d.{}".format( + tempdir.replace("%", "%%"), self._frames[0]["file"].rpartition(".")[2] ), ] def _finalize_mkvmerge(self, pathfmt, tempdir): args = [ self.mkvmerge, - "-o", pathfmt.path, # mkvmerge does not support "raw" paths - "--timecodes", "0:" + self._write_mkvmerge_timecodes(tempdir), + "-o", + pathfmt.path, # mkvmerge does not support "raw" paths + "--timecodes", + "0:" + self._write_mkvmerge_timecodes(tempdir), ] if self.extension == "webm": args.append("--webm") @@ -383,8 +386,7 @@ def _write_ffmpeg_concat(self, tempdir): append = content.append for frame in self._frames: - append("file '{}'\nduration {}".format( - frame["file"], frame["delay"] / 1000)) + append("file '{}'\nduration {}".format(frame["file"], frame["delay"] / 1000)) if self.repeat: append("file '{}'".format(frame["file"])) append("") @@ -417,7 +419,7 @@ def calculate_framerate(self, frames): if not self.uniform: gcd = self._delay_gcd(frames) if gcd >= 10: - return (None, "1000/{}".format(gcd)) + return (None, f"1000/{gcd}") return (None, None) @@ -431,10 +433,7 @@ def _delay_gcd(frames): @staticmethod def _delay_is_uniform(frames): delay = frames[0]["delay"] - for f in frames: - if f["delay"] != delay: - return False - return True + return all(f["delay"] == delay for f in frames) __postprocessor__ = UgoiraPP diff --git a/gallery_dl/postprocessor/zip.py b/gallery_dl/postprocessor/zip.py index ce36f2a7a..89b383f53 100644 --- a/gallery_dl/postprocessor/zip.py +++ b/gallery_dl/postprocessor/zip.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,19 +6,19 @@ """Store files in ZIP archives""" -from .common import PostProcessor -from .. import util -import zipfile import os +import zipfile +from .. import util +from .common import PostProcessor -class ZipPP(PostProcessor): +class ZipPP(PostProcessor): COMPRESSION_ALGORITHMS = { "store": zipfile.ZIP_STORED, - "zip" : zipfile.ZIP_DEFLATED, + "zip": zipfile.ZIP_DEFLATED, "bzip2": zipfile.ZIP_BZIP2, - "lzma" : zipfile.ZIP_LZMA, + "lzma": zipfile.ZIP_LZMA, } def __init__(self, job, options): @@ -31,19 +29,20 @@ def __init__(self, job, options): algorithm = options.get("compression", "store") if algorithm not in self.COMPRESSION_ALGORITHMS: self.log.warning( - "unknown compression algorithm '%s'; falling back to 'store'", - algorithm) + "unknown compression algorithm '%s'; falling back to 'store'", algorithm + ) algorithm = "store" self.zfile = None self.path = job.pathfmt.realdirectory[:-1] - self.args = (self.path + ext, "a", - self.COMPRESSION_ALGORITHMS[algorithm], True) - - job.register_hooks({ - "file": (self.write_safe if options.get("mode") == "safe" else - self.write_fast), - }, options) + self.args = (self.path + ext, "a", self.COMPRESSION_ALGORITHMS[algorithm], True) + + job.register_hooks( + { + "file": (self.write_safe if options.get("mode") == "safe" else self.write_fast), + }, + options, + ) job.hooks["finalize"].append(self.finalize) def open(self): @@ -80,10 +79,8 @@ def write_extra(self, pathfmt, zfile, files): try: zfile.write(path, os.path.basename(path)) except OSError as exc: - self.log.warning( - "Unable to write %s to %s", path, zfile.filename) + self.log.warning("Unable to write %s to %s", path, zfile.filename) self.log.debug("%s: %s", exc, exc.__class__.__name__) - pass else: if self.delete: util.remove_file(path) diff --git a/gallery_dl/text.py b/gallery_dl/text.py index 5fd5a4071..2db7cfb6e 100644 --- a/gallery_dl/text.py +++ b/gallery_dl/text.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,11 +6,11 @@ """Collection of functions that work on strings/text""" +import datetime +import html import re import sys -import html import time -import datetime import urllib.parse HTML_RE = re.compile("<[^>]+>") @@ -32,11 +30,7 @@ def remove_html(txt, repl=" ", sep=" "): def split_html(txt): """Split input string by HTML tags""" try: - return [ - unescape(x).strip() - for x in HTML_RE.split(txt) - if x and not x.isspace() - ] + return [unescape(x).strip() for x in HTML_RE.split(txt) if x and not x.isspace()] except TypeError: return [] @@ -62,11 +56,11 @@ def root_from_url(url, scheme="https://"): """Extract scheme and domain from a URL""" if not url.startswith(("https://", "http://")): try: - return scheme + url[:url.index("/")] + return scheme + url[: url.index("/")] except ValueError: return scheme + url try: - return url[:url.index("/", 8)] + return url[: url.index("/", 8)] except ValueError: return url @@ -123,7 +117,7 @@ def extract(txt, begin, end, pos=0): try: first = txt.index(begin, pos) + len(begin) last = txt.index(end, first) - return txt[first:last], last+len(end) + return txt[first:last], last + len(end) except Exception: return None, pos @@ -132,7 +126,7 @@ def extr(txt, begin, end, default=""): """Stripped-down version of 'extract()'""" try: first = txt.index(begin) + len(begin) - return txt[first:txt.index(end, first)] + return txt[first : txt.index(end, first)] except Exception: return default @@ -142,7 +136,7 @@ def rextract(txt, begin, end, pos=-1): lbeg = len(begin) first = txt.rindex(begin, 0, pos) last = txt.index(end, first + lbeg) - return txt[first + lbeg:last], first + return txt[first + lbeg : last], first except Exception: return None, pos @@ -175,6 +169,7 @@ def extract_iter(txt, begin, end, pos=0): def extract_from(txt, pos=0, default=""): """Returns a function object that extracts from 'txt'""" + def extr(begin, end, index=txt.index, txt=txt): nonlocal pos try: @@ -184,6 +179,7 @@ def extr(begin, end, index=txt.index, txt=txt): return txt[first:last] except Exception: return default + return extr @@ -286,7 +282,7 @@ def parse_query_list(qs): return result -if sys.hexversion < 0x30c0000: +if sys.hexversion < 0x30C0000: # Python <= 3.11 def parse_timestamp(ts, default=None): """Create a datetime object from a Unix timestamp""" diff --git a/gallery_dl/update.py b/gallery_dl/update.py index b068e3750..297c4a609 100644 --- a/gallery_dl/update.py +++ b/gallery_dl/update.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -10,55 +8,54 @@ import re import sys -from .extractor.common import Extractor, Message +from . import exception +from . import util +from . import version +from .extractor.common import Extractor +from .extractor.common import Message from .job import DownloadJob -from . import util, version, exception REPOS = { - "stable" : "mikf/gallery-dl", - "dev" : "gdl-org/builds", + "stable": "mikf/gallery-dl", + "dev": "gdl-org/builds", "nightly": "gdl-org/builds", - "master" : "gdl-org/builds", + "master": "gdl-org/builds", } BINARIES_STABLE = { - "windows" : "gallery-dl.exe", + "windows": "gallery-dl.exe", "windows_x86": "gallery-dl.exe", "windows_x64": "gallery-dl.exe", - "linux" : "gallery-dl.bin", + "linux": "gallery-dl.bin", } BINARIES_DEV = { - "windows" : "gallery-dl_windows.exe", + "windows": "gallery-dl_windows.exe", "windows_x86": "gallery-dl_windows_x86.exe", "windows_x64": "gallery-dl_windows.exe", - "linux" : "gallery-dl_linux", - "macos" : "gallery-dl_macos", + "linux": "gallery-dl_linux", + "macos": "gallery-dl_macos", } BINARIES = { - "stable" : BINARIES_STABLE, - "dev" : BINARIES_DEV, + "stable": BINARIES_STABLE, + "dev": BINARIES_DEV, "nightly": BINARIES_DEV, - "master" : BINARIES_DEV, + "master": BINARIES_DEV, } class UpdateJob(DownloadJob): - def handle_url(self, url, kwdict): if not self._check_update(kwdict): if kwdict["_check"]: self.status |= 1 - return self.extractor.log.info( - "gallery-dl is up to date (%s)", version.__version__) + return self.extractor.log.info("gallery-dl is up to date (%s)", version.__version__) if kwdict["_check"]: return self.extractor.log.info( - "A new release is available: %s -> %s", - version.__version__, kwdict["tag_name"]) + "A new release is available: %s -> %s", version.__version__, kwdict["tag_name"] + ) - self.extractor.log.info( - "Updating from %s to %s", - version.__version__, kwdict["tag_name"]) + self.extractor.log.info("Updating from %s to %s", version.__version__, kwdict["tag_name"]) path_old = sys.executable + ".old" path_new = sys.executable + ".new" @@ -98,10 +95,13 @@ def handle_url(self, url, kwdict): import atexit import subprocess - cmd = 'ping 127.0.0.1 -n 5 -w 1000 & del /F "{}"'.format(path_old) + cmd = f'ping 127.0.0.1 -n 5 -w 1000 & del /F "{path_old}"' atexit.register( - util.Popen, cmd, shell=True, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + util.Popen, + cmd, + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, ) else: @@ -193,8 +193,7 @@ def items(self): raise exception.StopExtraction("Invalid channel '%s'", repo) path_tag = tag if tag == "latest" else "tags/" + tag - url = "{}/repos/{}/releases/{}".format( - self.root_api, path_repo, path_tag) + url = f"{self.root_api}/repos/{path_repo}/releases/{path_tag}" headers = { "Accept": "application/vnd.github+json", "User-Agent": util.USERAGENT, @@ -204,15 +203,14 @@ def items(self): data["_check"] = check data["_exact"] = exact - if binary == "linux" and \ - repo != "stable" and \ - data["tag_name"] <= "2024.05.28": + if binary == "linux" and repo != "stable" and data["tag_name"] <= "2024.05.28": binary_name = "gallery-dl_ubuntu" else: binary_name = BINARIES[repo][binary] url = "{}/{}/releases/download/{}/{}".format( - self.root, path_repo, data["tag_name"], binary_name) + self.root, path_repo, data["tag_name"], binary_name + ) yield Message.Directory, data yield Message.Url, url, data diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 44ac22e23..91f7e697a 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,24 +6,29 @@ """Utility functions and classes""" -import re -import os -import sys -import json -import time -import random -import getpass -import hashlib import binascii +import collections import datetime import functools +import getpass +import hashlib import itertools +import json +import os +import random +import re import subprocess -import collections +import sys +import time import urllib.parse +from contextlib import suppress +from email.utils import mktime_tz +from email.utils import parsedate_tz from http.cookiejar import Cookie -from email.utils import mktime_tz, parsedate_tz -from . import text, version, exception + +from . import exception +from . import text +from . import version def bencode(num, alphabet="0123456789"): @@ -49,7 +52,7 @@ def bdecode(data, alphabet="0123456789"): def advance(iterable, num): - """"Advance 'iterable' by 'num' steps""" + """ "Advance 'iterable' by 'num' steps""" iterator = iter(iterable) next(itertools.islice(iterator, num, num), None) return iterator @@ -89,16 +92,15 @@ def contains(values, elements, separator=" "): if not isinstance(elements, (tuple, list)): return elements in values - for e in elements: - if e in values: - return True - return False + return any(e in values for e in elements) def raises(cls): """Returns a function that raises 'cls' as exception""" + def wrap(*args): raise cls(*args) + return wrap @@ -151,8 +153,7 @@ def format_value(value, suffixes="kMGTPEZY"): index = value_len - 4 if index >= 0: offset = (value_len - 1) % 3 + 1 - return (value[:offset] + "." + value[offset:offset+2] + - suffixes[index // 3]) + return value[:offset] + "." + value[offset : offset + 2] + suffixes[index // 3] return value @@ -193,9 +194,9 @@ def enumerate_reversed(iterable, start=0, length=None): length = len(iterable) try: - iterable = zip(range(start-1+length, start-1, -1), reversed(iterable)) + iterable = zip(range(start - 1 + length, start - 1, -1), reversed(iterable)) except TypeError: - iterable = list(zip(range(start, start+length), iterable)) + iterable = list(zip(range(start, start + length), iterable)) iterable.reverse() return iterable @@ -231,7 +232,7 @@ def datetime_to_timestamp_string(dt): return "" -if sys.hexversion < 0x30c0000: +if sys.hexversion < 0x30C0000: # Python <= 3.11 datetime_utcfromtimestamp = datetime.datetime.utcfromtimestamp datetime_utcnow = datetime.datetime.utcnow @@ -264,7 +265,8 @@ def json_default(obj): def dump_json(obj, fp=sys.stdout, ensure_ascii=True, indent=4): """Serialize 'obj' as JSON and write it to 'fp'""" json.dump( - obj, fp, + obj, + fp, ensure_ascii=ensure_ascii, indent=indent, default=json_default, @@ -308,30 +310,31 @@ def dump_response(response, fp, headers=False, content=True, hide_auth=True): cookie = req_headers.get("Cookie") if cookie: req_headers["Cookie"] = ";".join( - c.partition("=")[0] + "=***" - for c in cookie.split(";") + c.partition("=")[0] + "=***" for c in cookie.split(";") ) set_cookie = res_headers.get("Set-Cookie") if set_cookie: res_headers["Set-Cookie"] = re.sub( - r"(^|, )([^ =]+)=[^,;]*", r"\1\2=***", set_cookie, + r"(^|, )([^ =]+)=[^,;]*", + r"\1\2=***", + set_cookie, ) fmt_nv = "{}: {}".format - fp.write(outfmt.format( - request=request, - response=response, - request_headers="\n".join( - fmt_nv(name, value) - for name, value in req_headers.items() - ), - response_headers="\n".join( - fmt_nv(name, value) - for name, value in res_headers.items() - ), - ).encode()) + fp.write( + outfmt.format( + request=request, + response=response, + request_headers="\n".join( + fmt_nv(name, value) for name, value in req_headers.items() + ), + response_headers="\n".join( + fmt_nv(name, value) for name, value in res_headers.items() + ), + ).encode() + ) if content: if headers: @@ -356,7 +359,7 @@ def extract_headers(response): return data -@functools.lru_cache(maxsize=None) +@functools.cache def git_head(): try: out, err = Popen( @@ -382,17 +385,13 @@ def expand_path(path): def remove_file(path): - try: + with suppress(OSError): os.unlink(path) - except OSError: - pass def remove_directory(path): - try: + with suppress(OSError): os.rmdir(path) - except OSError: - pass def set_mtime(path, mtime): @@ -409,7 +408,6 @@ def cookiestxt_load(fp): cookies = [] for line in fp: - line = line.lstrip(" ") # strip '#HttpOnly_' if line.startswith("#HttpOnly_"): @@ -421,24 +419,32 @@ def cookiestxt_load(fp): if line[-1] == "\n": line = line[:-1] - domain, domain_specified, path, secure, expires, name, value = \ - line.split("\t") + domain, domain_specified, path, secure, expires, name, value = line.split("\t") if not name: name = value value = None - cookies.append(Cookie( - 0, name, value, - None, False, - domain, - domain_specified == "TRUE", - domain[0] == "." if domain else False, - path, False, - secure == "TRUE", - None if expires == "0" or not expires else expires, - False, None, None, {}, - )) + cookies.append( + Cookie( + 0, + name, + value, + None, + False, + domain, + domain_specified == "TRUE", + domain[0] == "." if domain else False, + path, + False, + secure == "TRUE", + None if expires == "0" or not expires else expires, + False, + None, + None, + {}, + ) + ) return cookies @@ -460,15 +466,19 @@ def cookiestxt_store(fp, cookies): value = cookie.value domain = cookie.domain - write("\t".join(( - domain, - "TRUE" if domain and domain[0] == "." else "FALSE", - cookie.path, - "TRUE" if cookie.secure else "FALSE", - "0" if cookie.expires is None else str(cookie.expires), - name, - value + "\n", - ))) + write( + "\t".join( + ( + domain, + "TRUE" if domain and domain[0] == "." else "FALSE", + cookie.path, + "TRUE" if cookie.secure else "FALSE", + "0" if cookie.expires is None else str(cookie.expires), + name, + value + "\n", + ) + ) + ) def code_to_language(code, default=None): @@ -520,20 +530,23 @@ def language_to_code(lang, default=None): } -class HTTPBasicAuth(): +class HTTPBasicAuth: __slots__ = ("authorization",) def __init__(self, username, password): - self.authorization = b"Basic " + binascii.b2a_base64( - username.encode("latin1") + b":" + str(password).encode("latin1") - )[:-1] + self.authorization = ( + b"Basic " + + binascii.b2a_base64( + username.encode("latin1") + b":" + str(password).encode("latin1") + )[:-1] + ) def __call__(self, request): request.headers["Authorization"] = self.authorization return request -class ModuleProxy(): +class ModuleProxy: __slots__ = () def __getitem__(self, key, modules=sys.modules): @@ -551,14 +564,14 @@ def __getitem__(self, key, modules=sys.modules): __getattr__ = __getitem__ -class LazyPrompt(): +class LazyPrompt: __slots__ = () def __str__(self): return getpass.getpass() -class NullContext(): +class NullContext: __slots__ = () def __enter__(self): @@ -568,8 +581,9 @@ def __exit__(self, exc_type, exc_value, traceback): pass -class CustomNone(): +class CustomNone: """None-style type that supports more operations than regular None""" + __slots__ = () __getattribute__ = identity @@ -650,24 +664,24 @@ def __str__(): NONE = CustomNone() EPOCH = datetime.datetime(1970, 1, 1) SECOND = datetime.timedelta(0, 1) -WINDOWS = (os.name == "nt") +WINDOWS = os.name == "nt" SENTINEL = object() USERAGENT = "gallery-dl/" + version.__version__ EXECUTABLE = getattr(sys, "frozen", False) SPECIAL_EXTRACTORS = {"oauth", "recursive", "generic"} GLOBALS = { - "contains" : contains, + "contains": contains, "parse_int": text.parse_int, - "urlsplit" : urllib.parse.urlsplit, - "datetime" : datetime.datetime, + "urlsplit": urllib.parse.urlsplit, + "datetime": datetime.datetime, "timedelta": datetime.timedelta, - "abort" : raises(exception.StopExtraction), + "abort": raises(exception.StopExtraction), "terminate": raises(exception.TerminateExtraction), - "restart" : raises(exception.RestartExtraction), + "restart": raises(exception.RestartExtraction), "hash_sha1": sha1, - "hash_md5" : md5, - "std" : ModuleProxy(), - "re" : re, + "hash_md5": md5, + "std": ModuleProxy(), + "re": re, } @@ -703,7 +717,7 @@ def compile_expression_raw(expr, name="", globals=None): return functools.partial(eval, code_object, globals or GLOBALS) -def compile_expression_defaultdict(expr, name="", globals=None): +def compile_expression_defaultdict(expr, name="", globals=None): # noqa: F811 global GLOBALS_DEFAULT GLOBALS_DEFAULT = collections.defaultdict(lambda: NONE, GLOBALS) @@ -778,13 +792,11 @@ def build_duration_func(duration, min=0.0): upper = float(upper) return functools.partial( random.uniform, - lower if lower > min else min, - upper if upper > min else min, + max(min, lower), + max(min, upper), ) - else: - if lower < min: - lower = min - return lambda: lower + lower = max(lower, min) + return lambda: lower def build_extractor_filter(categories, negate=True, special=None): @@ -796,7 +808,7 @@ def build_extractor_filter(categories, negate=True, special=None): catset = set() # set of categories / basecategories subset = set() # set of subcategories - catsub = [] # list of category-subcategory pairs + catsub = [] # list of category-subcategory pairs for item in categories: category, _, subcategory = item.partition(":") @@ -817,35 +829,34 @@ def build_extractor_filter(categories, negate=True, special=None): if negate: if catset: - tests.append(lambda extr: - extr.category not in catset and - extr.basecategory not in catset) + tests.append( + lambda extr: extr.category not in catset and extr.basecategory not in catset + ) if subset: tests.append(lambda extr: extr.subcategory not in subset) else: if catset: - tests.append(lambda extr: - extr.category in catset or - extr.basecategory in catset) + tests.append(lambda extr: extr.category in catset or extr.basecategory in catset) if subset: tests.append(lambda extr: extr.subcategory in subset) if catsub: + def test(extr): for category, subcategory in catsub: if subcategory == extr.subcategory and ( - category == extr.category or - category == extr.basecategory): + category == extr.category or category == extr.basecategory + ): return not negate return negate + tests.append(test) if len(tests) == 1: return tests[0] if negate: return lambda extr: all(t(extr) for t in tests) - else: - return lambda extr: any(t(extr) for t in tests) + return lambda extr: any(t(extr) for t in tests) def build_proxy_map(proxies, log=None): @@ -871,19 +882,16 @@ def build_proxy_map(proxies, log=None): def build_predicate(predicates): if not predicates: return lambda url, kwdict: True - elif len(predicates) == 1: + if len(predicates) == 1: return predicates[0] return functools.partial(chain_predicates, predicates) def chain_predicates(predicates, url, kwdict): - for pred in predicates: - if not pred(url, kwdict): - return False - return True + return all(pred(url, kwdict) for pred in predicates) -class RangePredicate(): +class RangePredicate: """Predicate; True if the current index is in the given range(s)""" def __init__(self, rangespec): @@ -905,10 +913,7 @@ def __call__(self, _url, _kwdict): if index > self.upper: raise exception.StopExtraction() - for range in self.ranges: - if index in range: - return True - return False + return any(index in range for range in self.ranges) @staticmethod def _parse(rangespec): @@ -929,31 +934,36 @@ def _parse(rangespec): if not group: continue - elif ":" in group: + if ":" in group: start, _, stop = group.partition(":") stop, _, step = stop.partition(":") - append(range( - int(start) if start.strip() else 1, - int(stop) if stop.strip() else sys.maxsize, - int(step) if step.strip() else 1, - )) + append( + range( + int(start) if start.strip() else 1, + int(stop) if stop.strip() else sys.maxsize, + int(step) if step.strip() else 1, + ) + ) elif "-" in group: start, _, stop = group.partition("-") - append(range( - int(start) if start.strip() else 1, - int(stop) + 1 if stop.strip() else sys.maxsize, - )) + append( + range( + int(start) if start.strip() else 1, + int(stop) + 1 if stop.strip() else sys.maxsize, + ) + ) else: start = int(group) - append(range(start, start+1)) + append(range(start, start + 1)) return ranges -class UniquePredicate(): +class UniquePredicate: """Predicate; True if given URL has not been encountered before""" + def __init__(self): self.urls = set() @@ -966,11 +976,11 @@ def __call__(self, url, _): return False -class FilterPredicate(): +class FilterPredicate: """Predicate; True if evaluating the given expression returns True""" def __init__(self, expr, target="image"): - name = "<{} filter>".format(target) + name = f"<{target} filter>" self.expr = compile_filter(expr, name) def __call__(self, _, kwdict): diff --git a/gallery_dl/version.py b/gallery_dl/version.py index f85f22192..31cb5f879 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify diff --git a/gallery_dl/ytdl.py b/gallery_dl/ytdl.py index fe88c2cb4..7bd48f643 100644 --- a/gallery_dl/ytdl.py +++ b/gallery_dl/ytdl.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,10 +6,13 @@ """Helpers for interacting with youtube-dl""" +import itertools import re import shlex -import itertools -from . import text, util, exception + +from . import exception +from . import text +from . import util def import_module(module_name): @@ -71,7 +72,7 @@ def construct_YoutubeDL(module, obj, user_opts, system_opts=None): def parse_command_line(module, argv): parser, opts, args = module.parseOpts(argv) - ytdlp = (module.__name__ == "yt_dlp") + ytdlp = module.__name__ == "yt_dlp" std_headers = module.std_headers try: @@ -126,8 +127,7 @@ def parse_command_line(module, argv): opts.remuxvideo = opts.remuxvideo.replace(" ", "") if getattr(opts, "wait_for_video", None) is not None: min_wait, _, max_wait = opts.wait_for_video.partition("-") - opts.wait_for_video = (module.parse_duration(min_wait), - module.parse_duration(max_wait)) + opts.wait_for_video = (module.parse_duration(min_wait), module.parse_duration(max_wait)) if opts.date is not None: date = module.DateRange.day(opts.date) @@ -140,21 +140,19 @@ def _unused_compat_opt(name): if name not in compat_opts: return False compat_opts.discard(name) - compat_opts.update(["*%s" % name]) + compat_opts.update([f"*{name}"]) return True - def set_default_compat( - compat_name, opt_name, default=True, remove_compat=True): + def set_default_compat(compat_name, opt_name, default=True, remove_compat=True): attr = getattr(opts, opt_name, None) if compat_name in compat_opts: if attr is None: setattr(opts, opt_name, not default) return True - else: - if remove_compat: - _unused_compat_opt(compat_name) - return False - elif attr is None: + if remove_compat: + _unused_compat_opt(compat_name) + return False + if attr is None: setattr(opts, opt_name, default) return None @@ -164,11 +162,11 @@ def set_default_compat( if "format-sort" in compat_opts: opts.format_sort.extend(module.InfoExtractor.FormatSort.ytdl_default) _video_multistreams_set = set_default_compat( - "multistreams", "allow_multiple_video_streams", - False, remove_compat=False) + "multistreams", "allow_multiple_video_streams", False, remove_compat=False + ) _audio_multistreams_set = set_default_compat( - "multistreams", "allow_multiple_audio_streams", - False, remove_compat=False) + "multistreams", "allow_multiple_audio_streams", False, remove_compat=False + ) if _video_multistreams_set is False and _audio_multistreams_set is False: _unused_compat_opt("multistreams") @@ -188,6 +186,7 @@ def set_default_compat( opts.format = "bestaudio/best" if ytdlp: + def metadataparser_actions(f): if isinstance(f, str): yield module.MetadataFromFieldPP.to_action(f) @@ -204,27 +203,29 @@ def metadataparser_actions(f): if opts.metafromtitle is not None: if "pre_process" not in parse_metadata: parse_metadata["pre_process"] = [] - parse_metadata["pre_process"].append( - "title:%s" % opts.metafromtitle) + parse_metadata["pre_process"].append(f"title:{opts.metafromtitle}") opts.parse_metadata = { - k: list(itertools.chain.from_iterable(map( - metadataparser_actions, v))) + k: list(itertools.chain.from_iterable(map(metadataparser_actions, v))) for k, v in parse_metadata.items() } else: if parse_metadata is None: parse_metadata = [] if opts.metafromtitle is not None: - parse_metadata.append("title:%s" % opts.metafromtitle) - opts.parse_metadata = list(itertools.chain.from_iterable(map( - metadataparser_actions, parse_metadata))) + parse_metadata.append(f"title:{opts.metafromtitle}") + opts.parse_metadata = list( + itertools.chain.from_iterable(map(metadataparser_actions, parse_metadata)) + ) opts.metafromtitle = None else: opts.parse_metadata = () - download_archive_fn = module.expand_path(opts.download_archive) \ - if opts.download_archive is not None else opts.download_archive + download_archive_fn = ( + module.expand_path(opts.download_archive) + if opts.download_archive is not None + else opts.download_archive + ) if getattr(opts, "getcomments", None): opts.writeinfojson = True @@ -233,30 +234,30 @@ def metadataparser_actions(f): opts.sponsorblock_mark = set() opts.sponsorblock_remove = set() else: - opts.sponsorblock_mark = \ - getattr(opts, "sponsorblock_mark", None) or set() - opts.sponsorblock_remove = \ - getattr(opts, "sponsorblock_remove", None) or set() + opts.sponsorblock_mark = getattr(opts, "sponsorblock_mark", None) or set() + opts.sponsorblock_remove = getattr(opts, "sponsorblock_remove", None) or set() opts.remove_chapters = getattr(opts, "remove_chapters", None) or () try: postprocessors = list(module.get_postprocessors(opts)) except AttributeError: - postprocessors = legacy_postprocessors( - opts, module, ytdlp, compat_opts) + postprocessors = legacy_postprocessors(opts, module, ytdlp, compat_opts) match_filter = ( - None if opts.match_filter is None - else module.match_filter_func(opts.match_filter)) + None if opts.match_filter is None else module.match_filter_func(opts.match_filter) + ) cookiesfrombrowser = getattr(opts, "cookiesfrombrowser", None) if cookiesfrombrowser: - match = re.fullmatch(r"""(?x) + match = re.fullmatch( + r"""(?x) (?P[^+:]+) (?:\s*\+\s*(?P[^:]+))? (?:\s*:\s*(?!:)(?P.+?))? (?:\s*::\s*(?P.+))? - """, cookiesfrombrowser) + """, + cookiesfrombrowser, + ) if match: browser, keyring, profile, container = match.groups() if keyring is not None: @@ -286,23 +287,17 @@ def metadataparser_actions(f): "forcefilename": opts.getfilename, "forceformat": opts.getformat, "forceprint": getattr(opts, "forceprint", None) or (), - "force_write_download_archive": getattr( - opts, "force_write_download_archive", None), + "force_write_download_archive": getattr(opts, "force_write_download_archive", None), "simulate": opts.simulate, "skip_download": opts.skip_download, "format": opts.format, - "allow_unplayable_formats": getattr( - opts, "allow_unplayable_formats", None), - "ignore_no_formats_error": getattr( - opts, "ignore_no_formats_error", None), - "format_sort": getattr( - opts, "format_sort", None), - "format_sort_force": getattr( - opts, "format_sort_force", None), + "allow_unplayable_formats": getattr(opts, "allow_unplayable_formats", None), + "ignore_no_formats_error": getattr(opts, "ignore_no_formats_error", None), + "format_sort": getattr(opts, "format_sort", None), + "format_sort_force": getattr(opts, "format_sort_force", None), "allow_multiple_video_streams": opts.allow_multiple_video_streams, "allow_multiple_audio_streams": opts.allow_multiple_audio_streams, - "check_formats": getattr( - opts, "check_formats", None), + "check_formats": getattr(opts, "check_formats", None), "outtmpl": opts.outtmpl, "outtmpl_na_placeholder": opts.outtmpl_na_placeholder, "paths": getattr(opts, "paths", None), @@ -321,8 +316,7 @@ def metadataparser_actions(f): "extractor_retries": getattr(opts, "extractor_retries", None), "skip_unavailable_fragments": opts.skip_unavailable_fragments, "keep_fragments": opts.keep_fragments, - "concurrent_fragment_downloads": getattr( - opts, "concurrent_fragment_downloads", None), + "concurrent_fragment_downloads": getattr(opts, "concurrent_fragment_downloads", None), "buffersize": opts.buffersize, "noresizebuffer": opts.noresizebuffer, "http_chunk_size": opts.http_chunk_size, @@ -344,8 +338,8 @@ def metadataparser_actions(f): "clean_infojson": opts.clean_infojson, "getcomments": getattr(opts, "getcomments", None), "writethumbnail": opts.writethumbnail is True, - "write_all_thumbnails": getattr(opts, "write_all_thumbnails", None) or - opts.writethumbnail == "all", + "write_all_thumbnails": getattr(opts, "write_all_thumbnails", None) + or opts.writethumbnail == "all", "writelink": getattr(opts, "writelink", None), "writeurllink": getattr(opts, "writeurllink", None), "writewebloclink": getattr(opts, "writewebloclink", None), @@ -377,8 +371,7 @@ def metadataparser_actions(f): "break_on_existing": getattr(opts, "break_on_existing", None), "break_on_reject": getattr(opts, "break_on_reject", None), "break_per_url": getattr(opts, "break_per_url", None), - "skip_playlist_after_errors": getattr( - opts, "skip_playlist_after_errors", None), + "skip_playlist_after_errors": getattr(opts, "skip_playlist_after_errors", None), "cookiefile": opts.cookiefile, "cookiesfrombrowser": cookiesfrombrowser, "nocheckcertificate": opts.no_check_certificate, @@ -392,10 +385,8 @@ def metadataparser_actions(f): "default_search": opts.default_search, "dynamic_mpd": getattr(opts, "dynamic_mpd", None), "extractor_args": getattr(opts, "extractor_args", None), - "youtube_include_dash_manifest": getattr( - opts, "youtube_include_dash_manifest", None), - "youtube_include_hls_manifest": getattr( - opts, "youtube_include_hls_manifest", None), + "youtube_include_dash_manifest": getattr(opts, "youtube_include_dash_manifest", None), + "youtube_include_hls_manifest": getattr(opts, "youtube_include_hls_manifest", None), "encoding": opts.encoding, "extract_flat": opts.extract_flat, "live_from_start": getattr(opts, "live_from_start", None), @@ -405,12 +396,10 @@ def metadataparser_actions(f): "postprocessors": postprocessors, "fixup": opts.fixup, "source_address": opts.source_address, - "sleep_interval_requests": getattr( - opts, "sleep_interval_requests", None), + "sleep_interval_requests": getattr(opts, "sleep_interval_requests", None), "sleep_interval": opts.sleep_interval, "max_sleep_interval": opts.max_sleep_interval, - "sleep_interval_subtitles": getattr( - opts, "sleep_interval_subtitles", None), + "sleep_interval_subtitles": getattr(opts, "sleep_interval_subtitles", None), "external_downloader": opts.external_downloader, "playlist_items": opts.playlist_items, "xattr_set_filesize": opts.xattr_set_filesize, @@ -419,18 +408,14 @@ def metadataparser_actions(f): "ffmpeg_location": opts.ffmpeg_location, "hls_prefer_native": opts.hls_prefer_native, "hls_use_mpegts": opts.hls_use_mpegts, - "hls_split_discontinuity": getattr( - opts, "hls_split_discontinuity", None), + "hls_split_discontinuity": getattr(opts, "hls_split_discontinuity", None), "external_downloader_args": opts.external_downloader_args, "postprocessor_args": opts.postprocessor_args, "cn_verification_proxy": opts.cn_verification_proxy, "geo_verification_proxy": opts.geo_verification_proxy, - "geo_bypass": getattr( - opts, "geo_bypass", "default"), - "geo_bypass_country": getattr( - opts, "geo_bypass_country", None), - "geo_bypass_ip_block": getattr( - opts, "geo_bypass_ip_block", None), + "geo_bypass": getattr(opts, "geo_bypass", "default"), + "geo_bypass_country": getattr(opts, "geo_bypass_country", None), + "geo_bypass_ip_block": getattr(opts, "geo_bypass_ip_block", None), "compat_opts": compat_opts, } @@ -446,65 +431,79 @@ def legacy_postprocessors(opts, module, ytdlp, compat_opts): sponsorblock_query = opts.sponsorblock_mark | opts.sponsorblock_remove if opts.metafromtitle: - postprocessors.append({ - "key": "MetadataFromTitle", - "titleformat": opts.metafromtitle, - }) + postprocessors.append( + { + "key": "MetadataFromTitle", + "titleformat": opts.metafromtitle, + } + ) if getattr(opts, "add_postprocessors", None): postprocessors += list(opts.add_postprocessors) if sponsorblock_query: - postprocessors.append({ - "key": "SponsorBlock", - "categories": sponsorblock_query, - "api": opts.sponsorblock_api, - "when": "pre_process", - }) + postprocessors.append( + { + "key": "SponsorBlock", + "categories": sponsorblock_query, + "api": opts.sponsorblock_api, + "when": "pre_process", + } + ) if opts.parse_metadata: - postprocessors.append({ - "key": "MetadataParser", - "actions": opts.parse_metadata, - "when": "pre_process", - }) + postprocessors.append( + { + "key": "MetadataParser", + "actions": opts.parse_metadata, + "when": "pre_process", + } + ) if opts.convertsubtitles: - pp = {"key": "FFmpegSubtitlesConvertor", - "format": opts.convertsubtitles} + pp = {"key": "FFmpegSubtitlesConvertor", "format": opts.convertsubtitles} if ytdlp: pp["when"] = "before_dl" postprocessors.append(pp) if getattr(opts, "convertthumbnails", None): - postprocessors.append({ - "key": "FFmpegThumbnailsConvertor", - "format": opts.convertthumbnails, - "when": "before_dl", - }) + postprocessors.append( + { + "key": "FFmpegThumbnailsConvertor", + "format": opts.convertthumbnails, + "when": "before_dl", + } + ) if getattr(opts, "exec_before_dl_cmd", None): - postprocessors.append({ - "key": "Exec", - "exec_cmd": opts.exec_before_dl_cmd, - "when": "before_dl", - }) + postprocessors.append( + { + "key": "Exec", + "exec_cmd": opts.exec_before_dl_cmd, + "when": "before_dl", + } + ) if opts.extractaudio: - postprocessors.append({ - "key": "FFmpegExtractAudio", - "preferredcodec": opts.audioformat, - "preferredquality": opts.audioquality, - "nopostoverwrites": opts.nopostoverwrites, - }) + postprocessors.append( + { + "key": "FFmpegExtractAudio", + "preferredcodec": opts.audioformat, + "preferredquality": opts.audioquality, + "nopostoverwrites": opts.nopostoverwrites, + } + ) if getattr(opts, "remuxvideo", None): - postprocessors.append({ - "key": "FFmpegVideoRemuxer", - "preferedformat": opts.remuxvideo, - }) + postprocessors.append( + { + "key": "FFmpegVideoRemuxer", + "preferedformat": opts.remuxvideo, + } + ) if opts.recodevideo: - postprocessors.append({ - "key": "FFmpegVideoConvertor", - "preferedformat": opts.recodevideo, - }) + postprocessors.append( + { + "key": "FFmpegVideoConvertor", + "preferedformat": opts.recodevideo, + } + ) if opts.embedsubtitles: pp = {"key": "FFmpegEmbedSubtitle"} if ytdlp: - pp["already_have_subtitle"] = ( - opts.writesubtitles and "no-keep-subs" not in compat_opts) + pp["already_have_subtitle"] = opts.writesubtitles and "no-keep-subs" not in compat_opts postprocessors.append(pp) if not opts.writeautomaticsub and "no-keep-subs" not in compat_opts: opts.writesubtitles = True @@ -519,14 +518,16 @@ def legacy_postprocessors(opts, module, ytdlp, compat_opts): continue remove_chapters_patterns.append(re.compile(regex)) if opts.remove_chapters or sponsorblock_query: - postprocessors.append({ - "key": "ModifyChapters", - "remove_chapters_patterns": remove_chapters_patterns, - "remove_sponsor_segments": opts.sponsorblock_remove, - "remove_ranges": remove_ranges, - "sponsorblock_chapter_title": opts.sponsorblock_chapter_title, - "force_keyframes": opts.force_keyframes_at_cuts, - }) + postprocessors.append( + { + "key": "ModifyChapters", + "remove_chapters_patterns": remove_chapters_patterns, + "remove_sponsor_segments": opts.sponsorblock_remove, + "remove_ranges": remove_ranges, + "sponsorblock_chapter_title": opts.sponsorblock_chapter_title, + "force_keyframes": opts.force_keyframes_at_cuts, + } + ) addchapters = getattr(opts, "addchapters", None) embed_infojson = getattr(opts, "embed_infojson", None) if opts.addmetadata or addchapters or embed_infojson: @@ -540,38 +541,45 @@ def legacy_postprocessors(opts, module, ytdlp, compat_opts): postprocessors.append(pp) if getattr(opts, "sponskrub", False) is not False: - postprocessors.append({ - "key": "SponSkrub", - "path": opts.sponskrub_path, - "args": opts.sponskrub_args, - "cut": opts.sponskrub_cut, - "force": opts.sponskrub_force, - "ignoreerror": opts.sponskrub is None, - "_from_cli": True, - }) + postprocessors.append( + { + "key": "SponSkrub", + "path": opts.sponskrub_path, + "args": opts.sponskrub_args, + "cut": opts.sponskrub_cut, + "force": opts.sponskrub_force, + "ignoreerror": opts.sponskrub is None, + "_from_cli": True, + } + ) if opts.embedthumbnail: - already_have_thumbnail = (opts.writethumbnail or - getattr(opts, "write_all_thumbnails", False)) - postprocessors.append({ - "key": "EmbedThumbnail", - "already_have_thumbnail": already_have_thumbnail, - }) + already_have_thumbnail = opts.writethumbnail or getattr(opts, "write_all_thumbnails", False) + postprocessors.append( + { + "key": "EmbedThumbnail", + "already_have_thumbnail": already_have_thumbnail, + } + ) if not already_have_thumbnail: opts.writethumbnail = True if isinstance(opts.outtmpl, dict): opts.outtmpl["pl_thumbnail"] = "" if getattr(opts, "split_chapters", None): - postprocessors.append({ - "key": "FFmpegSplitChapters", - "force_keyframes": opts.force_keyframes_at_cuts, - }) + postprocessors.append( + { + "key": "FFmpegSplitChapters", + "force_keyframes": opts.force_keyframes_at_cuts, + } + ) if opts.xattrs: postprocessors.append({"key": "XAttrMetadata"}) if opts.exec_cmd: - postprocessors.append({ - "key": "Exec", - "exec_cmd": opts.exec_cmd, - "when": "after_move", - }) + postprocessors.append( + { + "key": "Exec", + "exec_cmd": opts.exec_cmd, + "when": "after_move", + } + ) return postprocessors diff --git a/pyproject.toml b/pyproject.toml index fed528d4a..ea2947a6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,108 @@ [build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" + build-backend = "setuptools.build_meta" + requires = ["setuptools"] + + +# ============================================================================ # +# Ruff # +# ============================================================================ # +# +# Reference: +# - [Configuration](https://docs.astral.sh/ruff/configuration/) +# +# - [Settings](https://docs.astral.sh/ruff/settings) +# - [Rules](https://docs.astral.sh/ruff/rules/) +# +[tool.ruff] + + # Set the maximum line length to 100 + # Reference: https://knox.codes/posts/line-length-limits + line-length = 100 + + # Target Python 3.9 (the latest supported version) + # Reference: https://devguide.python.org/versions/ + target-version = "py39" + +[tool.ruff.lint] + select = [ + "C4", # flake8-comprehensions + "COM", # flake8-commas + "E", # pycodestyle (errors) + "F", # Pyflakes + "FA", # flake8-future-annotations + "FLY", # flynt + "FURB", # refurb + "G", # flake8-logging-format + "I", # isort + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PYI", # flake8-pyi + "Q", # flake8-quotes + "SIM", # flake8-simplify + "T10", # flake8-debugger + "TCH", # flake8-type-checking + "UP", # pyupgrade + + # The following checks are recommended but would require a lot of changes: + # "A", # flake8-builtins + # "ANN", # flake8-annotations + # "ARG", # flake8-unused-arguments + # "B", # flake8-bugbear + # "BLE", # flake8-blind-except + # "C", # mccabe (complexity) + # "D", # pydocstyle + # "DTZ", # flake8-datetimez + # "EM", # flake8-errmsg + # "ERA", # eradicate + # "EXE", # flake8-executable + # "FBT", # flake8-boolean-trap + # "INP", # flake8-no-pep420 + # "N", # pep8-naming + # "PL", # Pylint + # "PTH", # flake8-use-pathlib + # "RET", # flake8-return + # "RUF", # Ruff-specific rules + # "S", # flake8-bandit + # "SLF", # flake8-self + # "T20", # flake8-print + # "TD", # flake8-todos + # "W", # pycodestyle (warnings) + ] + + # ignore: + ignore = [ + "ANN101", # Missing type annotation for self in method (otherwise it's just t.Self) + "ANN102", # Missing type annotation for cls in classmethod + "COM812", # Trailing comma missing; ignored for compatibility with the Ruff formatter + "ISC001", # Implicitly concatenated string literals on one line; ignored for compatibility with the Ruff formatter + "SIM115", # Use a context manager for opening files; code is structured to return open file handle + ] + +[tool.ruff.lint.isort] + + # Each import should be on a separate line to minimize merge conflicts (see + # [reorder_python_imports](https://github.com/asottile/reorder_python_imports#why-this-style)). + force-single-line = true + known-first-party = ["gallery_dl"] + +[tool.ruff.lint.per-file-ignores] + "gallery_dl/extractor/500px.py" = [ + "E501", # Line too long; file uses large query strings + ] + "gallery_dl/extractor/mangapark.py" = [ + "E501", # Line too long; file uses large query strings + ] + "test/**" = [ + "E501", # Line too long + ] + +[tool.ruff.lint.pydocstyle] + # Use Google-style docstrings (https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). + convention = "google" + +[tool.ruff.format] + # Enable auto-formatting of code examples in docstrings. Markdown, + # reStructuredText code/literal blocks and doctests are all supported. + docstring-code-format = true diff --git a/scripts/build_testresult_db.py b/scripts/build_testresult_db.py index 85b332e09..247ebdbc1 100755 --- a/scripts/build_testresult_db.py +++ b/scripts/build_testresult_db.py @@ -1,26 +1,25 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """Collect results of extractor unit tests""" -import sys -import os.path import datetime +import os.path +import sys import util -from gallery_dl import extractor, job, config -from test.test_results import setup_test_config +from gallery_dl import config +from gallery_dl import extractor +from gallery_dl import job +from test.test_results import setup_test_config # filter test cases tests = [ (idx, extr, url, result) - for extr in extractor.extractors() if hasattr(extr, "test") and extr.test if len(sys.argv) <= 1 or extr.category in sys.argv - for idx, (url, result) in enumerate(extr._get_tests()) if result ] @@ -33,9 +32,8 @@ for idx, extr, url, result in tests: - # filename - name = "{}-{}-{}.json".format(extr.category, extr.subcategory, idx) + name = f"{extr.category}-{extr.subcategory}-{idx}.json" print(name) # config values @@ -46,7 +44,7 @@ key = key.split(".") config.set(key[:-1], key[-1], value) if "range" in result: - config.set((), "image-range" , result["range"]) + config.set((), "image-range", result["range"]) config.set((), "chapter-range", result["range"]) # write test data diff --git a/scripts/completion_bash.py b/scripts/completion_bash.py index b9921dc07..b92192328 100755 --- a/scripts/completion_bash.py +++ b/scripts/completion_bash.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019 Mike Fährmann # @@ -10,8 +9,8 @@ """Generate bash completion script from gallery-dl's argument parser""" import util -from gallery_dl import option +from gallery_dl import option TEMPLATE = """_gallery_dl() { @@ -36,7 +35,6 @@ diropts = [] fileopts = [] for action in option.build_parser()._actions: - if action.metavar in ("DEST",): diropts.extend(action.option_strings) @@ -49,8 +47,11 @@ PATH = util.path("data/completion/gallery-dl") with util.lazy(PATH) as fp: - fp.write(TEMPLATE % { - "opts" : " ".join(opts), - "diropts" : "|".join(diropts), - "fileopts": "|".join(fileopts), - }) + fp.write( + TEMPLATE + % { + "opts": " ".join(opts), + "diropts": "|".join(diropts), + "fileopts": "|".join(fileopts), + } + ) diff --git a/scripts/completion_fish.py b/scripts/completion_fish.py index a5dd47b95..92c7c0cb8 100755 --- a/scripts/completion_fish.py +++ b/scripts/completion_fish.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -8,8 +7,8 @@ """Generate fish completion script from gallery-dl's argument parser""" import util -from gallery_dl import option +from gallery_dl import option TEMPLATE = """complete -c gallery-dl -x %(opts)s diff --git a/scripts/completion_zsh.py b/scripts/completion_zsh.py index d96ed7334..a617daa21 100755 --- a/scripts/completion_zsh.py +++ b/scripts/completion_zsh.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2020 Mike Fährmann # @@ -9,8 +8,10 @@ """Generate zsh completion script from gallery-dl's argument parser""" -import util import argparse + +import util + from gallery_dl import option TEMPLATE = """#compdef gallery-dl @@ -25,16 +26,17 @@ return rc """ -TR = str.maketrans({ - "'": "'\\''", - "[": "\\[", - "]": "\\]", -}) +TR = str.maketrans( + { + "'": "'\\''", + "[": "\\[", + "]": "\\]", + } +) opts = [] for action in option.build_parser()._actions: - if not action.option_strings or action.help == argparse.SUPPRESS: continue elif len(action.option_strings) == 1: diff --git a/scripts/create_test_data.py b/scripts/create_test_data.py index 34d585ff4..eac28e809 100755 --- a/scripts/create_test_data.py +++ b/scripts/create_test_data.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2015-2019 Mike Fährmann # @@ -11,10 +10,9 @@ import argparse -import util # noqa from gallery_dl import extractor -from test.test_results import ResultJob, setup_test_config - +from test.test_results import ResultJob +from test.test_results import setup_test_config TESTDATA_FMT = """ test = ("{}", {{ @@ -41,7 +39,8 @@ def main(): if args.recreate: urls = [ test[0] - for extr in extractor.extractors() if extr.category in args.urls + for extr in extractor.extractors() + if extr.category in args.urls for test in extr.test ] else: @@ -58,12 +57,14 @@ def main(): data = (exc.__class__.__name__,) else: fmt = TESTDATA_FMT - data = (tjob.url_hash.hexdigest(), - tjob.kwdict_hash.hexdigest(), - tjob.content_hash.hexdigest()) + data = ( + tjob.url_hash.hexdigest(), + tjob.kwdict_hash.hexdigest(), + tjob.content_hash.hexdigest(), + ) print(tjob.extractor.__class__.__name__) print(fmt.format(url, *data)) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/scripts/export_tests.py b/scripts/export_tests.py index 7ea62852c..437821801 100755 --- a/scripts/export_tests.py +++ b/scripts/export_tests.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2023 Mike Fährmann # @@ -7,18 +6,18 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import collections +import itertools import os import re import sys -import itertools -import collections import util from pyprint import pyprint -from gallery_dl import extractor +from gallery_dl import extractor -FORMAT = '''\ +FORMAT = """\ # -*- coding: utf-8 -*- # This program is free software; you can redistribute it and/or modify @@ -31,34 +30,32 @@ __tests__ = ( {tests}\ ) -''' +""" def extract_tests_from_source(lines): tests = {} - match_url = re.compile( - r''' (?:test = | )?\(\(?"([^"]+)"(.*)''').match - match_end = re.compile( - r" (\}\)| \}\),)\n$").match + match_url = re.compile(r""" (?:test = | )?\(\(?"([^"]+)"(.*)""").match + match_end = re.compile(r" (\}\)| \}\),)\n$").match first = 0 url = "" for index, line in enumerate(lines): if first and match_end(line): - tests[url] = lines[first-1:index+1] + tests[url] = lines[first - 1 : index + 1] first = 0 - elif (m := match_url(line)): + elif m := match_url(line): offset = index while not m[2]: offset += 1 next = lines[offset] - line = line[:-2] + next[next.index('"')+1:] + line = line[:-2] + next[next.index('"') + 1 :] m = match_url(line) url = m[1] if m[2] in (",)", "),"): - tests[url] = lines[index-1:index+1] + tests[url] = lines[index - 1 : index + 1] first = 0 else: first = index @@ -88,12 +85,10 @@ def build_test(extr, data): comment = comment_from_source(source) head = { - "#url" : extr.url, - "#comment" : comment.replace('"', "'"), - "#category": (extr.basecategory, - extr.category, - extr.subcategory), - "#class" : extr.__class__, + "#url": extr.url, + "#comment": comment.replace('"', "'"), + "#category": (extr.basecategory, extr.category, extr.subcategory), + "#class": extr.__class__, } if not comment: @@ -103,19 +98,16 @@ def build_test(extr, data): if not data: data = {} - if (options := data.pop("options", None)): - instr["#options"] = { - name: value - for name, value in options - } - if (pattern := data.pop("pattern", None)): + if options := data.pop("options", None): + instr["#options"] = dict(options) + if pattern := data.pop("pattern", None): if pattern in PATTERNS: cls = PATTERNS[pattern] pattern = f"lit:{pyprint(cls)}.pattern" instr["#pattern"] = pattern - if (exception := data.pop("exception", None)): + if exception := data.pop("exception", None): instr["#exception"] = exception - if (range := data.pop("range", None)): + if range := data.pop("range", None): instr["#range"] = range if (count := data.pop("count", None)) is not None: instr["#count"] = count @@ -123,13 +115,12 @@ def build_test(extr, data): instr["#archive"] = archive if (extractor := data.pop("extractor", None)) is not None: instr["#extractor"] = extractor - if (url := data.pop("url", None)): + if url := data.pop("url", None): instr["#sha1_url"] = url - if (metadata := data.pop("keyword", None)): - if isinstance(metadata, str) and len(metadata) == 40: - instr["#sha1_metadata"] = metadata - metadata = {} - if (content := data.pop("content", None)): + if metadata := data.pop("keyword", None) and isinstance(metadata, str) and len(metadata) == 40: # noqa: F821 + instr["#sha1_metadata"] = metadata + metadata = {} + if content := data.pop("content", None): if isinstance(content, tuple): content = list(content) instr["#sha1_content"] = content @@ -144,10 +135,7 @@ def build_test(extr, data): def collect_patterns(): - return { - cls.pattern.pattern: cls - for cls in extractor._list_classes() - } + return {cls.pattern.pattern: cls for cls in extractor._list_classes()} def collect_tests(whitelist=None): @@ -155,7 +143,6 @@ def collect_tests(whitelist=None): for cls in extractor._list_classes(): for url, data in cls._get_tests(): - extr = cls.from_url(url) if whitelist and extr.category not in whitelist: continue @@ -170,7 +157,6 @@ def export_tests(data): tests = [] for head, instr, metadata in data: - for v in itertools.chain( head.values(), instr.values() if instr else (), @@ -181,9 +167,9 @@ def export_tests(data): module, _, name = v.__module__.rpartition(".") if name[0].isdecimal(): - stmt = f'''\ + stmt = f"""\ {module.partition(".")[0]} = __import__("{v.__module__}") -_{name} = getattr({module}, "{name}")''' +_{name} = getattr({module}, "{name}")""" elif module: stmt = f"from {module} import {name}" else: @@ -217,11 +203,14 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument( - "-t", "--target", + "-t", + "--target", help="target directory", ) parser.add_argument( - "-c", "--category", action="append", + "-c", + "--category", + action="append", help="extractor categories to export", ) @@ -230,7 +219,8 @@ def main(): if not args.target: args.target = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "test", "results", + "test", + "results", ) global PATTERNS diff --git a/scripts/hook-gallery_dl.py b/scripts/hook-gallery_dl.py index ae51b068a..f89d708bb 100644 --- a/scripts/hook-gallery_dl.py +++ b/scripts/hook-gallery_dl.py @@ -1,6 +1,6 @@ -# -*- coding: utf-8 -*- - -from gallery_dl import extractor, downloader, postprocessor +from gallery_dl import downloader +from gallery_dl import extractor +from gallery_dl import postprocessor hiddenimports = [ package.__name__ + "." + module diff --git a/scripts/man.py b/scripts/man.py index 2aff221d8..e9ea94754 100755 --- a/scripts/man.py +++ b/scripts/man.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019-2020 Mike Fährmann # @@ -9,16 +8,16 @@ """Generate man pages""" -import re import datetime +import re import util + import gallery_dl.option import gallery_dl.version def build_gallery_dl_1(path=None): - OPTS_FMT = """.TP\n.B "{}" {}\n{}""" TEMPLATE = r""" @@ -92,24 +91,28 @@ def build_gallery_dl_1(path=None): for action in gallery_dl.option.build_parser()._actions: if action.help.startswith("=="): continue - options.append(OPTS_FMT.format( - ", ".join(action.option_strings).replace("-", r"\-"), - r"\f[I]{}\f[]".format(action.metavar) if action.metavar else "", - action.help, - )) + options.append( + OPTS_FMT.format( + ", ".join(action.option_strings).replace("-", r"\-"), + rf"\f[I]{action.metavar}\f[]" if action.metavar else "", + action.help, + ) + ) if not path: path = util.path("data/man/gallery-dl.1") with util.lazy(path) as fp: - fp.write(TEMPLATE.lstrip() % { - "options": "\n".join(options), - "version": gallery_dl.version.__version__, - "date" : datetime.datetime.now().strftime("%Y-%m-%d"), - }) + fp.write( + TEMPLATE.lstrip() + % { + "options": "\n".join(options), + "version": gallery_dl.version.__version__, + "date": datetime.datetime.now().strftime("%Y-%m-%d"), + } + ) def build_gallery_dl_conf_5(path=None): - TEMPLATE = r""" .TH "GALLERY-DL.CONF" "5" "%(date)s" "%(version)s" "gallery-dl Manual" .\" disable hyphenation @@ -210,24 +213,26 @@ def build_gallery_dl_conf_5(path=None): for field, text in option.items(): if field in ("Type", "Default"): - content.append('.IP "{}:" {}'.format(field, len(field)+2)) + content.append(f'.IP "{field}:" {len(field) + 2}') content.append(strip_rst(text)) else: - content.append('.IP "{}:" 4'.format(field)) + content.append(f'.IP "{field}:" 4') content.append(strip_rst(text, field != "Example")) if not path: path = util.path("data/man/gallery-dl.conf.5") with util.lazy(path) as fp: - fp.write(TEMPLATE.lstrip() % { - "options": "\n".join(content), - "version": gallery_dl.version.__version__, - "date" : datetime.datetime.now().strftime("%Y-%m-%d"), - }) + fp.write( + TEMPLATE.lstrip() + % { + "options": "\n".join(content), + "version": gallery_dl.version.__version__, + "date": datetime.datetime.now().strftime("%Y-%m-%d"), + } + ) def parse_docs_configuration(): - doc_path = util.path("docs", "configuration.rst") with open(doc_path, encoding="utf-8") as fp: doc_lines = fp.readlines() @@ -240,12 +245,11 @@ def parse_docs_configuration(): name = None last = None for line in doc_lines: - if line[0] == ".": continue # start of new section - elif re.match(r"^=+$", line): + if re.match(r"^=+$", line): if sec_name and options: sections[sec_name] = options sec_name = last.strip() @@ -284,7 +288,6 @@ def parse_docs_configuration(): def strip_rst(text, extended=True, *, ITALIC=r"\\f[I]\1\\f[]", REGULAR=r"\1"): - text = text.replace("\\", "\\\\") # ``foo`` diff --git a/scripts/options.py b/scripts/options.py index f9f0ace1c..b33df31b9 100755 --- a/scripts/options.py +++ b/scripts/options.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2023 Mike Fährmann # @@ -16,9 +15,9 @@ import util import gallery_dl.util -gallery_dl.util.EXECUTABLE = True -from gallery_dl import option # noqa E402 +gallery_dl.util.EXECUTABLE = True +from gallery_dl import option # noqa: E402 TEMPLATE = """# Command-Line Options @@ -36,10 +35,11 @@ opts = opts.replace("\n ", "\n ") # indent by 4 -PATH = (sys.argv[1] if len(sys.argv) > 1 else - util.path("docs", "options.md")) +PATH = sys.argv[1] if len(sys.argv) > 1 else util.path("docs", "options.md") with util.lazy(PATH) as fp: - fp.write(TEMPLATE.format( - "/".join(os.path.normpath(__file__).split(os.sep)[-2:]), - opts, - )) + fp.write( + TEMPLATE.format( + "/".join(os.path.normpath(__file__).split(os.sep)[-2:]), + opts, + ) + ) diff --git a/scripts/pyinstaller.py b/scripts/pyinstaller.py index 58303547a..7c6c08ad7 100755 --- a/scripts/pyinstaller.py +++ b/scripts/pyinstaller.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """Build a standalone executable using PyInstaller""" import argparse -import util import sys +import util + def main(): parser = argparse.ArgumentParser() @@ -34,21 +34,29 @@ def main(): name = "gallery-dl" if label: - name = "{}_{}".format(name, label) + name = f"{name}_{label}" if args.extension: - name = "{}.{}".format(name, args.extension.lower()) + name = f"{name}.{args.extension.lower()}" import PyInstaller.__main__ - return PyInstaller.__main__.run([ - "--onefile", - "--console", - "--name", name, - "--additional-hooks-dir", util.path("scripts"), - "--distpath", util.path("dist"), - "--workpath", util.path("build"), - "--specpath", util.path("build"), - util.path("gallery_dl", "__main__.py"), - ]) + + return PyInstaller.__main__.run( + [ + "--onefile", + "--console", + "--name", + name, + "--additional-hooks-dir", + util.path("scripts"), + "--distpath", + util.path("dist"), + "--workpath", + util.path("build"), + "--specpath", + util.path("build"), + util.path("gallery_dl", "__main__.py"), + ] + ) if __name__ == "__main__": diff --git a/scripts/pyprint.py b/scripts/pyprint.py index 99d31e41e..369dad414 100644 --- a/scripts/pyprint.py +++ b/scripts/pyprint.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2024 Mike Fährmann # @@ -11,15 +10,11 @@ def pyprint(obj, indent=0, lmin=9, lmax=16): - if isinstance(obj, str): if obj.startswith("lit:"): - return f'''{obj[4:]}''' + return f"""{obj[4:]}""" - if "\\" in obj or obj.startswith("re:"): - prefix = "r" - else: - prefix = "" + prefix = "r" if "\\" in obj or obj.startswith("re:") else "" if "\n" in obj: quote = '"""' @@ -29,19 +24,19 @@ def pyprint(obj, indent=0, lmin=9, lmax=16): else: quote = '"' - return f'''{prefix}{quote}{obj}{quote}''' + return f"""{prefix}{quote}{obj}{quote}""" if isinstance(obj, bytes): return f'''b"{str(obj)[2:-1]}"''' if isinstance(obj, type): if obj.__module__ == "builtins": - return f'''{obj.__name__}''' + return f"""{obj.__name__}""" name = obj.__module__.rpartition(".")[2] if name[0].isdecimal(): name = f"_{name}" - return f'''{name}.{obj.__name__}''' + return f"""{name}.{obj.__name__}""" if isinstance(obj, dict): if not obj: @@ -65,14 +60,13 @@ def pyprint(obj, indent=0, lmin=9, lmax=16): else: lines.append( f'''{ws} "{key}"''' - f'''{' '*(lkey - len(key))}: ''' - f'''{pyprint(value, indent+4)},''' + f"""{' '*(lkey - len(key))}: """ + f"""{pyprint(value, indent+4)},""" ) - lines.append(f'''{ws}}}''') + lines.append(f"""{ws}}}""") return "\n".join(lines) - else: - key, value = obj.popitem() - return f'''{{"{key}": {pyprint(value)}}}''' + key, value = obj.popitem() + return f"""{{"{key}": {pyprint(value)}}}""" if isinstance(obj, list): if not obj: @@ -83,19 +77,14 @@ def pyprint(obj, indent=0, lmin=9, lmax=16): lines = [] lines.append("[") - lines.extend( - f'''{ws} {pyprint(value, indent+4)},''' - for value in obj - ) - lines.append(f'''{ws}]''') + lines.extend(f"""{ws} {pyprint(value, indent+4)},""" for value in obj) + lines.append(f"""{ws}]""") return "\n".join(lines) - else: - return f'''[{pyprint(obj[0])}]''' + return f"""[{pyprint(obj[0])}]""" if isinstance(obj, tuple): if len(obj) == 1: - return f'''({pyprint(obj[0], indent+4)},)''' - return f'''({", ".join(pyprint(v, indent+4) for v in obj)})''' + return f"""({pyprint(obj[0], indent+4)},)""" + return f"""({", ".join(pyprint(v, indent+4) for v in obj)})""" - else: - return f'''{obj}''' + return f"""{obj}""" diff --git a/scripts/run_tests.py b/scripts/run_tests.py index d1fd1f14c..d867a35e6 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2021 Mike Fährmann # @@ -11,8 +10,7 @@ import sys import unittest -TEST_DIRECTORY = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "test") +TEST_DIRECTORY = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "test") sys.path.insert(0, TEST_DIRECTORY) @@ -23,10 +21,7 @@ if file.startswith("test_") and file != "test_results.py" ] else: - TESTS = [ - name if name.startswith("test_") else "test_" + name - for name in sys.argv[1:] - ] + TESTS = [name if name.startswith("test_") else "test_" + name for name in sys.argv[1:]] suite = unittest.TestSuite() diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 4ba5a763f..ba98d9bd9 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -7,11 +6,12 @@ """Generate a Markdown document listing all supported sites""" +import collections import os import sys -import collections import util + from gallery_dl import extractor try: @@ -21,175 +21,174 @@ CATEGORY_MAP = { - "2chan" : "Futaba Channel", - "35photo" : "35PHOTO", - "adultempire" : "Adult Empire", - "agnph" : "AGNPH", - "allgirlbooru" : "All girl", - "ao3" : "Archive of Our Own", - "archivedmoe" : "Archived.Moe", - "archiveofsins" : "Archive of Sins", - "artstation" : "ArtStation", - "aryion" : "Eka's Portal", - "atfbooru" : "ATFBooru", - "azurlanewiki" : "Azur Lane Wiki", - "b4k" : "arch.b4k.co", - "baraag" : "baraag", - "batoto" : "BATO.TO", - "bbc" : "BBC", - "cien" : "Ci-en", - "cohost" : "cohost!", - "comicvine" : "Comic Vine", - "coomerparty" : "Coomer", - "deltaporno" : "DeltaPorno", - "deviantart" : "DeviantArt", - "drawfriends" : "Draw Friends", - "dynastyscans" : "Dynasty Reader", - "e621" : "e621", - "e926" : "e926", - "e6ai" : "e6AI", - "erome" : "EroMe", - "everia" : "EVERIA.CLUB", - "e-hentai" : "E-Hentai", - "exhentai" : "ExHentai", - "fallenangels" : "Fallen Angels Scans", - "fanbox" : "pixivFANBOX", - "fashionnova" : "Fashion Nova", - "furaffinity" : "Fur Affinity", - "hatenablog" : "HatenaBlog", - "hbrowse" : "HBrowse", - "hentai2read" : "Hentai2Read", - "hentaicosplays" : "Hentai Cosplay", - "hentaifoundry" : "Hentai Foundry", - "hentaifox" : "HentaiFox", - "hentaihand" : "HentaiHand", - "hentaihere" : "HentaiHere", - "hentaiimg" : "Hentai Image", - "hentainexus" : "HentaiNexus", - "hiperdex" : "Hipertoon", - "hitomi" : "Hitomi.la", - "horne" : "horne", - "idolcomplex" : "Idol Complex", + "2chan": "Futaba Channel", + "35photo": "35PHOTO", + "adultempire": "Adult Empire", + "agnph": "AGNPH", + "allgirlbooru": "All girl", + "ao3": "Archive of Our Own", + "archivedmoe": "Archived.Moe", + "archiveofsins": "Archive of Sins", + "artstation": "ArtStation", + "aryion": "Eka's Portal", + "atfbooru": "ATFBooru", + "azurlanewiki": "Azur Lane Wiki", + "b4k": "arch.b4k.co", + "baraag": "baraag", + "batoto": "BATO.TO", + "bbc": "BBC", + "cien": "Ci-en", + "cohost": "cohost!", + "comicvine": "Comic Vine", + "coomerparty": "Coomer", + "deltaporno": "DeltaPorno", + "deviantart": "DeviantArt", + "drawfriends": "Draw Friends", + "dynastyscans": "Dynasty Reader", + "e621": "e621", + "e926": "e926", + "e6ai": "e6AI", + "erome": "EroMe", + "everia": "EVERIA.CLUB", + "e-hentai": "E-Hentai", + "exhentai": "ExHentai", + "fallenangels": "Fallen Angels Scans", + "fanbox": "pixivFANBOX", + "fashionnova": "Fashion Nova", + "furaffinity": "Fur Affinity", + "hatenablog": "HatenaBlog", + "hbrowse": "HBrowse", + "hentai2read": "Hentai2Read", + "hentaicosplays": "Hentai Cosplay", + "hentaifoundry": "Hentai Foundry", + "hentaifox": "HentaiFox", + "hentaihand": "HentaiHand", + "hentaihere": "HentaiHere", + "hentaiimg": "Hentai Image", + "hentainexus": "HentaiNexus", + "hiperdex": "Hipertoon", + "hitomi": "Hitomi.la", + "horne": "horne", + "idolcomplex": "Idol Complex", "illusioncardsbooru": "Illusion Game Cards", - "imagebam" : "ImageBam", - "imagefap" : "ImageFap", - "imgbb" : "ImgBB", - "imgbox" : "imgbox", - "imagechest" : "ImageChest", - "imgkiwi" : "IMG.Kiwi", - "imgth" : "imgth", - "imgur" : "imgur", - "joyreactor" : "JoyReactor", - "itchio" : "itch.io", - "jpgfish" : "JPG Fish", - "kabeuchi" : "かべうち", - "kemonoparty" : "Kemono", - "koharu" : "SchaleNetwork", - "livedoor" : "livedoor Blog", - "ohpolly" : "Oh Polly", + "imagebam": "ImageBam", + "imagefap": "ImageFap", + "imgbb": "ImgBB", + "imgbox": "imgbox", + "imagechest": "ImageChest", + "imgkiwi": "IMG.Kiwi", + "imgth": "imgth", + "imgur": "imgur", + "joyreactor": "JoyReactor", + "itchio": "itch.io", + "jpgfish": "JPG Fish", + "kabeuchi": "かべうち", + "kemonoparty": "Kemono", + "koharu": "SchaleNetwork", + "livedoor": "livedoor Blog", + "ohpolly": "Oh Polly", "omgmiamiswimwear": "Omg Miami Swimwear", - "mangadex" : "MangaDex", - "mangafox" : "Manga Fox", - "mangahere" : "Manga Here", - "mangakakalot" : "MangaKakalot", - "mangalife" : "MangaLife", - "manganelo" : "Manganato", - "mangapark" : "MangaPark", - "mangaread" : "MangaRead", - "mangasee" : "MangaSee", - "mariowiki" : "Super Mario Wiki", + "mangadex": "MangaDex", + "mangafox": "Manga Fox", + "mangahere": "Manga Here", + "mangakakalot": "MangaKakalot", + "mangalife": "MangaLife", + "manganelo": "Manganato", + "mangapark": "MangaPark", + "mangaread": "MangaRead", + "mangasee": "MangaSee", + "mariowiki": "Super Mario Wiki", "mastodon.social": "mastodon.social", - "mediawiki" : "MediaWiki", - "micmicidol" : "MIC MIC IDOL", + "mediawiki": "MediaWiki", + "micmicidol": "MIC MIC IDOL", "myhentaigallery": "My Hentai Gallery", - "myportfolio" : "Adobe Portfolio", - "naverwebtoon" : "NaverWebtoon", - "nhentai" : "nhentai", - "nijie" : "nijie", - "nozomi" : "Nozomi.la", - "nsfwalbum" : "NSFWalbum.com", - "paheal" : "rule #34", - "photovogue" : "PhotoVogue", - "pidgiwiki" : "PidgiWiki", - "pixeldrain" : "pixeldrain", - "pornimagesxxx" : "Porn Image", - "pornpics" : "PornPics.com", - "pornreactor" : "PornReactor", + "myportfolio": "Adobe Portfolio", + "naverwebtoon": "NaverWebtoon", + "nhentai": "nhentai", + "nijie": "nijie", + "nozomi": "Nozomi.la", + "nsfwalbum": "NSFWalbum.com", + "paheal": "rule #34", + "photovogue": "PhotoVogue", + "pidgiwiki": "PidgiWiki", + "pixeldrain": "pixeldrain", + "pornimagesxxx": "Porn Image", + "pornpics": "PornPics.com", + "pornreactor": "PornReactor", "readcomiconline": "Read Comic Online", - "rbt" : "RebeccaBlackTech", - "redgifs" : "RedGIFs", - "rozenarcana" : "Rozen Arcana", - "rule34" : "Rule 34", - "rule34hentai" : "Rule34Hentai", - "rule34us" : "Rule 34", - "rule34vault" : "R34 Vault", - "rule34xyz" : "Rule 34 XYZ", - "sankaku" : "Sankaku Channel", - "sankakucomplex" : "Sankaku Complex", - "seiga" : "Niconico Seiga", - "senmanga" : "Sen Manga", - "sensescans" : "Sense-Scans", - "sexcom" : "Sex.com", - "simplyhentai" : "Simply Hentai", - "slickpic" : "SlickPic", - "slideshare" : "SlideShare", - "smugmug" : "SmugMug", - "speakerdeck" : "Speaker Deck", - "steamgriddb" : "SteamGridDB", - "subscribestar" : "SubscribeStar", - "tbib" : "The Big ImageBoard", - "tcbscans" : "TCB Scans", - "tco" : "Twitter t.co", - "tmohentai" : "TMOHentai", - "thatpervert" : "ThatPervert", - "thebarchive" : "The /b/ Archive", - "thecollection" : "The /co/llection", - "tumblrgallery" : "TumblrGallery", - "vanillarock" : "もえぴりあ", - "vidyart2" : "/v/idyart2", - "vidyapics" : "Vidya Booru", - "vk" : "VK", - "vsco" : "VSCO", - "wallpapercave" : "Wallpaper Cave", - "webmshare" : "webmshare", - "webtoons" : "Webtoon", - "wikiart" : "WikiArt.org", - "wikigg" : "wiki.gg", + "rbt": "RebeccaBlackTech", + "redgifs": "RedGIFs", + "rozenarcana": "Rozen Arcana", + "rule34": "Rule 34", + "rule34hentai": "Rule34Hentai", + "rule34us": "Rule 34", + "rule34vault": "R34 Vault", + "rule34xyz": "Rule 34 XYZ", + "sankaku": "Sankaku Channel", + "sankakucomplex": "Sankaku Complex", + "seiga": "Niconico Seiga", + "senmanga": "Sen Manga", + "sensescans": "Sense-Scans", + "sexcom": "Sex.com", + "simplyhentai": "Simply Hentai", + "slickpic": "SlickPic", + "slideshare": "SlideShare", + "smugmug": "SmugMug", + "speakerdeck": "Speaker Deck", + "steamgriddb": "SteamGridDB", + "subscribestar": "SubscribeStar", + "tbib": "The Big ImageBoard", + "tcbscans": "TCB Scans", + "tco": "Twitter t.co", + "tmohentai": "TMOHentai", + "thatpervert": "ThatPervert", + "thebarchive": "The /b/ Archive", + "thecollection": "The /co/llection", + "tumblrgallery": "TumblrGallery", + "vanillarock": "もえぴりあ", + "vidyart2": "/v/idyart2", + "vidyapics": "Vidya Booru", + "vk": "VK", + "vsco": "VSCO", + "wallpapercave": "Wallpaper Cave", + "webmshare": "webmshare", + "webtoons": "Webtoon", + "wikiart": "WikiArt.org", + "wikigg": "wiki.gg", "wikimediacommons": "Wikimedia Commons", - "xbunkr" : "xBunkr", - "xhamster" : "xHamster", - "xvideos" : "XVideos", - "yandere" : "yande.re", + "xbunkr": "xBunkr", + "xhamster": "xHamster", + "xvideos": "XVideos", + "yandere": "yande.re", } SUBCATEGORY_MAP = { - "" : "", - "art" : "Art", - "audio" : "Audio", - "doujin" : "Doujin", - "home" : "Home Feed", - "image" : "individual Images", - "index" : "Site Index", - "info" : "User Profile Information", - "issue" : "Comic Issues", - "manga" : "Manga", - "media" : "Media Files", - "note" : "Images from Notes", + "": "", + "art": "Art", + "audio": "Audio", + "doujin": "Doujin", + "home": "Home Feed", + "image": "individual Images", + "index": "Site Index", + "info": "User Profile Information", + "issue": "Comic Issues", + "manga": "Manga", + "media": "Media Files", + "note": "Images from Notes", "popular": "Popular Images", - "recent" : "Recent Images", - "search" : "Search Results", - "status" : "Images from Statuses", - "tag" : "Tag Searches", - "tweets" : "", - "user" : "User Profiles", - "watch" : "Watches", - "following" : "Followed Users", - "related-pin" : "related Pins", + "recent": "Recent Images", + "search": "Search Results", + "status": "Images from Statuses", + "tag": "Tag Searches", + "tweets": "", + "user": "User Profiles", + "watch": "Watches", + "following": "Followed Users", + "related-pin": "related Pins", "related-board": "", - "ao3": { - "user-works" : "", - "user-series" : "", + "user-works": "", + "user-series": "", "user-bookmark": "Bookmarks", }, "artstation": { @@ -210,25 +209,25 @@ "images": "Image Listings", "user-models": "User Models", "user-images": "User Images", - "user-posts" : "User Posts", + "user-posts": "User Posts", }, "coomerparty": { - "discord" : "", + "discord": "", "discord-server": "", - "posts" : "", + "posts": "", }, "desktopography": { "site": "", }, "deviantart": { "gallery-search": "Gallery Searches", - "stash" : "Sta.sh", + "stash": "Sta.sh", "status": "Status Updates", "watch-posts": "", }, "fanbox": { "supporting": "Supported User Feed", - "redirect" : "Pixiv Redirects", + "redirect": "Pixiv Redirects", }, "fapello": { "path": ["Videos", "Trending Posts", "Popular Videos", "Top Models"], @@ -238,7 +237,7 @@ }, "hatenablog": { "archive": "Archive", - "entry" : "Individual Posts", + "entry": "Individual Posts", }, "hentaifoundry": { "story": "", @@ -255,19 +254,19 @@ "tagged": "Tagged Posts", }, "kemonoparty": { - "discord" : "Discord Servers", + "discord": "Discord Servers", "discord-server": "", - "posts" : "", + "posts": "", }, "lensdump": { "albums": "", }, "mangadex": { - "feed" : "Followed Feed", + "feed": "Followed Feed", }, "nijie": { "followed": "Followed Users", - "nuita" : "Nuita History", + "nuita": "Nuita History", }, "pinterest": { "board": "", @@ -276,7 +275,7 @@ "allpins": "All Pins", }, "pixiv": { - "me" : "pixiv.me Links", + "me": "pixiv.me Links", "novel-bookmark": "Novel Bookmarks", "novel-series": "Novel Series", "novel-user": "", @@ -293,8 +292,8 @@ }, "raddle": { "usersubmissions": "User Profiles", - "post" : "Individual Posts", - "shorturl" : "", + "post": "Individual Posts", + "shorturl": "", }, "redgifs": { "collections": "", @@ -309,7 +308,7 @@ "pins": "User Pins", }, "skeb": { - "following" : "Followed Creators", + "following": "Followed Creators", "following-users": "Followed Users", }, "smugmug": { @@ -336,13 +335,13 @@ }, "wallhaven": { "collections": "", - "uploads" : "", + "uploads": "", }, "wallpapercave": { "image": ["individual Images", "Search Results"], }, "weasyl": { - "journals" : "", + "journals": "", "submissions": "", }, "weibo": { @@ -358,88 +357,94 @@ } BASE_MAP = { - "E621" : "e621 Instances", - "foolfuuka" : "FoolFuuka 4chan Archives", - "foolslide" : "FoOlSlide Instances", + "E621": "e621 Instances", + "foolfuuka": "FoolFuuka 4chan Archives", + "foolslide": "FoOlSlide Instances", "gelbooru_v01": "Gelbooru Beta 0.1.11", "gelbooru_v02": "Gelbooru Beta 0.2", - "jschan" : "jschan Imageboards", - "lolisafe" : "lolisafe and chibisafe", - "lynxchan" : "LynxChan Imageboards", - "moebooru" : "Moebooru and MyImouto", - "szurubooru" : "szurubooru Instances", + "jschan": "jschan Imageboards", + "lolisafe": "lolisafe and chibisafe", + "lynxchan": "LynxChan Imageboards", + "moebooru": "Moebooru and MyImouto", + "szurubooru": "szurubooru Instances", "urlshortener": "URL Shorteners", - "vichan" : "vichan Imageboards", + "vichan": "vichan Imageboards", } URL_MAP = { - "blogspot" : "https://www.blogger.com/", + "blogspot": "https://www.blogger.com/", "wikimedia": "https://www.wikimedia.org/", } _OAUTH = 'OAuth' _COOKIES = 'Cookies' -_APIKEY_DB = ('API Key') -_APIKEY_WH = ('API Key') -_APIKEY_WY = ('API Key') +_APIKEY_DB = ( + 'API Key' +) +_APIKEY_WH = ( + 'API Key' +) +_APIKEY_WY = ( + 'API Key' +) AUTH_MAP = { - "aibooru" : "Supported", - "ao3" : "Supported", - "aryion" : "Supported", - "atfbooru" : "Supported", - "baraag" : _OAUTH, - "bluesky" : "Supported", - "booruvar" : "Supported", - "boosty" : _COOKIES, - "coomerparty" : "Supported", - "danbooru" : "Supported", - "derpibooru" : _APIKEY_DB, - "deviantart" : _OAUTH, - "e621" : "Supported", - "e6ai" : "Supported", - "e926" : "Supported", - "e-hentai" : "Supported", - "exhentai" : "Supported", - "fanbox" : _COOKIES, - "fantia" : _COOKIES, - "flickr" : _OAUTH, - "furaffinity" : _COOKIES, - "furbooru" : "API Key", - "horne" : "Required", - "idolcomplex" : "Supported", - "imgbb" : "Supported", - "inkbunny" : "Supported", - "instagram" : _COOKIES, - "kemonoparty" : "Supported", - "mangadex" : "Supported", - "mangoxo" : "Supported", + "aibooru": "Supported", + "ao3": "Supported", + "aryion": "Supported", + "atfbooru": "Supported", + "baraag": _OAUTH, + "bluesky": "Supported", + "booruvar": "Supported", + "boosty": _COOKIES, + "coomerparty": "Supported", + "danbooru": "Supported", + "derpibooru": _APIKEY_DB, + "deviantart": _OAUTH, + "e621": "Supported", + "e6ai": "Supported", + "e926": "Supported", + "e-hentai": "Supported", + "exhentai": "Supported", + "fanbox": _COOKIES, + "fantia": _COOKIES, + "flickr": _OAUTH, + "furaffinity": _COOKIES, + "furbooru": "API Key", + "horne": "Required", + "idolcomplex": "Supported", + "imgbb": "Supported", + "inkbunny": "Supported", + "instagram": _COOKIES, + "kemonoparty": "Supported", + "mangadex": "Supported", + "mangoxo": "Supported", "mastodon.social": _OAUTH, - "newgrounds" : "Supported", - "nijie" : "Required", - "patreon" : _COOKIES, - "pawoo" : _OAUTH, - "pillowfort" : "Supported", - "pinterest" : _COOKIES, - "pixiv" : _OAUTH, - "ponybooru" : "API Key", - "reddit" : _OAUTH, - "sankaku" : "Supported", - "scrolller" : "Supported", - "seiga" : "Supported", - "smugmug" : _OAUTH, - "subscribestar" : "Supported", - "tapas" : "Supported", - "tsumino" : "Supported", - "tumblr" : _OAUTH, - "twitter" : "Supported", - "vipergirls" : "Supported", - "wallhaven" : _APIKEY_WH, - "weasyl" : _APIKEY_WY, - "zerochan" : "Supported", + "newgrounds": "Supported", + "nijie": "Required", + "patreon": _COOKIES, + "pawoo": _OAUTH, + "pillowfort": "Supported", + "pinterest": _COOKIES, + "pixiv": _OAUTH, + "ponybooru": "API Key", + "reddit": _OAUTH, + "sankaku": "Supported", + "scrolller": "Supported", + "seiga": "Supported", + "smugmug": _OAUTH, + "subscribestar": "Supported", + "tapas": "Supported", + "tsumino": "Supported", + "tumblr": _OAUTH, + "twitter": "Supported", + "vipergirls": "Supported", + "wallhaven": _APIKEY_WH, + "weasyl": _APIKEY_WY, + "zerochan": "Supported", } IGNORE_LIST = ( @@ -466,7 +471,7 @@ def domain(cls): return cls.root + "/" url = cls.example - return url[:url.find("/", 8)+1] + return url[: url.find("/", 8) + 1] def category_text(c): @@ -577,20 +582,20 @@ def build_extractor_list(): # define table columns COLUMNS = ( - ("Site", 20, - lambda bc, c, scs, d: category_text(c)), - ("URL" , 35, - lambda bc, c, scs, d: d), - ("Capabilities", 50, - lambda bc, c, scs, d: ", ".join(subcategory_text(bc, c, sc) for sc in scs - if subcategory_text(bc, c, sc))), - ("Authentication", 16, - lambda bc, c, scs, d: AUTH_MAP.get(c, "")), + ("Site", 20, lambda bc, c, scs, d: category_text(c)), + ("URL", 35, lambda bc, c, scs, d: d), + ( + "Capabilities", + 50, + lambda bc, c, scs, d: ", ".join( + subcategory_text(bc, c, sc) for sc in scs if subcategory_text(bc, c, sc) + ), + ), + ("Authentication", 16, lambda bc, c, scs, d: AUTH_MAP.get(c, "")), ) def generate_output(columns, categories, domains): - thead = [] append = thead.append append("") @@ -602,11 +607,9 @@ def generate_output(columns, categories, domains): append = tbody.append for bcat, base in categories.items(): - if bcat and base: name = BASE_MAP.get(bcat) or (bcat.capitalize() + " Instances") - append('\n\n ' + - name + '\n') + append('\n\n ' + name + "\n") clist = base.items() else: clist = sorted(base.items(), key=category_key) @@ -641,7 +644,6 @@ def generate_output(columns, categories, domains): categories, domains = build_extractor_list() -PATH = (sys.argv[1] if len(sys.argv) > 1 else - util.path("docs", "supportedsites.md")) +PATH = sys.argv[1] if len(sys.argv) > 1 else util.path("docs", "supportedsites.md") with util.lazy(PATH) as fp: fp.write(generate_output(COLUMNS, categories, domains)) diff --git a/scripts/util.py b/scripts/util.py index 48b8d3193..6c4c9c08a 100644 --- a/scripts/util.py +++ b/scripts/util.py @@ -1,11 +1,9 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import os import io +import os import sys ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -18,8 +16,7 @@ def path(*segments, join=os.path.join): return result -class lazy(): - +class lazy: def __init__(self, path): self.path = path self.buffer = io.StringIO() diff --git a/setup.py b/setup.py index 44acef9a6..92ee240de 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- +import os.path import re import sys -import os.path import warnings @@ -18,8 +17,8 @@ def check_file(fname): if os.path.exists(path): return True warnings.warn( - "Not including file '{}' since it is not present. " - "Run 'make' to build all automatically generated files.".format(fname) + f"Not including file '{fname}' since it is not present. " + "Run 'make' to build all automatically generated files." ) return False @@ -34,10 +33,10 @@ def check_file(fname): (path, [f for f in files if check_file(f)]) for (path, files) in [ ("share/bash-completion/completions", ["data/completion/gallery-dl"]), - ("share/zsh/site-functions" , ["data/completion/_gallery-dl"]), - ("share/fish/vendor_completions.d" , ["data/completion/gallery-dl.fish"]), - ("share/man/man1" , ["data/man/gallery-dl.1"]), - ("share/man/man5" , ["data/man/gallery-dl.conf.5"]), + ("share/zsh/site-functions", ["data/completion/_gallery-dl"]), + ("share/fish/vendor_completions.d", ["data/completion/gallery-dl.fish"]), + ("share/man/man1", ["data/man/gallery-dl.1"]), + ("share/man/man5", ["data/man/gallery-dl.conf.5"]), ] ] @@ -48,10 +47,13 @@ def check_file(fname): "gallery_dl.postprocessor", ] -DESCRIPTION = ("Command-line program to download image galleries and " - "collections from several image hosting sites") +DESCRIPTION = ( + "Command-line program to download image galleries and " + "collections from several image hosting sites" +) LONG_DESCRIPTION = read("README.rst").replace( - "= 2", - - "board" : "70", - "board_name": "新板提案", - "com" : str, - "fsize" : r"re:\d+", - "name" : "名無し", - "no" : r"re:17\d\d\d", - "now" : r"re:2[34]/../..\(.\)..:..:..", - "post" : "無題", - "server" : "dec", - "thread" : "17222", - "tim" : r"re:^\d{13}$", - "time" : r"re:^\d{10}$", - "title" : "画像会話板", -}, - + { + "#url": "https://dec.2chan.net/70/res/17222.htm", + "#category": ("", "2chan", "thread"), + "#class": _2chan._2chanThreadExtractor, + "#pattern": r"https://dec\.2chan\.net/70/src/\d{13}\.jpg", + "#count": ">= 2", + "board": "70", + "board_name": "新板提案", + "com": str, + "fsize": r"re:\d+", + "name": "名無し", + "no": r"re:17\d\d\d", + "now": r"re:2[34]/../..\(.\)..:..:..", + "post": "無題", + "server": "dec", + "thread": "17222", + "tim": r"re:^\d{13}$", + "time": r"re:^\d{10}$", + "title": "画像会話板", + }, ) diff --git a/test/results/2chen.py b/test/results/2chen.py index 589053fa0..adce745f8 100644 --- a/test/results/2chen.py +++ b/test/results/2chen.py @@ -1,68 +1,57 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import datetime gallery_dl = __import__("gallery_dl.extractor.2chen") _2chen = getattr(gallery_dl.extractor, "2chen") -import datetime - __tests__ = ( -{ - "#url" : "https://sturdychan.help/tv/268929", - "#category": ("", "2chen", "thread"), - "#class" : _2chen._2chenThreadExtractor, - "#pattern" : r"https://sturdychan\.help/assets/images/src/\w{40}\.\w+$", - "#count" : ">= 179", - - "board" : "tv", - "date" : datetime.datetime, - "hash" : r"re:[0-9a-f]{40}", - "name" : "Anonymous", - "no" : r"re:\d+", - "thread": "268929", - "time" : int, - "title" : "「/ttg/ #118: 🇧🇷 edition」", - "url" : str, -}, - -{ - "#url" : "https://2chen.club/tv/1", - "#category": ("", "2chen", "thread"), - "#class" : _2chen._2chenThreadExtractor, -}, - -{ - "#url" : "https://2chen.moe/jp/303786", - "#category": ("", "2chen", "thread"), - "#class" : _2chen._2chenThreadExtractor, -}, - -{ - "#url" : "https://sturdychan.help/co/", - "#category": ("", "2chen", "board"), - "#class" : _2chen._2chenBoardExtractor, - "#pattern" : _2chen._2chenThreadExtractor.pattern, -}, - -{ - "#url" : "https://2chen.moe/co", - "#category": ("", "2chen", "board"), - "#class" : _2chen._2chenBoardExtractor, -}, - -{ - "#url" : "https://2chen.club/tv", - "#category": ("", "2chen", "board"), - "#class" : _2chen._2chenBoardExtractor, -}, - -{ - "#url" : "https://2chen.moe/co/catalog", - "#category": ("", "2chen", "board"), - "#class" : _2chen._2chenBoardExtractor, -}, - + { + "#url": "https://sturdychan.help/tv/268929", + "#category": ("", "2chen", "thread"), + "#class": _2chen._2chenThreadExtractor, + "#pattern": r"https://sturdychan\.help/assets/images/src/\w{40}\.\w+$", + "#count": ">= 179", + "board": "tv", + "date": datetime.datetime, + "hash": r"re:[0-9a-f]{40}", + "name": "Anonymous", + "no": r"re:\d+", + "thread": "268929", + "time": int, + "title": "「/ttg/ #118: 🇧🇷 edition」", + "url": str, + }, + { + "#url": "https://2chen.club/tv/1", + "#category": ("", "2chen", "thread"), + "#class": _2chen._2chenThreadExtractor, + }, + { + "#url": "https://2chen.moe/jp/303786", + "#category": ("", "2chen", "thread"), + "#class": _2chen._2chenThreadExtractor, + }, + { + "#url": "https://sturdychan.help/co/", + "#category": ("", "2chen", "board"), + "#class": _2chen._2chenBoardExtractor, + "#pattern": _2chen._2chenThreadExtractor.pattern, + }, + { + "#url": "https://2chen.moe/co", + "#category": ("", "2chen", "board"), + "#class": _2chen._2chenBoardExtractor, + }, + { + "#url": "https://2chen.club/tv", + "#category": ("", "2chen", "board"), + "#class": _2chen._2chenBoardExtractor, + }, + { + "#url": "https://2chen.moe/co/catalog", + "#category": ("", "2chen", "board"), + "#class": _2chen._2chenBoardExtractor, + }, ) diff --git a/test/results/35photo.py b/test/results/35photo.py index c0e29d53b..d2e516722 100644 --- a/test/results/35photo.py +++ b/test/results/35photo.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -9,69 +7,61 @@ __tests__ = ( -{ - "#url" : "https://35photo.pro/liya", - "#category": ("", "35photo", "user"), - "#class" : _35photo._35photoUserExtractor, - "#pattern" : r"https://([a-z][0-9]\.)?35photo\.pro/photos_(main|series)/.*\.jpg", - "#count" : 9, -}, - -{ - "#url" : "https://35photo.pro/suhoveev", - "#comment" : "last photo ID (1267028) isn't given as 'photo-id=\"\" - " - "there are only 23 photos without the last one", - "#category": ("", "35photo", "user"), - "#class" : _35photo._35photoUserExtractor, - "#count" : ">= 33", -}, - -{ - "#url" : "https://en.35photo.pro/liya", - "#category": ("", "35photo", "user"), - "#class" : _35photo._35photoUserExtractor, -}, - -{ - "#url" : "https://ru.35photo.pro/liya", - "#category": ("", "35photo", "user"), - "#class" : _35photo._35photoUserExtractor, -}, - -{ - "#url" : "https://35photo.pro/tags/landscape/", - "#category": ("", "35photo", "tag"), - "#class" : _35photo._35photoTagExtractor, - "#range" : "1-25", - "#count" : 25, - "#archive" : False, -}, - -{ - "#url" : "https://35photo.pro/genre_109/", - "#category": ("", "35photo", "genre"), - "#class" : _35photo._35photoGenreExtractor, -}, - -{ - "#url" : "https://35photo.pro/photo_753340/", - "#category": ("", "35photo", "image"), - "#class" : _35photo._35photoImageExtractor, - "#count" : 1, - - "url" : r"re:https://35photo\.pro/photos_main/.*\.jpg", - "id" : 753340, - "title" : "Winter walk", - "description": str, - "tags" : list, - "views" : int, - "favorites" : int, - "score" : int, - "type" : 0, - "date" : "15 авг, 2014", - "user" : "liya", - "user_id" : 20415, - "user_name" : "Liya Mirzaeva", -}, - + { + "#url": "https://35photo.pro/liya", + "#category": ("", "35photo", "user"), + "#class": _35photo._35photoUserExtractor, + "#pattern": r"https://([a-z][0-9]\.)?35photo\.pro/photos_(main|series)/.*\.jpg", + "#count": 9, + }, + { + "#url": "https://35photo.pro/suhoveev", + "#comment": "last photo ID (1267028) isn't given as 'photo-id=\"\" - " + "there are only 23 photos without the last one", + "#category": ("", "35photo", "user"), + "#class": _35photo._35photoUserExtractor, + "#count": ">= 33", + }, + { + "#url": "https://en.35photo.pro/liya", + "#category": ("", "35photo", "user"), + "#class": _35photo._35photoUserExtractor, + }, + { + "#url": "https://ru.35photo.pro/liya", + "#category": ("", "35photo", "user"), + "#class": _35photo._35photoUserExtractor, + }, + { + "#url": "https://35photo.pro/tags/landscape/", + "#category": ("", "35photo", "tag"), + "#class": _35photo._35photoTagExtractor, + "#range": "1-25", + "#count": 25, + "#archive": False, + }, + { + "#url": "https://35photo.pro/genre_109/", + "#category": ("", "35photo", "genre"), + "#class": _35photo._35photoGenreExtractor, + }, + { + "#url": "https://35photo.pro/photo_753340/", + "#category": ("", "35photo", "image"), + "#class": _35photo._35photoImageExtractor, + "#count": 1, + "url": r"re:https://35photo\.pro/photos_main/.*\.jpg", + "id": 753340, + "title": "Winter walk", + "description": str, + "tags": list, + "views": int, + "favorites": int, + "score": int, + "type": 0, + "date": "15 авг, 2014", + "user": "liya", + "user_id": 20415, + "user_name": "Liya Mirzaeva", + }, ) diff --git a/test/results/3dbooru.py b/test/results/3dbooru.py index d3d407bd2..0eb72c83e 100644 --- a/test/results/3dbooru.py +++ b/test/results/3dbooru.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -9,42 +7,37 @@ __tests__ = ( -{ - "#url" : "http://behoimi.org/post?tags=himekawa_azuru+dress", - "#category": ("booru", "3dbooru", "tag"), - "#class" : _3dbooru._3dbooruTagExtractor, - "#sha1_url" : "ecb30c6aaaf8a6ff8f55255737a9840832a483c1", - "#sha1_content": "11cbda40c287e026c1ce4ca430810f761f2d0b2a", -}, - -{ - "#url" : "http://behoimi.org/pool/show/27", - "#category": ("booru", "3dbooru", "pool"), - "#class" : _3dbooru._3dbooruPoolExtractor, - "#sha1_url" : "da75d2d1475449d5ef0c266cb612683b110a30f2", - "#sha1_content": "fd5b37c5c6c2de4b4d6f1facffdefa1e28176554", -}, - -{ - "#url" : "http://behoimi.org/post/show/140852", - "#category": ("booru", "3dbooru", "post"), - "#class" : _3dbooru._3dbooruPostExtractor, - "#options" : {"tags": True}, - "#sha1_url" : "ce874ea26f01d6c94795f3cc3aaaaa9bc325f2f6", - "#sha1_content": "26549d55b82aa9a6c1686b96af8bfcfa50805cd4", - - "tags_character": "furude_rika", - "tags_copyright": "higurashi_no_naku_koro_ni", - "tags_model" : "himekawa_azuru", - "tags_general" : str, -}, - -{ - "#url" : "http://behoimi.org/post/popular_by_month?month=2&year=2013", - "#category": ("booru", "3dbooru", "popular"), - "#class" : _3dbooru._3dbooruPopularExtractor, - "#pattern" : r"http://behoimi\.org/data/../../[0-9a-f]{32}\.jpg", - "#count" : 20, -}, - + { + "#url": "http://behoimi.org/post?tags=himekawa_azuru+dress", + "#category": ("booru", "3dbooru", "tag"), + "#class": _3dbooru._3dbooruTagExtractor, + "#sha1_url": "ecb30c6aaaf8a6ff8f55255737a9840832a483c1", + "#sha1_content": "11cbda40c287e026c1ce4ca430810f761f2d0b2a", + }, + { + "#url": "http://behoimi.org/pool/show/27", + "#category": ("booru", "3dbooru", "pool"), + "#class": _3dbooru._3dbooruPoolExtractor, + "#sha1_url": "da75d2d1475449d5ef0c266cb612683b110a30f2", + "#sha1_content": "fd5b37c5c6c2de4b4d6f1facffdefa1e28176554", + }, + { + "#url": "http://behoimi.org/post/show/140852", + "#category": ("booru", "3dbooru", "post"), + "#class": _3dbooru._3dbooruPostExtractor, + "#options": {"tags": True}, + "#sha1_url": "ce874ea26f01d6c94795f3cc3aaaaa9bc325f2f6", + "#sha1_content": "26549d55b82aa9a6c1686b96af8bfcfa50805cd4", + "tags_character": "furude_rika", + "tags_copyright": "higurashi_no_naku_koro_ni", + "tags_model": "himekawa_azuru", + "tags_general": str, + }, + { + "#url": "http://behoimi.org/post/popular_by_month?month=2&year=2013", + "#category": ("booru", "3dbooru", "popular"), + "#class": _3dbooru._3dbooruPopularExtractor, + "#pattern": r"http://behoimi\.org/data/../../[0-9a-f]{32}\.jpg", + "#count": 20, + }, ) diff --git a/test/results/4archive.py b/test/results/4archive.py index cebec6fc4..404ed37d6 100644 --- a/test/results/4archive.py +++ b/test/results/4archive.py @@ -1,60 +1,53 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -gallery_dl = __import__("gallery_dl.extractor.4archive") -_4archive = getattr(gallery_dl.extractor, "4archive") import datetime +gallery_dl = __import__("gallery_dl.extractor.4archive") +_4archive = getattr(gallery_dl.extractor, "4archive") __tests__ = ( -{ - "#url" : "https://4archive.org/board/u/thread/2397221", - "#category": ("", "4archive", "thread"), - "#class" : _4archive._4archiveThreadExtractor, - "#pattern" : r"https://(cdn\.4archive\.org/u/image/150\d/\d\d\d/\d+\.\w+|4archive\.org/image/image-404\.png)", - "#count" : 16, - - "board" : "u", - "com" : str, - "date" : datetime.datetime, - "name" : "Anonymous", - "no" : range(2397221, 2418158), - "thread": 2397221, - "time" : int, - "title" : "best anime", - "url" : str, - "width" : int, - "height": int, - "size" : int, -}, - -{ - "#url" : "https://4archive.org/board/jp/thread/17611798", - "#category": ("", "4archive", "thread"), - "#class" : _4archive._4archiveThreadExtractor, - "#pattern" : r"https://(cdn\.4archive\.org/jp/image/\d\d\d\d/\d\d\d/\d+\.\w+|4archive\.org/image/image-404\.png)", - "#count" : 85, -}, - -{ - "#url" : "https://4archive.org/board/u", - "#category": ("", "4archive", "board"), - "#class" : _4archive._4archiveBoardExtractor, - "#pattern" : _4archive._4archiveThreadExtractor.pattern, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://4archive.org/board/jp/10", - "#category": ("", "4archive", "board"), - "#class" : _4archive._4archiveBoardExtractor, - "#pattern" : _4archive._4archiveThreadExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -} - + { + "#url": "https://4archive.org/board/u/thread/2397221", + "#category": ("", "4archive", "thread"), + "#class": _4archive._4archiveThreadExtractor, + "#pattern": r"https://(cdn\.4archive\.org/u/image/150\d/\d\d\d/\d+\.\w+|4archive\.org/image/image-404\.png)", + "#count": 16, + "board": "u", + "com": str, + "date": datetime.datetime, + "name": "Anonymous", + "no": range(2397221, 2418158), + "thread": 2397221, + "time": int, + "title": "best anime", + "url": str, + "width": int, + "height": int, + "size": int, + }, + { + "#url": "https://4archive.org/board/jp/thread/17611798", + "#category": ("", "4archive", "thread"), + "#class": _4archive._4archiveThreadExtractor, + "#pattern": r"https://(cdn\.4archive\.org/jp/image/\d\d\d\d/\d\d\d/\d+\.\w+|4archive\.org/image/image-404\.png)", + "#count": 85, + }, + { + "#url": "https://4archive.org/board/u", + "#category": ("", "4archive", "board"), + "#class": _4archive._4archiveBoardExtractor, + "#pattern": _4archive._4archiveThreadExtractor.pattern, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://4archive.org/board/jp/10", + "#category": ("", "4archive", "board"), + "#class": _4archive._4archiveBoardExtractor, + "#pattern": _4archive._4archiveThreadExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, ) diff --git a/test/results/4chan.py b/test/results/4chan.py index 6219e1652..343fd5a42 100644 --- a/test/results/4chan.py +++ b/test/results/4chan.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -9,29 +7,26 @@ __tests__ = ( -{ - "#url" : "https://boards.4chan.org/tg/thread/15396072/", - "#category": ("", "4chan", "thread"), - "#class" : _4chan._4chanThreadExtractor, - "#sha1_url" : "39082ad166161966d7ba8e37f2173a824eb540f0", - "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", - "#sha1_content" : "551e432d52700ff3711f14752124e9af86ecbbdf", -}, - -{ - "#url" : "https://boards.4channel.org/tg/thread/15396072/", - "#category": ("", "4chan", "thread"), - "#class" : _4chan._4chanThreadExtractor, - "#sha1_url" : "39082ad166161966d7ba8e37f2173a824eb540f0", - "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", -}, - -{ - "#url" : "https://boards.4channel.org/po/", - "#category": ("", "4chan", "board"), - "#class" : _4chan._4chanBoardExtractor, - "#pattern" : _4chan._4chanThreadExtractor.pattern, - "#count" : ">= 100", -}, - + { + "#url": "https://boards.4chan.org/tg/thread/15396072/", + "#category": ("", "4chan", "thread"), + "#class": _4chan._4chanThreadExtractor, + "#sha1_url": "39082ad166161966d7ba8e37f2173a824eb540f0", + "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", + "#sha1_content": "551e432d52700ff3711f14752124e9af86ecbbdf", + }, + { + "#url": "https://boards.4channel.org/tg/thread/15396072/", + "#category": ("", "4chan", "thread"), + "#class": _4chan._4chanThreadExtractor, + "#sha1_url": "39082ad166161966d7ba8e37f2173a824eb540f0", + "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", + }, + { + "#url": "https://boards.4channel.org/po/", + "#category": ("", "4chan", "board"), + "#class": _4chan._4chanBoardExtractor, + "#pattern": _4chan._4chanThreadExtractor.pattern, + "#count": ">= 100", + }, ) diff --git a/test/results/4chanarchives.py b/test/results/4chanarchives.py index 8aa8befd2..78dd96f40 100644 --- a/test/results/4chanarchives.py +++ b/test/results/4chanarchives.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -9,41 +7,36 @@ __tests__ = ( -{ - "#url" : "https://4chanarchives.com/board/c/thread/2707110", - "#category": ("", "4chanarchives", "thread"), - "#class" : _4chanarchives._4chanarchivesThreadExtractor, - "#pattern" : r"https://i\.imgur\.com/(0wLGseE|qbByWDc)\.jpg", - "#count" : 2, - - "board" : "c", - "com" : str, - "name" : "Anonymous", - "no" : int, - "thread": "2707110", - "time" : r"re:2016-07-1\d \d\d:\d\d:\d\d", - "title" : "Ren Kagami from 'Oyako Neburi'", -}, - -{ - "#url" : "https://4chanarchives.com/board/c/", - "#category": ("", "4chanarchives", "board"), - "#class" : _4chanarchives._4chanarchivesBoardExtractor, - "#pattern" : _4chanarchives._4chanarchivesThreadExtractor.pattern, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://4chanarchives.com/board/c", - "#category": ("", "4chanarchives", "board"), - "#class" : _4chanarchives._4chanarchivesBoardExtractor, -}, - -{ - "#url" : "https://4chanarchives.com/board/c/10", - "#category": ("", "4chanarchives", "board"), - "#class" : _4chanarchives._4chanarchivesBoardExtractor, -}, - + { + "#url": "https://4chanarchives.com/board/c/thread/2707110", + "#category": ("", "4chanarchives", "thread"), + "#class": _4chanarchives._4chanarchivesThreadExtractor, + "#pattern": r"https://i\.imgur\.com/(0wLGseE|qbByWDc)\.jpg", + "#count": 2, + "board": "c", + "com": str, + "name": "Anonymous", + "no": int, + "thread": "2707110", + "time": r"re:2016-07-1\d \d\d:\d\d:\d\d", + "title": "Ren Kagami from 'Oyako Neburi'", + }, + { + "#url": "https://4chanarchives.com/board/c/", + "#category": ("", "4chanarchives", "board"), + "#class": _4chanarchives._4chanarchivesBoardExtractor, + "#pattern": _4chanarchives._4chanarchivesThreadExtractor.pattern, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "https://4chanarchives.com/board/c", + "#category": ("", "4chanarchives", "board"), + "#class": _4chanarchives._4chanarchivesBoardExtractor, + }, + { + "#url": "https://4chanarchives.com/board/c/10", + "#category": ("", "4chanarchives", "board"), + "#class": _4chanarchives._4chanarchivesBoardExtractor, + }, ) diff --git a/test/results/4plebs.py b/test/results/4plebs.py index affe14d83..083b26d1b 100644 --- a/test/results/4plebs.py +++ b/test/results/4plebs.py @@ -1,37 +1,30 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://archive.4plebs.org/tg/thread/54059290", - "#category": ("foolfuuka", "4plebs", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#pattern" : r"https://i\.4pcdn\.org/tg/1[34]\d{11}\.(jpg|png|gif)", - "#count" : 30, -}, - -{ - "#url" : "https://archive.4plebs.org/tg/", - "#category": ("foolfuuka", "4plebs", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://archive.4plebs.org/_/search/text/test/", - "#category": ("foolfuuka", "4plebs", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://archive.4plebs.org/tg/gallery/1", - "#category": ("foolfuuka", "4plebs", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://archive.4plebs.org/tg/thread/54059290", + "#category": ("foolfuuka", "4plebs", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#pattern": r"https://i\.4pcdn\.org/tg/1[34]\d{11}\.(jpg|png|gif)", + "#count": 30, + }, + { + "#url": "https://archive.4plebs.org/tg/", + "#category": ("foolfuuka", "4plebs", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://archive.4plebs.org/_/search/text/test/", + "#category": ("foolfuuka", "4plebs", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://archive.4plebs.org/tg/gallery/1", + "#category": ("foolfuuka", "4plebs", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/500px.py b/test/results/500px.py index 5630e78ea..e025ee315 100644 --- a/test/results/500px.py +++ b/test/results/500px.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -9,95 +7,86 @@ __tests__ = ( -{ - "#url" : "https://500px.com/p/fashvamp", - "#category": ("", "500px", "user"), - "#class" : _500px._500pxUserExtractor, - "#pattern" : r"https?://drscdn.500px.org/photo/\d+/m%3D4096(_k%3D1)?/v2\?sig=", - "#range" : "1-99", - "#count" : 99, -}, - -{ - "#url" : "https://500px.com/fashvamp", - "#category": ("", "500px", "user"), - "#class" : _500px._500pxUserExtractor, -}, - -{ - "#url" : "https://web.500px.com/fashvamp", - "#category": ("", "500px", "user"), - "#class" : _500px._500pxUserExtractor, -}, - -{ - "#url" : "https://500px.com/p/fashvamp/galleries/lera", - "#category": ("", "500px", "gallery"), - "#class" : _500px._500pxGalleryExtractor, - "#pattern" : r"https?://drscdn.500px.org/photo/\d+/m%3D4096_k%3D1/v2\?sig=", - "#count" : 3, - - "gallery": dict, - "user" : dict, -}, - -{ - "#url" : "https://500px.com/fashvamp/galleries/lera", - "#category": ("", "500px", "gallery"), - "#class" : _500px._500pxGalleryExtractor, -}, - -{ - "#url" : "https://500px.com/liked", - "#category": ("", "500px", "favorite"), - "#class" : _500px._500pxFavoriteExtractor, -}, - -{ - "#url" : "https://500px.com/photo/222049255/queen-of-coasts", - "#category": ("", "500px", "image"), - "#class" : _500px._500pxImageExtractor, - "#pattern" : r"https://drscdn\.500px\.org/photo/222049255/m%3D4096_k%3D1/v2\?sig=\w+", - "#count" : 1, - - "camera" : "Canon EOS 600D", - "camera_info" : dict, - "comments" : list, - "comments_count" : int, - "created_at" : "2017-08-01T08:40:05+00:00", - "description" : str, - "editored_by" : None, - "editors_choice" : False, - "extension" : "jpg", - "feature" : "popular", - "feature_date" : "2017-08-01T09:58:28+00:00", - "focal_length" : "208", - "height" : 3111, - "id" : 222049255, - "image_format" : "jpg", - "image_url" : list, - "images" : list, - "iso" : "100", - "lens" : "EF-S55-250mm f/4-5.6 IS II", - "lens_info" : dict, - "liked" : None, - "location" : None, - "location_details": dict, - "name" : "Queen Of Coasts", - "nsfw" : False, - "privacy" : False, - "profile" : True, - "rating" : float, - "status" : 1, - "tags" : list, - "taken_at" : "2017-05-04T17:36:51+00:00", - "times_viewed" : int, - "url" : "/photo/222049255/Queen-Of-Coasts-by-Alice-Nabieva", - "user" : dict, - "user_id" : 12847235, - "votes_count" : int, - "watermark" : True, - "width" : 4637, -}, - + { + "#url": "https://500px.com/p/fashvamp", + "#category": ("", "500px", "user"), + "#class": _500px._500pxUserExtractor, + "#pattern": r"https?://drscdn.500px.org/photo/\d+/m%3D4096(_k%3D1)?/v2\?sig=", + "#range": "1-99", + "#count": 99, + }, + { + "#url": "https://500px.com/fashvamp", + "#category": ("", "500px", "user"), + "#class": _500px._500pxUserExtractor, + }, + { + "#url": "https://web.500px.com/fashvamp", + "#category": ("", "500px", "user"), + "#class": _500px._500pxUserExtractor, + }, + { + "#url": "https://500px.com/p/fashvamp/galleries/lera", + "#category": ("", "500px", "gallery"), + "#class": _500px._500pxGalleryExtractor, + "#pattern": r"https?://drscdn.500px.org/photo/\d+/m%3D4096_k%3D1/v2\?sig=", + "#count": 3, + "gallery": dict, + "user": dict, + }, + { + "#url": "https://500px.com/fashvamp/galleries/lera", + "#category": ("", "500px", "gallery"), + "#class": _500px._500pxGalleryExtractor, + }, + { + "#url": "https://500px.com/liked", + "#category": ("", "500px", "favorite"), + "#class": _500px._500pxFavoriteExtractor, + }, + { + "#url": "https://500px.com/photo/222049255/queen-of-coasts", + "#category": ("", "500px", "image"), + "#class": _500px._500pxImageExtractor, + "#pattern": r"https://drscdn\.500px\.org/photo/222049255/m%3D4096_k%3D1/v2\?sig=\w+", + "#count": 1, + "camera": "Canon EOS 600D", + "camera_info": dict, + "comments": list, + "comments_count": int, + "created_at": "2017-08-01T08:40:05+00:00", + "description": str, + "editored_by": None, + "editors_choice": False, + "extension": "jpg", + "feature": "popular", + "feature_date": "2017-08-01T09:58:28+00:00", + "focal_length": "208", + "height": 3111, + "id": 222049255, + "image_format": "jpg", + "image_url": list, + "images": list, + "iso": "100", + "lens": "EF-S55-250mm f/4-5.6 IS II", + "lens_info": dict, + "liked": None, + "location": None, + "location_details": dict, + "name": "Queen Of Coasts", + "nsfw": False, + "privacy": False, + "profile": True, + "rating": float, + "status": 1, + "tags": list, + "taken_at": "2017-05-04T17:36:51+00:00", + "times_viewed": int, + "url": "/photo/222049255/Queen-Of-Coasts-by-Alice-Nabieva", + "user": dict, + "user_id": 12847235, + "votes_count": int, + "watermark": True, + "width": 4637, + }, ) diff --git a/test/results/8chan.py b/test/results/8chan.py index 1e26c71ac..dd0673ab7 100644 --- a/test/results/8chan.py +++ b/test/results/8chan.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -9,81 +7,72 @@ __tests__ = ( -{ - "#url" : "https://8chan.moe/vhs/res/4.html", - "#class": _8chan._8chanThreadExtractor, - "#pattern": r"https://8chan\.moe/\.media/[0-9a-f]{64}\.\w+$", - "#count" : 14, - - "archived" : False, - "autoSage" : False, - "boardDescription": "Film and Cinema", - "boardMarkdown" : None, - "boardName" : "Movies", - "boardUri" : "vhs", - "creation" : r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z", - "cyclic" : False, - "email" : None, - "id" : r"re:^[0-9a-f]{6}$", - "locked" : False, - "markdown" : str, - "maxFileCount" : 5, - "maxFileSize" : "32.00 MB", - "maxMessageLength": 12000, - "message" : str, - "mime" : str, - "name" : "Anonymous", - "num" : int, - "originalName" : str, - "path" : r"re:/.media/[0-9a-f]{64}\.\w+$", - "pinned" : False, - "postId" : int, - "signedRole" : None, - "size" : int, - "threadId" : 4, - "thumb" : r"re:/.media/t_[0-9a-f]{64}$", - "uniquePosters" : 9, - "usesCustomCss" : True, - "usesCustomJs" : False, - "?wsPort" : int, - "?wssPort" : int, -}, - -{ - "#url" : "https://8chan.moe/vhs/last/4.html", - "#class": _8chan._8chanThreadExtractor, -}, - -{ - "#url" : "https://8chan.se/vhs/res/4.html", - "#class": _8chan._8chanThreadExtractor, -}, - -{ - "#url" : "https://8chan.cc/vhs/res/4.html", - "#class": _8chan._8chanThreadExtractor, -}, - -{ - "#url" : "https://8chan.moe/vhs/", - "#class": _8chan._8chanBoardExtractor, -}, - -{ - "#url" : "https://8chan.moe/vhs/2.html", - "#class": _8chan._8chanBoardExtractor, - "#pattern": _8chan._8chanThreadExtractor.pattern, - "#count" : range(24, 32), -}, - -{ - "#url" : "https://8chan.se/vhs/", - "#class": _8chan._8chanBoardExtractor, -}, - -{ - "#url" : "https://8chan.cc/vhs/", - "#class": _8chan._8chanBoardExtractor, -}, - + { + "#url": "https://8chan.moe/vhs/res/4.html", + "#class": _8chan._8chanThreadExtractor, + "#pattern": r"https://8chan\.moe/\.media/[0-9a-f]{64}\.\w+$", + "#count": 14, + "archived": False, + "autoSage": False, + "boardDescription": "Film and Cinema", + "boardMarkdown": None, + "boardName": "Movies", + "boardUri": "vhs", + "creation": r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z", + "cyclic": False, + "email": None, + "id": r"re:^[0-9a-f]{6}$", + "locked": False, + "markdown": str, + "maxFileCount": 5, + "maxFileSize": "32.00 MB", + "maxMessageLength": 12000, + "message": str, + "mime": str, + "name": "Anonymous", + "num": int, + "originalName": str, + "path": r"re:/.media/[0-9a-f]{64}\.\w+$", + "pinned": False, + "postId": int, + "signedRole": None, + "size": int, + "threadId": 4, + "thumb": r"re:/.media/t_[0-9a-f]{64}$", + "uniquePosters": 9, + "usesCustomCss": True, + "usesCustomJs": False, + "?wsPort": int, + "?wssPort": int, + }, + { + "#url": "https://8chan.moe/vhs/last/4.html", + "#class": _8chan._8chanThreadExtractor, + }, + { + "#url": "https://8chan.se/vhs/res/4.html", + "#class": _8chan._8chanThreadExtractor, + }, + { + "#url": "https://8chan.cc/vhs/res/4.html", + "#class": _8chan._8chanThreadExtractor, + }, + { + "#url": "https://8chan.moe/vhs/", + "#class": _8chan._8chanBoardExtractor, + }, + { + "#url": "https://8chan.moe/vhs/2.html", + "#class": _8chan._8chanBoardExtractor, + "#pattern": _8chan._8chanThreadExtractor.pattern, + "#count": range(24, 32), + }, + { + "#url": "https://8chan.se/vhs/", + "#class": _8chan._8chanBoardExtractor, + }, + { + "#url": "https://8chan.cc/vhs/", + "#class": _8chan._8chanBoardExtractor, + }, ) diff --git a/test/results/8kun.py b/test/results/8kun.py index 53d16e13d..9923b6a74 100644 --- a/test/results/8kun.py +++ b/test/results/8kun.py @@ -1,39 +1,32 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import vichan - __tests__ = ( -{ - "#url" : "https://8kun.top/test/res/65248.html", - "#category": ("vichan", "8kun", "thread"), - "#class" : vichan.VichanThreadExtractor, - "#pattern" : r"https://media\.128ducks\.com/file_store/\w{64}\.\w+", - "#count" : ">= 8", -}, - -{ - "#url" : "https://8kun.top/v/index.html", - "#category": ("vichan", "8kun", "board"), - "#class" : vichan.VichanBoardExtractor, - "#pattern" : vichan.VichanThreadExtractor.pattern, - "#count" : ">= 100", -}, - -{ - "#url" : "https://8kun.top/v/2.html", - "#category": ("vichan", "8kun", "board"), - "#class" : vichan.VichanBoardExtractor, -}, - -{ - "#url" : "https://8kun.top/v/index.html?PageSpeed=noscript", - "#category": ("vichan", "8kun", "board"), - "#class" : vichan.VichanBoardExtractor, -}, - + { + "#url": "https://8kun.top/test/res/65248.html", + "#category": ("vichan", "8kun", "thread"), + "#class": vichan.VichanThreadExtractor, + "#pattern": r"https://media\.128ducks\.com/file_store/\w{64}\.\w+", + "#count": ">= 8", + }, + { + "#url": "https://8kun.top/v/index.html", + "#category": ("vichan", "8kun", "board"), + "#class": vichan.VichanBoardExtractor, + "#pattern": vichan.VichanThreadExtractor.pattern, + "#count": ">= 100", + }, + { + "#url": "https://8kun.top/v/2.html", + "#category": ("vichan", "8kun", "board"), + "#class": vichan.VichanBoardExtractor, + }, + { + "#url": "https://8kun.top/v/index.html?PageSpeed=noscript", + "#category": ("vichan", "8kun", "board"), + "#class": vichan.VichanBoardExtractor, + }, ) diff --git a/test/results/8muses.py b/test/results/8muses.py index 7dfb84608..b1038b2b4 100644 --- a/test/results/8muses.py +++ b/test/results/8muses.py @@ -1,72 +1,62 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +from gallery_dl import exception gallery_dl = __import__("gallery_dl.extractor.8muses") _8muses = getattr(gallery_dl.extractor, "8muses") -from gallery_dl import exception - __tests__ = ( -{ - "#url" : "https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", - "#category": ("", "8muses", "album"), - "#class" : _8muses._8musesAlbumExtractor, - "#pattern" : r"https://comics.8muses.com/image/fl/[\w-]+", - "#sha1_url": "6286ac33087c236c5a7e51f8a9d4e4d5548212d4", - - "url" : str, - "hash" : str, - "page" : int, - "count": 6, - "album": { - "id" : 10467, - "title" : "Liar", - "path" : "Fakku Comics/mogg/Liar", - "parts" : [ - "Fakku Comics", - "mogg", - "Liar", - ], + { + "#url": "https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", + "#category": ("", "8muses", "album"), + "#class": _8muses._8musesAlbumExtractor, + "#pattern": r"https://comics.8muses.com/image/fl/[\w-]+", + "#sha1_url": "6286ac33087c236c5a7e51f8a9d4e4d5548212d4", + "url": str, + "hash": str, + "page": int, + "count": 6, + "album": { + "id": 10467, + "title": "Liar", + "path": "Fakku Comics/mogg/Liar", + "parts": [ + "Fakku Comics", + "mogg", + "Liar", + ], + "private": False, + "url": "https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", + "parent": 10464, + "views": int, + "likes": int, + "date": "dt:2018-07-10 00:00:00", + }, + }, + { + "#url": "https://www.8muses.com/comics/album/Fakku-Comics/santa", + "#category": ("", "8muses", "album"), + "#class": _8muses._8musesAlbumExtractor, + "#pattern": _8muses._8musesAlbumExtractor.pattern, + "#count": ">= 3", + "url": str, + "name": str, "private": False, - "url" : "https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", - "parent" : 10464, - "views" : int, - "likes" : int, - "date" : "dt:2018-07-10 00:00:00", }, -}, - -{ - "#url" : "https://www.8muses.com/comics/album/Fakku-Comics/santa", - "#category": ("", "8muses", "album"), - "#class" : _8muses._8musesAlbumExtractor, - "#pattern" : _8muses._8musesAlbumExtractor.pattern, - "#count" : ">= 3", - - "url" : str, - "name" : str, - "private": False, -}, - -{ - "#url" : "https://www.8muses.com/comics/album/Fakku-Comics/11?sort=az", - "#comment" : "custom sorting", - "#category": ("", "8muses", "album"), - "#class" : _8muses._8musesAlbumExtractor, - "#count" : ">= 70", - - "name": r"re:^[R-Zr-z]", -}, - -{ - "#url" : "https://comics.8muses.com/comics/album/Various-Authors/Chessire88/From-Trainers-to-Pokmons", - "#comment" : "non-ASCII characters", - "#category": ("", "8muses", "album"), - "#class" : _8muses._8musesAlbumExtractor, - "#exception": exception.HttpError, -}, - + { + "#url": "https://www.8muses.com/comics/album/Fakku-Comics/11?sort=az", + "#comment": "custom sorting", + "#category": ("", "8muses", "album"), + "#class": _8muses._8musesAlbumExtractor, + "#count": ">= 70", + "name": r"re:^[R-Zr-z]", + }, + { + "#url": "https://comics.8muses.com/comics/album/Various-Authors/Chessire88/From-Trainers-to-Pokmons", + "#comment": "non-ASCII characters", + "#category": ("", "8muses", "album"), + "#class": _8muses._8musesAlbumExtractor, + "#exception": exception.HttpError, + }, ) diff --git a/test/results/94chan.py b/test/results/94chan.py index 6e792e305..b69835555 100644 --- a/test/results/94chan.py +++ b/test/results/94chan.py @@ -1,45 +1,37 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import jschan - __tests__ = ( -{ - "#url" : "https://94chan.org/art/thread/25.html", - "#category": ("jschan", "94chan", "thread"), - "#class" : jschan.JschanThreadExtractor, - "#pattern" : r"https://94chan.org/file/[0-9a-f]{64}(\.\w+)?", - "#count" : ">= 15", -}, - -{ - "#url" : "https://94chan.org/art/", - "#category": ("jschan", "94chan", "board"), - "#class" : jschan.JschanBoardExtractor, - "#pattern" : jschan.JschanThreadExtractor.pattern, - "#count" : ">= 30", -}, - -{ - "#url" : "https://94chan.org/art/2.html", - "#category": ("jschan", "94chan", "board"), - "#class" : jschan.JschanBoardExtractor, -}, - -{ - "#url" : "https://94chan.org/art/catalog.html", - "#category": ("jschan", "94chan", "board"), - "#class" : jschan.JschanBoardExtractor, -}, - -{ - "#url" : "https://94chan.org/art/index.html", - "#category": ("jschan", "94chan", "board"), - "#class" : jschan.JschanBoardExtractor, -}, - + { + "#url": "https://94chan.org/art/thread/25.html", + "#category": ("jschan", "94chan", "thread"), + "#class": jschan.JschanThreadExtractor, + "#pattern": r"https://94chan.org/file/[0-9a-f]{64}(\.\w+)?", + "#count": ">= 15", + }, + { + "#url": "https://94chan.org/art/", + "#category": ("jschan", "94chan", "board"), + "#class": jschan.JschanBoardExtractor, + "#pattern": jschan.JschanThreadExtractor.pattern, + "#count": ">= 30", + }, + { + "#url": "https://94chan.org/art/2.html", + "#category": ("jschan", "94chan", "board"), + "#class": jschan.JschanBoardExtractor, + }, + { + "#url": "https://94chan.org/art/catalog.html", + "#category": ("jschan", "94chan", "board"), + "#class": jschan.JschanBoardExtractor, + }, + { + "#url": "https://94chan.org/art/index.html", + "#category": ("jschan", "94chan", "board"), + "#class": jschan.JschanBoardExtractor, + }, ) diff --git a/test/results/__init__.py b/test/results/__init__.py index 0865693ba..8ca52b325 100644 --- a/test/results/__init__.py +++ b/test/results/__init__.py @@ -1,16 +1,14 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import os import functools +import os __directory__ = os.path.dirname(__file__) -@functools.lru_cache(maxsize=None) +@functools.cache def tests(name): module = __import__(name, globals(), None, (), 1) return module.__tests__ diff --git a/test/results/acidimg.py b/test/results/acidimg.py index d61a7736d..c3b4c49d8 100644 --- a/test/results/acidimg.py +++ b/test/results/acidimg.py @@ -1,20 +1,16 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://acidimg.cc/img-5acb6b9de4640.html", - "#category": ("imagehost", "acidimg", "image"), - "#class" : imagehosts.AcidimgImageExtractor, - "#sha1_url" : "f132a630006e8d84f52d59555191ed82b3b64c04", - "#sha1_metadata": "135347ab4345002fc013863c0d9419ba32d98f78", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - + { + "#url": "https://acidimg.cc/img-5acb6b9de4640.html", + "#category": ("imagehost", "acidimg", "image"), + "#class": imagehosts.AcidimgImageExtractor, + "#sha1_url": "f132a630006e8d84f52d59555191ed82b3b64c04", + "#sha1_metadata": "135347ab4345002fc013863c0d9419ba32d98f78", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, ) diff --git a/test/results/adultempire.py b/test/results/adultempire.py index 1a2953ef8..936707d25 100644 --- a/test/results/adultempire.py +++ b/test/results/adultempire.py @@ -1,28 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import adultempire - __tests__ = ( -{ - "#url" : "https://www.adultempire.com/5998/gallery.html", - "#category": ("", "adultempire", "gallery"), - "#class" : adultempire.AdultempireGalleryExtractor, - "#range" : "1", - "#sha1_metadata": "5b3266e69801db0d78c22181da23bc102886e027", - "#sha1_content" : "5c6beb31e5e3cdc90ee5910d5c30f9aaec977b9e", -}, - -{ - "#url" : "https://www.adultdvdempire.com/5683/gallery.html", - "#category": ("", "adultempire", "gallery"), - "#class" : adultempire.AdultempireGalleryExtractor, - "#sha1_url" : "b12cd1a65cae8019d837505adb4d6a2c1ed4d70d", - "#sha1_metadata": "8d448d79c4ac5f5b10a3019d5b5129ddb43655e5", -}, - + { + "#url": "https://www.adultempire.com/5998/gallery.html", + "#category": ("", "adultempire", "gallery"), + "#class": adultempire.AdultempireGalleryExtractor, + "#range": "1", + "#sha1_metadata": "5b3266e69801db0d78c22181da23bc102886e027", + "#sha1_content": "5c6beb31e5e3cdc90ee5910d5c30f9aaec977b9e", + }, + { + "#url": "https://www.adultdvdempire.com/5683/gallery.html", + "#category": ("", "adultempire", "gallery"), + "#class": adultempire.AdultempireGalleryExtractor, + "#sha1_url": "b12cd1a65cae8019d837505adb4d6a2c1ed4d70d", + "#sha1_metadata": "8d448d79c4ac5f5b10a3019d5b5129ddb43655e5", + }, ) diff --git a/test/results/agnph.py b/test/results/agnph.py index ae5c0d24c..ee4a0b3c5 100644 --- a/test/results/agnph.py +++ b/test/results/agnph.py @@ -1,54 +1,47 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import agnph - __tests__ = ( -{ - "#url" : "https://agn.ph/gallery/post/?search=azuu", - "#category": ("booru", "agnph", "tag"), - "#class" : agnph.AgnphTagExtractor, - "#pattern" : r"http://agn\.ph/gallery/data/../../\w{32}\.jpg", - "#count" : ">= 50", -}, - -{ - "#url" : "https://agn.ph/gallery/post/show/501604/", - "#category": ("booru", "agnph", "post"), - "#class" : agnph.AgnphPostExtractor, - "#options" : {"tags": True}, - "#urls" : "http://agn.ph/gallery/data/7d/a5/7da50021f3e86f6cf1c215652060d772.png", - "#sha1_content": "93c8b2d3f53e891ad8fa68d5f60f8c7a70acd836", - - "artist" : "reyn_goldfur", - "created_at" : "1722041591", - "creator_id" : "-1", - "date" : "dt:2024-07-27 00:53:11", - "description" : None, - "fav_count" : "0", - "file_ext" : "png", - "file_url" : "http://agn.ph/gallery/data/7d/a5/7da50021f3e86f6cf1c215652060d772.png", - "has_children": False, - "height" : "1000", - "id" : "501604", - "md5" : "7da50021f3e86f6cf1c215652060d772", - "num_comments": "0", - "parent_id" : None, - "rating" : "e", - "source" : "https://inkbunny.net/s/2886519", - "status" : "approved", - "tags" : "anthro female hisuian_sneasel regional_form reyn_goldfur shelly_the_sneasel sneasel solo", - "tags_artist" : "reyn_goldfur", - "tags_character": "shelly_the_sneasel", - "tags_general": "anthro female solo", - "tags_species": "hisuian_sneasel regional_form sneasel", - "thumbnail_url": "http://agn.ph/gallery/data/thumb/7d/a5/7da50021f3e86f6cf1c215652060d772.png", - "width" : "953", - -}, - + { + "#url": "https://agn.ph/gallery/post/?search=azuu", + "#category": ("booru", "agnph", "tag"), + "#class": agnph.AgnphTagExtractor, + "#pattern": r"http://agn\.ph/gallery/data/../../\w{32}\.jpg", + "#count": ">= 50", + }, + { + "#url": "https://agn.ph/gallery/post/show/501604/", + "#category": ("booru", "agnph", "post"), + "#class": agnph.AgnphPostExtractor, + "#options": {"tags": True}, + "#urls": "http://agn.ph/gallery/data/7d/a5/7da50021f3e86f6cf1c215652060d772.png", + "#sha1_content": "93c8b2d3f53e891ad8fa68d5f60f8c7a70acd836", + "artist": "reyn_goldfur", + "created_at": "1722041591", + "creator_id": "-1", + "date": "dt:2024-07-27 00:53:11", + "description": None, + "fav_count": "0", + "file_ext": "png", + "file_url": "http://agn.ph/gallery/data/7d/a5/7da50021f3e86f6cf1c215652060d772.png", + "has_children": False, + "height": "1000", + "id": "501604", + "md5": "7da50021f3e86f6cf1c215652060d772", + "num_comments": "0", + "parent_id": None, + "rating": "e", + "source": "https://inkbunny.net/s/2886519", + "status": "approved", + "tags": "anthro female hisuian_sneasel regional_form reyn_goldfur shelly_the_sneasel sneasel solo", + "tags_artist": "reyn_goldfur", + "tags_character": "shelly_the_sneasel", + "tags_general": "anthro female solo", + "tags_species": "hisuian_sneasel regional_form sneasel", + "thumbnail_url": "http://agn.ph/gallery/data/thumb/7d/a5/7da50021f3e86f6cf1c215652060d772.png", + "width": "953", + }, ) diff --git a/test/results/aibooru.py b/test/results/aibooru.py index 414084239..b27ca4609 100644 --- a/test/results/aibooru.py +++ b/test/results/aibooru.py @@ -1,44 +1,36 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import danbooru - __tests__ = ( -{ - "#url" : "https://aibooru.online/posts?tags=center_frills&z=1", - "#category": ("Danbooru", "aibooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, - "#pattern" : r"https://cdn\.aibooru\.download/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", - "#count" : ">= 50", -}, - -{ - "#url" : "https://safe.aibooru.online/posts?tags=center_frills", - "#category": ("Danbooru", "aibooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, -}, - -{ - "#url" : "https://aibooru.online/pools/1", - "#category": ("Danbooru", "aibooru", "pool"), - "#class" : danbooru.DanbooruPoolExtractor, -}, - -{ - "#url" : "https://aibooru.online/posts/1", - "#category": ("Danbooru", "aibooru", "post"), - "#class" : danbooru.DanbooruPostExtractor, - "#sha1_content": "54d548743cd67799a62c77cbae97cfa0fec1b7e9", -}, - -{ - "#url" : "https://aibooru.online/explore/posts/popular", - "#category": ("Danbooru", "aibooru", "popular"), - "#class" : danbooru.DanbooruPopularExtractor, -}, - + { + "#url": "https://aibooru.online/posts?tags=center_frills&z=1", + "#category": ("Danbooru", "aibooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + "#pattern": r"https://cdn\.aibooru\.download/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", + "#count": ">= 50", + }, + { + "#url": "https://safe.aibooru.online/posts?tags=center_frills", + "#category": ("Danbooru", "aibooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + }, + { + "#url": "https://aibooru.online/pools/1", + "#category": ("Danbooru", "aibooru", "pool"), + "#class": danbooru.DanbooruPoolExtractor, + }, + { + "#url": "https://aibooru.online/posts/1", + "#category": ("Danbooru", "aibooru", "post"), + "#class": danbooru.DanbooruPostExtractor, + "#sha1_content": "54d548743cd67799a62c77cbae97cfa0fec1b7e9", + }, + { + "#url": "https://aibooru.online/explore/posts/popular", + "#category": ("Danbooru", "aibooru", "popular"), + "#class": danbooru.DanbooruPopularExtractor, + }, ) diff --git a/test/results/allgirlbooru.py b/test/results/allgirlbooru.py index e1bfc11fd..83887f7c2 100644 --- a/test/results/allgirlbooru.py +++ b/test/results/allgirlbooru.py @@ -1,47 +1,40 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v01 - __tests__ = ( -{ - "#url" : "https://allgirl.booru.org/index.php?page=post&s=list&tags=dress", - "#category": ("gelbooru_v01", "allgirlbooru", "tag"), - "#class" : gelbooru_v01.GelbooruV01TagExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://allgirl.booru.org/index.php?page=favorites&s=view&id=380", - "#category": ("gelbooru_v01", "allgirlbooru", "favorite"), - "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, - "#count" : 4, -}, - -{ - "#url" : "https://allgirl.booru.org/index.php?page=post&s=view&id=107213", - "#category": ("gelbooru_v01", "allgirlbooru", "post"), - "#class" : gelbooru_v01.GelbooruV01PostExtractor, - "#sha1_url" : "b416800d2d2b072f80d3b37cfca9cb806fb25d51", - "#sha1_content": "3e3c65e0854a988696e11adf0de52f8fa90a51c7", - - "created_at": "2021-02-13 16:27:39", - "date" : "dt:2021-02-13 16:27:39", - "file_url" : "https://img.booru.org/allgirl//images/107/2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb.jpg", - "height" : "1200", - "id" : "107213", - "md5" : "2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb", - "rating" : "s", - "score" : str, - "source" : "", - "tags" : "blush dress green_eyes green_hair hatsune_miku long_hair twintails vocaloid", - "uploader" : "Honochi31", - "width" : "1600", -}, - + { + "#url": "https://allgirl.booru.org/index.php?page=post&s=list&tags=dress", + "#category": ("gelbooru_v01", "allgirlbooru", "tag"), + "#class": gelbooru_v01.GelbooruV01TagExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://allgirl.booru.org/index.php?page=favorites&s=view&id=380", + "#category": ("gelbooru_v01", "allgirlbooru", "favorite"), + "#class": gelbooru_v01.GelbooruV01FavoriteExtractor, + "#count": 4, + }, + { + "#url": "https://allgirl.booru.org/index.php?page=post&s=view&id=107213", + "#category": ("gelbooru_v01", "allgirlbooru", "post"), + "#class": gelbooru_v01.GelbooruV01PostExtractor, + "#sha1_url": "b416800d2d2b072f80d3b37cfca9cb806fb25d51", + "#sha1_content": "3e3c65e0854a988696e11adf0de52f8fa90a51c7", + "created_at": "2021-02-13 16:27:39", + "date": "dt:2021-02-13 16:27:39", + "file_url": "https://img.booru.org/allgirl//images/107/2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb.jpg", + "height": "1200", + "id": "107213", + "md5": "2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb", + "rating": "s", + "score": str, + "source": "", + "tags": "blush dress green_eyes green_hair hatsune_miku long_hair twintails vocaloid", + "uploader": "Honochi31", + "width": "1600", + }, ) diff --git a/test/results/animereactor.py b/test/results/animereactor.py index a83ac6ff8..12305d9eb 100644 --- a/test/results/animereactor.py +++ b/test/results/animereactor.py @@ -1,29 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import reactor - __tests__ = ( -{ - "#url" : "http://anime.reactor.cc/tag/Anime+Art", - "#category": ("reactor", "anime.reactor", "tag"), - "#class" : reactor.ReactorTagExtractor, -}, - -{ - "#url" : "http://anime.reactor.cc/user/Shuster", - "#category": ("reactor", "anime.reactor", "user"), - "#class" : reactor.ReactorUserExtractor, -}, - -{ - "#url" : "http://anime.reactor.cc/post/3576250", - "#category": ("reactor", "anime.reactor", "post"), - "#class" : reactor.ReactorPostExtractor, -}, - + { + "#url": "http://anime.reactor.cc/tag/Anime+Art", + "#category": ("reactor", "anime.reactor", "tag"), + "#class": reactor.ReactorTagExtractor, + }, + { + "#url": "http://anime.reactor.cc/user/Shuster", + "#category": ("reactor", "anime.reactor", "user"), + "#class": reactor.ReactorUserExtractor, + }, + { + "#url": "http://anime.reactor.cc/post/3576250", + "#category": ("reactor", "anime.reactor", "post"), + "#class": reactor.ReactorPostExtractor, + }, ) diff --git a/test/results/ao3.py b/test/results/ao3.py index 4b3636738..d0f0f157a 100644 --- a/test/results/ao3.py +++ b/test/results/ao3.py @@ -1,248 +1,228 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import ao3 - __tests__ = ( -{ - "#url" : "https://archiveofourown.org/works/47802076", - "#class" : ao3.Ao3WorkExtractor, - "#urls" : "https://archiveofourown.org/downloads/47802076/The_Wildcard.pdf?updated_at=1720398424", - - "author" : "Flowers_for_ghouls", - "bookmarks": range(100, 300), - "chapters": { - "120506833": "1. Showtime", - "120866506": "2. A Comedy of Errors", - "121739140": "3. Gifts", - "121941313": "4. Date Night", - "123054364": "5. Breaking the News", - "123579898": "6. Isolated Events", - "124258153": "7. The Home Stretch", - "124886536": "8. Domestic Bliss", - "125335270": "9. The Offer", - "125871166": "10. The Promise", - "126223879": "11. Gifts II", - "126692398": "12. On the Move", - "127471375": "13. The Fruit Vignettes", - "128496448": "14. Respite", - "128994919": "15. Changes", - "129492154": "16. Halloween", - "130379002": "17. GIfts III", - "131066743": "18. R.A.S.B.E.W.", - "131884072": "19. The Longest Night", - "132730264": "20. Meeting the Pack", - "133714876": "21. A Mystery", - "134663854": "22. Growing Pains", - "135499822": "23. Presentation Day", - "136500946": "24. Revelations", - "137857876": "25. The Retirement Plan", - "139463056": "26. Two Birds, One Stone", - "141697141": "27. New Management", - }, - "comments" : range(800, 2000), - "date" : "dt:2023-06-11 00:00:00", - "date_completed": "dt:2024-05-10 00:00:00", - "date_updated" : "dt:2024-07-08 00:27:04", - "extension": "pdf", - "filename" : "The_Wildcard", - "id" : 47802076, - "lang" : "en", - "language" : "English", - "likes" : range(1000, 2000), - "series" : { - "id" : "4237024", - "prev" : "", - "next" : "57205801", - "index": "1", - "name" : "The Wildcard Universe", - }, - "title" : "The Wildcard", - "views" : range(34000, 50000), - "words" : 217549, - - "categories": [ - "Gen", - "M/M", - ], - "characters": [ - "Dewdrop Ghoul | Fire Ghoul", - "Aether | Quintessence Ghoul", - "Multi Ghoul | Swiss Army Ghoul", - "Rain | Water Ghoul", - "Cirrus | Air Ghoulette", - "Cumulus | Air Ghoulette", - "Sunshine Ghoulette", - "Mountain | Earth Ghoul", - "Cardinal Copia", - "Phantom Ghoul", - "Aurora Ghoulette", - "Sister Imperator (Ghost Sweden Band)", - ], - "fandom": [ - "Ghost (Sweden Band)", - ], - "rating": [ - "Mature", - ], - "relationships": [ - "Aether | Quintessence Ghoul/Dewdrop Ghoul | Fire Ghoul", - "Multi Ghoul | Swiss Army Ghoul/Rain | Water Ghoul", - ], - "summary": [ - "Aether has been asked to stay at the ministry to manage the renovation of the new infirmary. It couldn’t have been worse timing. Barely days into the new tour, Dew realizes he’s carrying their first kit.", - ], - "tags": [ - "Domestic Fluff", - "Pack Dynamics", - "gratuitous fluff", - "How do ghouls work?", - "they don't even know", - "but it's cute", - "Pregnant Dewdrop", - "Recreational Drug Use", - "Cowbell!", - "Protective Ghouls", - "no beta we die like Nihil", - "sick dewdrop", - "TW: Vomiting", - "Aether really loves Dew", - "Nesting", - "Ghoul Piles (Ghost Sweden Band)", - "Angst", - "Hurt/Comfort", - "Original Ghoul Kit(s) (Ghost Sweden Band)", - "Kit fic", - ], - "warnings": [ - "No Archive Warnings Apply", - ], -}, - -{ - "#url" : "https://archiveofourown.org/works/47802076", - "#class" : ao3.Ao3WorkExtractor, - "#options" : {"formats": ["epub", "mobi", "azw3", "pdf", "html"]}, - "#urls" : ( - "https://archiveofourown.org/downloads/47802076/The_Wildcard.epub?updated_at=1720398424", - "https://archiveofourown.org/downloads/47802076/The_Wildcard.mobi?updated_at=1720398424", - "https://archiveofourown.org/downloads/47802076/The_Wildcard.azw3?updated_at=1720398424", - "https://archiveofourown.org/downloads/47802076/The_Wildcard.pdf?updated_at=1720398424", - "https://archiveofourown.org/downloads/47802076/The_Wildcard.html?updated_at=1720398424", - ), -}, - -{ - "#url" : "https://archiveofourown.org/works/12345", - "#comment" : "restricted work / login required", - "#class" : ao3.Ao3WorkExtractor, - "#auth" : True, - "#urls" : "https://archiveofourown.org/downloads/12345/Unquenchable.pdf?updated_at=1716029699", -}, - -{ - "#url" : "https://archiveofourown.org/series/1903930", - "#class" : ao3.Ao3SeriesExtractor, - "#urls" : ( - "https://archiveofourown.org/works/26131546", - "https://archiveofourown.org/works/26291101", - "https://archiveofourown.org/works/26325292", - ), -}, - -{ - "#url" : "https://archiveofourown.org/tags/Sunshine%20(Ghost%20Sweden%20Band)/works", - "#class" : ao3.Ao3TagExtractor, - "#pattern" : ao3.Ao3WorkExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://archiveofourown.org/works/search?work_search%5Bquery%5D=air+fire+ice+water", - "#class" : ao3.Ao3SearchExtractor, - "#pattern" : ao3.Ao3WorkExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://archiveofourown.org/users/Fyrelass", - "#class" : ao3.Ao3UserExtractor, - "#urls" : ( - "https://archiveofourown.org/users/Fyrelass/works", - "https://archiveofourown.org/users/Fyrelass/series", - ), -}, -{ - "#url" : "https://archiveofourown.com/users/Fyrelass", - "#class" : ao3.Ao3UserExtractor, -}, -{ - "#url" : "https://archiveofourown.net/users/Fyrelass", - "#class" : ao3.Ao3UserExtractor, -}, -{ - "#url" : "https://ao3.org/users/Fyrelass", - "#class" : ao3.Ao3UserExtractor, -}, - -{ - "#url" : "https://archiveofourown.org/users/Fyrelass/profile", - "#class" : ao3.Ao3UserExtractor, -}, - -{ - "#url" : "https://archiveofourown.org/users/Fyrelass/pseuds/Aileen%20Autarkeia", - "#class" : ao3.Ao3UserExtractor, -}, - -{ - "#url" : "https://archiveofourown.org/users/Fyrelass/works", - "#class" : ao3.Ao3UserWorksExtractor, - "#auth" : False, - "#urls" : ( - "https://archiveofourown.org/works/55035061", - "https://archiveofourown.org/works/58979287", - "https://archiveofourown.org/works/52704457", - "https://archiveofourown.org/works/52502743", - "https://archiveofourown.org/works/52170409", - "https://archiveofourown.org/works/52078558", - "https://archiveofourown.org/works/51699982", - "https://archiveofourown.org/works/51975193", - "https://archiveofourown.org/works/51633877", - "https://archiveofourown.org/works/51591436", - "https://archiveofourown.org/works/50860891", - ), -}, - -{ - "#url" : "https://archiveofourown.org/users/Fyrelass/series", - "#class" : ao3.Ao3UserSeriesExtractor, - "#auth" : False, - "#urls" : ( - "https://archiveofourown.org/series/3821575", - ), -}, - -{ - "#url" : "https://archiveofourown.org/users/Fyrelass/bookmarks", - "#class" : ao3.Ao3UserBookmarkExtractor, - "#pattern" : r"https://archiveofourown\.org/(work|serie)s/\d+", - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://archiveofourown.org/users/mikf/subscriptions", - "#class" : ao3.Ao3SubscriptionsExtractor, - "#auth" : True, - "#pattern" : r"https://archiveofourown\.org/(work|serie|user)s/\w+", - "#count" : range(20, 30), -}, - + { + "#url": "https://archiveofourown.org/works/47802076", + "#class": ao3.Ao3WorkExtractor, + "#urls": "https://archiveofourown.org/downloads/47802076/The_Wildcard.pdf?updated_at=1720398424", + "author": "Flowers_for_ghouls", + "bookmarks": range(100, 300), + "chapters": { + "120506833": "1. Showtime", + "120866506": "2. A Comedy of Errors", + "121739140": "3. Gifts", + "121941313": "4. Date Night", + "123054364": "5. Breaking the News", + "123579898": "6. Isolated Events", + "124258153": "7. The Home Stretch", + "124886536": "8. Domestic Bliss", + "125335270": "9. The Offer", + "125871166": "10. The Promise", + "126223879": "11. Gifts II", + "126692398": "12. On the Move", + "127471375": "13. The Fruit Vignettes", + "128496448": "14. Respite", + "128994919": "15. Changes", + "129492154": "16. Halloween", + "130379002": "17. GIfts III", + "131066743": "18. R.A.S.B.E.W.", + "131884072": "19. The Longest Night", + "132730264": "20. Meeting the Pack", + "133714876": "21. A Mystery", + "134663854": "22. Growing Pains", + "135499822": "23. Presentation Day", + "136500946": "24. Revelations", + "137857876": "25. The Retirement Plan", + "139463056": "26. Two Birds, One Stone", + "141697141": "27. New Management", + }, + "comments": range(800, 2000), + "date": "dt:2023-06-11 00:00:00", + "date_completed": "dt:2024-05-10 00:00:00", + "date_updated": "dt:2024-07-08 00:27:04", + "extension": "pdf", + "filename": "The_Wildcard", + "id": 47802076, + "lang": "en", + "language": "English", + "likes": range(1000, 2000), + "series": { + "id": "4237024", + "prev": "", + "next": "57205801", + "index": "1", + "name": "The Wildcard Universe", + }, + "title": "The Wildcard", + "views": range(34000, 50000), + "words": 217549, + "categories": [ + "Gen", + "M/M", + ], + "characters": [ + "Dewdrop Ghoul | Fire Ghoul", + "Aether | Quintessence Ghoul", + "Multi Ghoul | Swiss Army Ghoul", + "Rain | Water Ghoul", + "Cirrus | Air Ghoulette", + "Cumulus | Air Ghoulette", + "Sunshine Ghoulette", + "Mountain | Earth Ghoul", + "Cardinal Copia", + "Phantom Ghoul", + "Aurora Ghoulette", + "Sister Imperator (Ghost Sweden Band)", + ], + "fandom": [ + "Ghost (Sweden Band)", + ], + "rating": [ + "Mature", + ], + "relationships": [ + "Aether | Quintessence Ghoul/Dewdrop Ghoul | Fire Ghoul", + "Multi Ghoul | Swiss Army Ghoul/Rain | Water Ghoul", + ], + "summary": [ + "Aether has been asked to stay at the ministry to manage the renovation of the new infirmary. It couldn’t have been worse timing. Barely days into the new tour, Dew realizes he’s carrying their first kit.", + ], + "tags": [ + "Domestic Fluff", + "Pack Dynamics", + "gratuitous fluff", + "How do ghouls work?", + "they don't even know", + "but it's cute", + "Pregnant Dewdrop", + "Recreational Drug Use", + "Cowbell!", + "Protective Ghouls", + "no beta we die like Nihil", + "sick dewdrop", + "TW: Vomiting", + "Aether really loves Dew", + "Nesting", + "Ghoul Piles (Ghost Sweden Band)", + "Angst", + "Hurt/Comfort", + "Original Ghoul Kit(s) (Ghost Sweden Band)", + "Kit fic", + ], + "warnings": [ + "No Archive Warnings Apply", + ], + }, + { + "#url": "https://archiveofourown.org/works/47802076", + "#class": ao3.Ao3WorkExtractor, + "#options": {"formats": ["epub", "mobi", "azw3", "pdf", "html"]}, + "#urls": ( + "https://archiveofourown.org/downloads/47802076/The_Wildcard.epub?updated_at=1720398424", + "https://archiveofourown.org/downloads/47802076/The_Wildcard.mobi?updated_at=1720398424", + "https://archiveofourown.org/downloads/47802076/The_Wildcard.azw3?updated_at=1720398424", + "https://archiveofourown.org/downloads/47802076/The_Wildcard.pdf?updated_at=1720398424", + "https://archiveofourown.org/downloads/47802076/The_Wildcard.html?updated_at=1720398424", + ), + }, + { + "#url": "https://archiveofourown.org/works/12345", + "#comment": "restricted work / login required", + "#class": ao3.Ao3WorkExtractor, + "#auth": True, + "#urls": "https://archiveofourown.org/downloads/12345/Unquenchable.pdf?updated_at=1716029699", + }, + { + "#url": "https://archiveofourown.org/series/1903930", + "#class": ao3.Ao3SeriesExtractor, + "#urls": ( + "https://archiveofourown.org/works/26131546", + "https://archiveofourown.org/works/26291101", + "https://archiveofourown.org/works/26325292", + ), + }, + { + "#url": "https://archiveofourown.org/tags/Sunshine%20(Ghost%20Sweden%20Band)/works", + "#class": ao3.Ao3TagExtractor, + "#pattern": ao3.Ao3WorkExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://archiveofourown.org/works/search?work_search%5Bquery%5D=air+fire+ice+water", + "#class": ao3.Ao3SearchExtractor, + "#pattern": ao3.Ao3WorkExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://archiveofourown.org/users/Fyrelass", + "#class": ao3.Ao3UserExtractor, + "#urls": ( + "https://archiveofourown.org/users/Fyrelass/works", + "https://archiveofourown.org/users/Fyrelass/series", + ), + }, + { + "#url": "https://archiveofourown.com/users/Fyrelass", + "#class": ao3.Ao3UserExtractor, + }, + { + "#url": "https://archiveofourown.net/users/Fyrelass", + "#class": ao3.Ao3UserExtractor, + }, + { + "#url": "https://ao3.org/users/Fyrelass", + "#class": ao3.Ao3UserExtractor, + }, + { + "#url": "https://archiveofourown.org/users/Fyrelass/profile", + "#class": ao3.Ao3UserExtractor, + }, + { + "#url": "https://archiveofourown.org/users/Fyrelass/pseuds/Aileen%20Autarkeia", + "#class": ao3.Ao3UserExtractor, + }, + { + "#url": "https://archiveofourown.org/users/Fyrelass/works", + "#class": ao3.Ao3UserWorksExtractor, + "#auth": False, + "#urls": ( + "https://archiveofourown.org/works/55035061", + "https://archiveofourown.org/works/58979287", + "https://archiveofourown.org/works/52704457", + "https://archiveofourown.org/works/52502743", + "https://archiveofourown.org/works/52170409", + "https://archiveofourown.org/works/52078558", + "https://archiveofourown.org/works/51699982", + "https://archiveofourown.org/works/51975193", + "https://archiveofourown.org/works/51633877", + "https://archiveofourown.org/works/51591436", + "https://archiveofourown.org/works/50860891", + ), + }, + { + "#url": "https://archiveofourown.org/users/Fyrelass/series", + "#class": ao3.Ao3UserSeriesExtractor, + "#auth": False, + "#urls": ("https://archiveofourown.org/series/3821575",), + }, + { + "#url": "https://archiveofourown.org/users/Fyrelass/bookmarks", + "#class": ao3.Ao3UserBookmarkExtractor, + "#pattern": r"https://archiveofourown\.org/(work|serie)s/\d+", + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://archiveofourown.org/users/mikf/subscriptions", + "#class": ao3.Ao3SubscriptionsExtractor, + "#auth": True, + "#pattern": r"https://archiveofourown\.org/(work|serie|user)s/\w+", + "#count": range(20, 30), + }, ) diff --git a/test/results/architizer.py b/test/results/architizer.py index 3a7515569..7f2d0bdbe 100644 --- a/test/results/architizer.py +++ b/test/results/architizer.py @@ -1,40 +1,34 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import architizer - __tests__ = ( -{ - "#url" : "https://architizer.com/projects/house-lo/", - "#category": ("", "architizer", "project"), - "#class" : architizer.ArchitizerProjectExtractor, - "#pattern" : r"https://architizer-prod\.imgix\.net/media/mediadata/uploads/.+\.jpg$", - - "count" : 27, - "description": str, - "firm" : "Atelier Lina Bellovicova", - "gid" : "225496", - "location" : "Czechia", - "num" : int, - "size" : "1000 sqft - 3000 sqft", - "slug" : "house-lo", - "status" : "Built", - "subcategory": "project", - "title" : "House LO", - "type" : "Residential › Private House", - "year" : "2020", -}, - -{ - "#url" : "https://architizer.com/firms/olson-kundig/", - "#category": ("", "architizer", "firm"), - "#class" : architizer.ArchitizerFirmExtractor, - "#pattern" : architizer.ArchitizerProjectExtractor.pattern, - "#count" : ">= 90", -}, - + { + "#url": "https://architizer.com/projects/house-lo/", + "#category": ("", "architizer", "project"), + "#class": architizer.ArchitizerProjectExtractor, + "#pattern": r"https://architizer-prod\.imgix\.net/media/mediadata/uploads/.+\.jpg$", + "count": 27, + "description": str, + "firm": "Atelier Lina Bellovicova", + "gid": "225496", + "location": "Czechia", + "num": int, + "size": "1000 sqft - 3000 sqft", + "slug": "house-lo", + "status": "Built", + "subcategory": "project", + "title": "House LO", + "type": "Residential › Private House", + "year": "2020", + }, + { + "#url": "https://architizer.com/firms/olson-kundig/", + "#category": ("", "architizer", "firm"), + "#class": architizer.ArchitizerFirmExtractor, + "#pattern": architizer.ArchitizerProjectExtractor.pattern, + "#count": ">= 90", + }, ) diff --git a/test/results/archivedmoe.py b/test/results/archivedmoe.py index 42aae1815..afa539c6a 100644 --- a/test/results/archivedmoe.py +++ b/test/results/archivedmoe.py @@ -1,56 +1,47 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://archived.moe/gd/thread/309639/", - "#category": ("foolfuuka", "archivedmoe", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url" : "fdd533840e2d535abd162c02d6dfadbc12e2dcd8", - "#sha1_content": "c27e2a7be3bc989b5dd859f7789cc854db3f5573", -}, - -{ - "#url" : "https://archived.moe/a/thread/159767162/", - "#category": ("foolfuuka", "archivedmoe", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "ffec05a1a1b906b5ca85992513671c9155ee9e87", -}, - -{ - "#url" : "https://archived.moe/b/thread/912594917/", - "#comment" : "broken thebarchive .webm URLs (#5116)", - "#category": ("foolfuuka", "archivedmoe", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#urls" : ( - "https://thebarchive.com/b/full_image/1705625299234839.gif", - "https://thebarchive.com/b/full_image/1705625431133806.web", - "https://thebarchive.com/b/full_image/1705626190307840.web", - ), -}, - -{ - "#url" : "https://archived.moe/gd/", - "#category": ("foolfuuka", "archivedmoe", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://archived.moe/_/search/text/test/", - "#category": ("foolfuuka", "archivedmoe", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://archived.moe/gd/gallery/2", - "#category": ("foolfuuka", "archivedmoe", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://archived.moe/gd/thread/309639/", + "#category": ("foolfuuka", "archivedmoe", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "fdd533840e2d535abd162c02d6dfadbc12e2dcd8", + "#sha1_content": "c27e2a7be3bc989b5dd859f7789cc854db3f5573", + }, + { + "#url": "https://archived.moe/a/thread/159767162/", + "#category": ("foolfuuka", "archivedmoe", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "ffec05a1a1b906b5ca85992513671c9155ee9e87", + }, + { + "#url": "https://archived.moe/b/thread/912594917/", + "#comment": "broken thebarchive .webm URLs (#5116)", + "#category": ("foolfuuka", "archivedmoe", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#urls": ( + "https://thebarchive.com/b/full_image/1705625299234839.gif", + "https://thebarchive.com/b/full_image/1705625431133806.web", + "https://thebarchive.com/b/full_image/1705626190307840.web", + ), + }, + { + "#url": "https://archived.moe/gd/", + "#category": ("foolfuuka", "archivedmoe", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://archived.moe/_/search/text/test/", + "#category": ("foolfuuka", "archivedmoe", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://archived.moe/gd/gallery/2", + "#category": ("foolfuuka", "archivedmoe", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/archiveofsins.py b/test/results/archiveofsins.py index 3f3fcb939..77a15dde7 100644 --- a/test/results/archiveofsins.py +++ b/test/results/archiveofsins.py @@ -1,43 +1,35 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://archiveofsins.com/h/thread/4668813/", - "#category": ("foolfuuka", "archiveofsins", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url" : "f612d287087e10a228ef69517cf811539db9a102", - "#sha1_content": "0dd92d0d8a7bf6e2f7d1f5ac8954c1bcf18c22a4", -}, - -{ - "#url" : "https://archiveofsins.com/h/", - "#category": ("foolfuuka", "archiveofsins", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://archiveofsins.com/_/search/text/test/", - "#category": ("foolfuuka", "archiveofsins", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://archiveofsins.com/_/search/text/test/", - "#category": ("foolfuuka", "archiveofsins", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://archiveofsins.com/h/gallery/3", - "#category": ("foolfuuka", "archiveofsins", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://archiveofsins.com/h/thread/4668813/", + "#category": ("foolfuuka", "archiveofsins", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "f612d287087e10a228ef69517cf811539db9a102", + "#sha1_content": "0dd92d0d8a7bf6e2f7d1f5ac8954c1bcf18c22a4", + }, + { + "#url": "https://archiveofsins.com/h/", + "#category": ("foolfuuka", "archiveofsins", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://archiveofsins.com/_/search/text/test/", + "#category": ("foolfuuka", "archiveofsins", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://archiveofsins.com/_/search/text/test/", + "#category": ("foolfuuka", "archiveofsins", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://archiveofsins.com/h/gallery/3", + "#category": ("foolfuuka", "archiveofsins", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/artstation.py b/test/results/artstation.py index 4844bd74e..7d0ebeccb 100644 --- a/test/results/artstation.py +++ b/test/results/artstation.py @@ -1,232 +1,201 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import artstation from gallery_dl import exception - +from gallery_dl.extractor import artstation __tests__ = ( -{ - "#url" : "https://www.artstation.com/sungchoi/", - "#category": ("", "artstation", "user"), - "#class" : artstation.ArtstationUserExtractor, - "#pattern" : r"https://\w+\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", - "#range" : "1-10", - "#count" : ">= 10", -}, - -{ - "#url" : "https://www.artstation.com/sungchoi/albums/all/", - "#category": ("", "artstation", "user"), - "#class" : artstation.ArtstationUserExtractor, -}, - -{ - "#url" : "https://sungchoi.artstation.com/", - "#category": ("", "artstation", "user"), - "#class" : artstation.ArtstationUserExtractor, -}, - -{ - "#url" : "https://sungchoi.artstation.com/projects/", - "#category": ("", "artstation", "user"), - "#class" : artstation.ArtstationUserExtractor, -}, - -{ - "#url" : "https://www.artstation.com/huimeiye/albums/770899", - "#category": ("", "artstation", "album"), - "#class" : artstation.ArtstationAlbumExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://www.artstation.com/huimeiye/albums/770898", - "#category": ("", "artstation", "album"), - "#class" : artstation.ArtstationAlbumExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://huimeiye.artstation.com/albums/770899", - "#category": ("", "artstation", "album"), - "#class" : artstation.ArtstationAlbumExtractor, -}, - -{ - "#url" : "https://www.artstation.com/mikf/likes", - "#category": ("", "artstation", "likes"), - "#class" : artstation.ArtstationLikesExtractor, - "#pattern" : r"https://\w+\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", - "#count" : 6, -}, - -{ - "#url" : "https://www.artstation.com/mikf/collections/2647023", - "#category": ("", "artstation", "collection"), - "#class" : artstation.ArtstationCollectionExtractor, - "#count" : 10, - - "collection": { - "id" : 2647023, - "is_private" : False, - "name" : "テスト", - "projects_count": 3, - "user_id" : 697975, - "active_projects_count" : 3, - "micro_square_image_url": "https://cdna.artstation.com/p/assets/images/images/005/131/434/micro_square/gaeri-kim-cat-front.jpg?1488720625", - "small_square_image_url": "https://cdna.artstation.com/p/assets/images/images/005/131/434/small_square/gaeri-kim-cat-front.jpg?1488720625", - }, - "user": "mikf", -}, - -{ - "#url" : "https://www.artstation.com/mikf/collections", - "#category": ("", "artstation", "collections"), - "#class" : artstation.ArtstationCollectionsExtractor, - "#urls" : ( - "https://www.artstation.com/mikf/collections/2647023", - "https://www.artstation.com/mikf/collections/2647719", - ), - - "id" : range(2647023, 2647719), - "is_private" : False, - "name" : r"re:テスト|empty", - "projects_count": int, - "user_id" : 697975, - "active_projects_count" : int, - "micro_square_image_url": str, - "small_square_image_url": str, -}, - -{ - "#url" : "https://www.artstation.com/sungchoi/likes", - "#comment" : "no likes", - "#category": ("", "artstation", "likes"), - "#class" : artstation.ArtstationLikesExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.artstation.com/contests/thu-2017/challenges/20", - "#category": ("", "artstation", "challenge"), - "#class" : artstation.ArtstationChallengeExtractor, -}, - -{ - "#url" : "https://www.artstation.com/contests/beyond-human/challenges/23?sorting=winners", - "#category": ("", "artstation", "challenge"), - "#class" : artstation.ArtstationChallengeExtractor, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://www.artstation.com/search?query=ancient&sort_by=rank", - "#category": ("", "artstation", "search"), - "#class" : artstation.ArtstationSearchExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.artstation.com/artwork?sorting=latest", - "#category": ("", "artstation", "artwork"), - "#class" : artstation.ArtstationArtworkExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.artstation.com/artwork/LQVJr", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, - "#pattern" : r"https?://\w+\.artstation\.com/p/assets/images/images/008/760/279/4k/.+", - "#sha1_content": "3f211ce0d6ecdb502db2cdf7bbeceb11d8421170", -}, - -{ - "#url" : "https://www.artstation.com/artwork/Db3dy", - "#comment" : "multiple images per project", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, - "#count" : 4, -}, - -{ - "#url" : "https://www.artstation.com/artwork/lR8b5k", - "#comment" : "artstation video clips (#2566)", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, - "#options" : {"videos": True}, - "#range" : "2-3", - "#urls" : ( - "https://cdn.artstation.com/p/video_sources/000/819/843/infection-4.mp4", - "https://cdn.artstation.com/p/video_sources/000/819/725/infection-veinonly-2.mp4", - ), -}, - -{ - "#url" : "https://www.artstation.com/artwork/g4WPK", - "#comment" : "embedded youtube video", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, - "#options" : {"external": True}, - "#pattern" : r"ytdl:https://www\.youtube(-nocookie)?\.com/embed/JNFfJtwwrU0", - "#range" : "2", -}, - -{ - "#url" : "https://www.artstation.com/artwork/3q3mXB", - "#comment" : "404 (#3016)", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://sungchoi.artstation.com/projects/LQVJr", - "#comment" : "alternate URL patterns", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, -}, - -{ - "#url" : "https://artstn.co/p/LQVJr", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, -}, - -{ - "#url" : "https://www.artstation.com/sungchoi/following", - "#category": ("", "artstation", "following"), - "#class" : artstation.ArtstationFollowingExtractor, - "#pattern" : artstation.ArtstationUserExtractor.pattern, - "#count" : ">= 40", -}, - -{ - "#url" : "https://fede-x-rojas.artstation.com/projects/WBdaZy", - "#comment" : "dash in username", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, -}, - -{ - "#url" : "https://fede-x-rojas.artstation.com/albums/8533110", - "#comment" : "dash in username", - "#category": ("", "artstation", "album"), - "#class" : artstation.ArtstationAlbumExtractor, -}, - -{ - "#url" : "https://fede-x-rojas.artstation.com/", - "#comment" : "dash in username", - "#category": ("", "artstation", "user"), - "#class" : artstation.ArtstationUserExtractor, -}, - + { + "#url": "https://www.artstation.com/sungchoi/", + "#category": ("", "artstation", "user"), + "#class": artstation.ArtstationUserExtractor, + "#pattern": r"https://\w+\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", + "#range": "1-10", + "#count": ">= 10", + }, + { + "#url": "https://www.artstation.com/sungchoi/albums/all/", + "#category": ("", "artstation", "user"), + "#class": artstation.ArtstationUserExtractor, + }, + { + "#url": "https://sungchoi.artstation.com/", + "#category": ("", "artstation", "user"), + "#class": artstation.ArtstationUserExtractor, + }, + { + "#url": "https://sungchoi.artstation.com/projects/", + "#category": ("", "artstation", "user"), + "#class": artstation.ArtstationUserExtractor, + }, + { + "#url": "https://www.artstation.com/huimeiye/albums/770899", + "#category": ("", "artstation", "album"), + "#class": artstation.ArtstationAlbumExtractor, + "#count": 2, + }, + { + "#url": "https://www.artstation.com/huimeiye/albums/770898", + "#category": ("", "artstation", "album"), + "#class": artstation.ArtstationAlbumExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://huimeiye.artstation.com/albums/770899", + "#category": ("", "artstation", "album"), + "#class": artstation.ArtstationAlbumExtractor, + }, + { + "#url": "https://www.artstation.com/mikf/likes", + "#category": ("", "artstation", "likes"), + "#class": artstation.ArtstationLikesExtractor, + "#pattern": r"https://\w+\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", + "#count": 6, + }, + { + "#url": "https://www.artstation.com/mikf/collections/2647023", + "#category": ("", "artstation", "collection"), + "#class": artstation.ArtstationCollectionExtractor, + "#count": 10, + "collection": { + "id": 2647023, + "is_private": False, + "name": "テスト", + "projects_count": 3, + "user_id": 697975, + "active_projects_count": 3, + "micro_square_image_url": "https://cdna.artstation.com/p/assets/images/images/005/131/434/micro_square/gaeri-kim-cat-front.jpg?1488720625", + "small_square_image_url": "https://cdna.artstation.com/p/assets/images/images/005/131/434/small_square/gaeri-kim-cat-front.jpg?1488720625", + }, + "user": "mikf", + }, + { + "#url": "https://www.artstation.com/mikf/collections", + "#category": ("", "artstation", "collections"), + "#class": artstation.ArtstationCollectionsExtractor, + "#urls": ( + "https://www.artstation.com/mikf/collections/2647023", + "https://www.artstation.com/mikf/collections/2647719", + ), + "id": range(2647023, 2647719), + "is_private": False, + "name": r"re:テスト|empty", + "projects_count": int, + "user_id": 697975, + "active_projects_count": int, + "micro_square_image_url": str, + "small_square_image_url": str, + }, + { + "#url": "https://www.artstation.com/sungchoi/likes", + "#comment": "no likes", + "#category": ("", "artstation", "likes"), + "#class": artstation.ArtstationLikesExtractor, + "#count": 0, + }, + { + "#url": "https://www.artstation.com/contests/thu-2017/challenges/20", + "#category": ("", "artstation", "challenge"), + "#class": artstation.ArtstationChallengeExtractor, + }, + { + "#url": "https://www.artstation.com/contests/beyond-human/challenges/23?sorting=winners", + "#category": ("", "artstation", "challenge"), + "#class": artstation.ArtstationChallengeExtractor, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://www.artstation.com/search?query=ancient&sort_by=rank", + "#category": ("", "artstation", "search"), + "#class": artstation.ArtstationSearchExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.artstation.com/artwork?sorting=latest", + "#category": ("", "artstation", "artwork"), + "#class": artstation.ArtstationArtworkExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.artstation.com/artwork/LQVJr", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + "#pattern": r"https?://\w+\.artstation\.com/p/assets/images/images/008/760/279/4k/.+", + "#sha1_content": "3f211ce0d6ecdb502db2cdf7bbeceb11d8421170", + }, + { + "#url": "https://www.artstation.com/artwork/Db3dy", + "#comment": "multiple images per project", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + "#count": 4, + }, + { + "#url": "https://www.artstation.com/artwork/lR8b5k", + "#comment": "artstation video clips (#2566)", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + "#options": {"videos": True}, + "#range": "2-3", + "#urls": ( + "https://cdn.artstation.com/p/video_sources/000/819/843/infection-4.mp4", + "https://cdn.artstation.com/p/video_sources/000/819/725/infection-veinonly-2.mp4", + ), + }, + { + "#url": "https://www.artstation.com/artwork/g4WPK", + "#comment": "embedded youtube video", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + "#options": {"external": True}, + "#pattern": r"ytdl:https://www\.youtube(-nocookie)?\.com/embed/JNFfJtwwrU0", + "#range": "2", + }, + { + "#url": "https://www.artstation.com/artwork/3q3mXB", + "#comment": "404 (#3016)", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + "#count": 0, + }, + { + "#url": "https://sungchoi.artstation.com/projects/LQVJr", + "#comment": "alternate URL patterns", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + }, + { + "#url": "https://artstn.co/p/LQVJr", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + }, + { + "#url": "https://www.artstation.com/sungchoi/following", + "#category": ("", "artstation", "following"), + "#class": artstation.ArtstationFollowingExtractor, + "#pattern": artstation.ArtstationUserExtractor.pattern, + "#count": ">= 40", + }, + { + "#url": "https://fede-x-rojas.artstation.com/projects/WBdaZy", + "#comment": "dash in username", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + }, + { + "#url": "https://fede-x-rojas.artstation.com/albums/8533110", + "#comment": "dash in username", + "#category": ("", "artstation", "album"), + "#class": artstation.ArtstationAlbumExtractor, + }, + { + "#url": "https://fede-x-rojas.artstation.com/", + "#comment": "dash in username", + "#category": ("", "artstation", "user"), + "#class": artstation.ArtstationUserExtractor, + }, ) diff --git a/test/results/aryion.py b/test/results/aryion.py index 113a4d95d..f099e150e 100644 --- a/test/results/aryion.py +++ b/test/results/aryion.py @@ -1,97 +1,84 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import aryion - __tests__ = ( -{ - "#url" : "https://aryion.com/g4/gallery/jameshoward", - "#category": ("", "aryion", "gallery"), - "#class" : aryion.AryionGalleryExtractor, - "#options" : {"recursive": False}, - "#pattern" : r"https://aryion\.com/g4/data\.php\?id=\d+$", - "#range" : "48-52", - "#count" : 5, -}, - -{ - "#url" : "https://aryion.com/g4/user/jameshoward", - "#category": ("", "aryion", "gallery"), - "#class" : aryion.AryionGalleryExtractor, -}, - -{ - "#url" : "https://aryion.com/g4/latest.php?name=jameshoward", - "#category": ("", "aryion", "gallery"), - "#class" : aryion.AryionGalleryExtractor, -}, - -{ - "#url" : "https://aryion.com/g4/favorites/jameshoward", - "#category": ("", "aryion", "favorite"), - "#class" : aryion.AryionFavoriteExtractor, - "#range" : "1-10", - "#count" : 10, - - "user" : "jameshoward", - "artist" : "re:^((?!jameshoward).)*$", -}, - -{ - "#url" : "https://aryion.com/g4/tags.php?tag=star+wars&p=28", - "#category": ("", "aryion", "tag"), - "#class" : aryion.AryionTagExtractor, - "#count" : ">= 5", -}, - -{ - "#url" : "https://aryion.com/g4/view/510079", - "#category": ("", "aryion", "post"), - "#class" : aryion.AryionPostExtractor, - "#sha1_url": "f233286fa5558c07ae500f7f2d5cb0799881450e", - - "artist" : "jameshoward", - "user" : "jameshoward", - "filename" : "jameshoward-510079-subscribestar_150", - "extension" : "jpg", - "id" : 510079, - "width" : 1665, - "height" : 1619, - "size" : 784239, - "title" : "I'm on subscribestar now too!", - "description": r"re:Doesn't hurt to have a backup, right\?", - "tags" : [ - "Non-Vore", - "subscribestar", - ], - "date" : "dt:2019-02-16 19:30:34", - "path" : [], - "views" : int, - "favorites" : int, - "comments" : int, - "_mtime" : "Sat, 16 Feb 2019 19:30:34 GMT", -}, - -{ - "#url" : "https://aryion.com/g4/view/588928", - "#comment" : "x-folder (#694)", - "#category": ("", "aryion", "post"), - "#class" : aryion.AryionPostExtractor, - "#pattern" : aryion.AryionPostExtractor.pattern, - "#count" : ">= 8", -}, - -{ - "#url" : "https://aryion.com/g4/view/537379", - "#comment" : "x-comic-folder (#945)", - "#category": ("", "aryion", "post"), - "#class" : aryion.AryionPostExtractor, - "#pattern" : aryion.AryionPostExtractor.pattern, - "#count" : 2, -}, - + { + "#url": "https://aryion.com/g4/gallery/jameshoward", + "#category": ("", "aryion", "gallery"), + "#class": aryion.AryionGalleryExtractor, + "#options": {"recursive": False}, + "#pattern": r"https://aryion\.com/g4/data\.php\?id=\d+$", + "#range": "48-52", + "#count": 5, + }, + { + "#url": "https://aryion.com/g4/user/jameshoward", + "#category": ("", "aryion", "gallery"), + "#class": aryion.AryionGalleryExtractor, + }, + { + "#url": "https://aryion.com/g4/latest.php?name=jameshoward", + "#category": ("", "aryion", "gallery"), + "#class": aryion.AryionGalleryExtractor, + }, + { + "#url": "https://aryion.com/g4/favorites/jameshoward", + "#category": ("", "aryion", "favorite"), + "#class": aryion.AryionFavoriteExtractor, + "#range": "1-10", + "#count": 10, + "user": "jameshoward", + "artist": "re:^((?!jameshoward).)*$", + }, + { + "#url": "https://aryion.com/g4/tags.php?tag=star+wars&p=28", + "#category": ("", "aryion", "tag"), + "#class": aryion.AryionTagExtractor, + "#count": ">= 5", + }, + { + "#url": "https://aryion.com/g4/view/510079", + "#category": ("", "aryion", "post"), + "#class": aryion.AryionPostExtractor, + "#sha1_url": "f233286fa5558c07ae500f7f2d5cb0799881450e", + "artist": "jameshoward", + "user": "jameshoward", + "filename": "jameshoward-510079-subscribestar_150", + "extension": "jpg", + "id": 510079, + "width": 1665, + "height": 1619, + "size": 784239, + "title": "I'm on subscribestar now too!", + "description": r"re:Doesn't hurt to have a backup, right\?", + "tags": [ + "Non-Vore", + "subscribestar", + ], + "date": "dt:2019-02-16 19:30:34", + "path": [], + "views": int, + "favorites": int, + "comments": int, + "_mtime": "Sat, 16 Feb 2019 19:30:34 GMT", + }, + { + "#url": "https://aryion.com/g4/view/588928", + "#comment": "x-folder (#694)", + "#category": ("", "aryion", "post"), + "#class": aryion.AryionPostExtractor, + "#pattern": aryion.AryionPostExtractor.pattern, + "#count": ">= 8", + }, + { + "#url": "https://aryion.com/g4/view/537379", + "#comment": "x-comic-folder (#945)", + "#category": ("", "aryion", "post"), + "#class": aryion.AryionPostExtractor, + "#pattern": aryion.AryionPostExtractor.pattern, + "#count": 2, + }, ) diff --git a/test/results/atfbooru.py b/test/results/atfbooru.py index 4bdd1b6d4..35d517979 100644 --- a/test/results/atfbooru.py +++ b/test/results/atfbooru.py @@ -1,39 +1,32 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import danbooru - __tests__ = ( -{ - "#url" : "https://booru.allthefallen.moe/posts?tags=yume_shokunin", - "#category": ("Danbooru", "atfbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, - "#count" : 12, -}, - -{ - "#url" : "https://booru.allthefallen.moe/pools/9", - "#category": ("Danbooru", "atfbooru", "pool"), - "#class" : danbooru.DanbooruPoolExtractor, - "#count" : 6, - "#sha1_url": "902549ffcdb00fe033c3f63e12bc3cb95c5fd8d5", -}, - -{ - "#url" : "https://booru.allthefallen.moe/posts/22", - "#category": ("Danbooru", "atfbooru", "post"), - "#class" : danbooru.DanbooruPostExtractor, - "#sha1_content": "21dda68e1d7e0a554078e62923f537d8e895cac8", -}, - -{ - "#url" : "https://booru.allthefallen.moe/explore/posts/popular", - "#category": ("Danbooru", "atfbooru", "popular"), - "#class" : danbooru.DanbooruPopularExtractor, -}, - + { + "#url": "https://booru.allthefallen.moe/posts?tags=yume_shokunin", + "#category": ("Danbooru", "atfbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + "#count": 12, + }, + { + "#url": "https://booru.allthefallen.moe/pools/9", + "#category": ("Danbooru", "atfbooru", "pool"), + "#class": danbooru.DanbooruPoolExtractor, + "#count": 6, + "#sha1_url": "902549ffcdb00fe033c3f63e12bc3cb95c5fd8d5", + }, + { + "#url": "https://booru.allthefallen.moe/posts/22", + "#category": ("Danbooru", "atfbooru", "post"), + "#class": danbooru.DanbooruPostExtractor, + "#sha1_content": "21dda68e1d7e0a554078e62923f537d8e895cac8", + }, + { + "#url": "https://booru.allthefallen.moe/explore/posts/popular", + "#category": ("Danbooru", "atfbooru", "popular"), + "#class": danbooru.DanbooruPopularExtractor, + }, ) diff --git a/test/results/azurlanewiki.py b/test/results/azurlanewiki.py index 176734207..35ebb7951 100644 --- a/test/results/azurlanewiki.py +++ b/test/results/azurlanewiki.py @@ -1,25 +1,20 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://azurlane.koumakan.jp/wiki/Azur_Lane_Wiki", - "#category": ("wikimedia", "azurlanewiki", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://azurlane.koumakan.jp/wiki/Louisville/Gallery", - "#comment" : "entries with missing 'imageinfo' (#5384)", - "#category": ("wikimedia", "azurlanewiki", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#count" : "> 10", -}, - + { + "#url": "https://azurlane.koumakan.jp/wiki/Azur_Lane_Wiki", + "#category": ("wikimedia", "azurlanewiki", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://azurlane.koumakan.jp/wiki/Louisville/Gallery", + "#comment": "entries with missing 'imageinfo' (#5384)", + "#category": ("wikimedia", "azurlanewiki", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#count": "> 10", + }, ) diff --git a/test/results/b4k.py b/test/results/b4k.py index 5a2c13c2c..212b5e185 100644 --- a/test/results/b4k.py +++ b/test/results/b4k.py @@ -1,30 +1,24 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://arch.b4k.co/meta/thread/196/", - "#category": ("foolfuuka", "b4k", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "d309713d2f838797096b3e9cb44fe514a9c9d07a", -}, - -{ - "#url" : "https://arch.b4k.co/meta/", - "#category": ("foolfuuka", "b4k", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://arch.b4k.co/meta/gallery/", - "#category": ("foolfuuka", "b4k", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://arch.b4k.co/meta/thread/196/", + "#category": ("foolfuuka", "b4k", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "d309713d2f838797096b3e9cb44fe514a9c9d07a", + }, + { + "#url": "https://arch.b4k.co/meta/", + "#category": ("foolfuuka", "b4k", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://arch.b4k.co/meta/gallery/", + "#category": ("foolfuuka", "b4k", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/baraag.py b/test/results/baraag.py index 567d12c53..30adde6c7 100644 --- a/test/results/baraag.py +++ b/test/results/baraag.py @@ -1,36 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mastodon - __tests__ = ( -{ - "#url" : "https://baraag.net/@pumpkinnsfw", - "#category": ("mastodon", "baraag", "user"), - "#class" : mastodon.MastodonUserExtractor, -}, - -{ - "#url" : "https://baraag.net/bookmarks", - "#category": ("mastodon", "baraag", "bookmark"), - "#class" : mastodon.MastodonBookmarkExtractor, -}, - -{ - "#url" : "https://baraag.net/users/pumpkinnsfw/following", - "#category": ("mastodon", "baraag", "following"), - "#class" : mastodon.MastodonFollowingExtractor, -}, - -{ - "#url" : "https://baraag.net/@pumpkinnsfw/104364170556898443", - "#category": ("mastodon", "baraag", "status"), - "#class" : mastodon.MastodonStatusExtractor, - "#sha1_content": "67748c1b828c58ad60d0fe5729b59fb29c872244", -}, - + { + "#url": "https://baraag.net/@pumpkinnsfw", + "#category": ("mastodon", "baraag", "user"), + "#class": mastodon.MastodonUserExtractor, + }, + { + "#url": "https://baraag.net/bookmarks", + "#category": ("mastodon", "baraag", "bookmark"), + "#class": mastodon.MastodonBookmarkExtractor, + }, + { + "#url": "https://baraag.net/users/pumpkinnsfw/following", + "#category": ("mastodon", "baraag", "following"), + "#class": mastodon.MastodonFollowingExtractor, + }, + { + "#url": "https://baraag.net/@pumpkinnsfw/104364170556898443", + "#category": ("mastodon", "baraag", "status"), + "#class": mastodon.MastodonStatusExtractor, + "#sha1_content": "67748c1b828c58ad60d0fe5729b59fb29c872244", + }, ) diff --git a/test/results/batoto.py b/test/results/batoto.py index 65c8401cd..f170645cd 100644 --- a/test/results/batoto.py +++ b/test/results/batoto.py @@ -1,290 +1,258 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import batoto from gallery_dl import exception +from gallery_dl.extractor import batoto __tests__ = ( -{ - "#url" : "https://bato.to/title/86408-i-shall-master-this-family-official/1681030-ch_8", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, - "#count" : 66, - - "chapter" : 8, - "chapter_id" : 1681030, - "chapter_minor": "", - "chapter_url" : "8", - "count" : 66, - "date" : "dt:2021-05-15 18:51:37", - "extension" : "webp", - "filename" : str, - "manga" : "I Shall Master this Family! [Official]", - "manga_id" : 86408, - "page" : range(1, 66), - "title" : "Observing", - "volume" : 0, - -}, - -{ - "#url" : "https://bato.to/title/104929-86-eighty-six-official/1943513-vol_1-ch_5", - "#comment" : "volume (vol) in url", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, - "#count" : 7, - - "manga" : "86--EIGHTY-SIX (Official)", - "title" : "The Spearhead Squadron's Power", - "volume" : 1, - "chapter": 5, -}, - -{ - "#url" : "https://mto.to/chapter/2584460", - "#comment" : "'-' in manga title (#5200)", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, - - "chapter" : 9, - "chapter_id": 2584460, - "chapter_minor": "", - "chapter_url": "9", - "count" : 18, - "date" : "dt:2023-11-26 11:01:12", - "manga" : "Isekai Teni shitara Aiken ga Saikyou ni narimashita - Silver Fenrir to Ore ga Isekai Kurashi wo Hajimetara (Official)", - "manga_id" : 126793, - "title" : "", - "volume" : 0 -}, - -{ - "#url" : "https://bato.to/title/90710-new-suitor-for-the-abandoned-wife/2089747-ch_76", - "#comment" : "duplicate info in chapter_minor / title (#5988)", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, - - "chapter" : 76, - "chapter_id" : 2089747, - "chapter_minor": "", - "chapter_url" : "76", - "title" : "Side Story 4 [END]", -}, - -{ - "#url" : "https://bato.to/title/115494-today-with-you/2631897-ch_38", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, - - "chapter" : 37, - "chapter_id" : 2631897, - "chapter_minor" : "", - "chapter_string": "S1 Episode 37 (End of season)", - "chapter_url" : "38", - "count" : 69, - "date" : "dt:2023-12-20 17:31:18", - "manga" : "Today With You", - "manga_id" : 115494, - "title" : "", - "volume" : 1, -}, - -{ - "#url" : "https://bato.to/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://bato.to/chapter/1681030", - "#comment" : "v2 URL", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://bato.to/title/113742-futsutsuka-na-akujo-de-wa-gozaimasu-ga-suuguu-chouso-torikae-den-official", - "#category": ("", "batoto", "manga"), - "#class" : batoto.BatotoMangaExtractor, - "#count" : ">= 21", - - "chapter" : int, - "chapter_minor": str, - "date" : "type:datetime", - "manga" : "Futsutsuka na Akujo de wa Gozaimasu ga - Suuguu Chouso Torikae Den", - "manga_id" : 113742, -}, - -{ - "#url" : "https://bato.to/title/104929-86-eighty-six-official", - "#comment" : "Manga with number in name", - "#category": ("", "batoto", "manga"), - "#class" : batoto.BatotoMangaExtractor, - "#count" : ">= 18", - - "manga": "86--EIGHTY-SIX (Official)", -}, - -{ - "#url" : "https://bato.to/title/140046-the-grand-duke-s-fox-princess-mgchan", - "#comment" : "Non-English translation (Indonesian)", - "#category": ("", "batoto", "manga"), - "#class" : batoto.BatotoMangaExtractor, - "#count" : ">= 29", - - "manga": "Grand Duke Dan Putri Rubah [cont by LUNABY]", -}, - -{ - "#url" : "https://bato.to/title/134270-removed", - "#comment" : "Deleted/removed manga", - "#category": ("", "batoto", "manga"), - "#class" : batoto.BatotoMangaExtractor, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://bato.to/title/86408-i-shall-master-this-family-official", - "#category": ("", "batoto", "manga"), - "#class" : batoto.BatotoMangaExtractor, -}, - -{ - "#url" : "https://bato.to/series/86408/i-shall-master-this-family-official", - "#comment" : "v2 URL", - "#category": ("", "batoto", "manga"), - "#class" : batoto.BatotoMangaExtractor, -}, - -{ - "#url" : "https://dto.to/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://hto.to/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://mto.to/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://wto.to/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://mangatoto.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://mangatoto.net/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://mangatoto.org/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://batocomic.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://batocomic.net/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://batocomic.org/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://readtoto.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://readtoto.net/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://readtoto.org/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://xbato.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://xbato.net/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://xbato.org/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://zbato.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://zbato.net/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://zbato.org/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://comiko.net/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://comiko.org/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://batotoo.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://batotwo.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://battwo.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - + { + "#url": "https://bato.to/title/86408-i-shall-master-this-family-official/1681030-ch_8", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + "#count": 66, + "chapter": 8, + "chapter_id": 1681030, + "chapter_minor": "", + "chapter_url": "8", + "count": 66, + "date": "dt:2021-05-15 18:51:37", + "extension": "webp", + "filename": str, + "manga": "I Shall Master this Family! [Official]", + "manga_id": 86408, + "page": range(1, 66), + "title": "Observing", + "volume": 0, + }, + { + "#url": "https://bato.to/title/104929-86-eighty-six-official/1943513-vol_1-ch_5", + "#comment": "volume (vol) in url", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + "#count": 7, + "manga": "86--EIGHTY-SIX (Official)", + "title": "The Spearhead Squadron's Power", + "volume": 1, + "chapter": 5, + }, + { + "#url": "https://mto.to/chapter/2584460", + "#comment": "'-' in manga title (#5200)", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + "chapter": 9, + "chapter_id": 2584460, + "chapter_minor": "", + "chapter_url": "9", + "count": 18, + "date": "dt:2023-11-26 11:01:12", + "manga": "Isekai Teni shitara Aiken ga Saikyou ni narimashita - Silver Fenrir to Ore ga Isekai Kurashi wo Hajimetara (Official)", + "manga_id": 126793, + "title": "", + "volume": 0, + }, + { + "#url": "https://bato.to/title/90710-new-suitor-for-the-abandoned-wife/2089747-ch_76", + "#comment": "duplicate info in chapter_minor / title (#5988)", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + "chapter": 76, + "chapter_id": 2089747, + "chapter_minor": "", + "chapter_url": "76", + "title": "Side Story 4 [END]", + }, + { + "#url": "https://bato.to/title/115494-today-with-you/2631897-ch_38", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + "chapter": 37, + "chapter_id": 2631897, + "chapter_minor": "", + "chapter_string": "S1 Episode 37 (End of season)", + "chapter_url": "38", + "count": 69, + "date": "dt:2023-12-20 17:31:18", + "manga": "Today With You", + "manga_id": 115494, + "title": "", + "volume": 1, + }, + { + "#url": "https://bato.to/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://bato.to/chapter/1681030", + "#comment": "v2 URL", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://bato.to/title/113742-futsutsuka-na-akujo-de-wa-gozaimasu-ga-suuguu-chouso-torikae-den-official", + "#category": ("", "batoto", "manga"), + "#class": batoto.BatotoMangaExtractor, + "#count": ">= 21", + "chapter": int, + "chapter_minor": str, + "date": "type:datetime", + "manga": "Futsutsuka na Akujo de wa Gozaimasu ga - Suuguu Chouso Torikae Den", + "manga_id": 113742, + }, + { + "#url": "https://bato.to/title/104929-86-eighty-six-official", + "#comment": "Manga with number in name", + "#category": ("", "batoto", "manga"), + "#class": batoto.BatotoMangaExtractor, + "#count": ">= 18", + "manga": "86--EIGHTY-SIX (Official)", + }, + { + "#url": "https://bato.to/title/140046-the-grand-duke-s-fox-princess-mgchan", + "#comment": "Non-English translation (Indonesian)", + "#category": ("", "batoto", "manga"), + "#class": batoto.BatotoMangaExtractor, + "#count": ">= 29", + "manga": "Grand Duke Dan Putri Rubah [cont by LUNABY]", + }, + { + "#url": "https://bato.to/title/134270-removed", + "#comment": "Deleted/removed manga", + "#category": ("", "batoto", "manga"), + "#class": batoto.BatotoMangaExtractor, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://bato.to/title/86408-i-shall-master-this-family-official", + "#category": ("", "batoto", "manga"), + "#class": batoto.BatotoMangaExtractor, + }, + { + "#url": "https://bato.to/series/86408/i-shall-master-this-family-official", + "#comment": "v2 URL", + "#category": ("", "batoto", "manga"), + "#class": batoto.BatotoMangaExtractor, + }, + { + "#url": "https://dto.to/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://hto.to/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://mto.to/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://wto.to/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://mangatoto.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://mangatoto.net/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://mangatoto.org/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://batocomic.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://batocomic.net/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://batocomic.org/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://readtoto.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://readtoto.net/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://readtoto.org/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://xbato.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://xbato.net/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://xbato.org/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://zbato.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://zbato.net/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://zbato.org/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://comiko.net/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://comiko.org/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://batotoo.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://batotwo.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://battwo.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, ) diff --git a/test/results/bbc.py b/test/results/bbc.py index 836786ae5..213d483cd 100644 --- a/test/results/bbc.py +++ b/test/results/bbc.py @@ -1,49 +1,41 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import bbc - __tests__ = ( -{ - "#url" : "https://www.bbc.co.uk/programmes/p084qtzs/p085g9kg", - "#category": ("", "bbc", "gallery"), - "#class" : bbc.BbcGalleryExtractor, - "#pattern" : r"https://ichef\.bbci\.co\.uk/images/ic/1920xn/\w+\.jpg", - "#count" : 37, - - "programme": "p084qtzs", - "path" : [ - "BBC One", - "Doctor Who (2005–2022)", - "The Timeless Children", - ], -}, - -{ - "#url" : "https://www.bbc.co.uk/programmes/p084qtzs", - "#category": ("", "bbc", "gallery"), - "#class" : bbc.BbcGalleryExtractor, -}, - -{ - "#url" : "https://www.bbc.co.uk/programmes/b006q2x0/galleries", - "#category": ("", "bbc", "programme"), - "#class" : bbc.BbcProgrammeExtractor, - "#pattern" : bbc.BbcGalleryExtractor.pattern, - "#range" : "1-50", - "#count" : ">= 50", -}, - -{ - "#url" : "https://www.bbc.co.uk/programmes/b006q2x0/galleries?page=25", - "#category": ("", "bbc", "programme"), - "#class" : bbc.BbcProgrammeExtractor, - "#pattern" : bbc.BbcGalleryExtractor.pattern, - "#count" : ">= 100", -}, - + { + "#url": "https://www.bbc.co.uk/programmes/p084qtzs/p085g9kg", + "#category": ("", "bbc", "gallery"), + "#class": bbc.BbcGalleryExtractor, + "#pattern": r"https://ichef\.bbci\.co\.uk/images/ic/1920xn/\w+\.jpg", + "#count": 37, + "programme": "p084qtzs", + "path": [ + "BBC One", + "Doctor Who (2005–2022)", + "The Timeless Children", + ], + }, + { + "#url": "https://www.bbc.co.uk/programmes/p084qtzs", + "#category": ("", "bbc", "gallery"), + "#class": bbc.BbcGalleryExtractor, + }, + { + "#url": "https://www.bbc.co.uk/programmes/b006q2x0/galleries", + "#category": ("", "bbc", "programme"), + "#class": bbc.BbcProgrammeExtractor, + "#pattern": bbc.BbcGalleryExtractor.pattern, + "#range": "1-50", + "#count": ">= 50", + }, + { + "#url": "https://www.bbc.co.uk/programmes/b006q2x0/galleries?page=25", + "#category": ("", "bbc", "programme"), + "#class": bbc.BbcProgrammeExtractor, + "#pattern": bbc.BbcGalleryExtractor.pattern, + "#count": ">= 100", + }, ) diff --git a/test/results/bbw-chan.py b/test/results/bbw-chan.py index 1d79a431a..1f736bbfd 100644 --- a/test/results/bbw-chan.py +++ b/test/results/bbw-chan.py @@ -1,39 +1,32 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import lynxchan - __tests__ = ( -{ - "#url" : "https://bbw-chan.link/bbwdraw/res/499.html", - "#category": ("lynxchan", "bbw-chan", "thread"), - "#class" : lynxchan.LynxchanThreadExtractor, - "#pattern" : r"https://bbw-chan\.link/\.media/[0-9a-f]{64}(\.\w+)?$", - "#count" : ">= 352", -}, - -{ - "#url" : "https://bbw-chan.nl/bbwdraw/res/489.html", - "#category": ("lynxchan", "bbw-chan", "thread"), - "#class" : lynxchan.LynxchanThreadExtractor, -}, - -{ - "#url" : "https://bbw-chan.link/bbwdraw/", - "#category": ("lynxchan", "bbw-chan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, - "#pattern" : lynxchan.LynxchanThreadExtractor.pattern, - "#count" : ">= 148", -}, - -{ - "#url" : "https://bbw-chan.nl/bbwdraw/2.html", - "#category": ("lynxchan", "bbw-chan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, -}, - + { + "#url": "https://bbw-chan.link/bbwdraw/res/499.html", + "#category": ("lynxchan", "bbw-chan", "thread"), + "#class": lynxchan.LynxchanThreadExtractor, + "#pattern": r"https://bbw-chan\.link/\.media/[0-9a-f]{64}(\.\w+)?$", + "#count": ">= 352", + }, + { + "#url": "https://bbw-chan.nl/bbwdraw/res/489.html", + "#category": ("lynxchan", "bbw-chan", "thread"), + "#class": lynxchan.LynxchanThreadExtractor, + }, + { + "#url": "https://bbw-chan.link/bbwdraw/", + "#category": ("lynxchan", "bbw-chan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + "#pattern": lynxchan.LynxchanThreadExtractor.pattern, + "#count": ">= 148", + }, + { + "#url": "https://bbw-chan.nl/bbwdraw/2.html", + "#category": ("lynxchan", "bbw-chan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + }, ) diff --git a/test/results/bcbnsfw.py b/test/results/bcbnsfw.py index e9fcf8b21..a1f7925c7 100644 --- a/test/results/bcbnsfw.py +++ b/test/results/bcbnsfw.py @@ -1,26 +1,21 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import szurubooru - __tests__ = ( -{ - "#url" : "https://booru.bcbnsfw.space/posts/query=simple_background", - "#category": ("szurubooru", "bcbnsfw", "tag"), - "#class" : szurubooru.SzurubooruTagExtractor, -}, - -{ - "#url" : "https://booru.bcbnsfw.space/post/1599", - "#comment" : "now only available as WebP", - "#category": ("szurubooru", "bcbnsfw", "post"), - "#class" : szurubooru.SzurubooruPostExtractor, - "#pattern" : r"https://booru\.bcbnsfw\.space/data/posts/1599_53784518e92086bd\.png", - "#sha1_content": "55f8b8d187adc82f2dcaf2aa89db0ae21b08c0b0", -}, - + { + "#url": "https://booru.bcbnsfw.space/posts/query=simple_background", + "#category": ("szurubooru", "bcbnsfw", "tag"), + "#class": szurubooru.SzurubooruTagExtractor, + }, + { + "#url": "https://booru.bcbnsfw.space/post/1599", + "#comment": "now only available as WebP", + "#category": ("szurubooru", "bcbnsfw", "post"), + "#class": szurubooru.SzurubooruPostExtractor, + "#pattern": r"https://booru\.bcbnsfw\.space/data/posts/1599_53784518e92086bd\.png", + "#sha1_content": "55f8b8d187adc82f2dcaf2aa89db0ae21b08c0b0", + }, ) diff --git a/test/results/behance.py b/test/results/behance.py index 3a805f813..bf72bc36f 100644 --- a/test/results/behance.py +++ b/test/results/behance.py @@ -1,99 +1,86 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import behance from gallery_dl import exception - +from gallery_dl.extractor import behance __tests__ = ( -{ - "#url" : "https://www.behance.net/gallery/17386197/A-Short-Story", - "#category": ("", "behance", "gallery"), - "#class" : behance.BehanceGalleryExtractor, - "#urls" : ( - "ytdl:https://player.vimeo.com/video/97189640?title=0&byline=0&portrait=0&color=ffffff", - "https://mir-s3-cdn-cf.behance.net/project_modules/source/a5a12417386197.562bc055a107d.jpg", - ), - - "id" : 17386197, - "name" : r"re:\"Hi\". A short story about the important things ", - "owners": [ - "Place Studio", - "Julio César Velazquez", - ], - "?fields": [ - "Animation", - "Character Design", - "Directing", - ], - "tags" : list, - "module": dict, - "date" : "dt:2014-06-03 15:41:51", -}, - -{ - "#url" : "https://www.behance.net/gallery/21324767/Nevada-City", - "#category": ("", "behance", "gallery"), - "#class" : behance.BehanceGalleryExtractor, - "#count" : 6, - "#sha1_url": "0258fe194fe7d828d6f2c7f6086a9a0a4140db1d", - - "owners": ["Alex Strohl"], -}, - -{ - "#url" : "https://www.behance.net/gallery/88276087/Audi-R8-RWD", - "#comment" : "'media_collection' modules", - "#category": ("", "behance", "gallery"), - "#class" : behance.BehanceGalleryExtractor, - "#pattern" : r"https://mir-s3-cdn-cf\.behance\.net/project_modules/source/[0-9a-f]+.[0-9a-f]+\.jpg", - "#count" : 20, - "#sha1_url": "6bebff0d37f85349f9ad28bd8b76fd66627c1e2f", -}, - -{ - "#url" : "https://www.behance.net/gallery/101185577/COLCCI", - "#comment" : "'video' modules (#1282)", - "#category": ("", "behance", "gallery"), - "#class" : behance.BehanceGalleryExtractor, - "#pattern" : r"ytdl:https://cdn-prod-ccv\.adobe\.com/\w+/rend/master\.m3u8\?", - "#count" : 3, -}, - -{ - "#url" : "https://www.behance.net/gallery/89270715/Moevir", - "#comment" : "'text' modules (#4799)", - "#category": ("", "behance", "gallery"), - "#class" : behance.BehanceGalleryExtractor, - "#options" : {"modules": "text"}, - "#urls" : """text:
          Make Shift
          https://www.moevir.com/News/make-shif
          Moevir Magazine November Issue 2019
          Photography by Caesar Lima @caephoto 
          Model: Bee @phamhuongbee 
          Makeup by Monica Alvarez @monicaalvarezmakeup 
          Styling by Jessica Boal @jessicaboal 
          Hair by James Gilbert @brandnewjames
          Shot at Vila Sophia
          """, -}, - -{ - "#url" : "https://www.behance.net/gallery/177464639/Kimori", - "#comment" : "mature content (#4417)", - "#category": ("", "behance", "gallery"), - "#class" : behance.BehanceGalleryExtractor, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://www.behance.net/alexstrohl", - "#category": ("", "behance", "user"), - "#class" : behance.BehanceUserExtractor, - "#pattern" : behance.BehanceGalleryExtractor.pattern, - "#count" : ">= 11", -}, - -{ - "#url" : "https://www.behance.net/collection/71340149/inspiration", - "#category": ("", "behance", "collection"), - "#class" : behance.BehanceCollectionExtractor, - "#pattern" : behance.BehanceGalleryExtractor.pattern, - "#count" : ">= 150", -}, - + { + "#url": "https://www.behance.net/gallery/17386197/A-Short-Story", + "#category": ("", "behance", "gallery"), + "#class": behance.BehanceGalleryExtractor, + "#urls": ( + "ytdl:https://player.vimeo.com/video/97189640?title=0&byline=0&portrait=0&color=ffffff", + "https://mir-s3-cdn-cf.behance.net/project_modules/source/a5a12417386197.562bc055a107d.jpg", + ), + "id": 17386197, + "name": r"re:\"Hi\". A short story about the important things ", + "owners": [ + "Place Studio", + "Julio César Velazquez", + ], + "?fields": [ + "Animation", + "Character Design", + "Directing", + ], + "tags": list, + "module": dict, + "date": "dt:2014-06-03 15:41:51", + }, + { + "#url": "https://www.behance.net/gallery/21324767/Nevada-City", + "#category": ("", "behance", "gallery"), + "#class": behance.BehanceGalleryExtractor, + "#count": 6, + "#sha1_url": "0258fe194fe7d828d6f2c7f6086a9a0a4140db1d", + "owners": ["Alex Strohl"], + }, + { + "#url": "https://www.behance.net/gallery/88276087/Audi-R8-RWD", + "#comment": "'media_collection' modules", + "#category": ("", "behance", "gallery"), + "#class": behance.BehanceGalleryExtractor, + "#pattern": r"https://mir-s3-cdn-cf\.behance\.net/project_modules/source/[0-9a-f]+.[0-9a-f]+\.jpg", + "#count": 20, + "#sha1_url": "6bebff0d37f85349f9ad28bd8b76fd66627c1e2f", + }, + { + "#url": "https://www.behance.net/gallery/101185577/COLCCI", + "#comment": "'video' modules (#1282)", + "#category": ("", "behance", "gallery"), + "#class": behance.BehanceGalleryExtractor, + "#pattern": r"ytdl:https://cdn-prod-ccv\.adobe\.com/\w+/rend/master\.m3u8\?", + "#count": 3, + }, + { + "#url": "https://www.behance.net/gallery/89270715/Moevir", + "#comment": "'text' modules (#4799)", + "#category": ("", "behance", "gallery"), + "#class": behance.BehanceGalleryExtractor, + "#options": {"modules": "text"}, + "#urls": """text:
          Make Shift
          https://www.moevir.com/News/make-shif
          Moevir Magazine November Issue 2019
          Photography by Caesar Lima @caephoto 
          Model: Bee @phamhuongbee 
          Makeup by Monica Alvarez @monicaalvarezmakeup 
          Styling by Jessica Boal @jessicaboal 
          Hair by James Gilbert @brandnewjames
          Shot at Vila Sophia
          """, + }, + { + "#url": "https://www.behance.net/gallery/177464639/Kimori", + "#comment": "mature content (#4417)", + "#category": ("", "behance", "gallery"), + "#class": behance.BehanceGalleryExtractor, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://www.behance.net/alexstrohl", + "#category": ("", "behance", "user"), + "#class": behance.BehanceUserExtractor, + "#pattern": behance.BehanceGalleryExtractor.pattern, + "#count": ">= 11", + }, + { + "#url": "https://www.behance.net/collection/71340149/inspiration", + "#category": ("", "behance", "collection"), + "#class": behance.BehanceCollectionExtractor, + "#pattern": behance.BehanceGalleryExtractor.pattern, + "#count": ">= 150", + }, ) diff --git a/test/results/bilibili.py b/test/results/bilibili.py index c32095fdd..4f1908096 100644 --- a/test/results/bilibili.py +++ b/test/results/bilibili.py @@ -1,45 +1,39 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import bilibili - __tests__ = ( -{ - "#url" : "https://www.bilibili.com/opus/988425412565532689", - "#class": bilibili.BilibiliArticleExtractor, - "#urls" : ( - "http://i0.hdslb.com/bfs/new_dyn/311264c4dcf45261f7d7a7fe451b05b9405279279.png", - "http://i0.hdslb.com/bfs/new_dyn/b60d8bc6996529613d617443a12c0a93405279279.png", - "http://i0.hdslb.com/bfs/new_dyn/d4494543210d9eee5310e11dc62581e4405279279.png", - "http://i0.hdslb.com/bfs/new_dyn/45268e63086b2d99811b2e6490130937405279279.png", - ), - - "count" : 4, - "detail" : dict, - "extension": "png", - "filename" : str, - "height" : 800, - "id" : "988425412565532689", - "isClient" : False, - "isPreview": False, - "num" : range(1, 4), - "size" : float, - "theme" : str, - "themeMode": "light", - "url" : str, - "username" : "平平出击", - "width" : 800, -}, - -{ - "#url" : "https://space.bilibili.com/405279279/article", - "#class" : bilibili.BilibiliUserArticlesExtractor, - "#pattern": bilibili.BilibiliArticleExtractor.pattern, - "#count" : range(50, 100), -}, - + { + "#url": "https://www.bilibili.com/opus/988425412565532689", + "#class": bilibili.BilibiliArticleExtractor, + "#urls": ( + "http://i0.hdslb.com/bfs/new_dyn/311264c4dcf45261f7d7a7fe451b05b9405279279.png", + "http://i0.hdslb.com/bfs/new_dyn/b60d8bc6996529613d617443a12c0a93405279279.png", + "http://i0.hdslb.com/bfs/new_dyn/d4494543210d9eee5310e11dc62581e4405279279.png", + "http://i0.hdslb.com/bfs/new_dyn/45268e63086b2d99811b2e6490130937405279279.png", + ), + "count": 4, + "detail": dict, + "extension": "png", + "filename": str, + "height": 800, + "id": "988425412565532689", + "isClient": False, + "isPreview": False, + "num": range(1, 4), + "size": float, + "theme": str, + "themeMode": "light", + "url": str, + "username": "平平出击", + "width": 800, + }, + { + "#url": "https://space.bilibili.com/405279279/article", + "#class": bilibili.BilibiliUserArticlesExtractor, + "#pattern": bilibili.BilibiliArticleExtractor.pattern, + "#count": range(50, 100), + }, ) diff --git a/test/results/bitly.py b/test/results/bitly.py index cbfcf0eb8..26f3ba2ff 100644 --- a/test/results/bitly.py +++ b/test/results/bitly.py @@ -1,19 +1,15 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import urlshortener - __tests__ = ( -{ - "#url" : "https://bit.ly/3cWIUgq", - "#category": ("urlshortener", "bitly", "link"), - "#class" : urlshortener.UrlshortenerLinkExtractor, - "#pattern" : "^https://gumroad.com/l/storm_b1", - "#count" : 1, -}, - + { + "#url": "https://bit.ly/3cWIUgq", + "#category": ("urlshortener", "bitly", "link"), + "#class": urlshortener.UrlshortenerLinkExtractor, + "#pattern": "^https://gumroad.com/l/storm_b1", + "#count": 1, + }, ) diff --git a/test/results/blogger.py b/test/results/blogger.py index eef964596..122f653de 100644 --- a/test/results/blogger.py +++ b/test/results/blogger.py @@ -1,37 +1,30 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import blogger - __tests__ = ( -{ - "#url" : "blogger:http://www.julianbunker.com/2010/12/moon-rise.html", - "#category": ("blogger", "www.julianbunker.com", "post"), - "#class" : blogger.BloggerPostExtractor, -}, - -{ - "#url" : "blogger:https://www.kefblog.com.ng/", - "#category": ("blogger", "www.kefblog.com.ng", "blog"), - "#class" : blogger.BloggerBlogExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "blogger:http://www.julianbunker.com/search?q=400mm", - "#category": ("blogger", "www.julianbunker.com", "search"), - "#class" : blogger.BloggerSearchExtractor, -}, - -{ - "#url" : "blogger:http://www.julianbunker.com/search/label/D%26D", - "#category": ("blogger", "www.julianbunker.com", "label"), - "#class" : blogger.BloggerLabelExtractor, -}, - + { + "#url": "blogger:http://www.julianbunker.com/2010/12/moon-rise.html", + "#category": ("blogger", "www.julianbunker.com", "post"), + "#class": blogger.BloggerPostExtractor, + }, + { + "#url": "blogger:https://www.kefblog.com.ng/", + "#category": ("blogger", "www.kefblog.com.ng", "blog"), + "#class": blogger.BloggerBlogExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "blogger:http://www.julianbunker.com/search?q=400mm", + "#category": ("blogger", "www.julianbunker.com", "search"), + "#class": blogger.BloggerSearchExtractor, + }, + { + "#url": "blogger:http://www.julianbunker.com/search/label/D%26D", + "#category": ("blogger", "www.julianbunker.com", "label"), + "#class": blogger.BloggerLabelExtractor, + }, ) diff --git a/test/results/blogspot.py b/test/results/blogspot.py index 3c7beadcf..c542f7fd0 100644 --- a/test/results/blogspot.py +++ b/test/results/blogspot.py @@ -1,94 +1,82 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import blogger - __tests__ = ( -{ - "#url" : "https://julianbphotography.blogspot.com/2010/12/moon-rise.html", - "#category": ("blogger", "blogspot", "post"), - "#class" : blogger.BloggerPostExtractor, - "#urls" : "https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjH9WkPvLJq2moxKtyt3ieJZWSDFQwOi3PHRdlHVHEQHRwy-d86Jg6HWSMhxaa6EgvlXq-zDMmKM4kIPn27eJ9Hepk2X9e9HQhqwMfrT8RYTnFe65uexw7KSk5FdWHxRVp5crz3p_qph3Bj/s0/Icy-Moonrise---For-Web.jpg", - - "blog": { - "date" : "dt:2010-11-21 18:19:42", - "description": "", - "id" : "5623928067739466034", - "kind" : "blogger#blog", - "locale" : dict, - "name" : "Julian Bunker Photography", - "pages" : int, - "posts" : int, - "published" : "2010-11-21T10:19:42-08:00", - "updated" : str, - "url" : "http://julianbphotography.blogspot.com/", + { + "#url": "https://julianbphotography.blogspot.com/2010/12/moon-rise.html", + "#category": ("blogger", "blogspot", "post"), + "#class": blogger.BloggerPostExtractor, + "#urls": "https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjH9WkPvLJq2moxKtyt3ieJZWSDFQwOi3PHRdlHVHEQHRwy-d86Jg6HWSMhxaa6EgvlXq-zDMmKM4kIPn27eJ9Hepk2X9e9HQhqwMfrT8RYTnFe65uexw7KSk5FdWHxRVp5crz3p_qph3Bj/s0/Icy-Moonrise---For-Web.jpg", + "blog": { + "date": "dt:2010-11-21 18:19:42", + "description": "", + "id": "5623928067739466034", + "kind": "blogger#blog", + "locale": dict, + "name": "Julian Bunker Photography", + "pages": int, + "posts": int, + "published": "2010-11-21T10:19:42-08:00", + "updated": str, + "url": "http://julianbphotography.blogspot.com/", + }, + "post": { + "author": "Julian Bunker", + "content": str, + "date": "dt:2010-12-26 01:08:00", + "etag": str, + "id": "6955139236418998998", + "kind": "blogger#post", + "published": "2010-12-25T17:08:00-08:00", + "replies": "0", + "title": "Moon Rise", + "updated": "2011-12-06T05:21:24-08:00", + "url": "http://julianbphotography.blogspot.com/2010/12/moon-rise.html", + }, + "extension": "jpg", + "filename": "Icy-Moonrise---For-Web", + "num": 1, + "url": "https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjH9WkPvLJq2moxKtyt3ieJZWSDFQwOi3PHRdlHVHEQHRwy-d86Jg6HWSMhxaa6EgvlXq-zDMmKM4kIPn27eJ9Hepk2X9e9HQhqwMfrT8RYTnFe65uexw7KSk5FdWHxRVp5crz3p_qph3Bj/s0/Icy-Moonrise---For-Web.jpg", }, - "post": { - "author" : "Julian Bunker", - "content" : str, - "date" : "dt:2010-12-26 01:08:00", - "etag" : str, - "id" : "6955139236418998998", - "kind" : "blogger#post", - "published": "2010-12-25T17:08:00-08:00", - "replies" : "0", - "title" : "Moon Rise", - "updated" : "2011-12-06T05:21:24-08:00", - "url" : "http://julianbphotography.blogspot.com/2010/12/moon-rise.html", + { + "#url": "http://cfnmscenesinmovies.blogspot.com/2011/11/cfnm-scene-jenna-fischer-in-office.html", + "#comment": "video (#587)", + "#category": ("blogger", "blogspot", "post"), + "#class": blogger.BloggerPostExtractor, + "#pattern": r"https://.+\.googlevideo\.com/videoplayback", + }, + { + "#url": "https://randomthingsthroughmyletterbox.blogspot.com/2022/01/bitter-flowers-by-gunnar-staalesen-blog.html", + "#comment": "new image domain (#2204)", + "#category": ("blogger", "blogspot", "post"), + "#class": blogger.BloggerPostExtractor, + "#pattern": r"https://blogger\.googleusercontent\.com/img/.+=s0$", + "#count": 8, + }, + { + "#url": "https://julianbphotography.blogspot.com/", + "#category": ("blogger", "blogspot", "blog"), + "#class": blogger.BloggerBlogExtractor, + "#pattern": r"https://blogger\.googleusercontent\.com/img/.+/s0/", + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://julianbphotography.blogspot.com/search?q=400mm", + "#category": ("blogger", "blogspot", "search"), + "#class": blogger.BloggerSearchExtractor, + "#count": "< 10", + "query": "400mm", + }, + { + "#url": "https://dmmagazine.blogspot.com/search/label/D%26D", + "#category": ("blogger", "blogspot", "label"), + "#class": blogger.BloggerLabelExtractor, + "#range": "1-25", + "#count": 25, + "label": "D&D", }, - "extension": "jpg", - "filename" : "Icy-Moonrise---For-Web", - "num" : 1, - "url" : "https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjH9WkPvLJq2moxKtyt3ieJZWSDFQwOi3PHRdlHVHEQHRwy-d86Jg6HWSMhxaa6EgvlXq-zDMmKM4kIPn27eJ9Hepk2X9e9HQhqwMfrT8RYTnFe65uexw7KSk5FdWHxRVp5crz3p_qph3Bj/s0/Icy-Moonrise---For-Web.jpg", -}, - -{ - "#url" : "http://cfnmscenesinmovies.blogspot.com/2011/11/cfnm-scene-jenna-fischer-in-office.html", - "#comment" : "video (#587)", - "#category": ("blogger", "blogspot", "post"), - "#class" : blogger.BloggerPostExtractor, - "#pattern" : r"https://.+\.googlevideo\.com/videoplayback", -}, - -{ - "#url" : "https://randomthingsthroughmyletterbox.blogspot.com/2022/01/bitter-flowers-by-gunnar-staalesen-blog.html", - "#comment" : "new image domain (#2204)", - "#category": ("blogger", "blogspot", "post"), - "#class" : blogger.BloggerPostExtractor, - "#pattern" : r"https://blogger\.googleusercontent\.com/img/.+=s0$", - "#count" : 8, -}, - -{ - "#url" : "https://julianbphotography.blogspot.com/", - "#category": ("blogger", "blogspot", "blog"), - "#class" : blogger.BloggerBlogExtractor, - "#pattern" : r"https://blogger\.googleusercontent\.com/img/.+/s0/", - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://julianbphotography.blogspot.com/search?q=400mm", - "#category": ("blogger", "blogspot", "search"), - "#class" : blogger.BloggerSearchExtractor, - "#count" : "< 10", - - "query": "400mm", -}, - -{ - "#url" : "https://dmmagazine.blogspot.com/search/label/D%26D", - "#category": ("blogger", "blogspot", "label"), - "#class" : blogger.BloggerLabelExtractor, - "#range" : "1-25", - "#count" : 25, - - "label": "D&D", -}, - ) diff --git a/test/results/bluesky.py b/test/results/bluesky.py index e2da3c0c7..591cc26e7 100644 --- a/test/results/bluesky.py +++ b/test/results/bluesky.py @@ -1,396 +1,352 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import bluesky - __tests__ = ( -{ - "#url" : "https://bsky.app/profile/bsky.app", - "#category": ("", "bluesky", "user"), - "#class" : bluesky.BlueskyUserExtractor, - "#urls" : ( - "https://bsky.app/profile/bsky.app/media", - ), -}, - -{ - "#url" : "https://www.bsky.app/profile/bsky.app", - "#class" : bluesky.BlueskyUserExtractor, -}, - -{ - "#url" : "https://main.bsky.dev/profile/bsky.app", - "#class" : bluesky.BlueskyUserExtractor, -}, - -{ - "#url" : "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur", - "#category": ("", "bluesky", "user"), - "#class" : bluesky.BlueskyUserExtractor, - "#options" : {"include": "all"}, - "#urls" : ( - "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/avatar", - "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/banner", - "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/posts", - "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/replies", - "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/media", - "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/likes", - ), -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/avatar", - "#category": ("", "bluesky", "avatar"), - "#class" : bluesky.BlueskyAvatarExtractor, - "#urls" : "https://puffball.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:z72i7hdynmk6r22z27h6tvur&cid=bafkreihagr2cmvl2jt4mgx3sppwe2it3fwolkrbtjrhcnwjk4jdijhsoze", -}, - -{ - "#url" : "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/banner", - "#category": ("", "bluesky", "background"), - "#class" : bluesky.BlueskyBackgroundExtractor, - "#urls" : "https://puffball.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:z72i7hdynmk6r22z27h6tvur&cid=bafkreichzyovokfzmymz36p5jibbjrhsur6n7hjnzxrpbt5jaydp2szvna", -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/posts", - "#category": ("", "bluesky", "posts"), - "#class" : bluesky.BlueskyPostsExtractor, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/replies", - "#category": ("", "bluesky", "replies"), - "#class" : bluesky.BlueskyRepliesExtractor, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/media", - "#category": ("", "bluesky", "media"), - "#class" : bluesky.BlueskyMediaExtractor, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://bsky.app/profile/did:plc:jfhpnnst6flqway4eaeqzj2a/feed/for-science", - "#category": ("", "bluesky", "feed"), - "#class" : bluesky.BlueskyFeedExtractor, - "#range" : "1-40", - "#count" : 40, - "#archive" : False, -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/follows", - "#category": ("", "bluesky", "following"), - "#class" : bluesky.BlueskyFollowingExtractor, - "#urls" : ( - "https://bsky.app/profile/did:plc:eon2iu7v3x2ukgxkqaf7e5np", - "https://bsky.app/profile/did:plc:ewvi7nxzyoun6zhxrhs64oiz", - ), -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/likes", - "#category": ("", "bluesky", "likes"), - "#class" : bluesky.BlueskyLikesExtractor, -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/lists/abcdefghijklm", - "#category": ("", "bluesky", "list"), - "#class" : bluesky.BlueskyListExtractor, -}, - -{ - "#url" : "https://bsky.app/search?q=nature", - "#category": ("", "bluesky", "search"), - "#class" : bluesky.BlueskySearchExtractor, - "#range" : "1-40", - "#count" : 40, - "#archive" : False, -}, - -{ - "#url" : "https://bsky.app/hashtag/nature", - "#class" : bluesky.BlueskyHashtagExtractor, - "#range" : "1-40", - "#count" : 40, - "#archive" : False, -}, -{ - "#url" : "https://bsky.app/hashtag/top", - "#class" : bluesky.BlueskyHashtagExtractor, -}, -{ - "#url" : "https://bsky.app/hashtag/nature/latest", - "#class" : bluesky.BlueskyHashtagExtractor, -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, - "#options" : {"metadata": True}, - "#urls" : "https://puffball.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:z72i7hdynmk6r22z27h6tvur&cid=bafkreidypzoaybmfj5h7pnpiyct6ng5yae6ydp4czrm72ocg7ev6vbirri", - "#sha1_content": "ffcf25e7c511173a12de5276b85903309fcd8d14", - - "author": { - "avatar" : "https://cdn.bsky.app/img/avatar/plain/did:plc:z72i7hdynmk6r22z27h6tvur/bafkreihagr2cmvl2jt4mgx3sppwe2it3fwolkrbtjrhcnwjk4jdijhsoze@jpeg", - "did" : "did:plc:z72i7hdynmk6r22z27h6tvur", - "displayName": "Bluesky", - "handle" : "bsky.app", - "instance" : "bsky.app", - "labels" : [], - }, - "cid" : "bafyreihh7m6bfrwlcjfklwturmja7qfse5gte7lskpmgw76flivimbnoqm", - "count" : 1, - "createdAt" : "2023-12-22T18:58:32.715Z", - "date" : "dt:2023-12-22 18:58:32", - "description": "The bluesky logo with the blue butterfly", - "extension" : "jpeg", - "filename" : "bafkreidypzoaybmfj5h7pnpiyct6ng5yae6ydp4czrm72ocg7ev6vbirri", - "height" : 630, - "indexedAt" : "2023-12-22T18:58:32.715Z", - "instance" : "bsky.app", - "labels" : [], - "likeCount" : int, - "num" : 1, - "post_id" : "3kh5rarr3gn2n", - "replyCount" : int, - "repostCount": int, - "uri" : "at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.post/3kh5rarr3gn2n", - "width" : 1200, - "hashtags" : [], - "mentions" : [], - "uris" : ["https://blueskyweb.xyz/blog/12-21-2023-butterfly"], - "user" : { - "avatar" : str, - "banner" : str, - "description" : str, - "did" : "did:plc:z72i7hdynmk6r22z27h6tvur", - "displayName" : "Bluesky", - "followersCount": int, - "followsCount" : int, - "handle" : "bsky.app", - "instance" : "bsky.app", - "indexedAt" : str, - "labels" : [], - "postsCount" : int, - }, -}, - -{ - "#url" : "https://bsky.app/profile/mikf.bsky.social/post/3kkzc3xaf5m2w", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, - "#options" : {"metadata": "facets"}, - "#urls" : "https://conocybe.us-west.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:cslxjqkeexku6elp5xowxkq7&cid=bafkreib7ydpe3xxo4cq7nn32w7eqhcanfaanz6caepd2z4kzplxtx2ctgi", - "#sha1_content": "9cf5748f6d00aae83fbb3cc2c6eb3caa832b90f4", - - "author": { - "did" : "did:plc:cslxjqkeexku6elp5xowxkq7", - "displayName": "mikf", - "handle" : "mikf.bsky.social", - "instance" : "bsky.social", - "labels" : [], - }, - "cid" : "bafyreihtck7clocti2qshaiounadof74pxqhz7gnvbstxujqzhlodigqru", - "count" : 1, - "createdAt" : "2024-02-09T21:57:31.917Z", - "date" : "dt:2024-02-09 21:57:31", - "description": "reading lewd books", - "extension" : "jpeg", - "filename" : "bafkreib7ydpe3xxo4cq7nn32w7eqhcanfaanz6caepd2z4kzplxtx2ctgi", - "hashtags" : [ - "patchouli", - "patchy", - ], - "mentions" : [ - "did:plc:cslxjqkeexku6elp5xowxkq7", - ], - "uris" : [ - "https://seiga.nicovideo.jp/seiga/im5977527", - ], - "width" : 1024, - "height" : 768, - "langs" : ["en"], - "likeCount" : int, - "num" : 1, - "post_id" : "3kkzc3xaf5m2w", - "replyCount" : int, - "repostCount": int, - "text" : "testing \"facets\"\n\nsource: seiga.nicovideo.jp/seiga/im5977...\n#patchouli #patchy\n@mikf.bsky.social", - "uri" : "at://did:plc:cslxjqkeexku6elp5xowxkq7/app.bsky.feed.post/3kkzc3xaf5m2w", -}, - -{ - "#url" : "https://bsky.app/profile/go-guiltism.bsky.social/post/3klgth6lilt2l", - "#comment" : "different embed CID path", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, - "#urls" : "https://amanita.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:owc2r2dsewj3hk73rtd746zh&cid=bafkreieuhplc7fpbvi3suvacaf2dqxzvuu4hgl5o6eifqb76tf3uopldmi", -}, - -{ - "#url" : "https://bsky.app/profile/mikf.bsky.social/post/3l46q5glfex27", - "#comment" : "video (#6183)", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, - "#urls" : "https://conocybe.us-west.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:cslxjqkeexku6elp5xowxkq7&cid=bafkreihq2nsfocrnlpx4nykb4szouqszxwmy3ucnk4k46nx5t6hjnxlti4", - - "description": "kirby and reimu dance", - "text" : "video", - "width" : 1280, - "height" : 720, - "filename" : "bafkreihq2nsfocrnlpx4nykb4szouqszxwmy3ucnk4k46nx5t6hjnxlti4", - "extension" : "mp4", -}, - -{ - "#url" : "https://bsky.app/profile/mikf.bsky.social/post/3kmfodjotln2f", - "#comment" : "quote (#6183)", - "#class" : bluesky.BlueskyPostExtractor, - "#options" : {"quoted": True}, - "#urls" : "https://lionsmane.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:eyhmjdxsnthqhvvszdejaocz&cid=bafkreib6eb7tfozksquveaj3z5msyx3hkniubrulxdys3eftthvmuzrtme", - - "author": { - "associated" : dict, - "avatar" : "https://cdn.bsky.app/img/avatar/plain/did:plc:eyhmjdxsnthqhvvszdejaocz/bafkreigjrftlw7tabtpie32saydttpnoi7276v252vnycr6zt6euef7vdi@jpeg", - "createdAt" : "2024-01-11T00:27:37.404Z", - "did" : "did:plc:eyhmjdxsnthqhvvszdejaocz", - "displayName": "フナ", - "handle" : "ykfuna.bsky.social", - "labels" : list, - }, - "quote_by": { - "avatar" : "https://cdn.bsky.app/img/avatar/plain/did:plc:cslxjqkeexku6elp5xowxkq7/bafkreic5jqkn5ohqhgsm6zzi7vnapuz54trojv3io4tfkrcyaprl4b2ztm@jpeg", - "createdAt" : "2024-02-05T00:03:54.087Z", - "did" : "did:plc:cslxjqkeexku6elp5xowxkq7", - "displayName": "mikf", - "handle" : "mikf.bsky.social", - "labels" : list, - }, - "quote_id": "3kmfodjotln2f", - "post_id" : "3km4qy5y3jc2z", -}, - -{ - "#url" : "https://bsky.app/profile/mikf.bsky.social/post/3kmfp2qktil25", - "#comment" : "quote with media (#6183)", - "#class" : bluesky.BlueskyPostExtractor, - "#options" : {"quoted": True}, - "#urls" : ( - "https://conocybe.us-west.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:cslxjqkeexku6elp5xowxkq7&cid=bafkreiegcyremdrecmnpisci3a3nduc7lm3zdcl76z5o5rd4nstyolrxki", - "https://lionsmane.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:eyhmjdxsnthqhvvszdejaocz&cid=bafkreicojrnwiw5eqo3ko2q6duduyjaoyiqvdc25kuikcedlijtbgvlt5e", - - ), - - "text" : {"quote with media", ""}, -}, - -{ - "#url" : "https://bsky.app/profile/nytimes.com/post/3l7xvcjgdxg2g", - "#comment" : "instance metadata", - "#class" : bluesky.BlueskyPostExtractor, - "#options" : {"metadata": "user"}, - - "instance": "bsky.app", - "author": { - "createdAt" : "2023-06-05T18:50:31.498Z", - "did" : "did:plc:eclio37ymobqex2ncko63h4r", - "displayName": "The New York Times", - "handle" : "nytimes.com", - "instance" : "nytimes.com", - }, - "user": { - "avatar" : "https://cdn.bsky.app/img/avatar/plain/did:plc:eclio37ymobqex2ncko63h4r/bafkreidvvqj5jymmpaeklwkpq6gi532el447mjy2yultuukypzqm5ohfju@jpeg", - "banner" : "https://cdn.bsky.app/img/banner/plain/did:plc:eclio37ymobqex2ncko63h4r/bafkreiaiorkgl6t2j5w3sf6nj37drvwuvriq3e3vqwf4yn3pchpwfbekta@jpeg", - "createdAt" : "2023-06-05T18:50:31.498Z", - "description" : "In-depth, independent reporting to better understand the world, now on Bluesky. News tips? Share them here: http://nyti.ms/2FVHq9v", - "did" : "did:plc:eclio37ymobqex2ncko63h4r", - "displayName" : "The New York Times", - "followersCount": int, - "followsCount" : int, - "handle" : "nytimes.com", - "instance" : "nytimes.com", - "indexedAt" : "2024-01-20T05:04:46.757Z", - "labels" : [], - "postsCount" : int, - }, -}, - -{ - "#url" : "https://bsky.app/profile/stupidsaru.woke.cat/post/3l66wwwqw6u2w", - "#comment" : "instance metadata", - "#class" : bluesky.BlueskyPostExtractor, - - "author": { - "createdAt": "2023-08-31T23:28:42.305Z", - "did" : "did:plc:b7s3pdcjk6qvxmu3n674hlgj", - "handle" : "stupidsaru.woke.cat", - "instance" : "woke.cat", + { + "#url": "https://bsky.app/profile/bsky.app", + "#category": ("", "bluesky", "user"), + "#class": bluesky.BlueskyUserExtractor, + "#urls": ("https://bsky.app/profile/bsky.app/media",), }, -}, - -{ - "#url" : "https://bsky.app/profile/alt.bun.how/post/3l7rdfxhyds2f", - "#comment" : "non-bsky PDS (#6406)", - "#class" : bluesky.BlueskyPostExtractor, - "#urls" : "https://pds.bun.how/xrpc/com.atproto.sync.getBlob?did=did:plc:7x6rtuenkuvxq3zsvffp2ide&cid=bafkreielhgekjheckgjusx7x5hxkbrqryfdmzdwwp2zoxchovgnpzkxzae", - "#sha1_content": "1777956de0dc8cf0815c5c7eb574a24ce54a1d42", - - "author": { - "createdAt": "2024-10-17T13:55:48.833Z", - "did" : "did:plc:7x6rtuenkuvxq3zsvffp2ide", - "handle" : "alt.bun.how", - "instance" : "bun.how", + { + "#url": "https://www.bsky.app/profile/bsky.app", + "#class": bluesky.BlueskyUserExtractor, + }, + { + "#url": "https://main.bsky.dev/profile/bsky.app", + "#class": bluesky.BlueskyUserExtractor, + }, + { + "#url": "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur", + "#category": ("", "bluesky", "user"), + "#class": bluesky.BlueskyUserExtractor, + "#options": {"include": "all"}, + "#urls": ( + "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/avatar", + "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/banner", + "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/posts", + "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/replies", + "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/media", + "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/likes", + ), + }, + { + "#url": "https://bsky.app/profile/bsky.app/avatar", + "#category": ("", "bluesky", "avatar"), + "#class": bluesky.BlueskyAvatarExtractor, + "#urls": "https://puffball.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:z72i7hdynmk6r22z27h6tvur&cid=bafkreihagr2cmvl2jt4mgx3sppwe2it3fwolkrbtjrhcnwjk4jdijhsoze", + }, + { + "#url": "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/banner", + "#category": ("", "bluesky", "background"), + "#class": bluesky.BlueskyBackgroundExtractor, + "#urls": "https://puffball.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:z72i7hdynmk6r22z27h6tvur&cid=bafkreichzyovokfzmymz36p5jibbjrhsur6n7hjnzxrpbt5jaydp2szvna", + }, + { + "#url": "https://bsky.app/profile/bsky.app/posts", + "#category": ("", "bluesky", "posts"), + "#class": bluesky.BlueskyPostsExtractor, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "https://bsky.app/profile/bsky.app/replies", + "#category": ("", "bluesky", "replies"), + "#class": bluesky.BlueskyRepliesExtractor, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "https://bsky.app/profile/bsky.app/media", + "#category": ("", "bluesky", "media"), + "#class": bluesky.BlueskyMediaExtractor, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "https://bsky.app/profile/did:plc:jfhpnnst6flqway4eaeqzj2a/feed/for-science", + "#category": ("", "bluesky", "feed"), + "#class": bluesky.BlueskyFeedExtractor, + "#range": "1-40", + "#count": 40, + "#archive": False, + }, + { + "#url": "https://bsky.app/profile/bsky.app/follows", + "#category": ("", "bluesky", "following"), + "#class": bluesky.BlueskyFollowingExtractor, + "#urls": ( + "https://bsky.app/profile/did:plc:eon2iu7v3x2ukgxkqaf7e5np", + "https://bsky.app/profile/did:plc:ewvi7nxzyoun6zhxrhs64oiz", + ), + }, + { + "#url": "https://bsky.app/profile/bsky.app/likes", + "#category": ("", "bluesky", "likes"), + "#class": bluesky.BlueskyLikesExtractor, + }, + { + "#url": "https://bsky.app/profile/bsky.app/lists/abcdefghijklm", + "#category": ("", "bluesky", "list"), + "#class": bluesky.BlueskyListExtractor, + }, + { + "#url": "https://bsky.app/search?q=nature", + "#category": ("", "bluesky", "search"), + "#class": bluesky.BlueskySearchExtractor, + "#range": "1-40", + "#count": 40, + "#archive": False, + }, + { + "#url": "https://bsky.app/hashtag/nature", + "#class": bluesky.BlueskyHashtagExtractor, + "#range": "1-40", + "#count": 40, + "#archive": False, + }, + { + "#url": "https://bsky.app/hashtag/top", + "#class": bluesky.BlueskyHashtagExtractor, + }, + { + "#url": "https://bsky.app/hashtag/nature/latest", + "#class": bluesky.BlueskyHashtagExtractor, + }, + { + "#url": "https://bsky.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + "#options": {"metadata": True}, + "#urls": "https://puffball.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:z72i7hdynmk6r22z27h6tvur&cid=bafkreidypzoaybmfj5h7pnpiyct6ng5yae6ydp4czrm72ocg7ev6vbirri", + "#sha1_content": "ffcf25e7c511173a12de5276b85903309fcd8d14", + "author": { + "avatar": "https://cdn.bsky.app/img/avatar/plain/did:plc:z72i7hdynmk6r22z27h6tvur/bafkreihagr2cmvl2jt4mgx3sppwe2it3fwolkrbtjrhcnwjk4jdijhsoze@jpeg", + "did": "did:plc:z72i7hdynmk6r22z27h6tvur", + "displayName": "Bluesky", + "handle": "bsky.app", + "instance": "bsky.app", + "labels": [], + }, + "cid": "bafyreihh7m6bfrwlcjfklwturmja7qfse5gte7lskpmgw76flivimbnoqm", + "count": 1, + "createdAt": "2023-12-22T18:58:32.715Z", + "date": "dt:2023-12-22 18:58:32", + "description": "The bluesky logo with the blue butterfly", + "extension": "jpeg", + "filename": "bafkreidypzoaybmfj5h7pnpiyct6ng5yae6ydp4czrm72ocg7ev6vbirri", + "height": 630, + "indexedAt": "2023-12-22T18:58:32.715Z", + "instance": "bsky.app", + "labels": [], + "likeCount": int, + "num": 1, + "post_id": "3kh5rarr3gn2n", + "replyCount": int, + "repostCount": int, + "uri": "at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.post/3kh5rarr3gn2n", + "width": 1200, + "hashtags": [], + "mentions": [], + "uris": ["https://blueskyweb.xyz/blog/12-21-2023-butterfly"], + "user": { + "avatar": str, + "banner": str, + "description": str, + "did": "did:plc:z72i7hdynmk6r22z27h6tvur", + "displayName": "Bluesky", + "followersCount": int, + "followsCount": int, + "handle": "bsky.app", + "instance": "bsky.app", + "indexedAt": str, + "labels": [], + "postsCount": int, + }, + }, + { + "#url": "https://bsky.app/profile/mikf.bsky.social/post/3kkzc3xaf5m2w", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + "#options": {"metadata": "facets"}, + "#urls": "https://conocybe.us-west.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:cslxjqkeexku6elp5xowxkq7&cid=bafkreib7ydpe3xxo4cq7nn32w7eqhcanfaanz6caepd2z4kzplxtx2ctgi", + "#sha1_content": "9cf5748f6d00aae83fbb3cc2c6eb3caa832b90f4", + "author": { + "did": "did:plc:cslxjqkeexku6elp5xowxkq7", + "displayName": "mikf", + "handle": "mikf.bsky.social", + "instance": "bsky.social", + "labels": [], + }, + "cid": "bafyreihtck7clocti2qshaiounadof74pxqhz7gnvbstxujqzhlodigqru", + "count": 1, + "createdAt": "2024-02-09T21:57:31.917Z", + "date": "dt:2024-02-09 21:57:31", + "description": "reading lewd books", + "extension": "jpeg", + "filename": "bafkreib7ydpe3xxo4cq7nn32w7eqhcanfaanz6caepd2z4kzplxtx2ctgi", + "hashtags": [ + "patchouli", + "patchy", + ], + "mentions": [ + "did:plc:cslxjqkeexku6elp5xowxkq7", + ], + "uris": [ + "https://seiga.nicovideo.jp/seiga/im5977527", + ], + "width": 1024, + "height": 768, + "langs": ["en"], + "likeCount": int, + "num": 1, + "post_id": "3kkzc3xaf5m2w", + "replyCount": int, + "repostCount": int, + "text": 'testing "facets"\n\nsource: seiga.nicovideo.jp/seiga/im5977...\n#patchouli #patchy\n@mikf.bsky.social', + "uri": "at://did:plc:cslxjqkeexku6elp5xowxkq7/app.bsky.feed.post/3kkzc3xaf5m2w", + }, + { + "#url": "https://bsky.app/profile/go-guiltism.bsky.social/post/3klgth6lilt2l", + "#comment": "different embed CID path", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + "#urls": "https://amanita.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:owc2r2dsewj3hk73rtd746zh&cid=bafkreieuhplc7fpbvi3suvacaf2dqxzvuu4hgl5o6eifqb76tf3uopldmi", + }, + { + "#url": "https://bsky.app/profile/mikf.bsky.social/post/3l46q5glfex27", + "#comment": "video (#6183)", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + "#urls": "https://conocybe.us-west.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:cslxjqkeexku6elp5xowxkq7&cid=bafkreihq2nsfocrnlpx4nykb4szouqszxwmy3ucnk4k46nx5t6hjnxlti4", + "description": "kirby and reimu dance", + "text": "video", + "width": 1280, + "height": 720, + "filename": "bafkreihq2nsfocrnlpx4nykb4szouqszxwmy3ucnk4k46nx5t6hjnxlti4", + "extension": "mp4", + }, + { + "#url": "https://bsky.app/profile/mikf.bsky.social/post/3kmfodjotln2f", + "#comment": "quote (#6183)", + "#class": bluesky.BlueskyPostExtractor, + "#options": {"quoted": True}, + "#urls": "https://lionsmane.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:eyhmjdxsnthqhvvszdejaocz&cid=bafkreib6eb7tfozksquveaj3z5msyx3hkniubrulxdys3eftthvmuzrtme", + "author": { + "associated": dict, + "avatar": "https://cdn.bsky.app/img/avatar/plain/did:plc:eyhmjdxsnthqhvvszdejaocz/bafkreigjrftlw7tabtpie32saydttpnoi7276v252vnycr6zt6euef7vdi@jpeg", + "createdAt": "2024-01-11T00:27:37.404Z", + "did": "did:plc:eyhmjdxsnthqhvvszdejaocz", + "displayName": "フナ", + "handle": "ykfuna.bsky.social", + "labels": list, + }, + "quote_by": { + "avatar": "https://cdn.bsky.app/img/avatar/plain/did:plc:cslxjqkeexku6elp5xowxkq7/bafkreic5jqkn5ohqhgsm6zzi7vnapuz54trojv3io4tfkrcyaprl4b2ztm@jpeg", + "createdAt": "2024-02-05T00:03:54.087Z", + "did": "did:plc:cslxjqkeexku6elp5xowxkq7", + "displayName": "mikf", + "handle": "mikf.bsky.social", + "labels": list, + }, + "quote_id": "3kmfodjotln2f", + "post_id": "3km4qy5y3jc2z", + }, + { + "#url": "https://bsky.app/profile/mikf.bsky.social/post/3kmfp2qktil25", + "#comment": "quote with media (#6183)", + "#class": bluesky.BlueskyPostExtractor, + "#options": {"quoted": True}, + "#urls": ( + "https://conocybe.us-west.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:cslxjqkeexku6elp5xowxkq7&cid=bafkreiegcyremdrecmnpisci3a3nduc7lm3zdcl76z5o5rd4nstyolrxki", + "https://lionsmane.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:eyhmjdxsnthqhvvszdejaocz&cid=bafkreicojrnwiw5eqo3ko2q6duduyjaoyiqvdc25kuikcedlijtbgvlt5e", + ), + "text": {"quote with media", ""}, + }, + { + "#url": "https://bsky.app/profile/nytimes.com/post/3l7xvcjgdxg2g", + "#comment": "instance metadata", + "#class": bluesky.BlueskyPostExtractor, + "#options": {"metadata": "user"}, + "instance": "bsky.app", + "author": { + "createdAt": "2023-06-05T18:50:31.498Z", + "did": "did:plc:eclio37ymobqex2ncko63h4r", + "displayName": "The New York Times", + "handle": "nytimes.com", + "instance": "nytimes.com", + }, + "user": { + "avatar": "https://cdn.bsky.app/img/avatar/plain/did:plc:eclio37ymobqex2ncko63h4r/bafkreidvvqj5jymmpaeklwkpq6gi532el447mjy2yultuukypzqm5ohfju@jpeg", + "banner": "https://cdn.bsky.app/img/banner/plain/did:plc:eclio37ymobqex2ncko63h4r/bafkreiaiorkgl6t2j5w3sf6nj37drvwuvriq3e3vqwf4yn3pchpwfbekta@jpeg", + "createdAt": "2023-06-05T18:50:31.498Z", + "description": "In-depth, independent reporting to better understand the world, now on Bluesky. News tips? Share them here: http://nyti.ms/2FVHq9v", + "did": "did:plc:eclio37ymobqex2ncko63h4r", + "displayName": "The New York Times", + "followersCount": int, + "followsCount": int, + "handle": "nytimes.com", + "instance": "nytimes.com", + "indexedAt": "2024-01-20T05:04:46.757Z", + "labels": [], + "postsCount": int, + }, + }, + { + "#url": "https://bsky.app/profile/stupidsaru.woke.cat/post/3l66wwwqw6u2w", + "#comment": "instance metadata", + "#class": bluesky.BlueskyPostExtractor, + "author": { + "createdAt": "2023-08-31T23:28:42.305Z", + "did": "did:plc:b7s3pdcjk6qvxmu3n674hlgj", + "handle": "stupidsaru.woke.cat", + "instance": "woke.cat", + }, + }, + { + "#url": "https://bsky.app/profile/alt.bun.how/post/3l7rdfxhyds2f", + "#comment": "non-bsky PDS (#6406)", + "#class": bluesky.BlueskyPostExtractor, + "#urls": "https://pds.bun.how/xrpc/com.atproto.sync.getBlob?did=did:plc:7x6rtuenkuvxq3zsvffp2ide&cid=bafkreielhgekjheckgjusx7x5hxkbrqryfdmzdwwp2zoxchovgnpzkxzae", + "#sha1_content": "1777956de0dc8cf0815c5c7eb574a24ce54a1d42", + "author": { + "createdAt": "2024-10-17T13:55:48.833Z", + "did": "did:plc:7x6rtuenkuvxq3zsvffp2ide", + "handle": "alt.bun.how", + "instance": "bun.how", + }, + }, + { + "#url": "https://cbsky.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + }, + { + "#url": "https://bskye.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + }, + { + "#url": "https://bskyx.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + }, + { + "#url": "https://bsyy.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + }, + { + "#url": "https://fxbsky.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + }, + { + "#url": "https://vxbsky.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, }, -}, - -{ - "#url" : "https://cbsky.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, -}, - -{ - "#url" : "https://bskye.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, -}, - -{ - "#url" : "https://bskyx.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, -}, - -{ - "#url" : "https://bsyy.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, -}, - -{ - "#url" : "https://fxbsky.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, -}, - -{ - "#url" : "https://vxbsky.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, -}, - ) diff --git a/test/results/booruvar.py b/test/results/booruvar.py index 8beb45e17..dcb669e4c 100644 --- a/test/results/booruvar.py +++ b/test/results/booruvar.py @@ -1,40 +1,33 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import danbooru - __tests__ = ( -{ - "#url" : "https://booru.borvar.art/posts?tags=chibi&z=1", - "#category": ("Danbooru", "booruvar", "tag"), - "#class" : danbooru.DanbooruTagExtractor, - "#pattern" : r"https://booru\.borvar\.art/data/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", - "#count" : ">= 3", -}, - -{ - "#url" : "https://booru.borvar.art/pools/2", - "#category": ("Danbooru", "booruvar", "pool"), - "#class" : danbooru.DanbooruPoolExtractor, - "#count" : 4, - "#sha1_url": "77fa3559a3fc919f72611f4e3dd0f919d19d3e0d", -}, - -{ - "#url" : "https://booru.borvar.art/posts/1487", - "#category": ("Danbooru", "booruvar", "post"), - "#class" : danbooru.DanbooruPostExtractor, - "#sha1_content": "91273ac1ea413a12be468841e2b5804656a50bff", -}, - -{ - "#url" : "https://booru.borvar.art/explore/posts/popular", - "#category": ("Danbooru", "booruvar", "popular"), - "#class" : danbooru.DanbooruPopularExtractor, -}, - + { + "#url": "https://booru.borvar.art/posts?tags=chibi&z=1", + "#category": ("Danbooru", "booruvar", "tag"), + "#class": danbooru.DanbooruTagExtractor, + "#pattern": r"https://booru\.borvar\.art/data/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", + "#count": ">= 3", + }, + { + "#url": "https://booru.borvar.art/pools/2", + "#category": ("Danbooru", "booruvar", "pool"), + "#class": danbooru.DanbooruPoolExtractor, + "#count": 4, + "#sha1_url": "77fa3559a3fc919f72611f4e3dd0f919d19d3e0d", + }, + { + "#url": "https://booru.borvar.art/posts/1487", + "#category": ("Danbooru", "booruvar", "post"), + "#class": danbooru.DanbooruPostExtractor, + "#sha1_content": "91273ac1ea413a12be468841e2b5804656a50bff", + }, + { + "#url": "https://booru.borvar.art/explore/posts/popular", + "#category": ("Danbooru", "booruvar", "popular"), + "#class": danbooru.DanbooruPopularExtractor, + }, ) diff --git a/test/results/boosty.py b/test/results/boosty.py index 2da0e6b3c..3957166c7 100644 --- a/test/results/boosty.py +++ b/test/results/boosty.py @@ -1,135 +1,121 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import boosty - __tests__ = ( -{ - "#url" : "https://boosty.to/milshoo", - "#class" : boosty.BoostyUserExtractor, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://boosty.to/milshoo?postsFrom=1706742000&postsTo=1709247599", - "#class" : boosty.BoostyUserExtractor, - "#urls" : "https://images.boosty.to/image/ff0d2006-3ee7-483d-a5fc-2a05b531742c?change_time=1707829201", -}, - -{ - "#url" : "https://boosty.to/milshoo/media/all", - "#class" : boosty.BoostyMediaExtractor, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://boosty.to/milshoo/posts/4304d8f0-3f49-4f97-a3f3-9f064bc32b2f", - "#class" : boosty.BoostyPostExtractor, - "#urls" : "https://images.boosty.to/image/75f86086-fc67-4ed2-9365-2958d3d1a8f7?change_time=1711027786", - - "count" : 1, - "num" : 1, - "extension": "", - "filename" : "75f86086-fc67-4ed2-9365-2958d3d1a8f7", - - "file": { - "height" : 2048, - "id" : "75f86086-fc67-4ed2-9365-2958d3d1a8f7", - "rendition": "", - "size" : 1094903, - "type" : "image", - "url" : "https://images.boosty.to/image/75f86086-fc67-4ed2-9365-2958d3d1a8f7?change_time=1711027786", - "width" : 2048, + { + "#url": "https://boosty.to/milshoo", + "#class": boosty.BoostyUserExtractor, + "#range": "1-40", + "#count": 40, }, - "user": { - "avatarUrl": "https://images.boosty.to/user/173542/avatar?change_time=1580888689", - "blogUrl" : "milshoo", - "flags" : { - "showPostDonations": True, - }, - "hasAvatar": True, - "id" : 173542, - "name" : "Милшу", + { + "#url": "https://boosty.to/milshoo?postsFrom=1706742000&postsTo=1709247599", + "#class": boosty.BoostyUserExtractor, + "#urls": "https://images.boosty.to/image/ff0d2006-3ee7-483d-a5fc-2a05b531742c?change_time=1707829201", + }, + { + "#url": "https://boosty.to/milshoo/media/all", + "#class": boosty.BoostyMediaExtractor, + "#range": "1-40", + "#count": 40, }, - "post": { - "comments" : dict, - "content" : [ - "Привет! Это Милшу ) Я открываю комментарии в своём телеграм канале Милшу ( ", - "https://t.me/milshoonya", - " ) и хочу, чтобы вы первые протестировали его работу :3\nСсылку на вступление в чат оставлю здесь ", - "https://t.me/+Z_5ph-XnIQU2YWMy", - "\nТакже хотела напомнить, что мы собираем деньги на два арта от Ананаси: ", - "https://boosty.to/milshoo/single-payment/donation/550562/target?share=target_link", - "\nБуду очень благодарна за помощь :D  ", - ], - "contentCounters": list, - "count" : dict, - "createdAt" : 1711027834, - "currencyPrices": { - "RUB": 0, - "USD": 0, + { + "#url": "https://boosty.to/milshoo/posts/4304d8f0-3f49-4f97-a3f3-9f064bc32b2f", + "#class": boosty.BoostyPostExtractor, + "#urls": "https://images.boosty.to/image/75f86086-fc67-4ed2-9365-2958d3d1a8f7?change_time=1711027786", + "count": 1, + "num": 1, + "extension": "", + "filename": "75f86086-fc67-4ed2-9365-2958d3d1a8f7", + "file": { + "height": 2048, + "id": "75f86086-fc67-4ed2-9365-2958d3d1a8f7", + "rendition": "", + "size": 1094903, + "type": "image", + "url": "https://images.boosty.to/image/75f86086-fc67-4ed2-9365-2958d3d1a8f7?change_time=1711027786", + "width": 2048, + }, + "user": { + "avatarUrl": "https://images.boosty.to/user/173542/avatar?change_time=1580888689", + "blogUrl": "milshoo", + "flags": { + "showPostDonations": True, + }, + "hasAvatar": True, + "id": 173542, + "name": "Милшу", + }, + "post": { + "comments": dict, + "content": [ + "Привет! Это Милшу ) Я открываю комментарии в своём телеграм канале Милшу ( ", + "https://t.me/milshoonya", + " ) и хочу, чтобы вы первые протестировали его работу :3\nСсылку на вступление в чат оставлю здесь ", + "https://t.me/+Z_5ph-XnIQU2YWMy", + "\nТакже хотела напомнить, что мы собираем деньги на два арта от Ананаси: ", + "https://boosty.to/milshoo/single-payment/donation/550562/target?share=target_link", + "\nБуду очень благодарна за помощь :D  ", + ], + "contentCounters": list, + "count": dict, + "createdAt": 1711027834, + "currencyPrices": { + "RUB": 0, + "USD": 0, + }, + "date": "dt:2024-03-21 13:30:34", + "donations": 0, + "donators": dict, + "hasAccess": True, + "id": "4304d8f0-3f49-4f97-a3f3-9f064bc32b2f", + "int_id": 5547124, + "isBlocked": False, + "isCommentsDenied": False, + "isDeleted": False, + "isLiked": False, + "isPublished": True, + "isRecord": False, + "isWaitingVideo": False, + "links": [ + "https://t.me/milshoonya", + "https://t.me/+Z_5ph-XnIQU2YWMy", + "https://boosty.to/milshoo/single-payment/donation/550562/target?share=target_link", + ], + "price": 0, + "publishTime": 1711027834, + "showViewsCounter": False, + "signedQuery": "", + "tags": [], + "teaser": [], + "title": "Открываю чат в телеге", + "updatedAt": 1711027904, }, - "date" : "dt:2024-03-21 13:30:34", - "donations" : 0, - "donators" : dict, - "hasAccess" : True, - "id" : "4304d8f0-3f49-4f97-a3f3-9f064bc32b2f", - "int_id" : 5547124, - "isBlocked" : False, - "isCommentsDenied": False, - "isDeleted" : False, - "isLiked" : False, - "isPublished": True, - "isRecord" : False, - "isWaitingVideo": False, - "links" : [ - "https://t.me/milshoonya", - "https://t.me/+Z_5ph-XnIQU2YWMy", - "https://boosty.to/milshoo/single-payment/donation/550562/target?share=target_link", - ], - "price" : 0, - "publishTime": 1711027834, - "showViewsCounter": False, - "signedQuery": "", - "tags" : [], - "teaser" : [], - "title" : "Открываю чат в телеге", - "updatedAt" : 1711027904 }, -}, - -{ - "#url" : "https://boosty.to/geekmedia/posts/31bb8fb6-83f1-404f-a597-f84bbe611d1d", - "#comment" : "video", - "#class" : boosty.BoostyPostExtractor, -}, - -{ - "#url" : "https://boosty.to/xcang/posts/5d4d6f90-5d48-4442-a7e5-2164a858681d", - "#comment" : "audio", - "#class" : boosty.BoostyPostExtractor, -}, - -{ - "#url" : "https://boosty.to/", - "#class" : boosty.BoostyFeedExtractor, - "#auth" : True, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://boosty.to/app/settings/subscriptions", - "#class" : boosty.BoostyFollowingExtractor, - "#pattern" : boosty.BoostyUserExtractor, - "#auth" : True, -}, - - + { + "#url": "https://boosty.to/geekmedia/posts/31bb8fb6-83f1-404f-a597-f84bbe611d1d", + "#comment": "video", + "#class": boosty.BoostyPostExtractor, + }, + { + "#url": "https://boosty.to/xcang/posts/5d4d6f90-5d48-4442-a7e5-2164a858681d", + "#comment": "audio", + "#class": boosty.BoostyPostExtractor, + }, + { + "#url": "https://boosty.to/", + "#class": boosty.BoostyFeedExtractor, + "#auth": True, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "https://boosty.to/app/settings/subscriptions", + "#class": boosty.BoostyFollowingExtractor, + "#pattern": boosty.BoostyUserExtractor, + "#auth": True, + }, ) diff --git a/test/results/bulbapedia.py b/test/results/bulbapedia.py index 1549a0350..5bcbb6e65 100644 --- a/test/results/bulbapedia.py +++ b/test/results/bulbapedia.py @@ -1,28 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://bulbapedia.bulbagarden.net/wiki/Jet", - "#category": ("wikimedia", "bulbapedia", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#pattern" : r"https://archives\.bulbagarden\.net/media/upload/\w+/\w+/[^/?#]+", - "#count" : range(10, 30), -}, - -{ - "#url" : "https://archives.bulbagarden.net/wiki/File:0460Abomasnow-Mega.png", - "#category": ("wikimedia", "bulbapedia", "file"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#pattern" : r"https://archives\.bulbagarden\.net/media/upload/\w+/\w+/[^/?#]+", - "#count" : range(8, 12), - "#archive" : False, -}, - + { + "#url": "https://bulbapedia.bulbagarden.net/wiki/Jet", + "#category": ("wikimedia", "bulbapedia", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#pattern": r"https://archives\.bulbagarden\.net/media/upload/\w+/\w+/[^/?#]+", + "#count": range(10, 30), + }, + { + "#url": "https://archives.bulbagarden.net/wiki/File:0460Abomasnow-Mega.png", + "#category": ("wikimedia", "bulbapedia", "file"), + "#class": wikimedia.WikimediaArticleExtractor, + "#pattern": r"https://archives\.bulbagarden\.net/media/upload/\w+/\w+/[^/?#]+", + "#count": range(8, 12), + "#archive": False, + }, ) diff --git a/test/results/bunkr.py b/test/results/bunkr.py index 4c732a9ea..1d4ea8d98 100644 --- a/test/results/bunkr.py +++ b/test/results/bunkr.py @@ -1,211 +1,176 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import bunkr - __tests__ = ( -{ - "#url" : "https://bunkr.sk/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, - "#urls" : "https://i-burger.bunkr.ru/test-%E3%83%86%E3%82%B9%E3%83%88-%22%26%3E-QjgneIQv.png?download=true", - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", - - "album_id" : "Lktg9Keq", - "album_name" : "test テスト \"&>", - "album_size" : "182 B", - "count" : 1, - "extension" : "png", - "file" : "https://i-burger.bunkr.ru/test-%E3%83%86%E3%82%B9%E3%83%88-%22%26%3E-QjgneIQv.png?download=true", - "filename" : "test-テスト-\"&>-QjgneIQv", - "id" : "QjgneIQv", - "name" : "test-テスト-\"&>", - "num" : 1, -}, - -{ - "#url" : "https://bunkr.is/a/iXTTc1o2", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, - "#urls" : ( - "https://i-milkshake.bunkr.ru/image-sZrQUeOx.jpg?download=true", - ), - "#sha1_content": "caf7c3d3439d94e83b3c24ddaf5a3a48aa057519", - - "album_id" : "iXTTc1o2", - "album_name" : "test2", - "album_size" : "534.6 KB", - "count" : 1, - "filename" : r"image-sZrQUeOx", - "id" : r"sZrQUeOx", - "name" : r"image", - "num" : 1, -}, - -{ - "#url" : "https://bunkr.cat/a/j1G29CnD", - "#comment" : "cdn12 .ru TLD (#4147)", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, - "#pattern" : r"https://(i-)?meatballs.bunkr.ru/\w+", - "#count" : 7, -}, - -{ - "#url" : "https://bunkr.ph/a/Lktg9Keq", - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.ps/a/Lktg9Keq", - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.pk/a/Lktg9Keq", - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.ax/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkrrr.org/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.ci/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.cr/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.fi/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.si/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.ac/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.media/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.site/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.ws/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkrr.ru/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkrr.su/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.la/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.su/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.ru/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.is/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.to/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "bunkr:http://example.org/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.black/i/image-sZrQUeOx.jpg", - "#category": ("lolisafe", "bunkr", "media"), - "#class" : bunkr.BunkrMediaExtractor, - "#urls" : "https://i-milkshake.bunkr.ru/image-sZrQUeOx.jpg?download=true", - "#sha1_content": "caf7c3d3439d94e83b3c24ddaf5a3a48aa057519", - - "count" : 1, - "extension": "jpg", - "file" : "https://i-milkshake.bunkr.ru/image-sZrQUeOx.jpg?download=true", - "filename" : "image-sZrQUeOx", - "id" : "sZrQUeOx", - "name" : "image", -}, - -{ - "#url" : "https://bunkrrr.org/d/dJuETSzKLrUps", - "#category": ("lolisafe", "bunkr", "media"), - "#class" : bunkr.BunkrMediaExtractor, - "#urls" : "https://burger.bunkr.ru/file-r5fmwjdd.zip", - "#sha1_content": "102ddd7894fe39b3843098fc51f972a0af938f45", - - "count" : 1, - "extension": "zip", - "file" : "https://burger.bunkr.ru/file-r5fmwjdd.zip", - "filename" : "file-r5fmwjdd", - "id" : "r5fmwjdd", - "name" : "file", -}, - + { + "#url": "https://bunkr.sk/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + "#urls": "https://i-burger.bunkr.ru/test-%E3%83%86%E3%82%B9%E3%83%88-%22%26%3E-QjgneIQv.png?download=true", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + "album_id": "Lktg9Keq", + "album_name": 'test テスト "&>', + "album_size": "182 B", + "count": 1, + "extension": "png", + "file": "https://i-burger.bunkr.ru/test-%E3%83%86%E3%82%B9%E3%83%88-%22%26%3E-QjgneIQv.png?download=true", + "filename": 'test-テスト-"&>-QjgneIQv', + "id": "QjgneIQv", + "name": 'test-テスト-"&>', + "num": 1, + }, + { + "#url": "https://bunkr.is/a/iXTTc1o2", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + "#urls": ("https://i-milkshake.bunkr.ru/image-sZrQUeOx.jpg?download=true",), + "#sha1_content": "caf7c3d3439d94e83b3c24ddaf5a3a48aa057519", + "album_id": "iXTTc1o2", + "album_name": "test2", + "album_size": "534.6 KB", + "count": 1, + "filename": r"image-sZrQUeOx", + "id": r"sZrQUeOx", + "name": r"image", + "num": 1, + }, + { + "#url": "https://bunkr.cat/a/j1G29CnD", + "#comment": "cdn12 .ru TLD (#4147)", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + "#pattern": r"https://(i-)?meatballs.bunkr.ru/\w+", + "#count": 7, + }, + { + "#url": "https://bunkr.ph/a/Lktg9Keq", + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.ps/a/Lktg9Keq", + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.pk/a/Lktg9Keq", + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.ax/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkrrr.org/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.ci/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.cr/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.fi/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.si/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.ac/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.media/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.site/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.ws/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkrr.ru/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkrr.su/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.la/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.su/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.ru/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.is/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.to/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "bunkr:http://example.org/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.black/i/image-sZrQUeOx.jpg", + "#category": ("lolisafe", "bunkr", "media"), + "#class": bunkr.BunkrMediaExtractor, + "#urls": "https://i-milkshake.bunkr.ru/image-sZrQUeOx.jpg?download=true", + "#sha1_content": "caf7c3d3439d94e83b3c24ddaf5a3a48aa057519", + "count": 1, + "extension": "jpg", + "file": "https://i-milkshake.bunkr.ru/image-sZrQUeOx.jpg?download=true", + "filename": "image-sZrQUeOx", + "id": "sZrQUeOx", + "name": "image", + }, + { + "#url": "https://bunkrrr.org/d/dJuETSzKLrUps", + "#category": ("lolisafe", "bunkr", "media"), + "#class": bunkr.BunkrMediaExtractor, + "#urls": "https://burger.bunkr.ru/file-r5fmwjdd.zip", + "#sha1_content": "102ddd7894fe39b3843098fc51f972a0af938f45", + "count": 1, + "extension": "zip", + "file": "https://burger.bunkr.ru/file-r5fmwjdd.zip", + "filename": "file-r5fmwjdd", + "id": "r5fmwjdd", + "name": "file", + }, ) diff --git a/test/results/catbox.py b/test/results/catbox.py index b977a52ed..fad1c2abc 100644 --- a/test/results/catbox.py +++ b/test/results/catbox.py @@ -1,59 +1,49 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import catbox - __tests__ = ( -{ - "#url" : "https://catbox.moe/c/1igcbe", - "#category": ("", "catbox", "album"), - "#class" : catbox.CatboxAlbumExtractor, - "#pattern" : r"https://files\.catbox\.moe/\w+\.\w{3}$", - "#count" : 3, - "#sha1_url" : "35866a88c29462814f103bc22ec031eaeb380f8a", - "#sha1_content": "70ddb9de3872e2d17cc27e48e6bf395e5c8c0b32", - - "album_id" : "1igcbe", - "album_name" : "test", - "date" : "dt:2022-08-18 00:00:00", - "description": "album test &>", -}, - -{ - "#url" : "https://www.catbox.moe/c/cd90s1", - "#category": ("", "catbox", "album"), - "#class" : catbox.CatboxAlbumExtractor, -}, - -{ - "#url" : "https://catbox.moe/c/w7tm47#", - "#category": ("", "catbox", "album"), - "#class" : catbox.CatboxAlbumExtractor, -}, - -{ - "#url" : "https://files.catbox.moe/8ih3y7.png", - "#category": ("", "catbox", "file"), - "#class" : catbox.CatboxFileExtractor, - "#pattern" : r"^https://files\.catbox\.moe/8ih3y7\.png$", - "#count" : 1, - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", -}, - -{ - "#url" : "https://litter.catbox.moe/t8v3n9.png", - "#category": ("", "catbox", "file"), - "#class" : catbox.CatboxFileExtractor, -}, - -{ - "#url" : "https://de.catbox.moe/bjdmz1.jpg", - "#category": ("", "catbox", "file"), - "#class" : catbox.CatboxFileExtractor, -}, - + { + "#url": "https://catbox.moe/c/1igcbe", + "#category": ("", "catbox", "album"), + "#class": catbox.CatboxAlbumExtractor, + "#pattern": r"https://files\.catbox\.moe/\w+\.\w{3}$", + "#count": 3, + "#sha1_url": "35866a88c29462814f103bc22ec031eaeb380f8a", + "#sha1_content": "70ddb9de3872e2d17cc27e48e6bf395e5c8c0b32", + "album_id": "1igcbe", + "album_name": "test", + "date": "dt:2022-08-18 00:00:00", + "description": "album test &>", + }, + { + "#url": "https://www.catbox.moe/c/cd90s1", + "#category": ("", "catbox", "album"), + "#class": catbox.CatboxAlbumExtractor, + }, + { + "#url": "https://catbox.moe/c/w7tm47#", + "#category": ("", "catbox", "album"), + "#class": catbox.CatboxAlbumExtractor, + }, + { + "#url": "https://files.catbox.moe/8ih3y7.png", + "#category": ("", "catbox", "file"), + "#class": catbox.CatboxFileExtractor, + "#pattern": r"^https://files\.catbox\.moe/8ih3y7\.png$", + "#count": 1, + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, + { + "#url": "https://litter.catbox.moe/t8v3n9.png", + "#category": ("", "catbox", "file"), + "#class": catbox.CatboxFileExtractor, + }, + { + "#url": "https://de.catbox.moe/bjdmz1.jpg", + "#category": ("", "catbox", "file"), + "#class": catbox.CatboxFileExtractor, + }, ) diff --git a/test/results/cavemanon.py b/test/results/cavemanon.py index 3d065f43b..c41ca4e25 100644 --- a/test/results/cavemanon.py +++ b/test/results/cavemanon.py @@ -1,44 +1,37 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shimmie2 - __tests__ = ( -{ - "#url" : "https://booru.cavemanon.xyz/index.php?q=post/list/Amber/1", - "#category": ("shimmie2", "cavemanon", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, - "#pattern" : r"https://booru\.cavemanon\.xyz/index\.php\?q=image/\d+\.\w+", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://booru.cavemanon.xyz/post/list/Amber/1", - "#category": ("shimmie2", "cavemanon", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, -}, - -{ - "#url" : "https://booru.cavemanon.xyz/index.php?q=post/view/8335", - "#category": ("shimmie2", "cavemanon", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, - "#pattern" : r"https://booru\.cavemanon\.xyz/index\.php\?q=image/8335\.png", - "#sha1_content": "7158f7e4abbbf143bad5835eb93dbe4d68c1d4ab", - - "extension": "png", - "file_url" : "https://booru.cavemanon.xyz/index.php?q=image/8335.png", - "filename" : "8335", - "height" : 460, - "id" : 8335, - "md5" : "", - "size" : 0, - "tags" : "Color discord_emote Fang Food Pterodactyl transparent", - "width" : 459, -}, - + { + "#url": "https://booru.cavemanon.xyz/index.php?q=post/list/Amber/1", + "#category": ("shimmie2", "cavemanon", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + "#pattern": r"https://booru\.cavemanon\.xyz/index\.php\?q=image/\d+\.\w+", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://booru.cavemanon.xyz/post/list/Amber/1", + "#category": ("shimmie2", "cavemanon", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + }, + { + "#url": "https://booru.cavemanon.xyz/index.php?q=post/view/8335", + "#category": ("shimmie2", "cavemanon", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + "#pattern": r"https://booru\.cavemanon\.xyz/index\.php\?q=image/8335\.png", + "#sha1_content": "7158f7e4abbbf143bad5835eb93dbe4d68c1d4ab", + "extension": "png", + "file_url": "https://booru.cavemanon.xyz/index.php?q=image/8335.png", + "filename": "8335", + "height": 460, + "id": 8335, + "md5": "", + "size": 0, + "tags": "Color discord_emote Fang Food Pterodactyl transparent", + "width": 459, + }, ) diff --git a/test/results/chelseacrew.py b/test/results/chelseacrew.py index 7c12613fc..5add98dda 100644 --- a/test/results/chelseacrew.py +++ b/test/results/chelseacrew.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://chelseacrew.com/collections/flats", - "#category": ("shopify", "chelseacrew", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://chelseacrew.com/collections/flats/products/dora", - "#category": ("shopify", "chelseacrew", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://chelseacrew.com/collections/flats", + "#category": ("shopify", "chelseacrew", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://chelseacrew.com/collections/flats/products/dora", + "#category": ("shopify", "chelseacrew", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/cien.py b/test/results/cien.py index ef797216d..662414a31 100644 --- a/test/results/cien.py +++ b/test/results/cien.py @@ -1,92 +1,81 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import cien - __tests__ = ( -{ - "#url" : "https://ci-en.net/creator/7491/article/1194568", - "#category": ("", "cien", "article"), - "#class" : cien.CienArticleExtractor, - "#pattern" : r"https://media\.ci-en\.jp/private/attachment/creator/00007491/c0c212a93027c8863bdb40668071c1525a4567f94baca13c17989045e5a3d81d/video-web\.mp4\?px-time=.+", - - "author": { - "@type" : "Person", - "image" : "https://media.ci-en.jp/public/icon/creator/00007491/9601a2a224245156335aaa839fa408d52c32c87dae5787fc03f455b7fd1d3488/image-200-c.jpg", - "name" : "やかろ", - "url" : "https://ci-en.net/creator/7491", - "sameAs": [ - "https://pokapoka0802.wixsite.com/tunousaginoie82", - "https://www.freem.ne.jp/brand/6001", - "https://store.steampowered.com/search/?developer=%E3%83%84%E3%83%8E%E3%82%A6%E3%82%B5%E3%82%AE%E3%81%AE%E5%AE%B6", - "https://plicy.net/User/87381", - "https://twitter.com/pokapoka0802", - ], + { + "#url": "https://ci-en.net/creator/7491/article/1194568", + "#category": ("", "cien", "article"), + "#class": cien.CienArticleExtractor, + "#pattern": r"https://media\.ci-en\.jp/private/attachment/creator/00007491/c0c212a93027c8863bdb40668071c1525a4567f94baca13c17989045e5a3d81d/video-web\.mp4\?px-time=.+", + "author": { + "@type": "Person", + "image": "https://media.ci-en.jp/public/icon/creator/00007491/9601a2a224245156335aaa839fa408d52c32c87dae5787fc03f455b7fd1d3488/image-200-c.jpg", + "name": "やかろ", + "url": "https://ci-en.net/creator/7491", + "sameAs": [ + "https://pokapoka0802.wixsite.com/tunousaginoie82", + "https://www.freem.ne.jp/brand/6001", + "https://store.steampowered.com/search/?developer=%E3%83%84%E3%83%8E%E3%82%A6%E3%82%B5%E3%82%AE%E3%81%AE%E5%AE%B6", + "https://plicy.net/User/87381", + "https://twitter.com/pokapoka0802", + ], + }, + "articleBody": str, + "count": 1, + "date": "dt:2024-07-21 15:36:00", + "dateModified": "2024-07-22T03:28:40+09:00", + "datePublished": "2024-07-22T00:36:00+09:00", + "description": "お知らせ 今回は雨のピリオードの解説をしたいと思うのですが、その前にいくつかお知らせがあります。 電話を使って謎を解いていくフリーゲーム 電話を通して、様々なキャラクターを会話をしていく、ノベルゲーム……", + "extension": "mp4", + "filename": "無題の動画 (1)", + "headline": "角兎図書館「雨のピリオード」No,16", + "image": "https://media.ci-en.jp/public/article_cover/creator/00007491/cb4062e8d885ab93e0d0fb3133265a7ad1056c906fd4ab81da509220620901e1/image-1280-c.jpg", + "keywords": "お知らせ,角兎図書館", + "mainEntityOfPage": "https://ci-en.net/creator/7491/article/1194568", + "name": "角兎図書館「雨のピリオード」No,16", + "num": 1, + "post_id": 1194568, + "type": "video", + "url": str, + }, + { + "#url": "https://ci-en.dlsite.com/creator/25509/article/1172460", + "#category": ("", "cien", "article"), + "#class": cien.CienArticleExtractor, + "#options": {"files": "download"}, + "#pattern": r"https://media\.ci-en\.jp/private/attachment/creator/00025509/7fd3c039d2277ba9541e82592aca6f6751f6c268404038ccbf1112bcf2f93357/upload/.+\.zip\?px-time=.+", + "filename": "VP 1.05.4 Tim-v9 ENG rec v3", + "extension": "zip", + "type": "download", + }, + { + "#url": "https://ci-en.net/creator/11962", + "#category": ("", "cien", "creator"), + "#class": cien.CienCreatorExtractor, + "#pattern": cien.CienArticleExtractor.pattern, + "#count": "> 25", + }, + { + "#url": "https://ci-en.net/mypage/recent", + "#category": ("", "cien", "recent"), + "#class": cien.CienRecentExtractor, + "#auth": True, + }, + { + "#url": "https://ci-en.net/mypage/subscription/following", + "#category": ("", "cien", "following"), + "#class": cien.CienFollowingExtractor, + "#pattern": cien.CienCreatorExtractor.pattern, + "#count": "> 3", + "#auth": True, + }, + { + "#url": "https://ci-en.net/mypage/subscription", + "#category": ("", "cien", "following"), + "#class": cien.CienFollowingExtractor, + "#auth": True, }, - "articleBody": str, - "count" : 1, - "date" : "dt:2024-07-21 15:36:00", - "dateModified" : "2024-07-22T03:28:40+09:00", - "datePublished": "2024-07-22T00:36:00+09:00", - "description": "お知らせ 今回は雨のピリオードの解説をしたいと思うのですが、その前にいくつかお知らせがあります。 電話を使って謎を解いていくフリーゲーム 電話を通して、様々なキャラクターを会話をしていく、ノベルゲーム……", - "extension" : "mp4", - "filename" : "無題の動画 (1)", - "headline" : "角兎図書館「雨のピリオード」No,16", - "image" : "https://media.ci-en.jp/public/article_cover/creator/00007491/cb4062e8d885ab93e0d0fb3133265a7ad1056c906fd4ab81da509220620901e1/image-1280-c.jpg", - "keywords" : "お知らせ,角兎図書館", - "mainEntityOfPage": "https://ci-en.net/creator/7491/article/1194568", - "name" : "角兎図書館「雨のピリオード」No,16", - "num" : 1, - "post_id" : 1194568, - "type" : "video", - "url" : str, -}, - -{ - "#url" : "https://ci-en.dlsite.com/creator/25509/article/1172460", - "#category": ("", "cien", "article"), - "#class" : cien.CienArticleExtractor, - "#options" : {"files": "download"}, - "#pattern" : r"https://media\.ci-en\.jp/private/attachment/creator/00025509/7fd3c039d2277ba9541e82592aca6f6751f6c268404038ccbf1112bcf2f93357/upload/.+\.zip\?px-time=.+", - - "filename" : "VP 1.05.4 Tim-v9 ENG rec v3", - "extension": "zip", - "type" : "download", -}, - -{ - "#url" : "https://ci-en.net/creator/11962", - "#category": ("", "cien", "creator"), - "#class" : cien.CienCreatorExtractor, - "#pattern" : cien.CienArticleExtractor.pattern, - "#count" : "> 25", -}, - -{ - "#url" : "https://ci-en.net/mypage/recent", - "#category": ("", "cien", "recent"), - "#class" : cien.CienRecentExtractor, - "#auth" : True, -}, - -{ - "#url" : "https://ci-en.net/mypage/subscription/following", - "#category": ("", "cien", "following"), - "#class" : cien.CienFollowingExtractor, - "#pattern" : cien.CienCreatorExtractor.pattern, - "#count" : "> 3", - "#auth" : True, -}, - -{ - "#url" : "https://ci-en.net/mypage/subscription", - "#category": ("", "cien", "following"), - "#class" : cien.CienFollowingExtractor, - "#auth" : True, -}, - ) diff --git a/test/results/civitai.py b/test/results/civitai.py index 249767497..0de1a9818 100644 --- a/test/results/civitai.py +++ b/test/results/civitai.py @@ -1,244 +1,220 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import civitai from gallery_dl import exception - +from gallery_dl.extractor import civitai __tests__ = ( -{ - "#url" : "https://civitai.com/models/703211/maid-classic", - "#class": civitai.CivitaiModelExtractor, - "#urls" : [ - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/5c4efa68-bb58-47c5-a716-98cd0f51f047/original=true/00013-4238863814.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/original=true/00014-3150861441.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/2dd1dc69-45a6-4beb-b36b-2e2bc65e3cda/original=true/00015-2885514572.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/00004-822988489.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/00005-1059918744.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/00006-3467286319.png", - ], - - "model" : { - "description": "

          The strength of Lora is recommended to be around 1.0.

          ", - "id" : 703211, - "minor" : False, - "name" : "メイド　クラシック/maid classic", - "nsfwLevel" : 1, - "type" : "LORA", + { + "#url": "https://civitai.com/models/703211/maid-classic", + "#class": civitai.CivitaiModelExtractor, + "#urls": [ + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/5c4efa68-bb58-47c5-a716-98cd0f51f047/original=true/00013-4238863814.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/original=true/00014-3150861441.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/2dd1dc69-45a6-4beb-b36b-2e2bc65e3cda/original=true/00015-2885514572.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/00004-822988489.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/00005-1059918744.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/00006-3467286319.png", + ], + "model": { + "description": "

          The strength of Lora is recommended to be around 1.0.

          ", + "id": 703211, + "minor": False, + "name": "メイド　クラシック/maid classic", + "nsfwLevel": 1, + "type": "LORA", + }, + "user": {"image": None, "username": "bolero537"}, + "file": { + "uuid": str, + }, + "version": dict, + "num": range(1, 3), }, - "user" : { - "image" : None, - "username": "bolero537" + { + "#url": "https://civitai.com/models/703211?modelVersionId=786644", + "#comment": "model version ID", + "#class": civitai.CivitaiModelExtractor, + "#urls": [ + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/00004-822988489.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/00005-1059918744.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/00006-3467286319.png", + ], + "version": { + "baseModel": "Pony", + "createdAt": "2024-08-30T15:28:47.661Z", + "date": "dt:2024-08-30 15:28:47", + "files": list, + "id": 786644, + "name": "v1.0 pony", + }, + "user": {"image": None, "username": "bolero537"}, + "file": { + "id": {26887862, 26887856, 26887852}, + "uuid": { + "52b6efa7-801c-4901-90b4-fa3964d23480", + "c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c", + "68568d22-c4f3-45cb-ac32-82f1cedf968f", + }, + }, + "model": { + "id": 703211, + }, + "num": range(1, 3), }, - "file" : { - "uuid": str, + { + "#url": "https://civitai.com/images/26962948", + "#class": civitai.CivitaiImageExtractor, + "#options": {"quality": "w", "metadata": True}, + "#urls": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/w/00014-3150861441.png", + "#sha1_content": "a9a9d08f5fcdbc1e1eec7f203717f9df97b7a671", + "createdAt": "2024-08-31T01:11:47.021Z", + "date": "dt:2024-08-31 01:11:47", + "extension": "jpg", + "filename": "00014-3150861441", + "hash": "ULN0-w?b4nRjxGM{-;t7M_t7NGae~qRjMyt7", + "height": 1536, + "id": 26962948, + "nsfwLevel": 1, + "postId": 6030721, + "stats": dict, + "url": "69bf3279-df2c-4ec8-b795-479e9cd3db1b", + "uuid": "69bf3279-df2c-4ec8-b795-479e9cd3db1b", + "width": 1152, + "user": { + "username": "bolero537", + }, + "generation": { + "canRemix": True, + "external": None, + "generationProcess": "img2img", + "resources": list, + "techniques": [], + "tools": [], + "meta": { + "Denoising strength": "0.4", + "Model": "boleromix_XL_V1.3", + "Model hash": "afaf521da2", + "Size": "1152x1536", + "Tiled Diffusion scale factor": "1.5", + "Tiled Diffusion upscaler": "R-ESRGAN 4x+ Anime6B", + "VAE": "sdxl_vae.safetensors", + "Version": "v1.7.0", + "cfgScale": 7, + "negativePrompt": "negativeXL_D,(worst quality,extra legs,extra arms,extra ears,bad fingers,extra fingers,bad anatomy, missing fingers, lowres,username, artist name, text,pubic hair,bar censor,censored,multipul angle,split view,realistic,3D:1)", + "prompt": "masterpiece,ultra-detailed,best quality,8K,illustration,cute face,clean skin ,shiny hair,girl,ultra-detailed-eyes,simple background, maid, maid apron, maid headdress, long sleeves,tray,tea,cup,skirt lift", + "resources": list, + "sampler": "DPM++ 2M Karras", + "seed": 3150861441, + "steps": 20, + "hashes": { + "lora:add-detail-xl": "9c783c8ce46c", + "lora:classic maid_XL_V1.0": "e8f6e4297112", + "model": "afaf521da2", + "vae": "735e4c3a44", + }, + "TI hashes": { + "negativeXL_D": "fff5d51ab655", + }, + }, + }, }, - "version": dict, - "num" : range(1, 3), -}, - -{ - "#url" : "https://civitai.com/models/703211?modelVersionId=786644", - "#comment": "model version ID", - "#class" : civitai.CivitaiModelExtractor, - "#urls" : [ - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/00004-822988489.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/00005-1059918744.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/00006-3467286319.png", - ], - - "version": { - "baseModel" : "Pony", - "createdAt" : "2024-08-30T15:28:47.661Z", - "date" : "dt:2024-08-30 15:28:47", - "files" : list, - "id" : 786644, - "name" : "v1.0 pony", + { + "#url": "https://civitai.com/posts/6877551", + "#class": civitai.CivitaiPostExtractor, + "#options": {"metadata": "generation"}, + "#urls": [ + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6220fa0f-9037-4b1d-bfbd-a740a06eeb7c/original=true/30748752.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/cd1edb7f-7b50-4da5-bf23-d38f24d8aef0/original=true/30748747.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/cfd5b231-accd-49bd-8bde-370880f63aa6/original=true/30748733.png", + ], + "post": { + "id": 6877551, + "date": "dt:2024-09-22 12:54:15", + }, + "file": { + "id": {30748752, 30748747, 30748733}, + "uuid": { + "6220fa0f-9037-4b1d-bfbd-a740a06eeb7c", + "cd1edb7f-7b50-4da5-bf23-d38f24d8aef0", + "cfd5b231-accd-49bd-8bde-370880f63aa6", + }, + "generation": { + "resources": list, + "techniques": [], + "tools": [], + "meta": { + "prompt": str, + "negativePrompt": str, + }, + }, + }, }, - "user" : { - "image" : None, - "username": "bolero537" + { + "#url": "https://civitai.com/tag/mecha", + "#class": civitai.CivitaiTagExtractor, }, - "file" : { - "id" : {26887862, 26887856, 26887852}, - "uuid": {"52b6efa7-801c-4901-90b4-fa3964d23480", - "c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c", - "68568d22-c4f3-45cb-ac32-82f1cedf968f"}, + { + "#url": "https://civitai.com/images?tags=482", + "#class": civitai.CivitaiImagesExtractor, }, - "model" : { - "id": 703211, + { + "#url": "https://civitai.com/images?modelVersionId=786644", + "#class": civitai.CivitaiImagesExtractor, }, - "num" : range(1, 3), -}, - -{ - "#url" : "https://civitai.com/images/26962948", - "#class": civitai.CivitaiImageExtractor, - "#options" : {"quality": "w", "metadata": True}, - "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/w/00014-3150861441.png", - "#sha1_content": "a9a9d08f5fcdbc1e1eec7f203717f9df97b7a671", - - "createdAt": "2024-08-31T01:11:47.021Z", - "date" : "dt:2024-08-31 01:11:47", - "extension": "jpg", - "filename" : "00014-3150861441", - "hash" : "ULN0-w?b4nRjxGM{-;t7M_t7NGae~qRjMyt7", - "height" : 1536, - "id" : 26962948, - "nsfwLevel": 1, - "postId" : 6030721, - "stats" : dict, - "url" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b", - "uuid" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b", - "width" : 1152, - "user" : { - "username": "bolero537", + { + "#url": "https://civitai.com/models", + "#class": civitai.CivitaiModelsExtractor, }, - "generation": { - "canRemix" : True, - "external" : None, - "generationProcess": "img2img", - "resources" : list, - "techniques": [], - "tools" : [], - "meta" : { - "Denoising strength": "0.4", - "Model" : "boleromix_XL_V1.3", - "Model hash" : "afaf521da2", - "Size" : "1152x1536", - "Tiled Diffusion scale factor": "1.5", - "Tiled Diffusion upscaler": "R-ESRGAN 4x+ Anime6B", - "VAE" : "sdxl_vae.safetensors", - "Version" : "v1.7.0", - "cfgScale" : 7, - "negativePrompt": "negativeXL_D,(worst quality,extra legs,extra arms,extra ears,bad fingers,extra fingers,bad anatomy, missing fingers, lowres,username, artist name, text,pubic hair,bar censor,censored,multipul angle,split view,realistic,3D:1)", - "prompt" : "masterpiece,ultra-detailed,best quality,8K,illustration,cute face,clean skin ,shiny hair,girl,ultra-detailed-eyes,simple background, maid, maid apron, maid headdress, long sleeves,tray,tea,cup,skirt lift", - "resources" : list, - "sampler" : "DPM++ 2M Karras", - "seed" : 3150861441, - "steps" : 20, - "hashes" : { - "lora:add-detail-xl": "9c783c8ce46c", - "lora:classic maid_XL_V1.0": "e8f6e4297112", - "model": "afaf521da2", - "vae": "735e4c3a44", - }, - "TI hashes" : { - "negativeXL_D": "fff5d51ab655", - }, - }, + { + "#url": "https://civitai.com/search/models?sortBy=models_v9&query=mecha", + "#class": civitai.CivitaiSearchExtractor, }, -}, - -{ - "#url" : "https://civitai.com/posts/6877551", - "#class" : civitai.CivitaiPostExtractor, - "#options": {"metadata": "generation"}, - "#urls" : [ - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6220fa0f-9037-4b1d-bfbd-a740a06eeb7c/original=true/30748752.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/cd1edb7f-7b50-4da5-bf23-d38f24d8aef0/original=true/30748747.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/cfd5b231-accd-49bd-8bde-370880f63aa6/original=true/30748733.png", - ], - - "post": { - "id" : 6877551, - "date": "dt:2024-09-22 12:54:15", + { + "#url": "https://civitai.com/user/waomodder", + "#class": civitai.CivitaiUserExtractor, + "#urls": [ + "https://civitai.com/user/waomodder/models", + "https://civitai.com/user/waomodder/posts", + ], }, - "file": { - "id" : {30748752, 30748747, 30748733}, - "uuid": {"6220fa0f-9037-4b1d-bfbd-a740a06eeb7c", - "cd1edb7f-7b50-4da5-bf23-d38f24d8aef0", - "cfd5b231-accd-49bd-8bde-370880f63aa6"}, - "generation": { - "resources" : list, - "techniques": [], - "tools" : [], - "meta" : { - "prompt" : str, - "negativePrompt": str, - }, - }, + { + "#url": "https://civitai.com/user/waomodder/models", + "#class": civitai.CivitaiUserModelsExtractor, + "#pattern": civitai.CivitaiModelExtractor.pattern, + "#count": ">= 8", + }, + { + "#url": "https://civitai.com/user/waomodder/posts", + "#class": civitai.CivitaiUserPostsExtractor, + "#pattern": r"https://image\.civitai\.com/xG1nkqKTMzGDvpLrqFT7WA/[0-9a-f-]+/original=true/\S+\.(jpe?g|png)", + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://civitai.com/user/waomodder/images", + "#class": civitai.CivitaiUserImagesExtractor, + "#pattern": r"https://image\.civitai\.com/xG1nkqKTMzGDvpLrqFT7WA/[0-9a-f-]+/original=true/\S+\.png", + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://civitai.com/user/USER/images?section=reactions", + "#category": ("", "civitai", "reactions"), + "#class": civitai.CivitaiUserImagesExtractor, + "#auth": True, + "#urls": ( + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/dd29c97a-1e95-4186-8df5-632736cbae79/original=true/00012-2489035818.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/5c4efa68-bb58-47c5-a716-98cd0f51f047/original=true/00013-4238863814.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/original=true/00014-3150861441.png", + ), + }, + { + "#url": "https://civitai.com/user/USER/images?section=reactions", + "#category": ("", "civitai", "reactions"), + "#class": civitai.CivitaiUserImagesExtractor, + "#auth": False, + "#exception": exception.AuthorizationError, }, -}, - -{ - "#url" : "https://civitai.com/tag/mecha", - "#class": civitai.CivitaiTagExtractor, -}, - -{ - "#url" : "https://civitai.com/images?tags=482", - "#class": civitai.CivitaiImagesExtractor, -}, - -{ - "#url" : "https://civitai.com/images?modelVersionId=786644", - "#class": civitai.CivitaiImagesExtractor, -}, - -{ - "#url" : "https://civitai.com/models", - "#class": civitai.CivitaiModelsExtractor, -}, - -{ - "#url" : "https://civitai.com/search/models?sortBy=models_v9&query=mecha", - "#class": civitai.CivitaiSearchExtractor, -}, - -{ - "#url" : "https://civitai.com/user/waomodder", - "#class": civitai.CivitaiUserExtractor, - "#urls" : [ - "https://civitai.com/user/waomodder/models", - "https://civitai.com/user/waomodder/posts", - ], -}, - -{ - "#url" : "https://civitai.com/user/waomodder/models", - "#class": civitai.CivitaiUserModelsExtractor, - "#pattern": civitai.CivitaiModelExtractor.pattern, - "#count" : ">= 8", -}, - -{ - "#url" : "https://civitai.com/user/waomodder/posts", - "#class": civitai.CivitaiUserPostsExtractor, - "#pattern": r"https://image\.civitai\.com/xG1nkqKTMzGDvpLrqFT7WA/[0-9a-f-]+/original=true/\S+\.(jpe?g|png)", - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://civitai.com/user/waomodder/images", - "#class": civitai.CivitaiUserImagesExtractor, - "#pattern": r"https://image\.civitai\.com/xG1nkqKTMzGDvpLrqFT7WA/[0-9a-f-]+/original=true/\S+\.png", - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://civitai.com/user/USER/images?section=reactions", - "#category": ("", "civitai", "reactions"), - "#class" : civitai.CivitaiUserImagesExtractor, - "#auth" : True, - "#urls" : ( - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/dd29c97a-1e95-4186-8df5-632736cbae79/original=true/00012-2489035818.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/5c4efa68-bb58-47c5-a716-98cd0f51f047/original=true/00013-4238863814.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/original=true/00014-3150861441.png", - ), -}, - -{ - "#url" : "https://civitai.com/user/USER/images?section=reactions", - "#category": ("", "civitai", "reactions"), - "#class" : civitai.CivitaiUserImagesExtractor, - "#auth" : False, - "#exception": exception.AuthorizationError, -}, - ) diff --git a/test/results/cohost.py b/test/results/cohost.py index 1184f21c0..42c05c654 100644 --- a/test/results/cohost.py +++ b/test/results/cohost.py @@ -1,26 +1,21 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import cohost - __tests__ = ( -{ - "#url" : "https://cohost.org/infinitebrians", - "#category": ("", "cohost", "user"), - "#class" : cohost.CohostUserExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://cohost.org/infinitebrians/post/4957017-thank-you-akira-tori", - "#category": ("", "cohost", "post"), - "#class" : cohost.CohostPostExtractor, - "#urls" : "https://staging.cohostcdn.org/attachment/58f9aa96-d2b2-4838-b81c-9aa8bac0bea0/march%204%202024.png", -}, - + { + "#url": "https://cohost.org/infinitebrians", + "#category": ("", "cohost", "user"), + "#class": cohost.CohostUserExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://cohost.org/infinitebrians/post/4957017-thank-you-akira-tori", + "#category": ("", "cohost", "post"), + "#class": cohost.CohostPostExtractor, + "#urls": "https://staging.cohostcdn.org/attachment/58f9aa96-d2b2-4838-b81c-9aa8bac0bea0/march%204%202024.png", + }, ) diff --git a/test/results/comicvine.py b/test/results/comicvine.py index b41c24e88..4db4bd56f 100644 --- a/test/results/comicvine.py +++ b/test/results/comicvine.py @@ -1,27 +1,22 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import comicvine - __tests__ = ( -{ - "#url" : "https://comicvine.gamespot.com/jock/4040-5653/images/", - "#category": ("", "comicvine", "tag"), - "#class" : comicvine.ComicvineTagExtractor, - "#pattern" : r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+\.(jpe?g|png)", - "#count" : ">= 140", -}, - -{ - "#url" : "https://comicvine.gamespot.com/batman/4005-1699/images/?tag=Fan%20Art%20%26%20Cosplay", - "#category": ("", "comicvine", "tag"), - "#class" : comicvine.ComicvineTagExtractor, - "#pattern" : r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+", - "#count" : ">= 400", -}, - + { + "#url": "https://comicvine.gamespot.com/jock/4040-5653/images/", + "#category": ("", "comicvine", "tag"), + "#class": comicvine.ComicvineTagExtractor, + "#pattern": r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+\.(jpe?g|png)", + "#count": ">= 140", + }, + { + "#url": "https://comicvine.gamespot.com/batman/4005-1699/images/?tag=Fan%20Art%20%26%20Cosplay", + "#category": ("", "comicvine", "tag"), + "#class": comicvine.ComicvineTagExtractor, + "#pattern": r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+", + "#count": ">= 400", + }, ) diff --git a/test/results/coomerparty.py b/test/results/coomerparty.py index 87c932e83..ef93be58b 100644 --- a/test/results/coomerparty.py +++ b/test/results/coomerparty.py @@ -1,26 +1,21 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import kemonoparty - __tests__ = ( -{ - "#url" : "https://coomer.su/onlyfans/user/alinity/post/125962203", - "#comment" : "coomer (#2100)", - "#category": ("", "coomerparty", "onlyfans"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#urls" : "https://coomer.su/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8.jpg", -}, - -{ - "#url" : "https://coomer.party/onlyfans/user/alinity/post/125962203", - "#category": ("", "coomerparty", "onlyfans"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#urls" : "https://coomer.party/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8.jpg", -}, - + { + "#url": "https://coomer.su/onlyfans/user/alinity/post/125962203", + "#comment": "coomer (#2100)", + "#category": ("", "coomerparty", "onlyfans"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#urls": "https://coomer.su/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8.jpg", + }, + { + "#url": "https://coomer.party/onlyfans/user/alinity/post/125962203", + "#category": ("", "coomerparty", "onlyfans"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#urls": "https://coomer.party/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8.jpg", + }, ) diff --git a/test/results/cyberdrop.py b/test/results/cyberdrop.py index 7f6a8eb93..9d1dbf2a3 100644 --- a/test/results/cyberdrop.py +++ b/test/results/cyberdrop.py @@ -1,61 +1,54 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import cyberdrop - __tests__ = ( -{ - "#url" : "https://cyberdrop.me/a/8uE0wQiK", - "#category": ("lolisafe", "cyberdrop", "album"), - "#class" : cyberdrop.CyberdropAlbumExtractor, - "#pattern" : r"https://\w+\.cyberdrop\.ch/api/file/d/yyK9y8xpQK5dP\?token=ey.+", - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", - - "album_id" : "8uE0wQiK", - "album_name" : "test テスト \"&>", - "album_size" : 182, - "count" : 1, - "date" : "dt:2023-11-26 00:00:00", - "description" : "test テスト \"&>", - "extension" : "png", - "filename" : "test-テスト--22->-rwU3x9LU", - "id" : "rwU3x9LU", - "name" : "test-テスト--22->", - "num" : 1, - "size" : 182, - "slug" : "yyK9y8xpQK5dP", - "thumbnail_url": str, - "type" : "image/png", - "url" : str, -}, - -{ - "#url" : "https://cyberdrop.me/a/HriMgbuf", - "#category": ("lolisafe", "cyberdrop", "album"), - "#class" : cyberdrop.CyberdropAlbumExtractor, - "#pattern" : r"https://\w+\.cyberdrop\.ch/api/file/d/\w+\?token=ey.+", - "#count" : 3, - - "album_id" : "HriMgbuf", - "album_name" : "animations", - "album_size" : 1090519, - "count" : 3, - "date" : "dt:2023-11-26 00:00:00", - "description" : "animated stuff", - "extension" : r"re:gif|webm", - "filename" : r"re:danbooru_\d+_\w+-\w+", - "id" : str, - "name" : r"re:danbooru_\d+_\w+", - "num" : range(1, 3), - "size" : int, - "slug" : str, - "thumbnail_url": str, - "type" : r"re:image/gif|video/webm", - "url" : str, -}, - + { + "#url": "https://cyberdrop.me/a/8uE0wQiK", + "#category": ("lolisafe", "cyberdrop", "album"), + "#class": cyberdrop.CyberdropAlbumExtractor, + "#pattern": r"https://\w+\.cyberdrop\.ch/api/file/d/yyK9y8xpQK5dP\?token=ey.+", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + "album_id": "8uE0wQiK", + "album_name": 'test テスト "&>', + "album_size": 182, + "count": 1, + "date": "dt:2023-11-26 00:00:00", + "description": 'test テスト "&>', + "extension": "png", + "filename": "test-テスト--22->-rwU3x9LU", + "id": "rwU3x9LU", + "name": "test-テスト--22->", + "num": 1, + "size": 182, + "slug": "yyK9y8xpQK5dP", + "thumbnail_url": str, + "type": "image/png", + "url": str, + }, + { + "#url": "https://cyberdrop.me/a/HriMgbuf", + "#category": ("lolisafe", "cyberdrop", "album"), + "#class": cyberdrop.CyberdropAlbumExtractor, + "#pattern": r"https://\w+\.cyberdrop\.ch/api/file/d/\w+\?token=ey.+", + "#count": 3, + "album_id": "HriMgbuf", + "album_name": "animations", + "album_size": 1090519, + "count": 3, + "date": "dt:2023-11-26 00:00:00", + "description": "animated stuff", + "extension": r"re:gif|webm", + "filename": r"re:danbooru_\d+_\w+-\w+", + "id": str, + "name": r"re:danbooru_\d+_\w+", + "num": range(1, 3), + "size": int, + "slug": str, + "thumbnail_url": str, + "type": r"re:image/gif|video/webm", + "url": str, + }, ) diff --git a/test/results/danbooru.py b/test/results/danbooru.py index bad664136..001a93898 100644 --- a/test/results/danbooru.py +++ b/test/results/danbooru.py @@ -1,234 +1,216 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import danbooru - __tests__ = ( -{ - "#url" : "https://danbooru.donmai.us/posts?tags=bonocho", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, - "#sha1_content": "b196fb9f1668109d7774a0a82efea3ffdda07746", -}, - -{ - "#url" : "https://danbooru.donmai.us/posts?tags=mushishi", - "#comment" : "test page transitions", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, - "#count" : ">= 300", -}, - -{ - "#url" : "https://danbooru.donmai.us/posts?tags=pixiv_id%3A1476533", - "#comment" : "'external' option (#1747)", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, - "#options" : {"external": True}, - "#pattern" : r"https://i\.pximg\.net/img-original/img/2008/08/28/02/35/48/1476533_p0\.jpg", -}, - -{ - "#url" : "https://hijiribe.donmai.us/posts?tags=bonocho", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, -}, - -{ - "#url" : "https://sonohara.donmai.us/posts?tags=bonocho", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, -}, - -{ - "#url" : "https://safebooru.donmai.us/posts?tags=bonocho", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, -}, - -{ - "#url" : "https://donmai.moe/posts?tags=bonocho", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, -}, - -{ - "#url" : "https://danbooru.donmai.us/pools/7659", - "#category": ("Danbooru", "danbooru", "pool"), - "#class" : danbooru.DanbooruPoolExtractor, - "#sha1_content": "b16bab12bea5f7ea9e0a836bf8045f280e113d99", -}, - -{ - "#url" : "https://danbooru.donmai.us/pool/show/7659", - "#category": ("Danbooru", "danbooru", "pool"), - "#class" : danbooru.DanbooruPoolExtractor, -}, - -{ - "#url" : "https://danbooru.donmai.us/posts/294929", - "#category": ("Danbooru", "danbooru", "post"), - "#class" : danbooru.DanbooruPostExtractor, - "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", - - "approver_id": None, - "bit_flags": 0, - "created_at": "2008-08-12T00:46:05.385-04:00", - "date": "dt:2008-08-12 04:46:05", - "down_score": 0, - "extension": "jpg", - "fav_count": 9, - "file_ext": "jpg", - "file_size": 358232, - "file_url": "https://cdn.donmai.us/original/ac/8e/ac8e3b92ea328ce9cf7211e69c905bf9.jpg", - "filename": "ac8e3b92ea328ce9cf7211e69c905bf9", - "has_active_children": False, - "has_children": False, - "has_large": True, - "has_visible_children": False, - "id": 294929, - "image_height": 687, - "image_width": 895, - "is_banned": False, - "is_deleted": False, - "is_flagged": False, - "is_pending": False, - "large_file_url": "https://cdn.donmai.us/sample/ac/8e/sample-ac8e3b92ea328ce9cf7211e69c905bf9.jpg", - "last_comment_bumped_at": None, - "last_commented_at": None, - "last_noted_at": None, - "md5": "ac8e3b92ea328ce9cf7211e69c905bf9", - "media_asset": dict, - "parent_id": None, - "pixiv_id": 1129835, - "preview_file_url": "https://cdn.donmai.us/180x180/ac/8e/ac8e3b92ea328ce9cf7211e69c905bf9.jpg", - "rating": "s", - "score": 1, - "source": "https://i.pximg.net/img-original/img/2008/07/09/16/10/23/1129835_p0.jpg", - "subcategory": "post", - "tag_count": 32, - "tag_count_artist": 1, - "tag_count_character": 3, - "tag_count_copyright": 3, - "tag_count_general": 23, - "tag_count_meta": 2, - "tag_string": "2boys bat_(animal) batman batman_(series) black_bodysuit bodysuit bonocho brown_eyes closed_mouth collared_shirt commentary_request copyright_name dc_comics expressionless facepaint glasgow_smile heath_ledger joker_(dc) male_focus multiple_boys outline outstretched_arm parted_lips photoshop_(medium) pink_shirt shirt sketch smile the_dark_knight upper_body white_outline wing_collar", - "tag_string_artist": "bonocho", - "tag_string_character": "batman heath_ledger joker_(dc)", - "tag_string_copyright": "batman_(series) dc_comics the_dark_knight", - "tag_string_general": "2boys bat_(animal) black_bodysuit bodysuit brown_eyes closed_mouth collared_shirt copyright_name expressionless facepaint glasgow_smile male_focus multiple_boys outline outstretched_arm parted_lips pink_shirt shirt sketch smile upper_body white_outline wing_collar", - "tag_string_meta": "commentary_request photoshop_(medium)", - "tags": [ - "2boys", - "bat_(animal)", - "batman", - "batman_(series)", - "black_bodysuit", - "bodysuit", - "bonocho", - "brown_eyes", - "closed_mouth", - "collared_shirt", - "commentary_request", - "copyright_name", - "dc_comics", - "expressionless", - "facepaint", - "glasgow_smile", - "heath_ledger", - "joker_(dc)", - "male_focus", - "multiple_boys", - "outline", - "outstretched_arm", - "parted_lips", - "photoshop_(medium)", - "pink_shirt", - "shirt", - "sketch", - "smile", - "the_dark_knight", - "upper_body", - "white_outline", - "wing_collar", - ], - "tags_artist": [ - "bonocho", - ], - "tags_character": [ - "batman", - "heath_ledger", - "joker_(dc)", - ], - "tags_copyright": [ - "batman_(series)", - "dc_comics", - "the_dark_knight", - ], - "tags_general": [ - "2boys", - "bat_(animal)", - "black_bodysuit", - "bodysuit", - "brown_eyes", - "closed_mouth", - "collared_shirt", - "copyright_name", - "expressionless", - "facepaint", - "glasgow_smile", - "male_focus", - "multiple_boys", - "outline", - "outstretched_arm", - "parted_lips", - "pink_shirt", - "shirt", - "sketch", - "smile", - "upper_body", - "white_outline", - "wing_collar", - ], - "tags_meta": [ - "commentary_request", - "photoshop_(medium)", - ], - "up_score": range(1, 5), - "updated_at": "2022-07-11T23:42:31.881-04:00", - "uploader_id": 67005, -}, - -{ - "#url" : "https://danbooru.donmai.us/posts/3613024", - "#category": ("Danbooru", "danbooru", "post"), - "#class" : danbooru.DanbooruPostExtractor, - "#options" : {"ugoira": True}, - "#pattern" : r"https?://.+\.zip$", -}, - -{ - "#url" : "https://danbooru.donmai.us/post/show/294929", - "#category": ("Danbooru", "danbooru", "post"), - "#class" : danbooru.DanbooruPostExtractor, -}, - -{ - "#url" : "https://danbooru.donmai.us/explore/posts/popular", - "#category": ("Danbooru", "danbooru", "popular"), - "#class" : danbooru.DanbooruPopularExtractor, -}, - -{ - "#url" : "https://danbooru.donmai.us/explore/posts/popular?date=2013-06-06&scale=week", - "#category": ("Danbooru", "danbooru", "popular"), - "#class" : danbooru.DanbooruPopularExtractor, - "#range" : "1-120", - "#count" : 120, -}, - + { + "#url": "https://danbooru.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + "#sha1_content": "b196fb9f1668109d7774a0a82efea3ffdda07746", + }, + { + "#url": "https://danbooru.donmai.us/posts?tags=mushishi", + "#comment": "test page transitions", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + "#count": ">= 300", + }, + { + "#url": "https://danbooru.donmai.us/posts?tags=pixiv_id%3A1476533", + "#comment": "'external' option (#1747)", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + "#options": {"external": True}, + "#pattern": r"https://i\.pximg\.net/img-original/img/2008/08/28/02/35/48/1476533_p0\.jpg", + }, + { + "#url": "https://hijiribe.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + }, + { + "#url": "https://sonohara.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + }, + { + "#url": "https://safebooru.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + }, + { + "#url": "https://donmai.moe/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + }, + { + "#url": "https://danbooru.donmai.us/pools/7659", + "#category": ("Danbooru", "danbooru", "pool"), + "#class": danbooru.DanbooruPoolExtractor, + "#sha1_content": "b16bab12bea5f7ea9e0a836bf8045f280e113d99", + }, + { + "#url": "https://danbooru.donmai.us/pool/show/7659", + "#category": ("Danbooru", "danbooru", "pool"), + "#class": danbooru.DanbooruPoolExtractor, + }, + { + "#url": "https://danbooru.donmai.us/posts/294929", + "#category": ("Danbooru", "danbooru", "post"), + "#class": danbooru.DanbooruPostExtractor, + "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", + "approver_id": None, + "bit_flags": 0, + "created_at": "2008-08-12T00:46:05.385-04:00", + "date": "dt:2008-08-12 04:46:05", + "down_score": 0, + "extension": "jpg", + "fav_count": 9, + "file_ext": "jpg", + "file_size": 358232, + "file_url": "https://cdn.donmai.us/original/ac/8e/ac8e3b92ea328ce9cf7211e69c905bf9.jpg", + "filename": "ac8e3b92ea328ce9cf7211e69c905bf9", + "has_active_children": False, + "has_children": False, + "has_large": True, + "has_visible_children": False, + "id": 294929, + "image_height": 687, + "image_width": 895, + "is_banned": False, + "is_deleted": False, + "is_flagged": False, + "is_pending": False, + "large_file_url": "https://cdn.donmai.us/sample/ac/8e/sample-ac8e3b92ea328ce9cf7211e69c905bf9.jpg", + "last_comment_bumped_at": None, + "last_commented_at": None, + "last_noted_at": None, + "md5": "ac8e3b92ea328ce9cf7211e69c905bf9", + "media_asset": dict, + "parent_id": None, + "pixiv_id": 1129835, + "preview_file_url": "https://cdn.donmai.us/180x180/ac/8e/ac8e3b92ea328ce9cf7211e69c905bf9.jpg", + "rating": "s", + "score": 1, + "source": "https://i.pximg.net/img-original/img/2008/07/09/16/10/23/1129835_p0.jpg", + "subcategory": "post", + "tag_count": 32, + "tag_count_artist": 1, + "tag_count_character": 3, + "tag_count_copyright": 3, + "tag_count_general": 23, + "tag_count_meta": 2, + "tag_string": "2boys bat_(animal) batman batman_(series) black_bodysuit bodysuit bonocho brown_eyes closed_mouth collared_shirt commentary_request copyright_name dc_comics expressionless facepaint glasgow_smile heath_ledger joker_(dc) male_focus multiple_boys outline outstretched_arm parted_lips photoshop_(medium) pink_shirt shirt sketch smile the_dark_knight upper_body white_outline wing_collar", + "tag_string_artist": "bonocho", + "tag_string_character": "batman heath_ledger joker_(dc)", + "tag_string_copyright": "batman_(series) dc_comics the_dark_knight", + "tag_string_general": "2boys bat_(animal) black_bodysuit bodysuit brown_eyes closed_mouth collared_shirt copyright_name expressionless facepaint glasgow_smile male_focus multiple_boys outline outstretched_arm parted_lips pink_shirt shirt sketch smile upper_body white_outline wing_collar", + "tag_string_meta": "commentary_request photoshop_(medium)", + "tags": [ + "2boys", + "bat_(animal)", + "batman", + "batman_(series)", + "black_bodysuit", + "bodysuit", + "bonocho", + "brown_eyes", + "closed_mouth", + "collared_shirt", + "commentary_request", + "copyright_name", + "dc_comics", + "expressionless", + "facepaint", + "glasgow_smile", + "heath_ledger", + "joker_(dc)", + "male_focus", + "multiple_boys", + "outline", + "outstretched_arm", + "parted_lips", + "photoshop_(medium)", + "pink_shirt", + "shirt", + "sketch", + "smile", + "the_dark_knight", + "upper_body", + "white_outline", + "wing_collar", + ], + "tags_artist": [ + "bonocho", + ], + "tags_character": [ + "batman", + "heath_ledger", + "joker_(dc)", + ], + "tags_copyright": [ + "batman_(series)", + "dc_comics", + "the_dark_knight", + ], + "tags_general": [ + "2boys", + "bat_(animal)", + "black_bodysuit", + "bodysuit", + "brown_eyes", + "closed_mouth", + "collared_shirt", + "copyright_name", + "expressionless", + "facepaint", + "glasgow_smile", + "male_focus", + "multiple_boys", + "outline", + "outstretched_arm", + "parted_lips", + "pink_shirt", + "shirt", + "sketch", + "smile", + "upper_body", + "white_outline", + "wing_collar", + ], + "tags_meta": [ + "commentary_request", + "photoshop_(medium)", + ], + "up_score": range(1, 5), + "updated_at": "2022-07-11T23:42:31.881-04:00", + "uploader_id": 67005, + }, + { + "#url": "https://danbooru.donmai.us/posts/3613024", + "#category": ("Danbooru", "danbooru", "post"), + "#class": danbooru.DanbooruPostExtractor, + "#options": {"ugoira": True}, + "#pattern": r"https?://.+\.zip$", + }, + { + "#url": "https://danbooru.donmai.us/post/show/294929", + "#category": ("Danbooru", "danbooru", "post"), + "#class": danbooru.DanbooruPostExtractor, + }, + { + "#url": "https://danbooru.donmai.us/explore/posts/popular", + "#category": ("Danbooru", "danbooru", "popular"), + "#class": danbooru.DanbooruPopularExtractor, + }, + { + "#url": "https://danbooru.donmai.us/explore/posts/popular?date=2013-06-06&scale=week", + "#category": ("Danbooru", "danbooru", "popular"), + "#class": danbooru.DanbooruPopularExtractor, + "#range": "1-120", + "#count": 120, + }, ) diff --git a/test/results/derpibooru.py b/test/results/derpibooru.py index 4fe775bb2..51acddb0f 100644 --- a/test/results/derpibooru.py +++ b/test/results/derpibooru.py @@ -1,128 +1,113 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import philomena - __tests__ = ( -{ - "#url" : "https://derpibooru.org/images/1", - "#category": ("philomena", "derpibooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, - "#count" : 1, - "#sha1_content": "88449eeb0c4fa5d3583d0b794f6bc1d70bf7f889", - - "animated" : False, - "aspect_ratio" : 1.0, - "comment_count" : int, - "created_at" : "2012-01-02T03:12:33Z", - "date" : "dt:2012-01-02 03:12:33", - "deletion_reason" : None, - "description" : "", - "downvotes" : int, - "duplicate_of" : None, - "duration" : 0.04, - "extension" : "png", - "faves" : int, - "first_seen_at" : "2012-01-02T03:12:33Z", - "format" : "png", - "height" : 900, - "hidden_from_users": False, - "id" : 1, - "mime_type" : "image/png", - "name" : "1__safe_fluttershy_solo_cloud_happy_flying_upvotes+galore_artist-colon-speccysy_get_sunshine", - "orig_sha512_hash": None, - "processed" : True, - "representations" : dict, - "score" : int, - "sha512_hash" : "f16c98e2848c2f1bfff3985e8f1a54375cc49f78125391aeb80534ce011ead14e3e452a5c4bc98a66f56bdfcd07ef7800663b994f3f343c572da5ecc22a9660f", - "size" : 860914, - "source_url" : "https://web.archive.org/web/20110702164313/http://speccysy.deviantart.com:80/art/Afternoon-Flight-215193985", - "spoilered" : False, - "tag_count" : int, - "tag_ids" : list, - "tags" : list, - "thumbnails_generated": True, - "updated_at" : r"re:\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ", - "uploader" : "Clover the Clever", - "uploader_id" : 211188, - "upvotes" : int, - "view_url" : str, - "width" : 900, - "wilson_score" : float, -}, - -{ - "#url" : "https://derpibooru.org/images/3334658", - "#comment" : "svg (#5643)", - "#category": ("philomena", "derpibooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, - "#urls" : "https://derpicdn.net/img/view/2024/4/1/3334658__safe_alternate+version_artist-colon-jp_derpibooru+exclusive_twilight+sparkle_alicorn_pony_amending+fences_g4_season+5_-dot-svg+available_female_grin_lo.svg", - "#sha1_content": "eec5adf02e2a4fe83b9211c0444d57dc03e21f50", - - "extension": "svg", - "format" : "svg", -}, - -{ - "#url" : "https://derpibooru.org/1", - "#category": ("philomena", "derpibooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, -}, - -{ - "#url" : "https://www.derpibooru.org/1", - "#category": ("philomena", "derpibooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, -}, - -{ - "#url" : "https://www.derpibooru.org/images/1", - "#category": ("philomena", "derpibooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, -}, - -{ - "#url" : "https://derpibooru.org/search?q=cute", - "#category": ("philomena", "derpibooru", "search"), - "#class" : philomena.PhilomenaSearchExtractor, - "#range" : "40-60", - "#count" : 21, -}, - -{ - "#url" : "https://derpibooru.org/tags/cute", - "#category": ("philomena", "derpibooru", "search"), - "#class" : philomena.PhilomenaSearchExtractor, - "#range" : "40-60", - "#count" : 21, -}, - -{ - "#url" : "https://derpibooru.org/tags/artist-colon--dash-_-fwslash--fwslash-%255Bkorroki%255D_aternak", - "#category": ("philomena", "derpibooru", "search"), - "#class" : philomena.PhilomenaSearchExtractor, - "#count" : ">= 2", -}, - -{ - "#url" : "https://derpibooru.org/galleries/1", - "#category": ("philomena", "derpibooru", "gallery"), - "#class" : philomena.PhilomenaGalleryExtractor, - "#pattern" : r"https://derpicdn\.net/img/view/\d+/\d+/\d+/\d+[^/]+$", - - "gallery": { - "description" : "Indexes start at 1 :P", - "id" : 1, - "spoiler_warning": "", - "thumbnail_id" : 1, - "title" : "The Very First Gallery", - "user" : "DeliciousBlackInk", - "user_id" : 365446, + { + "#url": "https://derpibooru.org/images/1", + "#category": ("philomena", "derpibooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + "#count": 1, + "#sha1_content": "88449eeb0c4fa5d3583d0b794f6bc1d70bf7f889", + "animated": False, + "aspect_ratio": 1.0, + "comment_count": int, + "created_at": "2012-01-02T03:12:33Z", + "date": "dt:2012-01-02 03:12:33", + "deletion_reason": None, + "description": "", + "downvotes": int, + "duplicate_of": None, + "duration": 0.04, + "extension": "png", + "faves": int, + "first_seen_at": "2012-01-02T03:12:33Z", + "format": "png", + "height": 900, + "hidden_from_users": False, + "id": 1, + "mime_type": "image/png", + "name": "1__safe_fluttershy_solo_cloud_happy_flying_upvotes+galore_artist-colon-speccysy_get_sunshine", + "orig_sha512_hash": None, + "processed": True, + "representations": dict, + "score": int, + "sha512_hash": "f16c98e2848c2f1bfff3985e8f1a54375cc49f78125391aeb80534ce011ead14e3e452a5c4bc98a66f56bdfcd07ef7800663b994f3f343c572da5ecc22a9660f", + "size": 860914, + "source_url": "https://web.archive.org/web/20110702164313/http://speccysy.deviantart.com:80/art/Afternoon-Flight-215193985", + "spoilered": False, + "tag_count": int, + "tag_ids": list, + "tags": list, + "thumbnails_generated": True, + "updated_at": r"re:\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ", + "uploader": "Clover the Clever", + "uploader_id": 211188, + "upvotes": int, + "view_url": str, + "width": 900, + "wilson_score": float, + }, + { + "#url": "https://derpibooru.org/images/3334658", + "#comment": "svg (#5643)", + "#category": ("philomena", "derpibooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + "#urls": "https://derpicdn.net/img/view/2024/4/1/3334658__safe_alternate+version_artist-colon-jp_derpibooru+exclusive_twilight+sparkle_alicorn_pony_amending+fences_g4_season+5_-dot-svg+available_female_grin_lo.svg", + "#sha1_content": "eec5adf02e2a4fe83b9211c0444d57dc03e21f50", + "extension": "svg", + "format": "svg", + }, + { + "#url": "https://derpibooru.org/1", + "#category": ("philomena", "derpibooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + }, + { + "#url": "https://www.derpibooru.org/1", + "#category": ("philomena", "derpibooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + }, + { + "#url": "https://www.derpibooru.org/images/1", + "#category": ("philomena", "derpibooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + }, + { + "#url": "https://derpibooru.org/search?q=cute", + "#category": ("philomena", "derpibooru", "search"), + "#class": philomena.PhilomenaSearchExtractor, + "#range": "40-60", + "#count": 21, + }, + { + "#url": "https://derpibooru.org/tags/cute", + "#category": ("philomena", "derpibooru", "search"), + "#class": philomena.PhilomenaSearchExtractor, + "#range": "40-60", + "#count": 21, + }, + { + "#url": "https://derpibooru.org/tags/artist-colon--dash-_-fwslash--fwslash-%255Bkorroki%255D_aternak", + "#category": ("philomena", "derpibooru", "search"), + "#class": philomena.PhilomenaSearchExtractor, + "#count": ">= 2", + }, + { + "#url": "https://derpibooru.org/galleries/1", + "#category": ("philomena", "derpibooru", "gallery"), + "#class": philomena.PhilomenaGalleryExtractor, + "#pattern": r"https://derpicdn\.net/img/view/\d+/\d+/\d+/\d+[^/]+$", + "gallery": { + "description": "Indexes start at 1 :P", + "id": 1, + "spoiler_warning": "", + "thumbnail_id": 1, + "title": "The Very First Gallery", + "user": "DeliciousBlackInk", + "user_id": 365446, + }, }, -}, - ) diff --git a/test/results/desktopography.py b/test/results/desktopography.py index c6066692d..1c4497414 100644 --- a/test/results/desktopography.py +++ b/test/results/desktopography.py @@ -1,29 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import desktopography - __tests__ = ( -{ - "#url" : "https://desktopography.net/", - "#category": ("", "desktopography", "site"), - "#class" : desktopography.DesktopographySiteExtractor, -}, - -{ - "#url" : "https://desktopography.net/exhibition-2020/", - "#category": ("", "desktopography", "exhibition"), - "#class" : desktopography.DesktopographyExhibitionExtractor, -}, - -{ - "#url" : "https://desktopography.net/portfolios/new-era/", - "#category": ("", "desktopography", "entry"), - "#class" : desktopography.DesktopographyEntryExtractor, -}, - + { + "#url": "https://desktopography.net/", + "#category": ("", "desktopography", "site"), + "#class": desktopography.DesktopographySiteExtractor, + }, + { + "#url": "https://desktopography.net/exhibition-2020/", + "#category": ("", "desktopography", "exhibition"), + "#class": desktopography.DesktopographyExhibitionExtractor, + }, + { + "#url": "https://desktopography.net/portfolios/new-era/", + "#category": ("", "desktopography", "entry"), + "#class": desktopography.DesktopographyEntryExtractor, + }, ) diff --git a/test/results/desuarchive.py b/test/results/desuarchive.py index 43cd96895..fa56dfd9a 100644 --- a/test/results/desuarchive.py +++ b/test/results/desuarchive.py @@ -1,56 +1,46 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://desuarchive.org/a/thread/159542679/", - "#category": ("foolfuuka", "desuarchive", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "e7d624aded15a069194e38dc731ec23217a422fb", -}, - -{ - "#url" : "https://desuarchive.org/a", - "#category": ("foolfuuka", "desuarchive", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://desuarchive.org/a/", - "#category": ("foolfuuka", "desuarchive", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://desuarchive.org/a/2", - "#category": ("foolfuuka", "desuarchive", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://desuarchive.org/a/page/2", - "#category": ("foolfuuka", "desuarchive", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, - "#pattern" : foolfuuka.FoolfuukaThreadExtractor.pattern, - "#count" : 10, -}, - -{ - "#url" : "https://desuarchive.org/_/search/text/test/", - "#category": ("foolfuuka", "desuarchive", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://desuarchive.org/a/gallery/5", - "#category": ("foolfuuka", "desuarchive", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://desuarchive.org/a/thread/159542679/", + "#category": ("foolfuuka", "desuarchive", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "e7d624aded15a069194e38dc731ec23217a422fb", + }, + { + "#url": "https://desuarchive.org/a", + "#category": ("foolfuuka", "desuarchive", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://desuarchive.org/a/", + "#category": ("foolfuuka", "desuarchive", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://desuarchive.org/a/2", + "#category": ("foolfuuka", "desuarchive", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://desuarchive.org/a/page/2", + "#category": ("foolfuuka", "desuarchive", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + "#pattern": foolfuuka.FoolfuukaThreadExtractor.pattern, + "#count": 10, + }, + { + "#url": "https://desuarchive.org/_/search/text/test/", + "#category": ("foolfuuka", "desuarchive", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://desuarchive.org/a/gallery/5", + "#category": ("foolfuuka", "desuarchive", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/deviantart.py b/test/results/deviantart.py index 0e8e371c4..285111e67 100644 --- a/test/results/deviantart.py +++ b/test/results/deviantart.py @@ -1,1022 +1,895 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import deviantart import datetime -from gallery_dl import exception +from gallery_dl import exception +from gallery_dl.extractor import deviantart __tests__ = ( -{ - "#url" : "https://www.deviantart.com/shimoda7", - "#category": ("", "deviantart", "user"), - "#class" : deviantart.DeviantartUserExtractor, - "#urls" : "https://www.deviantart.com/shimoda7/gallery", -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7", - "#category": ("", "deviantart", "user"), - "#class" : deviantart.DeviantartUserExtractor, - "#options" : {"include": "all"}, - "#urls" : ( - "https://www.deviantart.com/shimoda7/avatar", - "https://www.deviantart.com/shimoda7/banner", - "https://www.deviantart.com/shimoda7/gallery", - "https://www.deviantart.com/shimoda7/gallery/scraps", - "https://www.deviantart.com/shimoda7/posts", - "https://www.deviantart.com/shimoda7/posts/statuses", - "https://www.deviantart.com/shimoda7/favourites", - ), -}, - -{ - "#url" : "https://shimoda7.deviantart.com/", - "#category": ("", "deviantart", "user"), - "#class" : deviantart.DeviantartUserExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, - "#pattern" : r"https://(images-)?wixmp-[^.]+\.wixmp\.com/f/.+/.+\.(jpg|png)\?token=.+", - "#count" : ">= 38", - - "allows_comments" : bool, - "author" : { - "type" : "premium", - "usericon": str, - "userid" : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", - "username": "shimoda7", + { + "#url": "https://www.deviantart.com/shimoda7", + "#category": ("", "deviantart", "user"), + "#class": deviantart.DeviantartUserExtractor, + "#urls": "https://www.deviantart.com/shimoda7/gallery", }, - "category_path" : str, - "content" : { - "filesize" : int, - "height" : int, - "src" : str, - "transparency": bool, - "width" : int, - }, - "date" : datetime.datetime, - "deviationid" : str, - "?download_filesize": int, - "extension" : str, - "index" : int, - "is_deleted" : bool, - "is_downloadable" : bool, - "is_favourited" : bool, - "is_mature" : bool, - "preview" : { - "height" : int, - "src" : str, - "transparency": bool, - "width" : int, - }, - "published_time" : int, - "stats" : { - "comments" : int, - "favourites": int, - }, - "target" : dict, - "thumbs" : list, - "title" : str, - "url" : r"re:https://www.deviantart.com/shimoda7/art/[^/]+-\d+", - "username" : "shimoda7", -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/", - "#comment" : "range/skip (#4557)", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, - "#options" : {"original": False}, - "#pattern" : r"https://images-wixmp-[0-9a-f]+\.wixmp\.com/f/0e474835-ec35-4937-b647-b6830ed58bd1/d2idul-6158ded2-37ac-413d-802e-0689f0f020ad\.jpg\?token=[\w.]+", - "#range" : "38-", - "#count" : 1, -}, - -{ - "#url" : "https://www.deviantart.com/AlloyRabbit/gallery", - "#comment" : "deactivated account", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/Shydude/gallery", - "#comment" : "deactivated account", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/zapor666/gallery", - "#comment" : "deactivated account", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/yakuzafc/gallery", - "#comment" : "group", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, - "#pattern" : r"https://www.deviantart.com/yakuzafc/gallery/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/", - "#count" : ">= 15", -}, - -{ - "#url" : "https://www.deviantart.com/yakuzafc/gallery", - "#comment" : "'group': 'skip' (#4630)", - "#category" : ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, - "#options" : {"group": "skip"}, - "#exception": exception.StopExtraction, - "#count" : 0, -}, - -{ - "#url" : "https://www.deviantart.com/justatest235723/gallery", - "#comment" : "'folders' option (#276)", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, - "#options" : { - "metadata": 1, - "folders" : 1, - "original": 0, - }, - "#count" : 3, - - "description": str, - "folders" : list, - "is_watching": bool, - "license" : str, - "tags" : list, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda8/gallery/", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/all", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/?catpath=/", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/gallery/", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/gallery/all/", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/gallery/?catpath=/", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://deviantart.com/shimoda7/avatar", - "#category": ("", "deviantart", "avatar"), - "#class" : deviantart.DeviantartAvatarExtractor, - "#urls" : "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4", - "#sha1_content": "abf2cc79b842315f2e54bfdd93bf794a0f612b6f", - - "author" : { - "type" : "premium", - "usericon": "https://a.deviantart.net/avatars/s/h/shimoda7.jpg?4", - "userid" : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", + { + "#url": "https://www.deviantart.com/shimoda7", + "#category": ("", "deviantart", "user"), + "#class": deviantart.DeviantartUserExtractor, + "#options": {"include": "all"}, + "#urls": ( + "https://www.deviantart.com/shimoda7/avatar", + "https://www.deviantart.com/shimoda7/banner", + "https://www.deviantart.com/shimoda7/gallery", + "https://www.deviantart.com/shimoda7/gallery/scraps", + "https://www.deviantart.com/shimoda7/posts", + "https://www.deviantart.com/shimoda7/posts/statuses", + "https://www.deviantart.com/shimoda7/favourites", + ), + }, + { + "#url": "https://shimoda7.deviantart.com/", + "#category": ("", "deviantart", "user"), + "#class": deviantart.DeviantartUserExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + "#pattern": r"https://(images-)?wixmp-[^.]+\.wixmp\.com/f/.+/.+\.(jpg|png)\?token=.+", + "#count": ">= 38", + "allows_comments": bool, + "author": { + "type": "premium", + "usericon": str, + "userid": "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", + "username": "shimoda7", + }, + "category_path": str, + "content": { + "filesize": int, + "height": int, + "src": str, + "transparency": bool, + "width": int, + }, + "date": datetime.datetime, + "deviationid": str, + "?download_filesize": int, + "extension": str, + "index": int, + "is_deleted": bool, + "is_downloadable": bool, + "is_favourited": bool, + "is_mature": bool, + "preview": { + "height": int, + "src": str, + "transparency": bool, + "width": int, + }, + "published_time": int, + "stats": { + "comments": int, + "favourites": int, + }, + "target": dict, + "thumbs": list, + "title": str, + "url": r"re:https://www.deviantart.com/shimoda7/art/[^/]+-\d+", "username": "shimoda7", }, - "content" : { - "src": "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4" - }, - "da_category" : "avatar", - "date" : "dt:1970-01-01 00:00:00", - "extension" : "jpg", - "filename" : "avatar_by_shimoda7-d4", - "index" : 4, - "index_base36" : "4", - "is_deleted" : False, - "is_downloadable": False, - "is_original" : True, - "published_time" : 0, - "target" : { + { + "#url": "https://www.deviantart.com/shimoda7/gallery/", + "#comment": "range/skip (#4557)", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + "#options": {"original": False}, + "#pattern": r"https://images-wixmp-[0-9a-f]+\.wixmp\.com/f/0e474835-ec35-4937-b647-b6830ed58bd1/d2idul-6158ded2-37ac-413d-802e-0689f0f020ad\.jpg\?token=[\w.]+", + "#range": "38-", + "#count": 1, + }, + { + "#url": "https://www.deviantart.com/AlloyRabbit/gallery", + "#comment": "deactivated account", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://www.deviantart.com/Shydude/gallery", + "#comment": "deactivated account", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://www.deviantart.com/zapor666/gallery", + "#comment": "deactivated account", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://www.deviantart.com/yakuzafc/gallery", + "#comment": "group", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + "#pattern": r"https://www.deviantart.com/yakuzafc/gallery/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/", + "#count": ">= 15", + }, + { + "#url": "https://www.deviantart.com/yakuzafc/gallery", + "#comment": "'group': 'skip' (#4630)", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + "#options": {"group": "skip"}, + "#exception": exception.StopExtraction, + "#count": 0, + }, + { + "#url": "https://www.deviantart.com/justatest235723/gallery", + "#comment": "'folders' option (#276)", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + "#options": { + "metadata": 1, + "folders": 1, + "original": 0, + }, + "#count": 3, + "description": str, + "folders": list, + "is_watching": bool, + "license": str, + "tags": list, + }, + { + "#url": "https://www.deviantart.com/shimoda8/gallery/", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/all", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/?catpath=/", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://shimoda7.deviantart.com/gallery/", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://shimoda7.deviantart.com/gallery/all/", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://shimoda7.deviantart.com/gallery/?catpath=/", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://deviantart.com/shimoda7/avatar", + "#category": ("", "deviantart", "avatar"), + "#class": deviantart.DeviantartAvatarExtractor, + "#urls": "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4", + "#sha1_content": "abf2cc79b842315f2e54bfdd93bf794a0f612b6f", + "author": { + "type": "premium", + "usericon": "https://a.deviantart.net/avatars/s/h/shimoda7.jpg?4", + "userid": "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", + "username": "shimoda7", + }, + "content": {"src": "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4"}, + "da_category": "avatar", + "date": "dt:1970-01-01 00:00:00", "extension": "jpg", - "filename" : "avatar_by_shimoda7-d4", - "src" : "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4" + "filename": "avatar_by_shimoda7-d4", + "index": 4, + "index_base36": "4", + "is_deleted": False, + "is_downloadable": False, + "is_original": True, + "published_time": 0, + "target": { + "extension": "jpg", + "filename": "avatar_by_shimoda7-d4", + "src": "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4", + }, + "title": "avatar", + "username": "shimoda7", }, - "title" : "avatar", - "username" : "shimoda7", -}, - -{ - "#url" : "https://deviantart.com/shimoda7/avatar", - "#comment" : "'formats' option", - "#category": ("", "deviantart", "avatar"), - "#class" : deviantart.DeviantartAvatarExtractor, - "#archive" : False, - "#options" : {"formats": ["original.jpg", "big.jpg", "big.png", "big.gif"]}, - "#urls" : ( - "https://a.deviantart.net/avatars-original/s/h/shimoda7.jpg?4", - "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4", - "https://a.deviantart.net/avatars-big/s/h/shimoda7.png?4", - "https://a.deviantart.net/avatars-big/s/h/shimoda7.gif?4", - ), -}, - -{ - "#url" : "https://deviantart.com/h3813067/avatar", - "#comment" : "default avatar (#5276)", - "#category": ("", "deviantart", "avatar"), - "#class" : deviantart.DeviantartAvatarExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://deviantart.com/gdldev/banner", - "#category": ("", "deviantart", "background"), - "#class" : deviantart.DeviantartBackgroundExtractor, - "#pattern" : r"https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$", - "#sha1_content": "980eaa76ce515f1b6bef60dfadf26a5bbe9c583f", - - "allows_comments" : True, - "author" : { - "type" : "regular", - "usericon": "https://a.deviantart.net/avatars/g/d/gdldev.jpg?12", - "userid" : "1A12BA26-33C2-AA0A-7678-0B6DFBA7AC8E", - "username": "gdldev" - }, - "category_path" : "", - "content" : { - "filename" : "banner_by_gdldev_dgntyqc.png", - "filesize" : 84510, - "height" : 4000, - "src" : r"re:https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$", - "transparency": False, - "width" : 6400 - }, - "date" : "dt:2024-01-02 21:16:06", - "deviationid" : "8C8D6B28-766A-DE21-7F7D-CE055C3BD50A", - "download_filesize": 84510, - "extension" : "png", - "filename" : "banner_by_gdldev-dgntyqc", - "index" : 1007488020, - "index_base36" : "gntyqc", - "is_blocked" : False, - "is_deleted" : False, - "is_downloadable" : True, - "is_favourited" : False, - "is_mature" : False, - "is_original" : True, - "is_published" : False, - "preview" : dict, - "printid" : None, - "published_time" : 1704230166, - "stats" : { - "comments" : 0, - "favourites": 0, - }, - "target" : dict, - "thumbs" : list, - "title" : "Banner", - "url" : "https://www.deviantart.com/stash/0198jippkeys", - "username" : "gdldev", -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous", - "#comment" : "user", - "#category": ("", "deviantart", "folder"), - "#class" : deviantart.DeviantartFolderExtractor, - "#options" : {"original": False}, - "#count" : 5, -}, - -{ - "#url" : "https://www.deviantart.com/yakuzafc/gallery/37412168/Crafts", - "#comment" : "group", - "#category": ("", "deviantart", "folder"), - "#class" : deviantart.DeviantartFolderExtractor, - "#options" : {"original": False}, - "#count" : ">= 4", -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/B38E3C6A-2029-6B45-757B-3C8D3422AD1A/misc", - "#comment" : "uuid", - "#category": ("", "deviantart", "folder"), - "#class" : deviantart.DeviantartFolderExtractor, - "#options" : {"original": False}, - "#count" : 5, -}, - -{ - "#url" : "https://www.deviantart.com/justatest235723/gallery/69302698/-test-b-c-d-e-f-", - "#comment" : "name starts with '_', special characters (#1451)", - "#category": ("", "deviantart", "folder"), - "#class" : deviantart.DeviantartFolderExtractor, - "#options" : {"original": False}, - "#count" : 1, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/gallery/722019/Miscellaneous", - "#category": ("", "deviantart", "folder"), - "#class" : deviantart.DeviantartFolderExtractor, -}, - -{ - "#url" : "https://yakuzafc.deviantart.com/gallery/37412168/Crafts", - "#category": ("", "deviantart", "folder"), - "#class" : deviantart.DeviantartFolderExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/stash/022c83odnaxc", - "#category": ("", "deviantart", "stash"), - "#class" : deviantart.DeviantartStashExtractor, - "#pattern" : r"https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/dcvdmbc-e506cdcf-3208-4c20-85ab-0bfa8a7bcb16.png\?token=ey.+", - "#count" : 1, - "#sha1_content": "057eb2f2861f6c8a96876b13cca1a4b7a408c11f", - - "content": { - "filename": "01_by_justatest235723_dcvdmbc.png", - "filesize": 380, - "width" : 128, - "height" : 128, - "src" : r"re:https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/dcvdmbc-e506cdcf-3208-4c20-85ab-0bfa8a7bcb16.png\?token=ey.+", - }, - "date" : "dt:2018-12-26 14:49:27", - "deviationid" : "A4A6AD52-8857-46EE-ABFE-86D49D4FF9D0", - "download_filesize": 380, - "extension" : "png", - "filename" : "01_by_justatest235723-dcvdmbc", - "index" : 778297656, - "index_base36" : "cvdmbc", - "published_time": 1545835767, - "title" : "01", - "url" : "https://www.deviantart.com/stash/022c83odnaxc", -}, - -{ - "#url" : "https://sta.sh/022c83odnaxc", - "#category": ("", "deviantart", "stash"), - "#class" : deviantart.DeviantartStashExtractor, -}, - -{ - "#url" : "https://sta.sh/21jf51j7pzl2", - "#comment" : "multiple stash items", - "#category": ("", "deviantart", "stash"), - "#class" : deviantart.DeviantartStashExtractor, - "#options" : {"original": False}, - "#count" : 4, -}, - -{ - "#url" : "https://sta.sh/024t4coz16mi", - "#comment" : "downloadable, but no 'content' field (#307)", - "#category": ("", "deviantart", "stash"), - "#class" : deviantart.DeviantartStashExtractor, - "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.rar\?token=.+", - "#count" : 1, -}, - -{ - "#url" : "https://sta.sh/215twi387vfj", - "#comment" : "mixed folders and images (#659)", - "#category": ("", "deviantart", "stash"), - "#class" : deviantart.DeviantartStashExtractor, - "#options" : {"original": False}, - "#count" : 4, -}, - -{ - "#url" : "https://sta.sh/abcdefghijkl", - "#category": ("", "deviantart", "stash"), - "#class" : deviantart.DeviantartStashExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.deviantart.com/h3813067/favourites/", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, - "#options" : { - "metadata": True, - "flat" : False, - }, - "#count" : 1, -}, - -{ - "#url" : "https://www.deviantart.com/h3813067/favourites/", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, - "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", -}, - -{ - "#url" : "https://www.deviantart.com/h3813067/favourites/all", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/h3813067/favourites/?catpath=/", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, -}, - -{ - "#url" : "https://h3813067.deviantart.com/favourites/", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, -}, - -{ - "#url" : "https://h3813067.deviantart.com/favourites/all", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, -}, - -{ - "#url" : "https://h3813067.deviantart.com/favourites/?catpath=/", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/pencilshadings/favourites/70595441/3D-Favorites", - "#category": ("", "deviantart", "collection"), - "#class" : deviantart.DeviantartCollectionExtractor, - "#options" : {"original": False}, - "#count" : ">= 15", -}, - -{ - "#url" : "https://www.deviantart.com/pencilshadings/favourites/F050486B-CB62-3C66-87FB-1105A7F6379F/3D Favorites", - "#category": ("", "deviantart", "collection"), - "#class" : deviantart.DeviantartCollectionExtractor, - "#options" : {"original": False}, - "#count" : ">= 15", -}, - -{ - "#url" : "https://pencilshadings.deviantart.com/favourites/70595441/3D-Favorites", - "#category": ("", "deviantart", "collection"), - "#class" : deviantart.DeviantartCollectionExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, - "#sha1_url": "48aeed5631763d96f5391d2177ea72d9fdbee4e5", -}, - -{ - "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, - "#options" : {"journals": "text"}, - "#sha1_url": "b2a8e74d275664b1a4acee0fca0a6fd33298571e", -}, - -{ - "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, - "#options" : {"journals": "none"}, - "#count" : 0, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/posts/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/journal/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/journal/?catpath=/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/journal/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/journal/?catpath=/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/t1na/posts/statuses", - "#category": ("", "deviantart", "status"), - "#class" : deviantart.DeviantartStatusExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.deviantart.com/justgalym/posts/statuses", - "#category": ("", "deviantart", "status"), - "#class" : deviantart.DeviantartStatusExtractor, - "#count" : 4, - "#sha1_url": "62ee48ff3405c7714dca70abf42e8e39731012fc", -}, - -{ - "#url" : "https://www.deviantart.com/justgalym/posts/statuses", - "#category": ("", "deviantart", "status"), - "#class" : deviantart.DeviantartStatusExtractor, - "#options" : {"journals": "none"}, - "#pattern" : r"https://images-wixmp-\w+\.wixmp\.com/intermediary/f/[^/]+/[^.]+\.jpg", - "#count" : 1, -}, - -{ - "#url" : "https://www.deviantart.com/vanillaghosties/posts/statuses", - "#comment" : "shared sta.sh item", - "#category": ("", "deviantart", "status"), - "#class" : deviantart.DeviantartStatusExtractor, - "#options" : { - "journals": "none", - "original": False, - }, - "#range" : "5-", - "#count" : 1, - - "index" : int, - "index_base36": r"re:^[0-9a-z]+$", - "url" : r"re:^https://www.deviantart.com/stash/\w+", -}, - -{ - "#url" : "https://www.deviantart.com/AndrejSKalin/posts/statuses", - "#comment" : "'deleted' deviations in 'items'", - "#category": ("", "deviantart", "status"), - "#class" : deviantart.DeviantartStatusExtractor, - "#options" : { - "journals" : "none", - "original" : 0, - "image-filter": "deviationid[:8] == '147C8B03'", - }, - "#count" : 2, - "#archive" : False, - - "deviationid": "147C8B03-7D34-AE93-9241-FA3C6DBBC655", -}, - -{ - "#url" : "https://www.deviantart.com/justgalym/posts/statuses", - "#category": ("", "deviantart", "status"), - "#class" : deviantart.DeviantartStatusExtractor, - "#options" : {"journals": "text"}, - "#sha1_url": "10a336bdee7b9692919461443a7dde44d495818c", -}, - -{ - "#url" : "https://www.deviantart.com/tag/nature", - "#category": ("", "deviantart", "tag"), - "#class" : deviantart.DeviantartTagExtractor, - "#options" : {"original": False}, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://www.deviantart.com/watch/deviations", - "#category": ("", "deviantart", "watch"), - "#class" : deviantart.DeviantartWatchExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/notifications/watch", - "#category": ("", "deviantart", "watch"), - "#class" : deviantart.DeviantartWatchExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/watch/posts", - "#category": ("", "deviantart", "watch-posts"), - "#class" : deviantart.DeviantartWatchPostsExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/art/For-the-sake-10073852", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : {"original": 0}, - "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/art/For-the-sake-10073852", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : {"metadata": "submission,camera,stats"}, - - "can_post_comment": False, - "description" : str, - "is_watching" : False, - "license" : "No License", - "stats": { - "comments" : int, - "downloads" : int, - "downloads_today": int, - "favourites" : int, - "views" : int, - "views_today" : int, - }, - "submission": { - "category" : "traditional/drawings/other", - "creation_time" : "2004-08-25T02:44:08-0700", - "file_size" : "133 KB", - "resolution" : "710x510", - "submitted_with": { - "app": "Unknown App", - "url": "" + { + "#url": "https://deviantart.com/shimoda7/avatar", + "#comment": "'formats' option", + "#category": ("", "deviantart", "avatar"), + "#class": deviantart.DeviantartAvatarExtractor, + "#archive": False, + "#options": {"formats": ["original.jpg", "big.jpg", "big.png", "big.gif"]}, + "#urls": ( + "https://a.deviantart.net/avatars-original/s/h/shimoda7.jpg?4", + "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4", + "https://a.deviantart.net/avatars-big/s/h/shimoda7.png?4", + "https://a.deviantart.net/avatars-big/s/h/shimoda7.gif?4", + ), + }, + { + "#url": "https://deviantart.com/h3813067/avatar", + "#comment": "default avatar (#5276)", + "#category": ("", "deviantart", "avatar"), + "#class": deviantart.DeviantartAvatarExtractor, + "#count": 0, + }, + { + "#url": "https://deviantart.com/gdldev/banner", + "#category": ("", "deviantart", "background"), + "#class": deviantart.DeviantartBackgroundExtractor, + "#pattern": r"https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$", + "#sha1_content": "980eaa76ce515f1b6bef60dfadf26a5bbe9c583f", + "allows_comments": True, + "author": { + "type": "regular", + "usericon": "https://a.deviantart.net/avatars/g/d/gdldev.jpg?12", + "userid": "1A12BA26-33C2-AA0A-7678-0B6DFBA7AC8E", + "username": "gdldev", + }, + "category_path": "", + "content": { + "filename": "banner_by_gdldev_dgntyqc.png", + "filesize": 84510, + "height": 4000, + "src": r"re:https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$", + "transparency": False, + "width": 6400, + }, + "date": "dt:2024-01-02 21:16:06", + "deviationid": "8C8D6B28-766A-DE21-7F7D-CE055C3BD50A", + "download_filesize": 84510, + "extension": "png", + "filename": "banner_by_gdldev-dgntyqc", + "index": 1007488020, + "index_base36": "gntyqc", + "is_blocked": False, + "is_deleted": False, + "is_downloadable": True, + "is_favourited": False, + "is_mature": False, + "is_original": True, + "is_published": False, + "preview": dict, + "printid": None, + "published_time": 1704230166, + "stats": { + "comments": 0, + "favourites": 0, }, + "target": dict, + "thumbs": list, + "title": "Banner", + "url": "https://www.deviantart.com/stash/0198jippkeys", + "username": "gdldev", }, - "tags": [], -}, - -{ - "#url" : "https://www.deviantart.com/zzz/art/zzz-1234567890", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.deviantart.com/justatest235723/art/archive-1103129101", - "#comment": "ZIP archive + preview image (#3782)", - "#class" : deviantart.DeviantartDeviationExtractor, - "#options": {"previews": True}, - "#pattern": [ - r"/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/di8rvv1-afe65948-16e1-4eca-b08d-9e6aaa9ed344\.zip", - r"/i/940f2d05-c5eb-4917-8192-7eb6a2d508c6/di8rvv1-bb9d891f-4374-4203-acd3-aea34b29a6a1\.png", - ], -}, - -{ - "#url" : "https://www.deviantart.com/myria-moon/art/Aime-Moi-261986576", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : {"comments": True}, - "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.jpg\?token=.+", - - "comments": "len:44", -}, - -{ - "#url" : "https://www.deviantart.com/justatest235723/art/Blue-811519058", - "#comment" : "nested comments (#4653)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : { - "original": False, - "comments": True, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous", + "#comment": "user", + "#category": ("", "deviantart", "folder"), + "#class": deviantart.DeviantartFolderExtractor, + "#options": {"original": False}, + "#count": 5, }, - - "comments": "len:20", -}, - -{ - "#url" : "https://www.deviantart.com/justatest235723/art/Blue-811519058", - "#comment" : "comment avatars (#4995)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : { - "original" : False, - "comments-avatars": True, - }, - "#range" : "5-", - "#pattern" : r"^https://www\.deviantart\.com/justatest235723/avatar/$", - "#count" : 16, -}, - -{ - "#url" : "https://www.deviantart.com/citizenfresh/art/Hverarond-789295466", - "#comment" : "wixmp URL rewrite /intermediary/", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#urls" : "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/4deb0f1a-cdef-444e-b194-c8d6b3f7e933/dd1xca2-7f835e62-6fd3-4b99-92c7-2bfd4e1b296f.jpg", - - "is_downloadable": False, - "is_original" : False, -}, - -{ - "#url" : "https://www.deviantart.com/skatergators/art/COM-Moni-781571783", - "#comment" : "GIF (#242)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#pattern" : r"https://wixmp-\w+\.wixmp\.com/f/03fd2413-efe9-4e5c-8734-2b72605b3fbb/dcxbsnb-1bbf0b38-42af-4070-8878-f30961955bec\.gif\?token=ey...", -}, - -{ - "#url" : "https://www.deviantart.com/yuumei/art/Flash-Comic-214724929", - "#comment" : "Flash animation with GIF preview (#1731)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.swf\?token=.+", - - "filename" : "flash_comic_tutorial_by_yuumei-d3juatd", - "extension": "swf", -}, - -{ - "#url" : "https://www.deviantart.com/justatest235723/art/video-1103119114", - "#comment" : "video", - "#class" : deviantart.DeviantartDeviationExtractor, - "#pattern" : r"/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/di8ro5m-e2a5bdf0-daee-4e18-bede-fbfc394d6c65\.mp4\?token=ey", - - "filename" : "video_63aebdd4bc0323da460796b9a2ac8522_by_justatest235723-di8ro5m", - "extension": "mp4", -}, - -{ - "#url" : "https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498", - "#comment" : "sta.sh URLs from description (#302)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : { - "extra" : 1, - "original": 0, - }, - "#pattern" : deviantart.DeviantartStashExtractor.pattern, - "#range" : "2-", - "#count" : 4, -}, - -{ - "#url" : "https://www.deviantart.com/cimar-wildehopps/art/Honorary-Vixen-859809305", - "#comment" : "sta.sh URL from deviation['text_content']['body']['features']", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : {"extra": 1}, - "#pattern" : r"""text: + { + "#url": "https://www.deviantart.com/yakuzafc/gallery/37412168/Crafts", + "#comment": "group", + "#category": ("", "deviantart", "folder"), + "#class": deviantart.DeviantartFolderExtractor, + "#options": {"original": False}, + "#count": ">= 4", + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/B38E3C6A-2029-6B45-757B-3C8D3422AD1A/misc", + "#comment": "uuid", + "#category": ("", "deviantart", "folder"), + "#class": deviantart.DeviantartFolderExtractor, + "#options": {"original": False}, + "#count": 5, + }, + { + "#url": "https://www.deviantart.com/justatest235723/gallery/69302698/-test-b-c-d-e-f-", + "#comment": "name starts with '_', special characters (#1451)", + "#category": ("", "deviantart", "folder"), + "#class": deviantart.DeviantartFolderExtractor, + "#options": {"original": False}, + "#count": 1, + }, + { + "#url": "https://shimoda7.deviantart.com/gallery/722019/Miscellaneous", + "#category": ("", "deviantart", "folder"), + "#class": deviantart.DeviantartFolderExtractor, + }, + { + "#url": "https://yakuzafc.deviantart.com/gallery/37412168/Crafts", + "#category": ("", "deviantart", "folder"), + "#class": deviantart.DeviantartFolderExtractor, + }, + { + "#url": "https://www.deviantart.com/stash/022c83odnaxc", + "#category": ("", "deviantart", "stash"), + "#class": deviantart.DeviantartStashExtractor, + "#pattern": r"https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/dcvdmbc-e506cdcf-3208-4c20-85ab-0bfa8a7bcb16.png\?token=ey.+", + "#count": 1, + "#sha1_content": "057eb2f2861f6c8a96876b13cca1a4b7a408c11f", + "content": { + "filename": "01_by_justatest235723_dcvdmbc.png", + "filesize": 380, + "width": 128, + "height": 128, + "src": r"re:https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/dcvdmbc-e506cdcf-3208-4c20-85ab-0bfa8a7bcb16.png\?token=ey.+", + }, + "date": "dt:2018-12-26 14:49:27", + "deviationid": "A4A6AD52-8857-46EE-ABFE-86D49D4FF9D0", + "download_filesize": 380, + "extension": "png", + "filename": "01_by_justatest235723-dcvdmbc", + "index": 778297656, + "index_base36": "cvdmbc", + "published_time": 1545835767, + "title": "01", + "url": "https://www.deviantart.com/stash/022c83odnaxc", + }, + { + "#url": "https://sta.sh/022c83odnaxc", + "#category": ("", "deviantart", "stash"), + "#class": deviantart.DeviantartStashExtractor, + }, + { + "#url": "https://sta.sh/21jf51j7pzl2", + "#comment": "multiple stash items", + "#category": ("", "deviantart", "stash"), + "#class": deviantart.DeviantartStashExtractor, + "#options": {"original": False}, + "#count": 4, + }, + { + "#url": "https://sta.sh/024t4coz16mi", + "#comment": "downloadable, but no 'content' field (#307)", + "#category": ("", "deviantart", "stash"), + "#class": deviantart.DeviantartStashExtractor, + "#pattern": r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.rar\?token=.+", + "#count": 1, + }, + { + "#url": "https://sta.sh/215twi387vfj", + "#comment": "mixed folders and images (#659)", + "#category": ("", "deviantart", "stash"), + "#class": deviantart.DeviantartStashExtractor, + "#options": {"original": False}, + "#count": 4, + }, + { + "#url": "https://sta.sh/abcdefghijkl", + "#category": ("", "deviantart", "stash"), + "#class": deviantart.DeviantartStashExtractor, + "#count": 0, + }, + { + "#url": "https://www.deviantart.com/h3813067/favourites/", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + "#options": { + "metadata": True, + "flat": False, + }, + "#count": 1, + }, + { + "#url": "https://www.deviantart.com/h3813067/favourites/", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", + }, + { + "#url": "https://www.deviantart.com/h3813067/favourites/all", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + }, + { + "#url": "https://www.deviantart.com/h3813067/favourites/?catpath=/", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + }, + { + "#url": "https://h3813067.deviantart.com/favourites/", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + }, + { + "#url": "https://h3813067.deviantart.com/favourites/all", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + }, + { + "#url": "https://h3813067.deviantart.com/favourites/?catpath=/", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + }, + { + "#url": "https://www.deviantart.com/pencilshadings/favourites/70595441/3D-Favorites", + "#category": ("", "deviantart", "collection"), + "#class": deviantart.DeviantartCollectionExtractor, + "#options": {"original": False}, + "#count": ">= 15", + }, + { + "#url": "https://www.deviantart.com/pencilshadings/favourites/F050486B-CB62-3C66-87FB-1105A7F6379F/3D Favorites", + "#category": ("", "deviantart", "collection"), + "#class": deviantart.DeviantartCollectionExtractor, + "#options": {"original": False}, + "#count": ">= 15", + }, + { + "#url": "https://pencilshadings.deviantart.com/favourites/70595441/3D-Favorites", + "#category": ("", "deviantart", "collection"), + "#class": deviantart.DeviantartCollectionExtractor, + }, + { + "#url": "https://www.deviantart.com/angrywhitewanker/posts/journals/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + "#sha1_url": "48aeed5631763d96f5391d2177ea72d9fdbee4e5", + }, + { + "#url": "https://www.deviantart.com/angrywhitewanker/posts/journals/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + "#options": {"journals": "text"}, + "#sha1_url": "b2a8e74d275664b1a4acee0fca0a6fd33298571e", + }, + { + "#url": "https://www.deviantart.com/angrywhitewanker/posts/journals/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + "#options": {"journals": "none"}, + "#count": 0, + }, + { + "#url": "https://www.deviantart.com/shimoda7/posts/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/journal/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/journal/?catpath=/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + }, + { + "#url": "https://shimoda7.deviantart.com/journal/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + }, + { + "#url": "https://shimoda7.deviantart.com/journal/?catpath=/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + }, + { + "#url": "https://www.deviantart.com/t1na/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class": deviantart.DeviantartStatusExtractor, + "#count": 0, + }, + { + "#url": "https://www.deviantart.com/justgalym/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class": deviantart.DeviantartStatusExtractor, + "#count": 4, + "#sha1_url": "62ee48ff3405c7714dca70abf42e8e39731012fc", + }, + { + "#url": "https://www.deviantart.com/justgalym/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class": deviantart.DeviantartStatusExtractor, + "#options": {"journals": "none"}, + "#pattern": r"https://images-wixmp-\w+\.wixmp\.com/intermediary/f/[^/]+/[^.]+\.jpg", + "#count": 1, + }, + { + "#url": "https://www.deviantart.com/vanillaghosties/posts/statuses", + "#comment": "shared sta.sh item", + "#category": ("", "deviantart", "status"), + "#class": deviantart.DeviantartStatusExtractor, + "#options": { + "journals": "none", + "original": False, + }, + "#range": "5-", + "#count": 1, + "index": int, + "index_base36": r"re:^[0-9a-z]+$", + "url": r"re:^https://www.deviantart.com/stash/\w+", + }, + { + "#url": "https://www.deviantart.com/AndrejSKalin/posts/statuses", + "#comment": "'deleted' deviations in 'items'", + "#category": ("", "deviantart", "status"), + "#class": deviantart.DeviantartStatusExtractor, + "#options": { + "journals": "none", + "original": 0, + "image-filter": "deviationid[:8] == '147C8B03'", + }, + "#count": 2, + "#archive": False, + "deviationid": "147C8B03-7D34-AE93-9241-FA3C6DBBC655", + }, + { + "#url": "https://www.deviantart.com/justgalym/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class": deviantart.DeviantartStatusExtractor, + "#options": {"journals": "text"}, + "#sha1_url": "10a336bdee7b9692919461443a7dde44d495818c", + }, + { + "#url": "https://www.deviantart.com/tag/nature", + "#category": ("", "deviantart", "tag"), + "#class": deviantart.DeviantartTagExtractor, + "#options": {"original": False}, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://www.deviantart.com/watch/deviations", + "#category": ("", "deviantart", "watch"), + "#class": deviantart.DeviantartWatchExtractor, + }, + { + "#url": "https://www.deviantart.com/notifications/watch", + "#category": ("", "deviantart", "watch"), + "#class": deviantart.DeviantartWatchExtractor, + }, + { + "#url": "https://www.deviantart.com/watch/posts", + "#category": ("", "deviantart", "watch-posts"), + "#class": deviantart.DeviantartWatchPostsExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/art/For-the-sake-10073852", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": {"original": 0}, + "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", + }, + { + "#url": "https://www.deviantart.com/shimoda7/art/For-the-sake-10073852", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": {"metadata": "submission,camera,stats"}, + "can_post_comment": False, + "description": str, + "is_watching": False, + "license": "No License", + "stats": { + "comments": int, + "downloads": int, + "downloads_today": int, + "favourites": int, + "views": int, + "views_today": int, + }, + "submission": { + "category": "traditional/drawings/other", + "creation_time": "2004-08-25T02:44:08-0700", + "file_size": "133 KB", + "resolution": "710x510", + "submitted_with": {"app": "Unknown App", "url": ""}, + }, + "tags": [], + }, + { + "#url": "https://www.deviantart.com/zzz/art/zzz-1234567890", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.deviantart.com/justatest235723/art/archive-1103129101", + "#comment": "ZIP archive + preview image (#3782)", + "#class": deviantart.DeviantartDeviationExtractor, + "#options": {"previews": True}, + "#pattern": [ + r"/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/di8rvv1-afe65948-16e1-4eca-b08d-9e6aaa9ed344\.zip", + r"/i/940f2d05-c5eb-4917-8192-7eb6a2d508c6/di8rvv1-bb9d891f-4374-4203-acd3-aea34b29a6a1\.png", + ], + }, + { + "#url": "https://www.deviantart.com/myria-moon/art/Aime-Moi-261986576", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": {"comments": True}, + "#pattern": r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.jpg\?token=.+", + "comments": "len:44", + }, + { + "#url": "https://www.deviantart.com/justatest235723/art/Blue-811519058", + "#comment": "nested comments (#4653)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": { + "original": False, + "comments": True, + }, + "comments": "len:20", + }, + { + "#url": "https://www.deviantart.com/justatest235723/art/Blue-811519058", + "#comment": "comment avatars (#4995)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": { + "original": False, + "comments-avatars": True, + }, + "#range": "5-", + "#pattern": r"^https://www\.deviantart\.com/justatest235723/avatar/$", + "#count": 16, + }, + { + "#url": "https://www.deviantart.com/citizenfresh/art/Hverarond-789295466", + "#comment": "wixmp URL rewrite /intermediary/", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#urls": "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/4deb0f1a-cdef-444e-b194-c8d6b3f7e933/dd1xca2-7f835e62-6fd3-4b99-92c7-2bfd4e1b296f.jpg", + "is_downloadable": False, + "is_original": False, + }, + { + "#url": "https://www.deviantart.com/skatergators/art/COM-Moni-781571783", + "#comment": "GIF (#242)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#pattern": r"https://wixmp-\w+\.wixmp\.com/f/03fd2413-efe9-4e5c-8734-2b72605b3fbb/dcxbsnb-1bbf0b38-42af-4070-8878-f30961955bec\.gif\?token=ey...", + }, + { + "#url": "https://www.deviantart.com/yuumei/art/Flash-Comic-214724929", + "#comment": "Flash animation with GIF preview (#1731)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#pattern": r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.swf\?token=.+", + "filename": "flash_comic_tutorial_by_yuumei-d3juatd", + "extension": "swf", + }, + { + "#url": "https://www.deviantart.com/justatest235723/art/video-1103119114", + "#comment": "video", + "#class": deviantart.DeviantartDeviationExtractor, + "#pattern": r"/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/di8ro5m-e2a5bdf0-daee-4e18-bede-fbfc394d6c65\.mp4\?token=ey", + "filename": "video_63aebdd4bc0323da460796b9a2ac8522_by_justatest235723-di8ro5m", + "extension": "mp4", + }, + { + "#url": "https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498", + "#comment": "sta.sh URLs from description (#302)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": { + "extra": 1, + "original": 0, + }, + "#pattern": deviantart.DeviantartStashExtractor.pattern, + "#range": "2-", + "#count": 4, + }, + { + "#url": "https://www.deviantart.com/cimar-wildehopps/art/Honorary-Vixen-859809305", + "#comment": "sta.sh URL from deviation['text_content']['body']['features']", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": {"extra": 1}, + "#pattern": r"""text: |(?:https?://)?sta\.sh/([a-z0-9]+)""", - "#count" : 2, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/journal/ARTility-583755752", - "#comment" : "journal", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#pattern" : """text:\n""", - "#sha1_url": "37302947642d1e53392ef8ee9b3f473a3c578e7c", -}, - -{ - "#url" : "https://www.deviantart.com/gliitchlord/art/brashstrokes-812942668", - "#comment" : "journal-like post with isJournal == False (#419)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#pattern" : """text:\n""", - "#sha1_url": "8ca1dc8df53d3707c778d08a604f9ad9ddba7469", -}, - -{ - "#url" : "https://www.deviantart.com/stash/09z3557z648", - "#comment" : "sta.sh journal (#6207)", - "#class" : deviantart.DeviantartStashExtractor, - "#pattern" : """text:\n""", -}, - -{ - "#url" : "https://www.deviantart.com/starvinglunatic/art/Against-the-world-chapter-1-50968347", - "#comment" : "literature (#6254)", - "#class" : deviantart.DeviantartDeviationExtractor, - "#pattern" : """text:\n""", -}, - - -{ - "#url" : "https://www.deviantart.com/neotypical/art/985226590", - "#comment" : "subscription locked (#4567)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#count" : 0, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.deviantart.com/colibriworkshop/art/Crimson-Pandaren-Phoenix-World-of-Warcraft-630984457", - "#comment" : "'png' option (#4846)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : {"quality": "png", "intermediary": False}, - "#sha1_content": "75fb92a820b154c061f7e1f9935260577b2365ec", - "#pattern" : r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com" - r"/f/d86d1faa-37a8-4bcb-b421-53331885d763/dafo6q1-5c4c999a-019e-4845-8c29-6fab2d05c8e8\.jpg" - r"/v1/fill/w_1024,h_1297,q_75,strp" - r"/crimson_pandaren_phoenix_world_of_warcraft_by_colibriworkshop_dafo6q1-fullview\.png" - r"\?token=ey.+", - - "extension": "png", -}, - -{ - "#url" : "https://deviantart.com/view/904858796/", - "#comment" : "/view/ URLs", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#sha1_content": "8770ec40ad1c1d60f6b602b16301d124f612948f", -}, - -{ - "#url" : "http://www.deviantart.com/view/890672057", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#sha1_content": "1497e13d925caeb13a250cd666b779a640209236", -}, - -{ - "#url" : "https://www.deviantart.com/view/706871727", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#sha1_content": "4d013515e72dec1e3977c82fd71ce4b15b8bd856", -}, - -{ - "#url" : "https://www.deviantart.com/view/1", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.deviantart.com/deviation/817215762", - "#comment" : "/deviation/ (#3558)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://fav.me/ddijrpu", - "#comment" : "fav.me (#3558)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#count" : 1, -}, - -{ - "#url" : "https://fav.me/dddd", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/art/For-the-sake-of-a-memory-10073852", - "#comment" : "old-style URLs", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://myria-moon.deviantart.com/art/Aime-Moi-part-en-vadrouille-261986576", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://zzz.deviantart.com/art/zzz-1234567890", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/view.php?id=14864502", - "#comment" : "old /view/ URLs from the Wayback Machine", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "http://www.deviantart.com/view-full.php?id=100842", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://www.fxdeviantart.com/zzz/art/zzz-1234567890", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://www.fxdeviantart.com/view/1234567890", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/scraps", - "#category": ("", "deviantart", "scraps"), - "#class" : deviantart.DeviantartScrapsExtractor, - "#count" : 12, -}, - -{ - "#url" : "https://www.deviantart.com/chain-man/gallery/scraps", - "#comment" : "deactivated account", - "#category": ("", "deviantart", "scraps"), - "#class" : deviantart.DeviantartScrapsExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/?catpath=scraps", - "#category": ("", "deviantart", "scraps"), - "#class" : deviantart.DeviantartScrapsExtractor, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/gallery/?catpath=scraps", - "#category": ("", "deviantart", "scraps"), - "#class" : deviantart.DeviantartScrapsExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/search?q=tree", - "#category": ("", "deviantart", "search"), - "#class" : deviantart.DeviantartSearchExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/search/deviations?order=popular-1-week", - "#category": ("", "deviantart", "search"), - "#class" : deviantart.DeviantartSearchExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery?q=memory", - "#category": ("", "deviantart", "gallery-search"), - "#class" : deviantart.DeviantartGallerySearchExtractor, - "#options" : {"original": 0}, - "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery?q=memory&sort=popular", - "#category": ("", "deviantart", "gallery-search"), - "#class" : deviantart.DeviantartGallerySearchExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/about#watching", - "#category": ("", "deviantart", "following"), - "#class" : deviantart.DeviantartFollowingExtractor, - "#pattern" : deviantart.DeviantartUserExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/watching", - "#category": ("", "deviantart", "following"), - "#class" : deviantart.DeviantartFollowingExtractor, -}, - + "#count": 2, + }, + { + "#url": "https://www.deviantart.com/shimoda7/journal/ARTility-583755752", + "#comment": "journal", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#pattern": """text:\n""", + "#sha1_url": "37302947642d1e53392ef8ee9b3f473a3c578e7c", + }, + { + "#url": "https://www.deviantart.com/gliitchlord/art/brashstrokes-812942668", + "#comment": "journal-like post with isJournal == False (#419)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#pattern": """text:\n""", + "#sha1_url": "8ca1dc8df53d3707c778d08a604f9ad9ddba7469", + }, + { + "#url": "https://www.deviantart.com/stash/09z3557z648", + "#comment": "sta.sh journal (#6207)", + "#class": deviantart.DeviantartStashExtractor, + "#pattern": """text:\n""", + }, + { + "#url": "https://www.deviantart.com/starvinglunatic/art/Against-the-world-chapter-1-50968347", + "#comment": "literature (#6254)", + "#class": deviantart.DeviantartDeviationExtractor, + "#pattern": """text:\n""", + }, + { + "#url": "https://www.deviantart.com/neotypical/art/985226590", + "#comment": "subscription locked (#4567)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#count": 0, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.deviantart.com/colibriworkshop/art/Crimson-Pandaren-Phoenix-World-of-Warcraft-630984457", + "#comment": "'png' option (#4846)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": {"quality": "png", "intermediary": False}, + "#sha1_content": "75fb92a820b154c061f7e1f9935260577b2365ec", + "#pattern": r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com" + r"/f/d86d1faa-37a8-4bcb-b421-53331885d763/dafo6q1-5c4c999a-019e-4845-8c29-6fab2d05c8e8\.jpg" + r"/v1/fill/w_1024,h_1297,q_75,strp" + r"/crimson_pandaren_phoenix_world_of_warcraft_by_colibriworkshop_dafo6q1-fullview\.png" + r"\?token=ey.+", + "extension": "png", + }, + { + "#url": "https://deviantart.com/view/904858796/", + "#comment": "/view/ URLs", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#sha1_content": "8770ec40ad1c1d60f6b602b16301d124f612948f", + }, + { + "#url": "http://www.deviantart.com/view/890672057", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#sha1_content": "1497e13d925caeb13a250cd666b779a640209236", + }, + { + "#url": "https://www.deviantart.com/view/706871727", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#sha1_content": "4d013515e72dec1e3977c82fd71ce4b15b8bd856", + }, + { + "#url": "https://www.deviantart.com/view/1", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.deviantart.com/deviation/817215762", + "#comment": "/deviation/ (#3558)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://fav.me/ddijrpu", + "#comment": "fav.me (#3558)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#count": 1, + }, + { + "#url": "https://fav.me/dddd", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://shimoda7.deviantart.com/art/For-the-sake-of-a-memory-10073852", + "#comment": "old-style URLs", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://myria-moon.deviantart.com/art/Aime-Moi-part-en-vadrouille-261986576", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://zzz.deviantart.com/art/zzz-1234567890", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://www.deviantart.com/view.php?id=14864502", + "#comment": "old /view/ URLs from the Wayback Machine", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "http://www.deviantart.com/view-full.php?id=100842", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://www.fxdeviantart.com/zzz/art/zzz-1234567890", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://www.fxdeviantart.com/view/1234567890", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/scraps", + "#category": ("", "deviantart", "scraps"), + "#class": deviantart.DeviantartScrapsExtractor, + "#count": 12, + }, + { + "#url": "https://www.deviantart.com/chain-man/gallery/scraps", + "#comment": "deactivated account", + "#category": ("", "deviantart", "scraps"), + "#class": deviantart.DeviantartScrapsExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/?catpath=scraps", + "#category": ("", "deviantart", "scraps"), + "#class": deviantart.DeviantartScrapsExtractor, + }, + { + "#url": "https://shimoda7.deviantart.com/gallery/?catpath=scraps", + "#category": ("", "deviantart", "scraps"), + "#class": deviantart.DeviantartScrapsExtractor, + }, + { + "#url": "https://www.deviantart.com/search?q=tree", + "#category": ("", "deviantart", "search"), + "#class": deviantart.DeviantartSearchExtractor, + }, + { + "#url": "https://www.deviantart.com/search/deviations?order=popular-1-week", + "#category": ("", "deviantart", "search"), + "#class": deviantart.DeviantartSearchExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery?q=memory", + "#category": ("", "deviantart", "gallery-search"), + "#class": deviantart.DeviantartGallerySearchExtractor, + "#options": {"original": 0}, + "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery?q=memory&sort=popular", + "#category": ("", "deviantart", "gallery-search"), + "#class": deviantart.DeviantartGallerySearchExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/about#watching", + "#category": ("", "deviantart", "following"), + "#class": deviantart.DeviantartFollowingExtractor, + "#pattern": deviantart.DeviantartUserExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://www.deviantart.com/shimoda7/watching", + "#category": ("", "deviantart", "following"), + "#class": deviantart.DeviantartFollowingExtractor, + }, ) diff --git a/test/results/directlink.py b/test/results/directlink.py index 3309cef9a..9995a3f0a 100644 --- a/test/results/directlink.py +++ b/test/results/directlink.py @@ -1,174 +1,164 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import directlink - __tests__ = ( -{ - "#url" : "https://en.wikipedia.org/static/images/project-logos/enwiki.png", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, - "#sha1_url" : "18c5d00077332e98e53be9fed2ee4be66154b88d", - "#sha1_metadata": "105770a3f4393618ab7b811b731b22663b5d3794", - "#sha1_content" : "e6f58aaec8f31eb222f9e10fa9e9f64b79ae888c", -}, - -{ - "#url" : "https://example.org/file.webm", - "#comment" : "empty path", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, - "#sha1_url" : "2d807ed7059d1b532f1bb71dc24b510b80ff943f", - "#sha1_metadata": "29dad729c40fb09349f83edafa498dba1297464a", -}, - -{ - "#url" : "https://example.org/path/to/file.webm?que=1?&ry=2/#fragment", - "#comment" : "more complex example", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, - "#sha1_url" : "6fb1061390f8aada3db01cb24b51797c7ee42b31", - "#sha1_metadata": "3d7abc31d45ba324e59bc599c3b4862452d5f29c", -}, - -{ - "#url" : "https://example.org/%27%3C%23/%23%3E%27.jpg?key=%3C%26%3E", - "#comment" : "percent-encoded characters", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, - "#sha1_url" : "2627e8140727fdf743f86fe18f69f99a052c9718", - "#sha1_metadata": "831790fddda081bdddd14f96985ab02dc5b5341f", -}, - -{ - "#url" : "https://post-phinf.pstatic.net/MjAxOTA1MjlfMTQ4/MDAxNTU5MTI2NjcyNTkw.JUzkGb4V6dj9DXjLclrOoqR64uDxHFUO5KDriRdKpGwg.88mCtd4iT1NHlpVKSCaUpPmZPiDgT8hmQdQ5K_gYyu0g.JPEG/2.JPG", - "#comment" : "upper case file extension (#296)", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, - -{ - "#url" : "https://räksmörgås.josefsson.org/raksmorgas.jpg", - "#comment" : "internationalized domain name", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, - "#sha1_url" : "a65667f670b194afbd1e3ea5e7a78938d36747da", - "#sha1_metadata": "fd5037fe86eebd4764e176cbaf318caec0f700be", -}, - -{ - "#url" : "https://example.org/file.gif", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.bmp", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.svg", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.webp", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.avif", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.heic", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.psd", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.mp4", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.m4v", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.mov", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.mkv", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.ogg", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.ogm", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.ogv", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.wav", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.mp3", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.opus", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.zip", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.rar", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.7z", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.pdf", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.swf", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, - + { + "#url": "https://en.wikipedia.org/static/images/project-logos/enwiki.png", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + "#sha1_url": "18c5d00077332e98e53be9fed2ee4be66154b88d", + "#sha1_metadata": "105770a3f4393618ab7b811b731b22663b5d3794", + "#sha1_content": "e6f58aaec8f31eb222f9e10fa9e9f64b79ae888c", + }, + { + "#url": "https://example.org/file.webm", + "#comment": "empty path", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + "#sha1_url": "2d807ed7059d1b532f1bb71dc24b510b80ff943f", + "#sha1_metadata": "29dad729c40fb09349f83edafa498dba1297464a", + }, + { + "#url": "https://example.org/path/to/file.webm?que=1?&ry=2/#fragment", + "#comment": "more complex example", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + "#sha1_url": "6fb1061390f8aada3db01cb24b51797c7ee42b31", + "#sha1_metadata": "3d7abc31d45ba324e59bc599c3b4862452d5f29c", + }, + { + "#url": "https://example.org/%27%3C%23/%23%3E%27.jpg?key=%3C%26%3E", + "#comment": "percent-encoded characters", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + "#sha1_url": "2627e8140727fdf743f86fe18f69f99a052c9718", + "#sha1_metadata": "831790fddda081bdddd14f96985ab02dc5b5341f", + }, + { + "#url": "https://post-phinf.pstatic.net/MjAxOTA1MjlfMTQ4/MDAxNTU5MTI2NjcyNTkw.JUzkGb4V6dj9DXjLclrOoqR64uDxHFUO5KDriRdKpGwg.88mCtd4iT1NHlpVKSCaUpPmZPiDgT8hmQdQ5K_gYyu0g.JPEG/2.JPG", + "#comment": "upper case file extension (#296)", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://räksmörgås.josefsson.org/raksmorgas.jpg", + "#comment": "internationalized domain name", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + "#sha1_url": "a65667f670b194afbd1e3ea5e7a78938d36747da", + "#sha1_metadata": "fd5037fe86eebd4764e176cbaf318caec0f700be", + }, + { + "#url": "https://example.org/file.gif", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.bmp", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.svg", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.webp", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.avif", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.heic", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.psd", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.mp4", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.m4v", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.mov", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.mkv", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.ogg", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.ogm", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.ogv", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.wav", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.mp3", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.opus", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.zip", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.rar", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.7z", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.pdf", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.swf", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, ) diff --git a/test/results/drawfriends.py b/test/results/drawfriends.py index d481fe70f..aa91eed1b 100644 --- a/test/results/drawfriends.py +++ b/test/results/drawfriends.py @@ -1,29 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v01 - __tests__ = ( -{ - "#url" : "https://drawfriends.booru.org/index.php?page=post&s=list&tags=all", - "#category": ("gelbooru_v01", "drawfriends", "tag"), - "#class" : gelbooru_v01.GelbooruV01TagExtractor, -}, - -{ - "#url" : "https://drawfriends.booru.org/index.php?page=favorites&s=view&id=1", - "#category": ("gelbooru_v01", "drawfriends", "favorite"), - "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, -}, - -{ - "#url" : "https://drawfriends.booru.org/index.php?page=post&s=view&id=107474", - "#category": ("gelbooru_v01", "drawfriends", "post"), - "#class" : gelbooru_v01.GelbooruV01PostExtractor, -}, - + { + "#url": "https://drawfriends.booru.org/index.php?page=post&s=list&tags=all", + "#category": ("gelbooru_v01", "drawfriends", "tag"), + "#class": gelbooru_v01.GelbooruV01TagExtractor, + }, + { + "#url": "https://drawfriends.booru.org/index.php?page=favorites&s=view&id=1", + "#category": ("gelbooru_v01", "drawfriends", "favorite"), + "#class": gelbooru_v01.GelbooruV01FavoriteExtractor, + }, + { + "#url": "https://drawfriends.booru.org/index.php?page=post&s=view&id=107474", + "#category": ("gelbooru_v01", "drawfriends", "post"), + "#class": gelbooru_v01.GelbooruV01PostExtractor, + }, ) diff --git a/test/results/dynastyscans.py b/test/results/dynastyscans.py index 0d4088005..6179a8414 100644 --- a/test/results/dynastyscans.py +++ b/test/results/dynastyscans.py @@ -1,59 +1,50 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import dynastyscans - __tests__ = ( -{ - "#url" : "http://dynasty-scans.com/chapters/hitoribocchi_no_oo_seikatsu_ch33", - "#category": ("", "dynastyscans", "chapter"), - "#class" : dynastyscans.DynastyscansChapterExtractor, - "#sha1_url" : "3cafa527fecec27a66f35e038c0c53e35d5e4317", - "#sha1_metadata": "7b134f2093813d45774cc68a3cd199ffce3e6fd3", -}, - -{ - "#url" : "http://dynasty-scans.com/chapters/new_game_the_spinoff_special_13", - "#category": ("", "dynastyscans", "chapter"), - "#class" : dynastyscans.DynastyscansChapterExtractor, - "#sha1_url" : "047fa6d58f90272883157a80fbf1e6f03ea5bbab", - "#sha1_metadata": "62dc42e9025c79bdd3e26e026a690f4c28548fd4", -}, - -{ - "#url" : "https://dynasty-scans.com/series/hitoribocchi_no_oo_seikatsu", - "#category": ("", "dynastyscans", "manga"), - "#class" : dynastyscans.DynastyscansMangaExtractor, - "#pattern" : dynastyscans.DynastyscansChapterExtractor.pattern, - "#count" : ">= 100", -}, - -{ - "#url" : "https://dynasty-scans.com/images?with[]=4930&with[]=5211", - "#category": ("", "dynastyscans", "search"), - "#class" : dynastyscans.DynastyscansSearchExtractor, - "#sha1_url" : "d2422163db7b1db94bf343f8cd50ba9cc08ae6b5", - "#sha1_metadata": "65f9948e7f29a1db2b3e6abb697f7476d2196708", -}, - -{ - "#url" : "https://dynasty-scans.com/images", - "#category": ("", "dynastyscans", "search"), - "#class" : dynastyscans.DynastyscansSearchExtractor, - "#range" : "1", - "#count" : 1, -}, - -{ - "#url" : "https://dynasty-scans.com/images/1245", - "#category": ("", "dynastyscans", "image"), - "#class" : dynastyscans.DynastyscansImageExtractor, - "#sha1_url" : "877054defac8ea2bbaeb632db176037668c73eea", - "#sha1_metadata": "9f6fd139c372203dcf7237e662a80963ab070cb0", -}, - + { + "#url": "http://dynasty-scans.com/chapters/hitoribocchi_no_oo_seikatsu_ch33", + "#category": ("", "dynastyscans", "chapter"), + "#class": dynastyscans.DynastyscansChapterExtractor, + "#sha1_url": "3cafa527fecec27a66f35e038c0c53e35d5e4317", + "#sha1_metadata": "7b134f2093813d45774cc68a3cd199ffce3e6fd3", + }, + { + "#url": "http://dynasty-scans.com/chapters/new_game_the_spinoff_special_13", + "#category": ("", "dynastyscans", "chapter"), + "#class": dynastyscans.DynastyscansChapterExtractor, + "#sha1_url": "047fa6d58f90272883157a80fbf1e6f03ea5bbab", + "#sha1_metadata": "62dc42e9025c79bdd3e26e026a690f4c28548fd4", + }, + { + "#url": "https://dynasty-scans.com/series/hitoribocchi_no_oo_seikatsu", + "#category": ("", "dynastyscans", "manga"), + "#class": dynastyscans.DynastyscansMangaExtractor, + "#pattern": dynastyscans.DynastyscansChapterExtractor.pattern, + "#count": ">= 100", + }, + { + "#url": "https://dynasty-scans.com/images?with[]=4930&with[]=5211", + "#category": ("", "dynastyscans", "search"), + "#class": dynastyscans.DynastyscansSearchExtractor, + "#sha1_url": "d2422163db7b1db94bf343f8cd50ba9cc08ae6b5", + "#sha1_metadata": "65f9948e7f29a1db2b3e6abb697f7476d2196708", + }, + { + "#url": "https://dynasty-scans.com/images", + "#category": ("", "dynastyscans", "search"), + "#class": dynastyscans.DynastyscansSearchExtractor, + "#range": "1", + "#count": 1, + }, + { + "#url": "https://dynasty-scans.com/images/1245", + "#category": ("", "dynastyscans", "image"), + "#class": dynastyscans.DynastyscansImageExtractor, + "#sha1_url": "877054defac8ea2bbaeb632db176037668c73eea", + "#sha1_metadata": "9f6fd139c372203dcf7237e662a80963ab070cb0", + }, ) diff --git a/test/results/e621.py b/test/results/e621.py index 5cd5c74da..040f8d7cc 100644 --- a/test/results/e621.py +++ b/test/results/e621.py @@ -1,134 +1,117 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import e621 - __tests__ = ( -{ - "#url" : "https://e621.net/posts?tags=anry", - "#category": ("E621", "e621", "tag"), - "#class" : e621.E621TagExtractor, - "#sha1_url" : "8021e5ea28d47c474c1ffc9bd44863c4d45700ba", - "#sha1_content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", -}, - -{ - "#url" : "https://e621.net/post/index/1/anry", - "#category": ("E621", "e621", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e621.net/post?tags=anry", - "#category": ("E621", "e621", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e621.net/pools/73", - "#category": ("E621", "e621", "pool"), - "#class" : e621.E621PoolExtractor, - "#sha1_url" : "1bd09a72715286a79eea3b7f09f51b3493eb579a", - "#sha1_content": "91abe5d5334425d9787811d7f06d34c77974cd22", -}, - -{ - "#url" : "https://e621.net/pool/show/73", - "#category": ("E621", "e621", "pool"), - "#class" : e621.E621PoolExtractor, -}, - -{ - "#url" : "https://e621.net/posts/535", - "#category": ("E621", "e621", "post"), - "#class" : e621.E621PostExtractor, - "#sha1_url" : "f7f78b44c9b88f8f09caac080adc8d6d9fdaa529", - "#sha1_content": "66f46e96a893fba8e694c4e049b23c2acc9af462", - - "date": "dt:2007-02-17 19:02:32", -}, - -{ - "#url" : "https://e621.net/posts/3181052", - "#category": ("E621", "e621", "post"), - "#class" : e621.E621PostExtractor, - "#options" : {"metadata": "notes,pools"}, - "#pattern" : r"https://static\d\.e621\.net/data/c6/8c/c68cca0643890b615f75fb2719589bff\.png", - - "notes": [ - { - "body" : "Little Legends 2", - "created_at" : "2022-05-16T13:58:38.877-04:00", - "creator_id" : 517450, - "creator_name": "EeveeCuddler69", - "height" : 475, - "id" : 321296, - "is_active" : True, - "post_id" : 3181052, - "updated_at" : "2022-05-16T13:59:02.050-04:00", - "version" : 3, - "width" : 809, - "x" : 83, - "y" : 117, - }, - ], - "pools": [ - { - "category" : "series", - "created_at" : "2022-02-17T00:29:22.669-05:00", - "creator_id" : 1077440, - "creator_name": "Yeetus90", - "description" : """\ + { + "#url": "https://e621.net/posts?tags=anry", + "#category": ("E621", "e621", "tag"), + "#class": e621.E621TagExtractor, + "#sha1_url": "8021e5ea28d47c474c1ffc9bd44863c4d45700ba", + "#sha1_content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", + }, + { + "#url": "https://e621.net/post/index/1/anry", + "#category": ("E621", "e621", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e621.net/post?tags=anry", + "#category": ("E621", "e621", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e621.net/pools/73", + "#category": ("E621", "e621", "pool"), + "#class": e621.E621PoolExtractor, + "#sha1_url": "1bd09a72715286a79eea3b7f09f51b3493eb579a", + "#sha1_content": "91abe5d5334425d9787811d7f06d34c77974cd22", + }, + { + "#url": "https://e621.net/pool/show/73", + "#category": ("E621", "e621", "pool"), + "#class": e621.E621PoolExtractor, + }, + { + "#url": "https://e621.net/posts/535", + "#category": ("E621", "e621", "post"), + "#class": e621.E621PostExtractor, + "#sha1_url": "f7f78b44c9b88f8f09caac080adc8d6d9fdaa529", + "#sha1_content": "66f46e96a893fba8e694c4e049b23c2acc9af462", + "date": "dt:2007-02-17 19:02:32", + }, + { + "#url": "https://e621.net/posts/3181052", + "#category": ("E621", "e621", "post"), + "#class": e621.E621PostExtractor, + "#options": {"metadata": "notes,pools"}, + "#pattern": r"https://static\d\.e621\.net/data/c6/8c/c68cca0643890b615f75fb2719589bff\.png", + "notes": [ + { + "body": "Little Legends 2", + "created_at": "2022-05-16T13:58:38.877-04:00", + "creator_id": 517450, + "creator_name": "EeveeCuddler69", + "height": 475, + "id": 321296, + "is_active": True, + "post_id": 3181052, + "updated_at": "2022-05-16T13:59:02.050-04:00", + "version": 3, + "width": 809, + "x": 83, + "y": 117, + }, + ], + "pools": [ + { + "category": "series", + "created_at": "2022-02-17T00:29:22.669-05:00", + "creator_id": 1077440, + "creator_name": "Yeetus90", + "description": """\ * "Little Legends":/pools/27971\r * Little Legends 2\r * "Little Legends 3":/pools/27481\ """, - "id" : 27492, - "is_active" : False, - "name" : "Little Legends 2", - "post_count" : 39, - "post_ids" : list, - "updated_at" : "2022-03-27T06:30:03.382-04:00", - }, - ], -}, - -{ - "#url" : "https://e621.net/post/show/535", - "#category": ("E621", "e621", "post"), - "#class" : e621.E621PostExtractor, -}, - -{ - "#url" : "https://e621.net/explore/posts/popular", - "#category": ("E621", "e621", "popular"), - "#class" : e621.E621PopularExtractor, -}, - -{ - "#url" : "https://e621.net/explore/posts/popular?date=2019-06-01&scale=month", - "#category": ("E621", "e621", "popular"), - "#class" : e621.E621PopularExtractor, - "#pattern" : r"https://static\d.e621.net/data/../../[0-9a-f]+", - "#count" : ">= 70", -}, - -{ - "#url" : "https://e621.net/favorites", - "#category": ("E621", "e621", "favorite"), - "#class" : e621.E621FavoriteExtractor, -}, - -{ - "#url" : "https://e621.net/favorites?page=2&user_id=53275", - "#category": ("E621", "e621", "favorite"), - "#class" : e621.E621FavoriteExtractor, - "#pattern" : r"https://static\d.e621.net/data/../../[0-9a-f]+", - "#count" : "> 260", -}, - + "id": 27492, + "is_active": False, + "name": "Little Legends 2", + "post_count": 39, + "post_ids": list, + "updated_at": "2022-03-27T06:30:03.382-04:00", + }, + ], + }, + { + "#url": "https://e621.net/post/show/535", + "#category": ("E621", "e621", "post"), + "#class": e621.E621PostExtractor, + }, + { + "#url": "https://e621.net/explore/posts/popular", + "#category": ("E621", "e621", "popular"), + "#class": e621.E621PopularExtractor, + }, + { + "#url": "https://e621.net/explore/posts/popular?date=2019-06-01&scale=month", + "#category": ("E621", "e621", "popular"), + "#class": e621.E621PopularExtractor, + "#pattern": r"https://static\d.e621.net/data/../../[0-9a-f]+", + "#count": ">= 70", + }, + { + "#url": "https://e621.net/favorites", + "#category": ("E621", "e621", "favorite"), + "#class": e621.E621FavoriteExtractor, + }, + { + "#url": "https://e621.net/favorites?page=2&user_id=53275", + "#category": ("E621", "e621", "favorite"), + "#class": e621.E621FavoriteExtractor, + "#pattern": r"https://static\d.e621.net/data/../../[0-9a-f]+", + "#count": "> 260", + }, ) diff --git a/test/results/e6ai.py b/test/results/e6ai.py index 291f3b3a2..6e1122c5c 100644 --- a/test/results/e6ai.py +++ b/test/results/e6ai.py @@ -1,68 +1,56 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import e621 - __tests__ = ( -{ - "#url" : "https://e6ai.net/posts?tags=anry", - "#category": ("E621", "e6ai", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e6ai.net/post/index/1/anry", - "#category": ("E621", "e6ai", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e6ai.net/post?tags=anry", - "#category": ("E621", "e6ai", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e6ai.net/pools/3", - "#category": ("E621", "e6ai", "pool"), - "#class" : e621.E621PoolExtractor, - "#sha1_url": "a6d1ad67a3fa9b9f73731d34d5f6f26f7e85855f", -}, - -{ - "#url" : "https://e6ai.net/pool/show/3", - "#category": ("E621", "e6ai", "pool"), - "#class" : e621.E621PoolExtractor, -}, - -{ - "#url" : "https://e6ai.net/posts/23", - "#category": ("E621", "e6ai", "post"), - "#class" : e621.E621PostExtractor, - "#sha1_url" : "3c85a806b3d9eec861948af421fe0e8ad6b8f881", - "#sha1_content": "a05a484e4eb64637d56d751c02e659b4bc8ea5d5", -}, - -{ - "#url" : "https://e6ai.net/post/show/23", - "#category": ("E621", "e6ai", "post"), - "#class" : e621.E621PostExtractor, -}, - -{ - "#url" : "https://e6ai.net/explore/posts/popular", - "#category": ("E621", "e6ai", "popular"), - "#class" : e621.E621PopularExtractor, -}, - -{ - "#url" : "https://e6ai.net/favorites", - "#category": ("E621", "e6ai", "favorite"), - "#class" : e621.E621FavoriteExtractor, -}, - + { + "#url": "https://e6ai.net/posts?tags=anry", + "#category": ("E621", "e6ai", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e6ai.net/post/index/1/anry", + "#category": ("E621", "e6ai", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e6ai.net/post?tags=anry", + "#category": ("E621", "e6ai", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e6ai.net/pools/3", + "#category": ("E621", "e6ai", "pool"), + "#class": e621.E621PoolExtractor, + "#sha1_url": "a6d1ad67a3fa9b9f73731d34d5f6f26f7e85855f", + }, + { + "#url": "https://e6ai.net/pool/show/3", + "#category": ("E621", "e6ai", "pool"), + "#class": e621.E621PoolExtractor, + }, + { + "#url": "https://e6ai.net/posts/23", + "#category": ("E621", "e6ai", "post"), + "#class": e621.E621PostExtractor, + "#sha1_url": "3c85a806b3d9eec861948af421fe0e8ad6b8f881", + "#sha1_content": "a05a484e4eb64637d56d751c02e659b4bc8ea5d5", + }, + { + "#url": "https://e6ai.net/post/show/23", + "#category": ("E621", "e6ai", "post"), + "#class": e621.E621PostExtractor, + }, + { + "#url": "https://e6ai.net/explore/posts/popular", + "#category": ("E621", "e6ai", "popular"), + "#class": e621.E621PopularExtractor, + }, + { + "#url": "https://e6ai.net/favorites", + "#category": ("E621", "e6ai", "favorite"), + "#class": e621.E621FavoriteExtractor, + }, ) diff --git a/test/results/e926.py b/test/results/e926.py index b046b459b..c2cd32f02 100644 --- a/test/results/e926.py +++ b/test/results/e926.py @@ -1,87 +1,73 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import e621 - __tests__ = ( -{ - "#url" : "https://e926.net/posts?tags=anry", - "#category": ("E621", "e926", "tag"), - "#class" : e621.E621TagExtractor, - "#sha1_url" : "12198b275c62ffe2de67cca676c8e64de80c425d", - "#sha1_content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", -}, - -{ - "#url" : "https://e926.net/post/index/1/anry", - "#category": ("E621", "e926", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e926.net/post?tags=anry", - "#category": ("E621", "e926", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e926.net/pools/73", - "#category": ("E621", "e926", "pool"), - "#class" : e621.E621PoolExtractor, - "#sha1_url" : "6936f1b6a18c5c25bee7cad700088dbc2503481b", - "#sha1_content": "91abe5d5334425d9787811d7f06d34c77974cd22", -}, - -{ - "#url" : "https://e926.net/pool/show/73", - "#category": ("E621", "e926", "pool"), - "#class" : e621.E621PoolExtractor, -}, - -{ - "#url" : "https://e926.net/posts/535", - "#category": ("E621", "e926", "post"), - "#class" : e621.E621PostExtractor, - "#sha1_url" : "17aec8ebd8fab098d321adcb62a2db59dab1f4bf", - "#sha1_content": "66f46e96a893fba8e694c4e049b23c2acc9af462", -}, - -{ - "#url" : "https://e926.net/post/show/535", - "#category": ("E621", "e926", "post"), - "#class" : e621.E621PostExtractor, -}, - -{ - "#url" : "https://e926.net/explore/posts/popular", - "#category": ("E621", "e926", "popular"), - "#class" : e621.E621PopularExtractor, -}, - -{ - "#url" : "https://e926.net/explore/posts/popular?date=2019-06-01&scale=month", - "#category": ("E621", "e926", "popular"), - "#class" : e621.E621PopularExtractor, - "#pattern" : r"https://static\d.e926.net/data/../../[0-9a-f]+", - "#count" : ">= 70", -}, - -{ - "#url" : "https://e926.net/favorites", - "#category": ("E621", "e926", "favorite"), - "#class" : e621.E621FavoriteExtractor, -}, - -{ - "#url" : "https://e926.net/favorites?page=2&user_id=53275", - "#category": ("E621", "e926", "favorite"), - "#class" : e621.E621FavoriteExtractor, - "#pattern" : r"https://static\d.e926.net/data/../../[0-9a-f]+", - "#count" : "> 260", -}, - + { + "#url": "https://e926.net/posts?tags=anry", + "#category": ("E621", "e926", "tag"), + "#class": e621.E621TagExtractor, + "#sha1_url": "12198b275c62ffe2de67cca676c8e64de80c425d", + "#sha1_content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", + }, + { + "#url": "https://e926.net/post/index/1/anry", + "#category": ("E621", "e926", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e926.net/post?tags=anry", + "#category": ("E621", "e926", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e926.net/pools/73", + "#category": ("E621", "e926", "pool"), + "#class": e621.E621PoolExtractor, + "#sha1_url": "6936f1b6a18c5c25bee7cad700088dbc2503481b", + "#sha1_content": "91abe5d5334425d9787811d7f06d34c77974cd22", + }, + { + "#url": "https://e926.net/pool/show/73", + "#category": ("E621", "e926", "pool"), + "#class": e621.E621PoolExtractor, + }, + { + "#url": "https://e926.net/posts/535", + "#category": ("E621", "e926", "post"), + "#class": e621.E621PostExtractor, + "#sha1_url": "17aec8ebd8fab098d321adcb62a2db59dab1f4bf", + "#sha1_content": "66f46e96a893fba8e694c4e049b23c2acc9af462", + }, + { + "#url": "https://e926.net/post/show/535", + "#category": ("E621", "e926", "post"), + "#class": e621.E621PostExtractor, + }, + { + "#url": "https://e926.net/explore/posts/popular", + "#category": ("E621", "e926", "popular"), + "#class": e621.E621PopularExtractor, + }, + { + "#url": "https://e926.net/explore/posts/popular?date=2019-06-01&scale=month", + "#category": ("E621", "e926", "popular"), + "#class": e621.E621PopularExtractor, + "#pattern": r"https://static\d.e926.net/data/../../[0-9a-f]+", + "#count": ">= 70", + }, + { + "#url": "https://e926.net/favorites", + "#category": ("E621", "e926", "favorite"), + "#class": e621.E621FavoriteExtractor, + }, + { + "#url": "https://e926.net/favorites?page=2&user_id=53275", + "#category": ("E621", "e926", "favorite"), + "#class": e621.E621FavoriteExtractor, + "#pattern": r"https://static\d.e926.net/data/../../[0-9a-f]+", + "#count": "> 260", + }, ) diff --git a/test/results/endchan.py b/test/results/endchan.py index 97d34c3bc..de17d442e 100644 --- a/test/results/endchan.py +++ b/test/results/endchan.py @@ -1,38 +1,31 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import lynxchan - __tests__ = ( -{ - "#url" : "https://endchan.org/yuri/res/33621.html", - "#category": ("lynxchan", "endchan", "thread"), - "#class" : lynxchan.LynxchanThreadExtractor, - "#urls" : "https://endchan.org/.media/358c089df4be990e9f7b636e1ce83d3e-imagejpeg.jpg", -}, - -{ - "#url" : "https://endchan.org/yuri/res/33621.html", - "#category": ("lynxchan", "endchan", "thread"), - "#class" : lynxchan.LynxchanThreadExtractor, -}, - -{ - "#url" : "https://endchan.org/yuri/", - "#category": ("lynxchan", "endchan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, - "#pattern" : lynxchan.LynxchanThreadExtractor.pattern, - "#count" : ">= 8", -}, - -{ - "#url" : "https://endchan.org/yuri/catalog.html", - "#category": ("lynxchan", "endchan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, -}, - + { + "#url": "https://endchan.org/yuri/res/33621.html", + "#category": ("lynxchan", "endchan", "thread"), + "#class": lynxchan.LynxchanThreadExtractor, + "#urls": "https://endchan.org/.media/358c089df4be990e9f7b636e1ce83d3e-imagejpeg.jpg", + }, + { + "#url": "https://endchan.org/yuri/res/33621.html", + "#category": ("lynxchan", "endchan", "thread"), + "#class": lynxchan.LynxchanThreadExtractor, + }, + { + "#url": "https://endchan.org/yuri/", + "#category": ("lynxchan", "endchan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + "#pattern": lynxchan.LynxchanThreadExtractor.pattern, + "#count": ">= 8", + }, + { + "#url": "https://endchan.org/yuri/catalog.html", + "#category": ("lynxchan", "endchan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + }, ) diff --git a/test/results/erome.py b/test/results/erome.py index 3624fe177..9a411897d 100644 --- a/test/results/erome.py +++ b/test/results/erome.py @@ -1,57 +1,48 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import erome - __tests__ = ( -{ - "#url" : "https://www.erome.com/a/NQgdlWvk", - "#category": ("", "erome", "album"), - "#class" : erome.EromeAlbumExtractor, - "#pattern" : r"https://v\d+\.erome\.com/\d+/NQgdlWvk/j7jlzmYB_480p\.mp4", - "#count" : 1, - - "album_id": "NQgdlWvk", - "date" : None, - "count" : 1, - "num" : 1, - "title" : "porn", - "user" : "yYgWBZw8o8qsMzM", -}, - -{ - "#url" : "https://www.erome.com/a/TdbZ4ogi", - "#category": ("", "erome", "album"), - "#class" : erome.EromeAlbumExtractor, - "#pattern" : r"https://s\d+\.erome\.com/\d+/TdbZ4ogi/\w+", - "#count" : 6, - - "album_id": "TdbZ4ogi", - "date" : "dt:2024-03-18 00:01:56", - "count" : 6, - "num" : int, - "title" : "82e78cfbb461ad87198f927fcb1fda9a1efac9ff.", - "user" : "yYgWBZw8o8qsMzM", -}, - -{ - "#url" : "https://www.erome.com/yYgWBZw8o8qsMzM", - "#category": ("", "erome", "user"), - "#class" : erome.EromeUserExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://www.erome.com/search?q=cute", - "#category": ("", "erome", "search"), - "#class" : erome.EromeSearchExtractor, - "#range" : "1-25", - "#count" : 25, -}, - + { + "#url": "https://www.erome.com/a/NQgdlWvk", + "#category": ("", "erome", "album"), + "#class": erome.EromeAlbumExtractor, + "#pattern": r"https://v\d+\.erome\.com/\d+/NQgdlWvk/j7jlzmYB_480p\.mp4", + "#count": 1, + "album_id": "NQgdlWvk", + "date": None, + "count": 1, + "num": 1, + "title": "porn", + "user": "yYgWBZw8o8qsMzM", + }, + { + "#url": "https://www.erome.com/a/TdbZ4ogi", + "#category": ("", "erome", "album"), + "#class": erome.EromeAlbumExtractor, + "#pattern": r"https://s\d+\.erome\.com/\d+/TdbZ4ogi/\w+", + "#count": 6, + "album_id": "TdbZ4ogi", + "date": "dt:2024-03-18 00:01:56", + "count": 6, + "num": int, + "title": "82e78cfbb461ad87198f927fcb1fda9a1efac9ff.", + "user": "yYgWBZw8o8qsMzM", + }, + { + "#url": "https://www.erome.com/yYgWBZw8o8qsMzM", + "#category": ("", "erome", "user"), + "#class": erome.EromeUserExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://www.erome.com/search?q=cute", + "#category": ("", "erome", "search"), + "#class": erome.EromeSearchExtractor, + "#range": "1-25", + "#count": 25, + }, ) diff --git a/test/results/everia.py b/test/results/everia.py index af22be000..77fa1b802 100644 --- a/test/results/everia.py +++ b/test/results/everia.py @@ -1,51 +1,42 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import everia - __tests__ = ( -{ - "#url" : "https://everia.club/2024/09/23/mikacho-조미카-joapictures-someday/", - "#class": everia.EveriaPostExtractor, - "#count": 32, - - "title" : "Mikacho 조미카, JOApictures ‘Someday’", - "post_category": "Korea", - "tags" : ["[JOApictures]", "Mikacho 조미카"] -}, - -{ - "#url" : "https://everia.club/tag/miku-tanaka-%e7%94%b0%e4%b8%ad%e7%be%8e%e4%b9%85/", - "#class" : everia.EveriaTagExtractor, - "#pattern": everia.EveriaPostExtractor.pattern, - "#count" : "> 50", -}, - -{ - "#url" : "https://everia.club/category/japan/", - "#class" : everia.EveriaCategoryExtractor, - "#pattern": everia.EveriaPostExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://everia.club/2023/10/05/", - "#class" : everia.EveriaDateExtractor, - "#pattern": everia.EveriaPostExtractor.pattern, - "#count" : 34, -}, - -{ - "#url" : "https://everia.club/?s=saika", - "#class" : everia.EveriaSearchExtractor, - "#pattern": everia.EveriaPostExtractor.pattern, - "#range" : "1-15", - "#count" : 15, -}, - + { + "#url": "https://everia.club/2024/09/23/mikacho-조미카-joapictures-someday/", + "#class": everia.EveriaPostExtractor, + "#count": 32, + "title": "Mikacho 조미카, JOApictures ‘Someday’", + "post_category": "Korea", + "tags": ["[JOApictures]", "Mikacho 조미카"], + }, + { + "#url": "https://everia.club/tag/miku-tanaka-%e7%94%b0%e4%b8%ad%e7%be%8e%e4%b9%85/", + "#class": everia.EveriaTagExtractor, + "#pattern": everia.EveriaPostExtractor.pattern, + "#count": "> 50", + }, + { + "#url": "https://everia.club/category/japan/", + "#class": everia.EveriaCategoryExtractor, + "#pattern": everia.EveriaPostExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://everia.club/2023/10/05/", + "#class": everia.EveriaDateExtractor, + "#pattern": everia.EveriaPostExtractor.pattern, + "#count": 34, + }, + { + "#url": "https://everia.club/?s=saika", + "#class": everia.EveriaSearchExtractor, + "#pattern": everia.EveriaPostExtractor.pattern, + "#range": "1-15", + "#count": 15, + }, ) diff --git a/test/results/exhentai.py b/test/results/exhentai.py index df5935738..507bf6b70 100644 --- a/test/results/exhentai.py +++ b/test/results/exhentai.py @@ -1,137 +1,120 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import exhentai from gallery_dl import exception - +from gallery_dl.extractor import exhentai __tests__ = ( -{ - "#url" : "https://exhentai.org/g/1200119/d55c44d3d0/", - "#category": ("", "exhentai", "gallery"), - "#class" : exhentai.ExhentaiGalleryExtractor, - "#options" : {"original": False, "tags": True}, - "#sha1_content": [ - "2c68cff8a7ca540a78c36fdbf5fbae0260484f87", - "e9891a4c017ed0bb734cd1efba5cd03f594d31ff", - ], - - "cost" : int, - "date" : "dt:2018-03-18 20:14:00", - "eh_category" : "Non-H", - "expunged" : False, - "favorites" : r"re:^[12]\d$", - "filecount" : "4", - "filesize" : 1488978, - "gid" : 1200119, - "height" : int, - "image_token" : r"re:[0-9a-f]{10}", - "lang" : "ja", - "language" : "Japanese", - "parent" : "", - "rating" : r"re:\d\.\d+", - "size" : int, - "tags" : [ - "parody:komi-san wa komyushou desu.", - "character:shouko komi", - "group:seventh lowlife", - "other:sample", - ], - "tags_parody" : ["komi-san wa komyushou desu."], - "tags_character": ["shouko komi"], - "tags_group" : ["seventh lowlife"], - "tags_other" : ["sample"], - "thumb" : "https://s.exhentai.org/t/ce/0a/ce0a5bcb583229a9b07c0f83bcb1630ab1350640-624622-736-1036-jpg_250.jpg", - "title" : "C93 [Seventh_Lowlife] Komi-san ha Tokidoki Daitan desu (Komi-san wa Komyushou desu) [Sample]", - "title_jpn" : "(C93) [Comiketjack (わ！)] 古見さんは、時々大胆です。 (古見さんは、コミュ症です。) [見本]", - "token" : "d55c44d3d0", - "torrentcount": "0", - "uploader" : "klorpa", - "width" : int, -}, - -{ - "#url" : "https://exhentai.org/g/960461/4f0e369d82/", - "#category": ("", "exhentai", "gallery"), - "#class" : exhentai.ExhentaiGalleryExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "http://exhentai.org/g/962698/7f02358e00/", - "#category": ("", "exhentai", "gallery"), - "#class" : exhentai.ExhentaiGalleryExtractor, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://exhentai.org/s/f68367b4c8/1200119-3", - "#category": ("", "exhentai", "gallery"), - "#class" : exhentai.ExhentaiGalleryExtractor, - "#options" : {"original": False}, - "#count" : 2, -}, - -{ - "#url" : "https://e-hentai.org/s/f68367b4c8/1200119-3", - "#category": ("", "exhentai", "gallery"), - "#class" : exhentai.ExhentaiGalleryExtractor, - "#options" : {"original": False}, - "#count" : 2, -}, - -{ - "#url" : "https://g.e-hentai.org/g/1200119/d55c44d3d0/", - "#category": ("", "exhentai", "gallery"), - "#class" : exhentai.ExhentaiGalleryExtractor, -}, - -{ - "#url" : "https://e-hentai.org/?f_search=touhou", - "#category": ("", "exhentai", "search"), - "#class" : exhentai.ExhentaiSearchExtractor, -}, - -{ - "#url" : "https://exhentai.org/?f_cats=767&f_search=touhou", - "#category": ("", "exhentai", "search"), - "#class" : exhentai.ExhentaiSearchExtractor, -}, - -{ - "#url" : "https://exhentai.org/tag/parody:touhou+project", - "#category": ("", "exhentai", "search"), - "#class" : exhentai.ExhentaiSearchExtractor, -}, - -{ - "#url" : "https://exhentai.org/?f_doujinshi=0&f_manga=0&f_artistcg=0&f_gamecg=0&f_western=0&f_non-h=1&f_imageset=0&f_cosplay=0&f_asianporn=0&f_misc=0&f_search=touhou&f_apply=Apply+Filter", - "#category": ("", "exhentai", "search"), - "#class" : exhentai.ExhentaiSearchExtractor, - "#pattern" : exhentai.ExhentaiGalleryExtractor.pattern, - "#auth" : True, - "#range" : "1-30", - "#count" : 30, - - "gallery_id" : int, - "gallery_token": r"re:^[0-9a-f]{10}$", -}, - -{ - "#url" : "https://e-hentai.org/favorites.php", - "#category": ("", "exhentai", "favorite"), - "#class" : exhentai.ExhentaiFavoriteExtractor, - "#auth" : True, - "#urls" : "https://e-hentai.org/g/1200119/d55c44d3d0/", -}, - -{ - "#url" : "https://exhentai.org/favorites.php?favcat=1&f_search=touhou&f_apply=Search+Favorites", - "#category": ("", "exhentai", "favorite"), - "#class" : exhentai.ExhentaiFavoriteExtractor, -}, - + { + "#url": "https://exhentai.org/g/1200119/d55c44d3d0/", + "#category": ("", "exhentai", "gallery"), + "#class": exhentai.ExhentaiGalleryExtractor, + "#options": {"original": False, "tags": True}, + "#sha1_content": [ + "2c68cff8a7ca540a78c36fdbf5fbae0260484f87", + "e9891a4c017ed0bb734cd1efba5cd03f594d31ff", + ], + "cost": int, + "date": "dt:2018-03-18 20:14:00", + "eh_category": "Non-H", + "expunged": False, + "favorites": r"re:^[12]\d$", + "filecount": "4", + "filesize": 1488978, + "gid": 1200119, + "height": int, + "image_token": r"re:[0-9a-f]{10}", + "lang": "ja", + "language": "Japanese", + "parent": "", + "rating": r"re:\d\.\d+", + "size": int, + "tags": [ + "parody:komi-san wa komyushou desu.", + "character:shouko komi", + "group:seventh lowlife", + "other:sample", + ], + "tags_parody": ["komi-san wa komyushou desu."], + "tags_character": ["shouko komi"], + "tags_group": ["seventh lowlife"], + "tags_other": ["sample"], + "thumb": "https://s.exhentai.org/t/ce/0a/ce0a5bcb583229a9b07c0f83bcb1630ab1350640-624622-736-1036-jpg_250.jpg", + "title": "C93 [Seventh_Lowlife] Komi-san ha Tokidoki Daitan desu (Komi-san wa Komyushou desu) [Sample]", + "title_jpn": "(C93) [Comiketjack (わ！)] 古見さんは、時々大胆です。 (古見さんは、コミュ症です。) [見本]", + "token": "d55c44d3d0", + "torrentcount": "0", + "uploader": "klorpa", + "width": int, + }, + { + "#url": "https://exhentai.org/g/960461/4f0e369d82/", + "#category": ("", "exhentai", "gallery"), + "#class": exhentai.ExhentaiGalleryExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "http://exhentai.org/g/962698/7f02358e00/", + "#category": ("", "exhentai", "gallery"), + "#class": exhentai.ExhentaiGalleryExtractor, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://exhentai.org/s/f68367b4c8/1200119-3", + "#category": ("", "exhentai", "gallery"), + "#class": exhentai.ExhentaiGalleryExtractor, + "#options": {"original": False}, + "#count": 2, + }, + { + "#url": "https://e-hentai.org/s/f68367b4c8/1200119-3", + "#category": ("", "exhentai", "gallery"), + "#class": exhentai.ExhentaiGalleryExtractor, + "#options": {"original": False}, + "#count": 2, + }, + { + "#url": "https://g.e-hentai.org/g/1200119/d55c44d3d0/", + "#category": ("", "exhentai", "gallery"), + "#class": exhentai.ExhentaiGalleryExtractor, + }, + { + "#url": "https://e-hentai.org/?f_search=touhou", + "#category": ("", "exhentai", "search"), + "#class": exhentai.ExhentaiSearchExtractor, + }, + { + "#url": "https://exhentai.org/?f_cats=767&f_search=touhou", + "#category": ("", "exhentai", "search"), + "#class": exhentai.ExhentaiSearchExtractor, + }, + { + "#url": "https://exhentai.org/tag/parody:touhou+project", + "#category": ("", "exhentai", "search"), + "#class": exhentai.ExhentaiSearchExtractor, + }, + { + "#url": "https://exhentai.org/?f_doujinshi=0&f_manga=0&f_artistcg=0&f_gamecg=0&f_western=0&f_non-h=1&f_imageset=0&f_cosplay=0&f_asianporn=0&f_misc=0&f_search=touhou&f_apply=Apply+Filter", + "#category": ("", "exhentai", "search"), + "#class": exhentai.ExhentaiSearchExtractor, + "#pattern": exhentai.ExhentaiGalleryExtractor.pattern, + "#auth": True, + "#range": "1-30", + "#count": 30, + "gallery_id": int, + "gallery_token": r"re:^[0-9a-f]{10}$", + }, + { + "#url": "https://e-hentai.org/favorites.php", + "#category": ("", "exhentai", "favorite"), + "#class": exhentai.ExhentaiFavoriteExtractor, + "#auth": True, + "#urls": "https://e-hentai.org/g/1200119/d55c44d3d0/", + }, + { + "#url": "https://exhentai.org/favorites.php?favcat=1&f_search=touhou&f_apply=Search+Favorites", + "#category": ("", "exhentai", "favorite"), + "#class": exhentai.ExhentaiFavoriteExtractor, + }, ) diff --git a/test/results/fanbox.py b/test/results/fanbox.py index f11747f23..5cf8b551c 100644 --- a/test/results/fanbox.py +++ b/test/results/fanbox.py @@ -1,183 +1,158 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import fanbox - __tests__ = ( -{ - "#url" : "https://xub.fanbox.cc", - "#category": ("", "fanbox", "creator"), - "#class" : fanbox.FanboxCreatorExtractor, - "#range" : "1-15", - "#count" : ">= 15", - - "creatorId": "xub", - "tags" : list, - "title" : str, -}, - -{ - "#url" : "https://xub.fanbox.cc/posts", - "#category": ("", "fanbox", "creator"), - "#class" : fanbox.FanboxCreatorExtractor, -}, - -{ - "#url" : "https://www.fanbox.cc/@xub/", - "#category": ("", "fanbox", "creator"), - "#class" : fanbox.FanboxCreatorExtractor, -}, - -{ - "#url" : "https://www.fanbox.cc/@xub/posts", - "#category": ("", "fanbox", "creator"), - "#class" : fanbox.FanboxCreatorExtractor, -}, - -{ - "#url" : "https://www.fanbox.cc/@xub/posts/1910054", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - "#count" : 3, - - "title" : "えま★おうがすと", - "tags" : list, - "hasAdultContent": True, - "isCoverImage" : False, -}, - -{ - "#url" : "https://nekoworks.fanbox.cc/posts/915", - "#comment" : "entry post type, image embedded in html of the post", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - "#count" : 2, - - "title" : "【SAYORI FAN CLUB】お届け内容", - "tags" : list, - "html" : str, - "hasAdultContent": True, -}, - -{ - "#url" : "https://steelwire.fanbox.cc/posts/285502", - "#comment" : "article post type, imageMap, 2 twitter embeds, fanbox embed", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - "#options" : {"embeds": True}, - "#count" : 8, - - "title" : "イラスト+SS｜【全体公開版】義足の探鉱夫男子が義足を見せてくれるだけ ", - "tags" : list, - "articleBody" : dict, - "hasAdultContent": True, -}, - -{ - "#url" : "https://www.fanbox.cc/@official-en/posts/4326303", - "#comment" : "'content' metadata (#3020)", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - - "content": r"re:(?s)^Greetings from FANBOX.\n \nAs of Monday, September 5th, 2022, we are happy to announce the start of the FANBOX hashtag event #MySetupTour ! \nAbout the event\nTo join this event .+ \nPlease check this page for further details regarding the Privacy & Terms.\nhttps://fanbox.pixiv.help/.+/10184952456601\n\n\nThank you for your continued support of FANBOX.$", -}, - -{ - "#url" : "https://official-en.fanbox.cc/posts/7022572", - "#comment" : "'plan' and 'user' metadata (#4921)", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - "#options" : {"metadata": True}, - - "plan": { - "coverImageUrl" : "", - "creatorId" : "official-en", - "description" : "", - "fee" : 0, - "hasAdultContent": None, - "id" : "", - "paymentMethod" : None, - "title" : "", + { + "#url": "https://xub.fanbox.cc", + "#category": ("", "fanbox", "creator"), + "#class": fanbox.FanboxCreatorExtractor, + "#range": "1-15", + "#count": ">= 15", + "creatorId": "xub", + "tags": list, + "title": str, + }, + { + "#url": "https://xub.fanbox.cc/posts", + "#category": ("", "fanbox", "creator"), + "#class": fanbox.FanboxCreatorExtractor, + }, + { + "#url": "https://www.fanbox.cc/@xub/", + "#category": ("", "fanbox", "creator"), + "#class": fanbox.FanboxCreatorExtractor, }, - "user": { - "coverImageUrl" : "https://pixiv.pximg.net/c/1620x580_90_a2_g5/fanbox/public/images/creator/74349833/cover/n9mX8q4tUXHXXj7sK1RPWyUu.jpeg", - "creatorId" : "official-en", - "description" : "This is the official English pixivFANBOX account! \n(official Japanese account: https://official.fanbox.cc/ )\n\npixivFANBOX is a subscription service for building a reliable fan community where creators can nurture creative lifestyles together with their fans.\nFollowers can be notified of the updates from their favorite creators they are following. Supporters can enjoy closer communication with creators through exclusive content and their latest information.\n", - "hasAdultContent" : False, - "hasBoothShop" : False, - "iconUrl" : "https://pixiv.pximg.net/c/160x160_90_a2_g5/fanbox/public/images/user/74349833/icon/oJH0OoGoSixLrJXlnneNvC95.jpeg", - "isAcceptingRequest": False, - "isFollowed" : False, - "isStopped" : False, - "isSupported" : False, - "name" : "pixivFANBOX English", - "profileItems" : [], - "profileLinks" : [ - "https://twitter.com/pixivfanbox", + { + "#url": "https://www.fanbox.cc/@xub/posts", + "#category": ("", "fanbox", "creator"), + "#class": fanbox.FanboxCreatorExtractor, + }, + { + "#url": "https://www.fanbox.cc/@xub/posts/1910054", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "#count": 3, + "title": "えま★おうがすと", + "tags": list, + "hasAdultContent": True, + "isCoverImage": False, + }, + { + "#url": "https://nekoworks.fanbox.cc/posts/915", + "#comment": "entry post type, image embedded in html of the post", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "#count": 2, + "title": "【SAYORI FAN CLUB】お届け内容", + "tags": list, + "html": str, + "hasAdultContent": True, + }, + { + "#url": "https://steelwire.fanbox.cc/posts/285502", + "#comment": "article post type, imageMap, 2 twitter embeds, fanbox embed", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "#options": {"embeds": True}, + "#count": 8, + "title": "イラスト+SS｜【全体公開版】義足の探鉱夫男子が義足を見せてくれるだけ ", + "tags": list, + "articleBody": dict, + "hasAdultContent": True, + }, + { + "#url": "https://www.fanbox.cc/@official-en/posts/4326303", + "#comment": "'content' metadata (#3020)", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "content": r"re:(?s)^Greetings from FANBOX.\n \nAs of Monday, September 5th, 2022, we are happy to announce the start of the FANBOX hashtag event #MySetupTour ! \nAbout the event\nTo join this event .+ \nPlease check this page for further details regarding the Privacy & Terms.\nhttps://fanbox.pixiv.help/.+/10184952456601\n\n\nThank you for your continued support of FANBOX.$", + }, + { + "#url": "https://official-en.fanbox.cc/posts/7022572", + "#comment": "'plan' and 'user' metadata (#4921)", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "#options": {"metadata": True}, + "plan": { + "coverImageUrl": "", + "creatorId": "official-en", + "description": "", + "fee": 0, + "hasAdultContent": None, + "id": "", + "paymentMethod": None, + "title": "", + }, + "user": { + "coverImageUrl": "https://pixiv.pximg.net/c/1620x580_90_a2_g5/fanbox/public/images/creator/74349833/cover/n9mX8q4tUXHXXj7sK1RPWyUu.jpeg", + "creatorId": "official-en", + "description": "This is the official English pixivFANBOX account! \n(official Japanese account: https://official.fanbox.cc/ )\n\npixivFANBOX is a subscription service for building a reliable fan community where creators can nurture creative lifestyles together with their fans.\nFollowers can be notified of the updates from their favorite creators they are following. Supporters can enjoy closer communication with creators through exclusive content and their latest information.\n", + "hasAdultContent": False, + "hasBoothShop": False, + "iconUrl": "https://pixiv.pximg.net/c/160x160_90_a2_g5/fanbox/public/images/user/74349833/icon/oJH0OoGoSixLrJXlnneNvC95.jpeg", + "isAcceptingRequest": False, + "isFollowed": False, + "isStopped": False, + "isSupported": False, + "name": "pixivFANBOX English", + "profileItems": [], + "profileLinks": [ + "https://twitter.com/pixivfanbox", + ], + "userId": "74349833", + }, + }, + { + "#url": "https://saki9184.fanbox.cc/posts/7754760", + "#comment": "missing plan for exact 'feeRequired' value (#5759)", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "#options": {"metadata": "plan"}, + "feeRequired": 300, + "plan": { + "creatorId": "saki9184", + "fee": 350, + "id": "414274", + "title": "入り浸りJKハルヒ", + }, + }, + { + "#url": "https://mochirong.fanbox.cc/posts/3746116", + "#comment": "imageMap file order (#2718); comments metadata (#6287)", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "#options": {"metadata": "comments"}, + "#sha1_url": "c92ddd06f2efc4a5fe30ec67e21544f79a5c4062", + "#urls": [ + "https://pixiv.pximg.net/fanbox/public/images/post/3746116/cover/6h5w7F1MJWLeED6ODfHo6ZYQ.jpeg", + "https://downloads.fanbox.cc/images/post/3746116/ouTz7XZIeVD3FBOzoLhJ3ZTA.jpeg", + "https://downloads.fanbox.cc/images/post/3746116/hBs9bXEg6HvbqWT8QLD9g5ne.jpeg", + "https://downloads.fanbox.cc/images/post/3746116/C93E7db3C3sBqbDw6gQoZBMz.jpeg", ], - "userId" : "74349833", + "comments": "len:4", }, -}, - -{ - "#url" : "https://saki9184.fanbox.cc/posts/7754760", - "#comment" : "missing plan for exact 'feeRequired' value (#5759)", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - "#options" : {"metadata": "plan"}, - - "feeRequired": 300, - "plan" : { - "creatorId": "saki9184", - "fee" : 350, - "id" : "414274", - "title" : "入り浸りJKハルヒ", + { + "#url": "https://fanbox.cc/", + "#category": ("", "fanbox", "home"), + "#class": fanbox.FanboxHomeExtractor, + "#auth": True, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://fanbox.cc/home/supporting", + "#category": ("", "fanbox", "supporting"), + "#class": fanbox.FanboxSupportingExtractor, + "#auth": True, + "#count": 0, + }, + { + "#url": "https://www.pixiv.net/fanbox/creator/52336352", + "#category": ("", "fanbox", "redirect"), + "#class": fanbox.FanboxRedirectExtractor, + "#pattern": fanbox.FanboxCreatorExtractor.pattern, }, -}, - -{ - "#url" : "https://mochirong.fanbox.cc/posts/3746116", - "#comment" : "imageMap file order (#2718); comments metadata (#6287)", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - "#options" : {"metadata": "comments"}, - "#sha1_url": "c92ddd06f2efc4a5fe30ec67e21544f79a5c4062", - "#urls" : [ - "https://pixiv.pximg.net/fanbox/public/images/post/3746116/cover/6h5w7F1MJWLeED6ODfHo6ZYQ.jpeg", - "https://downloads.fanbox.cc/images/post/3746116/ouTz7XZIeVD3FBOzoLhJ3ZTA.jpeg", - "https://downloads.fanbox.cc/images/post/3746116/hBs9bXEg6HvbqWT8QLD9g5ne.jpeg", - "https://downloads.fanbox.cc/images/post/3746116/C93E7db3C3sBqbDw6gQoZBMz.jpeg", - ], - - "comments": "len:4", -}, - -{ - "#url" : "https://fanbox.cc/", - "#category": ("", "fanbox", "home"), - "#class" : fanbox.FanboxHomeExtractor, - "#auth" : True, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://fanbox.cc/home/supporting", - "#category": ("", "fanbox", "supporting"), - "#class" : fanbox.FanboxSupportingExtractor, - "#auth" : True, - "#count" : 0, -}, - -{ - "#url" : "https://www.pixiv.net/fanbox/creator/52336352", - "#category": ("", "fanbox", "redirect"), - "#class" : fanbox.FanboxRedirectExtractor, - "#pattern" : fanbox.FanboxCreatorExtractor.pattern, -}, - ) diff --git a/test/results/fandom.py b/test/results/fandom.py index 2a314043d..84331f9e0 100644 --- a/test/results/fandom.py +++ b/test/results/fandom.py @@ -1,117 +1,106 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.fandom.com/wiki/Title", - "#comment" : "for scripts/supportedsites.py", - "#category": ("wikimedia", "fandom-www", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://mushishi.fandom.com/wiki/Yahagi", - "#category": ("wikimedia", "fandom-mushishi", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#urls" : "https://static.wikia.nocookie.net/mushi-shi/images/f/f8/Yahagi.png/revision/latest?cb=20150128052255", - - "bitdepth" : 8, - "canonicaltitle": "File:Yahagi.png", - "comment" : "", - "commonmetadata": { - "ResolutionUnit": 3, - "XResolution" : "3779/100", - "YResolution" : "3779/100", + { + "#url": "https://www.fandom.com/wiki/Title", + "#comment": "for scripts/supportedsites.py", + "#category": ("wikimedia", "fandom-www", "article"), + "#class": wikimedia.WikimediaArticleExtractor, }, - "date" : "dt:2015-01-28 05:22:55", - "descriptionshorturl": "https://mushishi.fandom.com/index.php?curid=2595", - "descriptionurl": "https://mushishi.fandom.com/wiki/File:Yahagi.png", - "extension" : "png", - "extmetadata" : { - "DateTime": { - "hidden": "", - "source": "mediawiki-metadata", - "value": "2015-01-28T05:22:55Z", - }, - "ObjectName": { - "hidden": "", - "source": "mediawiki-metadata", - "value": "Yahagi", + { + "#url": "https://mushishi.fandom.com/wiki/Yahagi", + "#category": ("wikimedia", "fandom-mushishi", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#urls": "https://static.wikia.nocookie.net/mushi-shi/images/f/f8/Yahagi.png/revision/latest?cb=20150128052255", + "bitdepth": 8, + "canonicaltitle": "File:Yahagi.png", + "comment": "", + "commonmetadata": { + "ResolutionUnit": 3, + "XResolution": "3779/100", + "YResolution": "3779/100", }, - }, - "filename" : "Yahagi", - "height" : 410, - "metadata" : { - "bitDepth" : 8, - "colorType" : "truecolour", - "duration" : 0, - "frameCount": 0, - "loopCount" : 1, - "metadata" : [ - { - "name" : "XResolution", - "value": "3779/100", - }, - { - "name" : "YResolution", - "value": "3779/100", - }, - { - "name" : "ResolutionUnit", - "value": 3, + "date": "dt:2015-01-28 05:22:55", + "descriptionshorturl": "https://mushishi.fandom.com/index.php?curid=2595", + "descriptionurl": "https://mushishi.fandom.com/wiki/File:Yahagi.png", + "extension": "png", + "extmetadata": { + "DateTime": { + "hidden": "", + "source": "mediawiki-metadata", + "value": "2015-01-28T05:22:55Z", }, - { - "name" : "_MW_PNG_VERSION", - "value": 1, + "ObjectName": { + "hidden": "", + "source": "mediawiki-metadata", + "value": "Yahagi", }, - ], + }, + "filename": "Yahagi", + "height": 410, + "metadata": { + "bitDepth": 8, + "colorType": "truecolour", + "duration": 0, + "frameCount": 0, + "loopCount": 1, + "metadata": [ + { + "name": "XResolution", + "value": "3779/100", + }, + { + "name": "YResolution", + "value": "3779/100", + }, + { + "name": "ResolutionUnit", + "value": 3, + }, + { + "name": "_MW_PNG_VERSION", + "value": 1, + }, + ], + }, + "mime": "image/png", + "page": "Yahagi", + "sha1": "e3078a97976215323dbabb0c86b7acc55b512d16", + "size": 429912, + "timestamp": "2015-01-28T05:22:55Z", + "url": "https://static.wikia.nocookie.net/mushi-shi/images/f/f8/Yahagi.png/revision/latest?cb=20150128052255", + "user": "ITHYRIAL", + "userid": 4637089, + "width": 728, + }, + { + "#url": "https://hearthstone.fandom.com/wiki/Flame_Juggler", + "#comment": "empty 'metadata'", + "#category": ("wikimedia", "fandom-hearthstone", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "metadata": {}, + }, + { + "#url": "https://discogs.fandom.com/zh/wiki/File:CH-0430D2.jpg", + "#comment": "non-English language prefix (#6370)", + "#category": ("wikimedia", "fandom-discogs", "file"), + "#class": wikimedia.WikimediaArticleExtractor, + "#urls": "https://static.wikia.nocookie.net/discogs/images/a/ab/CH-0430D2.jpg/revision/latest?cb=20241007150151&path-prefix=zh", + }, + { + "#url": "https://projectsekai.fandom.com/wiki/Project_SEKAI_Wiki", + "#category": ("wikimedia", "fandom-projectsekai", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://youtube.fandom.com", + "#category": ("wikimedia", "fandom-youtube", "wiki"), + "#class": wikimedia.WikimediaWikiExtractor, + "#range": "1-20", + "#count": 20, }, - "mime" : "image/png", - "page" : "Yahagi", - "sha1" : "e3078a97976215323dbabb0c86b7acc55b512d16", - "size" : 429912, - "timestamp" : "2015-01-28T05:22:55Z", - "url" : "https://static.wikia.nocookie.net/mushi-shi/images/f/f8/Yahagi.png/revision/latest?cb=20150128052255", - "user" : "ITHYRIAL", - "userid" : 4637089, - "width" : 728, -}, - -{ - "#url" : "https://hearthstone.fandom.com/wiki/Flame_Juggler", - "#comment" : "empty 'metadata'", - "#category": ("wikimedia", "fandom-hearthstone", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - - "metadata" : {}, -}, - -{ - "#url" : "https://discogs.fandom.com/zh/wiki/File:CH-0430D2.jpg", - "#comment" : "non-English language prefix (#6370)", - "#category": ("wikimedia", "fandom-discogs", "file"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#urls" : "https://static.wikia.nocookie.net/discogs/images/a/ab/CH-0430D2.jpg/revision/latest?cb=20241007150151&path-prefix=zh", -}, - -{ - "#url" : "https://projectsekai.fandom.com/wiki/Project_SEKAI_Wiki", - "#category": ("wikimedia", "fandom-projectsekai", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://youtube.fandom.com", - "#category": ("wikimedia", "fandom-youtube", "wiki"), - "#class" : wikimedia.WikimediaWikiExtractor, - "#range" : "1-20", - "#count" : 20, -}, - ) diff --git a/test/results/fanleaks.py b/test/results/fanleaks.py index c420e0ca8..64fc09242 100644 --- a/test/results/fanleaks.py +++ b/test/results/fanleaks.py @@ -1,67 +1,56 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import fanleaks from gallery_dl import exception - +from gallery_dl.extractor import fanleaks __tests__ = ( -{ - "#url" : "https://fanleaks.club/selti/880", - "#category": ("", "fanleaks", "post"), - "#class" : fanleaks.FanleaksPostExtractor, - "#pattern" : r"https://fanleaks\.club//models/selti/images/selti_0880\.jpg", - - "model_id": "selti", - "model" : "Selti", - "id" : 880, - "type" : "photo", -}, - -{ - "#url" : "https://fanleaks.club/daisy-keech/1038", - "#category": ("", "fanleaks", "post"), - "#class" : fanleaks.FanleaksPostExtractor, - "#pattern" : r"https://fanleaks\.club//models/daisy-keech/videos/daisy-keech_1038\.mp4", - - "model_id": "daisy-keech", - "model" : "Daisy Keech", - "id" : 1038, - "type" : "video", -}, - -{ - "#url" : "https://fanleaks.club/hannahowo/000", - "#category": ("", "fanleaks", "post"), - "#class" : fanleaks.FanleaksPostExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://fanleaks.club/hannahowo", - "#category": ("", "fanleaks", "model"), - "#class" : fanleaks.FanleaksModelExtractor, - "#pattern" : r"https://fanleaks\.club//models/hannahowo/(images|videos)/hannahowo_\d+\.\w+", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://fanleaks.club/belle-delphine", - "#category": ("", "fanleaks", "model"), - "#class" : fanleaks.FanleaksModelExtractor, - "#pattern" : r"https://fanleaks\.club//models/belle-delphine/(images|videos)/belle-delphine_\d+\.\w+", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://fanleaks.club/daisy-keech", - "#category": ("", "fanleaks", "model"), - "#class" : fanleaks.FanleaksModelExtractor, -}, - + { + "#url": "https://fanleaks.club/selti/880", + "#category": ("", "fanleaks", "post"), + "#class": fanleaks.FanleaksPostExtractor, + "#pattern": r"https://fanleaks\.club//models/selti/images/selti_0880\.jpg", + "model_id": "selti", + "model": "Selti", + "id": 880, + "type": "photo", + }, + { + "#url": "https://fanleaks.club/daisy-keech/1038", + "#category": ("", "fanleaks", "post"), + "#class": fanleaks.FanleaksPostExtractor, + "#pattern": r"https://fanleaks\.club//models/daisy-keech/videos/daisy-keech_1038\.mp4", + "model_id": "daisy-keech", + "model": "Daisy Keech", + "id": 1038, + "type": "video", + }, + { + "#url": "https://fanleaks.club/hannahowo/000", + "#category": ("", "fanleaks", "post"), + "#class": fanleaks.FanleaksPostExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://fanleaks.club/hannahowo", + "#category": ("", "fanleaks", "model"), + "#class": fanleaks.FanleaksModelExtractor, + "#pattern": r"https://fanleaks\.club//models/hannahowo/(images|videos)/hannahowo_\d+\.\w+", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://fanleaks.club/belle-delphine", + "#category": ("", "fanleaks", "model"), + "#class": fanleaks.FanleaksModelExtractor, + "#pattern": r"https://fanleaks\.club//models/belle-delphine/(images|videos)/belle-delphine_\d+\.\w+", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://fanleaks.club/daisy-keech", + "#category": ("", "fanleaks", "model"), + "#class": fanleaks.FanleaksModelExtractor, + }, ) diff --git a/test/results/fantia.py b/test/results/fantia.py index 5867e7861..f3a816efc 100644 --- a/test/results/fantia.py +++ b/test/results/fantia.py @@ -1,67 +1,58 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import fantia - __tests__ = ( -{ - "#url" : "https://fantia.jp/fanclubs/6939", - "#category": ("", "fantia", "creator"), - "#class" : fantia.FantiaCreatorExtractor, - "#range" : "1-25", - "#count" : ">= 25", - - "fanclub_user_id": 52152, - "tags" : list, - "post_title" : str, -}, - -{ - "#url" : "https://fantia.jp/posts/1166373", - "#category": ("", "fantia", "post"), - "#class" : fantia.FantiaPostExtractor, - "#pattern" : r"https://(c\.fantia\.jp/uploads/post/file/1166373/|cc\.fantia\.jp/uploads/post_content_photo/file/732549[01]|fantia\.jp/posts/1166373/album_image\?)", - - "blogpost_text" : r"re:^$|This is a test.\n\n(This is a test.)?\n\n|Link to video:\nhttps://www.youtube.com/watch\?v=5SSdvNcAagI\n\nhtml img from another site:\n\n\n\n\n\n", - "comment" : "\n\n", - "content_category": r"re:thumb|blog|photo_gallery", - "content_comment" : str, - "content_count" : 5, - "content_filename": r"re:|", - "content_num" : range(1, 5), - "content_title" : r"re:Test (Blog Content \d+|Image Gallery)|thumb", - "date" : "dt:2022-03-09 16:46:12", - "fanclub_id" : 356320, - "fanclub_name" : "Test Fantia", - "fanclub_url" : "https://fantia.jp/fanclubs/356320", - "fanclub_user_id" : 7487131, - "fanclub_user_name": "2022/03/08 15:13:52の名無し", - "file_url" : str, - "filename" : str, - "num" : int, - "plan" : dict, - "post_id" : 1166373, - "post_title" : "Test Fantia Post", - "post_url" : "https://fantia.jp/posts/1166373", - "posted_at" : "Thu, 10 Mar 2022 01:46:12 +0900", - "rating" : "general", - "tags" : [], -}, - -{ - "#url" : "https://fantia.jp/posts/508363", - "#category": ("", "fantia", "post"), - "#class" : fantia.FantiaPostExtractor, - "#count" : 6, - - "post_title": "zunda逆バニーでおしりｺｯｼｮﾘ", - "tags" : list, - "rating" : "adult", - "post_id" : 508363, -}, - + { + "#url": "https://fantia.jp/fanclubs/6939", + "#category": ("", "fantia", "creator"), + "#class": fantia.FantiaCreatorExtractor, + "#range": "1-25", + "#count": ">= 25", + "fanclub_user_id": 52152, + "tags": list, + "post_title": str, + }, + { + "#url": "https://fantia.jp/posts/1166373", + "#category": ("", "fantia", "post"), + "#class": fantia.FantiaPostExtractor, + "#pattern": r"https://(c\.fantia\.jp/uploads/post/file/1166373/|cc\.fantia\.jp/uploads/post_content_photo/file/732549[01]|fantia\.jp/posts/1166373/album_image\?)", + "blogpost_text": r"re:^$|This is a test.\n\n(This is a test.)?\n\n|Link to video:\nhttps://www.youtube.com/watch\?v=5SSdvNcAagI\n\nhtml img from another site:\n\n\n\n\n\n", + "comment": "\n\n", + "content_category": r"re:thumb|blog|photo_gallery", + "content_comment": str, + "content_count": 5, + "content_filename": r"re:|", + "content_num": range(1, 5), + "content_title": r"re:Test (Blog Content \d+|Image Gallery)|thumb", + "date": "dt:2022-03-09 16:46:12", + "fanclub_id": 356320, + "fanclub_name": "Test Fantia", + "fanclub_url": "https://fantia.jp/fanclubs/356320", + "fanclub_user_id": 7487131, + "fanclub_user_name": "2022/03/08 15:13:52の名無し", + "file_url": str, + "filename": str, + "num": int, + "plan": dict, + "post_id": 1166373, + "post_title": "Test Fantia Post", + "post_url": "https://fantia.jp/posts/1166373", + "posted_at": "Thu, 10 Mar 2022 01:46:12 +0900", + "rating": "general", + "tags": [], + }, + { + "#url": "https://fantia.jp/posts/508363", + "#category": ("", "fantia", "post"), + "#class": fantia.FantiaPostExtractor, + "#count": 6, + "post_title": "zunda逆バニーでおしりｺｯｼｮﾘ", + "tags": list, + "rating": "adult", + "post_id": 508363, + }, ) diff --git a/test/results/fapachi.py b/test/results/fapachi.py index 8907e7f8b..6403625f1 100644 --- a/test/results/fapachi.py +++ b/test/results/fapachi.py @@ -1,44 +1,36 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import fapachi - __tests__ = ( -{ - "#url" : "https://fapachi.com/sonson/media/0082", - "#comment" : "NSFW", - "#category": ("", "fapachi", "post"), - "#class" : fapachi.FapachiPostExtractor, - "#pattern" : r"https://fapachi\.com/models/s/o/sonson/1/full/sonson_0082\.jpeg", - - "user": "sonson", - "id" : "0082", -}, - -{ - "#url" : "https://fapachi.com/ferxiita/media/0159", - "#comment" : "NSFW", - "#category": ("", "fapachi", "post"), - "#class" : fapachi.FapachiPostExtractor, -}, - -{ - "#url" : "https://fapachi.com/sonson", - "#category": ("", "fapachi", "user"), - "#class" : fapachi.FapachiUserExtractor, - "#pattern" : fapachi.FapachiPostExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://fapachi.com/ferxiita/page/3", - "#category": ("", "fapachi", "user"), - "#class" : fapachi.FapachiUserExtractor, -}, - + { + "#url": "https://fapachi.com/sonson/media/0082", + "#comment": "NSFW", + "#category": ("", "fapachi", "post"), + "#class": fapachi.FapachiPostExtractor, + "#pattern": r"https://fapachi\.com/models/s/o/sonson/1/full/sonson_0082\.jpeg", + "user": "sonson", + "id": "0082", + }, + { + "#url": "https://fapachi.com/ferxiita/media/0159", + "#comment": "NSFW", + "#category": ("", "fapachi", "post"), + "#class": fapachi.FapachiPostExtractor, + }, + { + "#url": "https://fapachi.com/sonson", + "#category": ("", "fapachi", "user"), + "#class": fapachi.FapachiUserExtractor, + "#pattern": fapachi.FapachiPostExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://fapachi.com/ferxiita/page/3", + "#category": ("", "fapachi", "user"), + "#class": fapachi.FapachiUserExtractor, + }, ) diff --git a/test/results/fapello.py b/test/results/fapello.py index 6764b9598..68634c35d 100644 --- a/test/results/fapello.py +++ b/test/results/fapello.py @@ -1,141 +1,119 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import fapello from gallery_dl import exception - +from gallery_dl.extractor import fapello __tests__ = ( -{ - "#url" : "https://fapello.com/carrykey/530/", - "#category": ("", "fapello", "post"), - "#class" : fapello.FapelloPostExtractor, - "#pattern" : r"https://fapello\.com/content/c/a/carrykey/1000/carrykey_0530\.jpg", - - "model" : "carrykey", - "id" : 530, - "type" : "photo", - "thumbnail": "", -}, - -{ - "#url" : "https://fapello.com/vladislava-661/693/", - "#category": ("", "fapello", "post"), - "#class" : fapello.FapelloPostExtractor, - "#pattern" : r"https://cdn\.fapello\.com/content/v/l/vladislava-661/1000/vladislava-661_0693\.mp4", - "#exception": exception.NotFoundError, - - "model" : "vladislava-661", - "id" : 693, - "type" : "video", - "thumbnail": "https://fapello.com/content/v/l/vladislava-661/1000/vladislava-661_0693.jpg", -}, - -{ - "#url" : "https://fapello.com/carrykey/000/", - "#category": ("", "fapello", "post"), - "#class" : fapello.FapelloPostExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://fapello.su/grace-charis-gracecharisxo/2038266/", - "#category": ("", "fapello", "post"), - "#class" : fapello.FapelloPostExtractor, - - "model" : "grace-charis-gracecharisxo", - "id" : 2038266, - "type" : "photo", -}, - -{ - "#url" : "https://fapello.com/hyoon/", - "#category": ("", "fapello", "model"), - "#class" : fapello.FapelloModelExtractor, - "#pattern" : fapello.FapelloPostExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://fapello.com/kobaebeefboo/", - "#category": ("", "fapello", "model"), - "#class" : fapello.FapelloModelExtractor, -}, - -{ - "#url" : "https://fapello.su/grace-charis-gracecharisxo/", - "#category": ("", "fapello", "model"), - "#class" : fapello.FapelloModelExtractor, - "#pattern" : fapello.FapelloPostExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://fapello.com/top-likes/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, - "#pattern" : fapello.FapelloModelExtractor.pattern, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://fapello.su/top-likes/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, - "#pattern" : fapello.FapelloModelExtractor.pattern, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://fapello.com/videos/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, - "#pattern" : fapello.FapelloPostExtractor.pattern, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://fapello.com/top-followers/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, -}, - -{ - "#url" : "https://fapello.su/top-followers/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, -}, - -{ - "#url" : "https://fapello.com/trending/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, -}, - -{ - "#url" : "https://fapello.su/trending/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, -}, - -{ - "#url" : "https://fapello.com/popular_videos/twelve_hours/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, -}, - -{ - "#url" : "https://fapello.com/popular_videos/week/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, -}, - + { + "#url": "https://fapello.com/carrykey/530/", + "#category": ("", "fapello", "post"), + "#class": fapello.FapelloPostExtractor, + "#pattern": r"https://fapello\.com/content/c/a/carrykey/1000/carrykey_0530\.jpg", + "model": "carrykey", + "id": 530, + "type": "photo", + "thumbnail": "", + }, + { + "#url": "https://fapello.com/vladislava-661/693/", + "#category": ("", "fapello", "post"), + "#class": fapello.FapelloPostExtractor, + "#pattern": r"https://cdn\.fapello\.com/content/v/l/vladislava-661/1000/vladislava-661_0693\.mp4", + "#exception": exception.NotFoundError, + "model": "vladislava-661", + "id": 693, + "type": "video", + "thumbnail": "https://fapello.com/content/v/l/vladislava-661/1000/vladislava-661_0693.jpg", + }, + { + "#url": "https://fapello.com/carrykey/000/", + "#category": ("", "fapello", "post"), + "#class": fapello.FapelloPostExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://fapello.su/grace-charis-gracecharisxo/2038266/", + "#category": ("", "fapello", "post"), + "#class": fapello.FapelloPostExtractor, + "model": "grace-charis-gracecharisxo", + "id": 2038266, + "type": "photo", + }, + { + "#url": "https://fapello.com/hyoon/", + "#category": ("", "fapello", "model"), + "#class": fapello.FapelloModelExtractor, + "#pattern": fapello.FapelloPostExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://fapello.com/kobaebeefboo/", + "#category": ("", "fapello", "model"), + "#class": fapello.FapelloModelExtractor, + }, + { + "#url": "https://fapello.su/grace-charis-gracecharisxo/", + "#category": ("", "fapello", "model"), + "#class": fapello.FapelloModelExtractor, + "#pattern": fapello.FapelloPostExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://fapello.com/top-likes/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + "#pattern": fapello.FapelloModelExtractor.pattern, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://fapello.su/top-likes/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + "#pattern": fapello.FapelloModelExtractor.pattern, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://fapello.com/videos/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + "#pattern": fapello.FapelloPostExtractor.pattern, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://fapello.com/top-followers/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + }, + { + "#url": "https://fapello.su/top-followers/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + }, + { + "#url": "https://fapello.com/trending/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + }, + { + "#url": "https://fapello.su/trending/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + }, + { + "#url": "https://fapello.com/popular_videos/twelve_hours/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + }, + { + "#url": "https://fapello.com/popular_videos/week/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + }, ) diff --git a/test/results/fappic.py b/test/results/fappic.py index b89c4e299..dbab88b57 100644 --- a/test/results/fappic.py +++ b/test/results/fappic.py @@ -1,26 +1,21 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://fappic.com/98wxqcklyh8k/test.png", - "#category": ("imagehost", "fappic", "image"), - "#class" : imagehosts.FappicImageExtractor, -}, - -{ - "#url" : "https://www.fappic.com/98wxqcklyh8k/test.png", - "#category": ("imagehost", "fappic", "image"), - "#class" : imagehosts.FappicImageExtractor, - "#pattern" : r"https://img\d+\.fappic\.com/img/\w+/test\.png", - "#sha1_metadata": "433b1d310b0ff12ad8a71ac7b9d8ba3f8cd1e898", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - + { + "#url": "https://fappic.com/98wxqcklyh8k/test.png", + "#category": ("imagehost", "fappic", "image"), + "#class": imagehosts.FappicImageExtractor, + }, + { + "#url": "https://www.fappic.com/98wxqcklyh8k/test.png", + "#category": ("imagehost", "fappic", "image"), + "#class": imagehosts.FappicImageExtractor, + "#pattern": r"https://img\d+\.fappic\.com/img/\w+/test\.png", + "#sha1_metadata": "433b1d310b0ff12ad8a71ac7b9d8ba3f8cd1e898", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, ) diff --git a/test/results/fashionnova.py b/test/results/fashionnova.py index 9cee0e23f..ca59496f5 100644 --- a/test/results/fashionnova.py +++ b/test/results/fashionnova.py @@ -1,45 +1,37 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://www.fashionnova.com/collections/mini-dresses", - "#category": ("shopify", "fashionnova", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.fashionnova.com/collections/mini-dresses/?page=1", - "#category": ("shopify", "fashionnova", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.fashionnova.com/collections/mini-dresses#1", - "#category": ("shopify", "fashionnova", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.fashionnova.com/products/all-my-life-legging-black", - "#category": ("shopify", "fashionnova", "product"), - "#class" : shopify.ShopifyProductExtractor, - "#pattern" : r"https?://cdn\d*\.shopify\.com/s/files/", - "#count" : 8, -}, - -{ - "#url" : "https://www.fashionnova.com/collections/flats/products/name", - "#category": ("shopify", "fashionnova", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://www.fashionnova.com/collections/mini-dresses", + "#category": ("shopify", "fashionnova", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.fashionnova.com/collections/mini-dresses/?page=1", + "#category": ("shopify", "fashionnova", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.fashionnova.com/collections/mini-dresses#1", + "#category": ("shopify", "fashionnova", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.fashionnova.com/products/all-my-life-legging-black", + "#category": ("shopify", "fashionnova", "product"), + "#class": shopify.ShopifyProductExtractor, + "#pattern": r"https?://cdn\d*\.shopify\.com/s/files/", + "#count": 8, + }, + { + "#url": "https://www.fashionnova.com/collections/flats/products/name", + "#category": ("shopify", "fashionnova", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/fireden.py b/test/results/fireden.py index 48549e18d..e90334d79 100644 --- a/test/results/fireden.py +++ b/test/results/fireden.py @@ -1,36 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://boards.fireden.net/sci/thread/11264294/", - "#category": ("foolfuuka", "fireden", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "61cab625c95584a12a30049d054931d64f8d20aa", -}, - -{ - "#url" : "https://boards.fireden.net/sci/", - "#category": ("foolfuuka", "fireden", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://boards.fireden.net/_/search/text/test/", - "#category": ("foolfuuka", "fireden", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://boards.fireden.net/sci/gallery/6", - "#category": ("foolfuuka", "fireden", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://boards.fireden.net/sci/thread/11264294/", + "#category": ("foolfuuka", "fireden", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "61cab625c95584a12a30049d054931d64f8d20aa", + }, + { + "#url": "https://boards.fireden.net/sci/", + "#category": ("foolfuuka", "fireden", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://boards.fireden.net/_/search/text/test/", + "#category": ("foolfuuka", "fireden", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://boards.fireden.net/sci/gallery/6", + "#category": ("foolfuuka", "fireden", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/flickr.py b/test/results/flickr.py index 5ae580502..092acfc19 100644 --- a/test/results/flickr.py +++ b/test/results/flickr.py @@ -1,171 +1,147 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import flickr from gallery_dl import exception - +from gallery_dl.extractor import flickr __tests__ = ( -{ - "#url" : "https://www.flickr.com/photos/departingyyz/16089302239", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, - "#options" : { - "contexts": True, - "exif": True, - }, - "#urls" : "https://live.staticflickr.com/7463/16089302239_de18cd8017_b_d.jpg", - "#pattern" : flickr.FlickrImageExtractor.pattern, - "#sha1_content": [ - "3133006c6d657fe54cf7d4c46b82abbcb0efaf9f", - "0821a28ee46386e85b02b67cf2720063440a228c", - ], - - "camera" : "Sony ILCE-7", - "comments" : int, - "description": str, - "exif" : list, - "extension" : "jpg", - "filename" : "16089302239_de18cd8017_b_d", - "id" : 16089302239, - "height" : 683, - "label" : "Large", - "media" : "photo", - "pool" : list, - "set" : list, - "url" : str, - "views" : int, - "width" : 1024, -}, - -{ - "#url" : "https://secure.flickr.com/photos/departingyyz/16089302239", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, -}, - -{ - "#url" : "https://m.flickr.com/photos/departingyyz/16089302239", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, -}, - -{ - "#url" : "https://flickr.com/photos/departingyyz/16089302239", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, -}, - -{ - "#url" : "https://www.flickr.com/photos/eliasroviello/52713899383/", - "#comment" : "video", - "#class" : flickr.FlickrImageExtractor, - "#pattern" : r"https://live.staticflickr\.com/video/52713899383/51dfffef79/1080p\.mp4\?s=ey.+", - - "media": "video", -}, - -{ - "#url" : "http://c2.staticflickr.com/2/1475/24531000464_9a7503ae68_b.jpg", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, -}, - -{ - "#url" : "https://farm2.static.flickr.com/1035/1188352415_cb139831d0.jpg", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, -}, - -{ - "#url" : "https://flic.kr/p/FPVo9U", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, -}, - -{ - "#url" : "https://www.flickr.com/photos/zzz/16089302238", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.flickr.com/photos/shona_s/albums/72157633471741607", - "#category": ("", "flickr", "album"), - "#class" : flickr.FlickrAlbumExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, - "#count" : 6, -}, - -{ - "#url" : "https://www.flickr.com/photos/shona_s/albums", - "#category": ("", "flickr", "album"), - "#class" : flickr.FlickrAlbumExtractor, - "#pattern" : flickr.FlickrAlbumExtractor.pattern, - "#count" : 2, -}, - -{ - "#url" : "https://secure.flickr.com/photos/shona_s/albums", - "#category": ("", "flickr", "album"), - "#class" : flickr.FlickrAlbumExtractor, -}, - -{ - "#url" : "https://m.flickr.com/photos/shona_s/albums", - "#category": ("", "flickr", "album"), - "#class" : flickr.FlickrAlbumExtractor, -}, - -{ - "#url" : "https://www.flickr.com/photos/flickr/galleries/72157681572514792/", - "#category": ("", "flickr", "gallery"), - "#class" : flickr.FlickrGalleryExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, - "#count" : ">= 10", -}, - -{ - "#url" : "https://www.flickr.com/groups/bird_headshots/", - "#category": ("", "flickr", "group"), - "#class" : flickr.FlickrGroupExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, - "#count" : "> 150", -}, - -{ - "#url" : "https://www.flickr.com/photos/shona_s/", - "#category": ("", "flickr", "user"), - "#class" : flickr.FlickrUserExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, - "#count" : 28, -}, - -{ - "#url" : "https://www.flickr.com/photos/shona_s/favorites", - "#category": ("", "flickr", "favorite"), - "#class" : flickr.FlickrFavoriteExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, - "#count" : 4, -}, - -{ - "#url" : "https://flickr.com/search/?text=mountain", - "#category": ("", "flickr", "search"), - "#class" : flickr.FlickrSearchExtractor, -}, - -{ - "#url" : "https://flickr.com/search/?text=tree%20cloud%20house&color_codes=4&styles=minimalism", - "#category": ("", "flickr", "search"), - "#class" : flickr.FlickrSearchExtractor, -}, - + { + "#url": "https://www.flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + "#options": { + "contexts": True, + "exif": True, + }, + "#urls": "https://live.staticflickr.com/7463/16089302239_de18cd8017_b_d.jpg", + "#pattern": flickr.FlickrImageExtractor.pattern, + "#sha1_content": [ + "3133006c6d657fe54cf7d4c46b82abbcb0efaf9f", + "0821a28ee46386e85b02b67cf2720063440a228c", + ], + "camera": "Sony ILCE-7", + "comments": int, + "description": str, + "exif": list, + "extension": "jpg", + "filename": "16089302239_de18cd8017_b_d", + "id": 16089302239, + "height": 683, + "label": "Large", + "media": "photo", + "pool": list, + "set": list, + "url": str, + "views": int, + "width": 1024, + }, + { + "#url": "https://secure.flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + }, + { + "#url": "https://m.flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + }, + { + "#url": "https://flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + }, + { + "#url": "https://www.flickr.com/photos/eliasroviello/52713899383/", + "#comment": "video", + "#class": flickr.FlickrImageExtractor, + "#pattern": r"https://live.staticflickr\.com/video/52713899383/51dfffef79/1080p\.mp4\?s=ey.+", + "media": "video", + }, + { + "#url": "http://c2.staticflickr.com/2/1475/24531000464_9a7503ae68_b.jpg", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + }, + { + "#url": "https://farm2.static.flickr.com/1035/1188352415_cb139831d0.jpg", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + }, + { + "#url": "https://flic.kr/p/FPVo9U", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + }, + { + "#url": "https://www.flickr.com/photos/zzz/16089302238", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.flickr.com/photos/shona_s/albums/72157633471741607", + "#category": ("", "flickr", "album"), + "#class": flickr.FlickrAlbumExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + "#count": 6, + }, + { + "#url": "https://www.flickr.com/photos/shona_s/albums", + "#category": ("", "flickr", "album"), + "#class": flickr.FlickrAlbumExtractor, + "#pattern": flickr.FlickrAlbumExtractor.pattern, + "#count": 2, + }, + { + "#url": "https://secure.flickr.com/photos/shona_s/albums", + "#category": ("", "flickr", "album"), + "#class": flickr.FlickrAlbumExtractor, + }, + { + "#url": "https://m.flickr.com/photos/shona_s/albums", + "#category": ("", "flickr", "album"), + "#class": flickr.FlickrAlbumExtractor, + }, + { + "#url": "https://www.flickr.com/photos/flickr/galleries/72157681572514792/", + "#category": ("", "flickr", "gallery"), + "#class": flickr.FlickrGalleryExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + "#count": ">= 10", + }, + { + "#url": "https://www.flickr.com/groups/bird_headshots/", + "#category": ("", "flickr", "group"), + "#class": flickr.FlickrGroupExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + "#count": "> 150", + }, + { + "#url": "https://www.flickr.com/photos/shona_s/", + "#category": ("", "flickr", "user"), + "#class": flickr.FlickrUserExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + "#count": 28, + }, + { + "#url": "https://www.flickr.com/photos/shona_s/favorites", + "#category": ("", "flickr", "favorite"), + "#class": flickr.FlickrFavoriteExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + "#count": 4, + }, + { + "#url": "https://flickr.com/search/?text=mountain", + "#category": ("", "flickr", "search"), + "#class": flickr.FlickrSearchExtractor, + }, + { + "#url": "https://flickr.com/search/?text=tree%20cloud%20house&color_codes=4&styles=minimalism", + "#category": ("", "flickr", "search"), + "#class": flickr.FlickrSearchExtractor, + }, ) diff --git a/test/results/foalcon.py b/test/results/foalcon.py index f2ee3a3af..9d6fb48eb 100644 --- a/test/results/foalcon.py +++ b/test/results/foalcon.py @@ -1,41 +1,34 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import szurubooru - __tests__ = ( -{ - "#url" : "https://booru.foalcon.com/posts/query=simple_background", - "#category": ("szurubooru", "foalcon", "tag"), - "#class" : szurubooru.SzurubooruTagExtractor, - "#pattern" : r"https://booru\.foalcon\.com/data/posts/\d+_[0-9a-f]{16}\.\w+", - "#range" : "1-150", - "#count" : 150, -}, - -{ - "#url" : "https://booru.foalcon.com/posts/query=", - "#category": ("szurubooru", "foalcon", "tag"), - "#class" : szurubooru.SzurubooruTagExtractor, -}, - -{ - "#url" : "https://booru.foalcon.com/posts", - "#category": ("szurubooru", "foalcon", "tag"), - "#class" : szurubooru.SzurubooruTagExtractor, -}, - -{ - "#url" : "https://booru.foalcon.com/post/30092", - "#category": ("szurubooru", "foalcon", "post"), - "#class" : szurubooru.SzurubooruPostExtractor, - "#pattern" : r"https://booru\.foalcon\.com/data/posts/30092_b7d56e941888b624\.png", - "#sha1_url" : "dad4d4c67d87cd9a4ac429b3414747c27a95d5cb", - "#sha1_content": "86d1514c0ca8197950cc4b74e7a59b2dc76ebf9c", -}, - + { + "#url": "https://booru.foalcon.com/posts/query=simple_background", + "#category": ("szurubooru", "foalcon", "tag"), + "#class": szurubooru.SzurubooruTagExtractor, + "#pattern": r"https://booru\.foalcon\.com/data/posts/\d+_[0-9a-f]{16}\.\w+", + "#range": "1-150", + "#count": 150, + }, + { + "#url": "https://booru.foalcon.com/posts/query=", + "#category": ("szurubooru", "foalcon", "tag"), + "#class": szurubooru.SzurubooruTagExtractor, + }, + { + "#url": "https://booru.foalcon.com/posts", + "#category": ("szurubooru", "foalcon", "tag"), + "#class": szurubooru.SzurubooruTagExtractor, + }, + { + "#url": "https://booru.foalcon.com/post/30092", + "#category": ("szurubooru", "foalcon", "post"), + "#class": szurubooru.SzurubooruPostExtractor, + "#pattern": r"https://booru\.foalcon\.com/data/posts/30092_b7d56e941888b624\.png", + "#sha1_url": "dad4d4c67d87cd9a4ac429b3414747c27a95d5cb", + "#sha1_content": "86d1514c0ca8197950cc4b74e7a59b2dc76ebf9c", + }, ) diff --git a/test/results/furaffinity.py b/test/results/furaffinity.py index 75535591d..53db402ff 100644 --- a/test/results/furaffinity.py +++ b/test/results/furaffinity.py @@ -1,248 +1,220 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import furaffinity - __tests__ = ( -{ - "#url" : "https://www.furaffinity.net/gallery/mirlinthloth/", - "#category": ("", "furaffinity", "gallery"), - "#class" : furaffinity.FuraffinityGalleryExtractor, - "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/mirlinthloth/\d+/\d+.\w+\.\w+", - "#range" : "45-50", - "#count" : 6, -}, - -{ - "#url" : "https://www.furaffinity.net/scraps/mirlinthloth/", - "#category": ("", "furaffinity", "scraps"), - "#class" : furaffinity.FuraffinityScrapsExtractor, - "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+(/stories)?/\d+/\d+.\w+.", - "#count" : ">= 3", -}, - -{ - "#url" : "https://www.furaffinity.net/favorites/mirlinthloth/", - "#category": ("", "furaffinity", "favorite"), - "#class" : furaffinity.FuraffinityFavoriteExtractor, - "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+/\d+/\d+.\w+\.\w+", - "#range" : "45-50", - "#count" : 6, - - "favorite_id": int, -}, - -{ - "#url" : "https://www.furaffinity.net/search/?q=cute", - "#category": ("", "furaffinity", "search"), - "#class" : furaffinity.FuraffinitySearchExtractor, - "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+/\d+/\d+.\w+\.\w+", - "#range" : "45-50", - "#count" : 6, -}, - -{ - "#url" : "https://www.furaffinity.net/search/?q=leaf&range=1day", - "#comment" : "first page of search results (#2402)", - "#category": ("", "furaffinity", "search"), - "#class" : furaffinity.FuraffinitySearchExtractor, - "#range" : "1-3", - "#count" : 3, -}, - -{ - "#url" : "https://www.furaffinity.net/view/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, - "#pattern" : r"https://d\d*\.f(uraffinity|acdn)\.net/(download/)?art/mirlinthloth/music/1488278723/1480267446.mirlinthloth_dj_fennmink_-_bude_s_4_ever\.mp3", - - "artist" : "mirlinthloth", - "artist_url" : "mirlinthloth", - "date" : "dt:2016-11-27 17:24:06", - "description": "A Song made playing the game Cosmic DJ.", - "extension" : "mp3", - "filename" : r"re:\d+\.\w+_dj_fennmink_-_bude_s_4_ever", - "id" : 21835115, - "tags" : list, - "title" : "Bude's 4 Ever", - "url" : r"re:https://d\d?\.f(uraffinity|acdn)\.net/art", - "user" : "mirlinthloth", - "views" : int, - "favorites" : int, - "comments" : int, - "rating" : "General", - "fa_category": "Music", - "theme" : "All", - "species" : "Unspecified / Any", - "gender" : "Any", - "width" : 120, - "height" : 120, -}, - -{ - "#url" : "https://www.furaffinity.net/view/42166511/", - "#comment" : "'external' option (#1492)", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, - "#options" : {"external": True}, - "#pattern" : r"https://d\d*\.f(uraffinity|acdn)\.net/|http://www\.postybirb\.com", - "#count" : 2, -}, - -{ - "#url" : "https://www.furaffinity.net/view/45331225/", - "#comment" : "no tags (#2277)", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, - - "artist" : "Kota_Remminders", - "artist_url" : "kotaremminders", - "date" : "dt:2022-01-03 17:49:33", - "fa_category": "Adoptables", - "filename" : "1641232173.kotaremminders_chidopts1", - "gender" : "Any", - "height" : 905, - "id" : 45331225, - "rating" : "General", - "species" : "Unspecified / Any", - "tags" : [], - "theme" : "All", - "title" : "REMINDER", - "width" : 1280, -}, - -{ - "#url" : "https://www.furaffinity.net/view/22964019/", - "#comment" : "get thumbnails for posts (#1284)", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, - - "artist" : "Dwale", - "artist_url" : "dwale", - "date" : "dt:2017-03-21 14:21:29", - "fa_category" : "Poetry", - "filename" : "1490106089.dwale_poem_for_children", - "folders" : [], - "height" : 50, - "id" : 22964019, - "rating" : "General", - "title" : "Poem for Children Wishing to Summon Evil Spirits", - "thumbnail" : "https://t.furaffinity.net/22964019@600-1490106089.jpg", - "width" : 50, -}, - -{ - "#url" : "https://www.furaffinity.net/view/34260156/", - "#comment" : "list gallery folders for image", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, - - "artist" : "dbd", - "artist_url" : "dbd", - "date" : "dt:2019-12-17 22:52:01", - "fa_category" : "All", - "filename" : "1576623121.dbd_patreoncustom-wdg13-web", - "folders" : ["By Year - 2019", - "Custom Character Folder - All Custom Characters", - "Custom Character Folder - Other Ungulates", - "Custom Character Folder - Female", - "Custom Character Folder - Patreon Supported Custom Characters"], - "height" : 900, - "id" : 34260156, - "rating" : "General", - "title" : "Patreon Custom Deer", - "thumbnail" : "https://t.furaffinity.net/34260156@600-1576623121.jpg", - "width" : 488, -}, - -{ - "#url" : "https://www.furaffinity.net/view/57587562", - "#comment" : "login required", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://furaffinity.net/view/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, -}, - -{ - "#url" : "https://fxfuraffinity.net/view/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, -}, - -{ - "#url" : "https://xfuraffinity.net/view/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, -}, - -{ - "#url" : "https://fxraffinity.net/view/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, -}, - -{ - "#url" : "https://sfw.furaffinity.net/view/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, -}, - -{ - "#url" : "https://www.furaffinity.net/full/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, -}, - -{ - "#url" : "https://www.furaffinity.net/user/mirlinthloth/", - "#category": ("", "furaffinity", "user"), - "#class" : furaffinity.FuraffinityUserExtractor, - "#pattern" : "/gallery/mirlinthloth/$", -}, - -{ - "#url" : "https://www.furaffinity.net/user/mirlinthloth/", - "#category": ("", "furaffinity", "user"), - "#class" : furaffinity.FuraffinityUserExtractor, - "#options" : {"include": "all"}, - "#pattern" : "/(gallery|scraps|favorites)/mirlinthloth/$", - "#count" : 3, -}, - -{ - "#url" : "https://www.furaffinity.net/watchlist/by/mirlinthloth/", - "#category": ("", "furaffinity", "following"), - "#class" : furaffinity.FuraffinityFollowingExtractor, - "#pattern" : furaffinity.FuraffinityUserExtractor.pattern, - "#range" : "176-225", - "#count" : 50, -}, - -{ - "#url" : "https://www.furaffinity.net/msg/submissions", - "#category": ("", "furaffinity", "submissions"), - "#class" : furaffinity.FuraffinitySubmissionsExtractor, - "#auth" : True, - "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/mirlinthloth/\d+/\d+.\w+\.\w+", - "#range" : "45-50", - "#count" : 6, -}, - -{ - "#url" : "https://www.furaffinity.net/msg/submissions/new~56789000@48/", - "#category": ("", "furaffinity", "submissions"), - "#class" : furaffinity.FuraffinitySubmissionsExtractor, - "#auth" : True, -}, - + { + "#url": "https://www.furaffinity.net/gallery/mirlinthloth/", + "#category": ("", "furaffinity", "gallery"), + "#class": furaffinity.FuraffinityGalleryExtractor, + "#pattern": r"https://d\d?\.f(uraffinity|acdn)\.net/art/mirlinthloth/\d+/\d+.\w+\.\w+", + "#range": "45-50", + "#count": 6, + }, + { + "#url": "https://www.furaffinity.net/scraps/mirlinthloth/", + "#category": ("", "furaffinity", "scraps"), + "#class": furaffinity.FuraffinityScrapsExtractor, + "#pattern": r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+(/stories)?/\d+/\d+.\w+.", + "#count": ">= 3", + }, + { + "#url": "https://www.furaffinity.net/favorites/mirlinthloth/", + "#category": ("", "furaffinity", "favorite"), + "#class": furaffinity.FuraffinityFavoriteExtractor, + "#pattern": r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+/\d+/\d+.\w+\.\w+", + "#range": "45-50", + "#count": 6, + "favorite_id": int, + }, + { + "#url": "https://www.furaffinity.net/search/?q=cute", + "#category": ("", "furaffinity", "search"), + "#class": furaffinity.FuraffinitySearchExtractor, + "#pattern": r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+/\d+/\d+.\w+\.\w+", + "#range": "45-50", + "#count": 6, + }, + { + "#url": "https://www.furaffinity.net/search/?q=leaf&range=1day", + "#comment": "first page of search results (#2402)", + "#category": ("", "furaffinity", "search"), + "#class": furaffinity.FuraffinitySearchExtractor, + "#range": "1-3", + "#count": 3, + }, + { + "#url": "https://www.furaffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + "#pattern": r"https://d\d*\.f(uraffinity|acdn)\.net/(download/)?art/mirlinthloth/music/1488278723/1480267446.mirlinthloth_dj_fennmink_-_bude_s_4_ever\.mp3", + "artist": "mirlinthloth", + "artist_url": "mirlinthloth", + "date": "dt:2016-11-27 17:24:06", + "description": "A Song made playing the game Cosmic DJ.", + "extension": "mp3", + "filename": r"re:\d+\.\w+_dj_fennmink_-_bude_s_4_ever", + "id": 21835115, + "tags": list, + "title": "Bude's 4 Ever", + "url": r"re:https://d\d?\.f(uraffinity|acdn)\.net/art", + "user": "mirlinthloth", + "views": int, + "favorites": int, + "comments": int, + "rating": "General", + "fa_category": "Music", + "theme": "All", + "species": "Unspecified / Any", + "gender": "Any", + "width": 120, + "height": 120, + }, + { + "#url": "https://www.furaffinity.net/view/42166511/", + "#comment": "'external' option (#1492)", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + "#options": {"external": True}, + "#pattern": r"https://d\d*\.f(uraffinity|acdn)\.net/|http://www\.postybirb\.com", + "#count": 2, + }, + { + "#url": "https://www.furaffinity.net/view/45331225/", + "#comment": "no tags (#2277)", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + "artist": "Kota_Remminders", + "artist_url": "kotaremminders", + "date": "dt:2022-01-03 17:49:33", + "fa_category": "Adoptables", + "filename": "1641232173.kotaremminders_chidopts1", + "gender": "Any", + "height": 905, + "id": 45331225, + "rating": "General", + "species": "Unspecified / Any", + "tags": [], + "theme": "All", + "title": "REMINDER", + "width": 1280, + }, + { + "#url": "https://www.furaffinity.net/view/22964019/", + "#comment": "get thumbnails for posts (#1284)", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + "artist": "Dwale", + "artist_url": "dwale", + "date": "dt:2017-03-21 14:21:29", + "fa_category": "Poetry", + "filename": "1490106089.dwale_poem_for_children", + "folders": [], + "height": 50, + "id": 22964019, + "rating": "General", + "title": "Poem for Children Wishing to Summon Evil Spirits", + "thumbnail": "https://t.furaffinity.net/22964019@600-1490106089.jpg", + "width": 50, + }, + { + "#url": "https://www.furaffinity.net/view/34260156/", + "#comment": "list gallery folders for image", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + "artist": "dbd", + "artist_url": "dbd", + "date": "dt:2019-12-17 22:52:01", + "fa_category": "All", + "filename": "1576623121.dbd_patreoncustom-wdg13-web", + "folders": [ + "By Year - 2019", + "Custom Character Folder - All Custom Characters", + "Custom Character Folder - Other Ungulates", + "Custom Character Folder - Female", + "Custom Character Folder - Patreon Supported Custom Characters", + ], + "height": 900, + "id": 34260156, + "rating": "General", + "title": "Patreon Custom Deer", + "thumbnail": "https://t.furaffinity.net/34260156@600-1576623121.jpg", + "width": 488, + }, + { + "#url": "https://www.furaffinity.net/view/57587562", + "#comment": "login required", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + "#count": 0, + }, + { + "#url": "https://furaffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + }, + { + "#url": "https://fxfuraffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + }, + { + "#url": "https://xfuraffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + }, + { + "#url": "https://fxraffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + }, + { + "#url": "https://sfw.furaffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + }, + { + "#url": "https://www.furaffinity.net/full/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + }, + { + "#url": "https://www.furaffinity.net/user/mirlinthloth/", + "#category": ("", "furaffinity", "user"), + "#class": furaffinity.FuraffinityUserExtractor, + "#pattern": "/gallery/mirlinthloth/$", + }, + { + "#url": "https://www.furaffinity.net/user/mirlinthloth/", + "#category": ("", "furaffinity", "user"), + "#class": furaffinity.FuraffinityUserExtractor, + "#options": {"include": "all"}, + "#pattern": "/(gallery|scraps|favorites)/mirlinthloth/$", + "#count": 3, + }, + { + "#url": "https://www.furaffinity.net/watchlist/by/mirlinthloth/", + "#category": ("", "furaffinity", "following"), + "#class": furaffinity.FuraffinityFollowingExtractor, + "#pattern": furaffinity.FuraffinityUserExtractor.pattern, + "#range": "176-225", + "#count": 50, + }, + { + "#url": "https://www.furaffinity.net/msg/submissions", + "#category": ("", "furaffinity", "submissions"), + "#class": furaffinity.FuraffinitySubmissionsExtractor, + "#auth": True, + "#pattern": r"https://d\d?\.f(uraffinity|acdn)\.net/art/mirlinthloth/\d+/\d+.\w+\.\w+", + "#range": "45-50", + "#count": 6, + }, + { + "#url": "https://www.furaffinity.net/msg/submissions/new~56789000@48/", + "#category": ("", "furaffinity", "submissions"), + "#class": furaffinity.FuraffinitySubmissionsExtractor, + "#auth": True, + }, ) diff --git a/test/results/furbooru.py b/test/results/furbooru.py index 682aa0054..449eea2f5 100644 --- a/test/results/furbooru.py +++ b/test/results/furbooru.py @@ -1,33 +1,27 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import philomena - __tests__ = ( -{ - "#url" : "https://furbooru.org/images/1", - "#category": ("philomena", "furbooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, - "#sha1_content": "9eaa1e1b32fa0f16520912257dbefaff238d5fd2", -}, - -{ - "#url" : "https://furbooru.org/search?q=cute", - "#category": ("philomena", "furbooru", "search"), - "#class" : philomena.PhilomenaSearchExtractor, - "#range" : "40-60", - "#count" : 21, -}, - -{ - "#url" : "https://furbooru.org/galleries/27", - "#category": ("philomena", "furbooru", "gallery"), - "#class" : philomena.PhilomenaGalleryExtractor, - "#count" : ">= 13", -}, - + { + "#url": "https://furbooru.org/images/1", + "#category": ("philomena", "furbooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + "#sha1_content": "9eaa1e1b32fa0f16520912257dbefaff238d5fd2", + }, + { + "#url": "https://furbooru.org/search?q=cute", + "#category": ("philomena", "furbooru", "search"), + "#class": philomena.PhilomenaSearchExtractor, + "#range": "40-60", + "#count": 21, + }, + { + "#url": "https://furbooru.org/galleries/27", + "#category": ("philomena", "furbooru", "gallery"), + "#class": philomena.PhilomenaGalleryExtractor, + "#count": ">= 13", + }, ) diff --git a/test/results/fuskator.py b/test/results/fuskator.py index cd68c9c2f..c51d40f22 100644 --- a/test/results/fuskator.py +++ b/test/results/fuskator.py @@ -1,47 +1,39 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import fuskator - __tests__ = ( -{ - "#url" : "https://fuskator.com/thumbs/d0GnIzXrSKU/", - "#category": ("", "fuskator", "gallery"), - "#class" : fuskator.FuskatorGalleryExtractor, - "#pattern" : r"https://i\d+.fuskator.com/large/d0GnIzXrSKU/.+\.jpg", - "#count" : 22, - - "gallery_id" : 473023, - "gallery_hash": "d0GnIzXrSKU", - "title" : r"re:Shaved Brunette Babe Maria Ryabushkina with ", - "views" : int, - "score" : float, - "count" : 22, - "tags" : list, -}, - -{ - "#url" : "https://fuskator.com/expanded/gXpKzjgIidA/index.html", - "#category": ("", "fuskator", "gallery"), - "#class" : fuskator.FuskatorGalleryExtractor, -}, - -{ - "#url" : "https://fuskator.com/search/red_swimsuit/", - "#category": ("", "fuskator", "search"), - "#class" : fuskator.FuskatorSearchExtractor, - "#pattern" : fuskator.FuskatorGalleryExtractor.pattern, - "#count" : ">= 40", -}, - -{ - "#url" : "https://fuskator.com/page/3/swimsuit/quality/", - "#category": ("", "fuskator", "search"), - "#class" : fuskator.FuskatorSearchExtractor, -}, - + { + "#url": "https://fuskator.com/thumbs/d0GnIzXrSKU/", + "#category": ("", "fuskator", "gallery"), + "#class": fuskator.FuskatorGalleryExtractor, + "#pattern": r"https://i\d+.fuskator.com/large/d0GnIzXrSKU/.+\.jpg", + "#count": 22, + "gallery_id": 473023, + "gallery_hash": "d0GnIzXrSKU", + "title": r"re:Shaved Brunette Babe Maria Ryabushkina with ", + "views": int, + "score": float, + "count": 22, + "tags": list, + }, + { + "#url": "https://fuskator.com/expanded/gXpKzjgIidA/index.html", + "#category": ("", "fuskator", "gallery"), + "#class": fuskator.FuskatorGalleryExtractor, + }, + { + "#url": "https://fuskator.com/search/red_swimsuit/", + "#category": ("", "fuskator", "search"), + "#class": fuskator.FuskatorSearchExtractor, + "#pattern": fuskator.FuskatorGalleryExtractor.pattern, + "#count": ">= 40", + }, + { + "#url": "https://fuskator.com/page/3/swimsuit/quality/", + "#category": ("", "fuskator", "search"), + "#class": fuskator.FuskatorSearchExtractor, + }, ) diff --git a/test/results/gelbooru.py b/test/results/gelbooru.py index 3f09ea690..b1808029c 100644 --- a/test/results/gelbooru.py +++ b/test/results/gelbooru.py @@ -1,192 +1,167 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru - __tests__ = ( -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", - "#category": ("booru", "gelbooru", "tag"), - "#class" : gelbooru.GelbooruTagExtractor, - "#count" : 5, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=all", - "#category": ("booru", "gelbooru", "tag"), - "#class" : gelbooru.GelbooruTagExtractor, - "#range" : "1-3", - "#count" : 3, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=", - "#category": ("booru", "gelbooru", "tag"), - "#class" : gelbooru.GelbooruTagExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=meiya_neon", - "#category": ("booru", "gelbooru", "tag"), - "#class" : gelbooru.GelbooruTagExtractor, - "#pattern" : r"https://img\d\.gelbooru\.com/images/../../[0-9a-f]{32}\.jpg", - "#range" : "196-204", - "#count" : 9, - "#sha1_url": "845a61aa1f90fb4ced841e8b7e62098be2e967bf", -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=id:>=67800+id:<=68000", - "#comment" : "meta tags (#5478)", - "#category": ("booru", "gelbooru", "tag"), - "#class" : gelbooru.GelbooruTagExtractor, - "#count" : 187, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=id:>=67800+id:<=68000+sort:id:asc", - "#comment" : "meta + sort tags (#5478)", - "#category": ("booru", "gelbooru", "tag"), - "#class" : gelbooru.GelbooruTagExtractor, - "#count" : 187, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=pool&s=show&id=761", - "#category": ("booru", "gelbooru", "pool"), - "#class" : gelbooru.GelbooruPoolExtractor, - "#count" : 6, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=favorites&s=view&id=1435674", - "#category": ("booru", "gelbooru", "favorite"), - "#class" : gelbooru.GelbooruFavoriteExtractor, - "#urls" : ( - "https://img3.gelbooru.com/images/5d/30/5d30fc056ed8668616b3c440df9bac89.jpg", - "https://img3.gelbooru.com/images/4c/2d/4c2da867ed643acdadd8105177dcdaf0.png", - "https://img3.gelbooru.com/images/c8/26/c826f3cb90d9aaca8d0632a96bf4abe8.jpg", - "https://img3.gelbooru.com/images/c1/fe/c1fe59c0bc8ce955dd353544b1015d0c.jpg", - "https://img3.gelbooru.com/images/e6/6d/e66d8883c184f5d3b2591dfcdf0d007c.jpg", - ), -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=favorites&s=view&id=1435674", - "#category": ("booru", "gelbooru", "favorite"), - "#class" : gelbooru.GelbooruFavoriteExtractor, - "#options" : {"order-posts": "reverse"}, - "#urls" : ( - "https://img3.gelbooru.com/images/e6/6d/e66d8883c184f5d3b2591dfcdf0d007c.jpg", - "https://img3.gelbooru.com/images/c1/fe/c1fe59c0bc8ce955dd353544b1015d0c.jpg", - "https://img3.gelbooru.com/images/c8/26/c826f3cb90d9aaca8d0632a96bf4abe8.jpg", - "https://img3.gelbooru.com/images/4c/2d/4c2da867ed643acdadd8105177dcdaf0.png", - "https://img3.gelbooru.com/images/5d/30/5d30fc056ed8668616b3c440df9bac89.jpg", - ), -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=313638", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, - "#count" : 1, - "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=313638", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?s=view&page=post&id=313638", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&id=313638&s=view", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?s=view&id=313638&page=post", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?id=313638&page=post&s=view", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?id=313638&s=view&page=post", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=6018318", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, - "#options" : {"tags": True}, - "#sha1_content": "977caf22f27c72a5d07ea4d4d9719acdab810991", - - "tags_artist" : "kirisaki_shuusei", - "tags_character": str, - "tags_copyright": "vocaloid", - "tags_general" : str, - "tags_metadata" : str, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=5938076", - "#comment" : "video", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, - "#pattern" : r"https://img\d\.gelbooru\.com/images/22/61/226111273615049235b001b381707bd0\.webm", - "#sha1_content": "6360452fa8c2f0c1137749e81471238564df832a", -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=5997331", - "#comment" : "notes", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, - "#options" : {"notes": True}, - - "notes": [ - { - "body" : "Look over this way when you talk~", - "height": 553, - "width" : 246, - "x" : 35, - "y" : 72, - }, - { - "body" : """Hey~ + { + "#url": "https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", + "#category": ("booru", "gelbooru", "tag"), + "#class": gelbooru.GelbooruTagExtractor, + "#count": 5, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=list&tags=all", + "#category": ("booru", "gelbooru", "tag"), + "#class": gelbooru.GelbooruTagExtractor, + "#range": "1-3", + "#count": 3, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=list&tags=", + "#category": ("booru", "gelbooru", "tag"), + "#class": gelbooru.GelbooruTagExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=list&tags=meiya_neon", + "#category": ("booru", "gelbooru", "tag"), + "#class": gelbooru.GelbooruTagExtractor, + "#pattern": r"https://img\d\.gelbooru\.com/images/../../[0-9a-f]{32}\.jpg", + "#range": "196-204", + "#count": 9, + "#sha1_url": "845a61aa1f90fb4ced841e8b7e62098be2e967bf", + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=list&tags=id:>=67800+id:<=68000", + "#comment": "meta tags (#5478)", + "#category": ("booru", "gelbooru", "tag"), + "#class": gelbooru.GelbooruTagExtractor, + "#count": 187, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=list&tags=id:>=67800+id:<=68000+sort:id:asc", + "#comment": "meta + sort tags (#5478)", + "#category": ("booru", "gelbooru", "tag"), + "#class": gelbooru.GelbooruTagExtractor, + "#count": 187, + }, + { + "#url": "https://gelbooru.com/index.php?page=pool&s=show&id=761", + "#category": ("booru", "gelbooru", "pool"), + "#class": gelbooru.GelbooruPoolExtractor, + "#count": 6, + }, + { + "#url": "https://gelbooru.com/index.php?page=favorites&s=view&id=1435674", + "#category": ("booru", "gelbooru", "favorite"), + "#class": gelbooru.GelbooruFavoriteExtractor, + "#urls": ( + "https://img3.gelbooru.com/images/5d/30/5d30fc056ed8668616b3c440df9bac89.jpg", + "https://img3.gelbooru.com/images/4c/2d/4c2da867ed643acdadd8105177dcdaf0.png", + "https://img3.gelbooru.com/images/c8/26/c826f3cb90d9aaca8d0632a96bf4abe8.jpg", + "https://img3.gelbooru.com/images/c1/fe/c1fe59c0bc8ce955dd353544b1015d0c.jpg", + "https://img3.gelbooru.com/images/e6/6d/e66d8883c184f5d3b2591dfcdf0d007c.jpg", + ), + }, + { + "#url": "https://gelbooru.com/index.php?page=favorites&s=view&id=1435674", + "#category": ("booru", "gelbooru", "favorite"), + "#class": gelbooru.GelbooruFavoriteExtractor, + "#options": {"order-posts": "reverse"}, + "#urls": ( + "https://img3.gelbooru.com/images/e6/6d/e66d8883c184f5d3b2591dfcdf0d007c.jpg", + "https://img3.gelbooru.com/images/c1/fe/c1fe59c0bc8ce955dd353544b1015d0c.jpg", + "https://img3.gelbooru.com/images/c8/26/c826f3cb90d9aaca8d0632a96bf4abe8.jpg", + "https://img3.gelbooru.com/images/4c/2d/4c2da867ed643acdadd8105177dcdaf0.png", + "https://img3.gelbooru.com/images/5d/30/5d30fc056ed8668616b3c440df9bac89.jpg", + ), + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=view&id=313638", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + "#count": 1, + "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=view&id=313638", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?s=view&page=post&id=313638", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&id=313638&s=view", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?s=view&id=313638&page=post", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?id=313638&page=post&s=view", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?id=313638&s=view&page=post", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=view&id=6018318", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + "#options": {"tags": True}, + "#sha1_content": "977caf22f27c72a5d07ea4d4d9719acdab810991", + "tags_artist": "kirisaki_shuusei", + "tags_character": str, + "tags_copyright": "vocaloid", + "tags_general": str, + "tags_metadata": str, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=view&id=5938076", + "#comment": "video", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + "#pattern": r"https://img\d\.gelbooru\.com/images/22/61/226111273615049235b001b381707bd0\.webm", + "#sha1_content": "6360452fa8c2f0c1137749e81471238564df832a", + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=view&id=5997331", + "#comment": "notes", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + "#options": {"notes": True}, + "notes": [ + { + "body": "Look over this way when you talk~", + "height": 553, + "width": 246, + "x": 35, + "y": 72, + }, + { + "body": """Hey~ Are you listening~?""", - "height": 557, - "width" : 246, - "x" : 1233, - "y" : 109, - }, - ], -}, - -{ - "#url" : "https://gelbooru.com/redirect.php?s=Ly9nZWxib29ydS5jb20vaW5kZXgucGhwP3BhZ2U9cG9zdCZzPXZpZXcmaWQ9MTgzMDA0Ng==", - "#category": ("booru", "gelbooru", "redirect"), - "#class" : gelbooru.GelbooruRedirectExtractor, - "#pattern" : r"https://gelbooru.com/index.php\?page=post&s=view&id=1830046", -}, - + "height": 557, + "width": 246, + "x": 1233, + "y": 109, + }, + ], + }, + { + "#url": "https://gelbooru.com/redirect.php?s=Ly9nZWxib29ydS5jb20vaW5kZXgucGhwP3BhZ2U9cG9zdCZzPXZpZXcmaWQ9MTgzMDA0Ng==", + "#category": ("booru", "gelbooru", "redirect"), + "#class": gelbooru.GelbooruRedirectExtractor, + "#pattern": r"https://gelbooru.com/index.php\?page=post&s=view&id=1830046", + }, ) diff --git a/test/results/generic.py b/test/results/generic.py index 4d940afee..e440ef8fe 100644 --- a/test/results/generic.py +++ b/test/results/generic.py @@ -1,68 +1,56 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import generic - __tests__ = ( -{ - "#url" : "generic:https://www.nongnu.org/lzip/", - "#category": ("", "generic", "www.nongnu.org"), - "#class" : generic.GenericExtractor, - "#count" : 1, - "#sha1_content": "40be5c77773d3e91db6e1c5df720ee30afb62368", - - "description": "Lossless data compressor", - "imageurl" : "https://www.nongnu.org/lzip/lzip.png", - "keywords" : "lzip, clzip, plzip, lzlib, LZMA, bzip2, gzip, data compression, GNU, free software", - "pageurl" : "https://www.nongnu.org/lzip/", -}, - -{ - "#url" : "generic:https://räksmörgås.josefsson.org/", - "#category": ("", "generic", "räksmörgås.josefsson.org"), - "#class" : generic.GenericExtractor, - "#pattern" : "^https://räksmörgås.josefsson.org/", - "#count" : 2, -}, - -{ - "#url" : "generic:https://en.wikipedia.org/Main_Page", - "#category": ("", "generic", "en.wikipedia.org"), - "#class" : generic.GenericExtractor, -}, - -{ - "#url" : "generic:https://example.org/path/to/file?que=1?&ry=2/#fragment", - "#category": ("", "generic", "example.org"), - "#class" : generic.GenericExtractor, -}, - -{ - "#url" : "generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", - "#category": ("", "generic", "example.org"), - "#class" : generic.GenericExtractor, -}, - -{ - "#url" : "generic:https://en.wikipedia.org/Main_Page", - "#category": ("", "generic", "en.wikipedia.org"), - "#class" : generic.GenericExtractor, -}, - -{ - "#url" : "generic:https://example.org/path/to/file?que=1?&ry=2/#fragment", - "#category": ("", "generic", "example.org"), - "#class" : generic.GenericExtractor, -}, - -{ - "#url" : "generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", - "#category": ("", "generic", "example.org"), - "#class" : generic.GenericExtractor, -}, - + { + "#url": "generic:https://www.nongnu.org/lzip/", + "#category": ("", "generic", "www.nongnu.org"), + "#class": generic.GenericExtractor, + "#count": 1, + "#sha1_content": "40be5c77773d3e91db6e1c5df720ee30afb62368", + "description": "Lossless data compressor", + "imageurl": "https://www.nongnu.org/lzip/lzip.png", + "keywords": "lzip, clzip, plzip, lzlib, LZMA, bzip2, gzip, data compression, GNU, free software", + "pageurl": "https://www.nongnu.org/lzip/", + }, + { + "#url": "generic:https://räksmörgås.josefsson.org/", + "#category": ("", "generic", "räksmörgås.josefsson.org"), + "#class": generic.GenericExtractor, + "#pattern": "^https://räksmörgås.josefsson.org/", + "#count": 2, + }, + { + "#url": "generic:https://en.wikipedia.org/Main_Page", + "#category": ("", "generic", "en.wikipedia.org"), + "#class": generic.GenericExtractor, + }, + { + "#url": "generic:https://example.org/path/to/file?que=1?&ry=2/#fragment", + "#category": ("", "generic", "example.org"), + "#class": generic.GenericExtractor, + }, + { + "#url": "generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", + "#category": ("", "generic", "example.org"), + "#class": generic.GenericExtractor, + }, + { + "#url": "generic:https://en.wikipedia.org/Main_Page", + "#category": ("", "generic", "en.wikipedia.org"), + "#class": generic.GenericExtractor, + }, + { + "#url": "generic:https://example.org/path/to/file?que=1?&ry=2/#fragment", + "#category": ("", "generic", "example.org"), + "#class": generic.GenericExtractor, + }, + { + "#url": "generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", + "#category": ("", "generic", "example.org"), + "#class": generic.GenericExtractor, + }, ) diff --git a/test/results/giantessbooru.py b/test/results/giantessbooru.py index e35b9d107..7683ddcc0 100644 --- a/test/results/giantessbooru.py +++ b/test/results/giantessbooru.py @@ -1,62 +1,52 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shimmie2 - __tests__ = ( -{ - "#url" : "https://sizechangebooru.com/index.php?q=/post/list/drawing/1", - "#category": ("shimmie2", "giantessbooru", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, - "#pattern" : r"https://sizechangebooru\.com/index\.php\?q=/image/\d+\.jpg", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://giantessbooru.com/index.php?q=/post/list/drawing/1", - "#category": ("shimmie2", "giantessbooru", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, -}, - -{ - "#url" : "https://giantessbooru.com/post/list/drawing/1", - "#category": ("shimmie2", "giantessbooru", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, -}, - -{ - "#url" : "https://sizechangebooru.com/index.php?q=/post/view/41", - "#category": ("shimmie2", "giantessbooru", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, - "#urls" : "https://sizechangebooru.com/index.php?q=/image/41.jpg", - "#sha1_content": "79115ed309d1f4e82e7bead6948760e889139c91", - - "extension": "jpg", - "file_url" : "https://sizechangebooru.com/index.php?q=/image/41.jpg", - "filename" : "41", - "height" : 0, - "id" : 41, - "md5" : "", - "size" : 0, - "tags" : "anime bare_midriff color drawing gentle giantess karbo looking_at_tinies negeyari outdoors smiling snake_girl white_hair", - "width" : 1387, -}, - -{ - "#url" : "https://giantessbooru.com/index.php?q=/post/view/41", - "#category": ("shimmie2", "giantessbooru", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, -}, - -{ - "#url" : "https://giantessbooru.com/post/view/41", - "#category": ("shimmie2", "giantessbooru", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, -}, - + { + "#url": "https://sizechangebooru.com/index.php?q=/post/list/drawing/1", + "#category": ("shimmie2", "giantessbooru", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + "#pattern": r"https://sizechangebooru\.com/index\.php\?q=/image/\d+\.jpg", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://giantessbooru.com/index.php?q=/post/list/drawing/1", + "#category": ("shimmie2", "giantessbooru", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + }, + { + "#url": "https://giantessbooru.com/post/list/drawing/1", + "#category": ("shimmie2", "giantessbooru", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + }, + { + "#url": "https://sizechangebooru.com/index.php?q=/post/view/41", + "#category": ("shimmie2", "giantessbooru", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + "#urls": "https://sizechangebooru.com/index.php?q=/image/41.jpg", + "#sha1_content": "79115ed309d1f4e82e7bead6948760e889139c91", + "extension": "jpg", + "file_url": "https://sizechangebooru.com/index.php?q=/image/41.jpg", + "filename": "41", + "height": 0, + "id": 41, + "md5": "", + "size": 0, + "tags": "anime bare_midriff color drawing gentle giantess karbo looking_at_tinies negeyari outdoors smiling snake_girl white_hair", + "width": 1387, + }, + { + "#url": "https://giantessbooru.com/index.php?q=/post/view/41", + "#category": ("shimmie2", "giantessbooru", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + }, + { + "#url": "https://giantessbooru.com/post/view/41", + "#category": ("shimmie2", "giantessbooru", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + }, ) diff --git a/test/results/gofile.py b/test/results/gofile.py index b7d267027..94d6acb95 100644 --- a/test/results/gofile.py +++ b/test/results/gofile.py @@ -1,57 +1,51 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gofile - __tests__ = ( -{ - "#url" : "https://gofile.io/d/k6BomI", - "#category": ("", "gofile", "folder"), - "#class" : gofile.GofileFolderExtractor, - "#pattern" : r"https://store\d+\.gofile\.io/download/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/test-%E3%83%86%E3%82%B9%E3%83%88-%2522%26!\.png", - - "createTime" : int, - "directLink" : r"re:https://store5.gofile.io/download/direct/.+", - "downloadCount": int, - "extension" : "png", - "filename" : "test-テスト-%22&!", - "folder" : { - "childs" : [ - "b0367d79-b8ba-407f-8342-aaf8eb815443", - "7fd4a36a-c1dd-49ff-9223-d93f7d24093f", - ], - "code" : "k6BomI", - "createTime" : 1654076165, - "id" : "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", - "name" : "root", - "public" : True, - "totalDownloadCount": int, - "totalSize" : 182, - "type" : "folder", + { + "#url": "https://gofile.io/d/k6BomI", + "#category": ("", "gofile", "folder"), + "#class": gofile.GofileFolderExtractor, + "#pattern": r"https://store\d+\.gofile\.io/download/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/test-%E3%83%86%E3%82%B9%E3%83%88-%2522%26!\.png", + "createTime": int, + "directLink": r"re:https://store5.gofile.io/download/direct/.+", + "downloadCount": int, + "extension": "png", + "filename": "test-テスト-%22&!", + "folder": { + "childs": [ + "b0367d79-b8ba-407f-8342-aaf8eb815443", + "7fd4a36a-c1dd-49ff-9223-d93f7d24093f", + ], + "code": "k6BomI", + "createTime": 1654076165, + "id": "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", + "name": "root", + "public": True, + "totalDownloadCount": int, + "totalSize": 182, + "type": "folder", + }, + "id": r"re:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}", + "link": r"re:https://store5.gofile.io/download/.+\.png", + "md5": r"re:[0-9a-f]{32}", + "mimetype": "image/png", + "name": "test-テスト-%22&!.png", + "num": int, + "parentFolder": "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", + "serverChoosen": "store5", + "size": 182, + "thumbnail": r"re:https://store5.gofile.io/download/.+\.png", + "type": "file", + }, + { + "#url": "https://gofile.io/d/7fd4a36a-c1dd-49ff-9223-d93f7d24093f", + "#category": ("", "gofile", "folder"), + "#class": gofile.GofileFolderExtractor, + "#options": {"website-token": None}, + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", }, - "id" : r"re:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}", - "link" : r"re:https://store5.gofile.io/download/.+\.png", - "md5" : r"re:[0-9a-f]{32}", - "mimetype" : "image/png", - "name" : "test-テスト-%22&!.png", - "num" : int, - "parentFolder" : "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", - "serverChoosen": "store5", - "size" : 182, - "thumbnail" : r"re:https://store5.gofile.io/download/.+\.png", - "type" : "file", -}, - -{ - "#url" : "https://gofile.io/d/7fd4a36a-c1dd-49ff-9223-d93f7d24093f", - "#category": ("", "gofile", "folder"), - "#class" : gofile.GofileFolderExtractor, - "#options" : {"website-token": None}, - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", -}, - ) diff --git a/test/results/hatenablog.py b/test/results/hatenablog.py index 4a306f9a1..f6e24f022 100644 --- a/test/results/hatenablog.py +++ b/test/results/hatenablog.py @@ -1,144 +1,123 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hatenablog - __tests__ = ( -{ - "#url" : "https://cosmiclatte.hatenablog.com/entry/2020/05/28/003227", - "#category": ("", "hatenablog", "entry"), - "#class" : hatenablog.HatenablogEntryExtractor, - "#count" : 20, -}, - -{ - "#url" : "https://moko0908.hatenablog.jp/entry/2023/12/31/083846", - "#category": ("", "hatenablog", "entry"), - "#class" : hatenablog.HatenablogEntryExtractor, -}, - -{ - "#url" : "https://p-shirokuma.hatenadiary.com/entry/20231227/1703685600", - "#category": ("", "hatenablog", "entry"), - "#class" : hatenablog.HatenablogEntryExtractor, -}, - -{ - "#url" : "https://urakatahero.hateblo.jp/entry/2ndlife", - "#category": ("", "hatenablog", "entry"), - "#class" : hatenablog.HatenablogEntryExtractor, -}, - -{ - "#url" : "hatenablog:https://blog.hyouhon.com/entry/2023/12/22/133549", - "#category": ("", "hatenablog", "entry"), - "#class" : hatenablog.HatenablogEntryExtractor, -}, - -{ - "#url" : "https://cetriolo.hatenablog.com", - "#category": ("", "hatenablog", "home"), - "#class" : hatenablog.HatenablogHomeExtractor, - "#range" : "1-7", - "#count" : 7, -}, - -{ - "#url" : "https://moko0908.hatenablog.jp/", - "#category": ("", "hatenablog", "home"), - "#class" : hatenablog.HatenablogHomeExtractor, -}, - -{ - "#url" : "https://p-shirokuma.hatenadiary.com/", - "#category": ("", "hatenablog", "home"), - "#class" : hatenablog.HatenablogHomeExtractor, -}, - -{ - "#url" : "https://urakatahero.hateblo.jp/", - "#category": ("", "hatenablog", "home"), - "#class" : hatenablog.HatenablogHomeExtractor, -}, - -{ - "#url" : "hatenablog:https://blog.hyouhon.com/", - "#category": ("", "hatenablog", "home"), - "#class" : hatenablog.HatenablogHomeExtractor, -}, - -{ - "#url" : ("https://8saki.hatenablog.com/archive/category/%E3%82%BB%E3" - "%83%AB%E3%83%95%E3%82%B8%E3%82%A7%E3%83%AB%E3%83%8D%E3%82" - "%A4%E3%83%AB"), - "#category": ("", "hatenablog", "archive"), - "#class" : hatenablog.HatenablogArchiveExtractor, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://moko0908.hatenablog.jp/archive/2023", - "#category": ("", "hatenablog", "archive"), - "#class" : hatenablog.HatenablogArchiveExtractor, - "#count" : 13, -}, - -{ - "#url" : "https://p-shirokuma.hatenadiary.com/archive/2023/01", - "#category": ("", "hatenablog", "archive"), - "#class" : hatenablog.HatenablogArchiveExtractor, - "#count" : 5, -}, - -{ - "#url" : "https://urakatahero.hateblo.jp/archive", - "#category": ("", "hatenablog", "archive"), - "#class" : hatenablog.HatenablogArchiveExtractor, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "hatenablog:https://blog.hyouhon.com/archive/2024/01/01", - "#category": ("", "hatenablog", "archive"), - "#class" : hatenablog.HatenablogArchiveExtractor, -}, - -{ - "#url" : "hatenablog:https://blog.hyouhon.com/search?q=a", - "#category": ("", "hatenablog", "search"), - "#class" : hatenablog.HatenablogSearchExtractor, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://cosmiclatte.hatenablog.com/search?q=a", - "#category": ("", "hatenablog", "search"), - "#class" : hatenablog.HatenablogSearchExtractor, -}, - -{ - "#url" : "https://moko0908.hatenablog.jp/search?q=a", - "#category": ("", "hatenablog", "search"), - "#class" : hatenablog.HatenablogSearchExtractor, -}, - -{ - "#url" : "https://p-shirokuma.hatenadiary.com/search?q=a", - "#category": ("", "hatenablog", "search"), - "#class" : hatenablog.HatenablogSearchExtractor, -}, - -{ - "#url" : "https://urakatahero.hateblo.jp/search?q=a", - "#category": ("", "hatenablog", "search"), - "#class" : hatenablog.HatenablogSearchExtractor, -}, - + { + "#url": "https://cosmiclatte.hatenablog.com/entry/2020/05/28/003227", + "#category": ("", "hatenablog", "entry"), + "#class": hatenablog.HatenablogEntryExtractor, + "#count": 20, + }, + { + "#url": "https://moko0908.hatenablog.jp/entry/2023/12/31/083846", + "#category": ("", "hatenablog", "entry"), + "#class": hatenablog.HatenablogEntryExtractor, + }, + { + "#url": "https://p-shirokuma.hatenadiary.com/entry/20231227/1703685600", + "#category": ("", "hatenablog", "entry"), + "#class": hatenablog.HatenablogEntryExtractor, + }, + { + "#url": "https://urakatahero.hateblo.jp/entry/2ndlife", + "#category": ("", "hatenablog", "entry"), + "#class": hatenablog.HatenablogEntryExtractor, + }, + { + "#url": "hatenablog:https://blog.hyouhon.com/entry/2023/12/22/133549", + "#category": ("", "hatenablog", "entry"), + "#class": hatenablog.HatenablogEntryExtractor, + }, + { + "#url": "https://cetriolo.hatenablog.com", + "#category": ("", "hatenablog", "home"), + "#class": hatenablog.HatenablogHomeExtractor, + "#range": "1-7", + "#count": 7, + }, + { + "#url": "https://moko0908.hatenablog.jp/", + "#category": ("", "hatenablog", "home"), + "#class": hatenablog.HatenablogHomeExtractor, + }, + { + "#url": "https://p-shirokuma.hatenadiary.com/", + "#category": ("", "hatenablog", "home"), + "#class": hatenablog.HatenablogHomeExtractor, + }, + { + "#url": "https://urakatahero.hateblo.jp/", + "#category": ("", "hatenablog", "home"), + "#class": hatenablog.HatenablogHomeExtractor, + }, + { + "#url": "hatenablog:https://blog.hyouhon.com/", + "#category": ("", "hatenablog", "home"), + "#class": hatenablog.HatenablogHomeExtractor, + }, + { + "#url": ( + "https://8saki.hatenablog.com/archive/category/%E3%82%BB%E3" + "%83%AB%E3%83%95%E3%82%B8%E3%82%A7%E3%83%AB%E3%83%8D%E3%82" + "%A4%E3%83%AB" + ), + "#category": ("", "hatenablog", "archive"), + "#class": hatenablog.HatenablogArchiveExtractor, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://moko0908.hatenablog.jp/archive/2023", + "#category": ("", "hatenablog", "archive"), + "#class": hatenablog.HatenablogArchiveExtractor, + "#count": 13, + }, + { + "#url": "https://p-shirokuma.hatenadiary.com/archive/2023/01", + "#category": ("", "hatenablog", "archive"), + "#class": hatenablog.HatenablogArchiveExtractor, + "#count": 5, + }, + { + "#url": "https://urakatahero.hateblo.jp/archive", + "#category": ("", "hatenablog", "archive"), + "#class": hatenablog.HatenablogArchiveExtractor, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "hatenablog:https://blog.hyouhon.com/archive/2024/01/01", + "#category": ("", "hatenablog", "archive"), + "#class": hatenablog.HatenablogArchiveExtractor, + }, + { + "#url": "hatenablog:https://blog.hyouhon.com/search?q=a", + "#category": ("", "hatenablog", "search"), + "#class": hatenablog.HatenablogSearchExtractor, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://cosmiclatte.hatenablog.com/search?q=a", + "#category": ("", "hatenablog", "search"), + "#class": hatenablog.HatenablogSearchExtractor, + }, + { + "#url": "https://moko0908.hatenablog.jp/search?q=a", + "#category": ("", "hatenablog", "search"), + "#class": hatenablog.HatenablogSearchExtractor, + }, + { + "#url": "https://p-shirokuma.hatenadiary.com/search?q=a", + "#category": ("", "hatenablog", "search"), + "#class": hatenablog.HatenablogSearchExtractor, + }, + { + "#url": "https://urakatahero.hateblo.jp/search?q=a", + "#category": ("", "hatenablog", "search"), + "#class": hatenablog.HatenablogSearchExtractor, + }, ) diff --git a/test/results/hentai2read.py b/test/results/hentai2read.py index 01349c2ec..eb8f544da 100644 --- a/test/results/hentai2read.py +++ b/test/results/hentai2read.py @@ -1,74 +1,64 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hentai2read - __tests__ = ( -{ - "#url" : "https://hentai2read.com/amazon_elixir/1/", - "#category": ("", "hentai2read", "chapter"), - "#class" : hentai2read.Hentai2readChapterExtractor, - "#sha1_url" : "964b942cf492b3a129d2fe2608abfc475bc99e71", - "#sha1_metadata": "85645b02d34aa11b3deb6dadd7536863476e1bad", -}, - -{ - "#url" : "https://hentai2read.com/popuni_kei_joshi_panic/2.5/", - "#category": ("", "hentai2read", "chapter"), - "#class" : hentai2read.Hentai2readChapterExtractor, - "#pattern" : r"https://hentaicdn\.com/hentai/13088/2\.5y/ccdn00\d+\.jpg", - "#count" : 36, - - "author" : "Kurisu", - "chapter" : 2, - "chapter_id" : 75152, - "chapter_minor": ".5", - "count" : 36, - "lang" : "en", - "language" : "English", - "manga" : "Popuni Kei Joshi Panic!", - "manga_id" : 13088, - "page" : int, - "title" : "Popuni Kei Joshi Panic! 2.5", - "type" : "Original", -}, - -{ - "#url" : "https://hentai2read.com/amazon_elixir/", - "#category": ("", "hentai2read", "manga"), - "#class" : hentai2read.Hentai2readMangaExtractor, - "#sha1_url" : "273073752d418ec887d7f7211e42b832e8c403ba", - "#sha1_metadata": "5c1b712258e78e120907121d3987c71f834d13e1", -}, - -{ - "#url" : "https://hentai2read.com/oshikage_riot/", - "#category": ("", "hentai2read", "manga"), - "#class" : hentai2read.Hentai2readMangaExtractor, - "#sha1_url" : "6595f920a3088a15c2819c502862d45f8eb6bea6", - "#sha1_metadata": "a2e9724acb221040d4b29bf9aa8cb75b2240d8af", -}, - -{ - "#url" : "https://hentai2read.com/popuni_kei_joshi_panic/", - "#category": ("", "hentai2read", "manga"), - "#class" : hentai2read.Hentai2readMangaExtractor, - "#pattern" : hentai2read.Hentai2readChapterExtractor.pattern, - "#range" : "2-3", - - "chapter" : int, - "chapter_id" : int, - "chapter_minor": ".5", - "lang" : "en", - "language" : "English", - "manga" : "Popuni Kei Joshi Panic!", - "manga_id" : 13088, - "title" : str, - "type" : "Original", -}, - + { + "#url": "https://hentai2read.com/amazon_elixir/1/", + "#category": ("", "hentai2read", "chapter"), + "#class": hentai2read.Hentai2readChapterExtractor, + "#sha1_url": "964b942cf492b3a129d2fe2608abfc475bc99e71", + "#sha1_metadata": "85645b02d34aa11b3deb6dadd7536863476e1bad", + }, + { + "#url": "https://hentai2read.com/popuni_kei_joshi_panic/2.5/", + "#category": ("", "hentai2read", "chapter"), + "#class": hentai2read.Hentai2readChapterExtractor, + "#pattern": r"https://hentaicdn\.com/hentai/13088/2\.5y/ccdn00\d+\.jpg", + "#count": 36, + "author": "Kurisu", + "chapter": 2, + "chapter_id": 75152, + "chapter_minor": ".5", + "count": 36, + "lang": "en", + "language": "English", + "manga": "Popuni Kei Joshi Panic!", + "manga_id": 13088, + "page": int, + "title": "Popuni Kei Joshi Panic! 2.5", + "type": "Original", + }, + { + "#url": "https://hentai2read.com/amazon_elixir/", + "#category": ("", "hentai2read", "manga"), + "#class": hentai2read.Hentai2readMangaExtractor, + "#sha1_url": "273073752d418ec887d7f7211e42b832e8c403ba", + "#sha1_metadata": "5c1b712258e78e120907121d3987c71f834d13e1", + }, + { + "#url": "https://hentai2read.com/oshikage_riot/", + "#category": ("", "hentai2read", "manga"), + "#class": hentai2read.Hentai2readMangaExtractor, + "#sha1_url": "6595f920a3088a15c2819c502862d45f8eb6bea6", + "#sha1_metadata": "a2e9724acb221040d4b29bf9aa8cb75b2240d8af", + }, + { + "#url": "https://hentai2read.com/popuni_kei_joshi_panic/", + "#category": ("", "hentai2read", "manga"), + "#class": hentai2read.Hentai2readMangaExtractor, + "#pattern": hentai2read.Hentai2readChapterExtractor.pattern, + "#range": "2-3", + "chapter": int, + "chapter_id": int, + "chapter_minor": ".5", + "lang": "en", + "language": "English", + "manga": "Popuni Kei Joshi Panic!", + "manga_id": 13088, + "title": str, + "type": "Original", + }, ) diff --git a/test/results/hentaicosplays.py b/test/results/hentaicosplays.py index 5964b445b..3afc0fe37 100644 --- a/test/results/hentaicosplays.py +++ b/test/results/hentaicosplays.py @@ -1,57 +1,46 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hentaicosplays - __tests__ = ( -{ - "#url" : "https://hentai-cosplay-xxx.com/image/---devilism--tide-kurihara-/", - "#category": ("", "hentaicosplays", "gallery"), - "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, - "#pattern" : r"https://static\d?\.hentai-cosplay-xxx\.com/upload/\d+/\d+/\d+/\d+\.jpg$", - - "count": 18, - "site" : "hentai-cosplay-xxx", - "slug" : "---devilism--tide-kurihara-", - "title": "艦 こ れ-devilism の tide Kurihara 憂", -}, - -{ - "#url" : "https://hentai-cosplays.com/image/---devilism--tide-kurihara-/", - "#category": ("", "hentaicosplays", "gallery"), - "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, - "#pattern" : r"https://static\d?\.hentai-cosplay-xxx\.com/upload/\d+/\d+/\d+/\d+\.jpg$", - - "count": 18, - "site" : "hentai-cosplays", - "slug" : "---devilism--tide-kurihara-", - "title": "艦 こ れ-devilism の tide Kurihara 憂", -}, - -{ - "#url" : "https://fr.porn-images-xxx.com/image/enako-enako-24/", - "#category": ("", "hentaicosplays", "gallery"), - "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, - "#pattern" : r"https://static\d?.porn-images-xxx.com/upload/\d+/\d+/\d+/\d+.jpg$", - - "count": 11, - "site" : "porn-images-xxx", - "title": str, -}, - -{ - "#url" : "https://ja.hentai-img.com/image/hollow-cora-502/", - "#category": ("", "hentaicosplays", "gallery"), - "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, - "#pattern" : r"https://static\d?.hentai-img.com/upload/\d+/\d+/\d+/\d+.jpg$", - - "count": 2, - "site" : "hentai-img", - "title": str, -}, - + { + "#url": "https://hentai-cosplay-xxx.com/image/---devilism--tide-kurihara-/", + "#category": ("", "hentaicosplays", "gallery"), + "#class": hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern": r"https://static\d?\.hentai-cosplay-xxx\.com/upload/\d+/\d+/\d+/\d+\.jpg$", + "count": 18, + "site": "hentai-cosplay-xxx", + "slug": "---devilism--tide-kurihara-", + "title": "艦 こ れ-devilism の tide Kurihara 憂", + }, + { + "#url": "https://hentai-cosplays.com/image/---devilism--tide-kurihara-/", + "#category": ("", "hentaicosplays", "gallery"), + "#class": hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern": r"https://static\d?\.hentai-cosplay-xxx\.com/upload/\d+/\d+/\d+/\d+\.jpg$", + "count": 18, + "site": "hentai-cosplays", + "slug": "---devilism--tide-kurihara-", + "title": "艦 こ れ-devilism の tide Kurihara 憂", + }, + { + "#url": "https://fr.porn-images-xxx.com/image/enako-enako-24/", + "#category": ("", "hentaicosplays", "gallery"), + "#class": hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern": r"https://static\d?.porn-images-xxx.com/upload/\d+/\d+/\d+/\d+.jpg$", + "count": 11, + "site": "porn-images-xxx", + "title": str, + }, + { + "#url": "https://ja.hentai-img.com/image/hollow-cora-502/", + "#category": ("", "hentaicosplays", "gallery"), + "#class": hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern": r"https://static\d?.hentai-img.com/upload/\d+/\d+/\d+/\d+.jpg$", + "count": 2, + "site": "hentai-img", + "title": str, + }, ) diff --git a/test/results/hentaifoundry.py b/test/results/hentaifoundry.py index 0335bfee6..a3c0506aa 100644 --- a/test/results/hentaifoundry.py +++ b/test/results/hentaifoundry.py @@ -1,196 +1,171 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import hentaifoundry import datetime +from gallery_dl.extractor import hentaifoundry __tests__ = ( -{ - "#url" : "https://www.hentai-foundry.com/user/Tenpura/profile", - "#category": ("", "hentaifoundry", "user"), - "#class" : hentaifoundry.HentaifoundryUserExtractor, -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura", - "#category": ("", "hentaifoundry", "pictures"), - "#class" : hentaifoundry.HentaifoundryPicturesExtractor, - "#sha1_url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28", -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura/page/3", - "#category": ("", "hentaifoundry", "pictures"), - "#class" : hentaifoundry.HentaifoundryPicturesExtractor, -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Ethevian/scraps", - "#category": ("", "hentaifoundry", "scraps"), - "#class" : hentaifoundry.HentaifoundryScrapsExtractor, - "#pattern" : r"https://pictures\.hentai-foundry\.com/e/Ethevian/.+", - "#count" : ">= 10", -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps/page/3", - "#category": ("", "hentaifoundry", "scraps"), - "#class" : hentaifoundry.HentaifoundryScrapsExtractor, -}, - -{ - "#url" : "https://www.hentai-foundry.com/user/Tenpura/faves/pictures", - "#category": ("", "hentaifoundry", "favorite"), - "#class" : hentaifoundry.HentaifoundryFavoriteExtractor, - "#sha1_url": "56f9ae2e89fe855e9fe1da9b81e5ec6212b0320b", -}, - -{ - "#url" : "https://www.hentai-foundry.com/user/Tenpura/faves/pictures/page/3", - "#category": ("", "hentaifoundry", "favorite"), - "#class" : hentaifoundry.HentaifoundryFavoriteExtractor, -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/tagged/kancolle", - "#category": ("", "hentaifoundry", "tag"), - "#class" : hentaifoundry.HentaifoundryTagExtractor, - "#pattern" : r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", - "#range" : "20-30", - - "search_tags": "kancolle", -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/recent/2018-09-20", - "#category": ("", "hentaifoundry", "recent"), - "#class" : hentaifoundry.HentaifoundryRecentExtractor, - "#pattern" : r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", - "#range" : "20-30", - - "date": "2018-09-20", -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/popular", - "#category": ("", "hentaifoundry", "popular"), - "#class" : hentaifoundry.HentaifoundryPopularExtractor, - "#pattern" : r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", - "#range" : "20-30", -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura/407501/shimakaze", - "#category": ("", "hentaifoundry", "image"), - "#class" : hentaifoundry.HentaifoundryImageExtractor, - "#sha1_url" : "fbf2fd74906738094e2575d2728e8dc3de18a8a3", - "#sha1_content": "91bf01497c39254b6dfb234a18e8f01629c77fd1", - - "artist" : "Tenpura", - "date" : "dt:2016-02-22 14:41:19", - "description": "Thank you!", - "height" : 700, - "index" : 407501, - "media" : "Other digital art", - "ratings" : [ - "Sexual content", - "Contains female nudity", - ], - "score" : int, - "tags" : [ - "collection", - "kancolle", - "kantai", - "shimakaze", - ], - "title" : "shimakaze", - "user" : "Tenpura", - "views" : int, - "width" : 495, -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Soloid/186714/Osaloop", - "#comment" : "SWF / rumble embed (#4641)", - "#category": ("", "hentaifoundry", "image"), - "#class" : hentaifoundry.HentaifoundryImageExtractor, - "#urls" : "https://pictures.hentai-foundry.com/s/Soloid/186714/Soloid-186714-Osaloop.swf", - - "artist" : "Soloid", - "date" : "dt:2013-02-07 17:25:54", - "description": "It took me ages.\nI hope you'll like it.\nSorry for the bad quality, I made it on after effect because Flash works like shit when you have 44 layers to animate, and the final ae SWF file is 55mo big.", - "extension" : "swf", - "filename" : "Soloid-186714-Osaloop", - "height" : 768, - "index" : 186714, - "media" : "Digital drawing or painting", - "ratings" : [ - "Nudity", - "Sexual content", - "Contains female nudity", - ], - "score" : range(80, 120), - "src" : "https://pictures.hentai-foundry.com/s/Soloid/186714/Soloid-186714-Osaloop.swf", - "tags" : [ - "soloid", - ], - "title" : "Osaloop", - "user" : "Soloid", - "views" : range(45000, 60000), - "width" : 613, -}, - -{ - "#url" : "http://www.hentai-foundry.com/pictures/user/Tenpura/407501/", - "#category": ("", "hentaifoundry", "image"), - "#class" : hentaifoundry.HentaifoundryImageExtractor, - "#pattern" : "http://pictures.hentai-foundry.com/t/Tenpura/407501/", -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura/407501/", - "#category": ("", "hentaifoundry", "image"), - "#class" : hentaifoundry.HentaifoundryImageExtractor, -}, - -{ - "#url" : "https://pictures.hentai-foundry.com/t/Tenpura/407501/Tenpura-407501-shimakaze.png", - "#category": ("", "hentaifoundry", "image"), - "#class" : hentaifoundry.HentaifoundryImageExtractor, -}, - -{ - "#url" : "https://www.hentai-foundry.com/stories/user/SnowWolf35", - "#category": ("", "hentaifoundry", "stories"), - "#class" : hentaifoundry.HentaifoundryStoriesExtractor, - "#count" : ">= 35", - - "author" : "SnowWolf35", - "chapters" : int, - "comments" : int, - "date" : datetime.datetime, - "description": str, - "index" : int, - "rating" : int, - "ratings" : list, - "status" : r"re:(Inc|C)omplete", - "title" : str, - "user" : "SnowWolf35", - "views" : int, - "words" : int, -}, - -{ - "#url" : "https://www.hentai-foundry.com/stories/user/SnowWolf35/26416/Overwatch-High-Chapter-Voting-Location", - "#category": ("", "hentaifoundry", "story"), - "#class" : hentaifoundry.HentaifoundryStoryExtractor, - "#sha1_url": "5a67cfa8c3bf7634c8af8485dd07c1ea74ee0ae8", - - "title": "Overwatch High Chapter Voting Location", -}, - + { + "#url": "https://www.hentai-foundry.com/user/Tenpura/profile", + "#category": ("", "hentaifoundry", "user"), + "#class": hentaifoundry.HentaifoundryUserExtractor, + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Tenpura", + "#category": ("", "hentaifoundry", "pictures"), + "#class": hentaifoundry.HentaifoundryPicturesExtractor, + "#sha1_url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28", + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Tenpura/page/3", + "#category": ("", "hentaifoundry", "pictures"), + "#class": hentaifoundry.HentaifoundryPicturesExtractor, + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Ethevian/scraps", + "#category": ("", "hentaifoundry", "scraps"), + "#class": hentaifoundry.HentaifoundryScrapsExtractor, + "#pattern": r"https://pictures\.hentai-foundry\.com/e/Ethevian/.+", + "#count": ">= 10", + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps/page/3", + "#category": ("", "hentaifoundry", "scraps"), + "#class": hentaifoundry.HentaifoundryScrapsExtractor, + }, + { + "#url": "https://www.hentai-foundry.com/user/Tenpura/faves/pictures", + "#category": ("", "hentaifoundry", "favorite"), + "#class": hentaifoundry.HentaifoundryFavoriteExtractor, + "#sha1_url": "56f9ae2e89fe855e9fe1da9b81e5ec6212b0320b", + }, + { + "#url": "https://www.hentai-foundry.com/user/Tenpura/faves/pictures/page/3", + "#category": ("", "hentaifoundry", "favorite"), + "#class": hentaifoundry.HentaifoundryFavoriteExtractor, + }, + { + "#url": "https://www.hentai-foundry.com/pictures/tagged/kancolle", + "#category": ("", "hentaifoundry", "tag"), + "#class": hentaifoundry.HentaifoundryTagExtractor, + "#pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", + "#range": "20-30", + "search_tags": "kancolle", + }, + { + "#url": "https://www.hentai-foundry.com/pictures/recent/2018-09-20", + "#category": ("", "hentaifoundry", "recent"), + "#class": hentaifoundry.HentaifoundryRecentExtractor, + "#pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", + "#range": "20-30", + "date": "2018-09-20", + }, + { + "#url": "https://www.hentai-foundry.com/pictures/popular", + "#category": ("", "hentaifoundry", "popular"), + "#class": hentaifoundry.HentaifoundryPopularExtractor, + "#pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", + "#range": "20-30", + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Tenpura/407501/shimakaze", + "#category": ("", "hentaifoundry", "image"), + "#class": hentaifoundry.HentaifoundryImageExtractor, + "#sha1_url": "fbf2fd74906738094e2575d2728e8dc3de18a8a3", + "#sha1_content": "91bf01497c39254b6dfb234a18e8f01629c77fd1", + "artist": "Tenpura", + "date": "dt:2016-02-22 14:41:19", + "description": "Thank you!", + "height": 700, + "index": 407501, + "media": "Other digital art", + "ratings": [ + "Sexual content", + "Contains female nudity", + ], + "score": int, + "tags": [ + "collection", + "kancolle", + "kantai", + "shimakaze", + ], + "title": "shimakaze", + "user": "Tenpura", + "views": int, + "width": 495, + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Soloid/186714/Osaloop", + "#comment": "SWF / rumble embed (#4641)", + "#category": ("", "hentaifoundry", "image"), + "#class": hentaifoundry.HentaifoundryImageExtractor, + "#urls": "https://pictures.hentai-foundry.com/s/Soloid/186714/Soloid-186714-Osaloop.swf", + "artist": "Soloid", + "date": "dt:2013-02-07 17:25:54", + "description": "It took me ages.\nI hope you'll like it.\nSorry for the bad quality, I made it on after effect because Flash works like shit when you have 44 layers to animate, and the final ae SWF file is 55mo big.", + "extension": "swf", + "filename": "Soloid-186714-Osaloop", + "height": 768, + "index": 186714, + "media": "Digital drawing or painting", + "ratings": [ + "Nudity", + "Sexual content", + "Contains female nudity", + ], + "score": range(80, 120), + "src": "https://pictures.hentai-foundry.com/s/Soloid/186714/Soloid-186714-Osaloop.swf", + "tags": [ + "soloid", + ], + "title": "Osaloop", + "user": "Soloid", + "views": range(45000, 60000), + "width": 613, + }, + { + "#url": "http://www.hentai-foundry.com/pictures/user/Tenpura/407501/", + "#category": ("", "hentaifoundry", "image"), + "#class": hentaifoundry.HentaifoundryImageExtractor, + "#pattern": "http://pictures.hentai-foundry.com/t/Tenpura/407501/", + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Tenpura/407501/", + "#category": ("", "hentaifoundry", "image"), + "#class": hentaifoundry.HentaifoundryImageExtractor, + }, + { + "#url": "https://pictures.hentai-foundry.com/t/Tenpura/407501/Tenpura-407501-shimakaze.png", + "#category": ("", "hentaifoundry", "image"), + "#class": hentaifoundry.HentaifoundryImageExtractor, + }, + { + "#url": "https://www.hentai-foundry.com/stories/user/SnowWolf35", + "#category": ("", "hentaifoundry", "stories"), + "#class": hentaifoundry.HentaifoundryStoriesExtractor, + "#count": ">= 35", + "author": "SnowWolf35", + "chapters": int, + "comments": int, + "date": datetime.datetime, + "description": str, + "index": int, + "rating": int, + "ratings": list, + "status": r"re:(Inc|C)omplete", + "title": str, + "user": "SnowWolf35", + "views": int, + "words": int, + }, + { + "#url": "https://www.hentai-foundry.com/stories/user/SnowWolf35/26416/Overwatch-High-Chapter-Voting-Location", + "#category": ("", "hentaifoundry", "story"), + "#class": hentaifoundry.HentaifoundryStoryExtractor, + "#sha1_url": "5a67cfa8c3bf7634c8af8485dd07c1ea74ee0ae8", + "title": "Overwatch High Chapter Voting Location", + }, ) diff --git a/test/results/hentaifox.py b/test/results/hentaifox.py index 123bd2774..385df8a2c 100644 --- a/test/results/hentaifox.py +++ b/test/results/hentaifox.py @@ -1,108 +1,93 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hentaifox - __tests__ = ( -{ - "#url" : "https://hentaifox.com/gallery/56622/", - "#category": ("", "hentaifox", "gallery"), - "#class" : hentaifox.HentaifoxGalleryExtractor, - "#pattern" : r"https://i\d*\.hentaifox\.com/\d+/\d+/\d+\.jpg", - "#count" : 24, - "#sha1_metadata": "bcd6b67284f378e5cc30b89b761140e3e60fcd92", -}, - -{ - "#url" : "https://hentaifox.com/gallery/630/", - "#comment" : "'split_tag' element (#1378)", - "#category": ("", "hentaifox", "gallery"), - "#class" : hentaifox.HentaifoxGalleryExtractor, - - "artist" : [ - "beti", - "betty", - "magi", - "mimikaki", - ], - "characters": [ - "aerith gainsborough", - "tifa lockhart", - "yuffie kisaragi", - ], - "count" : 32, - "gallery_id": 630, - "group" : ["cu-little2"], - "parody" : [ - "darkstalkers | vampire", - "final fantasy vii", - ], - "tags" : [ - "femdom", - "fingering", - "masturbation", - "yuri", - ], - "title" : "Cu-Little Bakanya～", - "type" : "doujinshi", -}, - -{ - "#url" : "https://hentaifox.com/gallery/35261/", - "#comment" : "email-protected title (#4201)", - "#category": ("", "hentaifox", "gallery"), - "#class" : hentaifox.HentaifoxGalleryExtractor, - - "gallery_id": 35261, - "title" : "ManageM@ster!", - "artist" : ["haritama hiroki"], - "group" : ["studio n.ball"], -}, - -{ - "#url" : "https://hentaifox.com/parody/touhou-project/", - "#category": ("", "hentaifox", "search"), - "#class" : hentaifox.HentaifoxSearchExtractor, -}, - -{ - "#url" : "https://hentaifox.com/character/reimu-hakurei/", - "#category": ("", "hentaifox", "search"), - "#class" : hentaifox.HentaifoxSearchExtractor, -}, - -{ - "#url" : "https://hentaifox.com/artist/distance/", - "#category": ("", "hentaifox", "search"), - "#class" : hentaifox.HentaifoxSearchExtractor, -}, - -{ - "#url" : "https://hentaifox.com/search/touhou/", - "#category": ("", "hentaifox", "search"), - "#class" : hentaifox.HentaifoxSearchExtractor, -}, - -{ - "#url" : "https://hentaifox.com/group/v-slash/", - "#category": ("", "hentaifox", "search"), - "#class" : hentaifox.HentaifoxSearchExtractor, -}, - -{ - "#url" : "https://hentaifox.com/tag/heterochromia/", - "#category": ("", "hentaifox", "search"), - "#class" : hentaifox.HentaifoxSearchExtractor, - "#pattern" : hentaifox.HentaifoxGalleryExtractor.pattern, - "#count" : ">= 60", - - "url" : str, - "gallery_id": int, - "title" : str, -}, - + { + "#url": "https://hentaifox.com/gallery/56622/", + "#category": ("", "hentaifox", "gallery"), + "#class": hentaifox.HentaifoxGalleryExtractor, + "#pattern": r"https://i\d*\.hentaifox\.com/\d+/\d+/\d+\.jpg", + "#count": 24, + "#sha1_metadata": "bcd6b67284f378e5cc30b89b761140e3e60fcd92", + }, + { + "#url": "https://hentaifox.com/gallery/630/", + "#comment": "'split_tag' element (#1378)", + "#category": ("", "hentaifox", "gallery"), + "#class": hentaifox.HentaifoxGalleryExtractor, + "artist": [ + "beti", + "betty", + "magi", + "mimikaki", + ], + "characters": [ + "aerith gainsborough", + "tifa lockhart", + "yuffie kisaragi", + ], + "count": 32, + "gallery_id": 630, + "group": ["cu-little2"], + "parody": [ + "darkstalkers | vampire", + "final fantasy vii", + ], + "tags": [ + "femdom", + "fingering", + "masturbation", + "yuri", + ], + "title": "Cu-Little Bakanya～", + "type": "doujinshi", + }, + { + "#url": "https://hentaifox.com/gallery/35261/", + "#comment": "email-protected title (#4201)", + "#category": ("", "hentaifox", "gallery"), + "#class": hentaifox.HentaifoxGalleryExtractor, + "gallery_id": 35261, + "title": "ManageM@ster!", + "artist": ["haritama hiroki"], + "group": ["studio n.ball"], + }, + { + "#url": "https://hentaifox.com/parody/touhou-project/", + "#category": ("", "hentaifox", "search"), + "#class": hentaifox.HentaifoxSearchExtractor, + }, + { + "#url": "https://hentaifox.com/character/reimu-hakurei/", + "#category": ("", "hentaifox", "search"), + "#class": hentaifox.HentaifoxSearchExtractor, + }, + { + "#url": "https://hentaifox.com/artist/distance/", + "#category": ("", "hentaifox", "search"), + "#class": hentaifox.HentaifoxSearchExtractor, + }, + { + "#url": "https://hentaifox.com/search/touhou/", + "#category": ("", "hentaifox", "search"), + "#class": hentaifox.HentaifoxSearchExtractor, + }, + { + "#url": "https://hentaifox.com/group/v-slash/", + "#category": ("", "hentaifox", "search"), + "#class": hentaifox.HentaifoxSearchExtractor, + }, + { + "#url": "https://hentaifox.com/tag/heterochromia/", + "#category": ("", "hentaifox", "search"), + "#class": hentaifox.HentaifoxSearchExtractor, + "#pattern": hentaifox.HentaifoxGalleryExtractor.pattern, + "#count": ">= 60", + "url": str, + "gallery_id": int, + "title": str, + }, ) diff --git a/test/results/hentaihand.py b/test/results/hentaihand.py index 3d9a34b6d..610596454 100644 --- a/test/results/hentaihand.py +++ b/test/results/hentaihand.py @@ -1,57 +1,48 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hentaihand - __tests__ = ( -{ - "#url" : "https://hentaihand.com/en/comic/c75-takumi-na-muchi-choudenji-hou-no-aishi-kata-how-to-love-a-super-electromagnetic-gun-toaru-kagaku-no-railgun-english", - "#category": ("", "hentaihand", "gallery"), - "#class" : hentaihand.HentaihandGalleryExtractor, - "#pattern" : r"https://cdn.hentaihand.com/.*/images/37387/\d+.jpg$", - "#count" : 50, - - "artists" : ["Takumi Na Muchi"], - "date" : "dt:2014-06-28 00:00:00", - "gallery_id" : 37387, - "lang" : "en", - "language" : "English", - "parodies" : ["Toaru Kagaku No Railgun"], - "relationships": list, - "tags" : list, - "title" : r"re:\(C75\) \[Takumi na Muchi\] Choudenji Hou ", - "title_alt" : r"re:\(C75\) \[たくみなむち\] 超電磁砲のあいしかた", - "type" : "Doujinshi", -}, - -{ - "#url" : "https://hentaihand.com/en/artist/takumi-na-muchi", - "#category": ("", "hentaihand", "tag"), - "#class" : hentaihand.HentaihandTagExtractor, - "#pattern" : hentaihand.HentaihandGalleryExtractor.pattern, - "#count" : ">= 6", -}, - -{ - "#url" : "https://hentaihand.com/en/tag/full-color", - "#category": ("", "hentaihand", "tag"), - "#class" : hentaihand.HentaihandTagExtractor, -}, - -{ - "#url" : "https://hentaihand.com/fr/language/japanese", - "#category": ("", "hentaihand", "tag"), - "#class" : hentaihand.HentaihandTagExtractor, -}, - -{ - "#url" : "https://hentaihand.com/zh/category/manga", - "#category": ("", "hentaihand", "tag"), - "#class" : hentaihand.HentaihandTagExtractor, -}, - + { + "#url": "https://hentaihand.com/en/comic/c75-takumi-na-muchi-choudenji-hou-no-aishi-kata-how-to-love-a-super-electromagnetic-gun-toaru-kagaku-no-railgun-english", + "#category": ("", "hentaihand", "gallery"), + "#class": hentaihand.HentaihandGalleryExtractor, + "#pattern": r"https://cdn.hentaihand.com/.*/images/37387/\d+.jpg$", + "#count": 50, + "artists": ["Takumi Na Muchi"], + "date": "dt:2014-06-28 00:00:00", + "gallery_id": 37387, + "lang": "en", + "language": "English", + "parodies": ["Toaru Kagaku No Railgun"], + "relationships": list, + "tags": list, + "title": r"re:\(C75\) \[Takumi na Muchi\] Choudenji Hou ", + "title_alt": r"re:\(C75\) \[たくみなむち\] 超電磁砲のあいしかた", + "type": "Doujinshi", + }, + { + "#url": "https://hentaihand.com/en/artist/takumi-na-muchi", + "#category": ("", "hentaihand", "tag"), + "#class": hentaihand.HentaihandTagExtractor, + "#pattern": hentaihand.HentaihandGalleryExtractor.pattern, + "#count": ">= 6", + }, + { + "#url": "https://hentaihand.com/en/tag/full-color", + "#category": ("", "hentaihand", "tag"), + "#class": hentaihand.HentaihandTagExtractor, + }, + { + "#url": "https://hentaihand.com/fr/language/japanese", + "#category": ("", "hentaihand", "tag"), + "#class": hentaihand.HentaihandTagExtractor, + }, + { + "#url": "https://hentaihand.com/zh/category/manga", + "#category": ("", "hentaihand", "tag"), + "#class": hentaihand.HentaihandTagExtractor, + }, ) diff --git a/test/results/hentaihere.py b/test/results/hentaihere.py index abe52fe91..659388efb 100644 --- a/test/results/hentaihere.py +++ b/test/results/hentaihere.py @@ -1,65 +1,56 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hentaihere - __tests__ = ( -{ - "#url" : "https://hentaihere.com/m/S13812/1/1/", - "#category": ("", "hentaihere", "chapter"), - "#class" : hentaihere.HentaihereChapterExtractor, - "#sha1_url" : "964b942cf492b3a129d2fe2608abfc475bc99e71", - "#sha1_metadata": "0207d20eea3a15d2a8d1496755bdfa49de7cfa9d", -}, - -{ - "#url" : "https://hentaihere.com/m/S23048/1.5/1/", - "#category": ("", "hentaihere", "chapter"), - "#class" : hentaihere.HentaihereChapterExtractor, - "#pattern" : r"https://hentaicdn\.com/hentai/23048/1\.5/ccdn00\d+\.jpg", - "#count" : 32, - - "author" : "Shinozuka Yuuji", - "chapter" : 1, - "chapter_id" : 80186, - "chapter_minor": ".5", - "count" : 32, - "lang" : "en", - "language" : "English", - "manga" : "High School Slut's Love Consultation", - "manga_id" : 23048, - "page" : int, - "title" : "High School Slut's Love Consultation + Girlfriend [Full Color]", - "type" : "Original", -}, - -{ - "#url" : "https://hentaihere.com/m/S13812", - "#category": ("", "hentaihere", "manga"), - "#class" : hentaihere.HentaihereMangaExtractor, - "#sha1_url" : "d1ba6e28bb2162e844f8559c2b2725ba0a093559", - "#sha1_metadata": "5c1b712258e78e120907121d3987c71f834d13e1", -}, - -{ - "#url" : "https://hentaihere.com/m/S7608", - "#category": ("", "hentaihere", "manga"), - "#class" : hentaihere.HentaihereMangaExtractor, - "#sha1_url": "6c5239758dc93f6b1b4175922836c10391b174f7", - - "chapter" : int, - "chapter_id" : int, - "chapter_minor": "", - "lang" : "en", - "language" : "English", - "manga" : "Oshikake Riot", - "manga_id" : 7608, - "title" : r"re:Oshikake Riot( \d+)?", - "type" : "Original", -}, - + { + "#url": "https://hentaihere.com/m/S13812/1/1/", + "#category": ("", "hentaihere", "chapter"), + "#class": hentaihere.HentaihereChapterExtractor, + "#sha1_url": "964b942cf492b3a129d2fe2608abfc475bc99e71", + "#sha1_metadata": "0207d20eea3a15d2a8d1496755bdfa49de7cfa9d", + }, + { + "#url": "https://hentaihere.com/m/S23048/1.5/1/", + "#category": ("", "hentaihere", "chapter"), + "#class": hentaihere.HentaihereChapterExtractor, + "#pattern": r"https://hentaicdn\.com/hentai/23048/1\.5/ccdn00\d+\.jpg", + "#count": 32, + "author": "Shinozuka Yuuji", + "chapter": 1, + "chapter_id": 80186, + "chapter_minor": ".5", + "count": 32, + "lang": "en", + "language": "English", + "manga": "High School Slut's Love Consultation", + "manga_id": 23048, + "page": int, + "title": "High School Slut's Love Consultation + Girlfriend [Full Color]", + "type": "Original", + }, + { + "#url": "https://hentaihere.com/m/S13812", + "#category": ("", "hentaihere", "manga"), + "#class": hentaihere.HentaihereMangaExtractor, + "#sha1_url": "d1ba6e28bb2162e844f8559c2b2725ba0a093559", + "#sha1_metadata": "5c1b712258e78e120907121d3987c71f834d13e1", + }, + { + "#url": "https://hentaihere.com/m/S7608", + "#category": ("", "hentaihere", "manga"), + "#class": hentaihere.HentaihereMangaExtractor, + "#sha1_url": "6c5239758dc93f6b1b4175922836c10391b174f7", + "chapter": int, + "chapter_id": int, + "chapter_minor": "", + "lang": "en", + "language": "English", + "manga": "Oshikake Riot", + "manga_id": 7608, + "title": r"re:Oshikake Riot( \d+)?", + "type": "Original", + }, ) diff --git a/test/results/hentainexus.py b/test/results/hentainexus.py index 01091350d..3b3e58357 100644 --- a/test/results/hentainexus.py +++ b/test/results/hentainexus.py @@ -1,81 +1,72 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hentainexus - __tests__ = ( -{ - "#url" : "https://hentainexus.com/view/5688", - "#category": ("", "hentainexus", "gallery"), - "#class" : hentainexus.HentainexusGalleryExtractor, - - "artist" : "Tsukiriran", - "book" : "", - "circle" : "", - "count" : 4, - "cover" : str, - "description": "The cherry blossom blooms for one final graduation memory. ❤", - "event" : "", - "extension" : "png", - "filename" : str, - "gallery_id" : 5688, - "image" : str, - "label" : str, - "lang" : "en", - "language" : "English", - "magazine" : "Comic Bavel 2018-08", - "num" : range(1, 4), - "parody" : "Original Work", - "publisher" : "FAKKU", - "tags" : [ - "busty", - "color", - "creampie", - "exhibitionism", - "hentai", - "kimono", - "pubic hair", - "uncensored", - "unlimited", - "vanilla", - ], - "title" : "Graduation!", - "title_conventional": "[Tsukiriran] Graduation! (Comic Bavel 2018-08)", - "type" : "image", - "url_label" : str, -}, - -{ - "#url" : "https://hentainexus.com/read/5688", - "#category": ("", "hentainexus", "gallery"), - "#class" : hentainexus.HentainexusGalleryExtractor, -}, - -{ - "#url" : "https://hentainexus.com/view/715", - "#comment" : "combined left-right pages (#5827)", - "#category": ("", "hentainexus", "gallery"), - "#class" : hentainexus.HentainexusGalleryExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://hentainexus.com/?q=tag:%22heart+pupils%22%20tag:group", - "#category": ("", "hentainexus", "search"), - "#class" : hentainexus.HentainexusSearchExtractor, - "#pattern" : hentainexus.HentainexusGalleryExtractor.pattern, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://hentainexus.com/page/3?q=tag:%22heart+pupils%22", - "#category": ("", "hentainexus", "search"), - "#class" : hentainexus.HentainexusSearchExtractor, -}, - + { + "#url": "https://hentainexus.com/view/5688", + "#category": ("", "hentainexus", "gallery"), + "#class": hentainexus.HentainexusGalleryExtractor, + "artist": "Tsukiriran", + "book": "", + "circle": "", + "count": 4, + "cover": str, + "description": "The cherry blossom blooms for one final graduation memory. ❤", + "event": "", + "extension": "png", + "filename": str, + "gallery_id": 5688, + "image": str, + "label": str, + "lang": "en", + "language": "English", + "magazine": "Comic Bavel 2018-08", + "num": range(1, 4), + "parody": "Original Work", + "publisher": "FAKKU", + "tags": [ + "busty", + "color", + "creampie", + "exhibitionism", + "hentai", + "kimono", + "pubic hair", + "uncensored", + "unlimited", + "vanilla", + ], + "title": "Graduation!", + "title_conventional": "[Tsukiriran] Graduation! (Comic Bavel 2018-08)", + "type": "image", + "url_label": str, + }, + { + "#url": "https://hentainexus.com/read/5688", + "#category": ("", "hentainexus", "gallery"), + "#class": hentainexus.HentainexusGalleryExtractor, + }, + { + "#url": "https://hentainexus.com/view/715", + "#comment": "combined left-right pages (#5827)", + "#category": ("", "hentainexus", "gallery"), + "#class": hentainexus.HentainexusGalleryExtractor, + "#count": 2, + }, + { + "#url": "https://hentainexus.com/?q=tag:%22heart+pupils%22%20tag:group", + "#category": ("", "hentainexus", "search"), + "#class": hentainexus.HentainexusSearchExtractor, + "#pattern": hentainexus.HentainexusGalleryExtractor.pattern, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://hentainexus.com/page/3?q=tag:%22heart+pupils%22", + "#category": ("", "hentainexus", "search"), + "#class": hentainexus.HentainexusSearchExtractor, + }, ) diff --git a/test/results/hiperdex.py b/test/results/hiperdex.py index a65c0928f..736b0c962 100644 --- a/test/results/hiperdex.py +++ b/test/results/hiperdex.py @@ -1,148 +1,125 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hiperdex - __tests__ = ( -{ - "#url" : "https://hipertoon.com/manga/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, - "#pattern" : r"https://(1st)?hiper(dex|toon)\d?.(com|net|info|top)/wp-content/uploads/WP-manga/data/manga_\w+/[0-9a-f]{32}/\d+\.webp", - "#count" : 9, - - "artist" : "Sasuga Kei", - "author" : "Sasuga Kei", - "chapter" : 154, - "chapter_minor": ".5", - "description" : "Natsuo Fujii is in love with his teacher, Hina. Attempting to forget his feelings towards her, Natsuo goes to a mixer with his classmates where he meets an odd girl named Rui Tachibana. In a strange turn of events, Rui asks Natsuo to sneak out with her and do her a favor. To his surprise, their destination is Rui’s house—and her request is for him to have sex with her. There’s no love behind the act; she just wants to learn from the experience. Thinking that it might help him forget about Hina, Natsuo hesitantly agrees. After this unusual encounter Natsuo now faces a new problem. With his father remarrying, he ends up with a new pair of stepsisters; unfortunately, he knows these two girls all too well. He soon finds out his new siblings are none other than Hina and Rui! Now living with both the teacher he loves and the girl with whom he had his “first time,” Natsuo finds himself in an unexpected love triangle as he climbs ever closer towards adulthood.", - "genre" : list, - "manga" : "Domestic na Kanojo", - "release" : 2014, - "score" : float, - "type" : "Manga", -}, - -{ - "#url" : "https://hiperdex.com/mangas/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, -}, - -{ - "#url" : "https://1sthiperdex.com/manga/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, -}, - -{ - "#url" : "https://hiperdex2.com/manga/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, -}, - -{ - "#url" : "https://hiperdex.net/manga/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, -}, - -{ - "#url" : "https://hiperdex.info/manga/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, -}, - -{ - "#url" : "https://hiperdex.top/manga/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, -}, - -{ - "#url" : "https://hiperdex.com/manga/1603231576-youre-not-that-special/", - "#category": ("", "hiperdex", "manga"), - "#class" : hiperdex.HiperdexMangaExtractor, - "#pattern" : hiperdex.HiperdexChapterExtractor.pattern, - "#count" : 51, - - "artist" : "Bolp", - "author" : "Abyo4", - "chapter" : int, - "chapter_minor": "", - "description" : r"re:I didn’t think much of the creepy girl in ", - "genre" : list, - "manga" : "You’re Not That Special!", - "release" : 2019, - "score" : float, - "status" : "Completed", - "type" : "Manhwa", -}, - -{ - "#url" : "https://hiperdex.com/manga/youre-not-that-special/", - "#category": ("", "hiperdex", "manga"), - "#class" : hiperdex.HiperdexMangaExtractor, -}, - -{ - "#url" : "https://1sthiperdex.com/manga/youre-not-that-special/", - "#category": ("", "hiperdex", "manga"), - "#class" : hiperdex.HiperdexMangaExtractor, -}, - -{ - "#url" : "https://hiperdex2.com/manga/youre-not-that-special/", - "#category": ("", "hiperdex", "manga"), - "#class" : hiperdex.HiperdexMangaExtractor, -}, - -{ - "#url" : "https://hiperdex.net/manga/youre-not-that-special/", - "#category": ("", "hiperdex", "manga"), - "#class" : hiperdex.HiperdexMangaExtractor, -}, - -{ - "#url" : "https://hiperdex.info/manga/youre-not-that-special/", - "#category": ("", "hiperdex", "manga"), - "#class" : hiperdex.HiperdexMangaExtractor, -}, - -{ - "#url" : "https://1sthiperdex.com/manga-artist/beck-ho-an/", - "#category": ("", "hiperdex", "artist"), - "#class" : hiperdex.HiperdexArtistExtractor, -}, - -{ - "#url" : "https://hiperdex.net/manga-artist/beck-ho-an/", - "#category": ("", "hiperdex", "artist"), - "#class" : hiperdex.HiperdexArtistExtractor, -}, - -{ - "#url" : "https://hiperdex2.com/manga-artist/beck-ho-an/", - "#category": ("", "hiperdex", "artist"), - "#class" : hiperdex.HiperdexArtistExtractor, -}, - -{ - "#url" : "https://hiperdex.info/manga-artist/beck-ho-an/", - "#category": ("", "hiperdex", "artist"), - "#class" : hiperdex.HiperdexArtistExtractor, -}, - -{ - "#url" : "https://hiperdex.com/manga-author/viagra/", - "#category": ("", "hiperdex", "artist"), - "#class" : hiperdex.HiperdexArtistExtractor, - "#pattern" : hiperdex.HiperdexMangaExtractor.pattern, - "#count" : ">= 6", -}, - + { + "#url": "https://hipertoon.com/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + "#pattern": r"https://(1st)?hiper(dex|toon)\d?.(com|net|info|top)/wp-content/uploads/WP-manga/data/manga_\w+/[0-9a-f]{32}/\d+\.webp", + "#count": 9, + "artist": "Sasuga Kei", + "author": "Sasuga Kei", + "chapter": 154, + "chapter_minor": ".5", + "description": "Natsuo Fujii is in love with his teacher, Hina. Attempting to forget his feelings towards her, Natsuo goes to a mixer with his classmates where he meets an odd girl named Rui Tachibana. In a strange turn of events, Rui asks Natsuo to sneak out with her and do her a favor. To his surprise, their destination is Rui’s house—and her request is for him to have sex with her. There’s no love behind the act; she just wants to learn from the experience. Thinking that it might help him forget about Hina, Natsuo hesitantly agrees. After this unusual encounter Natsuo now faces a new problem. With his father remarrying, he ends up with a new pair of stepsisters; unfortunately, he knows these two girls all too well. He soon finds out his new siblings are none other than Hina and Rui! Now living with both the teacher he loves and the girl with whom he had his “first time,” Natsuo finds himself in an unexpected love triangle as he climbs ever closer towards adulthood.", + "genre": list, + "manga": "Domestic na Kanojo", + "release": 2014, + "score": float, + "type": "Manga", + }, + { + "#url": "https://hiperdex.com/mangas/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + }, + { + "#url": "https://1sthiperdex.com/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + }, + { + "#url": "https://hiperdex2.com/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + }, + { + "#url": "https://hiperdex.net/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + }, + { + "#url": "https://hiperdex.info/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + }, + { + "#url": "https://hiperdex.top/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + }, + { + "#url": "https://hiperdex.com/manga/1603231576-youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class": hiperdex.HiperdexMangaExtractor, + "#pattern": hiperdex.HiperdexChapterExtractor.pattern, + "#count": 51, + "artist": "Bolp", + "author": "Abyo4", + "chapter": int, + "chapter_minor": "", + "description": r"re:I didn’t think much of the creepy girl in ", + "genre": list, + "manga": "You’re Not That Special!", + "release": 2019, + "score": float, + "status": "Completed", + "type": "Manhwa", + }, + { + "#url": "https://hiperdex.com/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class": hiperdex.HiperdexMangaExtractor, + }, + { + "#url": "https://1sthiperdex.com/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class": hiperdex.HiperdexMangaExtractor, + }, + { + "#url": "https://hiperdex2.com/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class": hiperdex.HiperdexMangaExtractor, + }, + { + "#url": "https://hiperdex.net/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class": hiperdex.HiperdexMangaExtractor, + }, + { + "#url": "https://hiperdex.info/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class": hiperdex.HiperdexMangaExtractor, + }, + { + "#url": "https://1sthiperdex.com/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class": hiperdex.HiperdexArtistExtractor, + }, + { + "#url": "https://hiperdex.net/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class": hiperdex.HiperdexArtistExtractor, + }, + { + "#url": "https://hiperdex2.com/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class": hiperdex.HiperdexArtistExtractor, + }, + { + "#url": "https://hiperdex.info/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class": hiperdex.HiperdexArtistExtractor, + }, + { + "#url": "https://hiperdex.com/manga-author/viagra/", + "#category": ("", "hiperdex", "artist"), + "#class": hiperdex.HiperdexArtistExtractor, + "#pattern": hiperdex.HiperdexMangaExtractor.pattern, + "#count": ">= 6", + }, ) diff --git a/test/results/hitomi.py b/test/results/hitomi.py index 1b0ffcba8..c5b6cead2 100644 --- a/test/results/hitomi.py +++ b/test/results/hitomi.py @@ -1,234 +1,207 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hitomi - __tests__ = ( -{ - "#url" : "https://hitomi.la/galleries/867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, - "#pattern" : r"https://[a-c]a\.hitomi\.la/webp/\d+/\d+/[0-9a-f]{64}\.webp", - "#count" : 16, - - "artist" : ["morris"], - "characters": [], - "count" : 16, - "date" : "dt:2015-10-27 19:20:00", - "extension" : "webp", - "extension_original" : "jpg", - "filename" : str, - "gallery_id": 867789, - "group" : [], - "lang" : "en", - "language" : "English", - "num" : range(1, 16), - "parody" : [], - "tags" : [ - "Cheating ♀", - "Drugs ♀", - "Drugs ♂", - "Incest", - "Milf ♀", - "Mother ♀", - "Sole Female ♀", - "Sole Male ♂", - "Uncensored" - ], - "title" : "Amazon no Hiyaku | Amazon Elixir (decensored)", - "title_jpn" : "", - "type" : "Manga", -}, - -{ - "#url" : "https://hitomi.la/galleries/1401410.html", - "#comment" : "download test", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, - "#range" : "1", - "#sha1_content": "d75d5a3d1302a48469016b20e53c26b714d17745", -}, - -{ - "#url" : "https://hitomi.la/galleries/733697.html", - "#comment" : "Game CG with scenes (#321)", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, - "#count" : 210, -}, - -{ - "#url" : "https://hitomi.la/galleries/1045954.html", - "#comment" : "fallback for galleries only available through /reader/ URLs", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, - "#count" : 1413, -}, - -{ - "#url" : "https://hitomi.la/cg/scathacha-sama-okuchi-ecchi-1291900.html", - "#comment" : "gallery with 'broken' redirect", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/cg/1615823.html", - "#comment" : "no tags", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, - "#options" : {"format": "avif"}, - "#pattern" : r"https://[a-c]a\.hitomi\.la/avif/\d+/\d+/[0-9a-f]{64}\.avif", - "#count" : 22, - - "artist" : ["sorairo len"], - "characters": [], - "count" : 22, - "date" : "dt:2020-04-19 06:33:00", - "extension" : "avif", - "filename" : str, - "gallery_id": 1615823, - "group" : [], - "lang" : "ja", - "language" : "Japanese", - "num" : range(1, 22), - "parody" : [], - "tags" : [ - "Blowjob ♀", - "Focus Blowjob ♀", - "Fox Girl ♀", - "Kemonomimi ♀", - "Loli ♀", - "Miko ♀", - "No Penetration", - "Unusual Pupils ♀", - "Variant Set" - ], - "title" : "Kouko-sama ga Okuchi de Reiryoku Hokyuu", - "title_jpn" : "コウコ様がお口で霊力補給♡", - "type" : "Artistcg", -}, - -{ - "#url" : "https://hitomi.la/manga/amazon-no-hiyaku-867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/manga/867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/doujinshi/867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/cg/867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/gamecg/867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/imageset/867789.html", - "#comment" : "/imageset/ gallery (#4756)", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/reader/867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/tag/screenshots-japanese.html", - "#category": ("", "hitomi", "tag"), - "#class" : hitomi.HitomiTagExtractor, - "#pattern" : hitomi.HitomiGalleryExtractor.pattern, - "#count" : ">= 35", -}, - -{ - "#url" : "https://hitomi.la/artist/a1-all-1.html", - "#category": ("", "hitomi", "tag"), - "#class" : hitomi.HitomiTagExtractor, -}, - -{ - "#url" : "https://hitomi.la/group/initial%2Dg-all-1.html", - "#category": ("", "hitomi", "tag"), - "#class" : hitomi.HitomiTagExtractor, -}, - -{ - "#url" : "https://hitomi.la/series/amnesia-all-1.html", - "#category": ("", "hitomi", "tag"), - "#class" : hitomi.HitomiTagExtractor, -}, - -{ - "#url" : "https://hitomi.la/type/doujinshi-all-1.html", - "#category": ("", "hitomi", "tag"), - "#class" : hitomi.HitomiTagExtractor, -}, - -{ - "#url" : "https://hitomi.la/character/a2-all-1.html", - "#category": ("", "hitomi", "tag"), - "#class" : hitomi.HitomiTagExtractor, -}, - -{ - "#url" : "https://hitomi.la/index-japanese.html", - "#class" : hitomi.HitomiIndexExtractor, - "#pattern" : hitomi.HitomiGalleryExtractor.pattern, - "#range" : "1-150", - "#count" : 150, -}, - -{ - "#url" : "https://hitomi.la/search.html?tag%3Ascreenshots%20language%3Ajapanese", - "#class" : hitomi.HitomiSearchExtractor, - "#pattern" : hitomi.HitomiGalleryExtractor.pattern, - "#range" : "1-150", - "#count" : 150, -}, - -{ - "#url" : "https://hitomi.la/search.html?language%3Ajapanese%20artist%3Asumiya", - "#class" : hitomi.HitomiSearchExtractor, -}, -{ - "#url" : "https://hitomi.la/search.html?group:initial_g", - "#class" : hitomi.HitomiSearchExtractor, -}, -{ - "#url" : "https://hitomi.la/search.html?series:amnesia", - "#class" : hitomi.HitomiSearchExtractor, -}, -{ - "#url" : "https://hitomi.la/search.html?type%3Adoujinshi", - "#class" : hitomi.HitomiSearchExtractor, -}, -{ - "#url" : "https://hitomi.la/search.html?character%3Aa2", - "#class" : hitomi.HitomiSearchExtractor, -}, - + { + "#url": "https://hitomi.la/galleries/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + "#pattern": r"https://[a-c]a\.hitomi\.la/webp/\d+/\d+/[0-9a-f]{64}\.webp", + "#count": 16, + "artist": ["morris"], + "characters": [], + "count": 16, + "date": "dt:2015-10-27 19:20:00", + "extension": "webp", + "extension_original": "jpg", + "filename": str, + "gallery_id": 867789, + "group": [], + "lang": "en", + "language": "English", + "num": range(1, 16), + "parody": [], + "tags": [ + "Cheating ♀", + "Drugs ♀", + "Drugs ♂", + "Incest", + "Milf ♀", + "Mother ♀", + "Sole Female ♀", + "Sole Male ♂", + "Uncensored", + ], + "title": "Amazon no Hiyaku | Amazon Elixir (decensored)", + "title_jpn": "", + "type": "Manga", + }, + { + "#url": "https://hitomi.la/galleries/1401410.html", + "#comment": "download test", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + "#range": "1", + "#sha1_content": "d75d5a3d1302a48469016b20e53c26b714d17745", + }, + { + "#url": "https://hitomi.la/galleries/733697.html", + "#comment": "Game CG with scenes (#321)", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + "#count": 210, + }, + { + "#url": "https://hitomi.la/galleries/1045954.html", + "#comment": "fallback for galleries only available through /reader/ URLs", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + "#count": 1413, + }, + { + "#url": "https://hitomi.la/cg/scathacha-sama-okuchi-ecchi-1291900.html", + "#comment": "gallery with 'broken' redirect", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/cg/1615823.html", + "#comment": "no tags", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + "#options": {"format": "avif"}, + "#pattern": r"https://[a-c]a\.hitomi\.la/avif/\d+/\d+/[0-9a-f]{64}\.avif", + "#count": 22, + "artist": ["sorairo len"], + "characters": [], + "count": 22, + "date": "dt:2020-04-19 06:33:00", + "extension": "avif", + "filename": str, + "gallery_id": 1615823, + "group": [], + "lang": "ja", + "language": "Japanese", + "num": range(1, 22), + "parody": [], + "tags": [ + "Blowjob ♀", + "Focus Blowjob ♀", + "Fox Girl ♀", + "Kemonomimi ♀", + "Loli ♀", + "Miko ♀", + "No Penetration", + "Unusual Pupils ♀", + "Variant Set", + ], + "title": "Kouko-sama ga Okuchi de Reiryoku Hokyuu", + "title_jpn": "コウコ様がお口で霊力補給♡", + "type": "Artistcg", + }, + { + "#url": "https://hitomi.la/manga/amazon-no-hiyaku-867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/manga/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/doujinshi/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/cg/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/gamecg/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/imageset/867789.html", + "#comment": "/imageset/ gallery (#4756)", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/reader/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/tag/screenshots-japanese.html", + "#category": ("", "hitomi", "tag"), + "#class": hitomi.HitomiTagExtractor, + "#pattern": hitomi.HitomiGalleryExtractor.pattern, + "#count": ">= 35", + }, + { + "#url": "https://hitomi.la/artist/a1-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class": hitomi.HitomiTagExtractor, + }, + { + "#url": "https://hitomi.la/group/initial%2Dg-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class": hitomi.HitomiTagExtractor, + }, + { + "#url": "https://hitomi.la/series/amnesia-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class": hitomi.HitomiTagExtractor, + }, + { + "#url": "https://hitomi.la/type/doujinshi-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class": hitomi.HitomiTagExtractor, + }, + { + "#url": "https://hitomi.la/character/a2-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class": hitomi.HitomiTagExtractor, + }, + { + "#url": "https://hitomi.la/index-japanese.html", + "#class": hitomi.HitomiIndexExtractor, + "#pattern": hitomi.HitomiGalleryExtractor.pattern, + "#range": "1-150", + "#count": 150, + }, + { + "#url": "https://hitomi.la/search.html?tag%3Ascreenshots%20language%3Ajapanese", + "#class": hitomi.HitomiSearchExtractor, + "#pattern": hitomi.HitomiGalleryExtractor.pattern, + "#range": "1-150", + "#count": 150, + }, + { + "#url": "https://hitomi.la/search.html?language%3Ajapanese%20artist%3Asumiya", + "#class": hitomi.HitomiSearchExtractor, + }, + { + "#url": "https://hitomi.la/search.html?group:initial_g", + "#class": hitomi.HitomiSearchExtractor, + }, + { + "#url": "https://hitomi.la/search.html?series:amnesia", + "#class": hitomi.HitomiSearchExtractor, + }, + { + "#url": "https://hitomi.la/search.html?type%3Adoujinshi", + "#class": hitomi.HitomiSearchExtractor, + }, + { + "#url": "https://hitomi.la/search.html?character%3Aa2", + "#class": hitomi.HitomiSearchExtractor, + }, ) diff --git a/test/results/horne.py b/test/results/horne.py index f6bddba86..f68c71cc9 100644 --- a/test/results/horne.py +++ b/test/results/horne.py @@ -1,133 +1,118 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import nijie import datetime +from gallery_dl.extractor import nijie __tests__ = ( -{ - "#url" : "https://horne.red/members.php?id=58000", - "#category": ("Nijie", "horne", "user"), - "#class" : nijie.NijieUserExtractor, - "#urls" : ( - "https://horne.red/members_illust.php?id=58000", - "https://horne.red/members_dojin.php?id=58000", - ), -}, - -{ - "#url" : "https://horne.red/members_illust.php?id=58000", - "#category": ("Nijie", "horne", "illustration"), - "#class" : nijie.NijieIllustrationExtractor, - "#pattern" : r"https://pic\.nijie\.net/\d+/horne/\w+/\d+/\d+/illust/\d+_\d+_[0-9a-f]+_[0-9a-f]+\.png", - "#range" : "1-20", - "#count" : 20, - - "artist_id" : 58000, - "artist_name": "のえるわ", - "date" : datetime.datetime, - "description": str, - "image_id" : int, - "num" : int, - "tags" : list, - "title" : str, - "url" : str, - "user_id" : 58000, - "user_name" : "のえるわ", -}, - -{ - "#url" : "https://horne.red/members_dojin.php?id=58000", - "#category": ("Nijie", "horne", "doujin"), - "#class" : nijie.NijieDoujinExtractor, -}, - -{ - "#url" : "https://horne.red/user_like_illust_view.php?id=58000", - "#category": ("Nijie", "horne", "favorite"), - "#class" : nijie.NijieFavoriteExtractor, - "#range" : "1-5", - "#count" : 5, - - "user_id" : 58000, - "user_name": "のえるわ", -}, - -{ - "#url" : "https://horne.red/history_nuita.php?id=58000", - "#category": ("Nijie", "horne", "nuita"), - "#class" : nijie.NijieNuitaExtractor, -}, - -{ - "#url" : "https://horne.red/like_user_view.php", - "#category": ("Nijie", "horne", "feed"), - "#class" : nijie.NijieFeedExtractor, -}, - -{ - "#url" : "https://horne.red/like_my.php", - "#category": ("Nijie", "horne", "followed"), - "#class" : nijie.NijieFollowedExtractor, -}, - -{ - "#url" : "https://horne.red/view.php?id=8708", - "#category": ("Nijie", "horne", "image"), - "#class" : nijie.NijieImageExtractor, - "#urls" : "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_c8f715a8f3d53943_db6231.png", - - "artist_id" : 58000, - "artist_name": "のえるわ", - "count" : 1, - "date" : "dt:2018-01-29 14:25:39", - "description": "前回とシチュがまるかぶり　\r\n竿野郎は塗るのだるかった", - "extension" : "png", - "filename" : "0_0_c8f715a8f3d53943_db6231", - "image_id" : 8708, - "num" : 0, - "tags" : [ - "男の娘", - "オリキャラ", - "うちのこ", - ], - "title" : "うちのこえっち", - "url" : "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_c8f715a8f3d53943_db6231.png", - "user_id" : 58000, - "user_name" : "のえるわ", -}, - -{ - "#url" : "https://horne.red/view.php?id=8716", - "#category": ("Nijie", "horne", "image"), - "#class" : nijie.NijieImageExtractor, - "#urls" : ( - "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_b4ffb4b6f7ec6d51_1a32c0.png", - "https://pic.nijie.net/07/horne/18/00/58000/illust/8716_0_2690972a4f6270bb_85ed8f.png", - "https://pic.nijie.net/07/horne/18/00/58000/illust/8716_1_09348508d8b76f36_f6cf47.png", - "https://pic.nijie.net/08/horne/18/00/58000/illust/8716_2_5151d956d3789277_d76e75.png", - ), - - "artist_id" : 58000, - "artist_name": "のえるわ", - "count" : 4, - "date" : "dt:2018-02-04 14:47:24", - "description": "ノエル「そんなことしなくても、言ってくれたら咥えるのに・・・♡」", - "image_id" : 8716, - "num" : range(0, 3), - "tags" : [ - "男の娘", - "フェラ", - "オリキャラ", - "うちのこ", - ], - "title" : "ノエル「いまどきそんな、恵方巻ネタなんてやらなくても・・・」", - "user_id" : 58000, - "user_name" : "のえるわ", -}, - + { + "#url": "https://horne.red/members.php?id=58000", + "#category": ("Nijie", "horne", "user"), + "#class": nijie.NijieUserExtractor, + "#urls": ( + "https://horne.red/members_illust.php?id=58000", + "https://horne.red/members_dojin.php?id=58000", + ), + }, + { + "#url": "https://horne.red/members_illust.php?id=58000", + "#category": ("Nijie", "horne", "illustration"), + "#class": nijie.NijieIllustrationExtractor, + "#pattern": r"https://pic\.nijie\.net/\d+/horne/\w+/\d+/\d+/illust/\d+_\d+_[0-9a-f]+_[0-9a-f]+\.png", + "#range": "1-20", + "#count": 20, + "artist_id": 58000, + "artist_name": "のえるわ", + "date": datetime.datetime, + "description": str, + "image_id": int, + "num": int, + "tags": list, + "title": str, + "url": str, + "user_id": 58000, + "user_name": "のえるわ", + }, + { + "#url": "https://horne.red/members_dojin.php?id=58000", + "#category": ("Nijie", "horne", "doujin"), + "#class": nijie.NijieDoujinExtractor, + }, + { + "#url": "https://horne.red/user_like_illust_view.php?id=58000", + "#category": ("Nijie", "horne", "favorite"), + "#class": nijie.NijieFavoriteExtractor, + "#range": "1-5", + "#count": 5, + "user_id": 58000, + "user_name": "のえるわ", + }, + { + "#url": "https://horne.red/history_nuita.php?id=58000", + "#category": ("Nijie", "horne", "nuita"), + "#class": nijie.NijieNuitaExtractor, + }, + { + "#url": "https://horne.red/like_user_view.php", + "#category": ("Nijie", "horne", "feed"), + "#class": nijie.NijieFeedExtractor, + }, + { + "#url": "https://horne.red/like_my.php", + "#category": ("Nijie", "horne", "followed"), + "#class": nijie.NijieFollowedExtractor, + }, + { + "#url": "https://horne.red/view.php?id=8708", + "#category": ("Nijie", "horne", "image"), + "#class": nijie.NijieImageExtractor, + "#urls": "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_c8f715a8f3d53943_db6231.png", + "artist_id": 58000, + "artist_name": "のえるわ", + "count": 1, + "date": "dt:2018-01-29 14:25:39", + "description": "前回とシチュがまるかぶり　\r\n竿野郎は塗るのだるかった", + "extension": "png", + "filename": "0_0_c8f715a8f3d53943_db6231", + "image_id": 8708, + "num": 0, + "tags": [ + "男の娘", + "オリキャラ", + "うちのこ", + ], + "title": "うちのこえっち", + "url": "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_c8f715a8f3d53943_db6231.png", + "user_id": 58000, + "user_name": "のえるわ", + }, + { + "#url": "https://horne.red/view.php?id=8716", + "#category": ("Nijie", "horne", "image"), + "#class": nijie.NijieImageExtractor, + "#urls": ( + "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_b4ffb4b6f7ec6d51_1a32c0.png", + "https://pic.nijie.net/07/horne/18/00/58000/illust/8716_0_2690972a4f6270bb_85ed8f.png", + "https://pic.nijie.net/07/horne/18/00/58000/illust/8716_1_09348508d8b76f36_f6cf47.png", + "https://pic.nijie.net/08/horne/18/00/58000/illust/8716_2_5151d956d3789277_d76e75.png", + ), + "artist_id": 58000, + "artist_name": "のえるわ", + "count": 4, + "date": "dt:2018-02-04 14:47:24", + "description": "ノエル「そんなことしなくても、言ってくれたら咥えるのに・・・♡」", + "image_id": 8716, + "num": range(3), + "tags": [ + "男の娘", + "フェラ", + "オリキャラ", + "うちのこ", + ], + "title": "ノエル「いまどきそんな、恵方巻ネタなんてやらなくても・・・」", + "user_id": 58000, + "user_name": "のえるわ", + }, ) diff --git a/test/results/hotleak.py b/test/results/hotleak.py index 7305180c1..e9ed093ae 100644 --- a/test/results/hotleak.py +++ b/test/results/hotleak.py @@ -1,104 +1,88 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import hotleak from gallery_dl import exception - +from gallery_dl.extractor import hotleak __tests__ = ( -{ - "#url" : "https://hotleak.vip/kaiyakawaii/photo/1617145", - "#category": ("", "hotleak", "post"), - "#class" : hotleak.HotleakPostExtractor, - "#urls" : "https://image-cdn.hotleak.vip/storage/images/e98/18ad68/18ad68.webp", - - "id" : 1617145, - "creator" : "kaiyakawaii", - "type" : "photo", - "filename" : "18ad68", - "extension": "webp", -}, - -{ - "#url" : "https://hotleak.vip/lilmochidoll/video/1625538", - "#category": ("", "hotleak", "post"), - "#class" : hotleak.HotleakPostExtractor, - "#pattern" : r"ytdl:https://cdn\d+-leak\.camhdxx\.com/.+,\d+/1661/1625538/index\.m3u8", - - "id" : 1625538, - "creator" : "lilmochidoll", - "type" : "video", - "filename" : "index", - "extension": "mp4", -}, - -{ - "#url" : "https://hotleak.vip/kaiyakawaii", - "#category": ("", "hotleak", "creator"), - "#class" : hotleak.HotleakCreatorExtractor, - "#range" : "1-200", - "#count" : 200, -}, - -{ - "#url" : "https://hotleak.vip/stellaviolet", - "#category": ("", "hotleak", "creator"), - "#class" : hotleak.HotleakCreatorExtractor, - "#count" : "> 600", -}, - -{ - "#url" : "https://hotleak.vip/doesnotexist", - "#category": ("", "hotleak", "creator"), - "#class" : hotleak.HotleakCreatorExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://hotleak.vip/photos", - "#category": ("", "hotleak", "category"), - "#class" : hotleak.HotleakCategoryExtractor, - "#pattern" : hotleak.HotleakPostExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://hotleak.vip/videos", - "#category": ("", "hotleak", "category"), - "#class" : hotleak.HotleakCategoryExtractor, -}, - -{ - "#url" : "https://hotleak.vip/creators", - "#category": ("", "hotleak", "category"), - "#class" : hotleak.HotleakCategoryExtractor, - "#pattern" : hotleak.HotleakCreatorExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://hotleak.vip/hot", - "#category": ("", "hotleak", "category"), - "#class" : hotleak.HotleakCategoryExtractor, -}, - -{ - "#url" : "https://hotleak.vip/search?search=gallery-dl", - "#category": ("", "hotleak", "search"), - "#class" : hotleak.HotleakSearchExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://hotleak.vip/search?search=hannah", - "#category": ("", "hotleak", "search"), - "#class" : hotleak.HotleakSearchExtractor, - "#count" : "> 30", -}, - + { + "#url": "https://hotleak.vip/kaiyakawaii/photo/1617145", + "#category": ("", "hotleak", "post"), + "#class": hotleak.HotleakPostExtractor, + "#urls": "https://image-cdn.hotleak.vip/storage/images/e98/18ad68/18ad68.webp", + "id": 1617145, + "creator": "kaiyakawaii", + "type": "photo", + "filename": "18ad68", + "extension": "webp", + }, + { + "#url": "https://hotleak.vip/lilmochidoll/video/1625538", + "#category": ("", "hotleak", "post"), + "#class": hotleak.HotleakPostExtractor, + "#pattern": r"ytdl:https://cdn\d+-leak\.camhdxx\.com/.+,\d+/1661/1625538/index\.m3u8", + "id": 1625538, + "creator": "lilmochidoll", + "type": "video", + "filename": "index", + "extension": "mp4", + }, + { + "#url": "https://hotleak.vip/kaiyakawaii", + "#category": ("", "hotleak", "creator"), + "#class": hotleak.HotleakCreatorExtractor, + "#range": "1-200", + "#count": 200, + }, + { + "#url": "https://hotleak.vip/stellaviolet", + "#category": ("", "hotleak", "creator"), + "#class": hotleak.HotleakCreatorExtractor, + "#count": "> 600", + }, + { + "#url": "https://hotleak.vip/doesnotexist", + "#category": ("", "hotleak", "creator"), + "#class": hotleak.HotleakCreatorExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://hotleak.vip/photos", + "#category": ("", "hotleak", "category"), + "#class": hotleak.HotleakCategoryExtractor, + "#pattern": hotleak.HotleakPostExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://hotleak.vip/videos", + "#category": ("", "hotleak", "category"), + "#class": hotleak.HotleakCategoryExtractor, + }, + { + "#url": "https://hotleak.vip/creators", + "#category": ("", "hotleak", "category"), + "#class": hotleak.HotleakCategoryExtractor, + "#pattern": hotleak.HotleakCreatorExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://hotleak.vip/hot", + "#category": ("", "hotleak", "category"), + "#class": hotleak.HotleakCategoryExtractor, + }, + { + "#url": "https://hotleak.vip/search?search=gallery-dl", + "#category": ("", "hotleak", "search"), + "#class": hotleak.HotleakSearchExtractor, + "#count": 0, + }, + { + "#url": "https://hotleak.vip/search?search=hannah", + "#category": ("", "hotleak", "search"), + "#class": hotleak.HotleakSearchExtractor, + "#count": "> 30", + }, ) diff --git a/test/results/hypnohub.py b/test/results/hypnohub.py index d979b620e..bc888fad3 100644 --- a/test/results/hypnohub.py +++ b/test/results/hypnohub.py @@ -1,77 +1,69 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v02 - __tests__ = ( -{ - "#url" : "https://hypnohub.net/index.php?page=post&s=list&tags=gonoike_biwa", - "#category": ("gelbooru_v02", "hypnohub", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#sha1_url": "fe662b86d38c331fcac9c62af100167d404937dc", -}, - -{ - "#url" : "https://hypnohub.net/index.php?page=pool&s=show&id=61", - "#category": ("gelbooru_v02", "hypnohub", "pool"), - "#class" : gelbooru_v02.GelbooruV02PoolExtractor, - "#count" : 3, - "#sha1_url": "d314826280073441a2da609f70ee814d1f4b9407", -}, - -{ - "#url" : "https://hypnohub.net/index.php?page=favorites&s=view&id=43546", - "#category": ("gelbooru_v02", "hypnohub", "favorite"), - "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, - "#count" : 3, -}, - -{ - "#url" : "https://hypnohub.net/index.php?page=post&s=view&id=1439", - "#category": ("gelbooru_v02", "hypnohub", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#options" : { - "tags" : True, - "notes": True, + { + "#url": "https://hypnohub.net/index.php?page=post&s=list&tags=gonoike_biwa", + "#category": ("gelbooru_v02", "hypnohub", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#sha1_url": "fe662b86d38c331fcac9c62af100167d404937dc", }, - "#pattern" : r"https://hypnohub\.net/images/90/24/90245c3c5250c2a8173255d3923a010b\.jpg", - "#sha1_content": "5987c5d2354f22e5fa9b7ee7ce4a6f7beb8b2b71", - - "tags_artist" : "brokenteapot", - "tags_character": "hsien-ko", - "tags_copyright": "capcom darkstalkers", - "tags_general" : str, - "tags_metadata" : "dialogue text translated", - "notes" : [ - { - "body" : "Master Master Master Master Master Master", - "height": 83, - "id" : 10577, - "width" : 129, - "x" : 259, - "y" : 20, - }, - { - "body" : "Response Response Response Response Response Response", - "height": 86, - "id" : 10578, - "width" : 125, - "x" : 126, - "y" : 20, - }, - { - "body" : "Obedience Obedience Obedience Obedience Obedience Obedience", - "height": 80, - "id" : 10579, - "width" : 98, - "x" : 20, - "y" : 20, + { + "#url": "https://hypnohub.net/index.php?page=pool&s=show&id=61", + "#category": ("gelbooru_v02", "hypnohub", "pool"), + "#class": gelbooru_v02.GelbooruV02PoolExtractor, + "#count": 3, + "#sha1_url": "d314826280073441a2da609f70ee814d1f4b9407", + }, + { + "#url": "https://hypnohub.net/index.php?page=favorites&s=view&id=43546", + "#category": ("gelbooru_v02", "hypnohub", "favorite"), + "#class": gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count": 3, + }, + { + "#url": "https://hypnohub.net/index.php?page=post&s=view&id=1439", + "#category": ("gelbooru_v02", "hypnohub", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#options": { + "tags": True, + "notes": True, }, - ], -}, - + "#pattern": r"https://hypnohub\.net/images/90/24/90245c3c5250c2a8173255d3923a010b\.jpg", + "#sha1_content": "5987c5d2354f22e5fa9b7ee7ce4a6f7beb8b2b71", + "tags_artist": "brokenteapot", + "tags_character": "hsien-ko", + "tags_copyright": "capcom darkstalkers", + "tags_general": str, + "tags_metadata": "dialogue text translated", + "notes": [ + { + "body": "Master Master Master Master Master Master", + "height": 83, + "id": 10577, + "width": 129, + "x": 259, + "y": 20, + }, + { + "body": "Response Response Response Response Response Response", + "height": 86, + "id": 10578, + "width": 125, + "x": 126, + "y": 20, + }, + { + "body": "Obedience Obedience Obedience Obedience Obedience Obedience", + "height": 80, + "id": 10579, + "width": 98, + "x": 20, + "y": 20, + }, + ], + }, ) diff --git a/test/results/idolcomplex.py b/test/results/idolcomplex.py index dfcff1994..a4832682b 100644 --- a/test/results/idolcomplex.py +++ b/test/results/idolcomplex.py @@ -1,122 +1,104 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import idolcomplex - __tests__ = ( -{ - "#url" : "https://idol.sankakucomplex.com/en/posts?tags=lyumos", - "#category": ("booru", "idolcomplex", "tag"), - "#class" : idolcomplex.IdolcomplexTagExtractor, - "#pattern" : r"https://i[sv]\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[^/]{32}\.\w+\?e=\d+&m=[^&#]+", - "#range" : "18-22", - "#count" : 5, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/posts/?tags=lyumos", - "#category": ("booru", "idolcomplex", "tag"), - "#class" : idolcomplex.IdolcomplexTagExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/en/?tags=lyumos", - "#category": ("booru", "idolcomplex", "tag"), - "#class" : idolcomplex.IdolcomplexTagExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/?tags=lyumos", - "#category": ("booru", "idolcomplex", "tag"), - "#class" : idolcomplex.IdolcomplexTagExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/?tags=lyumos+wreath&page=3&next=694215", - "#category": ("booru", "idolcomplex", "tag"), - "#class" : idolcomplex.IdolcomplexTagExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/en/pools/e9PMwnwRBK3", - "#category": ("booru", "idolcomplex", "pool"), - "#class" : idolcomplex.IdolcomplexPoolExtractor, - "#count" : 3, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/en/pools/show/145", - "#category": ("booru", "idolcomplex", "pool"), - "#class" : idolcomplex.IdolcomplexPoolExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/pool/show/145", - "#category": ("booru", "idolcomplex", "pool"), - "#class" : idolcomplex.IdolcomplexPoolExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/en/posts/vkr36qdOaZ4", - "#category": ("booru", "idolcomplex", "post"), - "#class" : idolcomplex.IdolcomplexPostExtractor, - "#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd", - - "created_at" : "2017-11-24 17:01:27.696", - "date" : "dt:2017-11-24 17:01:27", - "extension" : "jpg", - "file_url" : r"re:https://i[sv]\.sankakucomplex\.com/data/50/9e/509eccbba54a43cea6b275a65b93c51d\.jpg\?", - "filename" : "509eccbba54a43cea6b275a65b93c51d", - "height" : 683, - "id" : "vkr36qdOaZ4", # legacy ID: 694215 - "md5" : "509eccbba54a43cea6b275a65b93c51d", - "rating" : "g", - "tags" : "lyumos the_witcher shani_(the_witcher) 1girl green_eyes non-asian redhead waistcoat wreath cosplay 3:2_aspect_ratio", - "tags_character": "shani_(the_witcher)", - "tags_copyright": "the_witcher", - "tags_general" : "1girl green_eyes non-asian redhead waistcoat wreath", - "tags_genre" : "cosplay", - "tags_idol" : "lyumos", - "tags_medium" : "3:2_aspect_ratio", - "vote_average" : range(4, 5), - "vote_count" : range(25, 40), - "width" : 1024, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/en/posts/509eccbba54a43cea6b275a65b93c51d", - "#category": ("booru", "idolcomplex", "post"), - "#class" : idolcomplex.IdolcomplexPostExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/en/posts/show/509eccbba54a43cea6b275a65b93c51d", - "#category": ("booru", "idolcomplex", "post"), - "#class" : idolcomplex.IdolcomplexPostExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/posts/509eccbba54a43cea6b275a65b93c51d", - "#category": ("booru", "idolcomplex", "post"), - "#class" : idolcomplex.IdolcomplexPostExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/post/show/694215", - "#category": ("booru", "idolcomplex", "post"), - "#class" : idolcomplex.IdolcomplexPostExtractor, - "#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd", - - "id" : "vkr36qdOaZ4", # legacy ID: 694215 - "tags_character": "shani_(the_witcher)", - "tags_copyright": "the_witcher", - "tags_idol" : str, - "tags_medium" : str, - "tags_general" : str, -}, - + { + "#url": "https://idol.sankakucomplex.com/en/posts?tags=lyumos", + "#category": ("booru", "idolcomplex", "tag"), + "#class": idolcomplex.IdolcomplexTagExtractor, + "#pattern": r"https://i[sv]\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[^/]{32}\.\w+\?e=\d+&m=[^&#]+", + "#range": "18-22", + "#count": 5, + }, + { + "#url": "https://idol.sankakucomplex.com/posts/?tags=lyumos", + "#category": ("booru", "idolcomplex", "tag"), + "#class": idolcomplex.IdolcomplexTagExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/en/?tags=lyumos", + "#category": ("booru", "idolcomplex", "tag"), + "#class": idolcomplex.IdolcomplexTagExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/?tags=lyumos", + "#category": ("booru", "idolcomplex", "tag"), + "#class": idolcomplex.IdolcomplexTagExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/?tags=lyumos+wreath&page=3&next=694215", + "#category": ("booru", "idolcomplex", "tag"), + "#class": idolcomplex.IdolcomplexTagExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/en/pools/e9PMwnwRBK3", + "#category": ("booru", "idolcomplex", "pool"), + "#class": idolcomplex.IdolcomplexPoolExtractor, + "#count": 3, + }, + { + "#url": "https://idol.sankakucomplex.com/en/pools/show/145", + "#category": ("booru", "idolcomplex", "pool"), + "#class": idolcomplex.IdolcomplexPoolExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/pool/show/145", + "#category": ("booru", "idolcomplex", "pool"), + "#class": idolcomplex.IdolcomplexPoolExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/en/posts/vkr36qdOaZ4", + "#category": ("booru", "idolcomplex", "post"), + "#class": idolcomplex.IdolcomplexPostExtractor, + "#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd", + "created_at": "2017-11-24 17:01:27.696", + "date": "dt:2017-11-24 17:01:27", + "extension": "jpg", + "file_url": r"re:https://i[sv]\.sankakucomplex\.com/data/50/9e/509eccbba54a43cea6b275a65b93c51d\.jpg\?", + "filename": "509eccbba54a43cea6b275a65b93c51d", + "height": 683, + "id": "vkr36qdOaZ4", # legacy ID: 694215 + "md5": "509eccbba54a43cea6b275a65b93c51d", + "rating": "g", + "tags": "lyumos the_witcher shani_(the_witcher) 1girl green_eyes non-asian redhead waistcoat wreath cosplay 3:2_aspect_ratio", + "tags_character": "shani_(the_witcher)", + "tags_copyright": "the_witcher", + "tags_general": "1girl green_eyes non-asian redhead waistcoat wreath", + "tags_genre": "cosplay", + "tags_idol": "lyumos", + "tags_medium": "3:2_aspect_ratio", + "vote_average": range(4, 5), + "vote_count": range(25, 40), + "width": 1024, + }, + { + "#url": "https://idol.sankakucomplex.com/en/posts/509eccbba54a43cea6b275a65b93c51d", + "#category": ("booru", "idolcomplex", "post"), + "#class": idolcomplex.IdolcomplexPostExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/en/posts/show/509eccbba54a43cea6b275a65b93c51d", + "#category": ("booru", "idolcomplex", "post"), + "#class": idolcomplex.IdolcomplexPostExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/posts/509eccbba54a43cea6b275a65b93c51d", + "#category": ("booru", "idolcomplex", "post"), + "#class": idolcomplex.IdolcomplexPostExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/post/show/694215", + "#category": ("booru", "idolcomplex", "post"), + "#class": idolcomplex.IdolcomplexPostExtractor, + "#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd", + "id": "vkr36qdOaZ4", # legacy ID: 694215 + "tags_character": "shani_(the_witcher)", + "tags_copyright": "the_witcher", + "tags_idol": str, + "tags_medium": str, + "tags_general": str, + }, ) diff --git a/test/results/illusioncardsbooru.py b/test/results/illusioncardsbooru.py index ba0793641..c13e76c02 100644 --- a/test/results/illusioncardsbooru.py +++ b/test/results/illusioncardsbooru.py @@ -1,34 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v01 - __tests__ = ( -{ - "#url" : "https://illusioncards.booru.org/index.php?page=post&s=list&tags=koikatsu", - "#category": ("gelbooru_v01", "illusioncardsbooru", "tag"), - "#class" : gelbooru_v01.GelbooruV01TagExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://illusioncards.booru.org/index.php?page=favorites&s=view&id=84887", - "#category": ("gelbooru_v01", "illusioncardsbooru", "favorite"), - "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://illusioncards.booru.org/index.php?page=post&s=view&id=82746", - "#category": ("gelbooru_v01", "illusioncardsbooru", "post"), - "#class" : gelbooru_v01.GelbooruV01PostExtractor, - "#sha1_url" : "3f9cd2fadf78869b90bc5422f27b48f1af0e0909", - "#sha1_content": "159e60b92d05597bd1bb63510c2c3e4a4bada1dc", -}, - + { + "#url": "https://illusioncards.booru.org/index.php?page=post&s=list&tags=koikatsu", + "#category": ("gelbooru_v01", "illusioncardsbooru", "tag"), + "#class": gelbooru_v01.GelbooruV01TagExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://illusioncards.booru.org/index.php?page=favorites&s=view&id=84887", + "#category": ("gelbooru_v01", "illusioncardsbooru", "favorite"), + "#class": gelbooru_v01.GelbooruV01FavoriteExtractor, + "#count": 2, + }, + { + "#url": "https://illusioncards.booru.org/index.php?page=post&s=view&id=82746", + "#category": ("gelbooru_v01", "illusioncardsbooru", "post"), + "#class": gelbooru_v01.GelbooruV01PostExtractor, + "#sha1_url": "3f9cd2fadf78869b90bc5422f27b48f1af0e0909", + "#sha1_content": "159e60b92d05597bd1bb63510c2c3e4a4bada1dc", + }, ) diff --git a/test/results/imagebam.py b/test/results/imagebam.py index 759e2dc48..2de9cf813 100644 --- a/test/results/imagebam.py +++ b/test/results/imagebam.py @@ -1,78 +1,67 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imagebam from gallery_dl import exception - +from gallery_dl.extractor import imagebam __tests__ = ( -{ - "#url" : "https://www.imagebam.com/gallery/adz2y0f9574bjpmonaismyrhtjgvey4o", - "#category": ("", "imagebam", "gallery"), - "#class" : imagebam.ImagebamGalleryExtractor, - "#sha1_url" : "76d976788ae2757ac81694736b07b72356f5c4c8", - "#sha1_metadata": "b048478b1bbba3072a7fa9fcc40630b3efad1f6c", - "#sha1_content" : "596e6bfa157f2c7169805d50075c2986549973a8", -}, - -{ - "#url" : "http://www.imagebam.com/gallery/op9dwcklwdrrguibnkoe7jxgvig30o5p", - "#category": ("", "imagebam", "gallery"), - "#class" : imagebam.ImagebamGalleryExtractor, - "#count" : 107, - "#sha1_url": "32ae6fe5dc3e4ca73ff6252e522d16473595d1d1", -}, - -{ - "#url" : "http://www.imagebam.com/gallery/gsl8teckymt4vbvx1stjkyk37j70va2c", - "#category": ("", "imagebam", "gallery"), - "#class" : imagebam.ImagebamGalleryExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://www.imagebam.com/view/GA3MT1", - "#comment" : "/view/ path (#2378)", - "#category": ("", "imagebam", "gallery"), - "#class" : imagebam.ImagebamGalleryExtractor, - "#sha1_url" : "35018ce1e00a2d2825a33d3cd37857edaf804919", - "#sha1_metadata": "3a9f98178f73694c527890c0d7ca9a92b46987ba", -}, - -{ - "#url" : "https://www.imagebam.com/image/94d56c502511890", - "#category": ("", "imagebam", "image"), - "#class" : imagebam.ImagebamImageExtractor, - "#sha1_url" : "5e9ba3b1451f8ded0ae3a1b84402888893915d4a", - "#sha1_metadata": "2a4380d4b57554ff793898c2d6ec60987c86d1a1", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - -{ - "#url" : "http://images3.imagebam.com/1d/8c/44/94d56c502511890.png", - "#category": ("", "imagebam", "image"), - "#class" : imagebam.ImagebamImageExtractor, -}, - -{ - "#url" : "https://www.imagebam.com/image/0850951366904951", - "#comment" : "NSFW (#1534)", - "#category": ("", "imagebam", "image"), - "#class" : imagebam.ImagebamImageExtractor, - "#sha1_url": "d37297b17ed1615b4311c8ed511e50ce46e4c748", -}, - -{ - "#url" : "https://www.imagebam.com/view/ME8JOQP", - "#comment" : "/view/ path (#2378)", - "#category": ("", "imagebam", "image"), - "#class" : imagebam.ImagebamImageExtractor, - "#sha1_url" : "4dca72bbe61a0360185cf4ab2bed8265b49565b8", - "#sha1_metadata": "15a494c02fd30846b41b42a26117aedde30e4ceb", - "#sha1_content" : "f81008666b17a42d8834c4749b910e1dc10a6e83", -}, - + { + "#url": "https://www.imagebam.com/gallery/adz2y0f9574bjpmonaismyrhtjgvey4o", + "#category": ("", "imagebam", "gallery"), + "#class": imagebam.ImagebamGalleryExtractor, + "#sha1_url": "76d976788ae2757ac81694736b07b72356f5c4c8", + "#sha1_metadata": "b048478b1bbba3072a7fa9fcc40630b3efad1f6c", + "#sha1_content": "596e6bfa157f2c7169805d50075c2986549973a8", + }, + { + "#url": "http://www.imagebam.com/gallery/op9dwcklwdrrguibnkoe7jxgvig30o5p", + "#category": ("", "imagebam", "gallery"), + "#class": imagebam.ImagebamGalleryExtractor, + "#count": 107, + "#sha1_url": "32ae6fe5dc3e4ca73ff6252e522d16473595d1d1", + }, + { + "#url": "http://www.imagebam.com/gallery/gsl8teckymt4vbvx1stjkyk37j70va2c", + "#category": ("", "imagebam", "gallery"), + "#class": imagebam.ImagebamGalleryExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://www.imagebam.com/view/GA3MT1", + "#comment": "/view/ path (#2378)", + "#category": ("", "imagebam", "gallery"), + "#class": imagebam.ImagebamGalleryExtractor, + "#sha1_url": "35018ce1e00a2d2825a33d3cd37857edaf804919", + "#sha1_metadata": "3a9f98178f73694c527890c0d7ca9a92b46987ba", + }, + { + "#url": "https://www.imagebam.com/image/94d56c502511890", + "#category": ("", "imagebam", "image"), + "#class": imagebam.ImagebamImageExtractor, + "#sha1_url": "5e9ba3b1451f8ded0ae3a1b84402888893915d4a", + "#sha1_metadata": "2a4380d4b57554ff793898c2d6ec60987c86d1a1", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, + { + "#url": "http://images3.imagebam.com/1d/8c/44/94d56c502511890.png", + "#category": ("", "imagebam", "image"), + "#class": imagebam.ImagebamImageExtractor, + }, + { + "#url": "https://www.imagebam.com/image/0850951366904951", + "#comment": "NSFW (#1534)", + "#category": ("", "imagebam", "image"), + "#class": imagebam.ImagebamImageExtractor, + "#sha1_url": "d37297b17ed1615b4311c8ed511e50ce46e4c748", + }, + { + "#url": "https://www.imagebam.com/view/ME8JOQP", + "#comment": "/view/ path (#2378)", + "#category": ("", "imagebam", "image"), + "#class": imagebam.ImagebamImageExtractor, + "#sha1_url": "4dca72bbe61a0360185cf4ab2bed8265b49565b8", + "#sha1_metadata": "15a494c02fd30846b41b42a26117aedde30e4ceb", + "#sha1_content": "f81008666b17a42d8834c4749b910e1dc10a6e83", + }, ) diff --git a/test/results/imagechest.py b/test/results/imagechest.py index 41254a390..60cd7887a 100644 --- a/test/results/imagechest.py +++ b/test/results/imagechest.py @@ -1,52 +1,44 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imagechest from gallery_dl import exception - +from gallery_dl.extractor import imagechest __tests__ = ( -{ - "#url" : "https://imgchest.com/p/3na7kr3by8d", - "#category": ("", "imagechest", "gallery"), - "#class" : imagechest.ImagechestGalleryExtractor, - "#pattern" : r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", - "#count" : 3, - "#sha1_url" : "7328ca4ec2459378d725e3be19f661d2b045feda", - "#sha1_content": "076959e65be30249a2c651fbe6090dc30ba85193", - - "count" : 3, - "gallery_id": "3na7kr3by8d", - "num" : int, - "title" : "Wizardry - Video Game From The Mid 80's", -}, - -{ - "#url" : "https://imgchest.com/p/9p4n3q2z7nq", - "#comment" : "'Load More Files' button (#4028)", - "#category": ("", "imagechest", "gallery"), - "#class" : imagechest.ImagechestGalleryExtractor, - "#pattern" : r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", - "#count" : 52, - "#sha1_url": "f5674e8ba79d336193c9f698708d9dcc10e78cc7", -}, - -{ - "#url" : "https://imgchest.com/p/xxxxxxxxxxx", - "#category": ("", "imagechest", "gallery"), - "#class" : imagechest.ImagechestGalleryExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://imgchest.com/u/LunarLandr", - "#category": ("", "imagechest", "user"), - "#class" : imagechest.ImagechestUserExtractor, - "#pattern" : imagechest.ImagechestGalleryExtractor.pattern, - "#count" : 279, -}, - + { + "#url": "https://imgchest.com/p/3na7kr3by8d", + "#category": ("", "imagechest", "gallery"), + "#class": imagechest.ImagechestGalleryExtractor, + "#pattern": r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", + "#count": 3, + "#sha1_url": "7328ca4ec2459378d725e3be19f661d2b045feda", + "#sha1_content": "076959e65be30249a2c651fbe6090dc30ba85193", + "count": 3, + "gallery_id": "3na7kr3by8d", + "num": int, + "title": "Wizardry - Video Game From The Mid 80's", + }, + { + "#url": "https://imgchest.com/p/9p4n3q2z7nq", + "#comment": "'Load More Files' button (#4028)", + "#category": ("", "imagechest", "gallery"), + "#class": imagechest.ImagechestGalleryExtractor, + "#pattern": r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", + "#count": 52, + "#sha1_url": "f5674e8ba79d336193c9f698708d9dcc10e78cc7", + }, + { + "#url": "https://imgchest.com/p/xxxxxxxxxxx", + "#category": ("", "imagechest", "gallery"), + "#class": imagechest.ImagechestGalleryExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://imgchest.com/u/LunarLandr", + "#category": ("", "imagechest", "user"), + "#class": imagechest.ImagechestUserExtractor, + "#pattern": imagechest.ImagechestGalleryExtractor.pattern, + "#count": 279, + }, ) diff --git a/test/results/imagefap.py b/test/results/imagefap.py index 8cdaf9b1c..2b4768a33 100644 --- a/test/results/imagefap.py +++ b/test/results/imagefap.py @@ -1,211 +1,182 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imagefap from gallery_dl import exception - +from gallery_dl.extractor import imagefap __tests__ = ( -{ - "#url" : "https://www.imagefap.com/gallery/7102714", - "#category": ("", "imagefap", "gallery"), - "#class" : imagefap.ImagefapGalleryExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://www.imagefap.com/gallery/7876223", - "#category": ("", "imagefap", "gallery"), - "#class" : imagefap.ImagefapGalleryExtractor, - "#pattern" : r"https://cdn[ch]?\.imagefap\.com/images/full/\d+/\d+/\d+\.jpg", - "#count" : 44, - - "categories" : [ - "Asses", - "Softcore", - "Pornstars", - ], - "count" : 44, - "description": "", - "gallery_id" : 7876223, - "image_id" : int, - "num" : int, - "tags" : [ - "big ass", - "panties", - "horny", - "pussy", - "exposed", - "outdoor", - ], - "title" : "Kelsi Monroe in lingerie", - "uploader" : "BdRachel", -}, - -{ - "#url" : "https://www.imagefap.com/gallery/6706356", - "#comment" : "description (#3905)", - "#category": ("", "imagefap", "gallery"), - "#class" : imagefap.ImagefapGalleryExtractor, - "#range" : "1", - - "categories" : [ - "Lesbian", - "Fetish", - "Animated GIFS", - ], - "count" : 75, - "description": "A mixed collection of pics and gifs depicting lesbian femdom.\n\nAll images originally found on various Tumblr blogs and through the internet.\n\nObviously I don't own any of the images so if you do and you would like them removed please just let me know and I shall remove them straight away.", - "gallery_id" : 6706356, - "tags" : [ - "lesbian", - "femdom", - "lesbian femdom", - "lezdom", - "dominant women", - "submissive women", - ], - "title" : "Lezdom, Lesbian Femdom, Lesbian Domination - 3", - "uploader" : "pussysimon", -}, - -{ - "#url" : "https://www.imagefap.com/pictures/7102714", - "#category": ("", "imagefap", "gallery"), - "#class" : imagefap.ImagefapGalleryExtractor, -}, - -{ - "#url" : "https://www.imagefap.com/gallery.php?gid=7102714", - "#category": ("", "imagefap", "gallery"), - "#class" : imagefap.ImagefapGalleryExtractor, -}, - -{ - "#url" : "https://beta.imagefap.com/gallery.php?gid=7102714", - "#category": ("", "imagefap", "gallery"), - "#class" : imagefap.ImagefapGalleryExtractor, -}, - -{ - "#url" : "https://www.imagefap.com/photo/1962981893", - "#category": ("", "imagefap", "image"), - "#class" : imagefap.ImagefapImageExtractor, - "#pattern" : r"https://cdn[ch]?\.imagefap\.com/images/full/65/196/1962981893\.jpg", - - "date" : "21/08/2014", - "gallery_id": 7876223, - "height" : 1600, - "image_id" : 1962981893, - "title" : "Kelsi Monroe in lingerie", - "uploader" : "BdRachel", - "width" : 1066, -}, - -{ - "#url" : "https://beta.imagefap.com/photo/1962981893", - "#category": ("", "imagefap", "image"), - "#class" : imagefap.ImagefapImageExtractor, -}, - -{ - "#url" : "https://www.imagefap.com/organizer/409758", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#pattern" : r"https://www\.imagefap\.com/gallery/7876223", - "#count" : 1, - "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", -}, - -{ - "#url" : "https://www.imagefap.com/organizer/613950/Grace-Stout", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, - "#count" : 31, - - "title": r"re:Grace Stout .+", -}, - -{ - "#url" : "https://www.imagefap.com/usergallery.php?userid=1981976&folderid=409758", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#urls" : "https://www.imagefap.com/gallery/7876223", - - "folder" : "Softcore", - "gallery_id": "7876223", - "title" : "Kelsi Monroe in lingerie", -}, - -{ - "#url" : "https://www.imagefap.com/usergallery.php?user=BdRachel&folderid=409758", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", -}, - -{ - "#url" : "https://www.imagefap.com/profile/BdRachel/galleries?folderid=-1", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, - "#range" : "1-40", - - "folder": "Uncategorized", -}, - -{ - "#url" : "https://www.imagefap.com/usergallery.php?userid=1981976&folderid=-1", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, - "#range" : "1-40", -}, - -{ - "#url" : "https://www.imagefap.com/usergallery.php?user=BdRachel&folderid=-1", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, - "#range" : "1-40", -}, - -{ - "#url" : "https://www.imagefap.com/profile/BdRachel", - "#category": ("", "imagefap", "user"), - "#class" : imagefap.ImagefapUserExtractor, - "#pattern" : imagefap.ImagefapFolderExtractor.pattern, - "#count" : ">= 18", -}, - -{ - "#url" : "https://www.imagefap.com/usergallery.php?userid=1862791", - "#category": ("", "imagefap", "user"), - "#class" : imagefap.ImagefapUserExtractor, - "#pattern" : r"https://www\.imagefap\.com/profile/LucyRae/galleries\?folderid=-1", - "#count" : 1, -}, - -{ - "#url" : "https://www.imagefap.com/profile/BdRachel/galleries", - "#category": ("", "imagefap", "user"), - "#class" : imagefap.ImagefapUserExtractor, -}, - -{ - "#url" : "https://www.imagefap.com/profile.php?user=BdRachel", - "#category": ("", "imagefap", "user"), - "#class" : imagefap.ImagefapUserExtractor, -}, - -{ - "#url" : "https://beta.imagefap.com/profile.php?user=BdRachel", - "#category": ("", "imagefap", "user"), - "#class" : imagefap.ImagefapUserExtractor, -}, - + { + "#url": "https://www.imagefap.com/gallery/7102714", + "#category": ("", "imagefap", "gallery"), + "#class": imagefap.ImagefapGalleryExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://www.imagefap.com/gallery/7876223", + "#category": ("", "imagefap", "gallery"), + "#class": imagefap.ImagefapGalleryExtractor, + "#pattern": r"https://cdn[ch]?\.imagefap\.com/images/full/\d+/\d+/\d+\.jpg", + "#count": 44, + "categories": [ + "Asses", + "Softcore", + "Pornstars", + ], + "count": 44, + "description": "", + "gallery_id": 7876223, + "image_id": int, + "num": int, + "tags": [ + "big ass", + "panties", + "horny", + "pussy", + "exposed", + "outdoor", + ], + "title": "Kelsi Monroe in lingerie", + "uploader": "BdRachel", + }, + { + "#url": "https://www.imagefap.com/gallery/6706356", + "#comment": "description (#3905)", + "#category": ("", "imagefap", "gallery"), + "#class": imagefap.ImagefapGalleryExtractor, + "#range": "1", + "categories": [ + "Lesbian", + "Fetish", + "Animated GIFS", + ], + "count": 75, + "description": "A mixed collection of pics and gifs depicting lesbian femdom.\n\nAll images originally found on various Tumblr blogs and through the internet.\n\nObviously I don't own any of the images so if you do and you would like them removed please just let me know and I shall remove them straight away.", + "gallery_id": 6706356, + "tags": [ + "lesbian", + "femdom", + "lesbian femdom", + "lezdom", + "dominant women", + "submissive women", + ], + "title": "Lezdom, Lesbian Femdom, Lesbian Domination - 3", + "uploader": "pussysimon", + }, + { + "#url": "https://www.imagefap.com/pictures/7102714", + "#category": ("", "imagefap", "gallery"), + "#class": imagefap.ImagefapGalleryExtractor, + }, + { + "#url": "https://www.imagefap.com/gallery.php?gid=7102714", + "#category": ("", "imagefap", "gallery"), + "#class": imagefap.ImagefapGalleryExtractor, + }, + { + "#url": "https://beta.imagefap.com/gallery.php?gid=7102714", + "#category": ("", "imagefap", "gallery"), + "#class": imagefap.ImagefapGalleryExtractor, + }, + { + "#url": "https://www.imagefap.com/photo/1962981893", + "#category": ("", "imagefap", "image"), + "#class": imagefap.ImagefapImageExtractor, + "#pattern": r"https://cdn[ch]?\.imagefap\.com/images/full/65/196/1962981893\.jpg", + "date": "21/08/2014", + "gallery_id": 7876223, + "height": 1600, + "image_id": 1962981893, + "title": "Kelsi Monroe in lingerie", + "uploader": "BdRachel", + "width": 1066, + }, + { + "#url": "https://beta.imagefap.com/photo/1962981893", + "#category": ("", "imagefap", "image"), + "#class": imagefap.ImagefapImageExtractor, + }, + { + "#url": "https://www.imagefap.com/organizer/409758", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#pattern": r"https://www\.imagefap\.com/gallery/7876223", + "#count": 1, + "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", + }, + { + "#url": "https://www.imagefap.com/organizer/613950/Grace-Stout", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#pattern": imagefap.ImagefapGalleryExtractor.pattern, + "#count": 31, + "title": r"re:Grace Stout .+", + }, + { + "#url": "https://www.imagefap.com/usergallery.php?userid=1981976&folderid=409758", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#urls": "https://www.imagefap.com/gallery/7876223", + "folder": "Softcore", + "gallery_id": "7876223", + "title": "Kelsi Monroe in lingerie", + }, + { + "#url": "https://www.imagefap.com/usergallery.php?user=BdRachel&folderid=409758", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", + }, + { + "#url": "https://www.imagefap.com/profile/BdRachel/galleries?folderid=-1", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#pattern": imagefap.ImagefapGalleryExtractor.pattern, + "#range": "1-40", + "folder": "Uncategorized", + }, + { + "#url": "https://www.imagefap.com/usergallery.php?userid=1981976&folderid=-1", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#pattern": imagefap.ImagefapGalleryExtractor.pattern, + "#range": "1-40", + }, + { + "#url": "https://www.imagefap.com/usergallery.php?user=BdRachel&folderid=-1", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#pattern": imagefap.ImagefapGalleryExtractor.pattern, + "#range": "1-40", + }, + { + "#url": "https://www.imagefap.com/profile/BdRachel", + "#category": ("", "imagefap", "user"), + "#class": imagefap.ImagefapUserExtractor, + "#pattern": imagefap.ImagefapFolderExtractor.pattern, + "#count": ">= 18", + }, + { + "#url": "https://www.imagefap.com/usergallery.php?userid=1862791", + "#category": ("", "imagefap", "user"), + "#class": imagefap.ImagefapUserExtractor, + "#pattern": r"https://www\.imagefap\.com/profile/LucyRae/galleries\?folderid=-1", + "#count": 1, + }, + { + "#url": "https://www.imagefap.com/profile/BdRachel/galleries", + "#category": ("", "imagefap", "user"), + "#class": imagefap.ImagefapUserExtractor, + }, + { + "#url": "https://www.imagefap.com/profile.php?user=BdRachel", + "#category": ("", "imagefap", "user"), + "#class": imagefap.ImagefapUserExtractor, + }, + { + "#url": "https://beta.imagefap.com/profile.php?user=BdRachel", + "#category": ("", "imagefap", "user"), + "#class": imagefap.ImagefapUserExtractor, + }, ) diff --git a/test/results/imagetwist.py b/test/results/imagetwist.py index d00ad2729..a230d1ca7 100644 --- a/test/results/imagetwist.py +++ b/test/results/imagetwist.py @@ -1,56 +1,47 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://imagetwist.com/f1i2s4vhvbrq/test.png", - "#category": ("imagehost", "imagetwist", "image"), - "#class" : imagehosts.ImagetwistImageExtractor, - "#sha1_url" : "8d5e168c0bee30211f821c6f3b2116e419d42671", - "#sha1_metadata": "d1060a4c2e3b73b83044e20681712c0ffdd6cfef", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - -{ - "#url" : "https://www.imagetwist.com/f1i2s4vhvbrq/test.png", - "#category": ("imagehost", "imagetwist", "image"), - "#class" : imagehosts.ImagetwistImageExtractor, -}, - -{ - "#url" : "https://phun.imagetwist.com/f1i2s4vhvbrq/test.png", - "#category": ("imagehost", "imagetwist", "image"), - "#class" : imagehosts.ImagetwistImageExtractor, -}, - -{ - "#url" : "https://imagehaha.com/f1i2s4vhvbrq/test.png", - "#category": ("imagehost", "imagetwist", "image"), - "#class" : imagehosts.ImagetwistImageExtractor, -}, - -{ - "#url" : "https://www.imagehaha.com/f1i2s4vhvbrq/test.png", - "#category": ("imagehost", "imagetwist", "image"), - "#class" : imagehosts.ImagetwistImageExtractor, -}, - -{ - "#url" : "https://imagetwist.com/p/gdldev/747223/digits", - "#category": ("imagehost", "imagetwist", "gallery"), - "#class" : imagehosts.ImagetwistGalleryExtractor, - "#urls" : ( - "https://imagetwist.com/j6eu91sbl9bs", - "https://imagetwist.com/vx4oh119izyr", - "https://imagetwist.com/n3td3a6vzzed", - "https://imagetwist.com/8uz6lmg31nmc", - ), -}, - + { + "#url": "https://imagetwist.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class": imagehosts.ImagetwistImageExtractor, + "#sha1_url": "8d5e168c0bee30211f821c6f3b2116e419d42671", + "#sha1_metadata": "d1060a4c2e3b73b83044e20681712c0ffdd6cfef", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, + { + "#url": "https://www.imagetwist.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class": imagehosts.ImagetwistImageExtractor, + }, + { + "#url": "https://phun.imagetwist.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class": imagehosts.ImagetwistImageExtractor, + }, + { + "#url": "https://imagehaha.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class": imagehosts.ImagetwistImageExtractor, + }, + { + "#url": "https://www.imagehaha.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class": imagehosts.ImagetwistImageExtractor, + }, + { + "#url": "https://imagetwist.com/p/gdldev/747223/digits", + "#category": ("imagehost", "imagetwist", "gallery"), + "#class": imagehosts.ImagetwistGalleryExtractor, + "#urls": ( + "https://imagetwist.com/j6eu91sbl9bs", + "https://imagetwist.com/vx4oh119izyr", + "https://imagetwist.com/n3td3a6vzzed", + "https://imagetwist.com/8uz6lmg31nmc", + ), + }, ) diff --git a/test/results/imagevenue.py b/test/results/imagevenue.py index f57b1c947..e18b2c4fd 100644 --- a/test/results/imagevenue.py +++ b/test/results/imagevenue.py @@ -1,34 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://www.imagevenue.com/ME13LS07", - "#category": ("imagehost", "imagevenue", "image"), - "#class" : imagehosts.ImagevenueImageExtractor, - "#pattern" : r"https://cdn-images\.imagevenue\.com/10/ac/05/ME13LS07_o\.png", - "#sha1_metadata": "ae15d6e3b2095f019eee84cd896700cd34b09c36", - "#sha1_content" : "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", -}, - -{ - "#url" : "https://www.imagevenue.com/view/o?i=92518_13732377annakarina424200712535AM_122_486lo.jpg&h=img150&l=loc486", - "#category": ("imagehost", "imagevenue", "image"), - "#class" : imagehosts.ImagevenueImageExtractor, - "#sha1_url": "8bf0254e29250d8f5026c0105bbdda3ee3d84980", -}, - -{ - "#url" : "http://img28116.imagevenue.com/img.php?image=th_52709_test_122_64lo.jpg", - "#category": ("imagehost", "imagevenue", "image"), - "#class" : imagehosts.ImagevenueImageExtractor, - "#sha1_url": "f98e3091df7f48a05fb60fbd86f789fc5ec56331", -}, - + { + "#url": "https://www.imagevenue.com/ME13LS07", + "#category": ("imagehost", "imagevenue", "image"), + "#class": imagehosts.ImagevenueImageExtractor, + "#pattern": r"https://cdn-images\.imagevenue\.com/10/ac/05/ME13LS07_o\.png", + "#sha1_metadata": "ae15d6e3b2095f019eee84cd896700cd34b09c36", + "#sha1_content": "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", + }, + { + "#url": "https://www.imagevenue.com/view/o?i=92518_13732377annakarina424200712535AM_122_486lo.jpg&h=img150&l=loc486", + "#category": ("imagehost", "imagevenue", "image"), + "#class": imagehosts.ImagevenueImageExtractor, + "#sha1_url": "8bf0254e29250d8f5026c0105bbdda3ee3d84980", + }, + { + "#url": "http://img28116.imagevenue.com/img.php?image=th_52709_test_122_64lo.jpg", + "#category": ("imagehost", "imagevenue", "image"), + "#class": imagehosts.ImagevenueImageExtractor, + "#sha1_url": "f98e3091df7f48a05fb60fbd86f789fc5ec56331", + }, ) diff --git a/test/results/imgbb.py b/test/results/imgbb.py index e2d1bc33d..21a3e5d94 100644 --- a/test/results/imgbb.py +++ b/test/results/imgbb.py @@ -1,92 +1,80 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imgbb from gallery_dl import exception - +from gallery_dl.extractor import imgbb __tests__ = ( -{ - "#url" : "https://ibb.co/album/i5PggF", - "#category": ("", "imgbb", "album"), - "#class" : imgbb.ImgbbAlbumExtractor, - "#patten" : r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", - "#count" : 91, - "#sha1_url" : "efe7e5a76531436e3b82c87e4ebd34c4dfeb484c", - "#sha1_metadata": "f1ab5492adb6333409f3367566a6dd7110537e21", - - "album_id" : "i5PggF", - "album_name" : "British Scrap Book", - "extension" : "jpg", - "id" : r"re:^\w{7}$", - "title" : str, - "url" : r"re:https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", - "user" : "folkie", - "user_id" : "GvFMGK", - "displayname": "Folkie", - "width" : range(501, 1034), - "height" : range(335, 768), - "size" : range(74758, 439037), -}, - -{ - "#url" : "https://ibb.co/album/i5PggF?sort=title_asc", - "#comment" : "'sort' query argument", - "#category": ("", "imgbb", "album"), - "#class" : imgbb.ImgbbAlbumExtractor, - "#patten" : r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", - "#count" : 91, - "#sha1_url" : "cde36552cc132a27178f22a1b9aceaa4df7e1575", - "#sha1_metadata": "b98bbb7671e31ebf9c7585fb9fc691b71bcdb546", -}, - -{ - "#url" : "https://ibb.co/album/kYKpwF", - "#comment" : "no user data (#471)", - "#category": ("", "imgbb", "album"), - "#class" : imgbb.ImgbbAlbumExtractor, - "#sha1_url": "ac0abcfcb89f4df6adc2f7e4ff872f3b03ef1bc7", - - "displayname": "", - "user" : "", - "user_id" : "", -}, - -{ - "#url" : "https://ibb.co/album/hqgWrF", - "#comment" : "private", - "#category": ("", "imgbb", "album"), - "#class" : imgbb.ImgbbAlbumExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://folkie.imgbb.com", - "#category": ("", "imgbb", "user"), - "#class" : imgbb.ImgbbUserExtractor, - "#patten" : r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", - "#range" : "1-80", -}, - -{ - "#url" : "https://ibb.co/fUqh5b", - "#category": ("", "imgbb", "image"), - "#class" : imgbb.ImgbbImageExtractor, - "#pattern" : r"https://i\.ibb\.co/g3kvx80/Arundel-Ireeman-5\.jpg", - "#sha1_content": "c5a0965178a8b357acd8aa39660092918c63795e", - - "id" : "fUqh5b", - "title" : "Arundel Ireeman 5", - "url" : "https://i.ibb.co/g3kvx80/Arundel-Ireeman-5.jpg", - "width" : 960, - "height" : 719, - "user" : "folkie", - "user_id" : "GvFMGK", - "displayname": "Folkie", - "extension" : "jpg", -}, - + { + "#url": "https://ibb.co/album/i5PggF", + "#category": ("", "imgbb", "album"), + "#class": imgbb.ImgbbAlbumExtractor, + "#patten": r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", + "#count": 91, + "#sha1_url": "efe7e5a76531436e3b82c87e4ebd34c4dfeb484c", + "#sha1_metadata": "f1ab5492adb6333409f3367566a6dd7110537e21", + "album_id": "i5PggF", + "album_name": "British Scrap Book", + "extension": "jpg", + "id": r"re:^\w{7}$", + "title": str, + "url": r"re:https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", + "user": "folkie", + "user_id": "GvFMGK", + "displayname": "Folkie", + "width": range(501, 1034), + "height": range(335, 768), + "size": range(74758, 439037), + }, + { + "#url": "https://ibb.co/album/i5PggF?sort=title_asc", + "#comment": "'sort' query argument", + "#category": ("", "imgbb", "album"), + "#class": imgbb.ImgbbAlbumExtractor, + "#patten": r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", + "#count": 91, + "#sha1_url": "cde36552cc132a27178f22a1b9aceaa4df7e1575", + "#sha1_metadata": "b98bbb7671e31ebf9c7585fb9fc691b71bcdb546", + }, + { + "#url": "https://ibb.co/album/kYKpwF", + "#comment": "no user data (#471)", + "#category": ("", "imgbb", "album"), + "#class": imgbb.ImgbbAlbumExtractor, + "#sha1_url": "ac0abcfcb89f4df6adc2f7e4ff872f3b03ef1bc7", + "displayname": "", + "user": "", + "user_id": "", + }, + { + "#url": "https://ibb.co/album/hqgWrF", + "#comment": "private", + "#category": ("", "imgbb", "album"), + "#class": imgbb.ImgbbAlbumExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://folkie.imgbb.com", + "#category": ("", "imgbb", "user"), + "#class": imgbb.ImgbbUserExtractor, + "#patten": r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", + "#range": "1-80", + }, + { + "#url": "https://ibb.co/fUqh5b", + "#category": ("", "imgbb", "image"), + "#class": imgbb.ImgbbImageExtractor, + "#pattern": r"https://i\.ibb\.co/g3kvx80/Arundel-Ireeman-5\.jpg", + "#sha1_content": "c5a0965178a8b357acd8aa39660092918c63795e", + "id": "fUqh5b", + "title": "Arundel Ireeman 5", + "url": "https://i.ibb.co/g3kvx80/Arundel-Ireeman-5.jpg", + "width": 960, + "height": 719, + "user": "folkie", + "user_id": "GvFMGK", + "displayname": "Folkie", + "extension": "jpg", + }, ) diff --git a/test/results/imgbox.py b/test/results/imgbox.py index ed0be5938..8beb6a750 100644 --- a/test/results/imgbox.py +++ b/test/results/imgbox.py @@ -1,52 +1,44 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imgbox from gallery_dl import exception - +from gallery_dl.extractor import imgbox __tests__ = ( -{ - "#url" : "https://imgbox.com/g/JaX5V5HX7g", - "#category": ("", "imgbox", "gallery"), - "#class" : imgbox.ImgboxGalleryExtractor, - "#sha1_url" : "da4f15b161461119ee78841d4b8e8d054d95f906", - "#sha1_metadata": "4b1e62820ac2c6205b7ad0b6322cc8e00dbe1b0c", - "#sha1_content" : "d20307dc8511ac24d688859c55abf2e2cc2dd3cc", -}, - -{ - "#url" : "https://imgbox.com/g/cUGEkRbdZZ", - "#category": ("", "imgbox", "gallery"), - "#class" : imgbox.ImgboxGalleryExtractor, - "#sha1_url" : "76506a3aab175c456910851f66227e90484ca9f7", - "#sha1_metadata": "fb0427b87983197849fb2887905e758f3e50cb6e", -}, - -{ - "#url" : "https://imgbox.com/g/JaX5V5HX7h", - "#category": ("", "imgbox", "gallery"), - "#class" : imgbox.ImgboxGalleryExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://imgbox.com/qHhw7lpG", - "#category": ("", "imgbox", "image"), - "#class" : imgbox.ImgboxImageExtractor, - "#sha1_url" : "ee9cdea6c48ad0161c1b5f81f6b0c9110997038c", - "#sha1_metadata": "dfc72310026b45f3feb4f9cada20c79b2575e1af", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - -{ - "#url" : "https://imgbox.com/qHhw7lpH", - "#category": ("", "imgbox", "image"), - "#class" : imgbox.ImgboxImageExtractor, - "#exception": exception.NotFoundError, -}, - + { + "#url": "https://imgbox.com/g/JaX5V5HX7g", + "#category": ("", "imgbox", "gallery"), + "#class": imgbox.ImgboxGalleryExtractor, + "#sha1_url": "da4f15b161461119ee78841d4b8e8d054d95f906", + "#sha1_metadata": "4b1e62820ac2c6205b7ad0b6322cc8e00dbe1b0c", + "#sha1_content": "d20307dc8511ac24d688859c55abf2e2cc2dd3cc", + }, + { + "#url": "https://imgbox.com/g/cUGEkRbdZZ", + "#category": ("", "imgbox", "gallery"), + "#class": imgbox.ImgboxGalleryExtractor, + "#sha1_url": "76506a3aab175c456910851f66227e90484ca9f7", + "#sha1_metadata": "fb0427b87983197849fb2887905e758f3e50cb6e", + }, + { + "#url": "https://imgbox.com/g/JaX5V5HX7h", + "#category": ("", "imgbox", "gallery"), + "#class": imgbox.ImgboxGalleryExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://imgbox.com/qHhw7lpG", + "#category": ("", "imgbox", "image"), + "#class": imgbox.ImgboxImageExtractor, + "#sha1_url": "ee9cdea6c48ad0161c1b5f81f6b0c9110997038c", + "#sha1_metadata": "dfc72310026b45f3feb4f9cada20c79b2575e1af", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, + { + "#url": "https://imgbox.com/qHhw7lpH", + "#category": ("", "imgbox", "image"), + "#class": imgbox.ImgboxImageExtractor, + "#exception": exception.NotFoundError, + }, ) diff --git a/test/results/imgclick.py b/test/results/imgclick.py index 983f21f47..5191a1cdc 100644 --- a/test/results/imgclick.py +++ b/test/results/imgclick.py @@ -1,20 +1,16 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "http://imgclick.net/4tbrre1oxew9/test-_-_.png.html", - "#category": ("imagehost", "imgclick", "image"), - "#class" : imagehosts.ImgclickImageExtractor, - "#sha1_url" : "140dcb250a325f2d26b2d918c18b8ac6a2a0f6ab", - "#sha1_metadata": "6895256143eab955622fc149aa367777a8815ba3", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - + { + "#url": "http://imgclick.net/4tbrre1oxew9/test-_-_.png.html", + "#category": ("imagehost", "imgclick", "image"), + "#class": imagehosts.ImgclickImageExtractor, + "#sha1_url": "140dcb250a325f2d26b2d918c18b8ac6a2a0f6ab", + "#sha1_metadata": "6895256143eab955622fc149aa367777a8815ba3", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, ) diff --git a/test/results/imgkiwi.py b/test/results/imgkiwi.py index 16a4aa962..9d8a716a3 100644 --- a/test/results/imgkiwi.py +++ b/test/results/imgkiwi.py @@ -1,51 +1,43 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import chevereto - __tests__ = ( -{ - "#url" : "https://img.kiwi/image/79de2c41-70f9-4a87-bd6d-00fe9997c0c4.JR2wZz", - "#category": ("chevereto", "imgkiwi", "image"), - "#class" : chevereto.CheveretoImageExtractor, - "#urls" : "https://img.kiwi/images/2023/02/28/11ac1ebf28a2eae8265026b28e9c4413.jpg", - "#sha1_content": "9ea704a77e2038b9008350682cfad53a614a60bd", - - "album" : "Kins3y Wolansk1", - "extension": "jpg", - "filename" : "11ac1ebf28a2eae8265026b28e9c4413", - "id" : "JR2wZz", - "url" : "https://img.kiwi/images/2023/02/28/11ac1ebf28a2eae8265026b28e9c4413.jpg", - "user" : "johnirl", -}, - -{ - "#url" : "https://img.kiwi/album/kins3y-wolansk1.8Jxc", - "#category": ("chevereto", "imgkiwi", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, - "#pattern" : chevereto.CheveretoImageExtractor.pattern, - "#count" : 19, -}, - -{ - "#url" : "https://img.kiwi/johnirl", - "#category": ("chevereto", "imgkiwi", "user"), - "#class" : chevereto.CheveretoUserExtractor, - "#pattern" : chevereto.CheveretoImageExtractor.pattern, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://img.kiwi/johnirl/albums", - "#category": ("chevereto", "imgkiwi", "user"), - "#class" : chevereto.CheveretoUserExtractor, - "#pattern" : chevereto.CheveretoAlbumExtractor.pattern, - "#count" : 50, -}, - + { + "#url": "https://img.kiwi/image/79de2c41-70f9-4a87-bd6d-00fe9997c0c4.JR2wZz", + "#category": ("chevereto", "imgkiwi", "image"), + "#class": chevereto.CheveretoImageExtractor, + "#urls": "https://img.kiwi/images/2023/02/28/11ac1ebf28a2eae8265026b28e9c4413.jpg", + "#sha1_content": "9ea704a77e2038b9008350682cfad53a614a60bd", + "album": "Kins3y Wolansk1", + "extension": "jpg", + "filename": "11ac1ebf28a2eae8265026b28e9c4413", + "id": "JR2wZz", + "url": "https://img.kiwi/images/2023/02/28/11ac1ebf28a2eae8265026b28e9c4413.jpg", + "user": "johnirl", + }, + { + "#url": "https://img.kiwi/album/kins3y-wolansk1.8Jxc", + "#category": ("chevereto", "imgkiwi", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + "#pattern": chevereto.CheveretoImageExtractor.pattern, + "#count": 19, + }, + { + "#url": "https://img.kiwi/johnirl", + "#category": ("chevereto", "imgkiwi", "user"), + "#class": chevereto.CheveretoUserExtractor, + "#pattern": chevereto.CheveretoImageExtractor.pattern, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://img.kiwi/johnirl/albums", + "#category": ("chevereto", "imgkiwi", "user"), + "#class": chevereto.CheveretoUserExtractor, + "#pattern": chevereto.CheveretoAlbumExtractor.pattern, + "#count": 50, + }, ) diff --git a/test/results/imgspice.py b/test/results/imgspice.py index c90393c7b..adb75af22 100644 --- a/test/results/imgspice.py +++ b/test/results/imgspice.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://imgspice.com/q1taxkhxprrn/58410038_cal022jsp_308191001.jpg.html", - "#category": ("imagehost", "imgspice", "image"), - "#class" : imagehosts.ImgspiceImageExtractor, - "#urls" : "https://img30.imgspice.com/i/03792/q1taxkhxprrn.jpg", - "#sha1_content" : "f1de8e58a7c2ef747a206a38f96c5623b8a83edc", - - "extension": "jpg", - "filename" : "58410038_cal022jsp_308191001", - "token" : "q1taxkhxprrn", -}, - + { + "#url": "https://imgspice.com/q1taxkhxprrn/58410038_cal022jsp_308191001.jpg.html", + "#category": ("imagehost", "imgspice", "image"), + "#class": imagehosts.ImgspiceImageExtractor, + "#urls": "https://img30.imgspice.com/i/03792/q1taxkhxprrn.jpg", + "#sha1_content": "f1de8e58a7c2ef747a206a38f96c5623b8a83edc", + "extension": "jpg", + "filename": "58410038_cal022jsp_308191001", + "token": "q1taxkhxprrn", + }, ) diff --git a/test/results/imgth.py b/test/results/imgth.py index 333e13522..c79b67162 100644 --- a/test/results/imgth.py +++ b/test/results/imgth.py @@ -1,34 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imgth - __tests__ = ( -{ - "#url" : "https://imgth.com/gallery/37/wallpaper-anime", - "#category": ("", "imgth", "gallery"), - "#class" : imgth.ImgthGalleryExtractor, - "#pattern" : r"https://imgth\.com/images/2009/11/25/wallpaper-anime_\w+\.jpg", - "#sha1_url": "4ae1d281ca2b48952cf5cca57e9914402ad72748", - - "count" : 12, - "date" : "dt:2009-11-25 18:21:00", - "extension" : "jpg", - "filename" : r"re:wallpaper-anime_\w+", - "gallery_id": 37, - "num" : int, - "title" : "Wallpaper anime", - "user" : "celebrities", -}, - -{ - "#url" : "https://www.imgth.com/gallery/37/wallpaper-anime", - "#category": ("", "imgth", "gallery"), - "#class" : imgth.ImgthGalleryExtractor, -}, - + { + "#url": "https://imgth.com/gallery/37/wallpaper-anime", + "#category": ("", "imgth", "gallery"), + "#class": imgth.ImgthGalleryExtractor, + "#pattern": r"https://imgth\.com/images/2009/11/25/wallpaper-anime_\w+\.jpg", + "#sha1_url": "4ae1d281ca2b48952cf5cca57e9914402ad72748", + "count": 12, + "date": "dt:2009-11-25 18:21:00", + "extension": "jpg", + "filename": r"re:wallpaper-anime_\w+", + "gallery_id": 37, + "num": int, + "title": "Wallpaper anime", + "user": "celebrities", + }, + { + "#url": "https://www.imgth.com/gallery/37/wallpaper-anime", + "#category": ("", "imgth", "gallery"), + "#class": imgth.ImgthGalleryExtractor, + }, ) diff --git a/test/results/imgur.py b/test/results/imgur.py index 991cf5469..8ac6c91f6 100644 --- a/test/results/imgur.py +++ b/test/results/imgur.py @@ -1,410 +1,365 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imgur -from gallery_dl import exception import datetime +from gallery_dl import exception +from gallery_dl.extractor import imgur __tests__ = ( -{ - "#url" : "https://imgur.com/21yMxCS", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, - "#sha1_url" : "6f2dcfb86815bdd72808c313e5f715610bc7b9b2", - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", - "#urls" : "https://i.imgur.com/21yMxCS.png", - - "account_id" : 0, - "comment_count" : int, - "cover_id" : "21yMxCS", - "date" : "dt:2016-11-10 14:24:35", - "description" : "", - "downvote_count": int, - "duration" : 0, - "ext" : "png", - "favorite" : False, - "favorite_count": 0, - "has_sound" : False, - "height" : 32, - "id" : "21yMxCS", - "image_count" : 1, - "in_most_viral" : False, - "is_ad" : False, - "is_album" : False, - "is_animated" : False, - "is_looping" : False, - "is_mature" : False, - "is_pending" : False, - "mime_type" : "image/png", - "name" : "test-テスト", - "point_count" : int, - "privacy" : "", - "score" : int, - "size" : 182, - "title" : "Test", - "upvote_count" : int, - "url" : "https://i.imgur.com/21yMxCS.png", - "view_count" : int, - "width" : 64, -}, - -{ - "#url" : "http://imgur.com/0gybAXR", - "#comment" : "gifv/mp4 video", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, - "#sha1_url" : "a2220eb265a55b0c95e0d3d721ec7665460e3fd7", - "#sha1_content": "a3c080e43f58f55243ab830569ba02309d59abfc", -}, - -{ - "#url" : "https://imgur.com/XFfsmuC", - "#comment" : "missing title in API response (#467)", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, - - "title": "Tears are a natural response to irritants", -}, - -{ - "#url" : "https://imgur.com/1Nily2P", - "#comment" : "animated png", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, - "#pattern" : "https://i.imgur.com/1Nily2P.png", -}, - -{ - "#url" : "https://imgur.com/zzzzzzz", - "#comment" : "not found", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://imgur.com/test-21yMxCS", - "#comment" : "slug", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://m.imgur.com/r/Celebs/iHJ7tsM", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://www.imgur.com/21yMxCS", - "#comment" : "www", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://m.imgur.com/21yMxCS", - "#comment" : "mobile", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://imgur.com/zxaY6", - "#comment" : "5 character key", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://imgur.io/zxaY6", - "#comment" : ".io", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://i.imgur.com/21yMxCS.png", - "#comment" : "direct link", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://i.imgur.io/21yMxCS.png", - "#comment" : "direct link .io", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://i.imgur.com/21yMxCSh.png", - "#comment" : "direct link thumbnail", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://i.imgur.com/zxaY6.gif", - "#comment" : "direct link (short)", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://i.imgur.com/zxaY6s.gif", - "#comment" : "direct link (short; thumb)", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://imgur.com/a/TcBmP", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, - "#sha1_url": "ce3552f550a5b5316bd9c7ae02e21e39f30c0563", - "#urls" : ( - "https://i.imgur.com/693j2Kr.jpg", - "https://i.imgur.com/ZNalkAC.jpg", - "https://i.imgur.com/lMox9Ek.jpg", - "https://i.imgur.com/6PryGOv.jpg", - "https://i.imgur.com/ecasnH2.jpg", - "https://i.imgur.com/NlJDmFG.jpg", - "https://i.imgur.com/aCwKs8S.jpg", - "https://i.imgur.com/Oz4rpxo.jpg", - "https://i.imgur.com/hE93Xsn.jpg", - "https://i.imgur.com/A5uBLSx.jpg", - "https://i.imgur.com/zZghWiD.jpg", - "https://i.imgur.com/ALV4fYV.jpg", - "https://i.imgur.com/FDd90t9.jpg", - "https://i.imgur.com/Txw37NO.jpg", - "https://i.imgur.com/DcLw7Cl.jpg", - "https://i.imgur.com/a4VChy8.jpg", - "https://i.imgur.com/auCwCig.jpg", - "https://i.imgur.com/Z8VihIb.jpg", - "https://i.imgur.com/6WDRFne.jpg", - ), - - "album" : { - "account_id" : 0, - "comment_count" : int, - "cover_id" : "693j2Kr", - "date" : "dt:2015-10-09 10:37:50", - "description" : "", - "downvote_count": 0, - "favorite" : False, + { + "#url": "https://imgur.com/21yMxCS", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + "#sha1_url": "6f2dcfb86815bdd72808c313e5f715610bc7b9b2", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + "#urls": "https://i.imgur.com/21yMxCS.png", + "account_id": 0, + "comment_count": int, + "cover_id": "21yMxCS", + "date": "dt:2016-11-10 14:24:35", + "description": "", + "downvote_count": int, + "duration": 0, + "ext": "png", + "favorite": False, "favorite_count": 0, - "id" : "TcBmP", - "image_count" : 19, - "in_most_viral" : False, - "is_ad" : False, - "is_album" : True, - "is_mature" : False, - "is_pending" : False, - "privacy" : "private", - "score" : int, - "title" : "138", - "upvote_count" : int, - "url" : "https://imgur.com/a/TcBmP", - "view_count" : int, - "virality" : int, - }, - "account_id" : 0, - "count" : 19, - "date" : datetime.datetime, - "description": "", - "ext" : "jpg", - "has_sound" : False, - "height" : int, - "id" : str, - "is_animated": False, - "is_looping" : False, - "mime_type" : "image/jpeg", - "name" : str, - "num" : int, - "size" : int, - "title" : str, - "type" : "image", - "updated_at" : None, - "url" : str, - "width" : int, -}, - -{ - "#url" : "https://imgur.com/a/eD9CT", - "#comment" : "large album", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://imgur.com/a/RhJXhVT/all", - "#comment" : "7 character album hash", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, - "#sha1_url": "695ef0c950023362a0163ee5041796300db76674", -}, - -{ - "#url" : "https://imgur.com/a/TcBmQ", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://imgur.com/a/pjOnJA0", - "#comment" : "empty, no 'media' (#2557)", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://imgur.com/a/138-TcBmP", - "#comment" : "slug", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, -}, - -{ - "#url" : "https://www.imgur.com/a/TcBmP", - "#comment" : "www", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, -}, - -{ - "#url" : "https://imgur.io/a/TcBmP", - "#comment" : ".io", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, -}, - -{ - "#url" : "https://m.imgur.com/a/TcBmP", - "#comment" : "mobile", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, -}, - -{ - "#url" : "https://imgur.com/gallery/zf2fIms", - "#comment" : "non-album gallery (#380)", - "#category": ("", "imgur", "gallery"), - "#class" : imgur.ImgurGalleryExtractor, - "#pattern" : "https://imgur.com/zf2fIms", -}, - -{ - "#url" : "https://imgur.com/gallery/eD9CT", - "#category": ("", "imgur", "gallery"), - "#class" : imgur.ImgurGalleryExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://imgur.com/gallery/guy-gets-out-of-car-during-long-traffic-jam-to-pet-dog-zf2fIms", - "#comment" : "slug", - "#category": ("", "imgur", "gallery"), - "#class" : imgur.ImgurGalleryExtractor, -}, - -{ - "#url" : "https://imgur.com/t/unmuted/26sEhNr", - "#category": ("", "imgur", "gallery"), - "#class" : imgur.ImgurGalleryExtractor, -}, - -{ - "#url" : "https://imgur.com/t/cat/qSB8NbN", - "#category": ("", "imgur", "gallery"), - "#class" : imgur.ImgurGalleryExtractor, -}, - -{ - "#url" : "https://imgur.io/t/cat/qSB8NbN", - "#comment" : ".io", - "#category": ("", "imgur", "gallery"), - "#class" : imgur.ImgurGalleryExtractor, -}, - -{ - "#url" : "https://imgur.com/user/Miguenzo", - "#category": ("", "imgur", "user"), - "#class" : imgur.ImgurUserExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://imgur.com/user/Miguenzo/posts", - "#category": ("", "imgur", "user"), - "#class" : imgur.ImgurUserExtractor, -}, - -{ - "#url" : "https://imgur.com/user/Miguenzo/submitted", - "#category": ("", "imgur", "user"), - "#class" : imgur.ImgurUserExtractor, -}, - -{ - "#url" : "https://imgur.com/user/Miguenzo/favorites", - "#category": ("", "imgur", "favorite"), - "#class" : imgur.ImgurFavoriteExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://imgur.com/user/mikf1/favorites/folder/11896757/public", - "#category": ("", "imgur", "favorite-folder"), - "#class" : imgur.ImgurFavoriteFolderExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#count" : 3, -}, - -{ - "#url" : "https://imgur.com/user/mikf1/favorites/folder/11896741/private", - "#category": ("", "imgur", "favorite-folder"), - "#class" : imgur.ImgurFavoriteFolderExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#count" : 5, -}, - -{ - "#url" : "https://imgur.com/r/pics", - "#category": ("", "imgur", "subreddit"), - "#class" : imgur.ImgurSubredditExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://imgur.com/t/animals", - "#category": ("", "imgur", "tag"), - "#class" : imgur.ImgurTagExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://imgur.com/search?q=cute+cat", - "#category": ("", "imgur", "search"), - "#class" : imgur.ImgurSearchExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#range" : "1-100", - "#count" : 100, -}, - + "has_sound": False, + "height": 32, + "id": "21yMxCS", + "image_count": 1, + "in_most_viral": False, + "is_ad": False, + "is_album": False, + "is_animated": False, + "is_looping": False, + "is_mature": False, + "is_pending": False, + "mime_type": "image/png", + "name": "test-テスト", + "point_count": int, + "privacy": "", + "score": int, + "size": 182, + "title": "Test", + "upvote_count": int, + "url": "https://i.imgur.com/21yMxCS.png", + "view_count": int, + "width": 64, + }, + { + "#url": "http://imgur.com/0gybAXR", + "#comment": "gifv/mp4 video", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + "#sha1_url": "a2220eb265a55b0c95e0d3d721ec7665460e3fd7", + "#sha1_content": "a3c080e43f58f55243ab830569ba02309d59abfc", + }, + { + "#url": "https://imgur.com/XFfsmuC", + "#comment": "missing title in API response (#467)", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + "title": "Tears are a natural response to irritants", + }, + { + "#url": "https://imgur.com/1Nily2P", + "#comment": "animated png", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + "#pattern": "https://i.imgur.com/1Nily2P.png", + }, + { + "#url": "https://imgur.com/zzzzzzz", + "#comment": "not found", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://imgur.com/test-21yMxCS", + "#comment": "slug", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://m.imgur.com/r/Celebs/iHJ7tsM", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://www.imgur.com/21yMxCS", + "#comment": "www", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://m.imgur.com/21yMxCS", + "#comment": "mobile", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://imgur.com/zxaY6", + "#comment": "5 character key", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://imgur.io/zxaY6", + "#comment": ".io", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://i.imgur.com/21yMxCS.png", + "#comment": "direct link", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://i.imgur.io/21yMxCS.png", + "#comment": "direct link .io", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://i.imgur.com/21yMxCSh.png", + "#comment": "direct link thumbnail", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://i.imgur.com/zxaY6.gif", + "#comment": "direct link (short)", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://i.imgur.com/zxaY6s.gif", + "#comment": "direct link (short; thumb)", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://imgur.com/a/TcBmP", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + "#sha1_url": "ce3552f550a5b5316bd9c7ae02e21e39f30c0563", + "#urls": ( + "https://i.imgur.com/693j2Kr.jpg", + "https://i.imgur.com/ZNalkAC.jpg", + "https://i.imgur.com/lMox9Ek.jpg", + "https://i.imgur.com/6PryGOv.jpg", + "https://i.imgur.com/ecasnH2.jpg", + "https://i.imgur.com/NlJDmFG.jpg", + "https://i.imgur.com/aCwKs8S.jpg", + "https://i.imgur.com/Oz4rpxo.jpg", + "https://i.imgur.com/hE93Xsn.jpg", + "https://i.imgur.com/A5uBLSx.jpg", + "https://i.imgur.com/zZghWiD.jpg", + "https://i.imgur.com/ALV4fYV.jpg", + "https://i.imgur.com/FDd90t9.jpg", + "https://i.imgur.com/Txw37NO.jpg", + "https://i.imgur.com/DcLw7Cl.jpg", + "https://i.imgur.com/a4VChy8.jpg", + "https://i.imgur.com/auCwCig.jpg", + "https://i.imgur.com/Z8VihIb.jpg", + "https://i.imgur.com/6WDRFne.jpg", + ), + "album": { + "account_id": 0, + "comment_count": int, + "cover_id": "693j2Kr", + "date": "dt:2015-10-09 10:37:50", + "description": "", + "downvote_count": 0, + "favorite": False, + "favorite_count": 0, + "id": "TcBmP", + "image_count": 19, + "in_most_viral": False, + "is_ad": False, + "is_album": True, + "is_mature": False, + "is_pending": False, + "privacy": "private", + "score": int, + "title": "138", + "upvote_count": int, + "url": "https://imgur.com/a/TcBmP", + "view_count": int, + "virality": int, + }, + "account_id": 0, + "count": 19, + "date": datetime.datetime, + "description": "", + "ext": "jpg", + "has_sound": False, + "height": int, + "id": str, + "is_animated": False, + "is_looping": False, + "mime_type": "image/jpeg", + "name": str, + "num": int, + "size": int, + "title": str, + "type": "image", + "updated_at": None, + "url": str, + "width": int, + }, + { + "#url": "https://imgur.com/a/eD9CT", + "#comment": "large album", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://imgur.com/a/RhJXhVT/all", + "#comment": "7 character album hash", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + "#sha1_url": "695ef0c950023362a0163ee5041796300db76674", + }, + { + "#url": "https://imgur.com/a/TcBmQ", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://imgur.com/a/pjOnJA0", + "#comment": "empty, no 'media' (#2557)", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://imgur.com/a/138-TcBmP", + "#comment": "slug", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + }, + { + "#url": "https://www.imgur.com/a/TcBmP", + "#comment": "www", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + }, + { + "#url": "https://imgur.io/a/TcBmP", + "#comment": ".io", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + }, + { + "#url": "https://m.imgur.com/a/TcBmP", + "#comment": "mobile", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + }, + { + "#url": "https://imgur.com/gallery/zf2fIms", + "#comment": "non-album gallery (#380)", + "#category": ("", "imgur", "gallery"), + "#class": imgur.ImgurGalleryExtractor, + "#pattern": "https://imgur.com/zf2fIms", + }, + { + "#url": "https://imgur.com/gallery/eD9CT", + "#category": ("", "imgur", "gallery"), + "#class": imgur.ImgurGalleryExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://imgur.com/gallery/guy-gets-out-of-car-during-long-traffic-jam-to-pet-dog-zf2fIms", + "#comment": "slug", + "#category": ("", "imgur", "gallery"), + "#class": imgur.ImgurGalleryExtractor, + }, + { + "#url": "https://imgur.com/t/unmuted/26sEhNr", + "#category": ("", "imgur", "gallery"), + "#class": imgur.ImgurGalleryExtractor, + }, + { + "#url": "https://imgur.com/t/cat/qSB8NbN", + "#category": ("", "imgur", "gallery"), + "#class": imgur.ImgurGalleryExtractor, + }, + { + "#url": "https://imgur.io/t/cat/qSB8NbN", + "#comment": ".io", + "#category": ("", "imgur", "gallery"), + "#class": imgur.ImgurGalleryExtractor, + }, + { + "#url": "https://imgur.com/user/Miguenzo", + "#category": ("", "imgur", "user"), + "#class": imgur.ImgurUserExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://imgur.com/user/Miguenzo/posts", + "#category": ("", "imgur", "user"), + "#class": imgur.ImgurUserExtractor, + }, + { + "#url": "https://imgur.com/user/Miguenzo/submitted", + "#category": ("", "imgur", "user"), + "#class": imgur.ImgurUserExtractor, + }, + { + "#url": "https://imgur.com/user/Miguenzo/favorites", + "#category": ("", "imgur", "favorite"), + "#class": imgur.ImgurFavoriteExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://imgur.com/user/mikf1/favorites/folder/11896757/public", + "#category": ("", "imgur", "favorite-folder"), + "#class": imgur.ImgurFavoriteFolderExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#count": 3, + }, + { + "#url": "https://imgur.com/user/mikf1/favorites/folder/11896741/private", + "#category": ("", "imgur", "favorite-folder"), + "#class": imgur.ImgurFavoriteFolderExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#count": 5, + }, + { + "#url": "https://imgur.com/r/pics", + "#category": ("", "imgur", "subreddit"), + "#class": imgur.ImgurSubredditExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://imgur.com/t/animals", + "#category": ("", "imgur", "tag"), + "#class": imgur.ImgurTagExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://imgur.com/search?q=cute+cat", + "#category": ("", "imgur", "search"), + "#class": imgur.ImgurSearchExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#range": "1-100", + "#count": 100, + }, ) diff --git a/test/results/imxto.py b/test/results/imxto.py index 430b388b8..663757ec4 100644 --- a/test/results/imxto.py +++ b/test/results/imxto.py @@ -1,65 +1,54 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imagehosts from gallery_dl import exception - +from gallery_dl.extractor import imagehosts __tests__ = ( -{ - "#url" : "https://imx.to/i/1qdeva", - "#comment" : "new-style URL", - "#category": ("imagehost", "imxto", "image"), - "#class" : imagehosts.ImxtoImageExtractor, - "#sha1_url" : "ab2173088a6cdef631d7a47dec4a5da1c6a00130", - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", - - "size" : 18, - "width" : 64, - "height": 32, - "hash" : "94d56c599223c59f3feb71ea603484d1", -}, - -{ - "#url" : "https://imx.to/img-57a2050547b97.html", - "#comment" : "old-style URL", - "#category": ("imagehost", "imxto", "image"), - "#class" : imagehosts.ImxtoImageExtractor, - "#sha1_url" : "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", - "#sha1_content": "54592f2635674c25677c6872db3709d343cdf92f", - - "size" : 5284, - "width" : 320, - "height": 160, - "hash" : "40da6aaa7b8c42b18ef74309bbc713fc", -}, - -{ - "#url" : "https://img.yt/img-57a2050547b97.html", - "#comment" : "img.yt domain", - "#category": ("imagehost", "imxto", "image"), - "#class" : imagehosts.ImxtoImageExtractor, - "#sha1_url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", -}, - -{ - "#url" : "https://imx.to/img-57a2050547b98.html", - "#category": ("imagehost", "imxto", "image"), - "#class" : imagehosts.ImxtoImageExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://imx.to/g/ozdy", - "#category": ("imagehost", "imxto", "gallery"), - "#class" : imagehosts.ImxtoGalleryExtractor, - "#pattern" : imagehosts.ImxtoImageExtractor.pattern, - "#count" : 40, - - "title": "untitled gallery", -}, - + { + "#url": "https://imx.to/i/1qdeva", + "#comment": "new-style URL", + "#category": ("imagehost", "imxto", "image"), + "#class": imagehosts.ImxtoImageExtractor, + "#sha1_url": "ab2173088a6cdef631d7a47dec4a5da1c6a00130", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + "size": 18, + "width": 64, + "height": 32, + "hash": "94d56c599223c59f3feb71ea603484d1", + }, + { + "#url": "https://imx.to/img-57a2050547b97.html", + "#comment": "old-style URL", + "#category": ("imagehost", "imxto", "image"), + "#class": imagehosts.ImxtoImageExtractor, + "#sha1_url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", + "#sha1_content": "54592f2635674c25677c6872db3709d343cdf92f", + "size": 5284, + "width": 320, + "height": 160, + "hash": "40da6aaa7b8c42b18ef74309bbc713fc", + }, + { + "#url": "https://img.yt/img-57a2050547b97.html", + "#comment": "img.yt domain", + "#category": ("imagehost", "imxto", "image"), + "#class": imagehosts.ImxtoImageExtractor, + "#sha1_url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", + }, + { + "#url": "https://imx.to/img-57a2050547b98.html", + "#category": ("imagehost", "imxto", "image"), + "#class": imagehosts.ImxtoImageExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://imx.to/g/ozdy", + "#category": ("imagehost", "imxto", "gallery"), + "#class": imagehosts.ImxtoGalleryExtractor, + "#pattern": imagehosts.ImxtoImageExtractor.pattern, + "#count": 40, + "title": "untitled gallery", + }, ) diff --git a/test/results/inkbunny.py b/test/results/inkbunny.py index 4e05da7ca..06c5d6b5d 100644 --- a/test/results/inkbunny.py +++ b/test/results/inkbunny.py @@ -1,183 +1,157 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import inkbunny import datetime +from gallery_dl.extractor import inkbunny __tests__ = ( -{ - "#url" : "https://inkbunny.net/soina", - "#category": ("", "inkbunny", "user"), - "#class" : inkbunny.InkbunnyUserExtractor, - "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_soina_.+", - "#range" : "20-50", - - "date" : datetime.datetime, - "deleted" : bool, - "file_id" : r"re:[0-9]+", - "filename" : r"re:[0-9]+_soina_\w+", - "full_file_md5" : r"re:[0-9a-f]{32}", - "mimetype" : str, - "submission_id" : r"re:[0-9]+", - "user_id" : "20969", - "comments_count" : r"re:[0-9]+", - "favorite" : bool, - "favorites_count": r"re:[0-9]+", - "friends_only" : bool, - "guest_block" : bool, - "hidden" : bool, - "pagecount" : r"re:[0-9]+", - "pools" : list, - "pools_count" : int, - "public" : bool, - "rating_id" : r"re:[0-9]+", - "rating_name" : str, - "ratings" : list, - "scraps" : bool, - "tags" : list, - "title" : str, - "type_name" : str, - "username" : "soina", - "views" : str, -}, - -{ - "#url" : "https://inkbunny.net/gallery/soina", - "#category": ("", "inkbunny", "gallery"), - "#class" : inkbunny.InkbunnyUserExtractor, - "#range" : "1-25", - - "scraps": False, -}, - -{ - "#url" : "https://inkbunny.net/scraps/soina", - "#category": ("", "inkbunny", "scraps"), - "#class" : inkbunny.InkbunnyUserExtractor, - "#range" : "1-25", - - "scraps": True, -}, - -{ - "#url" : "https://inkbunny.net/poolview_process.php?pool_id=28985", - "#category": ("", "inkbunny", "pool"), - "#class" : inkbunny.InkbunnyPoolExtractor, - "#count" : 9, - - "pool_id": "28985", -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=pool&pool_id=28985&page=1&orderby=pool_order&random=no", - "#category": ("", "inkbunny", "pool"), - "#class" : inkbunny.InkbunnyPoolExtractor, -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?mode=pool&pool_id=28985", - "#category": ("", "inkbunny", "pool"), - "#class" : inkbunny.InkbunnyPoolExtractor, -}, - -{ - "#url" : "https://inkbunny.net/userfavorites_process.php?favs_user_id=20969", - "#category": ("", "inkbunny", "favorite"), - "#class" : inkbunny.InkbunnyFavoriteExtractor, - "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_\w+_.+", - "#range" : "20-50", - - "favs_user_id" : "20969", - "favs_username": "soina", -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=userfavs&random=no&orderby=fav_datetime&page=1&user_id=20969", - "#category": ("", "inkbunny", "favorite"), - "#class" : inkbunny.InkbunnyFavoriteExtractor, -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?mode=userfavs&user_id=20969", - "#category": ("", "inkbunny", "favorite"), - "#class" : inkbunny.InkbunnyFavoriteExtractor, -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=unreadsubs&page=1&orderby=unread_datetime", - "#category": ("", "inkbunny", "unread"), - "#class" : inkbunny.InkbunnyUnreadExtractor, -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?mode=unreadsubs", - "#category": ("", "inkbunny", "unread"), - "#class" : inkbunny.InkbunnyUnreadExtractor, -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=search&page=1&orderby=create_datetime&text=cute&stringtype=and&keywords=yes&title=yes&description=no&artist=&favsby=&type=&days=&keyword_id=&user_id=&random=&md5=", - "#category": ("", "inkbunny", "search"), - "#class" : inkbunny.InkbunnySearchExtractor, - "#range" : "1-10", - "#count" : 10, - - "search": { - "rid" : "ffffffffff", - "mode" : "search", - "page" : "1", - "orderby" : "create_datetime", - "text" : "cute", - "stringtype" : "and", - "keywords" : "yes", - "title" : "yes", - "description": "no", - }, -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?mode=search", - "#category": ("", "inkbunny", "search"), - "#class" : inkbunny.InkbunnySearchExtractor, -}, - -{ - "#url" : "https://inkbunny.net/watchlist_process.php?mode=watching&user_id=20969", - "#category": ("", "inkbunny", "following"), - "#class" : inkbunny.InkbunnyFollowingExtractor, - "#pattern" : inkbunny.InkbunnyUserExtractor.pattern, - "#count" : ">= 90", -}, - -{ - "#url" : "https://inkbunny.net/usersviewall.php?rid=ffffffffff&mode=watching&page=1&user_id=20969&orderby=added&namesonly=", - "#category": ("", "inkbunny", "following"), - "#class" : inkbunny.InkbunnyFollowingExtractor, -}, - -{ - "#url" : "https://inkbunny.net/usersviewall.php?mode=watching&user_id=20969", - "#category": ("", "inkbunny", "following"), - "#class" : inkbunny.InkbunnyFollowingExtractor, -}, - -{ - "#url" : "https://inkbunny.net/s/1829715", - "#category": ("", "inkbunny", "post"), - "#class" : inkbunny.InkbunnyPostExtractor, - "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/2626/2626843_soina_dscn2296\.jpg", - "#sha1_content": "cf69d8dddf0822a12b4eef1f4b2258bd600b36c8", -}, - -{ - "#url" : "https://inkbunny.net/s/2044094", - "#category": ("", "inkbunny", "post"), - "#class" : inkbunny.InkbunnyPostExtractor, - "#count" : 4, -}, - + { + "#url": "https://inkbunny.net/soina", + "#category": ("", "inkbunny", "user"), + "#class": inkbunny.InkbunnyUserExtractor, + "#pattern": r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_soina_.+", + "#range": "20-50", + "date": datetime.datetime, + "deleted": bool, + "file_id": r"re:[0-9]+", + "filename": r"re:[0-9]+_soina_\w+", + "full_file_md5": r"re:[0-9a-f]{32}", + "mimetype": str, + "submission_id": r"re:[0-9]+", + "user_id": "20969", + "comments_count": r"re:[0-9]+", + "favorite": bool, + "favorites_count": r"re:[0-9]+", + "friends_only": bool, + "guest_block": bool, + "hidden": bool, + "pagecount": r"re:[0-9]+", + "pools": list, + "pools_count": int, + "public": bool, + "rating_id": r"re:[0-9]+", + "rating_name": str, + "ratings": list, + "scraps": bool, + "tags": list, + "title": str, + "type_name": str, + "username": "soina", + "views": str, + }, + { + "#url": "https://inkbunny.net/gallery/soina", + "#category": ("", "inkbunny", "gallery"), + "#class": inkbunny.InkbunnyUserExtractor, + "#range": "1-25", + "scraps": False, + }, + { + "#url": "https://inkbunny.net/scraps/soina", + "#category": ("", "inkbunny", "scraps"), + "#class": inkbunny.InkbunnyUserExtractor, + "#range": "1-25", + "scraps": True, + }, + { + "#url": "https://inkbunny.net/poolview_process.php?pool_id=28985", + "#category": ("", "inkbunny", "pool"), + "#class": inkbunny.InkbunnyPoolExtractor, + "#count": 9, + "pool_id": "28985", + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=pool&pool_id=28985&page=1&orderby=pool_order&random=no", + "#category": ("", "inkbunny", "pool"), + "#class": inkbunny.InkbunnyPoolExtractor, + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?mode=pool&pool_id=28985", + "#category": ("", "inkbunny", "pool"), + "#class": inkbunny.InkbunnyPoolExtractor, + }, + { + "#url": "https://inkbunny.net/userfavorites_process.php?favs_user_id=20969", + "#category": ("", "inkbunny", "favorite"), + "#class": inkbunny.InkbunnyFavoriteExtractor, + "#pattern": r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_\w+_.+", + "#range": "20-50", + "favs_user_id": "20969", + "favs_username": "soina", + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=userfavs&random=no&orderby=fav_datetime&page=1&user_id=20969", + "#category": ("", "inkbunny", "favorite"), + "#class": inkbunny.InkbunnyFavoriteExtractor, + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?mode=userfavs&user_id=20969", + "#category": ("", "inkbunny", "favorite"), + "#class": inkbunny.InkbunnyFavoriteExtractor, + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=unreadsubs&page=1&orderby=unread_datetime", + "#category": ("", "inkbunny", "unread"), + "#class": inkbunny.InkbunnyUnreadExtractor, + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?mode=unreadsubs", + "#category": ("", "inkbunny", "unread"), + "#class": inkbunny.InkbunnyUnreadExtractor, + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=search&page=1&orderby=create_datetime&text=cute&stringtype=and&keywords=yes&title=yes&description=no&artist=&favsby=&type=&days=&keyword_id=&user_id=&random=&md5=", + "#category": ("", "inkbunny", "search"), + "#class": inkbunny.InkbunnySearchExtractor, + "#range": "1-10", + "#count": 10, + "search": { + "rid": "ffffffffff", + "mode": "search", + "page": "1", + "orderby": "create_datetime", + "text": "cute", + "stringtype": "and", + "keywords": "yes", + "title": "yes", + "description": "no", + }, + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?mode=search", + "#category": ("", "inkbunny", "search"), + "#class": inkbunny.InkbunnySearchExtractor, + }, + { + "#url": "https://inkbunny.net/watchlist_process.php?mode=watching&user_id=20969", + "#category": ("", "inkbunny", "following"), + "#class": inkbunny.InkbunnyFollowingExtractor, + "#pattern": inkbunny.InkbunnyUserExtractor.pattern, + "#count": ">= 90", + }, + { + "#url": "https://inkbunny.net/usersviewall.php?rid=ffffffffff&mode=watching&page=1&user_id=20969&orderby=added&namesonly=", + "#category": ("", "inkbunny", "following"), + "#class": inkbunny.InkbunnyFollowingExtractor, + }, + { + "#url": "https://inkbunny.net/usersviewall.php?mode=watching&user_id=20969", + "#category": ("", "inkbunny", "following"), + "#class": inkbunny.InkbunnyFollowingExtractor, + }, + { + "#url": "https://inkbunny.net/s/1829715", + "#category": ("", "inkbunny", "post"), + "#class": inkbunny.InkbunnyPostExtractor, + "#pattern": r"https://[\w.]+\.metapix\.net/files/full/2626/2626843_soina_dscn2296\.jpg", + "#sha1_content": "cf69d8dddf0822a12b4eef1f4b2258bd600b36c8", + }, + { + "#url": "https://inkbunny.net/s/2044094", + "#category": ("", "inkbunny", "post"), + "#class": inkbunny.InkbunnyPostExtractor, + "#count": 4, + }, ) diff --git a/test/results/instagram.py b/test/results/instagram.py index 3cdb7cd86..1d057ea2e 100644 --- a/test/results/instagram.py +++ b/test/results/instagram.py @@ -1,280 +1,244 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import instagram - __tests__ = ( -{ - "#url" : "https://www.instagram.com/instagram/", - "#category": ("", "instagram", "user"), - "#class" : instagram.InstagramUserExtractor, - "#auth" : False, - "#options" : {"include": "all"}, - "#urls": [ - "https://www.instagram.com/instagram/info/", - "https://www.instagram.com/instagram/avatar/", - "https://www.instagram.com/stories/instagram/", - "https://www.instagram.com/instagram/highlights/", - "https://www.instagram.com/instagram/posts/", - "https://www.instagram.com/instagram/reels/", - "https://www.instagram.com/instagram/tagged/", - ], -}, - -{ - "#url" : "https://www.instagram.com/instagram/?hl=en", - "#category": ("", "instagram", "user"), - "#class" : instagram.InstagramUserExtractor, -}, - -{ - "#url" : "https://www.instagram.com/id:25025320/", - "#category": ("", "instagram", "user"), - "#class" : instagram.InstagramUserExtractor, -}, - -{ - "#url" : "https://www.instagram.com/instagram/posts/", - "#category": ("", "instagram", "posts"), - "#class" : instagram.InstagramPostsExtractor, - "#range" : "1-16", - "#count" : ">= 16", -}, - -{ - "#url" : "https://www.instagram.com/instagram/reels/", - "#category": ("", "instagram", "reels"), - "#class" : instagram.InstagramReelsExtractor, - "#range" : "40-60", - "#count" : ">= 20", -}, - -{ - "#url" : "https://www.instagram.com/instagram/tagged/", - "#category": ("", "instagram", "tagged"), - "#class" : instagram.InstagramTaggedExtractor, - "#range" : "1-16", - "#count" : ">= 16", - - "tagged_owner_id" : "25025320", - "tagged_username" : "instagram", - "tagged_full_name": "Instagram", -}, - -{ - "#url" : "https://www.instagram.com/kadakaofficial/guide/knit-i-need-collection/18131821684305217/", - "#category": ("", "instagram", "guide"), - "#class" : instagram.InstagramGuideExtractor, - "#range" : "1-16", - "#count" : ">= 16", -}, - -{ - "#url" : "https://www.instagram.com/instagram/saved/", - "#category": ("", "instagram", "saved"), - "#class" : instagram.InstagramSavedExtractor, -}, - -{ - "#url" : "https://www.instagram.com/instagram/saved/all-posts/", - "#category": ("", "instagram", "saved"), - "#class" : instagram.InstagramSavedExtractor, -}, - -{ - "#url" : "https://www.instagram.com/instagram/saved/collection_name/123456789/", - "#category": ("", "instagram", "collection"), - "#class" : instagram.InstagramCollectionExtractor, -}, - -{ - "#url" : "https://www.instagram.com/stories/instagram/", - "#category": ("", "instagram", "stories"), - "#class" : instagram.InstagramStoriesExtractor, -}, - -{ - "#url" : "https://www.instagram.com/stories/highlights/18042509488170095/", - "#category": ("", "instagram", "highlights"), - "#class" : instagram.InstagramStoriesExtractor, -}, - -{ - "#url" : "https://instagram.com/stories/geekmig/2724343156064789461", - "#category": ("", "instagram", "stories"), - "#class" : instagram.InstagramStoriesExtractor, -}, - -{ - "#url" : "https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1", - "#category": ("", "instagram", "highlights"), - "#class" : instagram.InstagramStoriesExtractor, -}, - -{ - "#url" : "https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1?story_media_id=2724343156064789461", - "#category": ("", "instagram", "highlights"), - "#class" : instagram.InstagramStoriesExtractor, -}, - -{ - "#url" : "https://www.instagram.com/instagram/highlights", - "#category": ("", "instagram", "highlights"), - "#class" : instagram.InstagramHighlightsExtractor, -}, - -{ - "#url" : "https://www.instagram.com/instagram/following", - "#category": ("", "instagram", "following"), - "#class" : instagram.InstagramFollowingExtractor, - "#range" : "1-16", - "#count" : ">= 16", -}, - -{ - "#url" : "https://www.instagram.com/explore/tags/instagram/", - "#category": ("", "instagram", "tag"), - "#class" : instagram.InstagramTagExtractor, - "#range" : "1-16", - "#count" : ">= 16", -}, - -{ - "#url" : "https://www.instagram.com/instagram/info", - "#category": ("", "instagram", "info"), - "#class" : instagram.InstagramInfoExtractor, - "#auth" : False, -}, - -{ - "#url" : "https://www.instagram.com/instagram/avatar", - "#category": ("", "instagram", "avatar"), - "#class" : instagram.InstagramAvatarExtractor, - "#pattern" : r"https://instagram\.[\w.-]+\.fbcdn\.net/v/t51\.2885-19/281440578_1088265838702675_6233856337905829714_n\.jpg", -}, - -{ - "#url" : "https://www.instagram.com/p/BqvsDleB3lV/", - "#comment" : "GraphImage", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, - "#pattern" : r"https://[^/]+\.(cdninstagram\.com|fbcdn\.net)/v(p/[0-9a-f]+/[0-9A-F]+)?/t51.2885-15/e35/44877605_725955034447492_3123079845831750529_n.jpg", - - "date" : "dt:2018-11-29 01:04:04", - "description" : str, - "height" : int, - "likes" : int, - "location_id" : "214424288", - "location_slug" : "hong-kong", - "location_url" : r"re:/explore/locations/214424288/hong-kong/", - "media_id" : "1922949326347663701", - "shortcode" : "BqvsDleB3lV", - "post_id" : "1922949326347663701", - "post_shortcode": "BqvsDleB3lV", - "post_url" : "https://www.instagram.com/p/BqvsDleB3lV/", - "tags" : ["#WHPsquares"], - "typename" : "GraphImage", - "username" : "instagram", - "width" : int, -}, - -{ - "#url" : "https://www.instagram.com/p/BoHk1haB5tM/", - "#comment" : "GraphSidecar", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, - "#count" : 5, - - "sidecar_media_id": "1875629777499953996", - "sidecar_shortcode": "BoHk1haB5tM", - "post_id" : "1875629777499953996", - "post_shortcode" : "BoHk1haB5tM", - "post_url" : "https://www.instagram.com/p/BoHk1haB5tM/", - "num" : int, - "likes" : int, - "username" : "instagram", -}, - -{ - "#url" : "https://www.instagram.com/p/Bqxp0VSBgJg/", - "#comment" : "GraphVideo", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, - "#pattern" : r"/46840863_726311431074534_7805566102611403091_n\.mp4", - - "date" : "dt:2018-11-29 19:23:58", - "description": str, - "height" : int, - "likes" : int, - "media_id" : "1923502432034620000", - "post_url" : "https://www.instagram.com/p/Bqxp0VSBgJg/", - "shortcode" : "Bqxp0VSBgJg", - "tags" : ["#ASMR"], - "typename" : "GraphVideo", - "username" : "instagram", - "width" : int, -}, - -{ - "#url" : "https://www.instagram.com/tv/BkQjCfsBIzi/", - "#comment" : "GraphVideo (IGTV)", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, - "#pattern" : r"/10000000_597132547321814_702169244961988209_n\.mp4", - - "date" : "dt:2018-06-20 19:51:32", - "description": str, - "height" : int, - "likes" : int, - "media_id" : "1806097553666903266", - "post_url" : "https://www.instagram.com/p/BkQjCfsBIzi/", - "shortcode" : "BkQjCfsBIzi", - "typename" : "GraphVideo", - "username" : "instagram", - "width" : int, -}, - -{ - "#url" : "https://www.instagram.com/p/BtOvDOfhvRr/", - "#comment" : "GraphSidecar with 2 embedded GraphVideo objects", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, - "#count" : 2, - - "post_url" : "https://www.instagram.com/p/BtOvDOfhvRr/", - "sidecar_media_id": "1967717017113261163", - "sidecar_shortcode": "BtOvDOfhvRr", - "video_url" : str, -}, - -{ - "#url" : "https://www.instagram.com/p/B_2lf3qAd3y/", - "#comment" : "GraphImage with tagged user", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, - - "tagged_users": [{ - "id" : "1246468638", - "username" : "kaaymbl", - "full_name": "Call Me Kay", -}], -}, - -{ - "#url" : "https://www.instagram.com/dm/p/CW042g7B9CY/", - "#comment" : "URL with username (#2085)", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, -}, - -{ - "#url" : "https://www.instagram.com/reel/CDg_6Y1pxWu/", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, -}, - + { + "#url": "https://www.instagram.com/instagram/", + "#category": ("", "instagram", "user"), + "#class": instagram.InstagramUserExtractor, + "#auth": False, + "#options": {"include": "all"}, + "#urls": [ + "https://www.instagram.com/instagram/info/", + "https://www.instagram.com/instagram/avatar/", + "https://www.instagram.com/stories/instagram/", + "https://www.instagram.com/instagram/highlights/", + "https://www.instagram.com/instagram/posts/", + "https://www.instagram.com/instagram/reels/", + "https://www.instagram.com/instagram/tagged/", + ], + }, + { + "#url": "https://www.instagram.com/instagram/?hl=en", + "#category": ("", "instagram", "user"), + "#class": instagram.InstagramUserExtractor, + }, + { + "#url": "https://www.instagram.com/id:25025320/", + "#category": ("", "instagram", "user"), + "#class": instagram.InstagramUserExtractor, + }, + { + "#url": "https://www.instagram.com/instagram/posts/", + "#category": ("", "instagram", "posts"), + "#class": instagram.InstagramPostsExtractor, + "#range": "1-16", + "#count": ">= 16", + }, + { + "#url": "https://www.instagram.com/instagram/reels/", + "#category": ("", "instagram", "reels"), + "#class": instagram.InstagramReelsExtractor, + "#range": "40-60", + "#count": ">= 20", + }, + { + "#url": "https://www.instagram.com/instagram/tagged/", + "#category": ("", "instagram", "tagged"), + "#class": instagram.InstagramTaggedExtractor, + "#range": "1-16", + "#count": ">= 16", + "tagged_owner_id": "25025320", + "tagged_username": "instagram", + "tagged_full_name": "Instagram", + }, + { + "#url": "https://www.instagram.com/kadakaofficial/guide/knit-i-need-collection/18131821684305217/", + "#category": ("", "instagram", "guide"), + "#class": instagram.InstagramGuideExtractor, + "#range": "1-16", + "#count": ">= 16", + }, + { + "#url": "https://www.instagram.com/instagram/saved/", + "#category": ("", "instagram", "saved"), + "#class": instagram.InstagramSavedExtractor, + }, + { + "#url": "https://www.instagram.com/instagram/saved/all-posts/", + "#category": ("", "instagram", "saved"), + "#class": instagram.InstagramSavedExtractor, + }, + { + "#url": "https://www.instagram.com/instagram/saved/collection_name/123456789/", + "#category": ("", "instagram", "collection"), + "#class": instagram.InstagramCollectionExtractor, + }, + { + "#url": "https://www.instagram.com/stories/instagram/", + "#category": ("", "instagram", "stories"), + "#class": instagram.InstagramStoriesExtractor, + }, + { + "#url": "https://www.instagram.com/stories/highlights/18042509488170095/", + "#category": ("", "instagram", "highlights"), + "#class": instagram.InstagramStoriesExtractor, + }, + { + "#url": "https://instagram.com/stories/geekmig/2724343156064789461", + "#category": ("", "instagram", "stories"), + "#class": instagram.InstagramStoriesExtractor, + }, + { + "#url": "https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1", + "#category": ("", "instagram", "highlights"), + "#class": instagram.InstagramStoriesExtractor, + }, + { + "#url": "https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1?story_media_id=2724343156064789461", + "#category": ("", "instagram", "highlights"), + "#class": instagram.InstagramStoriesExtractor, + }, + { + "#url": "https://www.instagram.com/instagram/highlights", + "#category": ("", "instagram", "highlights"), + "#class": instagram.InstagramHighlightsExtractor, + }, + { + "#url": "https://www.instagram.com/instagram/following", + "#category": ("", "instagram", "following"), + "#class": instagram.InstagramFollowingExtractor, + "#range": "1-16", + "#count": ">= 16", + }, + { + "#url": "https://www.instagram.com/explore/tags/instagram/", + "#category": ("", "instagram", "tag"), + "#class": instagram.InstagramTagExtractor, + "#range": "1-16", + "#count": ">= 16", + }, + { + "#url": "https://www.instagram.com/instagram/info", + "#category": ("", "instagram", "info"), + "#class": instagram.InstagramInfoExtractor, + "#auth": False, + }, + { + "#url": "https://www.instagram.com/instagram/avatar", + "#category": ("", "instagram", "avatar"), + "#class": instagram.InstagramAvatarExtractor, + "#pattern": r"https://instagram\.[\w.-]+\.fbcdn\.net/v/t51\.2885-19/281440578_1088265838702675_6233856337905829714_n\.jpg", + }, + { + "#url": "https://www.instagram.com/p/BqvsDleB3lV/", + "#comment": "GraphImage", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + "#pattern": r"https://[^/]+\.(cdninstagram\.com|fbcdn\.net)/v(p/[0-9a-f]+/[0-9A-F]+)?/t51.2885-15/e35/44877605_725955034447492_3123079845831750529_n.jpg", + "date": "dt:2018-11-29 01:04:04", + "description": str, + "height": int, + "likes": int, + "location_id": "214424288", + "location_slug": "hong-kong", + "location_url": r"re:/explore/locations/214424288/hong-kong/", + "media_id": "1922949326347663701", + "shortcode": "BqvsDleB3lV", + "post_id": "1922949326347663701", + "post_shortcode": "BqvsDleB3lV", + "post_url": "https://www.instagram.com/p/BqvsDleB3lV/", + "tags": ["#WHPsquares"], + "typename": "GraphImage", + "username": "instagram", + "width": int, + }, + { + "#url": "https://www.instagram.com/p/BoHk1haB5tM/", + "#comment": "GraphSidecar", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + "#count": 5, + "sidecar_media_id": "1875629777499953996", + "sidecar_shortcode": "BoHk1haB5tM", + "post_id": "1875629777499953996", + "post_shortcode": "BoHk1haB5tM", + "post_url": "https://www.instagram.com/p/BoHk1haB5tM/", + "num": int, + "likes": int, + "username": "instagram", + }, + { + "#url": "https://www.instagram.com/p/Bqxp0VSBgJg/", + "#comment": "GraphVideo", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + "#pattern": r"/46840863_726311431074534_7805566102611403091_n\.mp4", + "date": "dt:2018-11-29 19:23:58", + "description": str, + "height": int, + "likes": int, + "media_id": "1923502432034620000", + "post_url": "https://www.instagram.com/p/Bqxp0VSBgJg/", + "shortcode": "Bqxp0VSBgJg", + "tags": ["#ASMR"], + "typename": "GraphVideo", + "username": "instagram", + "width": int, + }, + { + "#url": "https://www.instagram.com/tv/BkQjCfsBIzi/", + "#comment": "GraphVideo (IGTV)", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + "#pattern": r"/10000000_597132547321814_702169244961988209_n\.mp4", + "date": "dt:2018-06-20 19:51:32", + "description": str, + "height": int, + "likes": int, + "media_id": "1806097553666903266", + "post_url": "https://www.instagram.com/p/BkQjCfsBIzi/", + "shortcode": "BkQjCfsBIzi", + "typename": "GraphVideo", + "username": "instagram", + "width": int, + }, + { + "#url": "https://www.instagram.com/p/BtOvDOfhvRr/", + "#comment": "GraphSidecar with 2 embedded GraphVideo objects", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + "#count": 2, + "post_url": "https://www.instagram.com/p/BtOvDOfhvRr/", + "sidecar_media_id": "1967717017113261163", + "sidecar_shortcode": "BtOvDOfhvRr", + "video_url": str, + }, + { + "#url": "https://www.instagram.com/p/B_2lf3qAd3y/", + "#comment": "GraphImage with tagged user", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + "tagged_users": [ + { + "id": "1246468638", + "username": "kaaymbl", + "full_name": "Call Me Kay", + } + ], + }, + { + "#url": "https://www.instagram.com/dm/p/CW042g7B9CY/", + "#comment": "URL with username (#2085)", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + }, + { + "#url": "https://www.instagram.com/reel/CDg_6Y1pxWu/", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + }, ) diff --git a/test/results/issuu.py b/test/results/issuu.py index 4a90be13b..b43e324d3 100644 --- a/test/results/issuu.py +++ b/test/results/issuu.py @@ -1,46 +1,40 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import issuu - __tests__ = ( -{ - "#url" : "https://issuu.com/issuu/docs/motions-1-2019/", - "#category": ("", "issuu", "publication"), - "#class" : issuu.IssuuPublicationExtractor, - "#pattern" : r"https://image.isu.pub/190916155301-\w+/jpg/page_\d+.jpg", - "#count" : 36, - - "document" : { - "access" : "PUBLIC", - "contentRating": { - "isAdsafe" : True, - "isExplicit": False, - "isReviewed": True, + { + "#url": "https://issuu.com/issuu/docs/motions-1-2019/", + "#category": ("", "issuu", "publication"), + "#class": issuu.IssuuPublicationExtractor, + "#pattern": r"https://image.isu.pub/190916155301-\w+/jpg/page_\d+.jpg", + "#count": 36, + "document": { + "access": "PUBLIC", + "contentRating": { + "isAdsafe": True, + "isExplicit": False, + "isReviewed": True, + }, + "date": "dt:2019-09-16 00:00:00", + "description": r"re:Motions, the brand new publication by I", + "documentName": "motions-1-2019", + "pageCount": 36, + "publicationId": "d99ec95935f15091b040cb8060f05510", + "title": "Motions by Issuu - Issue 1", + "username": "issuu", }, - "date" : "dt:2019-09-16 00:00:00", - "description" : r"re:Motions, the brand new publication by I", - "documentName" : "motions-1-2019", - "pageCount" : 36, - "publicationId": "d99ec95935f15091b040cb8060f05510", - "title" : "Motions by Issuu - Issue 1", - "username" : "issuu", + "extension": "jpg", + "filename": r"re:page_\d+", + "num": int, + }, + { + "#url": "https://issuu.com/issuu", + "#category": ("", "issuu", "user"), + "#class": issuu.IssuuUserExtractor, + "#pattern": issuu.IssuuPublicationExtractor.pattern, + "#count": "> 25", }, - "extension": "jpg", - "filename" : r"re:page_\d+", - "num" : int, -}, - -{ - "#url" : "https://issuu.com/issuu", - "#category": ("", "issuu", "user"), - "#class" : issuu.IssuuUserExtractor, - "#pattern" : issuu.IssuuPublicationExtractor.pattern, - "#count" : "> 25", -}, - ) diff --git a/test/results/itaku.py b/test/results/itaku.py index 27b59414b..7cc6ef40d 100644 --- a/test/results/itaku.py +++ b/test/results/itaku.py @@ -1,81 +1,74 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import itaku - __tests__ = ( -{ - "#url" : "https://itaku.ee/profile/piku/gallery", - "#category": ("", "itaku", "gallery"), - "#class" : itaku.ItakuGalleryExtractor, - "#pattern" : r"https://itaku\.ee/api/media/gallery_imgs/[^/?#]+\.(jpg|png|gif)", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://itaku.ee/images/100471", - "#category": ("", "itaku", "image"), - "#class" : itaku.ItakuImageExtractor, - "#urls" : "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT.png", - - "already_pinned" : None, - "blacklisted" : { - "blacklisted_tags": [], - "is_blacklisted" : False, + { + "#url": "https://itaku.ee/profile/piku/gallery", + "#category": ("", "itaku", "gallery"), + "#class": itaku.ItakuGalleryExtractor, + "#pattern": r"https://itaku\.ee/api/media/gallery_imgs/[^/?#]+\.(jpg|png|gif)", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://itaku.ee/images/100471", + "#category": ("", "itaku", "image"), + "#class": itaku.ItakuImageExtractor, + "#urls": "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT.png", + "already_pinned": None, + "blacklisted": { + "blacklisted_tags": [], + "is_blacklisted": False, + }, + "can_reshare": True, + "date": "dt:2022-05-05 19:21:17", + "date_added": "2022-05-05T19:21:17.674148Z", + "date_edited": "2022-05-25T14:37:46.220612Z", + "description": "sketch from drawpile", + "extension": "png", + "filename": "220504_oUNIAFT", + "hotness_score": float, + "id": 100471, + "image": "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT.png", + "image_xl": "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT/lg.jpg", + "liked_by_you": False, + "maturity_rating": "SFW", + "num_comments": int, + "num_likes": int, + "num_reshares": int, + "obj_tags": 136446, + "owner": 16775, + "owner_avatar": "https://itaku.ee/api/media/profile_pics/av2022r_vKYVywc/md.jpg", + "owner_displayname": "Piku", + "owner_username": "piku", + "reshared_by_you": False, + "sections": ["Fanart/Miku"], + "tags": list, + "tags_character": ["hatsune_miku"], + "tags_copyright": ["vocaloid"], + "tags_general": [ + "twintails", + "green_hair", + "flag", + "gloves", + "green_eyes", + "female", + "racing_miku", + ], + "title": "Racing Miku 2022 Ver.", + "too_mature": False, + "uncompressed_filesize": "0.62", + "video": None, + "visibility": "PUBLIC", + }, + { + "#url": "https://itaku.ee/images/19465", + "#comment": "video", + "#category": ("", "itaku", "image"), + "#class": itaku.ItakuImageExtractor, + "#urls": "https://itaku.ee/api/media/gallery_vids/sleepy_af_OY5GHWw.mp4", }, - "can_reshare" : True, - "date" : "dt:2022-05-05 19:21:17", - "date_added" : "2022-05-05T19:21:17.674148Z", - "date_edited" : "2022-05-25T14:37:46.220612Z", - "description" : "sketch from drawpile", - "extension" : "png", - "filename" : "220504_oUNIAFT", - "hotness_score" : float, - "id" : 100471, - "image" : "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT.png", - "image_xl" : "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT/lg.jpg", - "liked_by_you" : False, - "maturity_rating" : "SFW", - "num_comments" : int, - "num_likes" : int, - "num_reshares" : int, - "obj_tags" : 136446, - "owner" : 16775, - "owner_avatar" : "https://itaku.ee/api/media/profile_pics/av2022r_vKYVywc/md.jpg", - "owner_displayname": "Piku", - "owner_username" : "piku", - "reshared_by_you" : False, - "sections" : ["Fanart/Miku"], - "tags" : list, - "tags_character" : ["hatsune_miku"], - "tags_copyright" : ["vocaloid"], - "tags_general": [ - "twintails", - "green_hair", - "flag", - "gloves", - "green_eyes", - "female", - "racing_miku", - ], - "title" : "Racing Miku 2022 Ver.", - "too_mature" : False, - "uncompressed_filesize": "0.62", - "video" : None, - "visibility" : "PUBLIC", -}, - -{ - "#url" : "https://itaku.ee/images/19465", - "#comment" : "video", - "#category": ("", "itaku", "image"), - "#class" : itaku.ItakuImageExtractor, - "#urls" : "https://itaku.ee/api/media/gallery_vids/sleepy_af_OY5GHWw.mp4", -}, - ) diff --git a/test/results/itchio.py b/test/results/itchio.py index f49bd69f3..445754d54 100644 --- a/test/results/itchio.py +++ b/test/results/itchio.py @@ -1,33 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import itchio - __tests__ = ( -{ - "#url" : "https://sirtartarus.itch.io/a-craft-of-mine", - "#category": ("", "itchio", "game"), - "#class" : itchio.ItchioGameExtractor, - "#pattern" : r"https://(dl.itch.zone|itchio-mirror.\w+.r2.cloudflarestorage.com)/upload2/game/1983311/\d+\?", - "#count" : 3, - - "extension": "", - "filename" : r"re:\d+", - "game" : { - "id" : 1983311, - "noun" : "game", - "title": "A Craft Of Mine", - "url" : "https://sirtartarus.itch.io/a-craft-of-mine", + { + "#url": "https://sirtartarus.itch.io/a-craft-of-mine", + "#category": ("", "itchio", "game"), + "#class": itchio.ItchioGameExtractor, + "#pattern": r"https://(dl.itch.zone|itchio-mirror.\w+.r2.cloudflarestorage.com)/upload2/game/1983311/\d+\?", + "#count": 3, + "extension": "", + "filename": r"re:\d+", + "game": { + "id": 1983311, + "noun": "game", + "title": "A Craft Of Mine", + "url": "https://sirtartarus.itch.io/a-craft-of-mine", + }, + "user": { + "id": 4060052, + "name": "SirTartarus", + "url": "https://sirtartarus.itch.io", + }, }, - "user" : { - "id" : 4060052, - "name": "SirTartarus", - "url" : "https://sirtartarus.itch.io", - }, -}, - ) diff --git a/test/results/joyreactor.py b/test/results/joyreactor.py index 4c29c2130..320e8e150 100644 --- a/test/results/joyreactor.py +++ b/test/results/joyreactor.py @@ -1,108 +1,93 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import reactor - __tests__ = ( -{ - "#url" : "http://joyreactor.cc/tag/Advent+Cirno", - "#category": ("reactor", "joyreactor", "tag"), - "#class" : reactor.ReactorTagExtractor, - "#count" : ">= 15", -}, - -{ - "#url" : "http://joyreactor.com/tag/Cirno", - "#category": ("reactor", "joyreactor", "tag"), - "#class" : reactor.ReactorTagExtractor, - "#sha1_url": "aa59090590b26f4654881301fe8fe748a51625a8", -}, - -{ - "#url" : "http://joyreactor.com/tag/Dark+Souls+2/best", - "#comment" : "'best' rating (#3073)", - "#category": ("reactor", "joyreactor", "tag"), - "#class" : reactor.ReactorTagExtractor, - "#count" : 4, -}, - -{ - "#url" : "http://joyreactor.cc/search/Nature", - "#category": ("reactor", "joyreactor", "search"), - "#class" : reactor.ReactorSearchExtractor, - "#range" : "1-25", - "#count" : ">= 20", -}, - -{ - "#url" : "http://joyreactor.com/search?q=Nature", - "#category": ("reactor", "joyreactor", "search"), - "#class" : reactor.ReactorSearchExtractor, - "#range" : "1-25", - "#count" : ">= 20", -}, - -{ - "#url" : "http://joyreactor.cc/user/hemantic", - "#category": ("reactor", "joyreactor", "user"), - "#class" : reactor.ReactorUserExtractor, -}, - -{ - "#url" : "http://joyreactor.com/user/Tacoman123", - "#category": ("reactor", "joyreactor", "user"), - "#class" : reactor.ReactorUserExtractor, - "#sha1_url": "60ce9a3e3db791a0899f7fb7643b5b87d09ae3b5", -}, - -{ - "#url" : "http://joyreactor.com/post/3721876", - "#comment" : "single image", - "#category": ("reactor", "joyreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#pattern" : r"http://img\d\.joyreactor\.com/pics/post/full/cartoon-painting-monster-lake-4841316.jpeg", - "#count" : 1, - "#sha1_metadata": "2207a7dfed55def2042b6c2554894c8d7fda386e", -}, - -{ - "#url" : "http://joyreactor.com/post/3713804", - "#comment" : "4 images", - "#category": ("reactor", "joyreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#pattern" : r"http://img\d\.joyreactor\.com/pics/post/full/movie-tv-godzilla-monsters-\d+\.jpeg", - "#count" : 4, - "#sha1_metadata": "d7da9ba7809004c809eedcf6f1c06ad0fbb3df21", -}, - -{ - "#url" : "http://joyreactor.com/post/3726210", - "#comment" : "gif / video", - "#category": ("reactor", "joyreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#sha1_url" : "60f3b9a0a3918b269bea9b4f8f1a5ab3c2c550f8", - "#sha1_metadata": "8949d9d5fc469dab264752432efbaa499561664a", -}, - -{ - "#url" : "http://joyreactor.com/post/3668724", - "#comment" : "youtube embed", - "#category": ("reactor", "joyreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#sha1_url" : "bf1666eddcff10c9b58f6be63fa94e4e13074214", - "#sha1_metadata": "e18b1ffbd79d76f9a0e90b6d474cc2499e343f0b", -}, - -{ - "#url" : "http://joyreactor.cc/post/1299", - "#comment" : "'malformed' JSON", - "#category": ("reactor", "joyreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#sha1_url": "ab02c6eb7b4035ad961b29ee0770ee41be2fcc39", -}, - + { + "#url": "http://joyreactor.cc/tag/Advent+Cirno", + "#category": ("reactor", "joyreactor", "tag"), + "#class": reactor.ReactorTagExtractor, + "#count": ">= 15", + }, + { + "#url": "http://joyreactor.com/tag/Cirno", + "#category": ("reactor", "joyreactor", "tag"), + "#class": reactor.ReactorTagExtractor, + "#sha1_url": "aa59090590b26f4654881301fe8fe748a51625a8", + }, + { + "#url": "http://joyreactor.com/tag/Dark+Souls+2/best", + "#comment": "'best' rating (#3073)", + "#category": ("reactor", "joyreactor", "tag"), + "#class": reactor.ReactorTagExtractor, + "#count": 4, + }, + { + "#url": "http://joyreactor.cc/search/Nature", + "#category": ("reactor", "joyreactor", "search"), + "#class": reactor.ReactorSearchExtractor, + "#range": "1-25", + "#count": ">= 20", + }, + { + "#url": "http://joyreactor.com/search?q=Nature", + "#category": ("reactor", "joyreactor", "search"), + "#class": reactor.ReactorSearchExtractor, + "#range": "1-25", + "#count": ">= 20", + }, + { + "#url": "http://joyreactor.cc/user/hemantic", + "#category": ("reactor", "joyreactor", "user"), + "#class": reactor.ReactorUserExtractor, + }, + { + "#url": "http://joyreactor.com/user/Tacoman123", + "#category": ("reactor", "joyreactor", "user"), + "#class": reactor.ReactorUserExtractor, + "#sha1_url": "60ce9a3e3db791a0899f7fb7643b5b87d09ae3b5", + }, + { + "#url": "http://joyreactor.com/post/3721876", + "#comment": "single image", + "#category": ("reactor", "joyreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#pattern": r"http://img\d\.joyreactor\.com/pics/post/full/cartoon-painting-monster-lake-4841316.jpeg", + "#count": 1, + "#sha1_metadata": "2207a7dfed55def2042b6c2554894c8d7fda386e", + }, + { + "#url": "http://joyreactor.com/post/3713804", + "#comment": "4 images", + "#category": ("reactor", "joyreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#pattern": r"http://img\d\.joyreactor\.com/pics/post/full/movie-tv-godzilla-monsters-\d+\.jpeg", + "#count": 4, + "#sha1_metadata": "d7da9ba7809004c809eedcf6f1c06ad0fbb3df21", + }, + { + "#url": "http://joyreactor.com/post/3726210", + "#comment": "gif / video", + "#category": ("reactor", "joyreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#sha1_url": "60f3b9a0a3918b269bea9b4f8f1a5ab3c2c550f8", + "#sha1_metadata": "8949d9d5fc469dab264752432efbaa499561664a", + }, + { + "#url": "http://joyreactor.com/post/3668724", + "#comment": "youtube embed", + "#category": ("reactor", "joyreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#sha1_url": "bf1666eddcff10c9b58f6be63fa94e4e13074214", + "#sha1_metadata": "e18b1ffbd79d76f9a0e90b6d474cc2499e343f0b", + }, + { + "#url": "http://joyreactor.cc/post/1299", + "#comment": "'malformed' JSON", + "#category": ("reactor", "joyreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#sha1_url": "ab02c6eb7b4035ad961b29ee0770ee41be2fcc39", + }, ) diff --git a/test/results/jpgfish.py b/test/results/jpgfish.py index bef1d6e05..81d4944cf 100644 --- a/test/results/jpgfish.py +++ b/test/results/jpgfish.py @@ -1,155 +1,129 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import chevereto - __tests__ = ( -{ - "#url" : "https://jpg4.su/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, - "#urls" : "https://simp3.host.church/images/funnymeme.jpg", - "#sha1_content": "098e5e9b17ad634358426e0ffd1c93871474d13c", - - "album" : "", - "extension": "jpg", - "filename" : "funnymeme", - "id" : "LecXGS", - "url" : "https://simp3.host.church/images/funnymeme.jpg", - "user" : "exearco", -}, - -{ - "#url" : "https://jpg.church/img/auCruA", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, - "#pattern" : r"https://simp2\.host\.church/hannahowo_00457\.jpg", - - "album": "401-500", -}, - -{ - "#url" : "https://jpg1.su/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, -}, - -{ - "#url" : "https://jpeg.pet/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, -}, - -{ - "#url" : "https://jpg.pet/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, -}, - -{ - "#url" : "https://jpg.fishing/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, -}, - -{ - "#url" : "https://jpg.fish/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, -}, - -{ - "#url" : "https://jpg.church/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, -}, - -{ - "#url" : "https://jpg1.su/album/CDilP/?sort=date_desc&page=1", - "#category": ("chevereto", "jpgfish", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://jpg.fishing/a/gunggingnsk.N9OOI", - "#category": ("chevereto", "jpgfish", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, - "#count" : 114, -}, - -{ - "#url" : "https://jpg.fish/a/101-200.aNJ6A/", - "#category": ("chevereto", "jpgfish", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, - "#count" : 100, -}, - -{ - "#url" : "https://jpg.church/a/hannahowo.aNTdH/sub", - "#category": ("chevereto", "jpgfish", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, - "#count" : 606, -}, - -{ - "#url" : "https://jpeg.pet/album/CDilP/?sort=date_desc&page=1", - "#category": ("chevereto", "jpgfish", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, -}, - -{ - "#url" : "https://jpg.pet/album/CDilP/?sort=date_desc&page=1", - "#category": ("chevereto", "jpgfish", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, -}, - -{ - "#url" : "https://jpg1.su/exearco", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, - "#count" : 3, -}, - -{ - "#url" : "https://jpg.church/exearco/albums", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, - "#count" : 1, -}, - -{ - "#url" : "https://jpeg.pet/exearco", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, -}, - -{ - "#url" : "https://jpg.pet/exearco", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, -}, - -{ - "#url" : "https://jpg.fishing/exearco", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, -}, - -{ - "#url" : "https://jpg.fish/exearco", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, -}, - -{ - "#url" : "https://jpg.church/exearco", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, -}, - + { + "#url": "https://jpg4.su/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + "#urls": "https://simp3.host.church/images/funnymeme.jpg", + "#sha1_content": "098e5e9b17ad634358426e0ffd1c93871474d13c", + "album": "", + "extension": "jpg", + "filename": "funnymeme", + "id": "LecXGS", + "url": "https://simp3.host.church/images/funnymeme.jpg", + "user": "exearco", + }, + { + "#url": "https://jpg.church/img/auCruA", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + "#pattern": r"https://simp2\.host\.church/hannahowo_00457\.jpg", + "album": "401-500", + }, + { + "#url": "https://jpg1.su/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + }, + { + "#url": "https://jpeg.pet/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + }, + { + "#url": "https://jpg.pet/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + }, + { + "#url": "https://jpg.fishing/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + }, + { + "#url": "https://jpg.fish/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + }, + { + "#url": "https://jpg.church/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + }, + { + "#url": "https://jpg1.su/album/CDilP/?sort=date_desc&page=1", + "#category": ("chevereto", "jpgfish", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + "#count": 2, + }, + { + "#url": "https://jpg.fishing/a/gunggingnsk.N9OOI", + "#category": ("chevereto", "jpgfish", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + "#count": 114, + }, + { + "#url": "https://jpg.fish/a/101-200.aNJ6A/", + "#category": ("chevereto", "jpgfish", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + "#count": 100, + }, + { + "#url": "https://jpg.church/a/hannahowo.aNTdH/sub", + "#category": ("chevereto", "jpgfish", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + "#count": 606, + }, + { + "#url": "https://jpeg.pet/album/CDilP/?sort=date_desc&page=1", + "#category": ("chevereto", "jpgfish", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + }, + { + "#url": "https://jpg.pet/album/CDilP/?sort=date_desc&page=1", + "#category": ("chevereto", "jpgfish", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + }, + { + "#url": "https://jpg1.su/exearco", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + "#count": 3, + }, + { + "#url": "https://jpg.church/exearco/albums", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + "#count": 1, + }, + { + "#url": "https://jpeg.pet/exearco", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + }, + { + "#url": "https://jpg.pet/exearco", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + }, + { + "#url": "https://jpg.fishing/exearco", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + }, + { + "#url": "https://jpg.fish/exearco", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + }, + { + "#url": "https://jpg.church/exearco", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + }, ) diff --git a/test/results/kabeuchi.py b/test/results/kabeuchi.py index 37b6a3d8f..dd5b00bb4 100644 --- a/test/results/kabeuchi.py +++ b/test/results/kabeuchi.py @@ -1,27 +1,22 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import kabeuchi from gallery_dl import exception - +from gallery_dl.extractor import kabeuchi __tests__ = ( -{ - "#url" : "https://kabe-uchiroom.com/mypage/?id=919865303848255493", - "#category": ("", "kabeuchi", "user"), - "#class" : kabeuchi.KabeuchiUserExtractor, - "#pattern" : r"https://kabe-uchiroom\.com/accounts/upfile/3/919865303848255493/\w+\.jpe?g", - "#count" : ">= 24", -}, - -{ - "#url" : "https://kabe-uchiroom.com/mypage/?id=123456789", - "#category": ("", "kabeuchi", "user"), - "#class" : kabeuchi.KabeuchiUserExtractor, - "#exception": exception.NotFoundError, -}, - + { + "#url": "https://kabe-uchiroom.com/mypage/?id=919865303848255493", + "#category": ("", "kabeuchi", "user"), + "#class": kabeuchi.KabeuchiUserExtractor, + "#pattern": r"https://kabe-uchiroom\.com/accounts/upfile/3/919865303848255493/\w+\.jpe?g", + "#count": ">= 24", + }, + { + "#url": "https://kabe-uchiroom.com/mypage/?id=123456789", + "#category": ("", "kabeuchi", "user"), + "#class": kabeuchi.KabeuchiUserExtractor, + "#exception": exception.NotFoundError, + }, ) diff --git a/test/results/keenspot.py b/test/results/keenspot.py index 9f756ad8b..241ca39c5 100644 --- a/test/results/keenspot.py +++ b/test/results/keenspot.py @@ -1,55 +1,47 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import keenspot - __tests__ = ( -{ - "#url" : "http://marksmen.keenspot.com/", - "#comment" : "link", - "#category": ("", "keenspot", "comic"), - "#class" : keenspot.KeenspotComicExtractor, - "#range" : "1-3", - "#sha1_url": "83bcf029103bf8bc865a1988afa4aaeb23709ba6", -}, - -{ - "#url" : "http://barkercomic.keenspot.com/", - "#comment" : "id", - "#category": ("", "keenspot", "comic"), - "#class" : keenspot.KeenspotComicExtractor, - "#range" : "1-3", - "#sha1_url": "c4080926db18d00bac641fdd708393b7d61379e6", -}, - -{ - "#url" : "http://crowscare.keenspot.com/", - "#comment" : "id v2", - "#category": ("", "keenspot", "comic"), - "#class" : keenspot.KeenspotComicExtractor, - "#range" : "1-3", - "#sha1_url": "a00e66a133dd39005777317da90cef921466fcaa", -}, - -{ - "#url" : "http://supernovas.keenspot.com/", - "#comment" : "ks", - "#category": ("", "keenspot", "comic"), - "#class" : keenspot.KeenspotComicExtractor, - "#range" : "1-3", - "#sha1_url": "de21b12887ef31ff82edccbc09d112e3885c3aab", -}, - -{ - "#url" : "http://twokinds.keenspot.com/comic/1066/", - "#category": ("", "keenspot", "comic"), - "#class" : keenspot.KeenspotComicExtractor, - "#range" : "1-3", - "#sha1_url": "6a784e11370abfb343dcad9adbb7718f9b7be350", -}, - + { + "#url": "http://marksmen.keenspot.com/", + "#comment": "link", + "#category": ("", "keenspot", "comic"), + "#class": keenspot.KeenspotComicExtractor, + "#range": "1-3", + "#sha1_url": "83bcf029103bf8bc865a1988afa4aaeb23709ba6", + }, + { + "#url": "http://barkercomic.keenspot.com/", + "#comment": "id", + "#category": ("", "keenspot", "comic"), + "#class": keenspot.KeenspotComicExtractor, + "#range": "1-3", + "#sha1_url": "c4080926db18d00bac641fdd708393b7d61379e6", + }, + { + "#url": "http://crowscare.keenspot.com/", + "#comment": "id v2", + "#category": ("", "keenspot", "comic"), + "#class": keenspot.KeenspotComicExtractor, + "#range": "1-3", + "#sha1_url": "a00e66a133dd39005777317da90cef921466fcaa", + }, + { + "#url": "http://supernovas.keenspot.com/", + "#comment": "ks", + "#category": ("", "keenspot", "comic"), + "#class": keenspot.KeenspotComicExtractor, + "#range": "1-3", + "#sha1_url": "de21b12887ef31ff82edccbc09d112e3885c3aab", + }, + { + "#url": "http://twokinds.keenspot.com/comic/1066/", + "#category": ("", "keenspot", "comic"), + "#class": keenspot.KeenspotComicExtractor, + "#range": "1-3", + "#sha1_url": "6a784e11370abfb343dcad9adbb7718f9b7be350", + }, ) diff --git a/test/results/kemonoparty.py b/test/results/kemonoparty.py index 4c3700897..34ff16e3b 100644 --- a/test/results/kemonoparty.py +++ b/test/results/kemonoparty.py @@ -1,414 +1,360 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import kemonoparty from gallery_dl import exception - +from gallery_dl.extractor import kemonoparty __tests__ = ( -{ - "#url" : "https://kemono.su/fanbox/user/6993449", - "#category": ("", "kemonoparty", "fanbox"), - "#class" : kemonoparty.KemonopartyUserExtractor, - "#range" : "1-500", - "#count" : 500, -}, - -{ - "#url" : "https://kemono.su/patreon/user/881792?o=150", - "#comment" : "'max-posts' option, 'o' query parameter (#1674)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyUserExtractor, - "#options" : {"max-posts": 100}, - "#count" : range(200, 300), -}, - -{ - "#url" : "https://kemono.su/fanbox/user/6993449?q=お蔵入りになった", - "#comment" : "search / 'q' query parameter (#3385, #4057)", - "#category": ("", "kemonoparty", "fanbox"), - "#class" : kemonoparty.KemonopartyUserExtractor, - "#urls" : ( - "https://kemono.su/data/ef/7b/ef7b4398a2f4ada597421fd3c116cff86e85695911f7cd2a459b0e566b864e46.png", - "https://kemono.su/data/73/e6/73e615f6645b9d1af6329448601673c9275f07fd11eb37670c97e307e29a9ee9.png", - ), - - "id": "8779", -}, - -{ - "#url" : "https://kemono.su/subscribestar/user/alcorart", - "#category": ("", "kemonoparty", "subscribestar"), - "#class" : kemonoparty.KemonopartyUserExtractor, -}, - -{ - "#url" : "https://kemono.su/subscribestar/user/alcorart", - "#category": ("", "kemonoparty", "subscribestar"), - "#class" : kemonoparty.KemonopartyUserExtractor, -}, - -{ - "#url" : "https://kemono.su/fanbox/user/6993449/post/506575", - "#category": ("", "kemonoparty", "fanbox"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#pattern" : r"https://kemono.su/data/21/0f/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg", - "#sha1_content": "900949cefc97ab8dc1979cc3664785aac5ba70dd", - - "added" : "2020-05-06T20:28:02.302000", - "content" : str, - "count" : 1, - "date" : "dt:2019-08-10 17:09:04", - "edited" : None, - "embed" : dict, - "extension" : "jpeg", - "filename" : "P058kDFYus7DbqAkGlfWTlOr", - "hash" : "210f35388e28bbcf756db18dd516e2d82ce758e0d32881eeee76d43e1716d382", - "id" : "506575", - "num" : 1, - "published" : "2019-08-10T17:09:04", - "service" : "fanbox", - "shared_file": False, - "subcategory": "fanbox", - "title" : "c96取り置き", - "type" : "file", - "user" : "6993449", -}, - -{ - "#url" : "https://kemono.su/fanbox/user/7356311/post/802343", - "#comment" : "inline image (#1286)", - "#category": ("", "kemonoparty", "fanbox"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#pattern" : r"https://kemono\.su/data/47/b5/47b5c014ecdcfabdf2c85eec53f1133a76336997ae8596f332e97d956a460ad2\.jpg", - - "hash": "47b5c014ecdcfabdf2c85eec53f1133a76336997ae8596f332e97d956a460ad2", -}, - -{ - "#url" : "https://kemono.su/gumroad/user/3101696181060/post/tOWyf", - "#category": ("", "kemonoparty", "gumroad"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#urls" : "https://kemono.su/data/6f/13/6f1394b19516396ea520254350662c254bbea30c1e111fd4b0f042c61c426d07.zip", -}, - -{ - "#url" : "https://kemono.party/gumroad/user/3252870377455/post/aJnAH", - "#comment" : "username (#1548, #1652)", - "#category": ("", "kemonoparty", "gumroad"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#options" : {"metadata": True}, - - "username": "Kudalyn's Creations", -}, - -{ - "#url" : "https://kemono.su/patreon/user/4158582/post/32099982", - "#comment" : "allow duplicates (#2440)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://kemono.su/patreon/user/4158582/post/32099982", - "#comment" : "allow duplicates (#2440)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#options" : {"duplicates": True}, - "#count" : 3, -}, - -{ - "#url" : "https://kemono.su/patreon/user/34134344/post/38129255", - "#comment" : "DMs (#2008)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#options" : {"dms": True}, - - "dms": [{ - "body": r"re:Hi! Thank you very much for supporting the work I did in May. Here's your reward pack! I hope you find something you enjoy in it. :\)\n\nhttps://www.mediafire.com/file/\w+/Set13_tier_2.zip/file", - "date": "2021-06", - }], -}, - -{ - "#url" : "https://kemono.su/patreon/user/3161935/post/68231671", - "#comment" : "announcements", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#options" : {"announcements": True}, - - "announcements": [{ - "body": "
          Thank you so much for the support!
          This Patreon is more of a tip jar for supporting what I make. I have to clarify that there are no exclusive Patreon animations because all are released for the public. You will get earlier access to WIPs. Direct downloads to my works are also available for $5 and $10 Tiers.
          ", - "date": "2023-02", - }], -}, - -{ - "#url" : "https://kemono.su/patreon/user/19623797/post/29035449", - "#comment" : "invalid file (#3510)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#pattern" : r"907ba78b4545338d3539683e63ecb51cf51c10adc9dabd86e92bd52339f298b9\.txt", - "#sha1_content": "da39a3ee5e6b4b0d3255bfef95601890afd80709", -}, - -{ - "#url" : "https://kemono.su/subscribestar/user/alcorart/post/184330", - "#category": ("", "kemonoparty", "subscribestar"), - "#class" : kemonoparty.KemonopartyPostExtractor, -}, - -{ - "#url" : "https://kemono.su/subscribestar/user/alcorart/post/184330", - "#category": ("", "kemonoparty", "subscribestar"), - "#class" : kemonoparty.KemonopartyPostExtractor, -}, - -{ - "#url" : "https://www.kemono.su/subscribestar/user/alcorart/post/184330", - "#category": ("", "kemonoparty", "subscribestar"), - "#class" : kemonoparty.KemonopartyPostExtractor, -}, - -{ - "#url" : "https://beta.kemono.su/subscribestar/user/alcorart/post/184330", - "#category": ("", "kemonoparty", "subscribestar"), - "#class" : kemonoparty.KemonopartyPostExtractor, -}, - -{ - "#url" : "https://kemono.su/patreon/user/3161935/post/68231671/revision/142470", - "#comment" : "revisions (#4498)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#urls" : "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", - - "file" : { - "hash": "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", - "name": "wip update.jpg", - "path": "/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", + { + "#url": "https://kemono.su/fanbox/user/6993449", + "#category": ("", "kemonoparty", "fanbox"), + "#class": kemonoparty.KemonopartyUserExtractor, + "#range": "1-500", + "#count": 500, + }, + { + "#url": "https://kemono.su/patreon/user/881792?o=150", + "#comment": "'max-posts' option, 'o' query parameter (#1674)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyUserExtractor, + "#options": {"max-posts": 100}, + "#count": range(200, 300), + }, + { + "#url": "https://kemono.su/fanbox/user/6993449?q=お蔵入りになった", + "#comment": "search / 'q' query parameter (#3385, #4057)", + "#category": ("", "kemonoparty", "fanbox"), + "#class": kemonoparty.KemonopartyUserExtractor, + "#urls": ( + "https://kemono.su/data/ef/7b/ef7b4398a2f4ada597421fd3c116cff86e85695911f7cd2a459b0e566b864e46.png", + "https://kemono.su/data/73/e6/73e615f6645b9d1af6329448601673c9275f07fd11eb37670c97e307e29a9ee9.png", + ), + "id": "8779", + }, + { + "#url": "https://kemono.su/subscribestar/user/alcorart", + "#category": ("", "kemonoparty", "subscribestar"), + "#class": kemonoparty.KemonopartyUserExtractor, + }, + { + "#url": "https://kemono.su/subscribestar/user/alcorart", + "#category": ("", "kemonoparty", "subscribestar"), + "#class": kemonoparty.KemonopartyUserExtractor, + }, + { + "#url": "https://kemono.su/fanbox/user/6993449/post/506575", + "#category": ("", "kemonoparty", "fanbox"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#pattern": r"https://kemono.su/data/21/0f/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg", + "#sha1_content": "900949cefc97ab8dc1979cc3664785aac5ba70dd", + "added": "2020-05-06T20:28:02.302000", + "content": str, + "count": 1, + "date": "dt:2019-08-10 17:09:04", + "edited": None, + "embed": dict, + "extension": "jpeg", + "filename": "P058kDFYus7DbqAkGlfWTlOr", + "hash": "210f35388e28bbcf756db18dd516e2d82ce758e0d32881eeee76d43e1716d382", + "id": "506575", + "num": 1, + "published": "2019-08-10T17:09:04", + "service": "fanbox", + "shared_file": False, + "subcategory": "fanbox", + "title": "c96取り置き", "type": "file", + "user": "6993449", + }, + { + "#url": "https://kemono.su/fanbox/user/7356311/post/802343", + "#comment": "inline image (#1286)", + "#category": ("", "kemonoparty", "fanbox"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#pattern": r"https://kemono\.su/data/47/b5/47b5c014ecdcfabdf2c85eec53f1133a76336997ae8596f332e97d956a460ad2\.jpg", + "hash": "47b5c014ecdcfabdf2c85eec53f1133a76336997ae8596f332e97d956a460ad2", + }, + { + "#url": "https://kemono.su/gumroad/user/3101696181060/post/tOWyf", + "#category": ("", "kemonoparty", "gumroad"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#urls": "https://kemono.su/data/6f/13/6f1394b19516396ea520254350662c254bbea30c1e111fd4b0f042c61c426d07.zip", }, - "attachments": [ - { + { + "#url": "https://kemono.party/gumroad/user/3252870377455/post/aJnAH", + "#comment": "username (#1548, #1652)", + "#category": ("", "kemonoparty", "gumroad"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#options": {"metadata": True}, + "username": "Kudalyn's Creations", + }, + { + "#url": "https://kemono.su/patreon/user/4158582/post/32099982", + "#comment": "allow duplicates (#2440)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#count": 2, + }, + { + "#url": "https://kemono.su/patreon/user/4158582/post/32099982", + "#comment": "allow duplicates (#2440)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#options": {"duplicates": True}, + "#count": 3, + }, + { + "#url": "https://kemono.su/patreon/user/34134344/post/38129255", + "#comment": "DMs (#2008)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#options": {"dms": True}, + "dms": [ + { + "body": r"re:Hi! Thank you very much for supporting the work I did in May. Here's your reward pack! I hope you find something you enjoy in it. :\)\n\nhttps://www.mediafire.com/file/\w+/Set13_tier_2.zip/file", + "date": "2021-06", + } + ], + }, + { + "#url": "https://kemono.su/patreon/user/3161935/post/68231671", + "#comment": "announcements", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#options": {"announcements": True}, + "announcements": [ + { + "body": "
          Thank you so much for the support!
          This Patreon is more of a tip jar for supporting what I make. I have to clarify that there are no exclusive Patreon animations because all are released for the public. You will get earlier access to WIPs. Direct downloads to my works are also available for $5 and $10 Tiers.
          ", + "date": "2023-02", + } + ], + }, + { + "#url": "https://kemono.su/patreon/user/19623797/post/29035449", + "#comment": "invalid file (#3510)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#pattern": r"907ba78b4545338d3539683e63ecb51cf51c10adc9dabd86e92bd52339f298b9\.txt", + "#sha1_content": "da39a3ee5e6b4b0d3255bfef95601890afd80709", + }, + { + "#url": "https://kemono.su/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class": kemonoparty.KemonopartyPostExtractor, + }, + { + "#url": "https://kemono.su/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class": kemonoparty.KemonopartyPostExtractor, + }, + { + "#url": "https://www.kemono.su/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class": kemonoparty.KemonopartyPostExtractor, + }, + { + "#url": "https://beta.kemono.su/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class": kemonoparty.KemonopartyPostExtractor, + }, + { + "#url": "https://kemono.su/patreon/user/3161935/post/68231671/revision/142470", + "#comment": "revisions (#4498)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#urls": "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", + "file": { "hash": "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", "name": "wip update.jpg", "path": "/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", - "type": "attachment", + "type": "file", }, - ], - "filename" : "wip update", - "extension" : "jpg", - "hash" : "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", - "revision_id" : 142470, - "revision_index": 2, - "revision_count": 9, - "revision_hash" : "e0e93281495e151b11636c156e52bfe9234c2a40", -}, - -{ - "#url" : "https://kemono.su/patreon/user/3161935/post/68231671", - "#comment" : "unique revisions (#5013)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#options" : {"revisions": "unique"}, - "#urls" : "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", - - "filename" : "wip update", - "hash" : "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", - "revision_id" : 0, - "revision_index": 1, - "revision_count": 1, - "revision_hash" : "e0e93281495e151b11636c156e52bfe9234c2a40", -}, - -{ - "#url" : "https://kemono.su/patreon/user/3161935/post/68231671/revisions", - "#comment" : "revisions (#4498)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#pattern" : r"https://kemono\.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86\.jpg", - "#count" : 9, - "#archive" : False, - - "revision_id": range(134996, 3052965), - "revision_index": range(1, 9), - "revision_count": 9, - "revision_hash": "e0e93281495e151b11636c156e52bfe9234c2a40", -}, - - -{ - "#url" : "https://kemono.su/patreon/user/3161935/post/68231671/revision/12345", - "#comment" : "revisions (#4498)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://kemono.su/patreon/user/6298789/post/69764693", - "#comment" : "'published' metadata with extra microsecond data", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - - "date" : "dt:2022-07-29 21:12:11", - "published": "2022-07-29T21:12:11.483000", -}, - -{ - "#url" : "https://kemono.su/gumroad/user/3267960360326/post/jwwag", - "#comment" : "empty 'file' with no 'path' (#5368)", - "#category": ("", "kemonoparty", "gumroad"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#count" : 8, - - "type" : "attachment", -}, - -{ - "#url" : "https://kemono.su/discord/server/488668827274444803#608504710906904576", - "#category": ("", "kemonoparty", "discord"), - "#class" : kemonoparty.KemonopartyDiscordExtractor, - "#count" : 4, - - "channel" : "608504710906904576", - "channel_name": "finish-work", -}, - -{ - "#url" : "https://kemono.su/discord/server/488668827274444803#finish-work", - "#category": ("", "kemonoparty", "discord"), - "#class" : kemonoparty.KemonopartyDiscordExtractor, - "#count" : 4, - - "channel" : "608504710906904576", - "channel_name": "finish-work", -}, - -{ - "#url" : "https://kemono.su/discord/server/488668827274444803/channel/608504710906904576#finish-work", - "#category": ("", "kemonoparty", "discord"), - "#class" : kemonoparty.KemonopartyDiscordExtractor, - "#count" : 4, - - "channel" : "608504710906904576", - "channel_name": "finish-work", - "date" : "type:datetime", -}, - -{ - "#url" : "https://kemono.su/discord/server/818188637329031199#818343747275456522", - "#comment" : "pagination", - "#category": ("", "kemonoparty", "discord"), - "#class" : kemonoparty.KemonopartyDiscordExtractor, - "#range" : "1-250", - "#count" : 250, - - "channel" : "818343747275456522", - "channel_name": "wraith-sfw-gallery", -}, - -{ - "#url" : "https://kemono.su/discord/server/256559665620451329/channel/462437519519383555#", - "#category": ("", "kemonoparty", "discord"), - "#class" : kemonoparty.KemonopartyDiscordExtractor, - "#pattern" : r"https://kemono\.su/data/(e3/77/e377e3525164559484ace2e64425b0cec1db08.*\.png|51/45/51453640a5e0a4d23fbf57fb85390f9c5ec154.*\.gif)", - "#count" : ">= 2", - - "hash": r"re:e377e3525164559484ace2e64425b0cec1db08|51453640a5e0a4d23fbf57fb85390f9c5ec154", -}, - -{ - "#url" : "https://kemono.su/discord/server/315262215055736843/channel/315262215055736843#general", - "#comment" : "'inline' files", - "#category": ("", "kemonoparty", "discord"), - "#class" : kemonoparty.KemonopartyDiscordExtractor, - "#options" : {"image-filter": "type == 'inline'"}, - "#pattern" : r"https://cdn\.discordapp\.com/attachments/\d+/\d+/.+$", - "#range" : "1-5", - - "hash": "", -}, - -{ - "#url" : "https://kemono.su/discord/server/488668827274444803", - "#category": ("", "kemonoparty", "discord-server"), - "#class" : kemonoparty.KemonopartyDiscordServerExtractor, - "#pattern" : kemonoparty.KemonopartyDiscordExtractor.pattern, - "#count" : 13, -}, - -{ - "#url" : "https://kemono.su/discord/server/488668827274444803", - "#category": ("", "kemonoparty", "discord-server"), - "#class" : kemonoparty.KemonopartyDiscordServerExtractor, - "#pattern" : kemonoparty.KemonopartyDiscordExtractor.pattern, - "#count" : 13, -}, - -{ - "#url" : "https://kemono.su/posts?q=foobar", - "#category": ("", "kemonoparty", "posts"), - "#class" : kemonoparty.KemonopartyPostsExtractor, - "#count" : range(60, 100), -}, - -{ - "#url" : "https://kemono.su/favorites", - "#category": ("", "kemonoparty", "favorite"), - "#class" : kemonoparty.KemonopartyFavoriteExtractor, - "#pattern" : kemonoparty.KemonopartyUserExtractor.pattern, - "#auth" : True, - "#urls" : ( - "https://kemono.su/patreon/user/881792", - "https://kemono.su/fanbox/user/6993449", - "https://kemono.su/subscribestar/user/alcorart", - ), -}, - -{ - "#url" : "https://kemono.su/favorites?type=artist&sort=faved_seq&order=asc", - "#category": ("", "kemonoparty", "favorite"), - "#class" : kemonoparty.KemonopartyFavoriteExtractor, - "#pattern" : kemonoparty.KemonopartyUserExtractor.pattern, - "#auth" : True, - "#urls" : ( - "https://kemono.su/fanbox/user/6993449", - "https://kemono.su/patreon/user/881792", - "https://kemono.su/subscribestar/user/alcorart", - ), -}, - -{ - "#url" : "https://kemono.su/favorites?type=post", - "#category": ("", "kemonoparty", "favorite"), - "#class" : kemonoparty.KemonopartyFavoriteExtractor, - "#pattern" : kemonoparty.KemonopartyPostExtractor.pattern, - "#auth" : True, - "#urls" : ( - "https://kemono.su/subscribestar/user/alcorart/post/184329", - "https://kemono.su/fanbox/user/6993449/post/23913", - "https://kemono.su/patreon/user/881792/post/4769638", - ), -}, - -{ - "#url" : "https://kemono.su/favorites?type=post&sort=published&order=asc", - "#category": ("", "kemonoparty", "favorite"), - "#class" : kemonoparty.KemonopartyFavoriteExtractor, - "#pattern" : kemonoparty.KemonopartyPostExtractor.pattern, - "#auth" : True, - "#urls" : ( - "https://kemono.su/patreon/user/881792/post/4769638", - "https://kemono.su/fanbox/user/6993449/post/23913", - "https://kemono.su/subscribestar/user/alcorart/post/184329", - ), -}, - + "attachments": [ + { + "hash": "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", + "name": "wip update.jpg", + "path": "/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", + "type": "attachment", + }, + ], + "filename": "wip update", + "extension": "jpg", + "hash": "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", + "revision_id": 142470, + "revision_index": 2, + "revision_count": 9, + "revision_hash": "e0e93281495e151b11636c156e52bfe9234c2a40", + }, + { + "#url": "https://kemono.su/patreon/user/3161935/post/68231671", + "#comment": "unique revisions (#5013)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#options": {"revisions": "unique"}, + "#urls": "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", + "filename": "wip update", + "hash": "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", + "revision_id": 0, + "revision_index": 1, + "revision_count": 1, + "revision_hash": "e0e93281495e151b11636c156e52bfe9234c2a40", + }, + { + "#url": "https://kemono.su/patreon/user/3161935/post/68231671/revisions", + "#comment": "revisions (#4498)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#pattern": r"https://kemono\.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86\.jpg", + "#count": 9, + "#archive": False, + "revision_id": range(134996, 3052965), + "revision_index": range(1, 9), + "revision_count": 9, + "revision_hash": "e0e93281495e151b11636c156e52bfe9234c2a40", + }, + { + "#url": "https://kemono.su/patreon/user/3161935/post/68231671/revision/12345", + "#comment": "revisions (#4498)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://kemono.su/patreon/user/6298789/post/69764693", + "#comment": "'published' metadata with extra microsecond data", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "date": "dt:2022-07-29 21:12:11", + "published": "2022-07-29T21:12:11.483000", + }, + { + "#url": "https://kemono.su/gumroad/user/3267960360326/post/jwwag", + "#comment": "empty 'file' with no 'path' (#5368)", + "#category": ("", "kemonoparty", "gumroad"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#count": 8, + "type": "attachment", + }, + { + "#url": "https://kemono.su/discord/server/488668827274444803#608504710906904576", + "#category": ("", "kemonoparty", "discord"), + "#class": kemonoparty.KemonopartyDiscordExtractor, + "#count": 4, + "channel": "608504710906904576", + "channel_name": "finish-work", + }, + { + "#url": "https://kemono.su/discord/server/488668827274444803#finish-work", + "#category": ("", "kemonoparty", "discord"), + "#class": kemonoparty.KemonopartyDiscordExtractor, + "#count": 4, + "channel": "608504710906904576", + "channel_name": "finish-work", + }, + { + "#url": "https://kemono.su/discord/server/488668827274444803/channel/608504710906904576#finish-work", + "#category": ("", "kemonoparty", "discord"), + "#class": kemonoparty.KemonopartyDiscordExtractor, + "#count": 4, + "channel": "608504710906904576", + "channel_name": "finish-work", + "date": "type:datetime", + }, + { + "#url": "https://kemono.su/discord/server/818188637329031199#818343747275456522", + "#comment": "pagination", + "#category": ("", "kemonoparty", "discord"), + "#class": kemonoparty.KemonopartyDiscordExtractor, + "#range": "1-250", + "#count": 250, + "channel": "818343747275456522", + "channel_name": "wraith-sfw-gallery", + }, + { + "#url": "https://kemono.su/discord/server/256559665620451329/channel/462437519519383555#", + "#category": ("", "kemonoparty", "discord"), + "#class": kemonoparty.KemonopartyDiscordExtractor, + "#pattern": r"https://kemono\.su/data/(e3/77/e377e3525164559484ace2e64425b0cec1db08.*\.png|51/45/51453640a5e0a4d23fbf57fb85390f9c5ec154.*\.gif)", + "#count": ">= 2", + "hash": r"re:e377e3525164559484ace2e64425b0cec1db08|51453640a5e0a4d23fbf57fb85390f9c5ec154", + }, + { + "#url": "https://kemono.su/discord/server/315262215055736843/channel/315262215055736843#general", + "#comment": "'inline' files", + "#category": ("", "kemonoparty", "discord"), + "#class": kemonoparty.KemonopartyDiscordExtractor, + "#options": {"image-filter": "type == 'inline'"}, + "#pattern": r"https://cdn\.discordapp\.com/attachments/\d+/\d+/.+$", + "#range": "1-5", + "hash": "", + }, + { + "#url": "https://kemono.su/discord/server/488668827274444803", + "#category": ("", "kemonoparty", "discord-server"), + "#class": kemonoparty.KemonopartyDiscordServerExtractor, + "#pattern": kemonoparty.KemonopartyDiscordExtractor.pattern, + "#count": 13, + }, + { + "#url": "https://kemono.su/discord/server/488668827274444803", + "#category": ("", "kemonoparty", "discord-server"), + "#class": kemonoparty.KemonopartyDiscordServerExtractor, + "#pattern": kemonoparty.KemonopartyDiscordExtractor.pattern, + "#count": 13, + }, + { + "#url": "https://kemono.su/posts?q=foobar", + "#category": ("", "kemonoparty", "posts"), + "#class": kemonoparty.KemonopartyPostsExtractor, + "#count": range(60, 100), + }, + { + "#url": "https://kemono.su/favorites", + "#category": ("", "kemonoparty", "favorite"), + "#class": kemonoparty.KemonopartyFavoriteExtractor, + "#pattern": kemonoparty.KemonopartyUserExtractor.pattern, + "#auth": True, + "#urls": ( + "https://kemono.su/patreon/user/881792", + "https://kemono.su/fanbox/user/6993449", + "https://kemono.su/subscribestar/user/alcorart", + ), + }, + { + "#url": "https://kemono.su/favorites?type=artist&sort=faved_seq&order=asc", + "#category": ("", "kemonoparty", "favorite"), + "#class": kemonoparty.KemonopartyFavoriteExtractor, + "#pattern": kemonoparty.KemonopartyUserExtractor.pattern, + "#auth": True, + "#urls": ( + "https://kemono.su/fanbox/user/6993449", + "https://kemono.su/patreon/user/881792", + "https://kemono.su/subscribestar/user/alcorart", + ), + }, + { + "#url": "https://kemono.su/favorites?type=post", + "#category": ("", "kemonoparty", "favorite"), + "#class": kemonoparty.KemonopartyFavoriteExtractor, + "#pattern": kemonoparty.KemonopartyPostExtractor.pattern, + "#auth": True, + "#urls": ( + "https://kemono.su/subscribestar/user/alcorart/post/184329", + "https://kemono.su/fanbox/user/6993449/post/23913", + "https://kemono.su/patreon/user/881792/post/4769638", + ), + }, + { + "#url": "https://kemono.su/favorites?type=post&sort=published&order=asc", + "#category": ("", "kemonoparty", "favorite"), + "#class": kemonoparty.KemonopartyFavoriteExtractor, + "#pattern": kemonoparty.KemonopartyPostExtractor.pattern, + "#auth": True, + "#urls": ( + "https://kemono.su/patreon/user/881792/post/4769638", + "https://kemono.su/fanbox/user/6993449/post/23913", + "https://kemono.su/subscribestar/user/alcorart/post/184329", + ), + }, ) diff --git a/test/results/khinsider.py b/test/results/khinsider.py index 7013069f2..f1057966a 100644 --- a/test/results/khinsider.py +++ b/test/results/khinsider.py @@ -1,30 +1,25 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import khinsider - __tests__ = ( -{ - "#url" : "https://downloads.khinsider.com/game-soundtracks/album/horizon-riders-wii", - "#category": ("", "khinsider", "soundtrack"), - "#class" : khinsider.KhinsiderSoundtrackExtractor, - "#pattern" : r"https?://(dl\.)?vgm(site|downloads)\.com/soundtracks/horizon-riders-wii/[^/]+/Horizon%20Riders%20Wii%20-%20Full%20Soundtrack\.mp3", - "#count" : 1, - - "album" : { - "count" : 1, - "date" : "Sep 18th, 2016", - "name" : "Horizon Riders", - "platform": "Wii", - "size" : 26214400, - "type" : "Gamerip", + { + "#url": "https://downloads.khinsider.com/game-soundtracks/album/horizon-riders-wii", + "#category": ("", "khinsider", "soundtrack"), + "#class": khinsider.KhinsiderSoundtrackExtractor, + "#pattern": r"https?://(dl\.)?vgm(site|downloads)\.com/soundtracks/horizon-riders-wii/[^/]+/Horizon%20Riders%20Wii%20-%20Full%20Soundtrack\.mp3", + "#count": 1, + "album": { + "count": 1, + "date": "Sep 18th, 2016", + "name": "Horizon Riders", + "platform": "Wii", + "size": 26214400, + "type": "Gamerip", + }, + "extension": "mp3", + "filename": "Horizon Riders Wii - Full Soundtrack", }, - "extension": "mp3", - "filename" : "Horizon Riders Wii - Full Soundtrack", -}, - ) diff --git a/test/results/koharu.py b/test/results/koharu.py index 0aed25f63..b5841d0c1 100644 --- a/test/results/koharu.py +++ b/test/results/koharu.py @@ -1,156 +1,144 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import koharu - __tests__ = ( -{ - "#url" : "https://niyaniya.moe/g/14216/6c67076fdd45", - "#category": ("", "koharu", "gallery"), - "#class" : koharu.KoharuGalleryExtractor, - "#options" : {"tags": True}, - "#pattern" : r"https://kisakisexo.xyz/download/59896/a4fbd1828229/f47639c6abaf1903dd69c36a3d961da84741a1831aa07a2906ce9c74156a5d75\?v=1721626410802&w=0", - "#count" : 1, - - "count" : 22, - "created_at": 1721626410802, - "date" : "dt:2024-07-22 05:33:30", - "extension" : "cbz", - "filename" : "f47639c6abaf1903dd69c36a3d961da84741a1831aa07a2906ce9c74156a5d75", - "id" : 14216, - "num" : 1, - "public_key": "6c67076fdd45", - "tags": [ - "general:beach", - "general:booty", - "general:dark skin", - "general:fingering", - "general:handjob", - "general:light hair", - "general:nakadashi", - "general:outdoors", - "general:ponytail", - "general:swimsuit", - "general:x-ray", - "artist:ouchi kaeru", - "magazine:comic kairakuten 2024-08", - "female:busty", - "language:english", - "language:translated", - "other:uncensored", - "other:vanilla", - ], - "tags_artist": [ - "ouchi kaeru", - ], - "tags_female": [ - "busty", - ], - "tags_general": [ - "beach", - "booty", - "dark skin", - "fingering", - "handjob", - "light hair", - "nakadashi", - "outdoors", - "ponytail", - "swimsuit", - "x-ray", - ], - "tags_language": [ - "english", - "translated", - ], - "tags_magazine": [ - "comic kairakuten 2024-08", - ], - "tags_other": [ - "uncensored", - "vanilla", - ], - "title" : "[Ouchi Kaeru] Summer Business (Comic Kairakuten 2024-08)", - "updated_at": 1721626410802, -}, - -{ - "#url" : "https://niyaniya.moe/g/14216/6c67076fdd45", - "#category": ("", "koharu", "gallery"), - "#class" : koharu.KoharuGalleryExtractor, - "#options" : {"cbz": False, "format": "780"}, - "#pattern" : r"https://koharusexo.xyz/data/59905/2df9110af7f1/a7cbeca3fb9c83aa87582a8a74cc8f8ce1b9e9b434dc1af293628871642f42df/[0-9a-f]+/.+", - "#count" : 22, -}, - -{ - "#url" : "https://niyaniya.moe/g/14216/6c67076fdd45", - "#category": ("", "koharu", "gallery"), - "#class" : koharu.KoharuGalleryExtractor, - "#options" : {"cbz": False, "format": "780"}, - "#range" : "1", - "#sha1_content": "08954e0ae18a900ee7ca144d1661c664468c2525", -}, - -{ - "#url" : "https://koharu.to/g/14216/6c67076fdd45", - "#class": koharu.KoharuGalleryExtractor, -}, -{ - "#url" : "https://anchira.to/g/14216/6c67076fdd45", - "#class": koharu.KoharuGalleryExtractor, -}, -{ - "#url" : "https://seia.to/g/14216/6c67076fdd45", - "#class": koharu.KoharuGalleryExtractor, -}, -{ - "#url" : "https://shupogaki.moe/g/14216/6c67076fdd45", - "#class": koharu.KoharuGalleryExtractor, -}, -{ - "#url" : "https://hoshino.one/g/14216/6c67076fdd45", - "#class": koharu.KoharuGalleryExtractor, -}, - -{ - "#url" : "https://niyaniya.moe/reader/14216/6c67076fdd45", - "#category": ("", "koharu", "gallery"), - "#class" : koharu.KoharuGalleryExtractor, -}, - -{ - "#url" : "https://niyaniya.moe/?s=tag:^beach$", - "#category": ("", "koharu", "search"), - "#class" : koharu.KoharuSearchExtractor, - "#pattern" : koharu.KoharuGalleryExtractor.pattern, - "#count" : ">= 50", -}, - -{ - "#url" : "https://niyaniya.moe/favorites", - "#category": ("", "koharu", "favorite"), - "#class" : koharu.KoharuFavoriteExtractor, - "#pattern" : koharu.KoharuGalleryExtractor.pattern, - "#auth" : True, - "#urls" : [ - "https://niyaniya.moe/g/14216/6c67076fdd45", - ], -}, - -{ - "#url" : "https://niyaniya.moe/favorites?cat=6&sort=4", - "#category": ("", "koharu", "favorite"), - "#class" : koharu.KoharuFavoriteExtractor, - "#pattern" : koharu.KoharuGalleryExtractor.pattern, - "#auth" : True, - "#urls" : [ - "https://niyaniya.moe/g/14216/6c67076fdd45", - ], -}, - + { + "#url": "https://niyaniya.moe/g/14216/6c67076fdd45", + "#category": ("", "koharu", "gallery"), + "#class": koharu.KoharuGalleryExtractor, + "#options": {"tags": True}, + "#pattern": r"https://kisakisexo.xyz/download/59896/a4fbd1828229/f47639c6abaf1903dd69c36a3d961da84741a1831aa07a2906ce9c74156a5d75\?v=1721626410802&w=0", + "#count": 1, + "count": 22, + "created_at": 1721626410802, + "date": "dt:2024-07-22 05:33:30", + "extension": "cbz", + "filename": "f47639c6abaf1903dd69c36a3d961da84741a1831aa07a2906ce9c74156a5d75", + "id": 14216, + "num": 1, + "public_key": "6c67076fdd45", + "tags": [ + "general:beach", + "general:booty", + "general:dark skin", + "general:fingering", + "general:handjob", + "general:light hair", + "general:nakadashi", + "general:outdoors", + "general:ponytail", + "general:swimsuit", + "general:x-ray", + "artist:ouchi kaeru", + "magazine:comic kairakuten 2024-08", + "female:busty", + "language:english", + "language:translated", + "other:uncensored", + "other:vanilla", + ], + "tags_artist": [ + "ouchi kaeru", + ], + "tags_female": [ + "busty", + ], + "tags_general": [ + "beach", + "booty", + "dark skin", + "fingering", + "handjob", + "light hair", + "nakadashi", + "outdoors", + "ponytail", + "swimsuit", + "x-ray", + ], + "tags_language": [ + "english", + "translated", + ], + "tags_magazine": [ + "comic kairakuten 2024-08", + ], + "tags_other": [ + "uncensored", + "vanilla", + ], + "title": "[Ouchi Kaeru] Summer Business (Comic Kairakuten 2024-08)", + "updated_at": 1721626410802, + }, + { + "#url": "https://niyaniya.moe/g/14216/6c67076fdd45", + "#category": ("", "koharu", "gallery"), + "#class": koharu.KoharuGalleryExtractor, + "#options": {"cbz": False, "format": "780"}, + "#pattern": r"https://koharusexo.xyz/data/59905/2df9110af7f1/a7cbeca3fb9c83aa87582a8a74cc8f8ce1b9e9b434dc1af293628871642f42df/[0-9a-f]+/.+", + "#count": 22, + }, + { + "#url": "https://niyaniya.moe/g/14216/6c67076fdd45", + "#category": ("", "koharu", "gallery"), + "#class": koharu.KoharuGalleryExtractor, + "#options": {"cbz": False, "format": "780"}, + "#range": "1", + "#sha1_content": "08954e0ae18a900ee7ca144d1661c664468c2525", + }, + { + "#url": "https://koharu.to/g/14216/6c67076fdd45", + "#class": koharu.KoharuGalleryExtractor, + }, + { + "#url": "https://anchira.to/g/14216/6c67076fdd45", + "#class": koharu.KoharuGalleryExtractor, + }, + { + "#url": "https://seia.to/g/14216/6c67076fdd45", + "#class": koharu.KoharuGalleryExtractor, + }, + { + "#url": "https://shupogaki.moe/g/14216/6c67076fdd45", + "#class": koharu.KoharuGalleryExtractor, + }, + { + "#url": "https://hoshino.one/g/14216/6c67076fdd45", + "#class": koharu.KoharuGalleryExtractor, + }, + { + "#url": "https://niyaniya.moe/reader/14216/6c67076fdd45", + "#category": ("", "koharu", "gallery"), + "#class": koharu.KoharuGalleryExtractor, + }, + { + "#url": "https://niyaniya.moe/?s=tag:^beach$", + "#category": ("", "koharu", "search"), + "#class": koharu.KoharuSearchExtractor, + "#pattern": koharu.KoharuGalleryExtractor.pattern, + "#count": ">= 50", + }, + { + "#url": "https://niyaniya.moe/favorites", + "#category": ("", "koharu", "favorite"), + "#class": koharu.KoharuFavoriteExtractor, + "#pattern": koharu.KoharuGalleryExtractor.pattern, + "#auth": True, + "#urls": [ + "https://niyaniya.moe/g/14216/6c67076fdd45", + ], + }, + { + "#url": "https://niyaniya.moe/favorites?cat=6&sort=4", + "#category": ("", "koharu", "favorite"), + "#class": koharu.KoharuFavoriteExtractor, + "#pattern": koharu.KoharuGalleryExtractor.pattern, + "#auth": True, + "#urls": [ + "https://niyaniya.moe/g/14216/6c67076fdd45", + ], + }, ) diff --git a/test/results/kohlchan.py b/test/results/kohlchan.py index 479ed7154..4314994a4 100644 --- a/test/results/kohlchan.py +++ b/test/results/kohlchan.py @@ -1,39 +1,32 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import lynxchan - __tests__ = ( -{ - "#url" : "https://kohlchan.net/a/res/4594.html", - "#category": ("lynxchan", "kohlchan", "thread"), - "#class" : lynxchan.LynxchanThreadExtractor, - "#pattern" : r"https://kohlchan\.net/\.media/[0-9a-f]{64}(\.\w+)?$", - "#count" : ">= 80", -}, - -{ - "#url" : "https://kohlchan.net/a/", - "#category": ("lynxchan", "kohlchan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, - "#pattern" : lynxchan.LynxchanThreadExtractor.pattern, - "#count" : ">= 100", -}, - -{ - "#url" : "https://kohlchan.net/a/2.html", - "#category": ("lynxchan", "kohlchan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, -}, - -{ - "#url" : "https://kohlchan.net/a/catalog.html", - "#category": ("lynxchan", "kohlchan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, -}, - + { + "#url": "https://kohlchan.net/a/res/4594.html", + "#category": ("lynxchan", "kohlchan", "thread"), + "#class": lynxchan.LynxchanThreadExtractor, + "#pattern": r"https://kohlchan\.net/\.media/[0-9a-f]{64}(\.\w+)?$", + "#count": ">= 80", + }, + { + "#url": "https://kohlchan.net/a/", + "#category": ("lynxchan", "kohlchan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + "#pattern": lynxchan.LynxchanThreadExtractor.pattern, + "#count": ">= 100", + }, + { + "#url": "https://kohlchan.net/a/2.html", + "#category": ("lynxchan", "kohlchan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + }, + { + "#url": "https://kohlchan.net/a/catalog.html", + "#category": ("lynxchan", "kohlchan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + }, ) diff --git a/test/results/komikcast.py b/test/results/komikcast.py index 0bd76121f..fdc97f6aa 100644 --- a/test/results/komikcast.py +++ b/test/results/komikcast.py @@ -1,85 +1,66 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import komikcast - __tests__ = ( -{ - "#url" : "https://komikcast.lol/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", - "#category": ("", "komikcast", "chapter"), - "#class" : komikcast.KomikcastChapterExtractor, - "#pattern" : r"https://svr\d+\.imgkc\d+\.my\.id/wp-content/img/A/Apotheosis/002-2/\d{3}\.jpg", - "#count" : 18, - - "chapter" : 2, - "chapter_minor": ".2", - "count" : 18, - "extension": "jpg", - "filename" : r"re:0\d{2}", - "lang" : "id", - "language" : "Indonesian", - "manga" : "Apotheosis", - "page" : range(1, 18), - "title" : "", -}, - -{ - "#url" : "https://komikcast.site/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", - "#category": ("", "komikcast", "chapter"), - "#class" : komikcast.KomikcastChapterExtractor, -}, - -{ - "#url" : "https://komikcast.me/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", - "#category": ("", "komikcast", "chapter"), - "#class" : komikcast.KomikcastChapterExtractor, -}, - -{ - "#url" : "https://komikcast.com/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", - "#category": ("", "komikcast", "chapter"), - "#class" : komikcast.KomikcastChapterExtractor, -}, - -{ - "#url" : "https://komikcast.me/chapter/soul-land-ii-chapter-300-1-bahasa-indonesia/", - "#category": ("", "komikcast", "chapter"), - "#class" : komikcast.KomikcastChapterExtractor, - "#pattern" : r"https://svr\d\.imgkc\d*\.my\.id/wp-content/img/S/Soul_Land_II/300\.1/\d\d\.jpg", - "#count" : 9, - "#sha1_metadata": "cb646cfed3d45105bd645ab38b2e9f7d8c436436", -}, - -{ - "#url" : "https://komikcast.site/komik/090-eko-to-issho/", - "#category": ("", "komikcast", "manga"), - "#class" : komikcast.KomikcastMangaExtractor, - "#pattern" : komikcast.KomikcastChapterExtractor.pattern, - "#count" : 12, - - "author" : "Asakura Maru", - "chapter": range(1, 12), - "chapter_minor": "", - "genres" : [ - "Comedy", - "Drama", - "Romance", - "School Life", - "Sci-Fi", - "Shounen" - ], - "manga" : "090 Eko to Issho", - "type" : "Manga", -}, - -{ - "#url" : "https://komikcast.me/tonari-no-kashiwagi-san/", - "#category": ("", "komikcast", "manga"), - "#class" : komikcast.KomikcastMangaExtractor, -}, - + { + "#url": "https://komikcast.lol/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class": komikcast.KomikcastChapterExtractor, + "#pattern": r"https://svr\d+\.imgkc\d+\.my\.id/wp-content/img/A/Apotheosis/002-2/\d{3}\.jpg", + "#count": 18, + "chapter": 2, + "chapter_minor": ".2", + "count": 18, + "extension": "jpg", + "filename": r"re:0\d{2}", + "lang": "id", + "language": "Indonesian", + "manga": "Apotheosis", + "page": range(1, 18), + "title": "", + }, + { + "#url": "https://komikcast.site/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class": komikcast.KomikcastChapterExtractor, + }, + { + "#url": "https://komikcast.me/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class": komikcast.KomikcastChapterExtractor, + }, + { + "#url": "https://komikcast.com/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class": komikcast.KomikcastChapterExtractor, + }, + { + "#url": "https://komikcast.me/chapter/soul-land-ii-chapter-300-1-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class": komikcast.KomikcastChapterExtractor, + "#pattern": r"https://svr\d\.imgkc\d*\.my\.id/wp-content/img/S/Soul_Land_II/300\.1/\d\d\.jpg", + "#count": 9, + "#sha1_metadata": "cb646cfed3d45105bd645ab38b2e9f7d8c436436", + }, + { + "#url": "https://komikcast.site/komik/090-eko-to-issho/", + "#category": ("", "komikcast", "manga"), + "#class": komikcast.KomikcastMangaExtractor, + "#pattern": komikcast.KomikcastChapterExtractor.pattern, + "#count": 12, + "author": "Asakura Maru", + "chapter": range(1, 12), + "chapter_minor": "", + "genres": ["Comedy", "Drama", "Romance", "School Life", "Sci-Fi", "Shounen"], + "manga": "090 Eko to Issho", + "type": "Manga", + }, + { + "#url": "https://komikcast.me/tonari-no-kashiwagi-san/", + "#category": ("", "komikcast", "manga"), + "#class": komikcast.KomikcastMangaExtractor, + }, ) diff --git a/test/results/konachan.py b/test/results/konachan.py index ed4d2d78a..2d51e6d90 100644 --- a/test/results/konachan.py +++ b/test/results/konachan.py @@ -1,92 +1,77 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import moebooru from gallery_dl import exception - +from gallery_dl.extractor import moebooru __tests__ = ( -{ - "#url" : "https://konachan.com/post/show/205189", - "#category": ("moebooru", "konachan", "post"), - "#class" : moebooru.MoebooruPostExtractor, - "#options" : {"tags": True}, - "#sha1_content": "674e75a753df82f5ad80803f575818b8e46e4b65", - - "tags_artist" : "patata", - "tags_character": "clownpiece", - "tags_copyright": "touhou", - "tags_general" : str, -}, - -{ - "#url" : "https://konachan.net/post/show/205189", - "#category": ("moebooru", "konachan", "post"), - "#class" : moebooru.MoebooruPostExtractor, -}, - -{ - "#url" : "https://konachan.com/post?tags=patata", - "#category": ("moebooru", "konachan", "tag"), - "#class" : moebooru.MoebooruTagExtractor, - "#sha1_content": "838cfb815e31f48160855435655ddf7bfc4ecb8d", -}, - -{ - "#url" : "https://konachan.com/post?tags=", - "#comment" : "empty 'tags' (#4354)", - "#category": ("moebooru", "konachan", "tag"), - "#class" : moebooru.MoebooruTagExtractor, -}, - -{ - "#url" : "https://konachan.net/post?tags=patata", - "#category": ("moebooru", "konachan", "tag"), - "#class" : moebooru.MoebooruTagExtractor, -}, - -{ - "#url" : "https://konachan.com/pool/show/95", - "#category": ("moebooru", "konachan", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, - "#sha1_content": "cf0546e38a93c2c510a478f8744e60687b7a8426", -}, - -{ - "#url" : "https://konachan.com/pool/show/95", - "#comment" : "'metadata' option (#4646)", - "#category": ("moebooru", "konachan", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, - "#options" : {"metadata": True}, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://konachan.net/pool/show/95", - "#category": ("moebooru", "konachan", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, -}, - -{ - "#url" : "https://konachan.com/post/popular_by_month?month=11&year=2010", - "#category": ("moebooru", "konachan", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, - "#count" : 20, -}, - -{ - "#url" : "https://konachan.com/post/popular_recent", - "#category": ("moebooru", "konachan", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, -}, - -{ - "#url" : "https://konachan.net/post/popular_recent", - "#category": ("moebooru", "konachan", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, -}, - + { + "#url": "https://konachan.com/post/show/205189", + "#category": ("moebooru", "konachan", "post"), + "#class": moebooru.MoebooruPostExtractor, + "#options": {"tags": True}, + "#sha1_content": "674e75a753df82f5ad80803f575818b8e46e4b65", + "tags_artist": "patata", + "tags_character": "clownpiece", + "tags_copyright": "touhou", + "tags_general": str, + }, + { + "#url": "https://konachan.net/post/show/205189", + "#category": ("moebooru", "konachan", "post"), + "#class": moebooru.MoebooruPostExtractor, + }, + { + "#url": "https://konachan.com/post?tags=patata", + "#category": ("moebooru", "konachan", "tag"), + "#class": moebooru.MoebooruTagExtractor, + "#sha1_content": "838cfb815e31f48160855435655ddf7bfc4ecb8d", + }, + { + "#url": "https://konachan.com/post?tags=", + "#comment": "empty 'tags' (#4354)", + "#category": ("moebooru", "konachan", "tag"), + "#class": moebooru.MoebooruTagExtractor, + }, + { + "#url": "https://konachan.net/post?tags=patata", + "#category": ("moebooru", "konachan", "tag"), + "#class": moebooru.MoebooruTagExtractor, + }, + { + "#url": "https://konachan.com/pool/show/95", + "#category": ("moebooru", "konachan", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + "#sha1_content": "cf0546e38a93c2c510a478f8744e60687b7a8426", + }, + { + "#url": "https://konachan.com/pool/show/95", + "#comment": "'metadata' option (#4646)", + "#category": ("moebooru", "konachan", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + "#options": {"metadata": True}, + "#exception": exception.HttpError, + }, + { + "#url": "https://konachan.net/pool/show/95", + "#category": ("moebooru", "konachan", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + }, + { + "#url": "https://konachan.com/post/popular_by_month?month=11&year=2010", + "#category": ("moebooru", "konachan", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + "#count": 20, + }, + { + "#url": "https://konachan.com/post/popular_recent", + "#category": ("moebooru", "konachan", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + }, + { + "#url": "https://konachan.net/post/popular_recent", + "#category": ("moebooru", "konachan", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + }, ) diff --git a/test/results/lensdump.py b/test/results/lensdump.py index 91f633c03..6f528392c 100644 --- a/test/results/lensdump.py +++ b/test/results/lensdump.py @@ -1,101 +1,86 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import lensdump - __tests__ = ( -{ - "#url" : "https://lensdump.com/a/1IhJr", - "#class" : lensdump.LensdumpAlbumExtractor, - "#pattern" : r"https://[abcd]\.l3n\.co/i/tq\w{4}\.png", - - "extension": "png", - "name" : str, - "num" : int, - "title" : str, - "url" : str, - "width" : int, -}, - -{ - "#url" : "https://lensdump.com/a/tA4lA", - "#comment" : "2 pages", - "#class" : lensdump.LensdumpAlbumExtractor, - "#pattern" : r"https://[abcd]\.l3n\.co/i/\w{6}\.(jpe?g|png)", - "#count" : 64, -}, - -{ - "#url" : "https://lensdump.com/vstar925", - "#class" : lensdump.LensdumpAlbumsExtractor, - "#urls" : ( - "https://lensdump.com/a/tX1uA", - "https://lensdump.com/a/R0gfK", - "https://lensdump.com/a/RSOMv", - "https://lensdump.com/a/9TbdT", - ), -}, - -{ - "#url" : "https://lensdump.com/vstar925/?sort=likes_desc&page=1", - "#comment" : "custom sort order", - "#class" : lensdump.LensdumpAlbumsExtractor, - "#urls" : ( - "https://lensdump.com/a/9TbdT", - "https://lensdump.com/a/RSOMv", - "https://lensdump.com/a/R0gfK", - "https://lensdump.com/a/tX1uA", - ), -}, - -{ - "#url" : "https://lensdump.com/vstar925/albums", - "#class" : lensdump.LensdumpAlbumsExtractor, -}, - -{ - "#url" : "https://lensdump.com/i/tyoAyM", - "#class" : lensdump.LensdumpImageExtractor, - "#urls" : "https://c.l3n.co/i/tyoAyM.webp", - "#sha1_content": "1aa749ed2c0cf679ec8e1df60068edaf3875de46", - - "date" : "dt:2022-08-01 08:24:28", - "extension": "webp", - "filename" : "tyoAyM", - "height" : 400, - "id" : "tyoAyM", - "title" : "MYOBI clovis bookcaseset", - "url" : "https://c.l3n.co/i/tyoAyM.webp", - "width" : 620, -}, - -{ - "#url" : "https://c.l3n.co/i/tyoAyM.webp", - "#class" : lensdump.LensdumpImageExtractor, - "#urls" : "https://c.l3n.co/i/tyoAyM.webp", - - "date" : "dt:2022-08-01 08:24:28", - "extension": "webp", - "filename" : "tyoAyM", - "height" : 400, - "id" : "tyoAyM", - "title" : "MYOBI clovis bookcaseset", - "url" : "https://c.l3n.co/i/tyoAyM.webp", - "width" : 620, -}, - -{ - "#url" : "https://i.lensdump.com/i/tyoAyM", - "#class" : lensdump.LensdumpImageExtractor, -}, - -{ - "#url" : "https://i3.lensdump.com/i/tyoAyM", - "#class" : lensdump.LensdumpImageExtractor, -}, - + { + "#url": "https://lensdump.com/a/1IhJr", + "#class": lensdump.LensdumpAlbumExtractor, + "#pattern": r"https://[abcd]\.l3n\.co/i/tq\w{4}\.png", + "extension": "png", + "name": str, + "num": int, + "title": str, + "url": str, + "width": int, + }, + { + "#url": "https://lensdump.com/a/tA4lA", + "#comment": "2 pages", + "#class": lensdump.LensdumpAlbumExtractor, + "#pattern": r"https://[abcd]\.l3n\.co/i/\w{6}\.(jpe?g|png)", + "#count": 64, + }, + { + "#url": "https://lensdump.com/vstar925", + "#class": lensdump.LensdumpAlbumsExtractor, + "#urls": ( + "https://lensdump.com/a/tX1uA", + "https://lensdump.com/a/R0gfK", + "https://lensdump.com/a/RSOMv", + "https://lensdump.com/a/9TbdT", + ), + }, + { + "#url": "https://lensdump.com/vstar925/?sort=likes_desc&page=1", + "#comment": "custom sort order", + "#class": lensdump.LensdumpAlbumsExtractor, + "#urls": ( + "https://lensdump.com/a/9TbdT", + "https://lensdump.com/a/RSOMv", + "https://lensdump.com/a/R0gfK", + "https://lensdump.com/a/tX1uA", + ), + }, + { + "#url": "https://lensdump.com/vstar925/albums", + "#class": lensdump.LensdumpAlbumsExtractor, + }, + { + "#url": "https://lensdump.com/i/tyoAyM", + "#class": lensdump.LensdumpImageExtractor, + "#urls": "https://c.l3n.co/i/tyoAyM.webp", + "#sha1_content": "1aa749ed2c0cf679ec8e1df60068edaf3875de46", + "date": "dt:2022-08-01 08:24:28", + "extension": "webp", + "filename": "tyoAyM", + "height": 400, + "id": "tyoAyM", + "title": "MYOBI clovis bookcaseset", + "url": "https://c.l3n.co/i/tyoAyM.webp", + "width": 620, + }, + { + "#url": "https://c.l3n.co/i/tyoAyM.webp", + "#class": lensdump.LensdumpImageExtractor, + "#urls": "https://c.l3n.co/i/tyoAyM.webp", + "date": "dt:2022-08-01 08:24:28", + "extension": "webp", + "filename": "tyoAyM", + "height": 400, + "id": "tyoAyM", + "title": "MYOBI clovis bookcaseset", + "url": "https://c.l3n.co/i/tyoAyM.webp", + "width": 620, + }, + { + "#url": "https://i.lensdump.com/i/tyoAyM", + "#class": lensdump.LensdumpImageExtractor, + }, + { + "#url": "https://i3.lensdump.com/i/tyoAyM", + "#class": lensdump.LensdumpImageExtractor, + }, ) diff --git a/test/results/lesbianenergy.py b/test/results/lesbianenergy.py index 650671f96..feef42c96 100644 --- a/test/results/lesbianenergy.py +++ b/test/results/lesbianenergy.py @@ -1,45 +1,37 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import misskey - __tests__ = ( -{ - "#url" : "https://lesbian.energy/@rerorero", - "#category": ("misskey", "lesbian.energy", "user"), - "#class" : misskey.MisskeyUserExtractor, - "#pattern" : r"https://(lesbian.energy/files/\w+|.+/media_attachments/files/.+)", - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://lesbian.energy/@nano@mk.yopo.work", - "#category": ("misskey", "lesbian.energy", "user"), - "#class" : misskey.MisskeyUserExtractor, -}, - -{ - "#url" : "https://lesbian.energy/notes/995ig09wqy", - "#category": ("misskey", "lesbian.energy", "note"), - "#class" : misskey.MisskeyNoteExtractor, - "#count" : 1, -}, - -{ - "#url" : "https://lesbian.energy/notes/96ynd9w5kc", - "#category": ("misskey", "lesbian.energy", "note"), - "#class" : misskey.MisskeyNoteExtractor, -}, - -{ - "#url" : "https://lesbian.energy/my/favorites", - "#category": ("misskey", "lesbian.energy", "favorite"), - "#class" : misskey.MisskeyFavoriteExtractor, -}, - + { + "#url": "https://lesbian.energy/@rerorero", + "#category": ("misskey", "lesbian.energy", "user"), + "#class": misskey.MisskeyUserExtractor, + "#pattern": r"https://(lesbian.energy/files/\w+|.+/media_attachments/files/.+)", + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://lesbian.energy/@nano@mk.yopo.work", + "#category": ("misskey", "lesbian.energy", "user"), + "#class": misskey.MisskeyUserExtractor, + }, + { + "#url": "https://lesbian.energy/notes/995ig09wqy", + "#category": ("misskey", "lesbian.energy", "note"), + "#class": misskey.MisskeyNoteExtractor, + "#count": 1, + }, + { + "#url": "https://lesbian.energy/notes/96ynd9w5kc", + "#category": ("misskey", "lesbian.energy", "note"), + "#class": misskey.MisskeyNoteExtractor, + }, + { + "#url": "https://lesbian.energy/my/favorites", + "#category": ("misskey", "lesbian.energy", "favorite"), + "#class": misskey.MisskeyFavoriteExtractor, + }, ) diff --git a/test/results/lexica.py b/test/results/lexica.py index 07183f675..4f26cdeb9 100644 --- a/test/results/lexica.py +++ b/test/results/lexica.py @@ -1,42 +1,37 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import lexica - __tests__ = ( -{ - "#url" : "https://lexica.art/?q=tree", - "#category": ("", "lexica", "search"), - "#class" : lexica.LexicaSearchExtractor, - "#pattern" : r"https://lexica-serve-encoded-images2\.sharif\.workers.dev/full_jpg/[0-9a-f-]{36}$", - "#range" : "1-80", - "#count" : 80, - - "height" : int, - "id" : str, - "upscaled_height": int, - "upscaled_width" : int, - "userid" : str, - "width" : int, - "prompt" : { - "c" : int, - "grid" : bool, - "height" : int, - "id" : str, - "images" : list, - "initImage" : None, - "initImageStrength": None, - "model" : "lexica-aperture-v2", - "negativePrompt" : str, - "prompt" : str, - "seed" : str, - "timestamp" : r"re:\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\dZ", - "width" : int, + { + "#url": "https://lexica.art/?q=tree", + "#category": ("", "lexica", "search"), + "#class": lexica.LexicaSearchExtractor, + "#pattern": r"https://lexica-serve-encoded-images2\.sharif\.workers.dev/full_jpg/[0-9a-f-]{36}$", + "#range": "1-80", + "#count": 80, + "height": int, + "id": str, + "upscaled_height": int, + "upscaled_width": int, + "userid": str, + "width": int, + "prompt": { + "c": int, + "grid": bool, + "height": int, + "id": str, + "images": list, + "initImage": None, + "initImageStrength": None, + "model": "lexica-aperture-v2", + "negativePrompt": str, + "prompt": str, + "seed": str, + "timestamp": r"re:\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\dZ", + "width": int, + }, }, -}, - ) diff --git a/test/results/lightroom.py b/test/results/lightroom.py index 4c1a05326..97e1da03d 100644 --- a/test/results/lightroom.py +++ b/test/results/lightroom.py @@ -1,31 +1,24 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import lightroom - __tests__ = ( -{ - "#url" : "https://lightroom.adobe.com/shares/0c9cce2033f24d24975423fe616368bf", - "#category": ("", "lightroom", "gallery"), - "#class" : lightroom.LightroomGalleryExtractor, - "#count" : ">= 55", - - "title": "Sterne und Nachtphotos", - "user" : "Christian Schrang", -}, - -{ - "#url" : "https://lightroom.adobe.com/shares/7ba68ad5a97e48608d2e6c57e6082813", - "#category": ("", "lightroom", "gallery"), - "#class" : lightroom.LightroomGalleryExtractor, - "#count" : ">= 180", - - "title": "HEBFC Snr/Res v Brighton", - "user" : "", -}, - + { + "#url": "https://lightroom.adobe.com/shares/0c9cce2033f24d24975423fe616368bf", + "#category": ("", "lightroom", "gallery"), + "#class": lightroom.LightroomGalleryExtractor, + "#count": ">= 55", + "title": "Sterne und Nachtphotos", + "user": "Christian Schrang", + }, + { + "#url": "https://lightroom.adobe.com/shares/7ba68ad5a97e48608d2e6c57e6082813", + "#category": ("", "lightroom", "gallery"), + "#class": lightroom.LightroomGalleryExtractor, + "#count": ">= 180", + "title": "HEBFC Snr/Res v Brighton", + "user": "", + }, ) diff --git a/test/results/livedoor.py b/test/results/livedoor.py index 7ab366f9c..c5c4335bf 100644 --- a/test/results/livedoor.py +++ b/test/results/livedoor.py @@ -1,66 +1,57 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import livedoor - __tests__ = ( -{ - "#url" : "http://blog.livedoor.jp/zatsu_ke/", - "#category": ("", "livedoor", "blog"), - "#class" : livedoor.LivedoorBlogExtractor, - "#pattern" : r"https?://livedoor.blogimg.jp/\w+/imgs/\w/\w/\w+\.\w+", - "#range" : "1-50", - "#count" : 50, - "#archive" : False, - - "post" : { - "categories" : tuple, - "date" : "type:datetime", - "description": str, - "id" : int, - "tags" : list, - "title" : str, - "user" : "zatsu_ke", + { + "#url": "http://blog.livedoor.jp/zatsu_ke/", + "#category": ("", "livedoor", "blog"), + "#class": livedoor.LivedoorBlogExtractor, + "#pattern": r"https?://livedoor.blogimg.jp/\w+/imgs/\w/\w/\w+\.\w+", + "#range": "1-50", + "#count": 50, + "#archive": False, + "post": { + "categories": tuple, + "date": "type:datetime", + "description": str, + "id": int, + "tags": list, + "title": str, + "user": "zatsu_ke", + }, + "filename": str, + "hash": r"re:\w{4,}", + "num": int, + }, + { + "#url": "http://blog.livedoor.jp/uotapo/", + "#category": ("", "livedoor", "blog"), + "#class": livedoor.LivedoorBlogExtractor, + "#range": "1-5", + "#count": 5, + }, + { + "#url": "http://blog.livedoor.jp/zatsu_ke/archives/51493859.html", + "#category": ("", "livedoor", "post"), + "#class": livedoor.LivedoorPostExtractor, + "#sha1_url": "9ca3bbba62722c8155be79ad7fc47be409e4a7a2", + "#sha1_metadata": "1f5b558492e0734f638b760f70bfc0b65c5a97b9", + }, + { + "#url": "http://blog.livedoor.jp/amaumauma/archives/7835811.html", + "#category": ("", "livedoor", "post"), + "#class": livedoor.LivedoorPostExtractor, + "#sha1_url": "204bbd6a9db4969c50e0923855aeede04f2e4a62", + "#sha1_metadata": "05821c7141360e6057ef2d382b046f28326a799d", + }, + { + "#url": "http://blog.livedoor.jp/uotapo/archives/1050616939.html", + "#category": ("", "livedoor", "post"), + "#class": livedoor.LivedoorPostExtractor, + "#sha1_url": "4b5ab144b7309eb870d9c08f8853d1abee9946d2", + "#sha1_metadata": "84fbf6e4eef16675013d6333039a7cfcb22c2d50", }, - "filename": str, - "hash" : r"re:\w{4,}", - "num" : int, -}, - -{ - "#url" : "http://blog.livedoor.jp/uotapo/", - "#category": ("", "livedoor", "blog"), - "#class" : livedoor.LivedoorBlogExtractor, - "#range" : "1-5", - "#count" : 5, -}, - -{ - "#url" : "http://blog.livedoor.jp/zatsu_ke/archives/51493859.html", - "#category": ("", "livedoor", "post"), - "#class" : livedoor.LivedoorPostExtractor, - "#sha1_url" : "9ca3bbba62722c8155be79ad7fc47be409e4a7a2", - "#sha1_metadata": "1f5b558492e0734f638b760f70bfc0b65c5a97b9", -}, - -{ - "#url" : "http://blog.livedoor.jp/amaumauma/archives/7835811.html", - "#category": ("", "livedoor", "post"), - "#class" : livedoor.LivedoorPostExtractor, - "#sha1_url" : "204bbd6a9db4969c50e0923855aeede04f2e4a62", - "#sha1_metadata": "05821c7141360e6057ef2d382b046f28326a799d", -}, - -{ - "#url" : "http://blog.livedoor.jp/uotapo/archives/1050616939.html", - "#category": ("", "livedoor", "post"), - "#class" : livedoor.LivedoorPostExtractor, - "#sha1_url" : "4b5ab144b7309eb870d9c08f8853d1abee9946d2", - "#sha1_metadata": "84fbf6e4eef16675013d6333039a7cfcb22c2d50", -}, - ) diff --git a/test/results/lolibooru.py b/test/results/lolibooru.py index f8750269b..f03ccf1c3 100644 --- a/test/results/lolibooru.py +++ b/test/results/lolibooru.py @@ -1,45 +1,36 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import moebooru - __tests__ = ( -{ - "#url" : "https://lolibooru.moe/post/show/281305/", - "#category": ("moebooru", "lolibooru", "post"), - "#class" : moebooru.MoebooruPostExtractor, - "#options" : {"notes": True}, - "#sha1_content": "a331430223ffc5b23c31649102e7d49f52489b57", - - "notes": list, -}, - -{ - "#url" : "https://lolibooru.moe/post/show/287835", - "#category": ("moebooru", "lolibooru", "post"), - "#class" : moebooru.MoebooruPostExtractor, -}, - -{ - "#url" : "https://lolibooru.moe/post?tags=ruu_%28tksymkw%29", - "#category": ("moebooru", "lolibooru", "tag"), - "#class" : moebooru.MoebooruTagExtractor, -}, - -{ - "#url" : "https://lolibooru.moe/pool/show/239", - "#category": ("moebooru", "lolibooru", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, -}, - -{ - "#url" : "https://lolibooru.moe/post/popular_recent", - "#category": ("moebooru", "lolibooru", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, -}, - + { + "#url": "https://lolibooru.moe/post/show/281305/", + "#category": ("moebooru", "lolibooru", "post"), + "#class": moebooru.MoebooruPostExtractor, + "#options": {"notes": True}, + "#sha1_content": "a331430223ffc5b23c31649102e7d49f52489b57", + "notes": list, + }, + { + "#url": "https://lolibooru.moe/post/show/287835", + "#category": ("moebooru", "lolibooru", "post"), + "#class": moebooru.MoebooruPostExtractor, + }, + { + "#url": "https://lolibooru.moe/post?tags=ruu_%28tksymkw%29", + "#category": ("moebooru", "lolibooru", "tag"), + "#class": moebooru.MoebooruTagExtractor, + }, + { + "#url": "https://lolibooru.moe/pool/show/239", + "#category": ("moebooru", "lolibooru", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + }, + { + "#url": "https://lolibooru.moe/post/popular_recent", + "#category": ("moebooru", "lolibooru", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + }, ) diff --git a/test/results/loungeunderwear.py b/test/results/loungeunderwear.py index 521de25cc..e55697237 100644 --- a/test/results/loungeunderwear.py +++ b/test/results/loungeunderwear.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://loungeunderwear.com/collections/apparel", - "#category": ("shopify", "loungeunderwear", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://de.loungeunderwear.com/products/ribbed-crop-top-black", - "#category": ("shopify", "loungeunderwear", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://loungeunderwear.com/collections/apparel", + "#category": ("shopify", "loungeunderwear", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://de.loungeunderwear.com/products/ribbed-crop-top-black", + "#category": ("shopify", "loungeunderwear", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/luscious.py b/test/results/luscious.py index 5e7a1460d..df82b0f8a 100644 --- a/test/results/luscious.py +++ b/test/results/luscious.py @@ -1,115 +1,104 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import luscious import datetime -from gallery_dl import exception +from gallery_dl import exception +from gallery_dl.extractor import luscious __tests__ = ( -{ - "#url" : "https://luscious.net/albums/okinami-no-koigokoro_277031/", - "#category": ("", "luscious", "album"), - "#class" : luscious.LusciousAlbumExtractor, - "#pattern" : r"https://storage\.bhs\.cloud\.ovh\.net/v1/AUTH_\w+/images/NTRshouldbeillegal/277031/luscious_net_\d+_\d+\.jpg$", - - "album" : { - "__typename" : "Album", - "audiences" : list, - "content" : "Hentai", - "cover" : r"re:https://storage\.bhs\.cloud\.ovh\.net/v1/.+/277031/", - "created" : 1479625853, - "created_by" : "Hive Mind", - "date" : "dt:2016-11-20 07:10:53", - "description" : "Enjoy.", - "download_url" : "/download/r/25/277031/", - "genres" : list, - "id" : 277031, - "is_manga" : True, - "labels" : list, - "language" : "English", - "like_status" : "none", - "modified" : int, - "permissions" : list, - "rating" : None, - "slug" : "okinami-no-koigokoro", - "status" : None, - "tags" : list, - "title" : "Okinami no Koigokoro", - "url" : "/albums/okinami-no-koigokoro_277031/", - "marked_for_deletion" : False, - "marked_for_processing" : False, - "number_of_animated_pictures": 0, - "number_of_favorites" : int, - "number_of_pictures" : 18, + { + "#url": "https://luscious.net/albums/okinami-no-koigokoro_277031/", + "#category": ("", "luscious", "album"), + "#class": luscious.LusciousAlbumExtractor, + "#pattern": r"https://storage\.bhs\.cloud\.ovh\.net/v1/AUTH_\w+/images/NTRshouldbeillegal/277031/luscious_net_\d+_\d+\.jpg$", + "album": { + "__typename": "Album", + "audiences": list, + "content": "Hentai", + "cover": r"re:https://storage\.bhs\.cloud\.ovh\.net/v1/.+/277031/", + "created": 1479625853, + "created_by": "Hive Mind", + "date": "dt:2016-11-20 07:10:53", + "description": "Enjoy.", + "download_url": "/download/r/25/277031/", + "genres": list, + "id": 277031, + "is_manga": True, + "labels": list, + "language": "English", + "like_status": "none", + "modified": int, + "permissions": list, + "rating": None, + "slug": "okinami-no-koigokoro", + "status": None, + "tags": list, + "title": "Okinami no Koigokoro", + "url": "/albums/okinami-no-koigokoro_277031/", + "marked_for_deletion": False, + "marked_for_processing": False, + "number_of_animated_pictures": 0, + "number_of_favorites": int, + "number_of_pictures": 18, + }, + "aspect_ratio": r"re:\d+:\d+", + "category": "luscious", + "created": int, + "date": datetime.datetime, + "height": int, + "id": int, + "is_animated": False, + "like_status": "none", + "position": int, + "resolution": r"re:\d+x\d+", + "status": None, + "tags": list, + "thumbnail": str, + "title": str, + "width": int, + "number_of_comments": int, + "number_of_favorites": int, + }, + { + "#url": "https://luscious.net/albums/not-found_277035/", + "#category": ("", "luscious", "album"), + "#class": luscious.LusciousAlbumExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://members.luscious.net/albums/login-required_323871/", + "#category": ("", "luscious", "album"), + "#class": luscious.LusciousAlbumExtractor, + "#count": 64, + }, + { + "#url": "https://www.luscious.net/albums/okinami_277031/", + "#category": ("", "luscious", "album"), + "#class": luscious.LusciousAlbumExtractor, + }, + { + "#url": "https://members.luscious.net/albums/okinami_277031/", + "#category": ("", "luscious", "album"), + "#class": luscious.LusciousAlbumExtractor, + }, + { + "#url": "https://luscious.net/pictures/c/video_game_manga/album/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1", + "#category": ("", "luscious", "album"), + "#class": luscious.LusciousAlbumExtractor, + }, + { + "#url": "https://members.luscious.net/albums/list/", + "#category": ("", "luscious", "search"), + "#class": luscious.LusciousSearchExtractor, + }, + { + "#url": "https://members.luscious.net/albums/list/?display=date_newest&language_ids=%2B1&tagged=+full_color&page=1", + "#category": ("", "luscious", "search"), + "#class": luscious.LusciousSearchExtractor, + "#pattern": luscious.LusciousAlbumExtractor.pattern, + "#range": "41-60", + "#count": 20, }, - "aspect_ratio" : r"re:\d+:\d+", - "category" : "luscious", - "created" : int, - "date" : datetime.datetime, - "height" : int, - "id" : int, - "is_animated" : False, - "like_status" : "none", - "position" : int, - "resolution" : r"re:\d+x\d+", - "status" : None, - "tags" : list, - "thumbnail" : str, - "title" : str, - "width" : int, - "number_of_comments": int, - "number_of_favorites": int, -}, - -{ - "#url" : "https://luscious.net/albums/not-found_277035/", - "#category": ("", "luscious", "album"), - "#class" : luscious.LusciousAlbumExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://members.luscious.net/albums/login-required_323871/", - "#category": ("", "luscious", "album"), - "#class" : luscious.LusciousAlbumExtractor, - "#count" : 64, -}, - -{ - "#url" : "https://www.luscious.net/albums/okinami_277031/", - "#category": ("", "luscious", "album"), - "#class" : luscious.LusciousAlbumExtractor, -}, - -{ - "#url" : "https://members.luscious.net/albums/okinami_277031/", - "#category": ("", "luscious", "album"), - "#class" : luscious.LusciousAlbumExtractor, -}, - -{ - "#url" : "https://luscious.net/pictures/c/video_game_manga/album/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1", - "#category": ("", "luscious", "album"), - "#class" : luscious.LusciousAlbumExtractor, -}, - -{ - "#url" : "https://members.luscious.net/albums/list/", - "#category": ("", "luscious", "search"), - "#class" : luscious.LusciousSearchExtractor, -}, - -{ - "#url" : "https://members.luscious.net/albums/list/?display=date_newest&language_ids=%2B1&tagged=+full_color&page=1", - "#category": ("", "luscious", "search"), - "#class" : luscious.LusciousSearchExtractor, - "#pattern" : luscious.LusciousAlbumExtractor.pattern, - "#range" : "41-60", - "#count" : 20, -}, - ) diff --git a/test/results/mangadex.py b/test/results/mangadex.py index 817b09413..2121fecc4 100644 --- a/test/results/mangadex.py +++ b/test/results/mangadex.py @@ -1,164 +1,145 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangadex -from gallery_dl import exception import datetime +from gallery_dl import exception +from gallery_dl.extractor import mangadex __tests__ = ( -{ - "#url" : "https://mangadex.org/chapter/f946ac53-0b71-4b5d-aeb2-7931b13c4aaa", - "#category": ("", "mangadex", "chapter"), - "#class" : mangadex.MangadexChapterExtractor, - "#sha1_metadata": "e86128a79ebe7201b648f1caa828496a2878dc8f", -}, - -{ - "#url" : "https://mangadex.org/chapter/61a88817-9c29-4281-bdf1-77b3c1be9831", - "#comment" : "oneshot", - "#category": ("", "mangadex", "chapter"), - "#class" : mangadex.MangadexChapterExtractor, - "#count" : 64, - "#sha1_metadata": "d11ed057a919854696853362be35fc0ba7dded4c", -}, - -{ - "#url" : "https://mangadex.org/chapter/74149a55-e7c4-44ea-8a37-98e879c1096f", - "#comment" : "MANGA Plus (#1154)", - "#category": ("", "mangadex", "chapter"), - "#class" : mangadex.MangadexChapterExtractor, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://mangadex.org/chapter/364728a4-6909-4164-9eea-6b56354f7c78", - "#comment" : "'externalUrl', but still downloadable / 404 (#2503)", - "#category": ("", "mangadex", "chapter"), - "#class" : mangadex.MangadexChapterExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", - "#comment" : "mutliple values for 'lang' (#4093)", - "#category": ("", "mangadex", "manga"), - "#class" : mangadex.MangadexMangaExtractor, - "#count" : ">= 5", - - "manga" : "Souten no Koumori", - "manga_id" : "f90c4398-8aad-4f51-8a1f-024ca09fdcbc", - "title" : r"re:One[Ss]hot", - "volume" : 0, - "chapter" : 0, - "chapter_minor": "", - "chapter_id" : str, - "date" : datetime.datetime, - "lang" : str, - "language" : str, - "artist" : ["Arakawa Hiromu"], - "author" : ["Arakawa Hiromu"], - "status" : "completed", - "tags" : [ - "Oneshot", - "Historical", - "Action", - "Martial Arts", - "Drama", - "Tragedy", - ], -}, - -{ - "#url" : "https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", - "#comment" : "mutliple values for 'lang' (#4093)", - "#category": ("", "mangadex", "manga"), - "#class" : mangadex.MangadexMangaExtractor, - "#options" : {"lang": "fr,it"}, - "#count" : 2, - - "manga" : "Souten no Koumori", - "lang" : {"fr", "it"}, - "language": {"French", "Italian"}, -}, - -{ - "#url" : "https://mangadex.cc/manga/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a/", - "#category": ("", "mangadex", "manga"), - "#class" : mangadex.MangadexMangaExtractor, - "#options" : {"lang": "en"}, - "#count" : ">= 100", -}, - -{ - "#url" : "https://mangadex.org/title/7c1e2742-a086-4fd3-a3be-701fd6cf0be9", - "#category": ("", "mangadex", "manga"), - "#class" : mangadex.MangadexMangaExtractor, - "#count" : ">= 25", -}, - -{ - "#url" : "https://mangadex.org/title/584ef094-b2ab-40ce-962c-bce341fb9d10", - "#category": ("", "mangadex", "manga"), - "#class" : mangadex.MangadexMangaExtractor, - "#count" : ">= 20", -}, - -{ - "#url" : "https://mangadex.org/title/feed", - "#category": ("", "mangadex", "feed"), - "#class" : mangadex.MangadexFeedExtractor, -}, - -{ - "#url" : "https://mangadex.org/list/3a0982c5-65aa-4de2-8a4a-2175be7383ab/test", - "#category": ("", "mangadex", "list"), - "#class" : mangadex.MangadexListExtractor, - "#urls" : ( - "https://mangadex.org/title/cba4e5d6-67a0-47a0-b37a-c06e9bf25d93", - "https://mangadex.org/title/cad76ec6-ca22-42f6-96f8-eca164da6545", - ), -}, - -{ - "#url" : "https://mangadex.org/list/3a0982c5-65aa-4de2-8a4a-2175be7383ab/test?tab=titles", - "#category": ("", "mangadex", "list"), - "#class" : mangadex.MangadexListExtractor, -}, - -{ - "#url" : "https://mangadex.org/list/3a0982c5-65aa-4de2-8a4a-2175be7383ab/test?tab=feed", - "#category": ("", "mangadex", "list-feed"), - "#class" : mangadex.MangadexListExtractor, - "#urls" : ( - "https://mangadex.org/chapter/fa8a695d-260f-4dcc-95a3-1f30e66d6571", - "https://mangadex.org/chapter/c765d6d5-5712-4360-be0b-0c8e0914fc94", - "https://mangadex.org/chapter/788766b9-41c6-422e-97ba-552f03ba9655", - ), -}, - -{ - "#url" : "https://mangadex.org/author/7222d0d5-836c-4bf3-9174-72bceade8c87/kotoyama", - "#class" : mangadex.MangadexAuthorExtractor, - "#urls" : ( - "https://mangadex.org/title/259dfd8a-f06a-4825-8fa6-a2dcd7274230", - "https://mangadex.org/title/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a", - "https://mangadex.org/title/f48bbb5f-8a23-4dea-8177-eb2dbbcbf4fa", - "https://mangadex.org/title/00b68132-4e69-4ff9-ad4b-29138b377dc8", - "https://mangadex.org/title/f1b70bba-3873-4c22-afa3-1d1c78299cd9", - "https://mangadex.org/title/41cd6fa7-3e53-4900-88e6-4a06cd7df9ad", - ), -}, - -{ - "#url" : "https://mangadex.org/author/254efca2-0ac0-432c-a3a3-55b7e207e87d/flipflops", - "#class" : mangadex.MangadexAuthorExtractor, - "#pattern" : mangadex.MangadexMangaExtractor.pattern, - "#options" : {"lang": "en"}, - "#count" : ">= 15", -}, - + { + "#url": "https://mangadex.org/chapter/f946ac53-0b71-4b5d-aeb2-7931b13c4aaa", + "#category": ("", "mangadex", "chapter"), + "#class": mangadex.MangadexChapterExtractor, + "#sha1_metadata": "e86128a79ebe7201b648f1caa828496a2878dc8f", + }, + { + "#url": "https://mangadex.org/chapter/61a88817-9c29-4281-bdf1-77b3c1be9831", + "#comment": "oneshot", + "#category": ("", "mangadex", "chapter"), + "#class": mangadex.MangadexChapterExtractor, + "#count": 64, + "#sha1_metadata": "d11ed057a919854696853362be35fc0ba7dded4c", + }, + { + "#url": "https://mangadex.org/chapter/74149a55-e7c4-44ea-8a37-98e879c1096f", + "#comment": "MANGA Plus (#1154)", + "#category": ("", "mangadex", "chapter"), + "#class": mangadex.MangadexChapterExtractor, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://mangadex.org/chapter/364728a4-6909-4164-9eea-6b56354f7c78", + "#comment": "'externalUrl', but still downloadable / 404 (#2503)", + "#category": ("", "mangadex", "chapter"), + "#class": mangadex.MangadexChapterExtractor, + "#count": 0, + }, + { + "#url": "https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", + "#comment": "mutliple values for 'lang' (#4093)", + "#category": ("", "mangadex", "manga"), + "#class": mangadex.MangadexMangaExtractor, + "#count": ">= 5", + "manga": "Souten no Koumori", + "manga_id": "f90c4398-8aad-4f51-8a1f-024ca09fdcbc", + "title": r"re:One[Ss]hot", + "volume": 0, + "chapter": 0, + "chapter_minor": "", + "chapter_id": str, + "date": datetime.datetime, + "lang": str, + "language": str, + "artist": ["Arakawa Hiromu"], + "author": ["Arakawa Hiromu"], + "status": "completed", + "tags": [ + "Oneshot", + "Historical", + "Action", + "Martial Arts", + "Drama", + "Tragedy", + ], + }, + { + "#url": "https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", + "#comment": "mutliple values for 'lang' (#4093)", + "#category": ("", "mangadex", "manga"), + "#class": mangadex.MangadexMangaExtractor, + "#options": {"lang": "fr,it"}, + "#count": 2, + "manga": "Souten no Koumori", + "lang": {"fr", "it"}, + "language": {"French", "Italian"}, + }, + { + "#url": "https://mangadex.cc/manga/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a/", + "#category": ("", "mangadex", "manga"), + "#class": mangadex.MangadexMangaExtractor, + "#options": {"lang": "en"}, + "#count": ">= 100", + }, + { + "#url": "https://mangadex.org/title/7c1e2742-a086-4fd3-a3be-701fd6cf0be9", + "#category": ("", "mangadex", "manga"), + "#class": mangadex.MangadexMangaExtractor, + "#count": ">= 25", + }, + { + "#url": "https://mangadex.org/title/584ef094-b2ab-40ce-962c-bce341fb9d10", + "#category": ("", "mangadex", "manga"), + "#class": mangadex.MangadexMangaExtractor, + "#count": ">= 20", + }, + { + "#url": "https://mangadex.org/title/feed", + "#category": ("", "mangadex", "feed"), + "#class": mangadex.MangadexFeedExtractor, + }, + { + "#url": "https://mangadex.org/list/3a0982c5-65aa-4de2-8a4a-2175be7383ab/test", + "#category": ("", "mangadex", "list"), + "#class": mangadex.MangadexListExtractor, + "#urls": ( + "https://mangadex.org/title/cba4e5d6-67a0-47a0-b37a-c06e9bf25d93", + "https://mangadex.org/title/cad76ec6-ca22-42f6-96f8-eca164da6545", + ), + }, + { + "#url": "https://mangadex.org/list/3a0982c5-65aa-4de2-8a4a-2175be7383ab/test?tab=titles", + "#category": ("", "mangadex", "list"), + "#class": mangadex.MangadexListExtractor, + }, + { + "#url": "https://mangadex.org/list/3a0982c5-65aa-4de2-8a4a-2175be7383ab/test?tab=feed", + "#category": ("", "mangadex", "list-feed"), + "#class": mangadex.MangadexListExtractor, + "#urls": ( + "https://mangadex.org/chapter/fa8a695d-260f-4dcc-95a3-1f30e66d6571", + "https://mangadex.org/chapter/c765d6d5-5712-4360-be0b-0c8e0914fc94", + "https://mangadex.org/chapter/788766b9-41c6-422e-97ba-552f03ba9655", + ), + }, + { + "#url": "https://mangadex.org/author/7222d0d5-836c-4bf3-9174-72bceade8c87/kotoyama", + "#class": mangadex.MangadexAuthorExtractor, + "#urls": ( + "https://mangadex.org/title/259dfd8a-f06a-4825-8fa6-a2dcd7274230", + "https://mangadex.org/title/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a", + "https://mangadex.org/title/f48bbb5f-8a23-4dea-8177-eb2dbbcbf4fa", + "https://mangadex.org/title/00b68132-4e69-4ff9-ad4b-29138b377dc8", + "https://mangadex.org/title/f1b70bba-3873-4c22-afa3-1d1c78299cd9", + "https://mangadex.org/title/41cd6fa7-3e53-4900-88e6-4a06cd7df9ad", + ), + }, + { + "#url": "https://mangadex.org/author/254efca2-0ac0-432c-a3a3-55b7e207e87d/flipflops", + "#class": mangadex.MangadexAuthorExtractor, + "#pattern": mangadex.MangadexMangaExtractor.pattern, + "#options": {"lang": "en"}, + "#count": ">= 15", + }, ) diff --git a/test/results/mangafox.py b/test/results/mangafox.py index dc0cc9a04..1e1a82604 100644 --- a/test/results/mangafox.py +++ b/test/results/mangafox.py @@ -1,71 +1,62 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangafox import datetime +from gallery_dl.extractor import mangafox __tests__ = ( -{ - "#url" : "http://fanfox.net/manga/kidou_keisatsu_patlabor/v05/c006.2/1.html", - "#category": ("", "mangafox", "chapter"), - "#class" : mangafox.MangafoxChapterExtractor, - "#sha1_metadata": "5661dab258d42d09d98f194f7172fb9851a49766", - "#sha1_content" : "5c50c252dcf12ffecf68801f4db8a2167265f66c", -}, - -{ - "#url" : "http://mangafox.me/manga/kidou_keisatsu_patlabor/v05/c006.2/", - "#category": ("", "mangafox", "chapter"), - "#class" : mangafox.MangafoxChapterExtractor, -}, - -{ - "#url" : "http://fanfox.net/manga/black_clover/vTBD/c295/1.html", - "#category": ("", "mangafox", "chapter"), - "#class" : mangafox.MangafoxChapterExtractor, -}, - -{ - "#url" : "https://fanfox.net/manga/kanojo_mo_kanojo", - "#category": ("", "mangafox", "manga"), - "#class" : mangafox.MangafoxMangaExtractor, - "#pattern" : mangafox.MangafoxChapterExtractor.pattern, - "#count" : ">=60", - - "author" : "HIROYUKI", - "chapter" : int, - "chapter_minor" : r"re:^(\.\d+)?$", - "chapter_string": r"re:(v\d+/)?c\d+", - "date" : datetime.datetime, - "description" : "High school boy Naoya gets a confession from Momi, a cute and friendly girl. However, Naoya already has a girlfriend, Seki... but Momi is too good a catch to let go. Momi and Nagoya's goal becomes clear: convince Seki to accept being an item with the two of them. Will she budge?", - "lang" : "en", - "language" : "English", - "manga" : "Kanojo mo Kanojo", - "tags" : [ - "Comedy", - "Romance", - "School Life", - "Shounen", - ], - "volume" : int, -}, - -{ - "#url" : "https://mangafox.me/manga/shangri_la_frontier", - "#category": ("", "mangafox", "manga"), - "#class" : mangafox.MangafoxMangaExtractor, - "#pattern" : mangafox.MangafoxChapterExtractor.pattern, - "#count" : ">=45", -}, - -{ - "#url" : "https://m.fanfox.net/manga/sentai_daishikkaku", - "#category": ("", "mangafox", "manga"), - "#class" : mangafox.MangafoxMangaExtractor, -}, - + { + "#url": "http://fanfox.net/manga/kidou_keisatsu_patlabor/v05/c006.2/1.html", + "#category": ("", "mangafox", "chapter"), + "#class": mangafox.MangafoxChapterExtractor, + "#sha1_metadata": "5661dab258d42d09d98f194f7172fb9851a49766", + "#sha1_content": "5c50c252dcf12ffecf68801f4db8a2167265f66c", + }, + { + "#url": "http://mangafox.me/manga/kidou_keisatsu_patlabor/v05/c006.2/", + "#category": ("", "mangafox", "chapter"), + "#class": mangafox.MangafoxChapterExtractor, + }, + { + "#url": "http://fanfox.net/manga/black_clover/vTBD/c295/1.html", + "#category": ("", "mangafox", "chapter"), + "#class": mangafox.MangafoxChapterExtractor, + }, + { + "#url": "https://fanfox.net/manga/kanojo_mo_kanojo", + "#category": ("", "mangafox", "manga"), + "#class": mangafox.MangafoxMangaExtractor, + "#pattern": mangafox.MangafoxChapterExtractor.pattern, + "#count": ">=60", + "author": "HIROYUKI", + "chapter": int, + "chapter_minor": r"re:^(\.\d+)?$", + "chapter_string": r"re:(v\d+/)?c\d+", + "date": datetime.datetime, + "description": "High school boy Naoya gets a confession from Momi, a cute and friendly girl. However, Naoya already has a girlfriend, Seki... but Momi is too good a catch to let go. Momi and Nagoya's goal becomes clear: convince Seki to accept being an item with the two of them. Will she budge?", + "lang": "en", + "language": "English", + "manga": "Kanojo mo Kanojo", + "tags": [ + "Comedy", + "Romance", + "School Life", + "Shounen", + ], + "volume": int, + }, + { + "#url": "https://mangafox.me/manga/shangri_la_frontier", + "#category": ("", "mangafox", "manga"), + "#class": mangafox.MangafoxMangaExtractor, + "#pattern": mangafox.MangafoxChapterExtractor.pattern, + "#count": ">=45", + }, + { + "#url": "https://m.fanfox.net/manga/sentai_daishikkaku", + "#category": ("", "mangafox", "manga"), + "#class": mangafox.MangafoxMangaExtractor, + }, ) diff --git a/test/results/mangahere.py b/test/results/mangahere.py index db17a6b08..cd78ad9a0 100644 --- a/test/results/mangahere.py +++ b/test/results/mangahere.py @@ -1,77 +1,65 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mangahere - __tests__ = ( -{ - "#url" : "https://www.mangahere.cc/manga/dongguo_xiaojie/c004.2/", - "#category": ("", "mangahere", "chapter"), - "#class" : mangahere.MangahereChapterExtractor, - "#sha1_metadata": "7c98d7b50a47e6757b089aa875a53aa970cac66f", - "#sha1_content" : "708d475f06893b88549cbd30df1e3f9428f2c884", -}, - -{ - "#url" : "https://www.mangahere.cc/manga/beastars/c196/1.html", - "#comment" : "URLs without HTTP scheme (#1070)", - "#category": ("", "mangahere", "chapter"), - "#class" : mangahere.MangahereChapterExtractor, - "#pattern" : "https://zjcdn.mangahere.org/.*", -}, - -{ - "#url" : "http://www.mangahere.co/manga/dongguo_xiaojie/c003.2/", - "#category": ("", "mangahere", "chapter"), - "#class" : mangahere.MangahereChapterExtractor, -}, - -{ - "#url" : "http://m.mangahere.co/manga/dongguo_xiaojie/c003.2/", - "#category": ("", "mangahere", "chapter"), - "#class" : mangahere.MangahereChapterExtractor, -}, - -{ - "#url" : "https://www.mangahere.cc/manga/aria/", - "#category": ("", "mangahere", "manga"), - "#class" : mangahere.MangahereMangaExtractor, - "#count" : 71, - "#sha1_url" : "9c2e54ec42e9a87ad53096c328b33c90750af3e4", - "#sha1_metadata": "71503c682c5d0c277a50409a8c5fd78e871e3d69", -}, - -{ - "#url" : "https://www.mangahere.cc/manga/hiyokoi/#50", - "#category": ("", "mangahere", "manga"), - "#class" : mangahere.MangahereMangaExtractor, - "#sha1_url" : "654850570aa03825cd57e2ae2904af489602c523", - "#sha1_metadata": "c8084d89a9ea6cf40353093669f9601a39bf5ca2", -}, - -{ - "#url" : "http://www.mangahere.cc/manga/gunnm_mars_chronicle/", - "#comment" : "adult filter (#556)", - "#category": ("", "mangahere", "manga"), - "#class" : mangahere.MangahereMangaExtractor, - "#pattern" : mangahere.MangahereChapterExtractor.pattern, - "#count" : ">= 50", -}, - -{ - "#url" : "https://www.mangahere.co/manga/aria/", - "#category": ("", "mangahere", "manga"), - "#class" : mangahere.MangahereMangaExtractor, -}, - -{ - "#url" : "https://m.mangahere.co/manga/aria/", - "#category": ("", "mangahere", "manga"), - "#class" : mangahere.MangahereMangaExtractor, -}, - + { + "#url": "https://www.mangahere.cc/manga/dongguo_xiaojie/c004.2/", + "#category": ("", "mangahere", "chapter"), + "#class": mangahere.MangahereChapterExtractor, + "#sha1_metadata": "7c98d7b50a47e6757b089aa875a53aa970cac66f", + "#sha1_content": "708d475f06893b88549cbd30df1e3f9428f2c884", + }, + { + "#url": "https://www.mangahere.cc/manga/beastars/c196/1.html", + "#comment": "URLs without HTTP scheme (#1070)", + "#category": ("", "mangahere", "chapter"), + "#class": mangahere.MangahereChapterExtractor, + "#pattern": "https://zjcdn.mangahere.org/.*", + }, + { + "#url": "http://www.mangahere.co/manga/dongguo_xiaojie/c003.2/", + "#category": ("", "mangahere", "chapter"), + "#class": mangahere.MangahereChapterExtractor, + }, + { + "#url": "http://m.mangahere.co/manga/dongguo_xiaojie/c003.2/", + "#category": ("", "mangahere", "chapter"), + "#class": mangahere.MangahereChapterExtractor, + }, + { + "#url": "https://www.mangahere.cc/manga/aria/", + "#category": ("", "mangahere", "manga"), + "#class": mangahere.MangahereMangaExtractor, + "#count": 71, + "#sha1_url": "9c2e54ec42e9a87ad53096c328b33c90750af3e4", + "#sha1_metadata": "71503c682c5d0c277a50409a8c5fd78e871e3d69", + }, + { + "#url": "https://www.mangahere.cc/manga/hiyokoi/#50", + "#category": ("", "mangahere", "manga"), + "#class": mangahere.MangahereMangaExtractor, + "#sha1_url": "654850570aa03825cd57e2ae2904af489602c523", + "#sha1_metadata": "c8084d89a9ea6cf40353093669f9601a39bf5ca2", + }, + { + "#url": "http://www.mangahere.cc/manga/gunnm_mars_chronicle/", + "#comment": "adult filter (#556)", + "#category": ("", "mangahere", "manga"), + "#class": mangahere.MangahereMangaExtractor, + "#pattern": mangahere.MangahereChapterExtractor.pattern, + "#count": ">= 50", + }, + { + "#url": "https://www.mangahere.co/manga/aria/", + "#category": ("", "mangahere", "manga"), + "#class": mangahere.MangahereMangaExtractor, + }, + { + "#url": "https://m.mangahere.co/manga/aria/", + "#category": ("", "mangahere", "manga"), + "#class": mangahere.MangahereMangaExtractor, + }, ) diff --git a/test/results/mangakakalot.py b/test/results/mangakakalot.py index b0b8badab..91b52d1f2 100644 --- a/test/results/mangakakalot.py +++ b/test/results/mangakakalot.py @@ -1,40 +1,33 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mangakakalot - __tests__ = ( -{ - "#url" : "https://ww3.mangakakalot.tv/chapter/manga-jk986845/chapter-34.2", - "#category": ("", "mangakakalot", "chapter"), - "#class" : mangakakalot.MangakakalotChapterExtractor, - "#pattern" : r"https://cm\.blazefast\.co/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.jpg", - "#count" : 9, - "#sha1_metadata": "0f1586ff52f0f9cbbb25306ae64ab718f8a6a633", -}, - -{ - "#url" : "https://mangakakalot.tv/chapter/hatarakanai_futari_the_jobless_siblings/chapter_20.1", - "#category": ("", "mangakakalot", "chapter"), - "#class" : mangakakalot.MangakakalotChapterExtractor, -}, - -{ - "#url" : "https://ww3.mangakakalot.tv/manga/manga-jk986845", - "#category": ("", "mangakakalot", "manga"), - "#class" : mangakakalot.MangakakalotMangaExtractor, - "#pattern" : mangakakalot.MangakakalotChapterExtractor.pattern, - "#count" : ">= 30", -}, - -{ - "#url" : "https://mangakakalot.tv/manga/lk921810", - "#category": ("", "mangakakalot", "manga"), - "#class" : mangakakalot.MangakakalotMangaExtractor, -}, - + { + "#url": "https://ww3.mangakakalot.tv/chapter/manga-jk986845/chapter-34.2", + "#category": ("", "mangakakalot", "chapter"), + "#class": mangakakalot.MangakakalotChapterExtractor, + "#pattern": r"https://cm\.blazefast\.co/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.jpg", + "#count": 9, + "#sha1_metadata": "0f1586ff52f0f9cbbb25306ae64ab718f8a6a633", + }, + { + "#url": "https://mangakakalot.tv/chapter/hatarakanai_futari_the_jobless_siblings/chapter_20.1", + "#category": ("", "mangakakalot", "chapter"), + "#class": mangakakalot.MangakakalotChapterExtractor, + }, + { + "#url": "https://ww3.mangakakalot.tv/manga/manga-jk986845", + "#category": ("", "mangakakalot", "manga"), + "#class": mangakakalot.MangakakalotMangaExtractor, + "#pattern": mangakakalot.MangakakalotChapterExtractor.pattern, + "#count": ">= 30", + }, + { + "#url": "https://mangakakalot.tv/manga/lk921810", + "#category": ("", "mangakakalot", "manga"), + "#class": mangakakalot.MangakakalotMangaExtractor, + }, ) diff --git a/test/results/mangalife.py b/test/results/mangalife.py index 80226afef..64bd40b8a 100644 --- a/test/results/mangalife.py +++ b/test/results/mangalife.py @@ -1,69 +1,63 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangasee import datetime +from gallery_dl.extractor import mangasee __tests__ = ( -{ - "#url" : "https://manga4life.com/read-online/One-Piece-chapter-1063-page-1.html", - "#category": ("", "mangalife", "chapter"), - "#class" : mangasee.MangaseeChapterExtractor, - "#pattern" : r"https://[^/]+/manga/One-Piece/1063-0\d\d\.png", - "#count" : 13, - - "author" : ["ODA Eiichiro"], - "chapter" : 1063, - "chapter_minor" : "", - "chapter_string": "110630", - "count" : 13, - "date" : "dt:2022-10-16 17:32:54", - "extension" : "png", - "filename" : r"re:1063-0\d\d", - "genre" : [ - "Action", - "Adventure", - "Comedy", - "Drama", - "Fantasy", - "Shounen", - ], - "index" : "1", - "lang" : "en", - "language" : "English", - "manga" : "One Piece", - "page" : int, - "title" : "", -}, - -{ - "#url" : "https://manga4life.com/manga/Ano-Musume-Ni-Kiss-To-Shirayuri-O", - "#category": ("", "mangalife", "manga"), - "#class" : mangasee.MangaseeMangaExtractor, - "#pattern" : mangasee.MangaseeChapterExtractor.pattern, - "#count" : ">= 50", - - "author" : ["Canno"], - "chapter" : int, - "chapter_minor" : r"re:^|\.5$", - "chapter_string": r"re:100\d\d\d", - "date" : datetime.datetime, - "genre" : [ - "Comedy", - "Romance", - "School Life", - "Seinen", - "Shoujo Ai", - ], - "index" : "1", - "lang" : "en", - "language" : "English", - "manga" : "Ano-Musume-Ni-Kiss-To-Shirayuri-O", - "title" : "", -}, - + { + "#url": "https://manga4life.com/read-online/One-Piece-chapter-1063-page-1.html", + "#category": ("", "mangalife", "chapter"), + "#class": mangasee.MangaseeChapterExtractor, + "#pattern": r"https://[^/]+/manga/One-Piece/1063-0\d\d\.png", + "#count": 13, + "author": ["ODA Eiichiro"], + "chapter": 1063, + "chapter_minor": "", + "chapter_string": "110630", + "count": 13, + "date": "dt:2022-10-16 17:32:54", + "extension": "png", + "filename": r"re:1063-0\d\d", + "genre": [ + "Action", + "Adventure", + "Comedy", + "Drama", + "Fantasy", + "Shounen", + ], + "index": "1", + "lang": "en", + "language": "English", + "manga": "One Piece", + "page": int, + "title": "", + }, + { + "#url": "https://manga4life.com/manga/Ano-Musume-Ni-Kiss-To-Shirayuri-O", + "#category": ("", "mangalife", "manga"), + "#class": mangasee.MangaseeMangaExtractor, + "#pattern": mangasee.MangaseeChapterExtractor.pattern, + "#count": ">= 50", + "author": ["Canno"], + "chapter": int, + "chapter_minor": r"re:^|\.5$", + "chapter_string": r"re:100\d\d\d", + "date": datetime.datetime, + "genre": [ + "Comedy", + "Romance", + "School Life", + "Seinen", + "Shoujo Ai", + ], + "index": "1", + "lang": "en", + "language": "English", + "manga": "Ano-Musume-Ni-Kiss-To-Shirayuri-O", + "title": "", + }, ) diff --git a/test/results/manganelo.py b/test/results/manganelo.py index b5772656c..a052c2e73 100644 --- a/test/results/manganelo.py +++ b/test/results/manganelo.py @@ -1,91 +1,76 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import manganelo - __tests__ = ( -{ - "#url" : "https://chapmanganato.com/manga-gn983696/chapter-23", - "#category": ("", "manganelo", "chapter"), - "#class" : manganelo.ManganeloChapterExtractor, - "#pattern" : r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/03/23/39/gn983696/vol_3_chapter_23_24_yen/\d+-[no]\.jpg", - "#count" : 25, - "#sha1_metadata": "17faaea7f0fb8c2675a327bf3aa0bcd7a6311d68", -}, - -{ - "#url" : "https://chapmanganelo.com/manga-ti107776/chapter-4", - "#category": ("", "manganelo", "chapter"), - "#class" : manganelo.ManganeloChapterExtractor, - "#pattern" : r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/01/92/08/ti970565/chapter_4_caster/\d+-o\.jpg", - "#count" : 45, - "#sha1_metadata": "06e01fa9b3fc9b5b954c0d4a98f0153b40922ded", -}, - -{ - "#url" : "https://chapmanganato.com/manga-no991297/chapter-8", - "#category": ("", "manganelo", "chapter"), - "#class" : manganelo.ManganeloChapterExtractor, - "#count" : 20, - - "chapter" : 8, - "chapter_minor": "-1", -}, - -{ - "#url" : "https://readmanganato.com/manga-gn983696/chapter-23", - "#category": ("", "manganelo", "chapter"), - "#class" : manganelo.ManganeloChapterExtractor, -}, - -{ - "#url" : "https://manganelo.com/chapter/gamers/chapter_15", - "#category": ("", "manganelo", "chapter"), - "#class" : manganelo.ManganeloChapterExtractor, -}, - -{ - "#url" : "https://manganelo.com/chapter/gq921227/chapter_23", - "#category": ("", "manganelo", "chapter"), - "#class" : manganelo.ManganeloChapterExtractor, -}, - -{ - "#url" : "https://chapmanganato.com/manga-gn983696", - "#category": ("", "manganelo", "manga"), - "#class" : manganelo.ManganeloMangaExtractor, - "#pattern" : manganelo.ManganeloChapterExtractor.pattern, - "#count" : ">= 25", -}, - -{ - "#url" : "https://m.manganelo.com/manga-ti107776", - "#category": ("", "manganelo", "manga"), - "#class" : manganelo.ManganeloMangaExtractor, - "#pattern" : manganelo.ManganeloChapterExtractor.pattern, - "#count" : ">= 12", -}, - -{ - "#url" : "https://readmanganato.com/manga-gn983696", - "#category": ("", "manganelo", "manga"), - "#class" : manganelo.ManganeloMangaExtractor, -}, - -{ - "#url" : "https://manganelo.com/manga/read_otome_no_teikoku", - "#category": ("", "manganelo", "manga"), - "#class" : manganelo.ManganeloMangaExtractor, -}, - -{ - "#url" : "https://manganelo.com/manga/ol921234/", - "#category": ("", "manganelo", "manga"), - "#class" : manganelo.ManganeloMangaExtractor, -}, - + { + "#url": "https://chapmanganato.com/manga-gn983696/chapter-23", + "#category": ("", "manganelo", "chapter"), + "#class": manganelo.ManganeloChapterExtractor, + "#pattern": r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/03/23/39/gn983696/vol_3_chapter_23_24_yen/\d+-[no]\.jpg", + "#count": 25, + "#sha1_metadata": "17faaea7f0fb8c2675a327bf3aa0bcd7a6311d68", + }, + { + "#url": "https://chapmanganelo.com/manga-ti107776/chapter-4", + "#category": ("", "manganelo", "chapter"), + "#class": manganelo.ManganeloChapterExtractor, + "#pattern": r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/01/92/08/ti970565/chapter_4_caster/\d+-o\.jpg", + "#count": 45, + "#sha1_metadata": "06e01fa9b3fc9b5b954c0d4a98f0153b40922ded", + }, + { + "#url": "https://chapmanganato.com/manga-no991297/chapter-8", + "#category": ("", "manganelo", "chapter"), + "#class": manganelo.ManganeloChapterExtractor, + "#count": 20, + "chapter": 8, + "chapter_minor": "-1", + }, + { + "#url": "https://readmanganato.com/manga-gn983696/chapter-23", + "#category": ("", "manganelo", "chapter"), + "#class": manganelo.ManganeloChapterExtractor, + }, + { + "#url": "https://manganelo.com/chapter/gamers/chapter_15", + "#category": ("", "manganelo", "chapter"), + "#class": manganelo.ManganeloChapterExtractor, + }, + { + "#url": "https://manganelo.com/chapter/gq921227/chapter_23", + "#category": ("", "manganelo", "chapter"), + "#class": manganelo.ManganeloChapterExtractor, + }, + { + "#url": "https://chapmanganato.com/manga-gn983696", + "#category": ("", "manganelo", "manga"), + "#class": manganelo.ManganeloMangaExtractor, + "#pattern": manganelo.ManganeloChapterExtractor.pattern, + "#count": ">= 25", + }, + { + "#url": "https://m.manganelo.com/manga-ti107776", + "#category": ("", "manganelo", "manga"), + "#class": manganelo.ManganeloMangaExtractor, + "#pattern": manganelo.ManganeloChapterExtractor.pattern, + "#count": ">= 12", + }, + { + "#url": "https://readmanganato.com/manga-gn983696", + "#category": ("", "manganelo", "manga"), + "#class": manganelo.ManganeloMangaExtractor, + }, + { + "#url": "https://manganelo.com/manga/read_otome_no_teikoku", + "#category": ("", "manganelo", "manga"), + "#class": manganelo.ManganeloMangaExtractor, + }, + { + "#url": "https://manganelo.com/manga/ol921234/", + "#category": ("", "manganelo", "manga"), + "#class": manganelo.ManganeloMangaExtractor, + }, ) diff --git a/test/results/mangapark.py b/test/results/mangapark.py index 432f535fa..3685c1ac9 100644 --- a/test/results/mangapark.py +++ b/test/results/mangapark.py @@ -1,140 +1,122 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangapark import datetime +from gallery_dl.extractor import mangapark __tests__ = ( -{ - "#url" : "https://mangapark.net/title/114972-aria/6710214-en-ch.60.2", - "#category": ("", "mangapark", "chapter"), - "#class" : mangapark.MangaparkChapterExtractor, - "#pattern" : r"https://[\w-]+\.mpcdn\.org/comic/2002/e67/61e29278a583b9227964076e/\d+_\d+_\d+_\d+\.jpeg\?acc=[^&#]+&exp=\d+", - "#count" : 70, - - "artist" : [], - "author" : ["Amano Kozue"], - "chapter" : 60, - "chapter_id" : 6710214, - "chapter_minor": ".2", - "count" : 70, - "date" : "dt:2022-01-15 09:25:03", - "extension" : "jpeg", - "filename" : str, - "genre" : [ - "adventure", - "comedy", - "drama", - "sci_fi", - "shounen", - "slice_of_life", - ], - "lang" : "en", - "language" : "English", - "manga" : "Aria", - "manga_id" : 114972, - "page" : int, - "source" : "Koala", - "title" : "Special Navigation - Aquaria Ii", - "volume" : 12, -}, - -{ - "#url" : "https://mangapark.com/title/114972-aria/6710214-en-ch.60.2", - "#category": ("", "mangapark", "chapter"), - "#class" : mangapark.MangaparkChapterExtractor, -}, - -{ - "#url" : "https://mangapark.org/title/114972-aria/6710214-en-ch.60.2", - "#category": ("", "mangapark", "chapter"), - "#class" : mangapark.MangaparkChapterExtractor, -}, - -{ - "#url" : "https://mangapark.io/title/114972-aria/6710214-en-ch.60.2", - "#category": ("", "mangapark", "chapter"), - "#class" : mangapark.MangaparkChapterExtractor, -}, - -{ - "#url" : "https://mangapark.me/title/114972-aria/6710214-en-ch.60.2", - "#category": ("", "mangapark", "chapter"), - "#class" : mangapark.MangaparkChapterExtractor, -}, - -{ - "#url" : "https://mangapark.net/title/114972-aria", - "#comment" : "'source' option", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, - "#pattern" : mangapark.MangaparkChapterExtractor.pattern, - "#count" : 141, - - "chapter" : int, - "chapter_id" : int, - "chapter_minor": str, - "date" : datetime.datetime, - "lang" : "en", - "language" : "English", - "manga_id" : 114972, - "source" : r"re:Horse|Koala", - "source_id" : int, - "title" : str, - "volume" : int, -}, - -{ - "#url" : "https://mangapark.net/title/114972-aria", - "#comment" : "'source' option", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, - "#options" : {"source": "koala"}, - "#pattern" : mangapark.MangaparkChapterExtractor.pattern, - "#count" : 70, - - "source" : "Koala", - "source_id": 15150116, -}, - -{ - "#url" : "https://mangapark.com/title/114972-", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, -}, - -{ - "#url" : "https://mangapark.com/title/114972", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, -}, - -{ - "#url" : "https://mangapark.com/title/114972-aria", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, -}, - -{ - "#url" : "https://mangapark.org/title/114972-aria", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, -}, - -{ - "#url" : "https://mangapark.io/title/114972-aria", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, -}, - -{ - "#url" : "https://mangapark.me/title/114972-aria", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, -}, - + { + "#url": "https://mangapark.net/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class": mangapark.MangaparkChapterExtractor, + "#pattern": r"https://[\w-]+\.mpcdn\.org/comic/2002/e67/61e29278a583b9227964076e/\d+_\d+_\d+_\d+\.jpeg\?acc=[^&#]+&exp=\d+", + "#count": 70, + "artist": [], + "author": ["Amano Kozue"], + "chapter": 60, + "chapter_id": 6710214, + "chapter_minor": ".2", + "count": 70, + "date": "dt:2022-01-15 09:25:03", + "extension": "jpeg", + "filename": str, + "genre": [ + "adventure", + "comedy", + "drama", + "sci_fi", + "shounen", + "slice_of_life", + ], + "lang": "en", + "language": "English", + "manga": "Aria", + "manga_id": 114972, + "page": int, + "source": "Koala", + "title": "Special Navigation - Aquaria Ii", + "volume": 12, + }, + { + "#url": "https://mangapark.com/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class": mangapark.MangaparkChapterExtractor, + }, + { + "#url": "https://mangapark.org/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class": mangapark.MangaparkChapterExtractor, + }, + { + "#url": "https://mangapark.io/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class": mangapark.MangaparkChapterExtractor, + }, + { + "#url": "https://mangapark.me/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class": mangapark.MangaparkChapterExtractor, + }, + { + "#url": "https://mangapark.net/title/114972-aria", + "#comment": "'source' option", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + "#pattern": mangapark.MangaparkChapterExtractor.pattern, + "#count": 141, + "chapter": int, + "chapter_id": int, + "chapter_minor": str, + "date": datetime.datetime, + "lang": "en", + "language": "English", + "manga_id": 114972, + "source": r"re:Horse|Koala", + "source_id": int, + "title": str, + "volume": int, + }, + { + "#url": "https://mangapark.net/title/114972-aria", + "#comment": "'source' option", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + "#options": {"source": "koala"}, + "#pattern": mangapark.MangaparkChapterExtractor.pattern, + "#count": 70, + "source": "Koala", + "source_id": 15150116, + }, + { + "#url": "https://mangapark.com/title/114972-", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + }, + { + "#url": "https://mangapark.com/title/114972", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + }, + { + "#url": "https://mangapark.com/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + }, + { + "#url": "https://mangapark.org/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + }, + { + "#url": "https://mangapark.io/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + }, + { + "#url": "https://mangapark.me/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + }, ) diff --git a/test/results/mangaread.py b/test/results/mangaread.py index 4330a13d3..f45804940 100644 --- a/test/results/mangaread.py +++ b/test/results/mangaread.py @@ -1,122 +1,107 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangaread from gallery_dl import exception - +from gallery_dl.extractor import mangaread __tests__ = ( -{ - "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1053-3/", - "#category": ("", "mangaread", "chapter"), - "#class" : mangaread.MangareadChapterExtractor, - "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", - "#count" : 11, - - "manga" : "One Piece", - "title" : "", - "chapter" : 1053, - "chapter_minor": ".3", - "tags" : ["Oda Eiichiro"], - "lang" : "en", - "language" : "English", -}, - -{ - "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1000000/", - "#category": ("", "mangaread", "chapter"), - "#class" : mangaread.MangareadChapterExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi/chapter-10/", - "#category": ("", "mangaread", "chapter"), - "#class" : mangaread.MangareadChapterExtractor, - "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", - "#count" : 9, - - "manga" : "Kanan-sama wa Akumade Choroi", - "title" : "", - "chapter" : 10, - "chapter_minor": "", - "tags" : list, - "lang" : "en", - "language" : "English", -}, - -{ - "#url" : "https://www.mangaread.org/manga/above-all-gods/chapter146-5/", - "#comment" : "^^ no whitespace", - "#category": ("", "mangaread", "chapter"), - "#class" : mangaread.MangareadChapterExtractor, - "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", - "#count" : 6, - - "manga" : "Above All Gods", - "title" : "", - "chapter" : 146, - "chapter_minor": ".5", - "tags" : list, - "lang" : "en", - "language" : "English", -}, - -{ - "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi", - "#category": ("", "mangaread", "manga"), - "#class" : mangaread.MangareadMangaExtractor, - "#pattern" : r"https://www\.mangaread\.org/manga/kanan-sama-wa-akumade-choroi/chapter-\d+([_-].+)?/", - "#count" : ">= 13", - - "manga" : "Kanan-sama wa Akumade Choroi", - "author" : ["nonco"], - "artist" : ["nonco"], - "type" : "Manga", - "genres" : [ - "Comedy", - "Romance", - "Shounen", - "Supernatural", - ], - "rating" : float, - "release" : 2022, - "status" : "OnGoing", - "lang" : "en", - "language" : "English", - "manga_alt" : list, - "description": str, -}, - -{ - "#url" : "https://www.mangaread.org/manga/one-piece", - "#category": ("", "mangaread", "manga"), - "#class" : mangaread.MangareadMangaExtractor, - "#pattern" : r"https://www\.mangaread\.org/manga/one-piece/chapter-\d+(-.+)?/", - "#count" : ">= 1066", - - "manga" : "One Piece", - "author" : ["Oda Eiichiro"], - "artist" : ["Oda Eiichiro"], - "type" : "Manga", - "genres" : list, - "rating" : float, - "release" : 1997, - "status" : "OnGoing", - "lang" : "en", - "language" : "English", - "manga_alt" : ["One Piece"], - "description": str, -}, - -{ - "#url" : "https://www.mangaread.org/manga/doesnotexist", - "#category": ("", "mangaread", "manga"), - "#class" : mangaread.MangareadMangaExtractor, - "#exception": exception.HttpError, -}, - + { + "#url": "https://www.mangaread.org/manga/one-piece/chapter-1053-3/", + "#category": ("", "mangaread", "chapter"), + "#class": mangaread.MangareadChapterExtractor, + "#pattern": r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count": 11, + "manga": "One Piece", + "title": "", + "chapter": 1053, + "chapter_minor": ".3", + "tags": ["Oda Eiichiro"], + "lang": "en", + "language": "English", + }, + { + "#url": "https://www.mangaread.org/manga/one-piece/chapter-1000000/", + "#category": ("", "mangaread", "chapter"), + "#class": mangaread.MangareadChapterExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi/chapter-10/", + "#category": ("", "mangaread", "chapter"), + "#class": mangaread.MangareadChapterExtractor, + "#pattern": r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count": 9, + "manga": "Kanan-sama wa Akumade Choroi", + "title": "", + "chapter": 10, + "chapter_minor": "", + "tags": list, + "lang": "en", + "language": "English", + }, + { + "#url": "https://www.mangaread.org/manga/above-all-gods/chapter146-5/", + "#comment": "^^ no whitespace", + "#category": ("", "mangaread", "chapter"), + "#class": mangaread.MangareadChapterExtractor, + "#pattern": r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count": 6, + "manga": "Above All Gods", + "title": "", + "chapter": 146, + "chapter_minor": ".5", + "tags": list, + "lang": "en", + "language": "English", + }, + { + "#url": "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi", + "#category": ("", "mangaread", "manga"), + "#class": mangaread.MangareadMangaExtractor, + "#pattern": r"https://www\.mangaread\.org/manga/kanan-sama-wa-akumade-choroi/chapter-\d+([_-].+)?/", + "#count": ">= 13", + "manga": "Kanan-sama wa Akumade Choroi", + "author": ["nonco"], + "artist": ["nonco"], + "type": "Manga", + "genres": [ + "Comedy", + "Romance", + "Shounen", + "Supernatural", + ], + "rating": float, + "release": 2022, + "status": "OnGoing", + "lang": "en", + "language": "English", + "manga_alt": list, + "description": str, + }, + { + "#url": "https://www.mangaread.org/manga/one-piece", + "#category": ("", "mangaread", "manga"), + "#class": mangaread.MangareadMangaExtractor, + "#pattern": r"https://www\.mangaread\.org/manga/one-piece/chapter-\d+(-.+)?/", + "#count": ">= 1066", + "manga": "One Piece", + "author": ["Oda Eiichiro"], + "artist": ["Oda Eiichiro"], + "type": "Manga", + "genres": list, + "rating": float, + "release": 1997, + "status": "OnGoing", + "lang": "en", + "language": "English", + "manga_alt": ["One Piece"], + "description": str, + }, + { + "#url": "https://www.mangaread.org/manga/doesnotexist", + "#category": ("", "mangaread", "manga"), + "#class": mangaread.MangareadMangaExtractor, + "#exception": exception.HttpError, + }, ) diff --git a/test/results/mangasee.py b/test/results/mangasee.py index 8cf51d4e4..0f226b105 100644 --- a/test/results/mangasee.py +++ b/test/results/mangasee.py @@ -1,69 +1,63 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangasee import datetime +from gallery_dl.extractor import mangasee __tests__ = ( -{ - "#url" : "https://mangasee123.com/read-online/Tokyo-Innocent-chapter-4.5-page-1.html", - "#category": ("", "mangasee", "chapter"), - "#class" : mangasee.MangaseeChapterExtractor, - "#pattern" : r"https://[^/]+/manga/Tokyo-Innocent/0004\.5-00\d\.png", - "#count" : 8, - - "author" : ["NARUMI Naru"], - "chapter" : 4, - "chapter_minor" : ".5", - "chapter_string": "100045", - "count" : 8, - "date" : "dt:2020-01-20 21:52:53", - "extension" : "png", - "filename" : r"re:0004\.5-00\d", - "genre" : [ - "Comedy", - "Fantasy", - "Harem", - "Romance", - "Shounen", - "Supernatural", - ], - "index" : "1", - "lang" : "en", - "language" : "English", - "manga" : "Tokyo Innocent", - "page" : int, - "title" : "", -}, - -{ - "#url" : "https://mangasee123.com/manga/Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", - "#category": ("", "mangasee", "manga"), - "#class" : mangasee.MangaseeMangaExtractor, - "#pattern" : mangasee.MangaseeChapterExtractor.pattern, - "#count" : ">= 17", - - "author" : ["TAKASE Masaya"], - "chapter" : int, - "chapter_minor" : r"re:^|\.5$", - "chapter_string": r"re:100\d\d\d", - "date" : datetime.datetime, - "genre" : [ - "Comedy", - "Romance", - "School Life", - "Shounen", - "Slice of Life", - ], - "index" : "1", - "lang" : "en", - "language" : "English", - "manga" : "Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", - "title" : "", -}, - + { + "#url": "https://mangasee123.com/read-online/Tokyo-Innocent-chapter-4.5-page-1.html", + "#category": ("", "mangasee", "chapter"), + "#class": mangasee.MangaseeChapterExtractor, + "#pattern": r"https://[^/]+/manga/Tokyo-Innocent/0004\.5-00\d\.png", + "#count": 8, + "author": ["NARUMI Naru"], + "chapter": 4, + "chapter_minor": ".5", + "chapter_string": "100045", + "count": 8, + "date": "dt:2020-01-20 21:52:53", + "extension": "png", + "filename": r"re:0004\.5-00\d", + "genre": [ + "Comedy", + "Fantasy", + "Harem", + "Romance", + "Shounen", + "Supernatural", + ], + "index": "1", + "lang": "en", + "language": "English", + "manga": "Tokyo Innocent", + "page": int, + "title": "", + }, + { + "#url": "https://mangasee123.com/manga/Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", + "#category": ("", "mangasee", "manga"), + "#class": mangasee.MangaseeMangaExtractor, + "#pattern": mangasee.MangaseeChapterExtractor.pattern, + "#count": ">= 17", + "author": ["TAKASE Masaya"], + "chapter": int, + "chapter_minor": r"re:^|\.5$", + "chapter_string": r"re:100\d\d\d", + "date": datetime.datetime, + "genre": [ + "Comedy", + "Romance", + "School Life", + "Shounen", + "Slice of Life", + ], + "index": "1", + "lang": "en", + "language": "English", + "manga": "Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", + "title": "", + }, ) diff --git a/test/results/mangoxo.py b/test/results/mangoxo.py index b7c392a4a..a88fb147c 100644 --- a/test/results/mangoxo.py +++ b/test/results/mangoxo.py @@ -1,42 +1,36 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mangoxo - __tests__ = ( -{ - "#url" : "https://www.mangoxo.com/album/lzVOv1Q9", - "#category": ("", "mangoxo", "album"), - "#class" : mangoxo.MangoxoAlbumExtractor, - "#sha1_url": "ad921fe62663b06e7d73997f7d00646cab7bdd0d", - - "channel": { - "id" : "gaxO16d8", - "name" : "Phoenix", - "cover": str, + { + "#url": "https://www.mangoxo.com/album/lzVOv1Q9", + "#category": ("", "mangoxo", "album"), + "#class": mangoxo.MangoxoAlbumExtractor, + "#sha1_url": "ad921fe62663b06e7d73997f7d00646cab7bdd0d", + "channel": { + "id": "gaxO16d8", + "name": "Phoenix", + "cover": str, + }, + "album": { + "id": "lzVOv1Q9", + "name": r"re:池永康晟 Ikenaga Yasunari 透出古朴", + "date": "dt:2019-03-22 14:42:00", + "description": str, + }, + "id": int, + "num": int, + "count": 65, }, - "album" : { - "id" : "lzVOv1Q9", - "name" : r"re:池永康晟 Ikenaga Yasunari 透出古朴", - "date" : "dt:2019-03-22 14:42:00", - "description": str, + { + "#url": "https://www.mangoxo.com/phoenix/album", + "#category": ("", "mangoxo", "channel"), + "#class": mangoxo.MangoxoChannelExtractor, + "#pattern": mangoxo.MangoxoAlbumExtractor.pattern, + "#range": "1-30", + "#count": "> 20", }, - "id" : int, - "num" : int, - "count" : 65, -}, - -{ - "#url" : "https://www.mangoxo.com/phoenix/album", - "#category": ("", "mangoxo", "channel"), - "#class" : mangoxo.MangoxoChannelExtractor, - "#pattern" : mangoxo.MangoxoAlbumExtractor.pattern, - "#range" : "1-30", - "#count" : "> 20", -}, - ) diff --git a/test/results/mariowiki.py b/test/results/mariowiki.py index 72c4cd523..15bc33b90 100644 --- a/test/results/mariowiki.py +++ b/test/results/mariowiki.py @@ -1,19 +1,15 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.mariowiki.com/Rabbit", - "#category": ("wikimedia", "mariowiki", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#pattern" : r"https://mario\.wiki\.gallery/images/.+", - "#count" : range(20, 50), -}, - + { + "#url": "https://www.mariowiki.com/Rabbit", + "#category": ("wikimedia", "mariowiki", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#pattern": r"https://mario\.wiki\.gallery/images/.+", + "#count": range(20, 50), + }, ) diff --git a/test/results/mastodon.py b/test/results/mastodon.py index b6cb34641..ad160c606 100644 --- a/test/results/mastodon.py +++ b/test/results/mastodon.py @@ -1,32 +1,25 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mastodon - __tests__ = ( -{ - "#url" : "mastodon:https://donotsta.re/@elly/AcoUaA7EH1igiYKmFU", - "#category": ("mastodon", "donotsta.re", "status"), - "#class" : mastodon.MastodonStatusExtractor, - "#urls" : "https://asdf.donotsta.re/media/917e7722dd30d510686ce9f3717a1f722dac96fd974b5af5ec2ccbc8cbd740c6.png", - - "instance": "donotsta.re", - "instance_remote": None, -}, - -{ - "#url" : "mastodon:https://wanderingwires.net/@quarc/9qppkxzyd1ee3i9p", - "#comment" : "null moved account", - "#category": ("mastodon", "wanderingwires.net", "status"), - "#class" : mastodon.MastodonStatusExtractor, - "#urls" : "https://s3.wanderingwires.net/null/4377e826-72ab-4659-885c-fa12945eb207.png", - - "instance": "wanderingwires.net", - "instance_remote": None, -}, - + { + "#url": "mastodon:https://donotsta.re/@elly/AcoUaA7EH1igiYKmFU", + "#category": ("mastodon", "donotsta.re", "status"), + "#class": mastodon.MastodonStatusExtractor, + "#urls": "https://asdf.donotsta.re/media/917e7722dd30d510686ce9f3717a1f722dac96fd974b5af5ec2ccbc8cbd740c6.png", + "instance": "donotsta.re", + "instance_remote": None, + }, + { + "#url": "mastodon:https://wanderingwires.net/@quarc/9qppkxzyd1ee3i9p", + "#comment": "null moved account", + "#category": ("mastodon", "wanderingwires.net", "status"), + "#class": mastodon.MastodonStatusExtractor, + "#urls": "https://s3.wanderingwires.net/null/4377e826-72ab-4659-885c-fa12945eb207.png", + "instance": "wanderingwires.net", + "instance_remote": None, + }, ) diff --git a/test/results/mastodonsocial.py b/test/results/mastodonsocial.py index 1eed8a246..da12a7510 100644 --- a/test/results/mastodonsocial.py +++ b/test/results/mastodonsocial.py @@ -1,199 +1,174 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mastodon - __tests__ = ( -{ - "#url" : "https://mastodon.social/@jk", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, - "#pattern" : r"https://files.mastodon.social/media_attachments/files/(\d+/){3,}original/\w+", - "#range" : "1-60", - "#count" : 60, -}, - -{ - "#url" : "https://mastodon.social/@ponapalt@ukadon.shillest.net", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, - "#pattern" : r"https://files\.mastodon\.social/cache/media_attachments/files/.+/original/\w{16}\.\w+$", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://mastodon.social/@gallerydl", - "#comment" : "reblogged/'boosted' posts (#4580)", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, - "#options" : {"reblogs": True}, - "#archive" : False, - "#urls": ( - "https://files.mastodon.social/media_attachments/files/111/330/852/486/713/967/original/2c25ade55a9d1af2.jpg", - "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", - "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", - ), -}, - -{ - "#url" : "https://mastodon.social/@id:10843", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, -}, - -{ - "#url" : "https://mastodon.social/users/id:10843", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, -}, - -{ - "#url" : "https://mastodon.social/users/jk", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, -}, - -{ - "#url" : "https://mastodon.social/users/yoru_nine@pawoo.net", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, -}, - -{ - "#url" : "https://mastodon.social/web/@jk", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, -}, - -{ - "#url" : "https://mastodon.social/bookmarks", - "#category": ("mastodon", "mastodon.social", "bookmark"), - "#class" : mastodon.MastodonBookmarkExtractor, - "#auth" : True, - "#urls" : "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", -}, - -{ - "#url" : "https://mastodon.social/favourites", - "#category": ("mastodon", "mastodon.social", "favorite"), - "#class" : mastodon.MastodonFavoriteExtractor, - "#auth" : True, - "#urls" : "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", -}, - -{ - "#url" : "https://mastodon.social/lists/92653", - "#category": ("mastodon", "mastodon.social", "list"), - "#class" : mastodon.MastodonListExtractor, - "#auth" : True, - "#pattern" : r"https://files\.mastodon\.social/media_attachments/files/(\d+/){3,}original/\w+", - "#range" : "1-10", -}, - -{ - "#url" : "https://mastodon.social/tags/mastodon", - "#category": ("mastodon", "mastodon.social", "hashtag"), - "#class" : mastodon.MastodonHashtagExtractor, - "#pattern" : r"https://files\.mastodon\.social/media_attachments/files/(\d+/){3,}original/\w+", - "#range" : "1-10", -}, - -{ - "#url" : "https://mastodon.social/@gallerydl/following", - "#category": ("mastodon", "mastodon.social", "following"), - "#class" : mastodon.MastodonFollowingExtractor, - "#extractor": False, - "#urls" : ( - "https://mastodon.ie/@RustyBertrand", - "https://ravenation.club/@soundwarrior20", - "https://mastodon.social/@0x4f", - "https://mastodon.social/@christianselig", - "https://saturation.social/@clive", - "https://mastodon.social/@sjvn", - ), - - "acct" : str, - "avatar" : r"re:https://files.mastodon.social/.+\.\w+$", - "avatar_static" : r"re:https://files.mastodon.social/.+\.\w+$", - "bot" : False, - "created_at" : r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z", - "discoverable" : True, - "display_name" : str, - "emojis" : [], - "fields" : list, - "followers_count": int, - "following_count": int, - "group" : False, - "header" : str, - "header_static" : str, - "id" : r"re:\d+", - "last_status_at": r"re:\d{4}-\d{2}-\d{2}", - "locked" : bool, - "note" : str, - "statuses_count": int, - "uri" : str, - "url" : str, - "username" : str, - -}, - -{ - "#url" : "https://mastodon.social/@0x4f/following", - "#category": ("mastodon", "mastodon.social", "following"), - "#class" : mastodon.MastodonFollowingExtractor, -}, - -{ - "#url" : "https://mastodon.social/users/id:10843/following", - "#category": ("mastodon", "mastodon.social", "following"), - "#class" : mastodon.MastodonFollowingExtractor, -}, - -{ - "#url" : "https://mastodon.social/@jk/103794036899778366", - "#category": ("mastodon", "mastodon.social", "status"), - "#class" : mastodon.MastodonStatusExtractor, - "#count" : 4, - - "count": 4, - "num" : int, -}, - -{ - "#url" : "https://mastodon.social/@technewsbot@assortedflotsam.com/112360601113258881", - "#comment" : "card image", - "#category": ("mastodon", "mastodon.social", "status"), - "#class" : mastodon.MastodonStatusExtractor, - "#options" : {"cards": True}, - "#urls" : "https://files.mastodon.social/cache/preview_cards/images/095/900/335/original/83f0b4a793c84123.jpg", - - "media": { - "author_name" : "Tom Warren", - "author_url" : "https://www.theverge.com/authors/tom-warren", - "blurhash" : "UHBDWMCjVGM0k,XjnPM#0h+vkpb^RkjYSh$*", - "description" : "Microsoft’s big Xbox games showcase will take place on June 9th. It will include more games than last year and a special Call of Duty Direct will follow.", - "embed_url" : "", - "height" : 628, - "html" : "", - "id" : "card95900335", - "image" : "https://files.mastodon.social/cache/preview_cards/images/095/900/335/original/83f0b4a793c84123.jpg", - "image_description": "The Xbox showcase illustration", - "language" : "en", - "provider_name": "The Verge", - "provider_url": "", - "published_at": "2024-04-30T14:15:30.341Z", - "title" : "The Xbox games showcase airs June 9th, followed by a Call of Duty Direct", - "type" : "link", - "url" : "https://files.mastodon.social/cache/preview_cards/images/095/900/335/original/83f0b4a793c84123.jpg", - "weburl" : "https://www.theverge.com/2024/4/30/24145262/xbox-games-showcase-summer-2024-call-of-duty-direct", - "width" : 1200, + { + "#url": "https://mastodon.social/@jk", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + "#pattern": r"https://files.mastodon.social/media_attachments/files/(\d+/){3,}original/\w+", + "#range": "1-60", + "#count": 60, + }, + { + "#url": "https://mastodon.social/@ponapalt@ukadon.shillest.net", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + "#pattern": r"https://files\.mastodon\.social/cache/media_attachments/files/.+/original/\w{16}\.\w+$", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://mastodon.social/@gallerydl", + "#comment": "reblogged/'boosted' posts (#4580)", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + "#options": {"reblogs": True}, + "#archive": False, + "#urls": ( + "https://files.mastodon.social/media_attachments/files/111/330/852/486/713/967/original/2c25ade55a9d1af2.jpg", + "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", + "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", + ), + }, + { + "#url": "https://mastodon.social/@id:10843", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + }, + { + "#url": "https://mastodon.social/users/id:10843", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + }, + { + "#url": "https://mastodon.social/users/jk", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + }, + { + "#url": "https://mastodon.social/users/yoru_nine@pawoo.net", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + }, + { + "#url": "https://mastodon.social/web/@jk", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + }, + { + "#url": "https://mastodon.social/bookmarks", + "#category": ("mastodon", "mastodon.social", "bookmark"), + "#class": mastodon.MastodonBookmarkExtractor, + "#auth": True, + "#urls": "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", + }, + { + "#url": "https://mastodon.social/favourites", + "#category": ("mastodon", "mastodon.social", "favorite"), + "#class": mastodon.MastodonFavoriteExtractor, + "#auth": True, + "#urls": "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", + }, + { + "#url": "https://mastodon.social/lists/92653", + "#category": ("mastodon", "mastodon.social", "list"), + "#class": mastodon.MastodonListExtractor, + "#auth": True, + "#pattern": r"https://files\.mastodon\.social/media_attachments/files/(\d+/){3,}original/\w+", + "#range": "1-10", + }, + { + "#url": "https://mastodon.social/tags/mastodon", + "#category": ("mastodon", "mastodon.social", "hashtag"), + "#class": mastodon.MastodonHashtagExtractor, + "#pattern": r"https://files\.mastodon\.social/media_attachments/files/(\d+/){3,}original/\w+", + "#range": "1-10", + }, + { + "#url": "https://mastodon.social/@gallerydl/following", + "#category": ("mastodon", "mastodon.social", "following"), + "#class": mastodon.MastodonFollowingExtractor, + "#extractor": False, + "#urls": ( + "https://mastodon.ie/@RustyBertrand", + "https://ravenation.club/@soundwarrior20", + "https://mastodon.social/@0x4f", + "https://mastodon.social/@christianselig", + "https://saturation.social/@clive", + "https://mastodon.social/@sjvn", + ), + "acct": str, + "avatar": r"re:https://files.mastodon.social/.+\.\w+$", + "avatar_static": r"re:https://files.mastodon.social/.+\.\w+$", + "bot": False, + "created_at": r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z", + "discoverable": True, + "display_name": str, + "emojis": [], + "fields": list, + "followers_count": int, + "following_count": int, + "group": False, + "header": str, + "header_static": str, + "id": r"re:\d+", + "last_status_at": r"re:\d{4}-\d{2}-\d{2}", + "locked": bool, + "note": str, + "statuses_count": int, + "uri": str, + "url": str, + "username": str, + }, + { + "#url": "https://mastodon.social/@0x4f/following", + "#category": ("mastodon", "mastodon.social", "following"), + "#class": mastodon.MastodonFollowingExtractor, + }, + { + "#url": "https://mastodon.social/users/id:10843/following", + "#category": ("mastodon", "mastodon.social", "following"), + "#class": mastodon.MastodonFollowingExtractor, + }, + { + "#url": "https://mastodon.social/@jk/103794036899778366", + "#category": ("mastodon", "mastodon.social", "status"), + "#class": mastodon.MastodonStatusExtractor, + "#count": 4, + "count": 4, + "num": int, + }, + { + "#url": "https://mastodon.social/@technewsbot@assortedflotsam.com/112360601113258881", + "#comment": "card image", + "#category": ("mastodon", "mastodon.social", "status"), + "#class": mastodon.MastodonStatusExtractor, + "#options": {"cards": True}, + "#urls": "https://files.mastodon.social/cache/preview_cards/images/095/900/335/original/83f0b4a793c84123.jpg", + "media": { + "author_name": "Tom Warren", + "author_url": "https://www.theverge.com/authors/tom-warren", + "blurhash": "UHBDWMCjVGM0k,XjnPM#0h+vkpb^RkjYSh$*", + "description": "Microsoft’s big Xbox games showcase will take place on June 9th. It will include more games than last year and a special Call of Duty Direct will follow.", + "embed_url": "", + "height": 628, + "html": "", + "id": "card95900335", + "image": "https://files.mastodon.social/cache/preview_cards/images/095/900/335/original/83f0b4a793c84123.jpg", + "image_description": "The Xbox showcase illustration", + "language": "en", + "provider_name": "The Verge", + "provider_url": "", + "published_at": "2024-04-30T14:15:30.341Z", + "title": "The Xbox games showcase airs June 9th, followed by a Call of Duty Direct", + "type": "link", + "url": "https://files.mastodon.social/cache/preview_cards/images/095/900/335/original/83f0b4a793c84123.jpg", + "weburl": "https://www.theverge.com/2024/4/30/24145262/xbox-games-showcase-summer-2024-call-of-duty-direct", + "width": 1200, + }, }, - -}, - ) diff --git a/test/results/mediawiki.py b/test/results/mediawiki.py index 683d0d36b..fd01685ec 100644 --- a/test/results/mediawiki.py +++ b/test/results/mediawiki.py @@ -1,24 +1,20 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.mediawiki.org/wiki/Help:Navigation", - "#category": ("wikimedia", "mediawiki", "help"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#urls" : ( - "https://upload.wikimedia.org/wikipedia/commons/e/ec/OOjs_UI_icon_information-progressive.svg", - "https://upload.wikimedia.org/wikipedia/commons/6/62/PD-icon.svg", - "https://upload.wikimedia.org/wikipedia/commons/0/0e/Vector_Sidebar.png", - "https://upload.wikimedia.org/wikipedia/commons/7/77/Vector_page_tabs.png", - "https://upload.wikimedia.org/wikipedia/commons/6/6e/Vector_user_links.png", - ), -}, - + { + "#url": "https://www.mediawiki.org/wiki/Help:Navigation", + "#category": ("wikimedia", "mediawiki", "help"), + "#class": wikimedia.WikimediaArticleExtractor, + "#urls": ( + "https://upload.wikimedia.org/wikipedia/commons/e/ec/OOjs_UI_icon_information-progressive.svg", + "https://upload.wikimedia.org/wikipedia/commons/6/62/PD-icon.svg", + "https://upload.wikimedia.org/wikipedia/commons/0/0e/Vector_Sidebar.png", + "https://upload.wikimedia.org/wikipedia/commons/7/77/Vector_page_tabs.png", + "https://upload.wikimedia.org/wikipedia/commons/6/6e/Vector_user_links.png", + ), + }, ) diff --git a/test/results/michaelscameras.py b/test/results/michaelscameras.py index 42c76bbc6..9c9fecd70 100644 --- a/test/results/michaelscameras.py +++ b/test/results/michaelscameras.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://michaels.com.au/collections/microphones", - "#category": ("shopify", "michaelscameras", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://michaels.com.au/collections/audio/products/boya-by-wm4-pro-k5-2-4ghz-mic-android-1-1-101281", - "#category": ("shopify", "michaelscameras", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://michaels.com.au/collections/microphones", + "#category": ("shopify", "michaelscameras", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://michaels.com.au/collections/audio/products/boya-by-wm4-pro-k5-2-4ghz-mic-android-1-1-101281", + "#category": ("shopify", "michaelscameras", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/misskeydesign.py b/test/results/misskeydesign.py index 3560597bd..9b54984ed 100644 --- a/test/results/misskeydesign.py +++ b/test/results/misskeydesign.py @@ -1,53 +1,44 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import misskey - __tests__ = ( -{ - "#url" : "https://misskey.design/@machina_3D", - "#category": ("misskey", "misskey.design", "user"), - "#class" : misskey.MisskeyUserExtractor, - "#pattern" : r"https://file\.misskey\.design/post/[\w-]{36}\.\w+", - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://misskey.design/@blooddj@pawoo.net", - "#category": ("misskey", "misskey.design", "user"), - "#class" : misskey.MisskeyUserExtractor, - "#count" : "> 30", -}, - -{ - "#url" : "https://misskey.design/@kujyo_t/following", - "#category": ("misskey", "misskey.design", "following"), - "#class" : misskey.MisskeyFollowingExtractor, - "#count" : ">= 250", -}, - -{ - "#url" : "https://misskey.design/notes/9jva1danjc", - "#category": ("misskey", "misskey.design", "note"), - "#class" : misskey.MisskeyNoteExtractor, - "#urls" : "https://file.misskey.design/post/a8d27901-24e1-42ab-b8a6-1e09c98c6f55.webp", -}, - -{ - "#url" : "https://misskey.design/my/favorites", - "#category": ("misskey", "misskey.design", "favorite"), - "#class" : misskey.MisskeyFavoriteExtractor, -}, - -{ - "#url" : "https://misskey.design/api/i/favorites", - "#category": ("misskey", "misskey.design", "favorite"), - "#class" : misskey.MisskeyFavoriteExtractor, -}, - + { + "#url": "https://misskey.design/@machina_3D", + "#category": ("misskey", "misskey.design", "user"), + "#class": misskey.MisskeyUserExtractor, + "#pattern": r"https://file\.misskey\.design/post/[\w-]{36}\.\w+", + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://misskey.design/@blooddj@pawoo.net", + "#category": ("misskey", "misskey.design", "user"), + "#class": misskey.MisskeyUserExtractor, + "#count": "> 30", + }, + { + "#url": "https://misskey.design/@kujyo_t/following", + "#category": ("misskey", "misskey.design", "following"), + "#class": misskey.MisskeyFollowingExtractor, + "#count": ">= 250", + }, + { + "#url": "https://misskey.design/notes/9jva1danjc", + "#category": ("misskey", "misskey.design", "note"), + "#class": misskey.MisskeyNoteExtractor, + "#urls": "https://file.misskey.design/post/a8d27901-24e1-42ab-b8a6-1e09c98c6f55.webp", + }, + { + "#url": "https://misskey.design/my/favorites", + "#category": ("misskey", "misskey.design", "favorite"), + "#class": misskey.MisskeyFavoriteExtractor, + }, + { + "#url": "https://misskey.design/api/i/favorites", + "#category": ("misskey", "misskey.design", "favorite"), + "#class": misskey.MisskeyFavoriteExtractor, + }, ) diff --git a/test/results/misskeyio.py b/test/results/misskeyio.py index 4de64aa43..006aae18e 100644 --- a/test/results/misskeyio.py +++ b/test/results/misskeyio.py @@ -1,62 +1,52 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import misskey - __tests__ = ( -{ - "#url" : "https://misskey.io/@lithla", - "#category": ("misskey", "misskey.io", "user"), - "#class" : misskey.MisskeyUserExtractor, - "#pattern" : r"https://(media.misskeyusercontent.com|s\d+\.arkjp\.net)/(misskey|io)/[\w-]+\.\w+", - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://misskey.io/@blooddj@pawoo.net", - "#category": ("misskey", "misskey.io", "user"), - "#class" : misskey.MisskeyUserExtractor, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://misskey.io/@blooddj@pawoo.net/following", - "#category": ("misskey", "misskey.io", "following"), - "#class" : misskey.MisskeyFollowingExtractor, - "#count" : ">= 6", - "#extractor": False, -}, - -{ - "#url" : "https://misskey.io/notes/9bhqfo835v", - "#category": ("misskey", "misskey.io", "note"), - "#class" : misskey.MisskeyNoteExtractor, - "#pattern" : r"https://(media\.misskeyusercontent\.com|s\d+\.arkjp\.net)/misskey/[\w-]+\.\w+", - "#count" : 4, -}, - -{ - "#url" : "https://misskey.io/notes/9brq7z1re6", - "#category": ("misskey", "misskey.io", "note"), - "#class" : misskey.MisskeyNoteExtractor, -}, - -{ - "#url" : "https://misskey.io/my/favorites", - "#category": ("misskey", "misskey.io", "favorite"), - "#class" : misskey.MisskeyFavoriteExtractor, -}, - -{ - "#url" : "https://misskey.io/api/i/favorites", - "#category": ("misskey", "misskey.io", "favorite"), - "#class" : misskey.MisskeyFavoriteExtractor, -}, - + { + "#url": "https://misskey.io/@lithla", + "#category": ("misskey", "misskey.io", "user"), + "#class": misskey.MisskeyUserExtractor, + "#pattern": r"https://(media.misskeyusercontent.com|s\d+\.arkjp\.net)/(misskey|io)/[\w-]+\.\w+", + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://misskey.io/@blooddj@pawoo.net", + "#category": ("misskey", "misskey.io", "user"), + "#class": misskey.MisskeyUserExtractor, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://misskey.io/@blooddj@pawoo.net/following", + "#category": ("misskey", "misskey.io", "following"), + "#class": misskey.MisskeyFollowingExtractor, + "#count": ">= 6", + "#extractor": False, + }, + { + "#url": "https://misskey.io/notes/9bhqfo835v", + "#category": ("misskey", "misskey.io", "note"), + "#class": misskey.MisskeyNoteExtractor, + "#pattern": r"https://(media\.misskeyusercontent\.com|s\d+\.arkjp\.net)/misskey/[\w-]+\.\w+", + "#count": 4, + }, + { + "#url": "https://misskey.io/notes/9brq7z1re6", + "#category": ("misskey", "misskey.io", "note"), + "#class": misskey.MisskeyNoteExtractor, + }, + { + "#url": "https://misskey.io/my/favorites", + "#category": ("misskey", "misskey.io", "favorite"), + "#class": misskey.MisskeyFavoriteExtractor, + }, + { + "#url": "https://misskey.io/api/i/favorites", + "#category": ("misskey", "misskey.io", "favorite"), + "#class": misskey.MisskeyFavoriteExtractor, + }, ) diff --git a/test/results/modcloth.py b/test/results/modcloth.py index 67ad082ef..fcb21b909 100644 --- a/test/results/modcloth.py +++ b/test/results/modcloth.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://modcloth.com/collections/shoes", - "#category": ("shopify", "modcloth", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://modcloth.com/collections/shoes/products/heidii-brn", - "#category": ("shopify", "modcloth", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://modcloth.com/collections/shoes", + "#category": ("shopify", "modcloth", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://modcloth.com/collections/shoes/products/heidii-brn", + "#category": ("shopify", "modcloth", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/myhentaigallery.py b/test/results/myhentaigallery.py index a90e067a6..e0e6aa494 100644 --- a/test/results/myhentaigallery.py +++ b/test/results/myhentaigallery.py @@ -1,38 +1,31 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import myhentaigallery - __tests__ = ( -{ - "#url" : "https://myhentaigallery.com/g/16247", - "#category": ("", "myhentaigallery", "gallery"), - "#class" : myhentaigallery.MyhentaigalleryGalleryExtractor, - "#pattern" : r"https://(cdn|images)\.myhentaicomics\.com/m\w\w/images/[^/]+/original/\d+\.jpg", - - "artist" : list, - "count" : 11, - "gallery_id": 16247, - "group" : list, - "parodies" : list, - "tags" : ["Giantess"], - "title" : "Attack Of The 50ft Woman 1", -}, - -{ - "#url" : "https://myhentaigallery.com/gallery/thumbnails/16247", - "#category": ("", "myhentaigallery", "gallery"), - "#class" : myhentaigallery.MyhentaigalleryGalleryExtractor, -}, - -{ - "#url" : "https://myhentaigallery.com/gallery/show/16247/1", - "#category": ("", "myhentaigallery", "gallery"), - "#class" : myhentaigallery.MyhentaigalleryGalleryExtractor, -}, - + { + "#url": "https://myhentaigallery.com/g/16247", + "#category": ("", "myhentaigallery", "gallery"), + "#class": myhentaigallery.MyhentaigalleryGalleryExtractor, + "#pattern": r"https://(cdn|images)\.myhentaicomics\.com/m\w\w/images/[^/]+/original/\d+\.jpg", + "artist": list, + "count": 11, + "gallery_id": 16247, + "group": list, + "parodies": list, + "tags": ["Giantess"], + "title": "Attack Of The 50ft Woman 1", + }, + { + "#url": "https://myhentaigallery.com/gallery/thumbnails/16247", + "#category": ("", "myhentaigallery", "gallery"), + "#class": myhentaigallery.MyhentaigalleryGalleryExtractor, + }, + { + "#url": "https://myhentaigallery.com/gallery/show/16247/1", + "#category": ("", "myhentaigallery", "gallery"), + "#class": myhentaigallery.MyhentaigalleryGalleryExtractor, + }, ) diff --git a/test/results/myportfolio.py b/test/results/myportfolio.py index 718512937..fc76e7a69 100644 --- a/test/results/myportfolio.py +++ b/test/results/myportfolio.py @@ -1,51 +1,43 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import myportfolio from gallery_dl import exception - +from gallery_dl.extractor import myportfolio __tests__ = ( -{ - "#url" : "https://andrewling.myportfolio.com/volvo-xc-90-hybrid", - "#category": ("", "myportfolio", "gallery"), - "#class" : myportfolio.MyportfolioGalleryExtractor, - "#sha1_url" : "acea0690c76db0e5cf267648cefd86e921bc3499", - "#sha1_metadata": "6ac6befe2ee0af921d24cf1dd4a4ed71be06db6d", -}, - -{ - "#url" : "https://andrewling.myportfolio.com/", - "#category": ("", "myportfolio", "gallery"), - "#class" : myportfolio.MyportfolioGalleryExtractor, - "#pattern" : r"https://andrewling\.myportfolio\.com/[^/?#+]+$", - "#count" : ">= 6", -}, - -{ - "#url" : "https://stevenilousphotography.myportfolio.com/society", - "#category": ("", "myportfolio", "gallery"), - "#class" : myportfolio.MyportfolioGalleryExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "myportfolio:https://tooco.com.ar/6-of-diamonds-paradise-bird", - "#comment" : "custom domain", - "#category": ("", "myportfolio", "gallery"), - "#class" : myportfolio.MyportfolioGalleryExtractor, - "#count" : 3, -}, - -{ - "#url" : "myportfolio:https://tooco.com.ar/", - "#category": ("", "myportfolio", "gallery"), - "#class" : myportfolio.MyportfolioGalleryExtractor, - "#pattern" : myportfolio.MyportfolioGalleryExtractor.pattern, - "#count" : ">= 40", -}, - + { + "#url": "https://andrewling.myportfolio.com/volvo-xc-90-hybrid", + "#category": ("", "myportfolio", "gallery"), + "#class": myportfolio.MyportfolioGalleryExtractor, + "#sha1_url": "acea0690c76db0e5cf267648cefd86e921bc3499", + "#sha1_metadata": "6ac6befe2ee0af921d24cf1dd4a4ed71be06db6d", + }, + { + "#url": "https://andrewling.myportfolio.com/", + "#category": ("", "myportfolio", "gallery"), + "#class": myportfolio.MyportfolioGalleryExtractor, + "#pattern": r"https://andrewling\.myportfolio\.com/[^/?#+]+$", + "#count": ">= 6", + }, + { + "#url": "https://stevenilousphotography.myportfolio.com/society", + "#category": ("", "myportfolio", "gallery"), + "#class": myportfolio.MyportfolioGalleryExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "myportfolio:https://tooco.com.ar/6-of-diamonds-paradise-bird", + "#comment": "custom domain", + "#category": ("", "myportfolio", "gallery"), + "#class": myportfolio.MyportfolioGalleryExtractor, + "#count": 3, + }, + { + "#url": "myportfolio:https://tooco.com.ar/", + "#category": ("", "myportfolio", "gallery"), + "#class": myportfolio.MyportfolioGalleryExtractor, + "#pattern": myportfolio.MyportfolioGalleryExtractor.pattern, + "#count": ">= 40", + }, ) diff --git a/test/results/naver.py b/test/results/naver.py index a763a5b52..4d517939c 100644 --- a/test/results/naver.py +++ b/test/results/naver.py @@ -1,84 +1,73 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import naver - __tests__ = ( -{ - "#url" : "https://blog.naver.com/rlfqjxm0/221430673006", - "#category": ("", "naver", "post"), - "#class" : naver.NaverPostExtractor, - "#sha1_url" : "6c694f3aced075ed5e9511f1e796d14cb26619cc", - "#sha1_metadata": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", -}, - -{ - "#url" : "https://blog.naver.com/PostView.nhn?blogId=rlfqjxm0&logNo=221430673006", - "#category": ("", "naver", "post"), - "#class" : naver.NaverPostExtractor, - "#sha1_url" : "6c694f3aced075ed5e9511f1e796d14cb26619cc", - "#sha1_metadata": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", -}, - -{ - "#url" : "https://blog.naver.com/PostView.nhn?blogId=rlfqjxm0&logNo=70161391809", - "#comment" : "filenames in EUC-KR encoding (#5126)", - "#category": ("", "naver", "post"), - "#class" : naver.NaverPostExtractor, - "#urls": ( - "https://blogfiles.pstatic.net/20130305_23/ping9303_1362411028002Dpz9z_PNG/1_사본.png", - "https://blogfiles.pstatic.net/20130305_46/rlfqjxm0_1362473322580x33zi_PNG/오마갓합작.png", - ), - - "blog": { - "id" : "rlfqjxm0", - "num" : 43030507, - "user": "에나", + { + "#url": "https://blog.naver.com/rlfqjxm0/221430673006", + "#category": ("", "naver", "post"), + "#class": naver.NaverPostExtractor, + "#sha1_url": "6c694f3aced075ed5e9511f1e796d14cb26619cc", + "#sha1_metadata": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", }, - "post": { - "date" : "dt:2013-03-05 17:48:00", - "description": " ◈ PROMOTER ：핑수 ˚ 아담 EDITOR：핑수 넵：이크：핑수...", - "num" : 70161391809, - "title" : "[공유] { 합작}  OH, MY GOD! ~ 아 또 무슨 종말을 한다 그래~", + { + "#url": "https://blog.naver.com/PostView.nhn?blogId=rlfqjxm0&logNo=221430673006", + "#category": ("", "naver", "post"), + "#class": naver.NaverPostExtractor, + "#sha1_url": "6c694f3aced075ed5e9511f1e796d14cb26619cc", + "#sha1_metadata": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", + }, + { + "#url": "https://blog.naver.com/PostView.nhn?blogId=rlfqjxm0&logNo=70161391809", + "#comment": "filenames in EUC-KR encoding (#5126)", + "#category": ("", "naver", "post"), + "#class": naver.NaverPostExtractor, + "#urls": ( + "https://blogfiles.pstatic.net/20130305_23/ping9303_1362411028002Dpz9z_PNG/1_사본.png", + "https://blogfiles.pstatic.net/20130305_46/rlfqjxm0_1362473322580x33zi_PNG/오마갓합작.png", + ), + "blog": { + "id": "rlfqjxm0", + "num": 43030507, + "user": "에나", + }, + "post": { + "date": "dt:2013-03-05 17:48:00", + "description": " ◈ PROMOTER ：핑수 ˚ 아담 EDITOR：핑수 넵：이크：핑수...", + "num": 70161391809, + "title": "[공유] { 합작}  OH, MY GOD! ~ 아 또 무슨 종말을 한다 그래~", + }, + "count": 2, + "num": range(1, 2), + "filename": r"re:1_사본|오마갓합작", + "extension": "png", + }, + { + "#url": "https://blog.naver.com/PostView.naver?blogId=rlfqjxm0&logNo=221430673006", + "#category": ("", "naver", "post"), + "#class": naver.NaverPostExtractor, + }, + { + "#url": "https://blog.naver.com/gukjung", + "#category": ("", "naver", "blog"), + "#class": naver.NaverBlogExtractor, + "#pattern": naver.NaverPostExtractor.pattern, + "#range": "1-12", + "#count": 12, + }, + { + "#url": "https://blog.naver.com/PostList.nhn?blogId=gukjung", + "#category": ("", "naver", "blog"), + "#class": naver.NaverBlogExtractor, + "#pattern": naver.NaverPostExtractor.pattern, + "#range": "1-12", + "#count": 12, + }, + { + "#url": "https://blog.naver.com/PostList.naver?blogId=gukjung", + "#category": ("", "naver", "blog"), + "#class": naver.NaverBlogExtractor, }, - "count" : 2, - "num" : range(1, 2), - "filename" : r"re:1_사본|오마갓합작", - "extension": "png", -}, - -{ - "#url" : "https://blog.naver.com/PostView.naver?blogId=rlfqjxm0&logNo=221430673006", - "#category": ("", "naver", "post"), - "#class" : naver.NaverPostExtractor, -}, - -{ - "#url" : "https://blog.naver.com/gukjung", - "#category": ("", "naver", "blog"), - "#class" : naver.NaverBlogExtractor, - "#pattern" : naver.NaverPostExtractor.pattern, - "#range" : "1-12", - "#count" : 12, -}, - -{ - "#url" : "https://blog.naver.com/PostList.nhn?blogId=gukjung", - "#category": ("", "naver", "blog"), - "#class" : naver.NaverBlogExtractor, - "#pattern" : naver.NaverPostExtractor.pattern, - "#range" : "1-12", - "#count" : 12, -}, - -{ - "#url" : "https://blog.naver.com/PostList.naver?blogId=gukjung", - "#category": ("", "naver", "blog"), - "#class" : naver.NaverBlogExtractor, -}, - ) diff --git a/test/results/naverwebtoon.py b/test/results/naverwebtoon.py index 36b670863..fdff61560 100644 --- a/test/results/naverwebtoon.py +++ b/test/results/naverwebtoon.py @@ -1,123 +1,109 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import naverwebtoon - __tests__ = ( -{ - "#url" : "https://comic.naver.com/webtoon/detail?titleId=26458&no=1&weekday=tue", - "#category": ("", "naverwebtoon", "episode"), - "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, - "#count" : 14, - "#sha1_url" : "47a956ba8c7a837213d5985f50c569fcff986f75", - "#sha1_content": "3806b6e8befbb1920048de9888dfce6220f69a60", - - "author" : ["김규삼"], - "artist" : ["김규삼"], - "comic" : "N의등대-눈의등대", - "count" : 14, - "episode" : "1", - "extension": "jpg", - "num" : int, - "tags" : [ - "스릴러", - "완결무료", - "완결스릴러", - ], - "title" : "n의 등대 - 눈의 등대 1화", - "title_id" : "26458", -}, - -{ - "#url" : "https://comic.naver.com/challenge/detail?titleId=765124&no=1", - "#category": ("", "naverwebtoon", "episode"), - "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, - "#pattern" : r"https://image-comic\.pstatic\.net/user_contents_data/challenge_comic/2021/01/19/342586/upload_7149856273586337846\.jpeg", - "#count" : 1, - - "author" : ["kemi****"], - "artist" : [], - "comic" : "우니 모두의 이야기", - "count" : 1, - "episode" : "1", - "extension": "jpeg", - "filename" : "upload_7149856273586337846", - "num" : 1, - "tags" : [ - "일상툰", - "우니모두의이야기", - "퇴사", - "입사", - "신입사원", - "사회초년생", - "회사원", - "20대", - ], - "title" : "퇴사하다", - "title_id" : "765124", -}, - -{ - "#url" : "https://comic.naver.com/bestChallenge/detail?titleId=620732&no=334", - "#comment" : "empty tags (#5120)", - "#category": ("", "naverwebtoon", "episode"), - "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, - "#count" : 9, - - "artist" : [], - "author" : ["안트로anthrokim"], - "comic" : "백일몽화원", - "count" : 9, - "episode" : "334", - "num" : range(1, 9), - "tags" : [], - "title" : "321화... 성(省)", - "title_id": "620732", -}, - -{ - "#url" : "https://comic.naver.com/bestChallenge/detail.nhn?titleId=771467&no=3", - "#category": ("", "naverwebtoon", "episode"), - "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, - "#pattern" : r"https://image-comic\.pstatic\.net/user_contents_data/challenge_comic/2021/04/28/345534/upload_3617293622396203109\.jpeg", - "#count" : 1, -}, - -{ - "#url" : "https://comic.naver.com/webtoon/list?titleId=22073", - "#category": ("", "naverwebtoon", "comic"), - "#class" : naverwebtoon.NaverwebtoonComicExtractor, - "#pattern" : naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, - "#count" : 32, -}, - -{ - "#url" : "https://comic.naver.com/webtoon/list?titleId=765124", - "#comment" : "/webtoon/ path for 'challenge' comic (#5123)", - "#category": ("", "naverwebtoon", "comic"), - "#class" : naverwebtoon.NaverwebtoonComicExtractor, - "#range" : "1", - "#urls" : "https://comic.naver.com/challenge/detail?titleId=765124&no=1", -}, - -{ - "#url" : "https://comic.naver.com/challenge/list?titleId=765124", - "#category": ("", "naverwebtoon", "comic"), - "#class" : naverwebtoon.NaverwebtoonComicExtractor, - "#pattern" : naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, - "#count" : 24, -}, - -{ - "#url" : "https://comic.naver.com/bestChallenge/list.nhn?titleId=789786", - "#category": ("", "naverwebtoon", "comic"), - "#class" : naverwebtoon.NaverwebtoonComicExtractor, - "#pattern" : naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, - "#count" : ">= 12", -}, - + { + "#url": "https://comic.naver.com/webtoon/detail?titleId=26458&no=1&weekday=tue", + "#category": ("", "naverwebtoon", "episode"), + "#class": naverwebtoon.NaverwebtoonEpisodeExtractor, + "#count": 14, + "#sha1_url": "47a956ba8c7a837213d5985f50c569fcff986f75", + "#sha1_content": "3806b6e8befbb1920048de9888dfce6220f69a60", + "author": ["김규삼"], + "artist": ["김규삼"], + "comic": "N의등대-눈의등대", + "count": 14, + "episode": "1", + "extension": "jpg", + "num": int, + "tags": [ + "스릴러", + "완결무료", + "완결스릴러", + ], + "title": "n의 등대 - 눈의 등대 1화", + "title_id": "26458", + }, + { + "#url": "https://comic.naver.com/challenge/detail?titleId=765124&no=1", + "#category": ("", "naverwebtoon", "episode"), + "#class": naverwebtoon.NaverwebtoonEpisodeExtractor, + "#pattern": r"https://image-comic\.pstatic\.net/user_contents_data/challenge_comic/2021/01/19/342586/upload_7149856273586337846\.jpeg", + "#count": 1, + "author": ["kemi****"], + "artist": [], + "comic": "우니 모두의 이야기", + "count": 1, + "episode": "1", + "extension": "jpeg", + "filename": "upload_7149856273586337846", + "num": 1, + "tags": [ + "일상툰", + "우니모두의이야기", + "퇴사", + "입사", + "신입사원", + "사회초년생", + "회사원", + "20대", + ], + "title": "퇴사하다", + "title_id": "765124", + }, + { + "#url": "https://comic.naver.com/bestChallenge/detail?titleId=620732&no=334", + "#comment": "empty tags (#5120)", + "#category": ("", "naverwebtoon", "episode"), + "#class": naverwebtoon.NaverwebtoonEpisodeExtractor, + "#count": 9, + "artist": [], + "author": ["안트로anthrokim"], + "comic": "백일몽화원", + "count": 9, + "episode": "334", + "num": range(1, 9), + "tags": [], + "title": "321화... 성(省)", + "title_id": "620732", + }, + { + "#url": "https://comic.naver.com/bestChallenge/detail.nhn?titleId=771467&no=3", + "#category": ("", "naverwebtoon", "episode"), + "#class": naverwebtoon.NaverwebtoonEpisodeExtractor, + "#pattern": r"https://image-comic\.pstatic\.net/user_contents_data/challenge_comic/2021/04/28/345534/upload_3617293622396203109\.jpeg", + "#count": 1, + }, + { + "#url": "https://comic.naver.com/webtoon/list?titleId=22073", + "#category": ("", "naverwebtoon", "comic"), + "#class": naverwebtoon.NaverwebtoonComicExtractor, + "#pattern": naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, + "#count": 32, + }, + { + "#url": "https://comic.naver.com/webtoon/list?titleId=765124", + "#comment": "/webtoon/ path for 'challenge' comic (#5123)", + "#category": ("", "naverwebtoon", "comic"), + "#class": naverwebtoon.NaverwebtoonComicExtractor, + "#range": "1", + "#urls": "https://comic.naver.com/challenge/detail?titleId=765124&no=1", + }, + { + "#url": "https://comic.naver.com/challenge/list?titleId=765124", + "#category": ("", "naverwebtoon", "comic"), + "#class": naverwebtoon.NaverwebtoonComicExtractor, + "#pattern": naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, + "#count": 24, + }, + { + "#url": "https://comic.naver.com/bestChallenge/list.nhn?titleId=789786", + "#category": ("", "naverwebtoon", "comic"), + "#class": naverwebtoon.NaverwebtoonComicExtractor, + "#pattern": naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, + "#count": ">= 12", + }, ) diff --git a/test/results/newgrounds.py b/test/results/newgrounds.py index 8ff37b2d4..bb5e358f1 100644 --- a/test/results/newgrounds.py +++ b/test/results/newgrounds.py @@ -1,405 +1,358 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import newgrounds - __tests__ = ( -{ - "#url" : "https://www.newgrounds.com/art/view/tomfulp/ryu-is-hawt", - "#category": ("", "newgrounds", "image"), - "#class" : newgrounds.NewgroundsImageExtractor, - "#urls" : "https://art.ngfiles.com/images/1993000/1993615_4474_tomfulp_ryu-is-hawt.44f81090378ae9c257a5e46a8e17cc4d.gif?f1695674895", - "#sha1_content": "8f395e08333eb2457ba8d8b715238f8910221365", - - "artist" : ["tomfulp"], - "comment" : "Consider this the bottom threshold for scouted artists.In fact consider it BELOW the bottom threshold.", - "date" : "dt:2009-06-04 14:44:05", - "description": "Consider this the bottom threshold for scouted artists. In fact consider it BELOW the bottom threshold. ", - "favorites" : int, - "filename" : "1993615_4474_tomfulp_ryu-is-hawt.44f81090378ae9c257a5e46a8e17cc4d", - "height" : 476, - "index" : 1993615, - "rating" : "e", - "score" : float, - "tags" : [ - "ryu", - "streetfighter", - ], - "title" : "Ryu is Hawt", - "type" : "article", - "user" : "tomfulp", - "width" : 447, -}, - -{ - "#url" : "https://art.ngfiles.com/images/0/94_tomfulp_ryu-is-hawt.gif", - "#category": ("", "newgrounds", "image"), - "#class" : newgrounds.NewgroundsImageExtractor, - "#urls" : "https://art.ngfiles.com/images/1993000/1993615_4474_tomfulp_ryu-is-hawt.44f81090378ae9c257a5e46a8e17cc4d.gif?f1695674895", -}, - -{ - "#url" : "https://www.newgrounds.com/art/view/sailoryon/yon-dream-buster", - "#comment" : "embedded file in 'comments' (#1033)", - "#category": ("", "newgrounds", "image"), - "#class" : newgrounds.NewgroundsImageExtractor, - "#urls" : ( - "https://art.ngfiles.com/images/1438000/1438673_sailoryon_yon-dream-buster.jpg?f1601058173", - "https://art.ngfiles.com/comments/172000/iu_172374_7112211.jpg", - ), -}, - -{ - "#url" : "https://www.newgrounds.com/art/view/zedrinbot/lewd-animation-tutorial", - "#comment" : "extra files in 'art-image-row' elements - WebP to GIF (#4642)", - "#category": ("", "newgrounds", "image"), - "#class" : newgrounds.NewgroundsImageExtractor, - "#urls" : ( - "https://art.ngfiles.com/images/5091000/5091275_45067_zedrinbot_untitled-5091275.0a9d27ed2bc265a7e89478ed6ad6f86f.gif?f1696187399", - "https://art.ngfiles.com/images/5091000/5091275_45071_zedrinbot_untitled-5091275.6fdc62eaef43528fb1c9bda624d30a3d.gif?f1696187437", - "https://art.ngfiles.com/images/5091000/5091275_45070_zedrinbot_untitled-5091275.0d7334746374465bd448908b88d1f810.gif?f1696187435", - "https://art.ngfiles.com/images/5091000/5091275_45072_zedrinbot_untitled-5091275.6fdc62eaef43528fb1c9bda624d30a3d.gif?f1696187438", - "https://art.ngfiles.com/images/5091000/5091275_45073_zedrinbot_untitled-5091275.20aa05c1cd22fd058e8c68ce58f5a302.gif?f1696187439", - ), -}, - -{ - "#url" : "https://www.newgrounds.com/art/view/zedrinbot/nazrin-tanlines", - "#comment" : "extra files in 'art-image-row' elements - native PNG files (#4642)", - "#category": ("", "newgrounds", "image"), - "#class" : newgrounds.NewgroundsImageExtractor, - "#auth" : True, - "#urls" : ( - "https://art.ngfiles.com/images/5009000/5009916_14628_zedrinbot_nazrin-tanlines.265f7b6beec5855a349e2646e90cbc01.png?f1695698131", - "https://art.ngfiles.com/images/5009000/5009916_14632_zedrinbot_nazrin-tanlines.40bd62fbf5875806cda6b004b348114a.png?f1695727318", - "https://art.ngfiles.com/images/5009000/5009916_14634_zedrinbot_nazrin-tanlines.40bd62fbf5875806cda6b004b348114a.png?f1695727321", - "https://art.ngfiles.com/images/5009000/5009916_14633_zedrinbot_nazrin-tanlines.40bd62fbf5875806cda6b004b348114a.png?f1695727318", - "https://art.ngfiles.com/images/5009000/5009916_14635_zedrinbot_nazrin-tanlines.6a7aa4fd63e5f8077ad29314568246cc.png?f1695727321", - "https://art.ngfiles.com/images/5009000/5009916_14636_zedrinbot_nazrin-tanlines.6a7aa4fd63e5f8077ad29314568246cc.png?f1695727322", - ), -}, - -{ - "#url" : "https://www.newgrounds.com/art/view/bacun/kill-la-kill-10th-anniversary", - "#comment" : "extra files in 'imageData' block (#4642)", - "#category": ("", "newgrounds", "image"), - "#class" : newgrounds.NewgroundsImageExtractor, - "#urls" : ( - "https://art.ngfiles.com/images/5127000/5127150_93307_bacun_kill-la-kill-10th-anniversary.61adfe309bec342f9db55fd44397235b.png?f1697310027", - "https://art.ngfiles.com/images/5127000/5127150_94250_bacun_kill-la-kill-10th-anniversary.64fdf525fa38c1ab34defac4b354bc7a.png?f1697332109", - ), -}, - -{ - "#url" : "https://www.newgrounds.com/art/view/sockdotclip/trickin-treats", - "#comment" : "extra files in comment section as '= 3", -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/art/page/3", - "#class" : newgrounds.NewgroundsArtExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/art?page=3", - "#class" : newgrounds.NewgroundsArtExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/audio", - "#class" : newgrounds.NewgroundsAudioExtractor, - "#pattern" : r"https://(audio\.ngfiles\.com/\d+/\d+_.+\.mp3|uploads\.ungrounded\.net/.+\.png)", - "#count" : ">= 10", -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/audio?page=3", - "#class" : newgrounds.NewgroundsAudioExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/movies", - "#class" : newgrounds.NewgroundsMoviesExtractor, - "#pattern" : r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/movies/?page=3", - "#class" : newgrounds.NewgroundsMoviesExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/games", - "#class" : newgrounds.NewgroundsGamesExtractor, - "#pattern" : r"https://uploads.ungrounded.net(/alternate)?/(\d+/\d+_.+|tmp/.+)", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/games?page=3", - "#class" : newgrounds.NewgroundsGamesExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com", - "#class" : newgrounds.NewgroundsUserExtractor, - "#urls" : "https://tomfulp.newgrounds.com/art", -}, - -{ - "#url" : "https://tomfulp.newgrounds.com", - "#class" : newgrounds.NewgroundsUserExtractor, - "#options" : {"include": "all"}, - "#urls" : ( - "https://tomfulp.newgrounds.com/art", - "https://tomfulp.newgrounds.com/audio", - "https://tomfulp.newgrounds.com/games", - "https://tomfulp.newgrounds.com/movies", - ), -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/art", - "#class" : newgrounds.NewgroundsFavoriteExtractor, - "#range" : "1-10", - "#count" : ">= 10", -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/art?page=3", - "#class" : newgrounds.NewgroundsFavoriteExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/audio", - "#class" : newgrounds.NewgroundsFavoriteExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/movies", - "#class" : newgrounds.NewgroundsFavoriteExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/", - "#class" : newgrounds.NewgroundsFavoriteExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/following", - "#class" : newgrounds.NewgroundsFollowingExtractor, - "#pattern" : newgrounds.NewgroundsUserExtractor.pattern, - "#range" : "76-125", - "#count" : 50, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/following?page=3", - "#class" : newgrounds.NewgroundsFollowingExtractor, -}, - - -{ - "#url" : "https://www.newgrounds.com/search/conduct/art?terms=tree", - "#class" : newgrounds.NewgroundsSearchExtractor, - "#pattern" : newgrounds.NewgroundsImageExtractor.pattern, - "#range" : "1-10", - "#count" : 10, - - "search_tags": "tree", -}, - -{ - "#url" : "https://www.newgrounds.com/search/conduct/movies?terms=tree", - "#class" : newgrounds.NewgroundsSearchExtractor, - "#pattern" : r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.newgrounds.com/search/conduct/audio?advanced=1&terms=tree+green+nature&match=tdtu&genre=5&suitabilities=e%2Cm", - "#class" : newgrounds.NewgroundsSearchExtractor, -}, - + "date": "dt:2015-02-23 19:31:59", + "description": "From The ZJ Report Show!", + "favorites": int, + "index": 609768, + "rating": "", + "score": float, + "tags": [ + "fulp", + "interview", + "tom", + "zj", + ], + "title": "ZJ Interviews Tom Fulp!", + "type": "music.song", + "user": "zj", + }, + { + "#url": "https://www.newgrounds.com/portal/view/161181/format/flash", + "#comment": "flash animation (#1257)", + "#category": ("", "newgrounds", "media"), + "#class": newgrounds.NewgroundsMediaExtractor, + "#urls": "https://uploads.ungrounded.net/161000/161181_ddautta_mask__550x281_.swf", + "type": "movie", + }, + { + "#url": "https://www.newgrounds.com/portal/view/758545", + "#comment": "format selection (#1729)", + "#category": ("", "newgrounds", "media"), + "#class": newgrounds.NewgroundsMediaExtractor, + "#options": {"format": "720p"}, + "#pattern": r"https://uploads\.ungrounded\.net/alternate/1482000/1482860_alternate_102516\.720p\.mp4\?\d+", + }, + { + "#url": "https://www.newgrounds.com/portal/view/717744", + "#comment": "'adult' rated (#2456)", + "#category": ("", "newgrounds", "media"), + "#class": newgrounds.NewgroundsMediaExtractor, + "#options": {"username": None}, + "#count": 1, + }, + { + "#url": "https://www.newgrounds.com/portal/view/829032", + "#comment": "flash game", + "#category": ("", "newgrounds", "media"), + "#class": newgrounds.NewgroundsMediaExtractor, + "#urls": ( + "https://uploads.ungrounded.net/829000/829032_picovsbeardx.swf", + "https://uploads.ungrounded.net/tmp/img/521000/iu_521265_5431202.gif", + ), + "artist": [ + "dungeonation", + "carpetbakery", + "animalspeakandrews", + "bill", + "chipollo", + "dylz49", + "gappyshamp", + "pinktophat", + "rad", + "shapeshiftingblob", + "tomfulp", + "voicesbycorey", + "psychogoldfish", + ], + "comment": r"re:The children are expendable. Take out the ", + "date": "dt:2022-01-10 23:00:57", + "description": "Bloodshed in The Big House that Blew...again!", + "favorites": int, + "index": 829032, + "post_url": "https://www.newgrounds.com/portal/view/829032", + "rating": "m", + "score": float, + "tags": [ + "assassin", + "boyfriend", + "darnell", + "nene", + "pico", + "picos-school", + ], + "title": "PICO VS BEAR DX", + "type": "game", + "url": "https://uploads.ungrounded.net/829000/829032_picovsbeardx.swf", + }, + { + "#url": "https://tomfulp.newgrounds.com/art", + "#class": newgrounds.NewgroundsArtExtractor, + "#pattern": newgrounds.NewgroundsImageExtractor.pattern, + "#count": ">= 3", + }, + { + "#url": "https://tomfulp.newgrounds.com/art/page/3", + "#class": newgrounds.NewgroundsArtExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/art?page=3", + "#class": newgrounds.NewgroundsArtExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/audio", + "#class": newgrounds.NewgroundsAudioExtractor, + "#pattern": r"https://(audio\.ngfiles\.com/\d+/\d+_.+\.mp3|uploads\.ungrounded\.net/.+\.png)", + "#count": ">= 10", + }, + { + "#url": "https://tomfulp.newgrounds.com/audio?page=3", + "#class": newgrounds.NewgroundsAudioExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/movies", + "#class": newgrounds.NewgroundsMoviesExtractor, + "#pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://tomfulp.newgrounds.com/movies/?page=3", + "#class": newgrounds.NewgroundsMoviesExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/games", + "#class": newgrounds.NewgroundsGamesExtractor, + "#pattern": r"https://uploads.ungrounded.net(/alternate)?/(\d+/\d+_.+|tmp/.+)", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://tomfulp.newgrounds.com/games?page=3", + "#class": newgrounds.NewgroundsGamesExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com", + "#class": newgrounds.NewgroundsUserExtractor, + "#urls": "https://tomfulp.newgrounds.com/art", + }, + { + "#url": "https://tomfulp.newgrounds.com", + "#class": newgrounds.NewgroundsUserExtractor, + "#options": {"include": "all"}, + "#urls": ( + "https://tomfulp.newgrounds.com/art", + "https://tomfulp.newgrounds.com/audio", + "https://tomfulp.newgrounds.com/games", + "https://tomfulp.newgrounds.com/movies", + ), + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/art", + "#class": newgrounds.NewgroundsFavoriteExtractor, + "#range": "1-10", + "#count": ">= 10", + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/art?page=3", + "#class": newgrounds.NewgroundsFavoriteExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/audio", + "#class": newgrounds.NewgroundsFavoriteExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/movies", + "#class": newgrounds.NewgroundsFavoriteExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/", + "#class": newgrounds.NewgroundsFavoriteExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/following", + "#class": newgrounds.NewgroundsFollowingExtractor, + "#pattern": newgrounds.NewgroundsUserExtractor.pattern, + "#range": "76-125", + "#count": 50, + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/following?page=3", + "#class": newgrounds.NewgroundsFollowingExtractor, + }, + { + "#url": "https://www.newgrounds.com/search/conduct/art?terms=tree", + "#class": newgrounds.NewgroundsSearchExtractor, + "#pattern": newgrounds.NewgroundsImageExtractor.pattern, + "#range": "1-10", + "#count": 10, + "search_tags": "tree", + }, + { + "#url": "https://www.newgrounds.com/search/conduct/movies?terms=tree", + "#class": newgrounds.NewgroundsSearchExtractor, + "#pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.newgrounds.com/search/conduct/audio?advanced=1&terms=tree+green+nature&match=tdtu&genre=5&suitabilities=e%2Cm", + "#class": newgrounds.NewgroundsSearchExtractor, + }, ) diff --git a/test/results/nhentai.py b/test/results/nhentai.py index 02667ac43..9a4addc36 100644 --- a/test/results/nhentai.py +++ b/test/results/nhentai.py @@ -1,110 +1,95 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import nhentai - __tests__ = ( -{ - "#url" : "https://nhentai.net/g/147850/", - "#category": ("", "nhentai", "gallery"), - "#class" : nhentai.NhentaiGalleryExtractor, - "#sha1_url": "5179dbf0f96af44005a0ff705a0ad64ac26547d0", - - "title" : r"re:\[Morris\] Amazon no Hiyaku \| Amazon Elixir", - "title_en" : str, - "title_ja" : str, - "gallery_id": 147850, - "media_id" : 867789, - "count" : 16, - "date" : 1446050915, - "scanlator" : "", - "artist" : ["morris"], - "group" : list, - "parody" : list, - "characters": list, - "tags" : list, - "type" : "manga", - "lang" : "en", - "language" : "English", - "width" : int, - "height" : int, -}, - -{ - "#url" : "https://nhentai.net/g/538045/", - "#comment" : "webp (#6442)", - "#class" : nhentai.NhentaiGalleryExtractor, - "#range" : "4-7", - "#urls" : ( - "https://i.nhentai.net/galleries/3115523/4.jpg", - "https://i.nhentai.net/galleries/3115523/5.webp", - "https://i.nhentai.net/galleries/3115523/6.webp", - "https://i.nhentai.net/galleries/3115523/7.jpg", - ), -}, - -{ - "#url" : "https://nhentai.net/tag/sole-female/", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, - "#pattern" : nhentai.NhentaiGalleryExtractor.pattern, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://nhentai.net/artist/itou-life/", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, -}, - -{ - "#url" : "https://nhentai.net/group/itou-life/", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, -}, - -{ - "#url" : "https://nhentai.net/parody/touhou-project/", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, -}, - -{ - "#url" : "https://nhentai.net/character/patchouli-knowledge/popular", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, -}, - -{ - "#url" : "https://nhentai.net/category/doujinshi/popular-today", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, -}, - -{ - "#url" : "https://nhentai.net/language/english/popular-week", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, -}, - -{ - "#url" : "https://nhentai.net/search/?q=touhou", - "#category": ("", "nhentai", "search"), - "#class" : nhentai.NhentaiSearchExtractor, - "#pattern" : nhentai.NhentaiGalleryExtractor.pattern, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://nhentai.net/favorites/", - "#category": ("", "nhentai", "favorite"), - "#class" : nhentai.NhentaiFavoriteExtractor, -}, - + { + "#url": "https://nhentai.net/g/147850/", + "#category": ("", "nhentai", "gallery"), + "#class": nhentai.NhentaiGalleryExtractor, + "#sha1_url": "5179dbf0f96af44005a0ff705a0ad64ac26547d0", + "title": r"re:\[Morris\] Amazon no Hiyaku \| Amazon Elixir", + "title_en": str, + "title_ja": str, + "gallery_id": 147850, + "media_id": 867789, + "count": 16, + "date": 1446050915, + "scanlator": "", + "artist": ["morris"], + "group": list, + "parody": list, + "characters": list, + "tags": list, + "type": "manga", + "lang": "en", + "language": "English", + "width": int, + "height": int, + }, + { + "#url": "https://nhentai.net/g/538045/", + "#comment": "webp (#6442)", + "#class": nhentai.NhentaiGalleryExtractor, + "#range": "4-7", + "#urls": ( + "https://i.nhentai.net/galleries/3115523/4.jpg", + "https://i.nhentai.net/galleries/3115523/5.webp", + "https://i.nhentai.net/galleries/3115523/6.webp", + "https://i.nhentai.net/galleries/3115523/7.jpg", + ), + }, + { + "#url": "https://nhentai.net/tag/sole-female/", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + "#pattern": nhentai.NhentaiGalleryExtractor.pattern, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://nhentai.net/artist/itou-life/", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + }, + { + "#url": "https://nhentai.net/group/itou-life/", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + }, + { + "#url": "https://nhentai.net/parody/touhou-project/", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + }, + { + "#url": "https://nhentai.net/character/patchouli-knowledge/popular", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + }, + { + "#url": "https://nhentai.net/category/doujinshi/popular-today", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + }, + { + "#url": "https://nhentai.net/language/english/popular-week", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + }, + { + "#url": "https://nhentai.net/search/?q=touhou", + "#category": ("", "nhentai", "search"), + "#class": nhentai.NhentaiSearchExtractor, + "#pattern": nhentai.NhentaiGalleryExtractor.pattern, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://nhentai.net/favorites/", + "#category": ("", "nhentai", "favorite"), + "#class": nhentai.NhentaiFavoriteExtractor, + }, ) diff --git a/test/results/nijie.py b/test/results/nijie.py index 829b62015..4d7280d69 100644 --- a/test/results/nijie.py +++ b/test/results/nijie.py @@ -1,193 +1,171 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import nijie import datetime -from gallery_dl import exception +from gallery_dl import exception +from gallery_dl.extractor import nijie __tests__ = ( -{ - "#url" : "https://nijie.info/members.php?id=44", - "#category": ("Nijie", "nijie", "user"), - "#class" : nijie.NijieUserExtractor, - "#urls" : ( - "https://nijie.info/members_illust.php?id=44", - "https://nijie.info/members_dojin.php?id=44", - ), -}, - -{ - "#url" : "https://nijie.info/members_illust.php?id=44", - "#category": ("Nijie", "nijie", "illustration"), - "#class" : nijie.NijieIllustrationExtractor, - "#urls": ( - "https://pic.nijie.net/04/nijie/14/44/44/illust/0_0_f46c08462568c2f1_be95d7.jpg", - "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg", - ), - - "artist_id" : 44, - "artist_name": "ED", - "count" : 1, - "date" : datetime.datetime, - "description": str, - "extension" : "jpg", - "filename" : str, - "image_id" : int, - "num" : 0, - "tags" : list, - "title" : str, - "url" : r"re:https://pic.nijie.net/\d+/nijie/.*jpg$", - "user_id" : 44, - "user_name" : "ED", -}, - -{ - "#url" : "https://nijie.info/members_illust.php?id=43", - "#category": ("Nijie", "nijie", "illustration"), - "#class" : nijie.NijieIllustrationExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://nijie.info/members_dojin.php?id=6782", - "#category": ("Nijie", "nijie", "doujin"), - "#class" : nijie.NijieDoujinExtractor, - "#count" : ">= 18", - - "user_id" : 6782, - "user_name": "ジョニー＠アビオン村", -}, - -{ - "#url" : "https://nijie.info/user_like_illust_view.php?id=44", - "#category": ("Nijie", "nijie", "favorite"), - "#class" : nijie.NijieFavoriteExtractor, - "#count" : ">= 16", - - "user_id" : 44, - "user_name": "ED", -}, - -{ - "#url" : "https://nijie.info/history_nuita.php?id=728995", - "#category": ("Nijie", "nijie", "nuita"), - "#class" : nijie.NijieNuitaExtractor, - "#range" : "1-10", - "#count" : 10, - - "user_id" : 728995, - "user_name": "莚", -}, - -{ - "#url" : "https://nijie.info/like_user_view.php", - "#category": ("Nijie", "nijie", "feed"), - "#class" : nijie.NijieFeedExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://nijie.info/like_my.php", - "#category": ("Nijie", "nijie", "followed"), - "#class" : nijie.NijieFollowedExtractor, -}, - -{ - "#url" : "https://nijie.info/view.php?id=70720", - "#category": ("Nijie", "nijie", "image"), - "#class" : nijie.NijieImageExtractor, - "#urls" : "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg", - "#sha1_url" : "3d654e890212ba823c9647754767336aebc0a743", - "#sha1_metadata": "58e716bcb03b431cae901178c198c787908e1c0c", - "#sha1_content" : "d85e3ea896ed5e4da0bca2390ad310a4df716ca6", - - "artist_id" : 44, - "artist_name": "ED", - "count" : 1, - "date" : "dt:2014-01-18 19:58:21", - "description": "租絵にてお邪魔いたし候\r\n是非ともこの”おっぱい”をご高覧賜りたく馳せ参じた次第\r\n長文にて失礼仕る\r\n\r\nまず全景でありますが、首を右に傾けてみて頂きたい\r\nこの絵図は茶碗を眺めていた私が思わぬ美しさにて昇天したときのものを、筆をとり、したためたものである（トレースではない）\r\n筆は疾風の如く走り、半刻過ぎには私好みの”おっぱい”になっていたのである！\r\n次に細部をみて頂きたい\r\n絵図を正面から見直して頂くと、なんとはんなりと美しいお椀型をしたおっぱいであろうか　　右手から緩やかに生まれる曲線は左手に進むにつれて、穏やかな歪みを含み流れる　　これは所謂轆轤目であるが三重の紐でおっぱいをぐるぐると巻きつけた情景そのままであり、この歪みから茶碗の均整は崩れ、たぷんたぷんのおっぱいの重量感を醸し出している！\r\nさらに左手に進めば梅花皮（カイラギ）を孕んだ高大が現れる　今回は点線にて表現するが、その姿は乳首から母乳が噴出するが如く　或は精子をぶっかけられたが如く　白くとろっとした釉薬の凝固が素晴しい景色をつくりだしているのである！\r\n最後には極めつけ、すくっと螺旋を帯びながらそそり立つ兜巾（ときん）！この情景はまさしく乳首である！　　全体をふんわりと盛り上げさせる乳輪にちょこっと存在する乳頭はぺろりと舌で確かめ勃起させたくなる風情がある！\r\n\r\nこれを”おっぱい”と呼ばずなんと呼ぼうや！？\r\n\r\n興奮のあまり失礼致した\r\n御免", - "extension" : "jpg", - "filename" : "0_0_28e8c02d921bee33_9222d3", - "image_id" : 70720, - "num" : 0, - "tags" : ["おっぱい"], - "title" : "俺好高麗井戸茶碗　銘おっぱい", - "url" : "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg", - "user_id" : 44, - "user_name" : "ED", -}, - -{ - "#url" : "https://nijie.info/view.php?id=594044", - "#category": ("Nijie", "nijie", "image"), - "#class" : nijie.NijieImageExtractor, - "#urls": ( - "https://pic.nijie.net/02/nijie/23m12/09/49509/illust/0_0_63568cc428259d50_45ca51.jpg", - "https://pic.nijie.net/01/nijie/23m12/09/49509/illust/594044_0_1c94b7cc4503589f_79c66c.jpg", - "https://pic.nijie.net/02/nijie/23m12/09/49509/illust/594044_1_9f4737ad48bf43c7_8f1e8e.jpg", - "https://pic.nijie.net/01/nijie/23m12/09/49509/illust/594044_2_a162861fac970a45_38c5f8.jpg", - ), - - "artist_id" : 49509, - "artist_name": "黒川　竜", - "count" : 4, - "date" : "dt:2023-12-02 04:19:29", - "description": "【DLサイトコム】ウィンターセール　30%OFF\r\n期間：2024年2月14日まで\r\n【toloveるドリンク】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ042727.html\r\n【toloveるドリンク２】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ043289.html\r\n【クランクランBIG】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ043564.html", - "image_id" : 594044, - "num" : range(0, 3), - "tags" : [ - "オリジナル", - "漫画", - "中出し", - "爆乳", - "巨乳", - "ToLOVEる", - "宣伝", - "クラン・クラン", - "マクロスF", - ], - "title" : "【DLサイトコム】ウィンターセール", - "url" : str, - "user_id" : 49509, - "user_name" : "黒川　竜", -}, - -{ - "#url" : "https://nijie.info/view.php?id=37078", - "#comment" : "'view_side_dojin' thumbnails (#5049)", - "#category": ("Nijie", "nijie", "image"), - "#class" : nijie.NijieImageExtractor, - "#urls" : "https://pic.nijie.net/03/nijie/13/98/498/illust/0_0_703023d18ca8d058_bca943.jpg", -}, - -{ - "#url" : "https://nijie.info/view.php?id=385585", - "#comment" : "video (#5707)", - "#category": ("Nijie", "nijie", "image"), - "#class" : nijie.NijieImageExtractor, - "#urls" : ( - "https://pic.nijie.net/01/nijie/20/82/59182/illust/0_0_162270ef49e2ee28_fab5ae.mp4", - "https://pic.nijie.net/04/nijie/20/82/59182/illust/385585_0_ff2d5d19129530d5_b2821e.jpg", - "https://pic.nijie.net/01/nijie/20/82/59182/illust/385585_1_7ee1a2a67bed2f84_212d67.jpg", - ), -}, - -{ - "#url" : "https://nijie.info/view.php?id=70724", - "#category": ("Nijie", "nijie", "image"), - "#class" : nijie.NijieImageExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://nijie.info/view_popup.php?id=70720", - "#category": ("Nijie", "nijie", "image"), - "#class" : nijie.NijieImageExtractor, -}, - + { + "#url": "https://nijie.info/members.php?id=44", + "#category": ("Nijie", "nijie", "user"), + "#class": nijie.NijieUserExtractor, + "#urls": ( + "https://nijie.info/members_illust.php?id=44", + "https://nijie.info/members_dojin.php?id=44", + ), + }, + { + "#url": "https://nijie.info/members_illust.php?id=44", + "#category": ("Nijie", "nijie", "illustration"), + "#class": nijie.NijieIllustrationExtractor, + "#urls": ( + "https://pic.nijie.net/04/nijie/14/44/44/illust/0_0_f46c08462568c2f1_be95d7.jpg", + "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg", + ), + "artist_id": 44, + "artist_name": "ED", + "count": 1, + "date": datetime.datetime, + "description": str, + "extension": "jpg", + "filename": str, + "image_id": int, + "num": 0, + "tags": list, + "title": str, + "url": r"re:https://pic.nijie.net/\d+/nijie/.*jpg$", + "user_id": 44, + "user_name": "ED", + }, + { + "#url": "https://nijie.info/members_illust.php?id=43", + "#category": ("Nijie", "nijie", "illustration"), + "#class": nijie.NijieIllustrationExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://nijie.info/members_dojin.php?id=6782", + "#category": ("Nijie", "nijie", "doujin"), + "#class": nijie.NijieDoujinExtractor, + "#count": ">= 18", + "user_id": 6782, + "user_name": "ジョニー＠アビオン村", + }, + { + "#url": "https://nijie.info/user_like_illust_view.php?id=44", + "#category": ("Nijie", "nijie", "favorite"), + "#class": nijie.NijieFavoriteExtractor, + "#count": ">= 16", + "user_id": 44, + "user_name": "ED", + }, + { + "#url": "https://nijie.info/history_nuita.php?id=728995", + "#category": ("Nijie", "nijie", "nuita"), + "#class": nijie.NijieNuitaExtractor, + "#range": "1-10", + "#count": 10, + "user_id": 728995, + "user_name": "莚", + }, + { + "#url": "https://nijie.info/like_user_view.php", + "#category": ("Nijie", "nijie", "feed"), + "#class": nijie.NijieFeedExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://nijie.info/like_my.php", + "#category": ("Nijie", "nijie", "followed"), + "#class": nijie.NijieFollowedExtractor, + }, + { + "#url": "https://nijie.info/view.php?id=70720", + "#category": ("Nijie", "nijie", "image"), + "#class": nijie.NijieImageExtractor, + "#urls": "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg", + "#sha1_url": "3d654e890212ba823c9647754767336aebc0a743", + "#sha1_metadata": "58e716bcb03b431cae901178c198c787908e1c0c", + "#sha1_content": "d85e3ea896ed5e4da0bca2390ad310a4df716ca6", + "artist_id": 44, + "artist_name": "ED", + "count": 1, + "date": "dt:2014-01-18 19:58:21", + "description": "租絵にてお邪魔いたし候\r\n是非ともこの”おっぱい”をご高覧賜りたく馳せ参じた次第\r\n長文にて失礼仕る\r\n\r\nまず全景でありますが、首を右に傾けてみて頂きたい\r\nこの絵図は茶碗を眺めていた私が思わぬ美しさにて昇天したときのものを、筆をとり、したためたものである（トレースではない）\r\n筆は疾風の如く走り、半刻過ぎには私好みの”おっぱい”になっていたのである！\r\n次に細部をみて頂きたい\r\n絵図を正面から見直して頂くと、なんとはんなりと美しいお椀型をしたおっぱいであろうか　　右手から緩やかに生まれる曲線は左手に進むにつれて、穏やかな歪みを含み流れる　　これは所謂轆轤目であるが三重の紐でおっぱいをぐるぐると巻きつけた情景そのままであり、この歪みから茶碗の均整は崩れ、たぷんたぷんのおっぱいの重量感を醸し出している！\r\nさらに左手に進めば梅花皮（カイラギ）を孕んだ高大が現れる　今回は点線にて表現するが、その姿は乳首から母乳が噴出するが如く　或は精子をぶっかけられたが如く　白くとろっとした釉薬の凝固が素晴しい景色をつくりだしているのである！\r\n最後には極めつけ、すくっと螺旋を帯びながらそそり立つ兜巾（ときん）！この情景はまさしく乳首である！　　全体をふんわりと盛り上げさせる乳輪にちょこっと存在する乳頭はぺろりと舌で確かめ勃起させたくなる風情がある！\r\n\r\nこれを”おっぱい”と呼ばずなんと呼ぼうや！？\r\n\r\n興奮のあまり失礼致した\r\n御免", + "extension": "jpg", + "filename": "0_0_28e8c02d921bee33_9222d3", + "image_id": 70720, + "num": 0, + "tags": ["おっぱい"], + "title": "俺好高麗井戸茶碗　銘おっぱい", + "url": "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg", + "user_id": 44, + "user_name": "ED", + }, + { + "#url": "https://nijie.info/view.php?id=594044", + "#category": ("Nijie", "nijie", "image"), + "#class": nijie.NijieImageExtractor, + "#urls": ( + "https://pic.nijie.net/02/nijie/23m12/09/49509/illust/0_0_63568cc428259d50_45ca51.jpg", + "https://pic.nijie.net/01/nijie/23m12/09/49509/illust/594044_0_1c94b7cc4503589f_79c66c.jpg", + "https://pic.nijie.net/02/nijie/23m12/09/49509/illust/594044_1_9f4737ad48bf43c7_8f1e8e.jpg", + "https://pic.nijie.net/01/nijie/23m12/09/49509/illust/594044_2_a162861fac970a45_38c5f8.jpg", + ), + "artist_id": 49509, + "artist_name": "黒川　竜", + "count": 4, + "date": "dt:2023-12-02 04:19:29", + "description": "【DLサイトコム】ウィンターセール　30%OFF\r\n期間：2024年2月14日まで\r\n【toloveるドリンク】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ042727.html\r\n【toloveるドリンク２】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ043289.html\r\n【クランクランBIG】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ043564.html", + "image_id": 594044, + "num": range(3), + "tags": [ + "オリジナル", + "漫画", + "中出し", + "爆乳", + "巨乳", + "ToLOVEる", + "宣伝", + "クラン・クラン", + "マクロスF", + ], + "title": "【DLサイトコム】ウィンターセール", + "url": str, + "user_id": 49509, + "user_name": "黒川　竜", + }, + { + "#url": "https://nijie.info/view.php?id=37078", + "#comment": "'view_side_dojin' thumbnails (#5049)", + "#category": ("Nijie", "nijie", "image"), + "#class": nijie.NijieImageExtractor, + "#urls": "https://pic.nijie.net/03/nijie/13/98/498/illust/0_0_703023d18ca8d058_bca943.jpg", + }, + { + "#url": "https://nijie.info/view.php?id=385585", + "#comment": "video (#5707)", + "#category": ("Nijie", "nijie", "image"), + "#class": nijie.NijieImageExtractor, + "#urls": ( + "https://pic.nijie.net/01/nijie/20/82/59182/illust/0_0_162270ef49e2ee28_fab5ae.mp4", + "https://pic.nijie.net/04/nijie/20/82/59182/illust/385585_0_ff2d5d19129530d5_b2821e.jpg", + "https://pic.nijie.net/01/nijie/20/82/59182/illust/385585_1_7ee1a2a67bed2f84_212d67.jpg", + ), + }, + { + "#url": "https://nijie.info/view.php?id=70724", + "#category": ("Nijie", "nijie", "image"), + "#class": nijie.NijieImageExtractor, + "#count": 0, + }, + { + "#url": "https://nijie.info/view_popup.php?id=70720", + "#category": ("Nijie", "nijie", "image"), + "#class": nijie.NijieImageExtractor, + }, ) diff --git a/test/results/noop.py b/test/results/noop.py index 935568177..b381952e8 100644 --- a/test/results/noop.py +++ b/test/results/noop.py @@ -1,28 +1,22 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import noop - __tests__ = ( -{ - "#url" : "noop", - "#class" : noop.NoopExtractor, - "#urls" : (), - "#count" : 0, -}, - -{ - "#url" : "nop", - "#class" : noop.NoopExtractor, -}, - -{ - "#url" : "NOOP", - "#class" : noop.NoopExtractor, -}, - + { + "#url": "noop", + "#class": noop.NoopExtractor, + "#urls": (), + "#count": 0, + }, + { + "#url": "nop", + "#class": noop.NoopExtractor, + }, + { + "#url": "NOOP", + "#class": noop.NoopExtractor, + }, ) diff --git a/test/results/nozomi.py b/test/results/nozomi.py index 042c79209..e21fc5439 100644 --- a/test/results/nozomi.py +++ b/test/results/nozomi.py @@ -1,106 +1,91 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import nozomi - __tests__ = ( -{ - "#url" : "https://nozomi.la/post/3649262.html", - "#category": ("", "nozomi", "post"), - "#class" : nozomi.NozomiPostExtractor, - "#pattern" : r"https://w\.nozomi\.la/2/15/aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5cf5a358caf604bf73152\.webp", - "#sha1_url" : "e5525e717aec712843be8b88592d6406ae9e60ba", - "#sha1_content": "6d62c4a7fea50c0a89d499603c4e7a2b4b9bffa8", - - "artist" : ["hammer (sunset beach)"], - "character": ["patchouli knowledge"], - "copyright": ["touhou"], - "dataid" : r"re:aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5", - "date" : "dt:2016-07-26 02:32:03", - "extension": "webp", - "filename" : str, - "height" : 768, - "is_video" : False, - "postid" : 3649262, - "tags" : list, - "type" : "jpg", - "url" : str, - "width" : 1024, -}, - -{ - "#url" : "https://nozomi.la/post/25588032.html", - "#comment" : "multiple images per post", - "#category": ("", "nozomi", "post"), - "#class" : nozomi.NozomiPostExtractor, - "#count" : 7, - "#sha1_url" : "fb956ccedcf2cf509739d26e2609e910244aa56c", - "#sha1_metadata": "516ca5cbd0d2a46a8ce26679d6e08de5ac42184b", -}, - -{ - "#url" : "https://nozomi.la/post/130309.html", - "#comment" : "empty 'date' (#1163)", - "#category": ("", "nozomi", "post"), - "#class" : nozomi.NozomiPostExtractor, - - "date": None, -}, - -{ - "#url" : "https://nozomi.la/post/1647.html", - "#comment" : "gif", - "#category": ("", "nozomi", "post"), - "#class" : nozomi.NozomiPostExtractor, - "#pattern" : r"https://g\.nozomi\.la/a/f0/d1b06469e00d72e4f6346209c149db459d76b58a074416c260ed93cc31fa9f0a\.gif", - "#sha1_content": "952efb78252bbc9fb56df2e8fafb68d5e6364181", -}, - -{ - "#url" : "https://nozomi.la/post/2269847.html", - "#comment" : "video", - "#category": ("", "nozomi", "post"), - "#class" : nozomi.NozomiPostExtractor, - "#pattern" : r"https://v\.nozomi\.la/d/0e/ff88398862669783691b31519f2bea3a35c24b6e62e3ba2d89b4409e41c660ed\.webm", - "#sha1_content": "57065e6c16da7b1c7098a63b36fb0c6c6f1b9bca", -}, - -{ - "#url" : "https://nozomi.la/", - "#category": ("", "nozomi", "index"), - "#class" : nozomi.NozomiIndexExtractor, -}, - -{ - "#url" : "https://nozomi.la/index-2.html", - "#category": ("", "nozomi", "index"), - "#class" : nozomi.NozomiIndexExtractor, -}, - -{ - "#url" : "https://nozomi.la/index-Popular-33.html", - "#category": ("", "nozomi", "index"), - "#class" : nozomi.NozomiIndexExtractor, -}, - -{ - "#url" : "https://nozomi.la/tag/3:1_aspect_ratio-1.html", - "#category": ("", "nozomi", "tag"), - "#class" : nozomi.NozomiTagExtractor, - "#pattern" : r"^https://[wgv]\.nozomi\.la/\w/\w\w/\w+\.\w+$", - "#range" : "1-25", - "#count" : ">= 25", -}, - -{ - "#url" : "https://nozomi.la/search.html?q=hibiscus%203:4_ratio#1", - "#category": ("", "nozomi", "search"), - "#class" : nozomi.NozomiSearchExtractor, - "#count" : ">= 5", -}, - + { + "#url": "https://nozomi.la/post/3649262.html", + "#category": ("", "nozomi", "post"), + "#class": nozomi.NozomiPostExtractor, + "#pattern": r"https://w\.nozomi\.la/2/15/aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5cf5a358caf604bf73152\.webp", + "#sha1_url": "e5525e717aec712843be8b88592d6406ae9e60ba", + "#sha1_content": "6d62c4a7fea50c0a89d499603c4e7a2b4b9bffa8", + "artist": ["hammer (sunset beach)"], + "character": ["patchouli knowledge"], + "copyright": ["touhou"], + "dataid": r"re:aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5", + "date": "dt:2016-07-26 02:32:03", + "extension": "webp", + "filename": str, + "height": 768, + "is_video": False, + "postid": 3649262, + "tags": list, + "type": "jpg", + "url": str, + "width": 1024, + }, + { + "#url": "https://nozomi.la/post/25588032.html", + "#comment": "multiple images per post", + "#category": ("", "nozomi", "post"), + "#class": nozomi.NozomiPostExtractor, + "#count": 7, + "#sha1_url": "fb956ccedcf2cf509739d26e2609e910244aa56c", + "#sha1_metadata": "516ca5cbd0d2a46a8ce26679d6e08de5ac42184b", + }, + { + "#url": "https://nozomi.la/post/130309.html", + "#comment": "empty 'date' (#1163)", + "#category": ("", "nozomi", "post"), + "#class": nozomi.NozomiPostExtractor, + "date": None, + }, + { + "#url": "https://nozomi.la/post/1647.html", + "#comment": "gif", + "#category": ("", "nozomi", "post"), + "#class": nozomi.NozomiPostExtractor, + "#pattern": r"https://g\.nozomi\.la/a/f0/d1b06469e00d72e4f6346209c149db459d76b58a074416c260ed93cc31fa9f0a\.gif", + "#sha1_content": "952efb78252bbc9fb56df2e8fafb68d5e6364181", + }, + { + "#url": "https://nozomi.la/post/2269847.html", + "#comment": "video", + "#category": ("", "nozomi", "post"), + "#class": nozomi.NozomiPostExtractor, + "#pattern": r"https://v\.nozomi\.la/d/0e/ff88398862669783691b31519f2bea3a35c24b6e62e3ba2d89b4409e41c660ed\.webm", + "#sha1_content": "57065e6c16da7b1c7098a63b36fb0c6c6f1b9bca", + }, + { + "#url": "https://nozomi.la/", + "#category": ("", "nozomi", "index"), + "#class": nozomi.NozomiIndexExtractor, + }, + { + "#url": "https://nozomi.la/index-2.html", + "#category": ("", "nozomi", "index"), + "#class": nozomi.NozomiIndexExtractor, + }, + { + "#url": "https://nozomi.la/index-Popular-33.html", + "#category": ("", "nozomi", "index"), + "#class": nozomi.NozomiIndexExtractor, + }, + { + "#url": "https://nozomi.la/tag/3:1_aspect_ratio-1.html", + "#category": ("", "nozomi", "tag"), + "#class": nozomi.NozomiTagExtractor, + "#pattern": r"^https://[wgv]\.nozomi\.la/\w/\w\w/\w+\.\w+$", + "#range": "1-25", + "#count": ">= 25", + }, + { + "#url": "https://nozomi.la/search.html?q=hibiscus%203:4_ratio#1", + "#category": ("", "nozomi", "search"), + "#class": nozomi.NozomiSearchExtractor, + "#count": ">= 5", + }, ) diff --git a/test/results/nsfwalbum.py b/test/results/nsfwalbum.py index cabd3e104..d4e88709a 100644 --- a/test/results/nsfwalbum.py +++ b/test/results/nsfwalbum.py @@ -1,36 +1,31 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import nsfwalbum - __tests__ = ( -{ - "#url" : "https://nsfwalbum.com/album/401611", - "#category": ("", "nsfwalbum", "album"), - "#class" : nsfwalbum.NsfwalbumAlbumExtractor, - "#range" : "1-5", - "#urls" : ( - "https://img70.imgspice.com/i/05457/mio2bu5xbrxe.jpg", - "https://img70.imgspice.com/i/05457/zgpxa8kr4h1d.jpg", - "https://img70.imgspice.com/i/05457/3379nxsm9lx8.jpg", - "https://img70.imgspice.com/i/05457/pncrkhspuoa3.jpg", - "https://img70.imgspice.com/i/05457/128b2odt216a.jpg", - ), - - "album_id" : 401611, - "extension": "jpg", - "filename" : str, - "height" : range(1365, 2048), - "id" : int, - "models" : [], - "num" : range(1, 5), - "studio" : "Met-Art", - "title" : "Met-Art - Katherine A - Difuza 25.05.2014 (134 photos)(4368 X 2912)", - "width" : range(1365, 2048), -}, - + { + "#url": "https://nsfwalbum.com/album/401611", + "#category": ("", "nsfwalbum", "album"), + "#class": nsfwalbum.NsfwalbumAlbumExtractor, + "#range": "1-5", + "#urls": ( + "https://img70.imgspice.com/i/05457/mio2bu5xbrxe.jpg", + "https://img70.imgspice.com/i/05457/zgpxa8kr4h1d.jpg", + "https://img70.imgspice.com/i/05457/3379nxsm9lx8.jpg", + "https://img70.imgspice.com/i/05457/pncrkhspuoa3.jpg", + "https://img70.imgspice.com/i/05457/128b2odt216a.jpg", + ), + "album_id": 401611, + "extension": "jpg", + "filename": str, + "height": range(1365, 2048), + "id": int, + "models": [], + "num": range(1, 5), + "studio": "Met-Art", + "title": "Met-Art - Katherine A - Difuza 25.05.2014 (134 photos)(4368 X 2912)", + "width": range(1365, 2048), + }, ) diff --git a/test/results/ohpolly.py b/test/results/ohpolly.py index 3ce1968b9..2430908cc 100644 --- a/test/results/ohpolly.py +++ b/test/results/ohpolly.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://www.ohpolly.com/collections/dresses-mini-dresses", - "#category": ("shopify", "ohpolly", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.ohpolly.com/products/edonia-ruched-triangle-cup-a-line-mini-dress-brown", - "#category": ("shopify", "ohpolly", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://www.ohpolly.com/collections/dresses-mini-dresses", + "#category": ("shopify", "ohpolly", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.ohpolly.com/products/edonia-ruched-triangle-cup-a-line-mini-dress-brown", + "#category": ("shopify", "ohpolly", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/omgmiamiswimwear.py b/test/results/omgmiamiswimwear.py index e92228da3..7deaf0fbc 100644 --- a/test/results/omgmiamiswimwear.py +++ b/test/results/omgmiamiswimwear.py @@ -1,25 +1,20 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://www.omgmiamiswimwear.com/collections/fajas", - "#category": ("shopify", "omgmiamiswimwear", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.omgmiamiswimwear.com/products/snatch-me-waist-belt", - "#category": ("shopify", "omgmiamiswimwear", "product"), - "#class" : shopify.ShopifyProductExtractor, - "#pattern" : r"https://cdn\.shopify\.com/s/files/1/1819/6171/", - "#count" : 3, -}, - + { + "#url": "https://www.omgmiamiswimwear.com/collections/fajas", + "#category": ("shopify", "omgmiamiswimwear", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.omgmiamiswimwear.com/products/snatch-me-waist-belt", + "#category": ("shopify", "omgmiamiswimwear", "product"), + "#class": shopify.ShopifyProductExtractor, + "#pattern": r"https://cdn\.shopify\.com/s/files/1/1819/6171/", + "#count": 3, + }, ) diff --git a/test/results/paheal.py b/test/results/paheal.py index 0d8262978..5f0f1e5c8 100644 --- a/test/results/paheal.py +++ b/test/results/paheal.py @@ -1,110 +1,96 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import paheal - __tests__ = ( -{ - "#url" : "https://rule34.paheal.net/post/list/Ayane_Suzuki/1", - "#category": ("shimmie2", "paheal", "tag"), - "#class" : paheal.PahealTagExtractor, - "#pattern" : r"https://[^.]+\.paheal\.net/_images/\w+/\d+%20-%20|https://r34i\.paheal-cdn\.net/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}$", - "#count" : range(70, 200), - - "date" : "type:datetime", - "extension": r"re:jpg|png", - "filename" : r"re:\d+ - \w+", - "duration" : float, - "height" : int, - "id" : int, - "md5" : r"re:[0-9a-f]{32}", - "search_tags": "Ayane_Suzuki", - "size" : int, - "tags" : str, - "width" : int, - -}, - -{ - "#url" : "https://rule34.paheal.net/post/list/Ayane_Suzuki/1", - "#category": ("shimmie2", "paheal", "tag"), - "#class" : paheal.PahealTagExtractor, - "#options" : {"metadata": True}, - "#range" : "1", - - "date" : "dt:2018-01-07 07:04:05", - "duration" : 0.0, - "extension" : "jpg", - "filename" : "2446128 - Ayane_Suzuki Idolmaster idolmaster_dearly_stars Zanzi", - "height" : 768, - "id" : 2446128, - "md5" : "b0ceda9d860df1d15b60293a7eb465c1", - "search_tags": "Ayane_Suzuki", - "size" : 204800, - "source" : "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=19957280", - "tags" : "Ayane_Suzuki Idolmaster idolmaster_dearly_stars Zanzi", - "uploader" : "XXXname", - "width" : 1024, -}, - -{ - "#url" : "https://rule34.paheal.net/post/view/481609", - "#category": ("shimmie2", "paheal", "post"), - "#class" : paheal.PahealPostExtractor, - "#urls" : "https://r34i.paheal-cdn.net/bb/dc/bbdc1c33410c2cdce7556c7990be26b7", - "#sha1_content": "7b924bcf150b352ac75c9d281d061e174c851a11", - - "date" : "dt:2010-06-17 15:40:23", - "extension": "jpg", - "file_url" : "https://r34i.paheal-cdn.net/bb/dc/bbdc1c33410c2cdce7556c7990be26b7", - "filename" : "481609 - Ayumu_Kasuga Azumanga_Daioh inanimate Vuvuzela", - "height" : 660, - "id" : 481609, - "md5" : "bbdc1c33410c2cdce7556c7990be26b7", - "size" : 157696, - "source" : "", - "tags" : "Ayumu_Kasuga Azumanga_Daioh inanimate Vuvuzela", - "uploader" : "CaptainButtface", - "width" : 614, -}, - -{ - "#url" : "https://rule34.paheal.net/post/view/488534", - "#category": ("shimmie2", "paheal", "post"), - "#class" : paheal.PahealPostExtractor, - - "date" : "dt:2010-06-25 13:51:17", - "height" : 800, - "md5" : "b39edfe455a0381110c710d6ed2ef57d", - "size" : 758784, - "source" : "http://www.furaffinity.net/view/4057821/", - "tags" : "inanimate thelost-dragon Vuvuzela", - "uploader": "leacheate_soup", - "width" : 1200, -}, - -{ - "#url" : "https://rule34.paheal.net/post/view/3864982", - "#comment" : "video", - "#category": ("shimmie2", "paheal", "post"), - "#class" : paheal.PahealPostExtractor, - "#urls" : "https://r34i.paheal-cdn.net/76/29/7629fc0ff77e32637dde5bf4f992b2cb", - - "date" : "dt:2020-09-06 01:59:03", - "duration" : 30.0, - "extension": "webm", - "height" : 2500, - "id" : 3864982, - "md5" : "7629fc0ff77e32637dde5bf4f992b2cb", - "size" : 18874368, - "source" : "https://twitter.com/VG_Worklog/status/1302407696294055936", - "tags" : "animated Metal_Gear Metal_Gear_Solid_V Quiet Vg_erotica webm", - "uploader" : "justausername", - "width" : 1768, -}, - + { + "#url": "https://rule34.paheal.net/post/list/Ayane_Suzuki/1", + "#category": ("shimmie2", "paheal", "tag"), + "#class": paheal.PahealTagExtractor, + "#pattern": r"https://[^.]+\.paheal\.net/_images/\w+/\d+%20-%20|https://r34i\.paheal-cdn\.net/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}$", + "#count": range(70, 200), + "date": "type:datetime", + "extension": r"re:jpg|png", + "filename": r"re:\d+ - \w+", + "duration": float, + "height": int, + "id": int, + "md5": r"re:[0-9a-f]{32}", + "search_tags": "Ayane_Suzuki", + "size": int, + "tags": str, + "width": int, + }, + { + "#url": "https://rule34.paheal.net/post/list/Ayane_Suzuki/1", + "#category": ("shimmie2", "paheal", "tag"), + "#class": paheal.PahealTagExtractor, + "#options": {"metadata": True}, + "#range": "1", + "date": "dt:2018-01-07 07:04:05", + "duration": 0.0, + "extension": "jpg", + "filename": "2446128 - Ayane_Suzuki Idolmaster idolmaster_dearly_stars Zanzi", + "height": 768, + "id": 2446128, + "md5": "b0ceda9d860df1d15b60293a7eb465c1", + "search_tags": "Ayane_Suzuki", + "size": 204800, + "source": "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=19957280", + "tags": "Ayane_Suzuki Idolmaster idolmaster_dearly_stars Zanzi", + "uploader": "XXXname", + "width": 1024, + }, + { + "#url": "https://rule34.paheal.net/post/view/481609", + "#category": ("shimmie2", "paheal", "post"), + "#class": paheal.PahealPostExtractor, + "#urls": "https://r34i.paheal-cdn.net/bb/dc/bbdc1c33410c2cdce7556c7990be26b7", + "#sha1_content": "7b924bcf150b352ac75c9d281d061e174c851a11", + "date": "dt:2010-06-17 15:40:23", + "extension": "jpg", + "file_url": "https://r34i.paheal-cdn.net/bb/dc/bbdc1c33410c2cdce7556c7990be26b7", + "filename": "481609 - Ayumu_Kasuga Azumanga_Daioh inanimate Vuvuzela", + "height": 660, + "id": 481609, + "md5": "bbdc1c33410c2cdce7556c7990be26b7", + "size": 157696, + "source": "", + "tags": "Ayumu_Kasuga Azumanga_Daioh inanimate Vuvuzela", + "uploader": "CaptainButtface", + "width": 614, + }, + { + "#url": "https://rule34.paheal.net/post/view/488534", + "#category": ("shimmie2", "paheal", "post"), + "#class": paheal.PahealPostExtractor, + "date": "dt:2010-06-25 13:51:17", + "height": 800, + "md5": "b39edfe455a0381110c710d6ed2ef57d", + "size": 758784, + "source": "http://www.furaffinity.net/view/4057821/", + "tags": "inanimate thelost-dragon Vuvuzela", + "uploader": "leacheate_soup", + "width": 1200, + }, + { + "#url": "https://rule34.paheal.net/post/view/3864982", + "#comment": "video", + "#category": ("shimmie2", "paheal", "post"), + "#class": paheal.PahealPostExtractor, + "#urls": "https://r34i.paheal-cdn.net/76/29/7629fc0ff77e32637dde5bf4f992b2cb", + "date": "dt:2020-09-06 01:59:03", + "duration": 30.0, + "extension": "webm", + "height": 2500, + "id": 3864982, + "md5": "7629fc0ff77e32637dde5bf4f992b2cb", + "size": 18874368, + "source": "https://twitter.com/VG_Worklog/status/1302407696294055936", + "tags": "animated Metal_Gear Metal_Gear_Solid_V Quiet Vg_erotica webm", + "uploader": "justausername", + "width": 1768, + }, ) diff --git a/test/results/palanq.py b/test/results/palanq.py index 046e6e882..eb6c9acef 100644 --- a/test/results/palanq.py +++ b/test/results/palanq.py @@ -1,36 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://archive.palanq.win/c/thread/4209598/", - "#category": ("foolfuuka", "palanq", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "1f9b5570d228f1f2991c827a6631030bc0e5933c", -}, - -{ - "#url" : "https://archive.palanq.win/c/", - "#category": ("foolfuuka", "palanq", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://archive.palanq.win/_/search/text/test/", - "#category": ("foolfuuka", "palanq", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://archive.palanq.win/c/gallery", - "#category": ("foolfuuka", "palanq", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://archive.palanq.win/c/thread/4209598/", + "#category": ("foolfuuka", "palanq", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "1f9b5570d228f1f2991c827a6631030bc0e5933c", + }, + { + "#url": "https://archive.palanq.win/c/", + "#category": ("foolfuuka", "palanq", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://archive.palanq.win/_/search/text/test/", + "#category": ("foolfuuka", "palanq", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://archive.palanq.win/c/gallery", + "#category": ("foolfuuka", "palanq", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/patreon.py b/test/results/patreon.py index f8cdaf10f..d98075740 100644 --- a/test/results/patreon.py +++ b/test/results/patreon.py @@ -1,124 +1,104 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import patreon import datetime -from gallery_dl import exception +from gallery_dl import exception +from gallery_dl.extractor import patreon __tests__ = ( -{ - "#url" : "https://www.patreon.com/koveliana", - "#class" : patreon.PatreonCreatorExtractor, - "#range" : "1-15", - "#count" : 15, - - "attachments" : list, - "comment_count": int, - "content" : str, - "creator" : dict, - "date" : datetime.datetime, - "id" : int, - "images" : list, - "like_count" : int, - "post_type" : str, - "published_at" : str, - "title" : str, -}, - -{ - "#url" : "https://www.patreon.com/koveliana/posts?filters[month]=2020-3", - "#class" : patreon.PatreonCreatorExtractor, - "#count" : 1, - - "date": "dt:2020-03-30 21:21:44", -}, - -{ - "#url" : "https://www.patreon.com/kovelianot", - "#class" : patreon.PatreonCreatorExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.patreon.com/c/koveliana", - "#class" : patreon.PatreonCreatorExtractor, -}, - -{ - "#url" : "https://www.patreon.com/user?u=2931440", - "#class" : patreon.PatreonCreatorExtractor, -}, - -{ - "#url" : "https://www.patreon.com/user/posts/?u=2931440", - "#class" : patreon.PatreonCreatorExtractor, -}, - -{ - "#url" : "https://www.patreon.com/user?c=369707", - "#class" : patreon.PatreonCreatorExtractor, -}, - -{ - "#url" : "https://www.patreon.com/id:369707", - "#class" : patreon.PatreonCreatorExtractor, -}, - -{ - "#url" : "https://www.patreon.com/home", - "#class" : patreon.PatreonUserExtractor, -}, - -{ - "#url" : "https://www.patreon.com/posts/precious-metal-23563293", - "#comment" : "postfile + attachments", - "#class" : patreon.PatreonPostExtractor, - "#count" : 4, -}, - -{ - "#url" : "https://www.patreon.com/posts/free-mari-8s-113049301", - "#comment" : "postfile + attachments_media (#6241)", - "#class" : patreon.PatreonPostExtractor, - "#pattern" : [ - r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/113049301/7ae4fd78d3374d849a80863f3d8eee89/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.jpg", - r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/113049301/b6ea96b18cbc47f78f9334d50d0877ea/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.mp4", - r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/113049301/62dc1d4194db4245aca31c56f71234ed/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.mp4", - ], -}, - -{ - "#url" : "https://www.patreon.com/posts/56127163", - "#comment" : "account suspended", - "#class" : patreon.PatreonPostExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.patreon.com/posts/free-post-12497641", - "#comment" : "tags (#1539)", - "#class" : patreon.PatreonPostExtractor, - - "tags": ["AWMedia"], -}, - -{ - "#url" : "https://www.patreon.com/posts/m3u8-94714289", - "#class" : patreon.PatreonPostExtractor, - "#pattern" : [ - r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/94714289/be3d8eb994ae44eca4baffcdc6dd25fc/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.png", - r"ytdl:https://www.patreon\.com/api/video/255859412/video\.m3u8", - ] -}, - -{ - "#url" : "https://www.patreon.com/posts/not-found-123", - "#class" : patreon.PatreonPostExtractor, - "#exception": exception.NotFoundError, -}, - + { + "#url": "https://www.patreon.com/koveliana", + "#class": patreon.PatreonCreatorExtractor, + "#range": "1-15", + "#count": 15, + "attachments": list, + "comment_count": int, + "content": str, + "creator": dict, + "date": datetime.datetime, + "id": int, + "images": list, + "like_count": int, + "post_type": str, + "published_at": str, + "title": str, + }, + { + "#url": "https://www.patreon.com/koveliana/posts?filters[month]=2020-3", + "#class": patreon.PatreonCreatorExtractor, + "#count": 1, + "date": "dt:2020-03-30 21:21:44", + }, + { + "#url": "https://www.patreon.com/kovelianot", + "#class": patreon.PatreonCreatorExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.patreon.com/c/koveliana", + "#class": patreon.PatreonCreatorExtractor, + }, + { + "#url": "https://www.patreon.com/user?u=2931440", + "#class": patreon.PatreonCreatorExtractor, + }, + { + "#url": "https://www.patreon.com/user/posts/?u=2931440", + "#class": patreon.PatreonCreatorExtractor, + }, + { + "#url": "https://www.patreon.com/user?c=369707", + "#class": patreon.PatreonCreatorExtractor, + }, + { + "#url": "https://www.patreon.com/id:369707", + "#class": patreon.PatreonCreatorExtractor, + }, + { + "#url": "https://www.patreon.com/home", + "#class": patreon.PatreonUserExtractor, + }, + { + "#url": "https://www.patreon.com/posts/precious-metal-23563293", + "#comment": "postfile + attachments", + "#class": patreon.PatreonPostExtractor, + "#count": 4, + }, + { + "#url": "https://www.patreon.com/posts/free-mari-8s-113049301", + "#comment": "postfile + attachments_media (#6241)", + "#class": patreon.PatreonPostExtractor, + "#pattern": [ + r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/113049301/7ae4fd78d3374d849a80863f3d8eee89/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.jpg", + r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/113049301/b6ea96b18cbc47f78f9334d50d0877ea/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.mp4", + r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/113049301/62dc1d4194db4245aca31c56f71234ed/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.mp4", + ], + }, + { + "#url": "https://www.patreon.com/posts/56127163", + "#comment": "account suspended", + "#class": patreon.PatreonPostExtractor, + "#count": 0, + }, + { + "#url": "https://www.patreon.com/posts/free-post-12497641", + "#comment": "tags (#1539)", + "#class": patreon.PatreonPostExtractor, + "tags": ["AWMedia"], + }, + { + "#url": "https://www.patreon.com/posts/m3u8-94714289", + "#class": patreon.PatreonPostExtractor, + "#pattern": [ + r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/94714289/be3d8eb994ae44eca4baffcdc6dd25fc/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.png", + r"ytdl:https://www.patreon\.com/api/video/255859412/video\.m3u8", + ], + }, + { + "#url": "https://www.patreon.com/posts/not-found-123", + "#class": patreon.PatreonPostExtractor, + "#exception": exception.NotFoundError, + }, ) diff --git a/test/results/pawoo.py b/test/results/pawoo.py index 5a9bfcaa2..18857a983 100644 --- a/test/results/pawoo.py +++ b/test/results/pawoo.py @@ -1,38 +1,31 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mastodon - __tests__ = ( -{ - "#url" : "https://pawoo.net/@yoru_nine/", - "#category": ("mastodon", "pawoo", "user"), - "#class" : mastodon.MastodonUserExtractor, - "#range" : "1-60", - "#count" : 60, -}, - -{ - "#url" : "https://pawoo.net/bookmarks", - "#category": ("mastodon", "pawoo", "bookmark"), - "#class" : mastodon.MastodonBookmarkExtractor, -}, - -{ - "#url" : "https://pawoo.net/users/yoru_nine/following", - "#category": ("mastodon", "pawoo", "following"), - "#class" : mastodon.MastodonFollowingExtractor, -}, - -{ - "#url" : "https://pawoo.net/@yoru_nine/105038878897832922", - "#category": ("mastodon", "pawoo", "status"), - "#class" : mastodon.MastodonStatusExtractor, - "#sha1_content": "b52e807f8ab548d6f896b09218ece01eba83987a", -}, - + { + "#url": "https://pawoo.net/@yoru_nine/", + "#category": ("mastodon", "pawoo", "user"), + "#class": mastodon.MastodonUserExtractor, + "#range": "1-60", + "#count": 60, + }, + { + "#url": "https://pawoo.net/bookmarks", + "#category": ("mastodon", "pawoo", "bookmark"), + "#class": mastodon.MastodonBookmarkExtractor, + }, + { + "#url": "https://pawoo.net/users/yoru_nine/following", + "#category": ("mastodon", "pawoo", "following"), + "#class": mastodon.MastodonFollowingExtractor, + }, + { + "#url": "https://pawoo.net/@yoru_nine/105038878897832922", + "#category": ("mastodon", "pawoo", "status"), + "#class": mastodon.MastodonStatusExtractor, + "#sha1_content": "b52e807f8ab548d6f896b09218ece01eba83987a", + }, ) diff --git a/test/results/photovogue.py b/test/results/photovogue.py index 6898f5d9e..902d358b2 100644 --- a/test/results/photovogue.py +++ b/test/results/photovogue.py @@ -1,50 +1,45 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import photovogue import datetime +from gallery_dl.extractor import photovogue __tests__ = ( -{ - "#url" : "https://www.vogue.com/photovogue/photographers/221252", - "#category": ("", "photovogue", "user"), - "#class" : photovogue.PhotovogueUserExtractor, -}, - -{ - "#url" : "https://vogue.com/photovogue/photographers/221252", - "#category": ("", "photovogue", "user"), - "#class" : photovogue.PhotovogueUserExtractor, - "#pattern" : "https://images.vogue.it/Photovogue/[^/]+_gallery.jpg", - - "date" : datetime.datetime, - "favorite_count" : int, - "favorited" : list, - "id" : int, - "image_id" : str, - "is_favorite" : False, - "orientation" : r"re:portrait|landscape", - "photographer" : { - "biography" : "Born in 1995. Live in Bologna.", - "city" : "Bologna", - "country_id" : 106, - "favoritedCount": int, - "id" : 221252, - "isGold" : bool, - "isPro" : bool, - "latitude" : str, - "longitude" : str, - "name" : "Arianna Mattarozzi", - "user_id" : "38cb0601-4a85-453c-b7dc-7650a037f2ab", - "websites" : list, + { + "#url": "https://www.vogue.com/photovogue/photographers/221252", + "#category": ("", "photovogue", "user"), + "#class": photovogue.PhotovogueUserExtractor, + }, + { + "#url": "https://vogue.com/photovogue/photographers/221252", + "#category": ("", "photovogue", "user"), + "#class": photovogue.PhotovogueUserExtractor, + "#pattern": "https://images.vogue.it/Photovogue/[^/]+_gallery.jpg", + "date": datetime.datetime, + "favorite_count": int, + "favorited": list, + "id": int, + "image_id": str, + "is_favorite": False, + "orientation": r"re:portrait|landscape", + "photographer": { + "biography": "Born in 1995. Live in Bologna.", + "city": "Bologna", + "country_id": 106, + "favoritedCount": int, + "id": 221252, + "isGold": bool, + "isPro": bool, + "latitude": str, + "longitude": str, + "name": "Arianna Mattarozzi", + "user_id": "38cb0601-4a85-453c-b7dc-7650a037f2ab", + "websites": list, + }, + "photographer_id": 221252, + "tags": list, + "title": str, }, - "photographer_id": 221252, - "tags" : list, - "title" : str, -}, - ) diff --git a/test/results/picarto.py b/test/results/picarto.py index 074562601..0a994ab6b 100644 --- a/test/results/picarto.py +++ b/test/results/picarto.py @@ -1,22 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import picarto import datetime +from gallery_dl.extractor import picarto __tests__ = ( -{ - "#url" : "https://picarto.tv/fnook/gallery/default/", - "#category": ("", "picarto", "gallery"), - "#class" : picarto.PicartoGalleryExtractor, - "#pattern" : r"https://images\.picarto\.tv/gallery/\d/\d\d/\d+/artwork/[0-9a-f-]+/large-[0-9a-f]+\.(jpg|png|gif)", - "#count" : ">= 7", - - "date": datetime.datetime, -}, - + { + "#url": "https://picarto.tv/fnook/gallery/default/", + "#category": ("", "picarto", "gallery"), + "#class": picarto.PicartoGalleryExtractor, + "#pattern": r"https://images\.picarto\.tv/gallery/\d/\d\d/\d+/artwork/[0-9a-f-]+/large-[0-9a-f]+\.(jpg|png|gif)", + "#count": ">= 7", + "date": datetime.datetime, + }, ) diff --git a/test/results/piczel.py b/test/results/piczel.py index 84d1d01b8..20f7b3c12 100644 --- a/test/results/piczel.py +++ b/test/results/piczel.py @@ -1,57 +1,50 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import piczel - __tests__ = ( -{ - "#url" : "https://piczel.tv/gallery/Bikupan", - "#category": ("", "piczel", "user"), - "#class" : piczel.PiczelUserExtractor, - "#range" : "1-100", - "#count" : ">= 100", -}, - -{ - "#url" : "https://piczel.tv/gallery/Lulena/1114", - "#category": ("", "piczel", "folder"), - "#class" : piczel.PiczelFolderExtractor, - "#count" : ">= 4", -}, - -{ - "#url" : "https://piczel.tv/gallery/image/7807", - "#category": ("", "piczel", "image"), - "#class" : piczel.PiczelImageExtractor, - "#pattern" : r"https://(\w+\.)?piczel\.tv/static/uploads/gallery_image/32920/image/7807/1532236438-Lulena\.png", - "#sha1_content": "df9a053a24234474a19bce2b7e27e0dec23bff87", - - "created_at" : "2018-07-22T05:13:58.000Z", - "date" : "dt:2018-07-22 05:13:58", - "description" : None, - "extension" : "png", - "favorites_count" : int, - "folder_id" : 1113, - "id" : 7807, - "is_flash" : False, - "is_video" : False, - "multi" : False, - "nsfw" : False, - "num" : 0, - "password_protected": False, - "tags" : [ - "fanart", - "commission", - "altair", - "recreators", - ], - "title" : "Altair", - "user" : dict, - "views" : int, -}, - + { + "#url": "https://piczel.tv/gallery/Bikupan", + "#category": ("", "piczel", "user"), + "#class": piczel.PiczelUserExtractor, + "#range": "1-100", + "#count": ">= 100", + }, + { + "#url": "https://piczel.tv/gallery/Lulena/1114", + "#category": ("", "piczel", "folder"), + "#class": piczel.PiczelFolderExtractor, + "#count": ">= 4", + }, + { + "#url": "https://piczel.tv/gallery/image/7807", + "#category": ("", "piczel", "image"), + "#class": piczel.PiczelImageExtractor, + "#pattern": r"https://(\w+\.)?piczel\.tv/static/uploads/gallery_image/32920/image/7807/1532236438-Lulena\.png", + "#sha1_content": "df9a053a24234474a19bce2b7e27e0dec23bff87", + "created_at": "2018-07-22T05:13:58.000Z", + "date": "dt:2018-07-22 05:13:58", + "description": None, + "extension": "png", + "favorites_count": int, + "folder_id": 1113, + "id": 7807, + "is_flash": False, + "is_video": False, + "multi": False, + "nsfw": False, + "num": 0, + "password_protected": False, + "tags": [ + "fanart", + "commission", + "altair", + "recreators", + ], + "title": "Altair", + "user": dict, + "views": int, + }, ) diff --git a/test/results/pidgiwiki.py b/test/results/pidgiwiki.py index fc837d4c7..7738d2675 100644 --- a/test/results/pidgiwiki.py +++ b/test/results/pidgiwiki.py @@ -1,24 +1,19 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.pidgi.net/wiki/File:Key_art_-_Fight_Knight.png", - "#category": ("wikimedia", "pidgiwiki", "file"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#urls" : "https://cdn.pidgi.net/images/0/0c/Key_art_-_Fight_Knight.png", -}, - -{ - "#url" : "https://pidgi.net/wiki/File:Key_art_-_Fight_Knight.png", - "#category": ("wikimedia", "pidgiwiki", "file"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.pidgi.net/wiki/File:Key_art_-_Fight_Knight.png", + "#category": ("wikimedia", "pidgiwiki", "file"), + "#class": wikimedia.WikimediaArticleExtractor, + "#urls": "https://cdn.pidgi.net/images/0/0c/Key_art_-_Fight_Knight.png", + }, + { + "#url": "https://pidgi.net/wiki/File:Key_art_-_Fight_Knight.png", + "#category": ("wikimedia", "pidgiwiki", "file"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/pillowfort.py b/test/results/pillowfort.py index 514697be7..485bbfc88 100644 --- a/test/results/pillowfort.py +++ b/test/results/pillowfort.py @@ -1,185 +1,175 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import pillowfort import datetime +from gallery_dl.extractor import pillowfort __tests__ = ( -{ - "#url" : "https://www.pillowfort.social/posts/27510", - "#category": ("", "pillowfort", "post"), - "#class" : pillowfort.PillowfortPostExtractor, - "#pattern" : r"https://img\d+\.pillowfort\.social/posts/\w+_out\d+\.png", - "#count" : 4, - - "avatar_url" : str, - "col" : 0, - "commentable" : True, - "comments_count" : int, - "community_id" : None, - "content" : str, - "count" : 4, - "created_at" : str, - "date" : datetime.datetime, - "deleted" : None, - "deleted_at" : None, - "deleted_by_mod" : None, - "deleted_for_flag_id": None, - "embed_code" : None, - "id" : int, - "last_activity" : str, - "last_activity_elapsed": str, - "last_edited_at" : str, - "likes_count" : int, - "media_type" : "picture", - "nsfw" : False, - "num" : range(1, 4), - "original_post_id": None, - "original_post_user_id": None, - "picture_content_type": None, - "picture_file_name": None, - "picture_file_size": None, - "picture_updated_at": None, - "post_id" : 27510, - "post_type" : "picture", - "privacy" : "public", - "reblog_copy_info": list, - "rebloggable" : True, - "reblogged_from_post_id": None, - "reblogged_from_user_id": None, - "reblogs_count" : int, - "row" : int, - "small_image_url" : None, - "tags" : list, - "time_elapsed" : str, - "timestamp" : str, - "title" : "What is Pillowfort.social?", - "updated_at" : str, - "url" : r"re:https://img3.pillowfort.social/posts/.*\.png", - "user_id" : 5, - "username" : "Staff", -}, - -{ - "#url" : "https://www.pillowfort.social/posts/1124584", - "#comment" : "'b2_lg_url' media URL (#4570)", - "#category": ("", "pillowfort", "post"), - "#class" : pillowfort.PillowfortPostExtractor, - "#pattern" : r"https://img2\.pillowfort\.social/posts/c8e834bc09e6_Brandee\.png", - "#count" : 1, - - "avatar_frame" : None, - "avatar_id" : None, - "avatar_url" : "https://img3.pillowfort.social/avatars/000/037/139/original/437.jpg?1545015697", - "b2_lg_url" : "https://img2.pillowfort.social/posts/c8e834bc09e6_Brandee.png", - "b2_sm_url" : "https://img2.pillowfort.social/posts/c8e834bc09e6_Brandee_small.png", - "cached_tag_list": "art, digital art, mermaid, mermaids, underwater, seaweed, illustration, speed paint", - "col" : 0, - "comm_screening_status": "not_applicable", - "commentable" : True, - "comments_count": 0, - "community_id" : None, - "concealed_comment_warning": None, - "content" : "

          Sea Bed

          ", - "count" : 1, - "created_at" : r"re:2020-02-.+", - "currentuser_default_avatar_url": None, - "currentuser_multi_avi": None, - "date" : "dt:2020-02-29 17:09:03", - "deleted" : None, - "deleted_at" : None, - "deleted_by_mod": None, - "deleted_for_flag_id": None, - "embed_code" : None, - "extension" : "png", - "filename" : "Brandee", - "hash" : "c8e834bc09e6", - "id" : 720167, - "last_activity" : r"re:2020-02-.+", - "last_activity_elapsed": r"re:\d+ months", - "last_edited_at": None, - "likes_count" : 8, - "media_type" : "picture", - "nsfw" : False, - "num" : 1, - "original_post_id": None, - "original_post_user_id": None, - "pic_row_last" : 1, - "picture_content_type": None, - "picture_file_name": None, - "picture_file_size": None, - "picture_updated_at": None, - "post_id" : 1124584, - "post_type" : "picture", - "privacy" : "public", - "reblog_copy_info": [], - "rebloggable" : True, - "reblogged_from_post_id": None, - "reblogged_from_user_id": None, - "reblogs_count" : int, - "row" : 1, - "small_image_url": None, - "tag_list" : None, - "tags" : [ - "art", - "digital art", - "mermaid", - "mermaids", - "underwater", - "seaweed", - "illustration", - "speed paint", - ], - "time_elapsed" : r"re:\d+ months", - "timestamp" : str, - "title" : "", - "updated_at" : r"re:2020-02-.+", - "url" : "", - "user_concealed": None, - "user_id" : 37201, - "username" : "Maclanahan", -}, - -{ - "#url" : "https://www.pillowfort.social/posts/1557500", - "#comment" : "'external' option", - "#category": ("", "pillowfort", "post"), - "#class" : pillowfort.PillowfortPostExtractor, - "#options" : { - "external": True, - "inline" : False, + { + "#url": "https://www.pillowfort.social/posts/27510", + "#category": ("", "pillowfort", "post"), + "#class": pillowfort.PillowfortPostExtractor, + "#pattern": r"https://img\d+\.pillowfort\.social/posts/\w+_out\d+\.png", + "#count": 4, + "avatar_url": str, + "col": 0, + "commentable": True, + "comments_count": int, + "community_id": None, + "content": str, + "count": 4, + "created_at": str, + "date": datetime.datetime, + "deleted": None, + "deleted_at": None, + "deleted_by_mod": None, + "deleted_for_flag_id": None, + "embed_code": None, + "id": int, + "last_activity": str, + "last_activity_elapsed": str, + "last_edited_at": str, + "likes_count": int, + "media_type": "picture", + "nsfw": False, + "num": range(1, 4), + "original_post_id": None, + "original_post_user_id": None, + "picture_content_type": None, + "picture_file_name": None, + "picture_file_size": None, + "picture_updated_at": None, + "post_id": 27510, + "post_type": "picture", + "privacy": "public", + "reblog_copy_info": list, + "rebloggable": True, + "reblogged_from_post_id": None, + "reblogged_from_user_id": None, + "reblogs_count": int, + "row": int, + "small_image_url": None, + "tags": list, + "time_elapsed": str, + "timestamp": str, + "title": "What is Pillowfort.social?", + "updated_at": str, + "url": r"re:https://img3.pillowfort.social/posts/.*\.png", + "user_id": 5, + "username": "Staff", + }, + { + "#url": "https://www.pillowfort.social/posts/1124584", + "#comment": "'b2_lg_url' media URL (#4570)", + "#category": ("", "pillowfort", "post"), + "#class": pillowfort.PillowfortPostExtractor, + "#pattern": r"https://img2\.pillowfort\.social/posts/c8e834bc09e6_Brandee\.png", + "#count": 1, + "avatar_frame": None, + "avatar_id": None, + "avatar_url": "https://img3.pillowfort.social/avatars/000/037/139/original/437.jpg?1545015697", + "b2_lg_url": "https://img2.pillowfort.social/posts/c8e834bc09e6_Brandee.png", + "b2_sm_url": "https://img2.pillowfort.social/posts/c8e834bc09e6_Brandee_small.png", + "cached_tag_list": "art, digital art, mermaid, mermaids, underwater, seaweed, illustration, speed paint", + "col": 0, + "comm_screening_status": "not_applicable", + "commentable": True, + "comments_count": 0, + "community_id": None, + "concealed_comment_warning": None, + "content": "

          Sea Bed

          ", + "count": 1, + "created_at": r"re:2020-02-.+", + "currentuser_default_avatar_url": None, + "currentuser_multi_avi": None, + "date": "dt:2020-02-29 17:09:03", + "deleted": None, + "deleted_at": None, + "deleted_by_mod": None, + "deleted_for_flag_id": None, + "embed_code": None, + "extension": "png", + "filename": "Brandee", + "hash": "c8e834bc09e6", + "id": 720167, + "last_activity": r"re:2020-02-.+", + "last_activity_elapsed": r"re:\d+ months", + "last_edited_at": None, + "likes_count": 8, + "media_type": "picture", + "nsfw": False, + "num": 1, + "original_post_id": None, + "original_post_user_id": None, + "pic_row_last": 1, + "picture_content_type": None, + "picture_file_name": None, + "picture_file_size": None, + "picture_updated_at": None, + "post_id": 1124584, + "post_type": "picture", + "privacy": "public", + "reblog_copy_info": [], + "rebloggable": True, + "reblogged_from_post_id": None, + "reblogged_from_user_id": None, + "reblogs_count": int, + "row": 1, + "small_image_url": None, + "tag_list": None, + "tags": [ + "art", + "digital art", + "mermaid", + "mermaids", + "underwater", + "seaweed", + "illustration", + "speed paint", + ], + "time_elapsed": r"re:\d+ months", + "timestamp": str, + "title": "", + "updated_at": r"re:2020-02-.+", + "url": "", + "user_concealed": None, + "user_id": 37201, + "username": "Maclanahan", + }, + { + "#url": "https://www.pillowfort.social/posts/1557500", + "#comment": "'external' option", + "#category": ("", "pillowfort", "post"), + "#class": pillowfort.PillowfortPostExtractor, + "#options": { + "external": True, + "inline": False, + }, + "#pattern": r"https://twitter\.com/Aliciawitdaart/status/1282862493841457152", + }, + { + "#url": "https://www.pillowfort.social/posts/1672518", + "#comment": "'inline' option", + "#category": ("", "pillowfort", "post"), + "#class": pillowfort.PillowfortPostExtractor, + "#options": {"inline": True}, + "#count": 3, + }, + { + "#url": "https://www.pillowfort.social/Pome", + "#category": ("", "pillowfort", "user"), + "#class": pillowfort.PillowfortUserExtractor, + "#pattern": r"https://img\d+\.pillowfort\.social/posts/", + "#range": "1-15", + "#count": 15, + }, + { + "#url": "https://www.pillowfort.social/Staff/tagged/funding", + "#category": ("", "pillowfort", "user"), + "#class": pillowfort.PillowfortUserExtractor, + "#pattern": r"https://img\d+\.pillowfort\.social/posts/", + "#count": range(30, 50), }, - "#pattern" : r"https://twitter\.com/Aliciawitdaart/status/1282862493841457152", -}, - -{ - "#url" : "https://www.pillowfort.social/posts/1672518", - "#comment" : "'inline' option", - "#category": ("", "pillowfort", "post"), - "#class" : pillowfort.PillowfortPostExtractor, - "#options" : {"inline": True}, - "#count" : 3, -}, - -{ - "#url" : "https://www.pillowfort.social/Pome", - "#category": ("", "pillowfort", "user"), - "#class" : pillowfort.PillowfortUserExtractor, - "#pattern" : r"https://img\d+\.pillowfort\.social/posts/", - "#range" : "1-15", - "#count" : 15, -}, - -{ - "#url" : "https://www.pillowfort.social/Staff/tagged/funding", - "#category": ("", "pillowfort", "user"), - "#class" : pillowfort.PillowfortUserExtractor, - "#pattern" : r"https://img\d+\.pillowfort\.social/posts/", - "#count" : range(30, 50), -}, - ) diff --git a/test/results/pinterest.py b/test/results/pinterest.py index 3ab5ebb3a..84408428f 100644 --- a/test/results/pinterest.py +++ b/test/results/pinterest.py @@ -1,202 +1,174 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import pinterest from gallery_dl import exception - +from gallery_dl.extractor import pinterest __tests__ = ( -{ - "#url" : "https://www.pinterest.com/pin/858146903966145189/", - "#category": ("", "pinterest", "pin"), - "#class" : pinterest.PinterestPinExtractor, - "#sha1_url" : "afb3c26719e3a530bb0e871c480882a801a4e8a5", - "#sha1_content": [ - "4c435a66f6bb82bb681db2ecc888f76cf6c5f9ca", - "d3e24bc9f7af585e8c23b9136956bd45a4d9b947", - ], -}, - -{ - "#url" : "https://www.pinterest.com/pin/422564377542934214/", - "#comment" : "video pin (#1189)", - "#class" : pinterest.PinterestPinExtractor, - "#pattern" : r"https://v\d*\.pinimg\.com/videos/mc/hls/d7/22/ff/d722ff00ab2352981b89974b37909de8.m3u8", - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://jp.pinterest.com/pin/858146904010573850/", - "#comment" : "story pin with images", - "#class" : pinterest.PinterestPinExtractor, - "#urls" : ( - "https://i.pinimg.com/originals/0f/b0/8c/0fb08c519067dd263a1fcfecea775450.jpg", - "https://i.pinimg.com/originals/2f/27/f3/2f27f3eb781b107ce58bf588c12a12b7.jpg", - "https://i.pinimg.com/originals/55/fd/df/55fddf8d26aa0d96071af52ac6a0c25f.jpg", - ), -}, - -{ - "#url" : "https://www.pinterest.com/pin/63824519713049795/", - "#comment" : "story pin with video (#6188)", - "#class" : pinterest.PinterestPinExtractor, - "#urls" : "ytdl:https://v1.pinimg.com/videos/iht/hls/7a/b0/cc/7ab0cc56dcbfc1508b8d650af7b0a593.m3u8", - - "extension" : "mp4", - "_ytdl_manifest": "hls", -}, - -{ - "#url" : "https://www.pinterest.com/pin/606508274845593025/", - "#comment" : "story pin with audio (#6188)", - "#class" : pinterest.PinterestPinExtractor, - "#range" : "2", - "#urls" : "https://v1.pinimg.com/audios/mp3/5d/37/74/5d37749bde03855c1292f8869c8d9387.mp3", - - "extension": "mp3", -}, - -{ - "#url" : "https://jp.pinterest.com/pin/851532242064221228/", - "#comment" : "story pin with text", - "#class" : pinterest.PinterestPinExtractor, - "#range" : "2", - "#urls" : "text:Everskies character+outfits i made", -}, - -{ - "#url" : "https://www.pinterest.com/pin/858146903966145188/", - "#category": ("", "pinterest", "pin"), - "#class" : pinterest.PinterestPinExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/test-/", - "#class" : pinterest.PinterestBoardExtractor, - "#urls" : "https://i.pinimg.com/originals/d4/f4/7f/d4f47fa2fce4c4c28475af5d94972904.jpg", -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/stuff/", - "#comment" : "board with sections (#835)", - "#category": ("", "pinterest", "board"), - "#class" : pinterest.PinterestBoardExtractor, - "#options" : {"sections": True}, - "#count" : 4, -}, - -{ - "#url" : "https://www.pinterest.jp/gdldev/bname/", - "#comment" : "board & section with /?# in name (#5104)", - "#category": ("", "pinterest", "board"), - "#class" : pinterest.PinterestBoardExtractor, - "#options" : {"sections": True}, - "#urls" : "https://www.pinterest.jp/gdldev/bname/id:5345901183739414095", -}, - -{ - "#url" : "https://www.pinterest.de/g1952849/secret/", - "#comment" : "secret board (#1055)", - "#category": ("", "pinterest", "board"), - "#class" : pinterest.PinterestBoardExtractor, - "#auth" : True, - "#count" : 2, -}, - -{ - "#url" : "https://www.pinterest.com/g1952848/test/", - "#category": ("", "pinterest", "board"), - "#class" : pinterest.PinterestBoardExtractor, - "#exception": exception.GalleryDLException, -}, - -{ - "#url" : "https://www.pinterest.co.uk/hextra7519/based-animals/", - "#comment" : ".co.uk TLD (#914)", - "#category": ("", "pinterest", "board"), - "#class" : pinterest.PinterestBoardExtractor, -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/", - "#category": ("", "pinterest", "user"), - "#class" : pinterest.PinterestUserExtractor, - "#pattern" : pinterest.PinterestBoardExtractor.pattern, - "#count" : ">= 2", -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/_saved/", - "#category": ("", "pinterest", "user"), - "#class" : pinterest.PinterestUserExtractor, -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/pins/", - "#category": ("", "pinterest", "allpins"), - "#class" : pinterest.PinterestAllpinsExtractor, - "#pattern" : r"https://i\.pinimg\.com/originals/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w{3}", - "#count" : 9, -}, - -{ - "#url" : "https://www.pinterest.de/digitalmomblog/_created/", - "#category": ("", "pinterest", "created"), - "#class" : pinterest.PinterestCreatedExtractor, - "#pattern" : r"ytdl:|https://i\.pinimg\.com/originals/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.(jpg|png|webp)", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/stuff/section", - "#category": ("", "pinterest", "section"), - "#class" : pinterest.PinterestSectionExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://www.pinterest.com/search/pins/?q=nature", - "#category": ("", "pinterest", "search"), - "#class" : pinterest.PinterestSearchExtractor, - "#range" : "1-50", - "#count" : ">= 50", -}, - -{ - "#url" : "https://www.pinterest.com/pin/858146903966145189/#related", - "#category": ("", "pinterest", "related-pin"), - "#class" : pinterest.PinterestRelatedPinExtractor, - "#range" : "31-70", - "#count" : 40, - "#archive" : False, -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/test-/#related", - "#category": ("", "pinterest", "related-board"), - "#class" : pinterest.PinterestRelatedBoardExtractor, - "#range" : "31-70", - "#count" : 40, - "#archive" : False, -}, - -{ - "#url" : "https://pin.it/Hvt8hgT", - "#category": ("", "pinterest", "pinit"), - "#class" : pinterest.PinterestPinitExtractor, - "#sha1_url": "8daad8558382c68f0868bdbd17d05205184632fa", -}, - -{ - "#url" : "https://pin.it/Hvt8hgS", - "#category": ("", "pinterest", "pinit"), - "#class" : pinterest.PinterestPinitExtractor, - "#exception": exception.NotFoundError, -}, - + { + "#url": "https://www.pinterest.com/pin/858146903966145189/", + "#category": ("", "pinterest", "pin"), + "#class": pinterest.PinterestPinExtractor, + "#sha1_url": "afb3c26719e3a530bb0e871c480882a801a4e8a5", + "#sha1_content": [ + "4c435a66f6bb82bb681db2ecc888f76cf6c5f9ca", + "d3e24bc9f7af585e8c23b9136956bd45a4d9b947", + ], + }, + { + "#url": "https://www.pinterest.com/pin/422564377542934214/", + "#comment": "video pin (#1189)", + "#class": pinterest.PinterestPinExtractor, + "#pattern": r"https://v\d*\.pinimg\.com/videos/mc/hls/d7/22/ff/d722ff00ab2352981b89974b37909de8.m3u8", + "#exception": exception.NotFoundError, + }, + { + "#url": "https://jp.pinterest.com/pin/858146904010573850/", + "#comment": "story pin with images", + "#class": pinterest.PinterestPinExtractor, + "#urls": ( + "https://i.pinimg.com/originals/0f/b0/8c/0fb08c519067dd263a1fcfecea775450.jpg", + "https://i.pinimg.com/originals/2f/27/f3/2f27f3eb781b107ce58bf588c12a12b7.jpg", + "https://i.pinimg.com/originals/55/fd/df/55fddf8d26aa0d96071af52ac6a0c25f.jpg", + ), + }, + { + "#url": "https://www.pinterest.com/pin/63824519713049795/", + "#comment": "story pin with video (#6188)", + "#class": pinterest.PinterestPinExtractor, + "#urls": "ytdl:https://v1.pinimg.com/videos/iht/hls/7a/b0/cc/7ab0cc56dcbfc1508b8d650af7b0a593.m3u8", + "extension": "mp4", + "_ytdl_manifest": "hls", + }, + { + "#url": "https://www.pinterest.com/pin/606508274845593025/", + "#comment": "story pin with audio (#6188)", + "#class": pinterest.PinterestPinExtractor, + "#range": "2", + "#urls": "https://v1.pinimg.com/audios/mp3/5d/37/74/5d37749bde03855c1292f8869c8d9387.mp3", + "extension": "mp3", + }, + { + "#url": "https://jp.pinterest.com/pin/851532242064221228/", + "#comment": "story pin with text", + "#class": pinterest.PinterestPinExtractor, + "#range": "2", + "#urls": "text:Everskies character+outfits i made", + }, + { + "#url": "https://www.pinterest.com/pin/858146903966145188/", + "#category": ("", "pinterest", "pin"), + "#class": pinterest.PinterestPinExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.pinterest.com/g1952849/test-/", + "#class": pinterest.PinterestBoardExtractor, + "#urls": "https://i.pinimg.com/originals/d4/f4/7f/d4f47fa2fce4c4c28475af5d94972904.jpg", + }, + { + "#url": "https://www.pinterest.com/g1952849/stuff/", + "#comment": "board with sections (#835)", + "#category": ("", "pinterest", "board"), + "#class": pinterest.PinterestBoardExtractor, + "#options": {"sections": True}, + "#count": 4, + }, + { + "#url": "https://www.pinterest.jp/gdldev/bname/", + "#comment": "board & section with /?# in name (#5104)", + "#category": ("", "pinterest", "board"), + "#class": pinterest.PinterestBoardExtractor, + "#options": {"sections": True}, + "#urls": "https://www.pinterest.jp/gdldev/bname/id:5345901183739414095", + }, + { + "#url": "https://www.pinterest.de/g1952849/secret/", + "#comment": "secret board (#1055)", + "#category": ("", "pinterest", "board"), + "#class": pinterest.PinterestBoardExtractor, + "#auth": True, + "#count": 2, + }, + { + "#url": "https://www.pinterest.com/g1952848/test/", + "#category": ("", "pinterest", "board"), + "#class": pinterest.PinterestBoardExtractor, + "#exception": exception.GalleryDLException, + }, + { + "#url": "https://www.pinterest.co.uk/hextra7519/based-animals/", + "#comment": ".co.uk TLD (#914)", + "#category": ("", "pinterest", "board"), + "#class": pinterest.PinterestBoardExtractor, + }, + { + "#url": "https://www.pinterest.com/g1952849/", + "#category": ("", "pinterest", "user"), + "#class": pinterest.PinterestUserExtractor, + "#pattern": pinterest.PinterestBoardExtractor.pattern, + "#count": ">= 2", + }, + { + "#url": "https://www.pinterest.com/g1952849/_saved/", + "#category": ("", "pinterest", "user"), + "#class": pinterest.PinterestUserExtractor, + }, + { + "#url": "https://www.pinterest.com/g1952849/pins/", + "#category": ("", "pinterest", "allpins"), + "#class": pinterest.PinterestAllpinsExtractor, + "#pattern": r"https://i\.pinimg\.com/originals/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w{3}", + "#count": 9, + }, + { + "#url": "https://www.pinterest.de/digitalmomblog/_created/", + "#category": ("", "pinterest", "created"), + "#class": pinterest.PinterestCreatedExtractor, + "#pattern": r"ytdl:|https://i\.pinimg\.com/originals/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.(jpg|png|webp)", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.pinterest.com/g1952849/stuff/section", + "#category": ("", "pinterest", "section"), + "#class": pinterest.PinterestSectionExtractor, + "#count": 2, + }, + { + "#url": "https://www.pinterest.com/search/pins/?q=nature", + "#category": ("", "pinterest", "search"), + "#class": pinterest.PinterestSearchExtractor, + "#range": "1-50", + "#count": ">= 50", + }, + { + "#url": "https://www.pinterest.com/pin/858146903966145189/#related", + "#category": ("", "pinterest", "related-pin"), + "#class": pinterest.PinterestRelatedPinExtractor, + "#range": "31-70", + "#count": 40, + "#archive": False, + }, + { + "#url": "https://www.pinterest.com/g1952849/test-/#related", + "#category": ("", "pinterest", "related-board"), + "#class": pinterest.PinterestRelatedBoardExtractor, + "#range": "31-70", + "#count": 40, + "#archive": False, + }, + { + "#url": "https://pin.it/Hvt8hgT", + "#category": ("", "pinterest", "pinit"), + "#class": pinterest.PinterestPinitExtractor, + "#sha1_url": "8daad8558382c68f0868bdbd17d05205184632fa", + }, + { + "#url": "https://pin.it/Hvt8hgS", + "#category": ("", "pinterest", "pinit"), + "#class": pinterest.PinterestPinitExtractor, + "#exception": exception.NotFoundError, + }, ) diff --git a/test/results/pinupgirlclothing.py b/test/results/pinupgirlclothing.py index ac82ad53e..e93826652 100644 --- a/test/results/pinupgirlclothing.py +++ b/test/results/pinupgirlclothing.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://pinupgirlclothing.com/collections/evening", - "#category": ("shopify", "pinupgirlclothing", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://pinupgirlclothing.com/collections/evening/products/clarice-coat-dress-in-olive-green-poly-crepe-laura-byrnes-design", - "#category": ("shopify", "pinupgirlclothing", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://pinupgirlclothing.com/collections/evening", + "#category": ("shopify", "pinupgirlclothing", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://pinupgirlclothing.com/collections/evening/products/clarice-coat-dress-in-olive-green-poly-crepe-laura-byrnes-design", + "#category": ("shopify", "pinupgirlclothing", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/pixeldrain.py b/test/results/pixeldrain.py index ed9448856..dde71144e 100644 --- a/test/results/pixeldrain.py +++ b/test/results/pixeldrain.py @@ -1,100 +1,92 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import pixeldrain import datetime -__tests__ = ( -{ - "#url" : "https://pixeldrain.com/u/jW9E6s4h", - "#category": ("", "pixeldrain", "file"), - "#class" : pixeldrain.PixeldrainFileExtractor, - "#urls" : "https://pixeldrain.com/api/file/jW9E6s4h?download", - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", - - "abuse_reporter_name" : "", - "abuse_type" : "", - "allow_video_player" : True, - "availability" : "", - "availability_message": "", - "bandwidth_used" : int, - "bandwidth_used_paid" : 0, - "can_download" : True, - "can_edit" : False, - "date" : "dt:2023-11-22 16:33:27", - "date_last_view" : r"re:\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z", - "date_upload" : "2023-11-22T16:33:27.744Z", - "delete_after_date" : "0001-01-01T00:00:00Z", - "delete_after_downloads": 0, - "download_speed_limit": 0, - "downloads" : int, - "extension" : "png", - "filename" : "test-テスト-\"&>", - "hash_sha256" : "eb359cd8f02a7d6762f9863798297ff6a22569c5c87a9d38c55bdb3a3e26003f", - "id" : "jW9E6s4h", - "mime_type" : "image/png", - "name" : "test-テスト-\"&>.png", - "show_ads" : True, - "size" : 182, - "success" : True, - "thumbnail_href" : "/file/jW9E6s4h/thumbnail", - "url" : "https://pixeldrain.com/api/file/jW9E6s4h?download", - "views" : int, -}, - -{ - "#url" : "https://pixeldrain.com/u/yEK1n2Qc", - "#category": ("", "pixeldrain", "file"), - "#class" : pixeldrain.PixeldrainFileExtractor, - "#urls" : "https://pixeldrain.com/api/file/yEK1n2Qc?download", - "#sha1_content": "08463261191d403de2133d829060050d8b04609f", - - "date" : "dt:2023-11-22 16:38:04", - "date_upload": "2023-11-22T16:38:04.928Z", - "extension" : "txt", - "filename" : '"&>', - "hash_sha256": "4c1e2bbcbe1dea8b6f895f5cdd8461c37c561bce4f1b3556ba58392d95964294", - "id" : "yEK1n2Qc", - "mime_type" : "text/plain; charset=utf-8", - "name" : '"&>.txt', - "size" : 14, -}, - -{ - "#url" : "https://pixeldrain.com/l/zQ7XpWfM", - "#category": ("", "pixeldrain", "album"), - "#class" : pixeldrain.PixeldrainAlbumExtractor, - "#urls" : ( - "https://pixeldrain.com/api/file/yEK1n2Qc?download", - "https://pixeldrain.com/api/file/jW9E6s4h?download", - ), +from gallery_dl.extractor import pixeldrain - "album" : { - "can_edit" : False, - "count" : 2, - "date" : "dt:2023-11-22 16:40:39", - "date_created": "2023-11-22T16:40:39.218Z", - "id" : "zQ7XpWfM", - "success" : True, - "title" : "アルバム", +__tests__ = ( + { + "#url": "https://pixeldrain.com/u/jW9E6s4h", + "#category": ("", "pixeldrain", "file"), + "#class": pixeldrain.PixeldrainFileExtractor, + "#urls": "https://pixeldrain.com/api/file/jW9E6s4h?download", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + "abuse_reporter_name": "", + "abuse_type": "", + "allow_video_player": True, + "availability": "", + "availability_message": "", + "bandwidth_used": int, + "bandwidth_used_paid": 0, + "can_download": True, + "can_edit": False, + "date": "dt:2023-11-22 16:33:27", + "date_last_view": r"re:\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z", + "date_upload": "2023-11-22T16:33:27.744Z", + "delete_after_date": "0001-01-01T00:00:00Z", + "delete_after_downloads": 0, + "download_speed_limit": 0, + "downloads": int, + "extension": "png", + "filename": 'test-テスト-"&>', + "hash_sha256": "eb359cd8f02a7d6762f9863798297ff6a22569c5c87a9d38c55bdb3a3e26003f", + "id": "jW9E6s4h", + "mime_type": "image/png", + "name": 'test-テスト-"&>.png', + "show_ads": True, + "size": 182, + "success": True, + "thumbnail_href": "/file/jW9E6s4h/thumbnail", + "url": "https://pixeldrain.com/api/file/jW9E6s4h?download", + "views": int, + }, + { + "#url": "https://pixeldrain.com/u/yEK1n2Qc", + "#category": ("", "pixeldrain", "file"), + "#class": pixeldrain.PixeldrainFileExtractor, + "#urls": "https://pixeldrain.com/api/file/yEK1n2Qc?download", + "#sha1_content": "08463261191d403de2133d829060050d8b04609f", + "date": "dt:2023-11-22 16:38:04", + "date_upload": "2023-11-22T16:38:04.928Z", + "extension": "txt", + "filename": '"&>', + "hash_sha256": "4c1e2bbcbe1dea8b6f895f5cdd8461c37c561bce4f1b3556ba58392d95964294", + "id": "yEK1n2Qc", + "mime_type": "text/plain; charset=utf-8", + "name": '"&>.txt', + "size": 14, + }, + { + "#url": "https://pixeldrain.com/l/zQ7XpWfM", + "#category": ("", "pixeldrain", "album"), + "#class": pixeldrain.PixeldrainAlbumExtractor, + "#urls": ( + "https://pixeldrain.com/api/file/yEK1n2Qc?download", + "https://pixeldrain.com/api/file/jW9E6s4h?download", + ), + "album": { + "can_edit": False, + "count": 2, + "date": "dt:2023-11-22 16:40:39", + "date_created": "2023-11-22T16:40:39.218Z", + "id": "zQ7XpWfM", + "success": True, + "title": "アルバム", + }, + "date": datetime.datetime, + "description": "", + "detail_href": r"re:/file/(yEK1n2Qc|jW9E6s4h)/info", + "hash_sha256": r"re:\w{64}", + "id": r"re:yEK1n2Qc|jW9E6s4h", + "mime_type": str, + }, + { + "#url": "https://pixeldrain.com/l/zQ7XpWfM#item=0", + "#category": ("", "pixeldrain", "album"), + "#class": pixeldrain.PixeldrainAlbumExtractor, + "#urls": "https://pixeldrain.com/api/file/jW9E6s4h?download", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", }, - "date" : datetime.datetime, - "description": "", - "detail_href": r"re:/file/(yEK1n2Qc|jW9E6s4h)/info", - "hash_sha256": r"re:\w{64}", - "id" : r"re:yEK1n2Qc|jW9E6s4h", - "mime_type" : str, -}, - -{ - "#url" : "https://pixeldrain.com/l/zQ7XpWfM#item=0", - "#category": ("", "pixeldrain", "album"), - "#class" : pixeldrain.PixeldrainAlbumExtractor, - "#urls" : "https://pixeldrain.com/api/file/jW9E6s4h?download", - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", -}, - ) diff --git a/test/results/pixhost.py b/test/results/pixhost.py index 82fbea985..790379a60 100644 --- a/test/results/pixhost.py +++ b/test/results/pixhost.py @@ -1,28 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://pixhost.to/show/190/130327671_test-.png", - "#category": ("imagehost", "pixhost", "image"), - "#class" : imagehosts.PixhostImageExtractor, - "#sha1_url" : "4e5470dcf6513944773044d40d883221bbc46cff", - "#sha1_metadata": "3bad6d59db42a5ebbd7842c2307e1c3ebd35e6b0", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - -{ - "#url" : "https://pixhost.to/gallery/jSMFq", - "#category": ("imagehost", "pixhost", "gallery"), - "#class" : imagehosts.PixhostGalleryExtractor, - "#pattern" : imagehosts.PixhostImageExtractor.pattern, - "#count" : 3, -}, - + { + "#url": "https://pixhost.to/show/190/130327671_test-.png", + "#category": ("imagehost", "pixhost", "image"), + "#class": imagehosts.PixhostImageExtractor, + "#sha1_url": "4e5470dcf6513944773044d40d883221bbc46cff", + "#sha1_metadata": "3bad6d59db42a5ebbd7842c2307e1c3ebd35e6b0", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, + { + "#url": "https://pixhost.to/gallery/jSMFq", + "#category": ("imagehost", "pixhost", "gallery"), + "#class": imagehosts.PixhostGalleryExtractor, + "#pattern": imagehosts.PixhostImageExtractor.pattern, + "#count": 3, + }, ) diff --git a/test/results/pixiv.py b/test/results/pixiv.py index d3f5bde49..b8fabc406 100644 --- a/test/results/pixiv.py +++ b/test/results/pixiv.py @@ -1,676 +1,592 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import pixiv from gallery_dl import exception - +from gallery_dl.extractor import pixiv __tests__ = ( -{ - "#url" : "https://www.pixiv.net/en/users/173530", - "#category": ("", "pixiv", "user"), - "#class" : pixiv.PixivUserExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/u/173530", - "#category": ("", "pixiv", "user"), - "#class" : pixiv.PixivUserExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/member.php?id=173530", - "#category": ("", "pixiv", "user"), - "#class" : pixiv.PixivUserExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/mypage.php#id=173530", - "#category": ("", "pixiv", "user"), - "#class" : pixiv.PixivUserExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/#id=173530", - "#category": ("", "pixiv", "user"), - "#class" : pixiv.PixivUserExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/artworks", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, - "#sha1_url": "852c31ad83b6840bacbce824d85f2a997889efb7", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/artworks/%E6%89%8B%E3%81%B6%E3%82%8D", - "#comment" : "illusts with specific tag", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, - "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", -}, - -{ - "#url" : "https://www.pixiv.net/member_illust.php?id=173530&tag=%E6%89%8B%E3%81%B6%E3%82%8D", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, - "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", -}, - -{ - "#url" : "http://www.pixiv.net/member_illust.php?id=173531", - "#comment" : "deleted account", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, - "#options" : {"metadata": True}, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/56514424/artworks", - "#comment" : "limit_sanity_level_360.png in artworks results (#5435, #6339)", - "#class" : pixiv.PixivArtworksExtractor, - "#count" : ">= 39", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/manga", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/illustrations", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/member_illust.php?id=173530", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, -}, - -{ - "#url" : "https://touch.pixiv.net/member_illust.php?id=173530", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/avatar", - "#category": ("", "pixiv", "avatar"), - "#class" : pixiv.PixivAvatarExtractor, - "#sha1_content": "4e57544480cc2036ea9608103e8f024fa737fe66", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/194921/background", - "#category": ("", "pixiv", "background"), - "#class" : pixiv.PixivBackgroundExtractor, - "#pattern" : r"https://i\.pximg\.net/background/img/2021/01/30/16/12/02/194921_af1f71e557a42f499213d4b9eaccc0f8\.jpg", -}, - -{ - "#url" : "https://pixiv.me/del_shannon", - "#category": ("", "pixiv", "me"), - "#class" : pixiv.PixivMeExtractor, - "#sha1_url": "29c295ce75150177e6b0a09089a949804c708fbf", -}, - -{ - "#url" : "https://pixiv.me/del_shanno", - "#category": ("", "pixiv", "me"), - "#class" : pixiv.PixivMeExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.pixiv.net/artworks/966412", - "#comment" : "related works (#1237)", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, - "#sha1_url" : "90c1715b07b0d1aad300bce256a0bc71f42540ba", - "#sha1_content": "69a8edfb717400d1c2e146ab2b30d2c235440c5a", - - "date" : "dt:2008-06-12 15:29:13", - "date_url": "dt:2008-06-12 15:29:13", -}, - -{ - "#url" : "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=966411", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=66806629", - "#comment" : "ugoira", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, - "#urls" : "https://i.pximg.net/img-zip-ugoira/img/2018/01/15/13/24/48/66806629_ugoira1920x1080.zip", - - "frames" : list, - "date" : "dt:2018-01-14 15:06:08", - "date_url": "dt:2018-01-15 04:24:48", -}, - -{ - "#url" : "https://www.pixiv.net/artworks/101003492", - "#comment" : "original ugoira frames (#6056)", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, - "#options" : {"ugoira": "original"}, - "#urls" : [ - "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira0.png", - "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira1.png", - "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira2.png", - "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira3.png", - "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira4.png", - "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira5.png", - ], - - "frames": list, -}, - -{ - "#url" : "https://www.pixiv.net/artworks/966412", - "#comment" : "related works (#1237)", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, - "#options" : {"related": True}, - "#range" : "1-10", - "#count" : ">= 10", -}, - -{ - "#url" : "https://www.pixiv.net/artworks/85960783", - "#comment" : "limit_sanity_level_360.png (#4327, #5180)", - "#class" : pixiv.PixivWorkExtractor, - "#options" : {"sanity": False}, - "#count" : 0, -}, - -{ - "#url" : "https://www.pixiv.net/en/artworks/102932581", - "#comment" : "limit_sanity_level_360.png (#4327, #5180)", - "#class" : pixiv.PixivWorkExtractor, - "#options" : {"sanity": True}, - "#urls" : "https://i.pximg.net/img-original/img/2022/11/20/00/00/49/102932581_p0.jpg", - - "caption" : "Meet a deer .", - "comment_access_control": 0, - "create_date" : "2022-11-19T15:00:00+00:00", - "date" : "dt:2022-11-19 15:00:00", - "date_url" : "dt:2022-11-19 15:00:49", - "extension" : "jpg", - "filename" : "102932581_p0", - "height" : 3840, - "id" : 102932581, - "illust_ai_type": 1, - "illust_book_style": 0, - "is_bookmarked" : False, - "is_muted" : False, - "num" : 0, - "page_count" : 1, - "rating" : "General", - "restrict" : 0, - "sanity_level" : 2, - "series" : None, - "suffix" : "", - "title" : "《 Bridge and Deer 》", - "tools" : [], - "total_bookmarks": range(1900, 3000), - "total_comments": range(3, 10), - "total_view" : range(11000, 20000), - "type" : "illust", - "url" : "https://i.pximg.net/img-original/img/2022/11/20/00/00/49/102932581_p0.jpg", - "visible" : False, - "width" : 2160, - "x_restrict" : 0, - "image_urls" : { - "mini" : "https://i.pximg.net/c/48x48/custom-thumb/img/2022/11/20/00/00/49/102932581_p0_custom1200.jpg", - "original": "https://i.pximg.net/img-original/img/2022/11/20/00/00/49/102932581_p0.jpg", - "regular" : "https://i.pximg.net/img-master/img/2022/11/20/00/00/49/102932581_p0_master1200.jpg", - "small" : "https://i.pximg.net/c/540x540_70/img-master/img/2022/11/20/00/00/49/102932581_p0_master1200.jpg", - "thumb" : "https://i.pximg.net/c/250x250_80_a2/custom-thumb/img/2022/11/20/00/00/49/102932581_p0_custom1200.jpg", - }, - "tags" : [ - "オリジナル", - "風景", - "イラスト", - "illustration", - "美しい", - "女の子", - "少女", - "deer", - "flower", - "spring", - ], - "user" : { - "account" : "805482263", - "id" : 7386235, - "is_followed": False, - "name" : "岛的鲸", - "profile_image_urls": {}, - }, -}, - -{ - "#url" : "https://www.pixiv.net/en/artworks/109487939", - "#comment" : "R-18 limit_sanity_level_360.png (#4327, #5180)", - "#class" : pixiv.PixivWorkExtractor, - "#urls" : [ - "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p0.png", - "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p1.png", - "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p2.png", - "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p3.png", - ], -}, - -{ - "#url" : "https://www.pixiv.net/en/artworks/104582860", - "#comment" : "deleted limit_sanity_level_360.png work (#6339)", - "#class" : pixiv.PixivWorkExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.pixiv.net/en/artworks/103983466", - "#comment" : "empty 'caption' in App API response (#4327, #5191)", - "#class" : pixiv.PixivWorkExtractor, - "#options" : {"captions": True}, - - "caption": r"re:Either she doesn't know how to pose or she can't move with that much clothing on her, in any case she's very well dressed for a holiday trip around town. Lots of stuff to see and a perfect day to grab some sweet pastries at the bakery.
          ...", -}, - -{ - "#url" : "https://www.pixiv.net/en/artworks/966412", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=96641", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "http://i1.pixiv.net/c/600x600/img-master/img/2008/06/13/00/29/13/966412_p0_master1200.jpg", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "https://i.pximg.net/img-original/img/2017/04/25/07/33/29/62568267_p0.png", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/i/966412", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "http://img.pixiv.net/img/soundcross/42626136.jpg", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "http://i2.pixiv.net/img76/img/snailrin/42672235.jpg", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/en/artworks/unlisted/eE3fTYaROT9IsZmep386", - "#class" : pixiv.PixivUnlistedExtractor, - "#urls" : "https://i.pximg.net/img-original/img/2020/10/15/00/46/12/85017704-149014193e4d3e23a6b8bd5e38b51ed4_p0.png", - - "id" : 85017704, - "id_unlisted": "eE3fTYaROT9IsZmep386", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/bookmarks/artworks", - "#category": ("", "pixiv", "favorite"), - "#class" : pixiv.PixivFavoriteExtractor, - "#urls" : [ - "https://i.pximg.net/img-original/img/2008/10/31/17/54/01/2005108_p0.jpg", - "https://i.pximg.net/img-original/img/2008/09/27/12/22/40/1719386_p0.jpg", - "https://i.pximg.net/img-original/img/2008/04/15/01/43/46/669358_p0.jpg", - "https://i.pximg.net/img-original/img/2008/06/19/21/52/15/1005851_p0.jpg", - "https://i.pximg.net/img-original/img/2008/06/17/22/16/54/994965_p0.jpg", - ], -}, - -{ - "#url" : "https://www.pixiv.net/bookmark.php?id=173530", - "#category": ("", "pixiv", "favorite"), - "#class" : pixiv.PixivFavoriteExtractor, - "#urls" : [ - "https://i.pximg.net/img-original/img/2008/10/31/17/54/01/2005108_p0.jpg", - "https://i.pximg.net/img-original/img/2008/09/27/12/22/40/1719386_p0.jpg", - "https://i.pximg.net/img-original/img/2008/04/15/01/43/46/669358_p0.jpg", - "https://i.pximg.net/img-original/img/2008/06/19/21/52/15/1005851_p0.jpg", - "https://i.pximg.net/img-original/img/2008/06/17/22/16/54/994965_p0.jpg", - ], -}, - -{ - "#url" : "https://www.pixiv.net/en/users/3137110/bookmarks/artworks/%E3%81%AF%E3%82%93%E3%82%82%E3%82%93", - "#comment" : "bookmarks with specific tag", - "#category": ("", "pixiv", "favorite"), - "#class" : pixiv.PixivFavoriteExtractor, - "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", -}, - -{ - "#url" : "https://www.pixiv.net/bookmark.php?id=3137110&tag=%E3%81%AF%E3%82%93%E3%82%82%E3%82%93&p=1", - "#comment" : "bookmarks with specific tag (legacy url)", - "#category": ("", "pixiv", "favorite"), - "#class" : pixiv.PixivFavoriteExtractor, - "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", -}, - -{ - "#url" : "https://www.pixiv.net/bookmark.php", - "#comment" : "own bookmarks", - "#category": ("", "pixiv", "bookmark"), - "#class" : pixiv.PixivFavoriteExtractor, - "#options" : {"metadata-bookmark": True}, - "#sha1_url": "90c1715b07b0d1aad300bce256a0bc71f42540ba", - - "tags_bookmark": [ - "47", - "hitman", - ], -}, - -{ - "#url" : "https://www.pixiv.net/bookmark.php?tag=foobar", - "#comment" : "own bookmarks with tag (#596)", - "#category": ("", "pixiv", "bookmark"), - "#class" : pixiv.PixivFavoriteExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/following", - "#comment" : "followed users (#515)", - "#category": ("", "pixiv", "following"), - "#class" : pixiv.PixivFavoriteExtractor, - "#pattern" : pixiv.PixivUserExtractor.pattern, - "#count" : ">= 12", -}, - -{ - "#url" : "https://www.pixiv.net/bookmark.php?id=173530&type=user", - "#comment" : "followed users (legacy url) (#515)", - "#category": ("", "pixiv", "following"), - "#class" : pixiv.PixivFavoriteExtractor, - "#pattern" : pixiv.PixivUserExtractor.pattern, - "#count" : ">= 12", -}, - -{ - "#url" : "https://touch.pixiv.net/bookmark.php?id=173530", - "#comment" : "touch URLs", - "#category": ("", "pixiv", "favorite"), - "#class" : pixiv.PixivFavoriteExtractor, -}, - -{ - "#url" : "https://touch.pixiv.net/bookmark.php", - "#category": ("", "pixiv", "bookmark"), - "#class" : pixiv.PixivFavoriteExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/ranking.php?mode=daily&date=20170818", - "#category": ("", "pixiv", "ranking"), - "#class" : pixiv.PixivRankingExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/ranking.php", - "#category": ("", "pixiv", "ranking"), - "#class" : pixiv.PixivRankingExtractor, -}, - -{ - "#url" : "https://touch.pixiv.net/ranking.php", - "#category": ("", "pixiv", "ranking"), - "#class" : pixiv.PixivRankingExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/ranking.php?mode=unknown", - "#category": ("", "pixiv", "ranking"), - "#class" : pixiv.PixivRankingExtractor, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://www.pixiv.net/en/tags/Original", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://pixiv.net/en/tags/foo/artworks?order=week&s_mode=s_tag", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://pixiv.net/en/tags/foo/artworks?order=date&s_mode=tag", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://www.pixiv.net/search.php?s_mode=s_tag&name=Original", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://www.pixiv.net/en/tags/foo/artworks?order=date&s_mode=s_tag", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/search.php?s_mode=s_tag&word=Original", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, -}, - -{ - "#url" : "https://touch.pixiv.net/search.php?word=Original", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/bookmark_new_illust.php", - "#category": ("", "pixiv", "follow"), - "#class" : pixiv.PixivFollowExtractor, -}, - -{ - "#url" : "https://touch.pixiv.net/bookmark_new_illust.php", - "#category": ("", "pixiv", "follow"), - "#class" : pixiv.PixivFollowExtractor, -}, - -{ - "#url" : "https://www.pixivision.net/en/a/2791", - "#category": ("", "pixiv", "pixivision"), - "#class" : pixiv.PixivPixivisionExtractor, -}, - -{ - "#url" : "https://pixivision.net/a/2791", - "#category": ("", "pixiv", "pixivision"), - "#class" : pixiv.PixivPixivisionExtractor, - "#count" : 7, - - "pixivision_id" : "2791", - "pixivision_title": "What's your favorite music? Editor’s picks featuring: “CD Covers”!", -}, - -{ - "#url" : "https://www.pixiv.net/user/10509347/series/21859", - "#category": ("", "pixiv", "series"), - "#class" : pixiv.PixivSeriesExtractor, - "#range" : "1-10", - "#count" : 10, - - "num_series": int, - "series" : { - "create_date": "2017-10-22T14:07:42+09:00", - "width" : 4250, - "height": 3009, - "id" : 21859, - "title" : "先輩がうざい後輩の話", - "total" : range(100, 500), - "user" : dict, - "watchlist_added": False, - }, -}, - -{ - "#url" : "https://www.pixiv.net/novel/show.php?id=12101012", - "#category": ("", "pixiv", "novel"), - "#class" : pixiv.PixivNovelExtractor, - "#count" : 1, - "#sha1_content": "20f4a62f0e87ae2cb9f5a787b6c641bfa4eabf93", - - "caption" : "
          第一印象から決めてました！

          素敵な表紙はいもこは妹さん(illust/53802907)からお借りしました。

          たくさんのコメント、タグありがとうございます、本当に嬉しいです。お返事できていませんが、一つ一つ目を通させていただいてます。タイトルも込みで読んでくださってすごく嬉しいです。ありがとうございます……！！

          ■12/19付けルキラン20位を頂きました…！大変混乱していますがすごく嬉しいです。ありがとうございます！　

          ■2019/12/20デイリー15位、女子に人気8位をを頂きました…！？！？！？！？て、手が震える…。ありがとうございます…ひえええ。感謝してもしきれないです…！", - "create_date" : "2019-12-19T23:14:36+09:00", - "date" : "dt:2019-12-19 14:14:36", - "extension" : "txt", - "id" : 12101012, - "image_urls" : dict, - "is_bookmarked" : False, - "is_muted" : False, - "is_mypixiv_only": False, - "is_original" : False, - "is_x_restricted": False, - "novel_ai_type" : 0, - "page_count" : 1, - "rating" : "General", - "restrict" : 0, - "series" : { - "id" : 1479656, - "title": "一目惚れした彼らの話", - }, - "tags" : [ - "鬼滅の夢", - "女主人公", - "煉獄杏寿郎", - "涙腺崩壊", - "なにこれすごい", - "来世で幸せになって欲しい", - "キメ学世界線できっと幸せになってる!!", - "あなたが神か!!", - "キメ学編を·····", - "鬼滅の夢小説10000users入り", - ], - "text_length" : 9569, - "title" : "本当は、一目惚れだった", - "total_bookmarks": range(17900, 20000), - "total_comments" : range(200, 400), - "total_view" : range(158000, 300000), - "user" : { - "account": "46_maru", - "id" : 888268, - }, - "visible" : True, - "x_restrict" : 0, -}, - -{ - "#url" : "https://www.pixiv.net/novel/show.php?id=16422450", - "#comment" : "embeds // covers (#5373)", - "#category": ("", "pixiv", "novel"), - "#class" : pixiv.PixivNovelExtractor, - "#options" : { - "embeds": True, - "covers": True, - }, - "#count" : 4, -}, - -{ - "#url" : "https://www.pixiv.net/novel/show.php?id=12101012", - "#comment" : "full series", - "#category": ("", "pixiv", "novel"), - "#class" : pixiv.PixivNovelExtractor, - "#options" : {"full-series": True}, - "#count" : 2, -}, - -{ - "#url" : "https://www.pixiv.net/n/19612040", - "#comment" : "short URL", - "#category": ("", "pixiv", "novel"), - "#class" : pixiv.PixivNovelExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/77055466/novels", - "#category": ("", "pixiv", "novel-user"), - "#class" : pixiv.PixivNovelUserExtractor, - "#pattern" : "^text:", - "#range" : "1-5", - "#count" : 5, -}, - -{ - "#url" : "https://www.pixiv.net/novel/series/1479656", - "#category": ("", "pixiv", "novel-series"), - "#class" : pixiv.PixivNovelSeriesExtractor, - "#count" : 2, - "#sha1_content": "243ce593333bbfe26e255e3372d9c9d8cea22d5b", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/77055466/bookmarks/novels", - "#category": ("", "pixiv", "novel-bookmark"), - "#class" : pixiv.PixivNovelBookmarkExtractor, - "#count" : 1, - "#sha1_content": "7194e8faa876b2b536f185ee271a2b6e46c69089", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/11/bookmarks/novels/TAG?rest=hide", - "#category": ("", "pixiv", "novel-bookmark"), - "#class" : pixiv.PixivNovelBookmarkExtractor, -}, - -{ - "#url" : "https://sketch.pixiv.net/@nicoby", - "#category": ("", "pixiv", "sketch"), - "#class" : pixiv.PixivSketchExtractor, - "#pattern" : r"https://img\-sketch\.pixiv\.net/uploads/medium/file/\d+/\d+\.(jpg|png)", - "#count" : ">= 35", -}, - + { + "#url": "https://www.pixiv.net/en/users/173530", + "#category": ("", "pixiv", "user"), + "#class": pixiv.PixivUserExtractor, + }, + { + "#url": "https://www.pixiv.net/u/173530", + "#category": ("", "pixiv", "user"), + "#class": pixiv.PixivUserExtractor, + }, + { + "#url": "https://www.pixiv.net/member.php?id=173530", + "#category": ("", "pixiv", "user"), + "#class": pixiv.PixivUserExtractor, + }, + { + "#url": "https://www.pixiv.net/mypage.php#id=173530", + "#category": ("", "pixiv", "user"), + "#class": pixiv.PixivUserExtractor, + }, + { + "#url": "https://www.pixiv.net/#id=173530", + "#category": ("", "pixiv", "user"), + "#class": pixiv.PixivUserExtractor, + }, + { + "#url": "https://www.pixiv.net/en/users/173530/artworks", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + "#sha1_url": "852c31ad83b6840bacbce824d85f2a997889efb7", + }, + { + "#url": "https://www.pixiv.net/en/users/173530/artworks/%E6%89%8B%E3%81%B6%E3%82%8D", + "#comment": "illusts with specific tag", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", + }, + { + "#url": "https://www.pixiv.net/member_illust.php?id=173530&tag=%E6%89%8B%E3%81%B6%E3%82%8D", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", + }, + { + "#url": "http://www.pixiv.net/member_illust.php?id=173531", + "#comment": "deleted account", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + "#options": {"metadata": True}, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.pixiv.net/en/users/56514424/artworks", + "#comment": "limit_sanity_level_360.png in artworks results (#5435, #6339)", + "#class": pixiv.PixivArtworksExtractor, + "#count": ">= 39", + }, + { + "#url": "https://www.pixiv.net/en/users/173530/manga", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + }, + { + "#url": "https://www.pixiv.net/en/users/173530/illustrations", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + }, + { + "#url": "https://www.pixiv.net/member_illust.php?id=173530", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + }, + { + "#url": "https://touch.pixiv.net/member_illust.php?id=173530", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + }, + { + "#url": "https://www.pixiv.net/en/users/173530/avatar", + "#category": ("", "pixiv", "avatar"), + "#class": pixiv.PixivAvatarExtractor, + "#sha1_content": "4e57544480cc2036ea9608103e8f024fa737fe66", + }, + { + "#url": "https://www.pixiv.net/en/users/194921/background", + "#category": ("", "pixiv", "background"), + "#class": pixiv.PixivBackgroundExtractor, + "#pattern": r"https://i\.pximg\.net/background/img/2021/01/30/16/12/02/194921_af1f71e557a42f499213d4b9eaccc0f8\.jpg", + }, + { + "#url": "https://pixiv.me/del_shannon", + "#category": ("", "pixiv", "me"), + "#class": pixiv.PixivMeExtractor, + "#sha1_url": "29c295ce75150177e6b0a09089a949804c708fbf", + }, + { + "#url": "https://pixiv.me/del_shanno", + "#category": ("", "pixiv", "me"), + "#class": pixiv.PixivMeExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.pixiv.net/artworks/966412", + "#comment": "related works (#1237)", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + "#sha1_url": "90c1715b07b0d1aad300bce256a0bc71f42540ba", + "#sha1_content": "69a8edfb717400d1c2e146ab2b30d2c235440c5a", + "date": "dt:2008-06-12 15:29:13", + "date_url": "dt:2008-06-12 15:29:13", + }, + { + "#url": "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=966411", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=66806629", + "#comment": "ugoira", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + "#urls": "https://i.pximg.net/img-zip-ugoira/img/2018/01/15/13/24/48/66806629_ugoira1920x1080.zip", + "frames": list, + "date": "dt:2018-01-14 15:06:08", + "date_url": "dt:2018-01-15 04:24:48", + }, + { + "#url": "https://www.pixiv.net/artworks/101003492", + "#comment": "original ugoira frames (#6056)", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + "#options": {"ugoira": "original"}, + "#urls": [ + "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira0.png", + "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira1.png", + "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira2.png", + "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira3.png", + "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira4.png", + "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira5.png", + ], + "frames": list, + }, + { + "#url": "https://www.pixiv.net/artworks/966412", + "#comment": "related works (#1237)", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + "#options": {"related": True}, + "#range": "1-10", + "#count": ">= 10", + }, + { + "#url": "https://www.pixiv.net/artworks/85960783", + "#comment": "limit_sanity_level_360.png (#4327, #5180)", + "#class": pixiv.PixivWorkExtractor, + "#options": {"sanity": False}, + "#count": 0, + }, + { + "#url": "https://www.pixiv.net/en/artworks/102932581", + "#comment": "limit_sanity_level_360.png (#4327, #5180)", + "#class": pixiv.PixivWorkExtractor, + "#options": {"sanity": True}, + "#urls": "https://i.pximg.net/img-original/img/2022/11/20/00/00/49/102932581_p0.jpg", + "caption": "Meet a deer .", + "comment_access_control": 0, + "create_date": "2022-11-19T15:00:00+00:00", + "date": "dt:2022-11-19 15:00:00", + "date_url": "dt:2022-11-19 15:00:49", + "extension": "jpg", + "filename": "102932581_p0", + "height": 3840, + "id": 102932581, + "illust_ai_type": 1, + "illust_book_style": 0, + "is_bookmarked": False, + "is_muted": False, + "num": 0, + "page_count": 1, + "rating": "General", + "restrict": 0, + "sanity_level": 2, + "series": None, + "suffix": "", + "title": "《 Bridge and Deer 》", + "tools": [], + "total_bookmarks": range(1900, 3000), + "total_comments": range(3, 10), + "total_view": range(11000, 20000), + "type": "illust", + "url": "https://i.pximg.net/img-original/img/2022/11/20/00/00/49/102932581_p0.jpg", + "visible": False, + "width": 2160, + "x_restrict": 0, + "image_urls": { + "mini": "https://i.pximg.net/c/48x48/custom-thumb/img/2022/11/20/00/00/49/102932581_p0_custom1200.jpg", + "original": "https://i.pximg.net/img-original/img/2022/11/20/00/00/49/102932581_p0.jpg", + "regular": "https://i.pximg.net/img-master/img/2022/11/20/00/00/49/102932581_p0_master1200.jpg", + "small": "https://i.pximg.net/c/540x540_70/img-master/img/2022/11/20/00/00/49/102932581_p0_master1200.jpg", + "thumb": "https://i.pximg.net/c/250x250_80_a2/custom-thumb/img/2022/11/20/00/00/49/102932581_p0_custom1200.jpg", + }, + "tags": [ + "オリジナル", + "風景", + "イラスト", + "illustration", + "美しい", + "女の子", + "少女", + "deer", + "flower", + "spring", + ], + "user": { + "account": "805482263", + "id": 7386235, + "is_followed": False, + "name": "岛的鲸", + "profile_image_urls": {}, + }, + }, + { + "#url": "https://www.pixiv.net/en/artworks/109487939", + "#comment": "R-18 limit_sanity_level_360.png (#4327, #5180)", + "#class": pixiv.PixivWorkExtractor, + "#urls": [ + "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p0.png", + "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p1.png", + "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p2.png", + "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p3.png", + ], + }, + { + "#url": "https://www.pixiv.net/en/artworks/104582860", + "#comment": "deleted limit_sanity_level_360.png work (#6339)", + "#class": pixiv.PixivWorkExtractor, + "#count": 0, + }, + { + "#url": "https://www.pixiv.net/en/artworks/103983466", + "#comment": "empty 'caption' in App API response (#4327, #5191)", + "#class": pixiv.PixivWorkExtractor, + "#options": {"captions": True}, + "caption": r"re:Either she doesn't know how to pose or she can't move with that much clothing on her, in any case she's very well dressed for a holiday trip around town. Lots of stuff to see and a perfect day to grab some sweet pastries at the bakery.
          ...", + }, + { + "#url": "https://www.pixiv.net/en/artworks/966412", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=96641", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "http://i1.pixiv.net/c/600x600/img-master/img/2008/06/13/00/29/13/966412_p0_master1200.jpg", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "https://i.pximg.net/img-original/img/2017/04/25/07/33/29/62568267_p0.png", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "https://www.pixiv.net/i/966412", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "http://img.pixiv.net/img/soundcross/42626136.jpg", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "http://i2.pixiv.net/img76/img/snailrin/42672235.jpg", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "https://www.pixiv.net/en/artworks/unlisted/eE3fTYaROT9IsZmep386", + "#class": pixiv.PixivUnlistedExtractor, + "#urls": "https://i.pximg.net/img-original/img/2020/10/15/00/46/12/85017704-149014193e4d3e23a6b8bd5e38b51ed4_p0.png", + "id": 85017704, + "id_unlisted": "eE3fTYaROT9IsZmep386", + }, + { + "#url": "https://www.pixiv.net/en/users/173530/bookmarks/artworks", + "#category": ("", "pixiv", "favorite"), + "#class": pixiv.PixivFavoriteExtractor, + "#urls": [ + "https://i.pximg.net/img-original/img/2008/10/31/17/54/01/2005108_p0.jpg", + "https://i.pximg.net/img-original/img/2008/09/27/12/22/40/1719386_p0.jpg", + "https://i.pximg.net/img-original/img/2008/04/15/01/43/46/669358_p0.jpg", + "https://i.pximg.net/img-original/img/2008/06/19/21/52/15/1005851_p0.jpg", + "https://i.pximg.net/img-original/img/2008/06/17/22/16/54/994965_p0.jpg", + ], + }, + { + "#url": "https://www.pixiv.net/bookmark.php?id=173530", + "#category": ("", "pixiv", "favorite"), + "#class": pixiv.PixivFavoriteExtractor, + "#urls": [ + "https://i.pximg.net/img-original/img/2008/10/31/17/54/01/2005108_p0.jpg", + "https://i.pximg.net/img-original/img/2008/09/27/12/22/40/1719386_p0.jpg", + "https://i.pximg.net/img-original/img/2008/04/15/01/43/46/669358_p0.jpg", + "https://i.pximg.net/img-original/img/2008/06/19/21/52/15/1005851_p0.jpg", + "https://i.pximg.net/img-original/img/2008/06/17/22/16/54/994965_p0.jpg", + ], + }, + { + "#url": "https://www.pixiv.net/en/users/3137110/bookmarks/artworks/%E3%81%AF%E3%82%93%E3%82%82%E3%82%93", + "#comment": "bookmarks with specific tag", + "#category": ("", "pixiv", "favorite"), + "#class": pixiv.PixivFavoriteExtractor, + "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", + }, + { + "#url": "https://www.pixiv.net/bookmark.php?id=3137110&tag=%E3%81%AF%E3%82%93%E3%82%82%E3%82%93&p=1", + "#comment": "bookmarks with specific tag (legacy url)", + "#category": ("", "pixiv", "favorite"), + "#class": pixiv.PixivFavoriteExtractor, + "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", + }, + { + "#url": "https://www.pixiv.net/bookmark.php", + "#comment": "own bookmarks", + "#category": ("", "pixiv", "bookmark"), + "#class": pixiv.PixivFavoriteExtractor, + "#options": {"metadata-bookmark": True}, + "#sha1_url": "90c1715b07b0d1aad300bce256a0bc71f42540ba", + "tags_bookmark": [ + "47", + "hitman", + ], + }, + { + "#url": "https://www.pixiv.net/bookmark.php?tag=foobar", + "#comment": "own bookmarks with tag (#596)", + "#category": ("", "pixiv", "bookmark"), + "#class": pixiv.PixivFavoriteExtractor, + "#count": 0, + }, + { + "#url": "https://www.pixiv.net/en/users/173530/following", + "#comment": "followed users (#515)", + "#category": ("", "pixiv", "following"), + "#class": pixiv.PixivFavoriteExtractor, + "#pattern": pixiv.PixivUserExtractor.pattern, + "#count": ">= 12", + }, + { + "#url": "https://www.pixiv.net/bookmark.php?id=173530&type=user", + "#comment": "followed users (legacy url) (#515)", + "#category": ("", "pixiv", "following"), + "#class": pixiv.PixivFavoriteExtractor, + "#pattern": pixiv.PixivUserExtractor.pattern, + "#count": ">= 12", + }, + { + "#url": "https://touch.pixiv.net/bookmark.php?id=173530", + "#comment": "touch URLs", + "#category": ("", "pixiv", "favorite"), + "#class": pixiv.PixivFavoriteExtractor, + }, + { + "#url": "https://touch.pixiv.net/bookmark.php", + "#category": ("", "pixiv", "bookmark"), + "#class": pixiv.PixivFavoriteExtractor, + }, + { + "#url": "https://www.pixiv.net/ranking.php?mode=daily&date=20170818", + "#category": ("", "pixiv", "ranking"), + "#class": pixiv.PixivRankingExtractor, + }, + { + "#url": "https://www.pixiv.net/ranking.php", + "#category": ("", "pixiv", "ranking"), + "#class": pixiv.PixivRankingExtractor, + }, + { + "#url": "https://touch.pixiv.net/ranking.php", + "#category": ("", "pixiv", "ranking"), + "#class": pixiv.PixivRankingExtractor, + }, + { + "#url": "https://www.pixiv.net/ranking.php?mode=unknown", + "#category": ("", "pixiv", "ranking"), + "#class": pixiv.PixivRankingExtractor, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://www.pixiv.net/en/tags/Original", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://pixiv.net/en/tags/foo/artworks?order=week&s_mode=s_tag", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://pixiv.net/en/tags/foo/artworks?order=date&s_mode=tag", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://www.pixiv.net/search.php?s_mode=s_tag&name=Original", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://www.pixiv.net/en/tags/foo/artworks?order=date&s_mode=s_tag", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + }, + { + "#url": "https://www.pixiv.net/search.php?s_mode=s_tag&word=Original", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + }, + { + "#url": "https://touch.pixiv.net/search.php?word=Original", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + }, + { + "#url": "https://www.pixiv.net/bookmark_new_illust.php", + "#category": ("", "pixiv", "follow"), + "#class": pixiv.PixivFollowExtractor, + }, + { + "#url": "https://touch.pixiv.net/bookmark_new_illust.php", + "#category": ("", "pixiv", "follow"), + "#class": pixiv.PixivFollowExtractor, + }, + { + "#url": "https://www.pixivision.net/en/a/2791", + "#category": ("", "pixiv", "pixivision"), + "#class": pixiv.PixivPixivisionExtractor, + }, + { + "#url": "https://pixivision.net/a/2791", + "#category": ("", "pixiv", "pixivision"), + "#class": pixiv.PixivPixivisionExtractor, + "#count": 7, + "pixivision_id": "2791", + "pixivision_title": "What's your favorite music? Editor’s picks featuring: “CD Covers”!", + }, + { + "#url": "https://www.pixiv.net/user/10509347/series/21859", + "#category": ("", "pixiv", "series"), + "#class": pixiv.PixivSeriesExtractor, + "#range": "1-10", + "#count": 10, + "num_series": int, + "series": { + "create_date": "2017-10-22T14:07:42+09:00", + "width": 4250, + "height": 3009, + "id": 21859, + "title": "先輩がうざい後輩の話", + "total": range(100, 500), + "user": dict, + "watchlist_added": False, + }, + }, + { + "#url": "https://www.pixiv.net/novel/show.php?id=12101012", + "#category": ("", "pixiv", "novel"), + "#class": pixiv.PixivNovelExtractor, + "#count": 1, + "#sha1_content": "20f4a62f0e87ae2cb9f5a787b6c641bfa4eabf93", + "caption": '
          第一印象から決めてました！

          素敵な表紙はいもこは妹さん(illust/53802907)からお借りしました。

          たくさんのコメント、タグありがとうございます、本当に嬉しいです。お返事できていませんが、一つ一つ目を通させていただいてます。タイトルも込みで読んでくださってすごく嬉しいです。ありがとうございます……！！

          ■12/19付けルキラン20位を頂きました…！大変混乱していますがすごく嬉しいです。ありがとうございます！　

          ■2019/12/20デイリー15位、女子に人気8位をを頂きました…！？！？！？！？て、手が震える…。ありがとうございます…ひえええ。感謝してもしきれないです…！', + "create_date": "2019-12-19T23:14:36+09:00", + "date": "dt:2019-12-19 14:14:36", + "extension": "txt", + "id": 12101012, + "image_urls": dict, + "is_bookmarked": False, + "is_muted": False, + "is_mypixiv_only": False, + "is_original": False, + "is_x_restricted": False, + "novel_ai_type": 0, + "page_count": 1, + "rating": "General", + "restrict": 0, + "series": { + "id": 1479656, + "title": "一目惚れした彼らの話", + }, + "tags": [ + "鬼滅の夢", + "女主人公", + "煉獄杏寿郎", + "涙腺崩壊", + "なにこれすごい", + "来世で幸せになって欲しい", + "キメ学世界線できっと幸せになってる!!", + "あなたが神か!!", + "キメ学編を·····", + "鬼滅の夢小説10000users入り", + ], + "text_length": 9569, + "title": "本当は、一目惚れだった", + "total_bookmarks": range(17900, 20000), + "total_comments": range(200, 400), + "total_view": range(158000, 300000), + "user": { + "account": "46_maru", + "id": 888268, + }, + "visible": True, + "x_restrict": 0, + }, + { + "#url": "https://www.pixiv.net/novel/show.php?id=16422450", + "#comment": "embeds // covers (#5373)", + "#category": ("", "pixiv", "novel"), + "#class": pixiv.PixivNovelExtractor, + "#options": { + "embeds": True, + "covers": True, + }, + "#count": 4, + }, + { + "#url": "https://www.pixiv.net/novel/show.php?id=12101012", + "#comment": "full series", + "#category": ("", "pixiv", "novel"), + "#class": pixiv.PixivNovelExtractor, + "#options": {"full-series": True}, + "#count": 2, + }, + { + "#url": "https://www.pixiv.net/n/19612040", + "#comment": "short URL", + "#category": ("", "pixiv", "novel"), + "#class": pixiv.PixivNovelExtractor, + }, + { + "#url": "https://www.pixiv.net/en/users/77055466/novels", + "#category": ("", "pixiv", "novel-user"), + "#class": pixiv.PixivNovelUserExtractor, + "#pattern": "^text:", + "#range": "1-5", + "#count": 5, + }, + { + "#url": "https://www.pixiv.net/novel/series/1479656", + "#category": ("", "pixiv", "novel-series"), + "#class": pixiv.PixivNovelSeriesExtractor, + "#count": 2, + "#sha1_content": "243ce593333bbfe26e255e3372d9c9d8cea22d5b", + }, + { + "#url": "https://www.pixiv.net/en/users/77055466/bookmarks/novels", + "#category": ("", "pixiv", "novel-bookmark"), + "#class": pixiv.PixivNovelBookmarkExtractor, + "#count": 1, + "#sha1_content": "7194e8faa876b2b536f185ee271a2b6e46c69089", + }, + { + "#url": "https://www.pixiv.net/en/users/11/bookmarks/novels/TAG?rest=hide", + "#category": ("", "pixiv", "novel-bookmark"), + "#class": pixiv.PixivNovelBookmarkExtractor, + }, + { + "#url": "https://sketch.pixiv.net/@nicoby", + "#category": ("", "pixiv", "sketch"), + "#class": pixiv.PixivSketchExtractor, + "#pattern": r"https://img\-sketch\.pixiv\.net/uploads/medium/file/\d+/\d+\.(jpg|png)", + "#count": ">= 35", + }, ) diff --git a/test/results/pixnet.py b/test/results/pixnet.py index 9c086526e..ad1fd0ca5 100644 --- a/test/results/pixnet.py +++ b/test/results/pixnet.py @@ -1,86 +1,73 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import pixnet - __tests__ = ( -{ - "#url" : "https://albertayu773.pixnet.net/album/photo/159443828", - "#category": ("", "pixnet", "image"), - "#class" : pixnet.PixnetImageExtractor, - "#sha1_url" : "156564c422138914c9fa5b42191677b45c414af4", - "#sha1_metadata": "19971bcd056dfef5593f4328a723a9602be0f087", - "#sha1_content" : "0e097bdf49e76dd9b9d57a016b08b16fa6a33280", -}, - -{ - "#url" : "https://albertayu773.pixnet.net/album/set/15078995", - "#category": ("", "pixnet", "set"), - "#class" : pixnet.PixnetSetExtractor, - "#sha1_url" : "6535712801af47af51110542f4938a7cef44557f", - "#sha1_metadata": "bf25d59e5b0959cb1f53e7fd2e2a25f2f67e5925", -}, - -{ - "#url" : "https://anrine910070.pixnet.net/album/set/5917493", - "#category": ("", "pixnet", "set"), - "#class" : pixnet.PixnetSetExtractor, - "#sha1_url" : "b3eb6431aea0bcf5003432a4a0f3a3232084fc13", - "#sha1_metadata": "bf7004faa1cea18cf9bd856f0955a69be51b1ec6", -}, - -{ - "#url" : "https://sky92100.pixnet.net/album/set/17492544", - "#comment" : "password-protected", - "#category": ("", "pixnet", "set"), - "#class" : pixnet.PixnetSetExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://albertayu773.pixnet.net/album/folder/1405768", - "#category": ("", "pixnet", "folder"), - "#class" : pixnet.PixnetFolderExtractor, - "#pattern" : pixnet.PixnetSetExtractor.pattern, - "#count" : ">= 15", -}, - -{ - "#url" : "https://albertayu773.pixnet.net/", - "#category": ("", "pixnet", "user"), - "#class" : pixnet.PixnetUserExtractor, -}, - -{ - "#url" : "https://albertayu773.pixnet.net/blog", - "#category": ("", "pixnet", "user"), - "#class" : pixnet.PixnetUserExtractor, -}, - -{ - "#url" : "https://albertayu773.pixnet.net/album", - "#category": ("", "pixnet", "user"), - "#class" : pixnet.PixnetUserExtractor, -}, - -{ - "#url" : "https://albertayu773.pixnet.net/album/list", - "#category": ("", "pixnet", "user"), - "#class" : pixnet.PixnetUserExtractor, - "#pattern" : pixnet.PixnetFolderExtractor.pattern, - "#count" : ">= 30", -}, - -{ - "#url" : "https://anrine910070.pixnet.net/album/list", - "#category": ("", "pixnet", "user"), - "#class" : pixnet.PixnetUserExtractor, - "#pattern" : pixnet.PixnetSetExtractor.pattern, - "#count" : ">= 14", -}, - + { + "#url": "https://albertayu773.pixnet.net/album/photo/159443828", + "#category": ("", "pixnet", "image"), + "#class": pixnet.PixnetImageExtractor, + "#sha1_url": "156564c422138914c9fa5b42191677b45c414af4", + "#sha1_metadata": "19971bcd056dfef5593f4328a723a9602be0f087", + "#sha1_content": "0e097bdf49e76dd9b9d57a016b08b16fa6a33280", + }, + { + "#url": "https://albertayu773.pixnet.net/album/set/15078995", + "#category": ("", "pixnet", "set"), + "#class": pixnet.PixnetSetExtractor, + "#sha1_url": "6535712801af47af51110542f4938a7cef44557f", + "#sha1_metadata": "bf25d59e5b0959cb1f53e7fd2e2a25f2f67e5925", + }, + { + "#url": "https://anrine910070.pixnet.net/album/set/5917493", + "#category": ("", "pixnet", "set"), + "#class": pixnet.PixnetSetExtractor, + "#sha1_url": "b3eb6431aea0bcf5003432a4a0f3a3232084fc13", + "#sha1_metadata": "bf7004faa1cea18cf9bd856f0955a69be51b1ec6", + }, + { + "#url": "https://sky92100.pixnet.net/album/set/17492544", + "#comment": "password-protected", + "#category": ("", "pixnet", "set"), + "#class": pixnet.PixnetSetExtractor, + "#count": 0, + }, + { + "#url": "https://albertayu773.pixnet.net/album/folder/1405768", + "#category": ("", "pixnet", "folder"), + "#class": pixnet.PixnetFolderExtractor, + "#pattern": pixnet.PixnetSetExtractor.pattern, + "#count": ">= 15", + }, + { + "#url": "https://albertayu773.pixnet.net/", + "#category": ("", "pixnet", "user"), + "#class": pixnet.PixnetUserExtractor, + }, + { + "#url": "https://albertayu773.pixnet.net/blog", + "#category": ("", "pixnet", "user"), + "#class": pixnet.PixnetUserExtractor, + }, + { + "#url": "https://albertayu773.pixnet.net/album", + "#category": ("", "pixnet", "user"), + "#class": pixnet.PixnetUserExtractor, + }, + { + "#url": "https://albertayu773.pixnet.net/album/list", + "#category": ("", "pixnet", "user"), + "#class": pixnet.PixnetUserExtractor, + "#pattern": pixnet.PixnetFolderExtractor.pattern, + "#count": ">= 30", + }, + { + "#url": "https://anrine910070.pixnet.net/album/list", + "#category": ("", "pixnet", "user"), + "#class": pixnet.PixnetUserExtractor, + "#pattern": pixnet.PixnetSetExtractor.pattern, + "#count": ">= 14", + }, ) diff --git a/test/results/plurk.py b/test/results/plurk.py index 11600baed..23a628a9f 100644 --- a/test/results/plurk.py +++ b/test/results/plurk.py @@ -1,35 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import plurk - __tests__ = ( -{ - "#url" : "https://www.plurk.com/plurkapi", - "#category": ("", "plurk", "timeline"), - "#class" : plurk.PlurkTimelineExtractor, - "#pattern" : "https?://.+", - "#count" : ">= 23", -}, - -{ - "#url" : "https://www.plurk.com/p/i701j1", - "#category": ("", "plurk", "post"), - "#class" : plurk.PlurkPostExtractor, - "#count" : 3, - "#sha1_url": "2115f208564591b8748525c2807a84596aaaaa5f", -}, - -{ - "#url" : "https://www.plurk.com/p/i701j1", - "#category": ("", "plurk", "post"), - "#class" : plurk.PlurkPostExtractor, - "#options" : {"comments": True}, - "#count" : ">= 210", -}, - + { + "#url": "https://www.plurk.com/plurkapi", + "#category": ("", "plurk", "timeline"), + "#class": plurk.PlurkTimelineExtractor, + "#pattern": "https?://.+", + "#count": ">= 23", + }, + { + "#url": "https://www.plurk.com/p/i701j1", + "#category": ("", "plurk", "post"), + "#class": plurk.PlurkPostExtractor, + "#count": 3, + "#sha1_url": "2115f208564591b8748525c2807a84596aaaaa5f", + }, + { + "#url": "https://www.plurk.com/p/i701j1", + "#category": ("", "plurk", "post"), + "#class": plurk.PlurkPostExtractor, + "#options": {"comments": True}, + "#count": ">= 210", + }, ) diff --git a/test/results/poipiku.py b/test/results/poipiku.py index dd9e744d4..114c46c39 100644 --- a/test/results/poipiku.py +++ b/test/results/poipiku.py @@ -1,76 +1,65 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import poipiku - __tests__ = ( -{ - "#url" : "https://poipiku.com/25049/", - "#category": ("", "poipiku", "user"), - "#class" : poipiku.PoipikuUserExtractor, - "#pattern" : r"https://img-org\.poipiku\.com/user_img\d+/000025049/\d+_\w+\.(jpe?g|png)$", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://poipiku.com/IllustListPcV.jsp?PG=1&ID=25049&KWD=", - "#category": ("", "poipiku", "user"), - "#class" : poipiku.PoipikuUserExtractor, -}, - -{ - "#url" : "https://poipiku.com/25049/5864576.html", - "#category": ("", "poipiku", "post"), - "#class" : poipiku.PoipikuPostExtractor, - "#pattern" : r"https://img-org\.poipiku\.com/user_img\d+/000025049/005864576_EWN1Y65gQ\.png$", - - "count" : 1, - "description" : "", - "extension" : "png", - "filename" : "005864576_EWN1Y65gQ", - "num" : 1, - "post_category": "DOODLE", - "post_id" : "5864576", - "user_id" : "25049", - "user_name" : "ユキウサギ", -}, - -{ - "#url" : "https://poipiku.com/2166245/6411749.html", - "#category": ("", "poipiku", "post"), - "#class" : poipiku.PoipikuPostExtractor, - "#pattern" : r"https://img-org\.poipiku\.com/user_img\d+/002166245/006411749_\w+\.jpeg$", - "#count" : 4, - - "count" : 4, - "description" : "絵茶の産物ネタバレあるやつ", - "num" : int, - "post_category": "SPOILER", - "post_id" : "6411749", - "user_id" : "2166245", - "user_name" : "wadahito", -}, - -{ - "#url" : "https://poipiku.com/3572553/5776587.html", - "#comment" : "different warning button style", - "#category": ("", "poipiku", "post"), - "#class" : poipiku.PoipikuPostExtractor, - "#pattern" : r"https://img-org\.poipiku.com/user_img\d+/003572553/005776587_(\d+_)?\w+\.jpeg$", - "#count" : 3, - - "count" : 3, - "description" : "ORANGE OASISボスネタバレ
          曲も大好き
          2枚目以降はほとんど見えなかった1枚目背景のヒエログリフ小ネタです𓀀", - "num" : int, - "post_category": "SPOILER", - "post_id" : "5776587", - "user_id" : "3572553", - "user_name" : "nagakun", -}, - + { + "#url": "https://poipiku.com/25049/", + "#category": ("", "poipiku", "user"), + "#class": poipiku.PoipikuUserExtractor, + "#pattern": r"https://img-org\.poipiku\.com/user_img\d+/000025049/\d+_\w+\.(jpe?g|png)$", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://poipiku.com/IllustListPcV.jsp?PG=1&ID=25049&KWD=", + "#category": ("", "poipiku", "user"), + "#class": poipiku.PoipikuUserExtractor, + }, + { + "#url": "https://poipiku.com/25049/5864576.html", + "#category": ("", "poipiku", "post"), + "#class": poipiku.PoipikuPostExtractor, + "#pattern": r"https://img-org\.poipiku\.com/user_img\d+/000025049/005864576_EWN1Y65gQ\.png$", + "count": 1, + "description": "", + "extension": "png", + "filename": "005864576_EWN1Y65gQ", + "num": 1, + "post_category": "DOODLE", + "post_id": "5864576", + "user_id": "25049", + "user_name": "ユキウサギ", + }, + { + "#url": "https://poipiku.com/2166245/6411749.html", + "#category": ("", "poipiku", "post"), + "#class": poipiku.PoipikuPostExtractor, + "#pattern": r"https://img-org\.poipiku\.com/user_img\d+/002166245/006411749_\w+\.jpeg$", + "#count": 4, + "count": 4, + "description": "絵茶の産物ネタバレあるやつ", + "num": int, + "post_category": "SPOILER", + "post_id": "6411749", + "user_id": "2166245", + "user_name": "wadahito", + }, + { + "#url": "https://poipiku.com/3572553/5776587.html", + "#comment": "different warning button style", + "#category": ("", "poipiku", "post"), + "#class": poipiku.PoipikuPostExtractor, + "#pattern": r"https://img-org\.poipiku.com/user_img\d+/003572553/005776587_(\d+_)?\w+\.jpeg$", + "#count": 3, + "count": 3, + "description": "ORANGE OASISボスネタバレ
          曲も大好き
          2枚目以降はほとんど見えなかった1枚目背景のヒエログリフ小ネタです𓀀", + "num": int, + "post_category": "SPOILER", + "post_id": "5776587", + "user_id": "3572553", + "user_name": "nagakun", + }, ) diff --git a/test/results/ponybooru.py b/test/results/ponybooru.py index 6955bb490..241985685 100644 --- a/test/results/ponybooru.py +++ b/test/results/ponybooru.py @@ -1,39 +1,32 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import philomena - __tests__ = ( -{ - "#url" : "https://ponybooru.org/images/1", - "#category": ("philomena", "ponybooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, - "#sha1_content": "bca26f58fafd791fe07adcd2a28efd7751824605", -}, - -{ - "#url" : "https://www.ponybooru.org/images/1", - "#category": ("philomena", "ponybooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, -}, - -{ - "#url" : "https://ponybooru.org/search?q=cute", - "#category": ("philomena", "ponybooru", "search"), - "#class" : philomena.PhilomenaSearchExtractor, - "#range" : "40-60", - "#count" : 21, -}, - -{ - "#url" : "https://ponybooru.org/galleries/27", - "#category": ("philomena", "ponybooru", "gallery"), - "#class" : philomena.PhilomenaGalleryExtractor, - "#count" : ">= 24", -}, - + { + "#url": "https://ponybooru.org/images/1", + "#category": ("philomena", "ponybooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + "#sha1_content": "bca26f58fafd791fe07adcd2a28efd7751824605", + }, + { + "#url": "https://www.ponybooru.org/images/1", + "#category": ("philomena", "ponybooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + }, + { + "#url": "https://ponybooru.org/search?q=cute", + "#category": ("philomena", "ponybooru", "search"), + "#class": philomena.PhilomenaSearchExtractor, + "#range": "40-60", + "#count": 21, + }, + { + "#url": "https://ponybooru.org/galleries/27", + "#category": ("philomena", "ponybooru", "gallery"), + "#class": philomena.PhilomenaGalleryExtractor, + "#count": ">= 24", + }, ) diff --git a/test/results/poringa.py b/test/results/poringa.py index 16c27e42b..db41098d5 100644 --- a/test/results/poringa.py +++ b/test/results/poringa.py @@ -1,54 +1,45 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import poringa - __tests__ = ( -{ - "#url" : "http://www.poringa.net/posts/imagenes/3051081/Turrita-alto-ojete.html", - "#category": ("", "poringa", "post"), - "#class" : poringa.PoringaPostExtractor, - "#count" : 26, - - "count" : 26, - "num" : range(1, 26), - "post_id" : "3051081", - "title" : "turrita alto ojete...", - "user" : "vipower1top", -}, - -{ - "#url" : "http://www.poringa.net/posts/imagenes/3095554/Otra-culona-de-instagram.html", - "#category": ("", "poringa", "post"), - "#class" : poringa.PoringaPostExtractor, - "#count" : 15, - - "count" : 15, - "num" : range(1, 15), - "post_id" : "3095554", - "title" : "Otra culona de instagram", - "user" : "Expectro007", -}, - -{ - "#url" : "http://www.poringa.net/Expectro007", - "#category": ("", "poringa", "user"), - "#class" : poringa.PoringaUserExtractor, - "#pattern" : r"https?://img-\d+\.poringa\.net/poringa/img/././././././Expectro007/\w{3}\.(jpg|png|gif)", - "#count" : range(500, 600), -}, - -{ - "#url" : "http://www.poringa.net/buscar/?&q=yuslopez", - "#category": ("", "poringa", "search"), - "#class" : poringa.PoringaSearchExtractor, - "#pattern" : r"https?://img-\d+\.poringa\.net/poringa/img/././././././\w+/\w{3}\.(jpg|png|gif)", - "#range" : "1-50", - "#count" : 50, -}, - + { + "#url": "http://www.poringa.net/posts/imagenes/3051081/Turrita-alto-ojete.html", + "#category": ("", "poringa", "post"), + "#class": poringa.PoringaPostExtractor, + "#count": 26, + "count": 26, + "num": range(1, 26), + "post_id": "3051081", + "title": "turrita alto ojete...", + "user": "vipower1top", + }, + { + "#url": "http://www.poringa.net/posts/imagenes/3095554/Otra-culona-de-instagram.html", + "#category": ("", "poringa", "post"), + "#class": poringa.PoringaPostExtractor, + "#count": 15, + "count": 15, + "num": range(1, 15), + "post_id": "3095554", + "title": "Otra culona de instagram", + "user": "Expectro007", + }, + { + "#url": "http://www.poringa.net/Expectro007", + "#category": ("", "poringa", "user"), + "#class": poringa.PoringaUserExtractor, + "#pattern": r"https?://img-\d+\.poringa\.net/poringa/img/././././././Expectro007/\w{3}\.(jpg|png|gif)", + "#count": range(500, 600), + }, + { + "#url": "http://www.poringa.net/buscar/?&q=yuslopez", + "#category": ("", "poringa", "search"), + "#class": poringa.PoringaSearchExtractor, + "#pattern": r"https?://img-\d+\.poringa\.net/poringa/img/././././././\w+/\w{3}\.(jpg|png|gif)", + "#range": "1-50", + "#count": 50, + }, ) diff --git a/test/results/pornhub.py b/test/results/pornhub.py index c60dc0e4e..2b1d527ed 100644 --- a/test/results/pornhub.py +++ b/test/results/pornhub.py @@ -1,137 +1,119 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import pornhub from gallery_dl import exception - +from gallery_dl.extractor import pornhub __tests__ = ( -{ - "#url" : "https://www.pornhub.com/album/19289801", - "#category": ("", "pornhub", "gallery"), - "#class" : pornhub.PornhubGalleryExtractor, - "#pattern" : r"https://\w+.phncdn.com/pics/albums/\d+/\d+/\d+/\d+/", - "#count" : ">= 300", - - "id" : int, - "num" : int, - "score" : int, - "views" : int, - "caption": str, - "user" : "Danika Mori", - "gallery": { - "id" : 19289801, + { + "#url": "https://www.pornhub.com/album/19289801", + "#category": ("", "pornhub", "gallery"), + "#class": pornhub.PornhubGalleryExtractor, + "#pattern": r"https://\w+.phncdn.com/pics/albums/\d+/\d+/\d+/\d+/", + "#count": ">= 300", + "id": int, + "num": int, "score": int, "views": int, - "tags" : list, - "title": "Danika Mori Best Moments", + "caption": str, + "user": "Danika Mori", + "gallery": { + "id": 19289801, + "score": int, + "views": int, + "tags": list, + "title": "Danika Mori Best Moments", + }, + }, + { + "#url": "https://www.pornhub.com/album/69606532", + "#comment": "KeyError due to missing image entry (#6299)", + "#class": pornhub.PornhubGalleryExtractor, + "#count": 6, + }, + { + "#url": "https://www.pornhub.com/album/69040172", + "#category": ("", "pornhub", "gallery"), + "#class": pornhub.PornhubGalleryExtractor, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://www.pornhub.com/gif/43726891", + "#category": ("", "pornhub", "gif"), + "#class": pornhub.PornhubGifExtractor, + "#pattern": r"https://\w+\.phncdn\.com/pics/gifs/043/726/891/43726891a\.webm", + "date": "dt:2023-04-20 00:00:00", + "extension": "webm", + "filename": "43726891a", + "id": "43726891", + "tags": [ + "sloppy deepthroat", + "perfect body", + "petite brunette", + "mouth fuck", + "big dick", + "natural big tits", + "deepthroat swallow", + "amateur couple", + "homemade", + "girls wanking boys", + "hardcore sex", + "babes 18 year", + ], + "timestamp": "5:07", + "title": "Intense sloppy blowjob of Danika Mori", + "url": "https://el.phncdn.com/pics/gifs/043/726/891/43726891a.webm", + "user": "Danika Mori", + "viewkey": "64367c8c78a4a", + }, + { + "#url": "https://www.pornhub.com/pornstar/danika-mori", + "#category": ("", "pornhub", "user"), + "#class": pornhub.PornhubUserExtractor, + }, + { + "#url": "https://www.pornhub.com/pornstar/danika-mori/photos", + "#category": ("", "pornhub", "photos"), + "#class": pornhub.PornhubPhotosExtractor, + "#pattern": pornhub.PornhubGalleryExtractor.pattern, + "#count": ">= 6", + }, + { + "#url": "https://www.pornhub.com/users/flyings0l0/photos/public", + "#category": ("", "pornhub", "photos"), + "#class": pornhub.PornhubPhotosExtractor, + }, + { + "#url": "https://www.pornhub.com/users/flyings0l0/photos/private", + "#category": ("", "pornhub", "photos"), + "#class": pornhub.PornhubPhotosExtractor, + }, + { + "#url": "https://www.pornhub.com/users/flyings0l0/photos/favorites", + "#category": ("", "pornhub", "photos"), + "#class": pornhub.PornhubPhotosExtractor, + }, + { + "#url": "https://www.pornhub.com/model/bossgirl/photos", + "#category": ("", "pornhub", "photos"), + "#class": pornhub.PornhubPhotosExtractor, + }, + { + "#url": "https://www.pornhub.com/pornstar/danika-mori/gifs", + "#category": ("", "pornhub", "gifs"), + "#class": pornhub.PornhubGifsExtractor, + "#pattern": pornhub.PornhubGifExtractor.pattern, + "#count": ">= 30", + }, + { + "#url": "https://www.pornhub.com/users/flyings0l0/gifs", + "#category": ("", "pornhub", "gifs"), + "#class": pornhub.PornhubGifsExtractor, + }, + { + "#url": "https://www.pornhub.com/model/bossgirl/gifs/video", + "#category": ("", "pornhub", "gifs"), + "#class": pornhub.PornhubGifsExtractor, }, -}, - -{ - "#url" : "https://www.pornhub.com/album/69606532", - "#comment" : "KeyError due to missing image entry (#6299)", - "#class" : pornhub.PornhubGalleryExtractor, - "#count" : 6, -}, - -{ - "#url" : "https://www.pornhub.com/album/69040172", - "#category": ("", "pornhub", "gallery"), - "#class" : pornhub.PornhubGalleryExtractor, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://www.pornhub.com/gif/43726891", - "#category": ("", "pornhub", "gif"), - "#class" : pornhub.PornhubGifExtractor, - "#pattern" : r"https://\w+\.phncdn\.com/pics/gifs/043/726/891/43726891a\.webm", - - "date" : "dt:2023-04-20 00:00:00", - "extension": "webm", - "filename" : "43726891a", - "id" : "43726891", - "tags" : [ - "sloppy deepthroat", - "perfect body", - "petite brunette", - "mouth fuck", - "big dick", - "natural big tits", - "deepthroat swallow", - "amateur couple", - "homemade", - "girls wanking boys", - "hardcore sex", - "babes 18 year", - ], - "timestamp": "5:07", - "title" : "Intense sloppy blowjob of Danika Mori", - "url" : "https://el.phncdn.com/pics/gifs/043/726/891/43726891a.webm", - "user" : "Danika Mori", - "viewkey" : "64367c8c78a4a", -}, - -{ - "#url" : "https://www.pornhub.com/pornstar/danika-mori", - "#category": ("", "pornhub", "user"), - "#class" : pornhub.PornhubUserExtractor, -}, - -{ - "#url" : "https://www.pornhub.com/pornstar/danika-mori/photos", - "#category": ("", "pornhub", "photos"), - "#class" : pornhub.PornhubPhotosExtractor, - "#pattern" : pornhub.PornhubGalleryExtractor.pattern, - "#count" : ">= 6", -}, - -{ - "#url" : "https://www.pornhub.com/users/flyings0l0/photos/public", - "#category": ("", "pornhub", "photos"), - "#class" : pornhub.PornhubPhotosExtractor, -}, - -{ - "#url" : "https://www.pornhub.com/users/flyings0l0/photos/private", - "#category": ("", "pornhub", "photos"), - "#class" : pornhub.PornhubPhotosExtractor, -}, - -{ - "#url" : "https://www.pornhub.com/users/flyings0l0/photos/favorites", - "#category": ("", "pornhub", "photos"), - "#class" : pornhub.PornhubPhotosExtractor, -}, - -{ - "#url" : "https://www.pornhub.com/model/bossgirl/photos", - "#category": ("", "pornhub", "photos"), - "#class" : pornhub.PornhubPhotosExtractor, -}, - -{ - "#url" : "https://www.pornhub.com/pornstar/danika-mori/gifs", - "#category": ("", "pornhub", "gifs"), - "#class" : pornhub.PornhubGifsExtractor, - "#pattern" : pornhub.PornhubGifExtractor.pattern, - "#count" : ">= 30", -}, - -{ - "#url" : "https://www.pornhub.com/users/flyings0l0/gifs", - "#category": ("", "pornhub", "gifs"), - "#class" : pornhub.PornhubGifsExtractor, -}, - -{ - "#url" : "https://www.pornhub.com/model/bossgirl/gifs/video", - "#category": ("", "pornhub", "gifs"), - "#class" : pornhub.PornhubGifsExtractor, -}, - ) diff --git a/test/results/pornpics.py b/test/results/pornpics.py index 6caf5dd11..644d83369 100644 --- a/test/results/pornpics.py +++ b/test/results/pornpics.py @@ -1,145 +1,128 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import pornpics - __tests__ = ( -{ - "#url" : "https://www.pornpics.com/galleries/british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest-62610699/", - "#category": ("", "pornpics", "gallery"), - "#class" : pornpics.PornpicsGalleryExtractor, - "#pattern" : r"https://cdni\.pornpics\.com/1280/7/160/62610699/62610699_\d+_[0-9a-f]{4}\.jpg", - - "categories": [ - "Outdoor", - "MILF", - "Boots", - "Amateur", - "Sexy", - ], - "channel" : ["FTV MILFs"], - "count" : 17, - "gallery_id": 62610699, - "models" : ["Danielle"], - "num" : int, - "slug" : "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", - "tags" : [ - "MILF Outdoor", - "Amateur MILF", - "Nature", - "Amateur Outdoor", - "First Time", - "Sexy MILF", - ], - "title" : "British beauty Danielle flashes hot breasts, ass and snatch in the forest", - "views" : int, -}, - -{ - "#url" : "https://pornpics.com/es/galleries/62610699", - "#category": ("", "pornpics", "gallery"), - "#class" : pornpics.PornpicsGalleryExtractor, - - "slug": "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", -}, - -{ - "#url" : "https://www.pornpics.com/galleries/four-american-hotties-in-swimsuit-showing-off-their-sexy-booty-and-bare-legs-59500405/", - "#comment" : "more than one 'channel' (#5195)", - "#category": ("", "pornpics", "gallery"), - "#class" : pornpics.PornpicsGalleryExtractor, - - "count" : 16, - "num" : range(1, 16), - "gallery_id": 59500405, - "slug" : "four-american-hotties-in-swimsuit-showing-off-their-sexy-booty-and-bare-legs", - "title" : "Four American hotties in swimsuit showing off their sexy booty and bare legs", - "views" : range(50000, 100000), - "models" : [ - "Kayla West", - "Layla Price", - "Marley Blaze", - "Mena Mason", - ], - "categories": [ - "Outdoor", - "Asian", - "Pornstar", - "Brunette", - "Blonde", - ], - "channel" : [ - "Adult Time", - "Fame Digital", - ], - "tags" : [ - "Nature", - "Asian Outdoor", - ], -}, - -{ - "#url" : "https://www.pornpics.com/tags/summer-dress/", - "#category": ("", "pornpics", "tag"), - "#class" : pornpics.PornpicsTagExtractor, - "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://pornpics.com/fr/tags/summer-dress", - "#category": ("", "pornpics", "tag"), - "#class" : pornpics.PornpicsTagExtractor, -}, - -{ - "#url" : "https://www.pornpics.com/?q=nature", - "#category": ("", "pornpics", "search"), - "#class" : pornpics.PornpicsSearchExtractor, - "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://www.pornpics.com/channels/femjoy/", - "#category": ("", "pornpics", "search"), - "#class" : pornpics.PornpicsSearchExtractor, - "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://www.pornpics.com/pornstars/emma-brown/", - "#category": ("", "pornpics", "search"), - "#class" : pornpics.PornpicsSearchExtractor, - "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://pornpics.com/jp/?q=nature", - "#category": ("", "pornpics", "search"), - "#class" : pornpics.PornpicsSearchExtractor, -}, - -{ - "#url" : "https://pornpics.com/it/channels/femjoy", - "#category": ("", "pornpics", "search"), - "#class" : pornpics.PornpicsSearchExtractor, -}, - -{ - "#url" : "https://pornpics.com/pt/pornstars/emma-brown", - "#category": ("", "pornpics", "search"), - "#class" : pornpics.PornpicsSearchExtractor, -}, - + { + "#url": "https://www.pornpics.com/galleries/british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest-62610699/", + "#category": ("", "pornpics", "gallery"), + "#class": pornpics.PornpicsGalleryExtractor, + "#pattern": r"https://cdni\.pornpics\.com/1280/7/160/62610699/62610699_\d+_[0-9a-f]{4}\.jpg", + "categories": [ + "Outdoor", + "MILF", + "Boots", + "Amateur", + "Sexy", + ], + "channel": ["FTV MILFs"], + "count": 17, + "gallery_id": 62610699, + "models": ["Danielle"], + "num": int, + "slug": "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", + "tags": [ + "MILF Outdoor", + "Amateur MILF", + "Nature", + "Amateur Outdoor", + "First Time", + "Sexy MILF", + ], + "title": "British beauty Danielle flashes hot breasts, ass and snatch in the forest", + "views": int, + }, + { + "#url": "https://pornpics.com/es/galleries/62610699", + "#category": ("", "pornpics", "gallery"), + "#class": pornpics.PornpicsGalleryExtractor, + "slug": "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", + }, + { + "#url": "https://www.pornpics.com/galleries/four-american-hotties-in-swimsuit-showing-off-their-sexy-booty-and-bare-legs-59500405/", + "#comment": "more than one 'channel' (#5195)", + "#category": ("", "pornpics", "gallery"), + "#class": pornpics.PornpicsGalleryExtractor, + "count": 16, + "num": range(1, 16), + "gallery_id": 59500405, + "slug": "four-american-hotties-in-swimsuit-showing-off-their-sexy-booty-and-bare-legs", + "title": "Four American hotties in swimsuit showing off their sexy booty and bare legs", + "views": range(50000, 100000), + "models": [ + "Kayla West", + "Layla Price", + "Marley Blaze", + "Mena Mason", + ], + "categories": [ + "Outdoor", + "Asian", + "Pornstar", + "Brunette", + "Blonde", + ], + "channel": [ + "Adult Time", + "Fame Digital", + ], + "tags": [ + "Nature", + "Asian Outdoor", + ], + }, + { + "#url": "https://www.pornpics.com/tags/summer-dress/", + "#category": ("", "pornpics", "tag"), + "#class": pornpics.PornpicsTagExtractor, + "#pattern": pornpics.PornpicsGalleryExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://pornpics.com/fr/tags/summer-dress", + "#category": ("", "pornpics", "tag"), + "#class": pornpics.PornpicsTagExtractor, + }, + { + "#url": "https://www.pornpics.com/?q=nature", + "#category": ("", "pornpics", "search"), + "#class": pornpics.PornpicsSearchExtractor, + "#pattern": pornpics.PornpicsGalleryExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://www.pornpics.com/channels/femjoy/", + "#category": ("", "pornpics", "search"), + "#class": pornpics.PornpicsSearchExtractor, + "#pattern": pornpics.PornpicsGalleryExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://www.pornpics.com/pornstars/emma-brown/", + "#category": ("", "pornpics", "search"), + "#class": pornpics.PornpicsSearchExtractor, + "#pattern": pornpics.PornpicsGalleryExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://pornpics.com/jp/?q=nature", + "#category": ("", "pornpics", "search"), + "#class": pornpics.PornpicsSearchExtractor, + }, + { + "#url": "https://pornpics.com/it/channels/femjoy", + "#category": ("", "pornpics", "search"), + "#class": pornpics.PornpicsSearchExtractor, + }, + { + "#url": "https://pornpics.com/pt/pornstars/emma-brown", + "#category": ("", "pornpics", "search"), + "#class": pornpics.PornpicsSearchExtractor, + }, ) diff --git a/test/results/pornreactor.py b/test/results/pornreactor.py index 0062d19aa..e4b37d2e8 100644 --- a/test/results/pornreactor.py +++ b/test/results/pornreactor.py @@ -1,66 +1,55 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import reactor - __tests__ = ( -{ - "#url" : "http://pornreactor.cc/tag/RiceGnat", - "#category": ("reactor", "pornreactor", "tag"), - "#class" : reactor.ReactorTagExtractor, - "#range" : "1-25", - "#count" : ">= 25", -}, - -{ - "#url" : "http://fapreactor.com/tag/RiceGnat", - "#category": ("reactor", "pornreactor", "tag"), - "#class" : reactor.ReactorTagExtractor, -}, - -{ - "#url" : "http://pornreactor.cc/search?q=ecchi+hentai", - "#category": ("reactor", "pornreactor", "search"), - "#class" : reactor.ReactorSearchExtractor, -}, - -{ - "#url" : "http://fapreactor.com/search/ecchi+hentai", - "#category": ("reactor", "pornreactor", "search"), - "#class" : reactor.ReactorSearchExtractor, -}, - -{ - "#url" : "http://pornreactor.cc/user/Disillusion", - "#category": ("reactor", "pornreactor", "user"), - "#class" : reactor.ReactorUserExtractor, - "#range" : "1-25", - "#count" : ">= 20", -}, - -{ - "#url" : "http://fapreactor.com/user/Disillusion", - "#category": ("reactor", "pornreactor", "user"), - "#class" : reactor.ReactorUserExtractor, -}, - -{ - "#url" : "http://pornreactor.cc/post/863166", - "#category": ("reactor", "pornreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#sha1_url" : "a09fb0577489e1f9564c25d0ad576f81b19c2ef3", - "#sha1_content": "ec6b0568bfb1803648744077da082d14de844340", -}, - -{ - "#url" : "http://fapreactor.com/post/863166", - "#category": ("reactor", "pornreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#sha1_url": "2a956ce0c90e8bc47b4392db4fa25ad1342f3e54", -}, - + { + "#url": "http://pornreactor.cc/tag/RiceGnat", + "#category": ("reactor", "pornreactor", "tag"), + "#class": reactor.ReactorTagExtractor, + "#range": "1-25", + "#count": ">= 25", + }, + { + "#url": "http://fapreactor.com/tag/RiceGnat", + "#category": ("reactor", "pornreactor", "tag"), + "#class": reactor.ReactorTagExtractor, + }, + { + "#url": "http://pornreactor.cc/search?q=ecchi+hentai", + "#category": ("reactor", "pornreactor", "search"), + "#class": reactor.ReactorSearchExtractor, + }, + { + "#url": "http://fapreactor.com/search/ecchi+hentai", + "#category": ("reactor", "pornreactor", "search"), + "#class": reactor.ReactorSearchExtractor, + }, + { + "#url": "http://pornreactor.cc/user/Disillusion", + "#category": ("reactor", "pornreactor", "user"), + "#class": reactor.ReactorUserExtractor, + "#range": "1-25", + "#count": ">= 20", + }, + { + "#url": "http://fapreactor.com/user/Disillusion", + "#category": ("reactor", "pornreactor", "user"), + "#class": reactor.ReactorUserExtractor, + }, + { + "#url": "http://pornreactor.cc/post/863166", + "#category": ("reactor", "pornreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#sha1_url": "a09fb0577489e1f9564c25d0ad576f81b19c2ef3", + "#sha1_content": "ec6b0568bfb1803648744077da082d14de844340", + }, + { + "#url": "http://fapreactor.com/post/863166", + "#category": ("reactor", "pornreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#sha1_url": "2a956ce0c90e8bc47b4392db4fa25ad1342f3e54", + }, ) diff --git a/test/results/postimg.py b/test/results/postimg.py index ba4b8a866..a045f7c4b 100644 --- a/test/results/postimg.py +++ b/test/results/postimg.py @@ -1,46 +1,38 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://postimages.org/Wtn2b3hC", - "#category": ("imagehost", "postimg", "image"), - "#class" : imagehosts.PostimgImageExtractor, -}, - -{ - "#url" : "https://www.postimages.org/Wtn2b3hC", - "#category": ("imagehost", "postimg", "image"), - "#class" : imagehosts.PostimgImageExtractor, -}, - -{ - "#url" : "https://pixxxels.cc/Wtn2b3hC", - "#category": ("imagehost", "postimg", "image"), - "#class" : imagehosts.PostimgImageExtractor, -}, - -{ - "#url" : "https://postimg.cc/Wtn2b3hC", - "#category": ("imagehost", "postimg", "image"), - "#class" : imagehosts.PostimgImageExtractor, - "#sha1_url" : "72f3c8b1d6c6601a20ad58f35635494b4891a99e", - "#sha1_metadata": "2d05808d04e4e83e33200db83521af06e3147a84", - "#sha1_content" : "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", -}, - -{ - "#url" : "https://postimg.cc/gallery/wxpDLgX", - "#category": ("imagehost", "postimg", "gallery"), - "#class" : imagehosts.PostimgGalleryExtractor, - "#pattern" : imagehosts.PostimgImageExtractor.pattern, - "#count" : 22, -}, - + { + "#url": "https://postimages.org/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class": imagehosts.PostimgImageExtractor, + }, + { + "#url": "https://www.postimages.org/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class": imagehosts.PostimgImageExtractor, + }, + { + "#url": "https://pixxxels.cc/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class": imagehosts.PostimgImageExtractor, + }, + { + "#url": "https://postimg.cc/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class": imagehosts.PostimgImageExtractor, + "#sha1_url": "72f3c8b1d6c6601a20ad58f35635494b4891a99e", + "#sha1_metadata": "2d05808d04e4e83e33200db83521af06e3147a84", + "#sha1_content": "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", + }, + { + "#url": "https://postimg.cc/gallery/wxpDLgX", + "#category": ("imagehost", "postimg", "gallery"), + "#class": imagehosts.PostimgGalleryExtractor, + "#pattern": imagehosts.PostimgImageExtractor.pattern, + "#count": 22, + }, ) diff --git a/test/results/raddle.py b/test/results/raddle.py index 6b3ccde82..7ee955a07 100644 --- a/test/results/raddle.py +++ b/test/results/raddle.py @@ -1,111 +1,96 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import postmill - __tests__ = ( -{ - "#url" : "https://raddle.me/", - "#category": ("postmill", "raddle", "home"), - "#class" : postmill.PostmillHomeExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://raddle.me/f/traa", - "#category": ("postmill", "raddle", "forum"), - "#class" : postmill.PostmillForumExtractor, - "#count" : 1, - "#pattern" : r"^https://raddle\.me/f/traa/156646/click-here-to-go-to-f-traaaaaaannnnnnnnnns$", -}, - -{ - "#url" : "https://raddle.me/user/Sam_the_enby/submissions", - "#category": ("postmill", "raddle", "usersubmissions"), - "#class" : postmill.PostmillUserSubmissionsExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://raddle.me/tag/Trans", - "#category": ("postmill", "raddle", "tag"), - "#class" : postmill.PostmillTagExtractor, -}, - -{ - "#url" : "https://raddle.me/search?q=tw", - "#category": ("postmill", "raddle", "search"), - "#class" : postmill.PostmillSearchExtractor, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://raddle.me/160845", - "#category": ("postmill", "raddle", "shorturl"), - "#class" : postmill.PostmillShortURLExtractor, - "#pattern" : r"^https://raddle\.me/f/egg_irl/160845/egg_irl$", -}, - -{ - "#url" : "https://raddle.me/f/NonBinary/179017/scattered-thoughts-would-appreciate-advice-immensely-tw", - "#comment" : "Text post", - "#category": ("postmill", "raddle", "post"), - "#class" : postmill.PostmillPostExtractor, - "#sha1_url" : "99277f815820810d9d7e219d455f818601858378", - "#sha1_content": "7a1159e1e45f2ce8e2c8b5959f6d66b042776f3b", - "#count" : 1, -}, - -{ - "#url" : "https://raddle.me/f/egg_irl/160845", - "#comment" : "Image post", - "#category": ("postmill", "raddle", "post"), - "#class" : postmill.PostmillPostExtractor, - "#sha1_content": "431e938082c2b59c44888a83cfc711cd1f0e910a", - "#urls" : "https://uploads-cdn.raddle.me/submission_images/30f4cf7d235d40c1daebf6dc2e58bef2a80bec2b5b2dab10f2021ea8e3f29e11.png", -}, - -{ - "#url" : "https://raddle.me/f/trans/177042/tw-vent-nsfw-suicide-i-lost-no-nut-november-tw-trauma", - "#comment" : "Image + text post (with text enabled)", - "#category": ("postmill", "raddle", "post"), - "#class" : postmill.PostmillPostExtractor, - "#options" : {"save-link-post-body": True}, - "#pattern" : r"^(text:[\s\S]+|https://(uploads-cdn\.)?raddle\.me/submission_images/[0-9a-f]+\.png)$", - "#count" : 2, -}, - -{ - "#url" : "https://raddle.me/f/videos/179541/raisins-and-sprite", - "#comment" : "Link post", - "#category": ("postmill", "raddle", "post"), - "#class" : postmill.PostmillPostExtractor, - "#urls" : "https://m.youtube.com/watch?v=RFJCA5zcZxI", - "#count" : 1, -}, - -{ - "#url" : "https://raddle.me/f/Anime/150698/neo-tokyo-1987-link-to-the-english-dub-version-last-link", - "#comment" : "Link + text post (with text disabled)", - "#category": ("postmill", "raddle", "post"), - "#class" : postmill.PostmillPostExtractor, - "#pattern" : r"^https://fantasyanime\.com/anime/neo-tokyo-dub$", - "#count" : 1, -}, - -{ - "#url" : "https://raddle.me/f/egg_irl/166855/4th-wall-breaking-please-let-this-be-a-flair-egg-irl", - "#comment" : "Post with multiple flairs", - "#category": ("postmill", "raddle", "post"), - "#class" : postmill.PostmillPostExtractor, - "flair" : ["Gender non-specific", "4th wall breaking"], -}, - + { + "#url": "https://raddle.me/", + "#category": ("postmill", "raddle", "home"), + "#class": postmill.PostmillHomeExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://raddle.me/f/traa", + "#category": ("postmill", "raddle", "forum"), + "#class": postmill.PostmillForumExtractor, + "#count": 1, + "#pattern": r"^https://raddle\.me/f/traa/156646/click-here-to-go-to-f-traaaaaaannnnnnnnnns$", + }, + { + "#url": "https://raddle.me/user/Sam_the_enby/submissions", + "#category": ("postmill", "raddle", "usersubmissions"), + "#class": postmill.PostmillUserSubmissionsExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://raddle.me/tag/Trans", + "#category": ("postmill", "raddle", "tag"), + "#class": postmill.PostmillTagExtractor, + }, + { + "#url": "https://raddle.me/search?q=tw", + "#category": ("postmill", "raddle", "search"), + "#class": postmill.PostmillSearchExtractor, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://raddle.me/160845", + "#category": ("postmill", "raddle", "shorturl"), + "#class": postmill.PostmillShortURLExtractor, + "#pattern": r"^https://raddle\.me/f/egg_irl/160845/egg_irl$", + }, + { + "#url": "https://raddle.me/f/NonBinary/179017/scattered-thoughts-would-appreciate-advice-immensely-tw", + "#comment": "Text post", + "#category": ("postmill", "raddle", "post"), + "#class": postmill.PostmillPostExtractor, + "#sha1_url": "99277f815820810d9d7e219d455f818601858378", + "#sha1_content": "7a1159e1e45f2ce8e2c8b5959f6d66b042776f3b", + "#count": 1, + }, + { + "#url": "https://raddle.me/f/egg_irl/160845", + "#comment": "Image post", + "#category": ("postmill", "raddle", "post"), + "#class": postmill.PostmillPostExtractor, + "#sha1_content": "431e938082c2b59c44888a83cfc711cd1f0e910a", + "#urls": "https://uploads-cdn.raddle.me/submission_images/30f4cf7d235d40c1daebf6dc2e58bef2a80bec2b5b2dab10f2021ea8e3f29e11.png", + }, + { + "#url": "https://raddle.me/f/trans/177042/tw-vent-nsfw-suicide-i-lost-no-nut-november-tw-trauma", + "#comment": "Image + text post (with text enabled)", + "#category": ("postmill", "raddle", "post"), + "#class": postmill.PostmillPostExtractor, + "#options": {"save-link-post-body": True}, + "#pattern": r"^(text:[\s\S]+|https://(uploads-cdn\.)?raddle\.me/submission_images/[0-9a-f]+\.png)$", + "#count": 2, + }, + { + "#url": "https://raddle.me/f/videos/179541/raisins-and-sprite", + "#comment": "Link post", + "#category": ("postmill", "raddle", "post"), + "#class": postmill.PostmillPostExtractor, + "#urls": "https://m.youtube.com/watch?v=RFJCA5zcZxI", + "#count": 1, + }, + { + "#url": "https://raddle.me/f/Anime/150698/neo-tokyo-1987-link-to-the-english-dub-version-last-link", + "#comment": "Link + text post (with text disabled)", + "#category": ("postmill", "raddle", "post"), + "#class": postmill.PostmillPostExtractor, + "#pattern": r"^https://fantasyanime\.com/anime/neo-tokyo-dub$", + "#count": 1, + }, + { + "#url": "https://raddle.me/f/egg_irl/166855/4th-wall-breaking-please-let-this-be-a-flair-egg-irl", + "#comment": "Post with multiple flairs", + "#category": ("postmill", "raddle", "post"), + "#class": postmill.PostmillPostExtractor, + "flair": ["Gender non-specific", "4th wall breaking"], + }, ) diff --git a/test/results/raidlondon.py b/test/results/raidlondon.py index 93aab265f..89e32ce0f 100644 --- a/test/results/raidlondon.py +++ b/test/results/raidlondon.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://www.raidlondon.com/collections/flats", - "#category": ("shopify", "raidlondon", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.raidlondon.com/collections/flats/products/raid-addyson-chunky-flat-shoe-in-white", - "#category": ("shopify", "raidlondon", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://www.raidlondon.com/collections/flats", + "#category": ("shopify", "raidlondon", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.raidlondon.com/collections/flats/products/raid-addyson-chunky-flat-shoe-in-white", + "#category": ("shopify", "raidlondon", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/rbt.py b/test/results/rbt.py index 5a81ad1af..2bf6eae83 100644 --- a/test/results/rbt.py +++ b/test/results/rbt.py @@ -1,43 +1,35 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://rbt.asia/g/thread/61487650/", - "#category": ("foolfuuka", "rbt", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", -}, - -{ - "#url" : "https://archive.rebeccablacktech.com/g/thread/61487650/", - "#category": ("foolfuuka", "rbt", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", -}, - -{ - "#url" : "https://rbt.asia/g/", - "#category": ("foolfuuka", "rbt", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://rbt.asia/_/search/text/test/", - "#category": ("foolfuuka", "rbt", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://rbt.asia/g/gallery/8", - "#category": ("foolfuuka", "rbt", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://rbt.asia/g/thread/61487650/", + "#category": ("foolfuuka", "rbt", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", + }, + { + "#url": "https://archive.rebeccablacktech.com/g/thread/61487650/", + "#category": ("foolfuuka", "rbt", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", + }, + { + "#url": "https://rbt.asia/g/", + "#category": ("foolfuuka", "rbt", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://rbt.asia/_/search/text/test/", + "#category": ("foolfuuka", "rbt", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://rbt.asia/g/gallery/8", + "#category": ("foolfuuka", "rbt", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/reactor.py b/test/results/reactor.py index 7ff6dcced..01741c9e2 100644 --- a/test/results/reactor.py +++ b/test/results/reactor.py @@ -1,36 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import reactor - __tests__ = ( -{ - "#url" : "http://reactor.cc/tag/gif", - "#category": ("reactor", "reactor", "tag"), - "#class" : reactor.ReactorTagExtractor, -}, - -{ - "#url" : "http://reactor.cc/search?q=Art", - "#category": ("reactor", "reactor", "search"), - "#class" : reactor.ReactorSearchExtractor, -}, - -{ - "#url" : "http://reactor.cc/user/Dioklet", - "#category": ("reactor", "reactor", "user"), - "#class" : reactor.ReactorUserExtractor, -}, - -{ - "#url" : "http://reactor.cc/post/4999736", - "#category": ("reactor", "reactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#sha1_url": "dfc74d150d7267384d8c229c4b82aa210755daa0", -}, - + { + "#url": "http://reactor.cc/tag/gif", + "#category": ("reactor", "reactor", "tag"), + "#class": reactor.ReactorTagExtractor, + }, + { + "#url": "http://reactor.cc/search?q=Art", + "#category": ("reactor", "reactor", "search"), + "#class": reactor.ReactorSearchExtractor, + }, + { + "#url": "http://reactor.cc/user/Dioklet", + "#category": ("reactor", "reactor", "user"), + "#class": reactor.ReactorUserExtractor, + }, + { + "#url": "http://reactor.cc/post/4999736", + "#category": ("reactor", "reactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#sha1_url": "dfc74d150d7267384d8c229c4b82aa210755daa0", + }, ) diff --git a/test/results/readcomiconline.py b/test/results/readcomiconline.py index 883ed3210..b92d710c1 100644 --- a/test/results/readcomiconline.py +++ b/test/results/readcomiconline.py @@ -1,36 +1,30 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import readcomiconline - __tests__ = ( -{ - "#url" : "https://readcomiconline.li/Comic/W-i-t-c-h/Issue-130?id=22289", - "#category": ("", "readcomiconline", "issue"), - "#class" : readcomiconline.ReadcomiconlineIssueExtractor, - "#pattern" : r"https://2\.bp\.blogspot\.com/[\w-]+=s0\?.+", - "#count" : 36, - "#sha1_metadata": "2d9ec81ce1b11fac06ebf96ce33cdbfca0e85eb5", -}, - -{ - "#url" : "https://readcomiconline.li/Comic/W-i-t-c-h", - "#category": ("", "readcomiconline", "comic"), - "#class" : readcomiconline.ReadcomiconlineComicExtractor, - "#sha1_url" : "74eb8b9504b4084fcc9367b341300b2c52260918", - "#sha1_metadata": "3986248e4458fa44a201ec073c3684917f48ee0c", -}, - -{ - "#url" : "https://readcomiconline.to/Comic/Bazooka-Jules", - "#category": ("", "readcomiconline", "comic"), - "#class" : readcomiconline.ReadcomiconlineComicExtractor, - "#sha1_url" : "2f66a467a772df4d4592e97a059ddbc3e8991799", - "#sha1_metadata": "f5ba5246cd787bb750924d9690cb1549199bd516", -}, - + { + "#url": "https://readcomiconline.li/Comic/W-i-t-c-h/Issue-130?id=22289", + "#category": ("", "readcomiconline", "issue"), + "#class": readcomiconline.ReadcomiconlineIssueExtractor, + "#pattern": r"https://2\.bp\.blogspot\.com/[\w-]+=s0\?.+", + "#count": 36, + "#sha1_metadata": "2d9ec81ce1b11fac06ebf96ce33cdbfca0e85eb5", + }, + { + "#url": "https://readcomiconline.li/Comic/W-i-t-c-h", + "#category": ("", "readcomiconline", "comic"), + "#class": readcomiconline.ReadcomiconlineComicExtractor, + "#sha1_url": "74eb8b9504b4084fcc9367b341300b2c52260918", + "#sha1_metadata": "3986248e4458fa44a201ec073c3684917f48ee0c", + }, + { + "#url": "https://readcomiconline.to/Comic/Bazooka-Jules", + "#category": ("", "readcomiconline", "comic"), + "#class": readcomiconline.ReadcomiconlineComicExtractor, + "#sha1_url": "2f66a467a772df4d4592e97a059ddbc3e8991799", + "#sha1_metadata": "f5ba5246cd787bb750924d9690cb1549199bd516", + }, ) diff --git a/test/results/realbooru.py b/test/results/realbooru.py index 1ff069e86..fbca6382f 100644 --- a/test/results/realbooru.py +++ b/test/results/realbooru.py @@ -1,85 +1,76 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v02 - __tests__ = ( -{ - "#url" : "https://realbooru.com/index.php?page=post&s=list&tags=wine", - "#category": ("gelbooru_v02", "realbooru", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#count" : ">= 64", -}, - -{ - "#url" : "https://realbooru.com/index.php?page=pool&s=show&id=1", - "#category": ("gelbooru_v02", "realbooru", "pool"), - "#class" : gelbooru_v02.GelbooruV02PoolExtractor, - "#urls" : ( - "https://realbooru.com//images/bf/d6/bfd682f338691e5254de796040fcba21.webm", - "https://realbooru.com//images/cb/7d/cb7d921673ba99f688031ac554777695.webm", - "https://realbooru.com//images/9e/14/9e140edc1cb2e4cc734ba5bdc4870955.webm", - ), -}, - -{ - "#url" : "https://realbooru.com/index.php?page=favorites&s=view&id=274", - "#category": ("gelbooru_v02", "realbooru", "favorite"), - "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, - "#urls" : ( - "https://realbooru.com//images/20/3e/0c2c4d8c978355c053602dc963eb13136c1614c1.jpeg", - ), -}, - -{ - "#url" : "https://realbooru.com/index.php?page=post&s=view&id=862054", - "#comment" : "regular post", - "#category": ("gelbooru_v02", "realbooru", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#options" : {"tags": True}, - "#urls" : "https://realbooru.com//images/8a/34/8a345820da989637c21ac013d522bf69.jpeg", - "#sha1_content": "f6213e6f25c3cb9e3cfefa6d4b3a78e44b9dea5b", - - "change" : "1705562002", - "created_at" : "Thu Jan 18 01:12:50 -0600 2024", - "creator_id" : "32011", - "date" : "dt:2024-01-18 07:12:50", - "file_url" : "https://realbooru.com//images/8a/34/8a345820da989637c21ac013d522bf69.jpeg", - "filename" : "8a345820da989637c21ac013d522bf69", - "has_children" : "false", - "has_comments" : "false", - "has_notes" : "false", - "height" : "1800", - "id" : "862054", - "md5" : "8a345820da989637c21ac013d522bf69", - "parent_id" : "", - "preview_height": "150", - "preview_url" : "https://realbooru.com/thumbnails/8a/34/thumbnail_8a345820da989637c21ac013d522bf69.jpg", - "preview_width" : "120", - "rating" : "e", - "sample_height" : "1063", - "sample_url" : "https://realbooru.com/samples/8a/34/sample_8a345820da989637c21ac013d522bf69.jpg", - "sample_width" : "850", - "score" : "", - "source" : "https://www.instagram.com/p/CwAO1UyJBnw", - "status" : "active", - "tags" : "1girl asian bikini black_hair breasts cleavage female female_only floral_print instagram japanese kurita_emi large_breasts looking_at_viewer navel sauna short_hair side-tie_bikini sitting solo", - "tags_copyright": "instagram", - "tags_general" : "1girl asian bikini black_hair breasts cleavage female female_only floral_print japanese large_breasts looking_at_viewer navel sauna short_hair side-tie_bikini sitting solo", - "tags_model" : "kurita_emi", - "width" : "1440", -}, - -{ - "#url" : "https://realbooru.com/index.php?page=post&s=view&id=568145", - "#comment" : "older post", - "#category": ("gelbooru_v02", "realbooru", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#sha1_content": "4a7424810f5f846c161b5d3b7c8b0a85a03368c8", -}, - + { + "#url": "https://realbooru.com/index.php?page=post&s=list&tags=wine", + "#category": ("gelbooru_v02", "realbooru", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#count": ">= 64", + }, + { + "#url": "https://realbooru.com/index.php?page=pool&s=show&id=1", + "#category": ("gelbooru_v02", "realbooru", "pool"), + "#class": gelbooru_v02.GelbooruV02PoolExtractor, + "#urls": ( + "https://realbooru.com//images/bf/d6/bfd682f338691e5254de796040fcba21.webm", + "https://realbooru.com//images/cb/7d/cb7d921673ba99f688031ac554777695.webm", + "https://realbooru.com//images/9e/14/9e140edc1cb2e4cc734ba5bdc4870955.webm", + ), + }, + { + "#url": "https://realbooru.com/index.php?page=favorites&s=view&id=274", + "#category": ("gelbooru_v02", "realbooru", "favorite"), + "#class": gelbooru_v02.GelbooruV02FavoriteExtractor, + "#urls": ( + "https://realbooru.com//images/20/3e/0c2c4d8c978355c053602dc963eb13136c1614c1.jpeg", + ), + }, + { + "#url": "https://realbooru.com/index.php?page=post&s=view&id=862054", + "#comment": "regular post", + "#category": ("gelbooru_v02", "realbooru", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#options": {"tags": True}, + "#urls": "https://realbooru.com//images/8a/34/8a345820da989637c21ac013d522bf69.jpeg", + "#sha1_content": "f6213e6f25c3cb9e3cfefa6d4b3a78e44b9dea5b", + "change": "1705562002", + "created_at": "Thu Jan 18 01:12:50 -0600 2024", + "creator_id": "32011", + "date": "dt:2024-01-18 07:12:50", + "file_url": "https://realbooru.com//images/8a/34/8a345820da989637c21ac013d522bf69.jpeg", + "filename": "8a345820da989637c21ac013d522bf69", + "has_children": "false", + "has_comments": "false", + "has_notes": "false", + "height": "1800", + "id": "862054", + "md5": "8a345820da989637c21ac013d522bf69", + "parent_id": "", + "preview_height": "150", + "preview_url": "https://realbooru.com/thumbnails/8a/34/thumbnail_8a345820da989637c21ac013d522bf69.jpg", + "preview_width": "120", + "rating": "e", + "sample_height": "1063", + "sample_url": "https://realbooru.com/samples/8a/34/sample_8a345820da989637c21ac013d522bf69.jpg", + "sample_width": "850", + "score": "", + "source": "https://www.instagram.com/p/CwAO1UyJBnw", + "status": "active", + "tags": "1girl asian bikini black_hair breasts cleavage female female_only floral_print instagram japanese kurita_emi large_breasts looking_at_viewer navel sauna short_hair side-tie_bikini sitting solo", + "tags_copyright": "instagram", + "tags_general": "1girl asian bikini black_hair breasts cleavage female female_only floral_print japanese large_breasts looking_at_viewer navel sauna short_hair side-tie_bikini sitting solo", + "tags_model": "kurita_emi", + "width": "1440", + }, + { + "#url": "https://realbooru.com/index.php?page=post&s=view&id=568145", + "#comment": "older post", + "#category": ("gelbooru_v02", "realbooru", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#sha1_content": "4a7424810f5f846c161b5d3b7c8b0a85a03368c8", + }, ) diff --git a/test/results/recursive.py b/test/results/recursive.py index 2ad31dd46..647e50fae 100644 --- a/test/results/recursive.py +++ b/test/results/recursive.py @@ -1,18 +1,14 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import recursive - __tests__ = ( -{ - "#url" : "recursive:https://pastebin.com/raw/FLwrCYsT", - "#category": ("", "recursive", ""), - "#class" : recursive.RecursiveExtractor, - "#sha1_url": "eee86d65c346361b818e8f4b2b307d9429f136a2", -}, - + { + "#url": "recursive:https://pastebin.com/raw/FLwrCYsT", + "#category": ("", "recursive", ""), + "#class": recursive.RecursiveExtractor, + "#sha1_url": "eee86d65c346361b818e8f4b2b307d9429f136a2", + }, ) diff --git a/test/results/reddit.py b/test/results/reddit.py index 457aaf4be..5982cb80f 100644 --- a/test/results/reddit.py +++ b/test/results/reddit.py @@ -1,287 +1,248 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import reddit - __tests__ = ( -{ - "#url" : "https://www.reddit.com/r/lavaporn/", - "#category": ("", "reddit", "subreddit"), - "#class" : reddit.RedditSubredditExtractor, - "#range" : "1-20", - "#count" : ">= 20", -}, - -{ - "#url" : "https://www.reddit.com/r/lavaporn/top/?sort=top&t=month", - "#category": ("", "reddit", "subreddit-top"), - "#class" : reddit.RedditSubredditExtractor, -}, - -{ - "#url" : "https://old.reddit.com/r/lavaporn/", - "#category": ("", "reddit", "subreddit"), - "#class" : reddit.RedditSubredditExtractor, -}, - -{ - "#url" : "https://np.reddit.com/r/lavaporn/", - "#category": ("", "reddit", "subreddit"), - "#class" : reddit.RedditSubredditExtractor, -}, - -{ - "#url" : "https://m.reddit.com/r/lavaporn/", - "#category": ("", "reddit", "subreddit"), - "#class" : reddit.RedditSubredditExtractor, -}, - -{ - "#url" : "https://www.reddit.com/", - "#category": ("", "reddit", "home"), - "#class" : reddit.RedditHomeExtractor, - "#range" : "1-20", - "#count" : ">= 20", - "#archive" : False, -}, - -{ - "#url" : "https://old.reddit.com/top/?sort=top&t=month", - "#category": ("", "reddit", "home-top"), - "#class" : reddit.RedditHomeExtractor, -}, - -{ - "#url" : "https://www.reddit.com/user/username/", - "#category": ("", "reddit", "user"), - "#class" : reddit.RedditUserExtractor, - "#count" : ">= 2", -}, - -{ - "#url" : "https://www.reddit.com/user/username/gilded/?sort=top&t=month", - "#category": ("", "reddit", "user-gilded"), - "#class" : reddit.RedditUserExtractor, -}, - -{ - "#url" : "https://old.reddit.com/user/username/", - "#category": ("", "reddit", "user"), - "#class" : reddit.RedditUserExtractor, -}, - -{ - "#url" : "https://www.reddit.com/u/username/", - "#category": ("", "reddit", "user"), - "#class" : reddit.RedditUserExtractor, -}, - -{ - "#url" : "https://www.reddit.com/r/lavaporn/comments/8cqhub/", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#pattern" : r"https://c2.staticflickr.com/8/7272/\w+_k.jpg", - "#count" : 1, -}, - -{ - "#url" : "https://www.reddit.com/r/lavaporn/comments/8cqhub/", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#options" : {"comments": 500}, - "#pattern" : "https://", - "#count" : 3, -}, - -{ - "#url" : "https://www.reddit.com/gallery/hrrh23", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#count" : 3, - "#sha1_url" : "25b91ede15459470274dd17291424b037ed8b0ae", - "#sha1_content": "1e7dde4ee7d5f4c4b45749abfd15b2dbfa27df3f", -}, - -{ - "#url" : "https://www.reddit.com/r/aww/comments/90bu6w/", - "#comment" : "video (dash)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#pattern" : "ytdl:https://v.redd.it/gyh95hiqc0b11", - "#count" : 1, -}, - -{ - "#url" : "https://www.reddit.com/r/aww/comments/90bu6w/", - "#comment" : "video (dash)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#options" : {"videos": "ytdl"}, - "#pattern" : "ytdl:https://www.reddit.com/r/aww/comments/90bu6w/heat_index_was_110_degrees_so_we_offered_him_a/", - "#count" : 1, -}, - -{ - "#url" : "https://www.reddit.com/r/aww/comments/90bu6w/", - "#comment" : "video (dash)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#options" : {"videos": "dash"}, - "#pattern" : r"ytdl:https://v.redd.it/gyh95hiqc0b11/DASHPlaylist.mpd\?a=", - "#count" : 1, -}, - -{ - "#url" : "https://www.reddit.com/gallery/icfgzv", - "#comment" : "deleted gallery (#953)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.reddit.com/r/araragi/comments/ib32hm", - "#comment" : "animated gallery items (#955)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#pattern" : r"https://i\.redd\.it/\w+\.gif", - "#count" : 2, -}, - -{ - "#url" : "https://www.reddit.com/r/cosplay/comments/jvwaqr", - "#comment" : "'failed' gallery item (#1127)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#count" : 1, -}, - -{ - "#url" : "https://www.reddit.com/r/kpopfap/comments/qjj04q/", - "#comment" : "gallery with no 'media_metadata' (#2001)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.reddit.com/r/RobloxArt/comments/15ko0qu/", - "#comment" : "comment embeds (#5366)", - "#class" : reddit.RedditSubmissionExtractor, - "#options" : {"comments": 10}, - "#urls" : ( - "https://i.redd.it/ppt5yciyipgb1.jpg", - "https://i.redd.it/u0ojzd69kpgb1.png", - ), -}, - -{ - "#url" : "https://www.reddit.com/r/RobloxArt/comments/15ko0qu/", - "#comment" : "disabled comment embeds (#6357)", - "#class" : reddit.RedditSubmissionExtractor, - "#options" : {"comments": 10, "embeds": False}, - "#urls" : "https://i.redd.it/ppt5yciyipgb1.jpg", -}, - -{ - "#url" : "https://www.reddit.com/user/TheSpiritTree/comments/srilyf/", - "#comment" : "user page submission (#2301)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#pattern" : "https://i.redd.it/8fpgv17yqlh81.jpg", - "#count" : 1, -}, - -{ - "#url" : "https://www.reddit.com/r/kittengifs/comments/12m0b8d", - "#comment" : "cross-posted video (#887, #3586, #3976)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#pattern" : r"ytdl:https://v\.redd\.it/cvabpjacrvta1", -}, - -{ - "#url" : "https://www.reddit.com/r/europe/comments/pm4531/the_name_of/", - "#comment" : "preview.redd.it (#4470)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#urls" : "https://preview.redd.it/u9ud4k6xaf271.jpg?auto=webp&s=19b1334cb4409111cda136c01f7b44c2c42bf9fb", -}, - -{ - "#url" : "https://old.reddit.com/r/lavaporn/comments/2a00np/", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, -}, - -{ - "#url" : "https://np.reddit.com/r/lavaporn/comments/2a00np/", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, -}, - -{ - "#url" : "https://m.reddit.com/r/lavaporn/comments/2a00np/", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, -}, - -{ - "#url" : "https://redd.it/2a00np/", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, -}, - -{ - "#url" : "https://i.redd.it/upjtjcx2npzz.jpg", - "#category": ("", "reddit", "image"), - "#class" : reddit.RedditImageExtractor, - "#sha1_url" : "0de614900feef103e580b632190458c0b62b641a", - "#sha1_content": "cc9a68cf286708d5ce23c68e79cd9cf7826db6a3", -}, - -{ - "#url" : "https://i.reddituploads.com/0f44f1b1fca2461f957c713d9592617d?fit=max&h=1536&w=1536&s=e96ce7846b3c8e1f921d2ce2671fb5e2", - "#category": ("", "reddit", "image"), - "#class" : reddit.RedditImageExtractor, - "#sha1_url" : "f24f25efcedaddeec802e46c60d77ef975dc52a5", - "#sha1_content": "541dbcc3ad77aa01ee21ca49843c5e382371fae7", -}, - -{ - "#url" : "https://preview.redd.it/00af44lpn0u51.jpg?width=960&crop=smart&auto=webp&v=enabled&s=dbca8ab84033f4a433772d9c15dbe0429c74e8ac", - "#comment" : "preview.redd.it -> i.redd.it", - "#category": ("", "reddit", "image"), - "#class" : reddit.RedditImageExtractor, - "#pattern" : r"^https://i\.redd\.it/00af44lpn0u51\.jpg$", -}, - -{ - "#url" : "https://www.reddit.com/r/analog/s/hKrTTvFVwZ", - "#comment" : "Mobile share URL", - "#category": ("", "reddit", "redirect"), - "#class" : reddit.RedditRedirectExtractor, - "#pattern" : r"^https://www\.reddit\.com/r/analog/comments/179exao/photographing_the_recent_annular_eclipse_with_a", -}, - -{ - "#url" : "https://www.reddit.com/u/Tailhook91/s/w4yAMbtOYm", - "#comment" : "Mobile share URL, user submission", - "#category": ("", "reddit", "redirect"), - "#class" : reddit.RedditRedirectExtractor, - "#pattern" : r"^https://www.reddit.com/user/Tailhook91/comments/znfxbr/prove_it/", -}, - -{ - "#url" : "https://www.reddit.com/user/Tailhook91/s/w4yAMbtOYm", - "#comment" : "Mobile share URL, user submission", - "#category": ("", "reddit", "redirect"), - "#class" : reddit.RedditRedirectExtractor, - "#pattern" : r"^https://www.reddit.com/user/Tailhook91/comments/znfxbr/prove_it/", -}, - + { + "#url": "https://www.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class": reddit.RedditSubredditExtractor, + "#range": "1-20", + "#count": ">= 20", + }, + { + "#url": "https://www.reddit.com/r/lavaporn/top/?sort=top&t=month", + "#category": ("", "reddit", "subreddit-top"), + "#class": reddit.RedditSubredditExtractor, + }, + { + "#url": "https://old.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class": reddit.RedditSubredditExtractor, + }, + { + "#url": "https://np.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class": reddit.RedditSubredditExtractor, + }, + { + "#url": "https://m.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class": reddit.RedditSubredditExtractor, + }, + { + "#url": "https://www.reddit.com/", + "#category": ("", "reddit", "home"), + "#class": reddit.RedditHomeExtractor, + "#range": "1-20", + "#count": ">= 20", + "#archive": False, + }, + { + "#url": "https://old.reddit.com/top/?sort=top&t=month", + "#category": ("", "reddit", "home-top"), + "#class": reddit.RedditHomeExtractor, + }, + { + "#url": "https://www.reddit.com/user/username/", + "#category": ("", "reddit", "user"), + "#class": reddit.RedditUserExtractor, + "#count": ">= 2", + }, + { + "#url": "https://www.reddit.com/user/username/gilded/?sort=top&t=month", + "#category": ("", "reddit", "user-gilded"), + "#class": reddit.RedditUserExtractor, + }, + { + "#url": "https://old.reddit.com/user/username/", + "#category": ("", "reddit", "user"), + "#class": reddit.RedditUserExtractor, + }, + { + "#url": "https://www.reddit.com/u/username/", + "#category": ("", "reddit", "user"), + "#class": reddit.RedditUserExtractor, + }, + { + "#url": "https://www.reddit.com/r/lavaporn/comments/8cqhub/", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#pattern": r"https://c2.staticflickr.com/8/7272/\w+_k.jpg", + "#count": 1, + }, + { + "#url": "https://www.reddit.com/r/lavaporn/comments/8cqhub/", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#options": {"comments": 500}, + "#pattern": "https://", + "#count": 3, + }, + { + "#url": "https://www.reddit.com/gallery/hrrh23", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#count": 3, + "#sha1_url": "25b91ede15459470274dd17291424b037ed8b0ae", + "#sha1_content": "1e7dde4ee7d5f4c4b45749abfd15b2dbfa27df3f", + }, + { + "#url": "https://www.reddit.com/r/aww/comments/90bu6w/", + "#comment": "video (dash)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#pattern": "ytdl:https://v.redd.it/gyh95hiqc0b11", + "#count": 1, + }, + { + "#url": "https://www.reddit.com/r/aww/comments/90bu6w/", + "#comment": "video (dash)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#options": {"videos": "ytdl"}, + "#pattern": "ytdl:https://www.reddit.com/r/aww/comments/90bu6w/heat_index_was_110_degrees_so_we_offered_him_a/", + "#count": 1, + }, + { + "#url": "https://www.reddit.com/r/aww/comments/90bu6w/", + "#comment": "video (dash)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#options": {"videos": "dash"}, + "#pattern": r"ytdl:https://v.redd.it/gyh95hiqc0b11/DASHPlaylist.mpd\?a=", + "#count": 1, + }, + { + "#url": "https://www.reddit.com/gallery/icfgzv", + "#comment": "deleted gallery (#953)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#count": 0, + }, + { + "#url": "https://www.reddit.com/r/araragi/comments/ib32hm", + "#comment": "animated gallery items (#955)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#pattern": r"https://i\.redd\.it/\w+\.gif", + "#count": 2, + }, + { + "#url": "https://www.reddit.com/r/cosplay/comments/jvwaqr", + "#comment": "'failed' gallery item (#1127)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#count": 1, + }, + { + "#url": "https://www.reddit.com/r/kpopfap/comments/qjj04q/", + "#comment": "gallery with no 'media_metadata' (#2001)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#count": 0, + }, + { + "#url": "https://www.reddit.com/r/RobloxArt/comments/15ko0qu/", + "#comment": "comment embeds (#5366)", + "#class": reddit.RedditSubmissionExtractor, + "#options": {"comments": 10}, + "#urls": ( + "https://i.redd.it/ppt5yciyipgb1.jpg", + "https://i.redd.it/u0ojzd69kpgb1.png", + ), + }, + { + "#url": "https://www.reddit.com/r/RobloxArt/comments/15ko0qu/", + "#comment": "disabled comment embeds (#6357)", + "#class": reddit.RedditSubmissionExtractor, + "#options": {"comments": 10, "embeds": False}, + "#urls": "https://i.redd.it/ppt5yciyipgb1.jpg", + }, + { + "#url": "https://www.reddit.com/user/TheSpiritTree/comments/srilyf/", + "#comment": "user page submission (#2301)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#pattern": "https://i.redd.it/8fpgv17yqlh81.jpg", + "#count": 1, + }, + { + "#url": "https://www.reddit.com/r/kittengifs/comments/12m0b8d", + "#comment": "cross-posted video (#887, #3586, #3976)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#pattern": r"ytdl:https://v\.redd\.it/cvabpjacrvta1", + }, + { + "#url": "https://www.reddit.com/r/europe/comments/pm4531/the_name_of/", + "#comment": "preview.redd.it (#4470)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#urls": "https://preview.redd.it/u9ud4k6xaf271.jpg?auto=webp&s=19b1334cb4409111cda136c01f7b44c2c42bf9fb", + }, + { + "#url": "https://old.reddit.com/r/lavaporn/comments/2a00np/", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + }, + { + "#url": "https://np.reddit.com/r/lavaporn/comments/2a00np/", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + }, + { + "#url": "https://m.reddit.com/r/lavaporn/comments/2a00np/", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + }, + { + "#url": "https://redd.it/2a00np/", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + }, + { + "#url": "https://i.redd.it/upjtjcx2npzz.jpg", + "#category": ("", "reddit", "image"), + "#class": reddit.RedditImageExtractor, + "#sha1_url": "0de614900feef103e580b632190458c0b62b641a", + "#sha1_content": "cc9a68cf286708d5ce23c68e79cd9cf7826db6a3", + }, + { + "#url": "https://i.reddituploads.com/0f44f1b1fca2461f957c713d9592617d?fit=max&h=1536&w=1536&s=e96ce7846b3c8e1f921d2ce2671fb5e2", + "#category": ("", "reddit", "image"), + "#class": reddit.RedditImageExtractor, + "#sha1_url": "f24f25efcedaddeec802e46c60d77ef975dc52a5", + "#sha1_content": "541dbcc3ad77aa01ee21ca49843c5e382371fae7", + }, + { + "#url": "https://preview.redd.it/00af44lpn0u51.jpg?width=960&crop=smart&auto=webp&v=enabled&s=dbca8ab84033f4a433772d9c15dbe0429c74e8ac", + "#comment": "preview.redd.it -> i.redd.it", + "#category": ("", "reddit", "image"), + "#class": reddit.RedditImageExtractor, + "#pattern": r"^https://i\.redd\.it/00af44lpn0u51\.jpg$", + }, + { + "#url": "https://www.reddit.com/r/analog/s/hKrTTvFVwZ", + "#comment": "Mobile share URL", + "#category": ("", "reddit", "redirect"), + "#class": reddit.RedditRedirectExtractor, + "#pattern": r"^https://www\.reddit\.com/r/analog/comments/179exao/photographing_the_recent_annular_eclipse_with_a", + }, + { + "#url": "https://www.reddit.com/u/Tailhook91/s/w4yAMbtOYm", + "#comment": "Mobile share URL, user submission", + "#category": ("", "reddit", "redirect"), + "#class": reddit.RedditRedirectExtractor, + "#pattern": r"^https://www.reddit.com/user/Tailhook91/comments/znfxbr/prove_it/", + }, + { + "#url": "https://www.reddit.com/user/Tailhook91/s/w4yAMbtOYm", + "#comment": "Mobile share URL, user submission", + "#category": ("", "reddit", "redirect"), + "#class": reddit.RedditRedirectExtractor, + "#pattern": r"^https://www.reddit.com/user/Tailhook91/comments/znfxbr/prove_it/", + }, ) diff --git a/test/results/redgifs.py b/test/results/redgifs.py index 610e32ebf..a17f4bc0c 100644 --- a/test/results/redgifs.py +++ b/test/results/redgifs.py @@ -1,182 +1,156 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import redgifs - __tests__ = ( -{ - "#url" : "https://www.redgifs.com/users/mmj", - "#category": ("", "redgifs", "user"), - "#class" : redgifs.RedgifsUserExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "#count" : range(50, 60), -}, - -{ - "#url" : "https://www.redgifs.com/users/mmj?order=old", - "#comment" : "'order' URL parameter (#4583)", - "#category": ("", "redgifs", "user"), - "#class" : redgifs.RedgifsUserExtractor, - "#range" : "1-5", - "#patterns": ( - r"https://thumbs\d+\.redgifs\.com/ShoddyOilyHarlequinbug\.mp4", - r"https://thumbs\d+\.redgifs\.com/UnevenPrestigiousKilldeer\.mp4", - r"https://thumbs\d+\.redgifs\.com/EveryShockingFlickertailsquirrel\.mp4", - r"https://thumbs\d+\.redgifs\.com/NegativeWarlikeAmericancurl\.mp4", - r"https://thumbs\d+\.redgifs\.com/PopularTerribleFritillarybutterfly\.mp4", - ), -}, - -{ - "#url" : "https://v3.redgifs.com/users/lamsinka89", - "#comment" : "'v3' subdomain (#3588, #3589)", - "#category": ("", "redgifs", "user"), - "#class" : redgifs.RedgifsUserExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", - "#count" : ">= 100", -}, - -{ - "#url" : "https://www.redgifs.com/users/boombah123/collections/2631326bbd", - "#category": ("", "redgifs", "collection"), - "#class" : redgifs.RedgifsCollectionExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.redgifs.com/users/boombah123/collections/9e6f7dd41f", - "#category": ("", "redgifs", "collection"), - "#class" : redgifs.RedgifsCollectionExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.redgifs.com/users/boombah123/collections", - "#category": ("", "redgifs", "collections"), - "#class" : redgifs.RedgifsCollectionsExtractor, - "#pattern" : r"https://www\.redgifs\.com/users/boombah123/collections/\w+", - "#count" : ">= 3", -}, - -{ - "#url" : "https://www.redgifs.com/niches/just-boobs", - "#category": ("", "redgifs", "niches"), - "#class" : redgifs.RedgifsNichesExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.redgifs.com/niches/thick-booty", - "#category": ("", "redgifs", "niches"), - "#class" : redgifs.RedgifsNichesExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.redgifs.com/gifs/jav", - "#category": ("", "redgifs", "search"), - "#class" : redgifs.RedgifsSearchExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.redgifs.com/browse?tags=JAV", - "#category": ("", "redgifs", "search"), - "#class" : redgifs.RedgifsSearchExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.redgifs.com/gifs/jav?order=best&verified=1", - "#category": ("", "redgifs", "search"), - "#class" : redgifs.RedgifsSearchExtractor, -}, - -{ - "#url" : "https://www.redgifs.com/browse?type=i&verified=y&order=top7", - "#category": ("", "redgifs", "search"), - "#class" : redgifs.RedgifsSearchExtractor, -}, - -{ - "#url" : "https://v3.redgifs.com/browse?tags=JAV", - "#category": ("", "redgifs", "search"), - "#class" : redgifs.RedgifsSearchExtractor, -}, - -{ - "#url" : "https://redgifs.com/watch/foolishforkedabyssiniancat", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/FoolishForkedAbyssiniancat\.mp4", - "#sha1_content": "f6e03f1df9a2ff2a74092f53ee7580d2fb943533", -}, - -{ - "#url" : "https://www.redgifs.com/watch/desertedbaregraywolf", - "#comment" : "gallery (#4021)", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[A-Za-z-]+\.jpg", - "#count" : 4, - - "num" : int, - "count" : 4, - "gallery": "187ad979693-1922-fc66-0000-a96fb07b8a5d", -}, - -{ - "#url" : "https://redgifs.com/ifr/FoolishForkedAbyssiniancat", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, -}, - -{ - "#url" : "https://i.redgifs.com/i/FoolishForkedAbyssiniancat", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, -}, - -{ - "#url" : "https://www.gifdeliverynetwork.com/foolishforkedabyssiniancat", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, -}, - -{ - "#url" : "https://v3.redgifs.com/watch/FoolishForkedAbyssiniancat", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, -}, - -{ - "#url" : "https://v3.redgifs.com/watch/605025947780972895", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, - - "id": "humblegrippingmole", -}, - -{ - "#url" : "https://www.gfycat.com/foolishforkedabyssiniancat", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, -}, - + { + "#url": "https://www.redgifs.com/users/mmj", + "#category": ("", "redgifs", "user"), + "#class": redgifs.RedgifsUserExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#count": range(50, 60), + }, + { + "#url": "https://www.redgifs.com/users/mmj?order=old", + "#comment": "'order' URL parameter (#4583)", + "#category": ("", "redgifs", "user"), + "#class": redgifs.RedgifsUserExtractor, + "#range": "1-5", + "#patterns": ( + r"https://thumbs\d+\.redgifs\.com/ShoddyOilyHarlequinbug\.mp4", + r"https://thumbs\d+\.redgifs\.com/UnevenPrestigiousKilldeer\.mp4", + r"https://thumbs\d+\.redgifs\.com/EveryShockingFlickertailsquirrel\.mp4", + r"https://thumbs\d+\.redgifs\.com/NegativeWarlikeAmericancurl\.mp4", + r"https://thumbs\d+\.redgifs\.com/PopularTerribleFritillarybutterfly\.mp4", + ), + }, + { + "#url": "https://v3.redgifs.com/users/lamsinka89", + "#comment": "'v3' subdomain (#3588, #3589)", + "#category": ("", "redgifs", "user"), + "#class": redgifs.RedgifsUserExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", + "#count": ">= 100", + }, + { + "#url": "https://www.redgifs.com/users/boombah123/collections/2631326bbd", + "#category": ("", "redgifs", "collection"), + "#class": redgifs.RedgifsCollectionExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.redgifs.com/users/boombah123/collections/9e6f7dd41f", + "#category": ("", "redgifs", "collection"), + "#class": redgifs.RedgifsCollectionExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.redgifs.com/users/boombah123/collections", + "#category": ("", "redgifs", "collections"), + "#class": redgifs.RedgifsCollectionsExtractor, + "#pattern": r"https://www\.redgifs\.com/users/boombah123/collections/\w+", + "#count": ">= 3", + }, + { + "#url": "https://www.redgifs.com/niches/just-boobs", + "#category": ("", "redgifs", "niches"), + "#class": redgifs.RedgifsNichesExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.redgifs.com/niches/thick-booty", + "#category": ("", "redgifs", "niches"), + "#class": redgifs.RedgifsNichesExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.redgifs.com/gifs/jav", + "#category": ("", "redgifs", "search"), + "#class": redgifs.RedgifsSearchExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.redgifs.com/browse?tags=JAV", + "#category": ("", "redgifs", "search"), + "#class": redgifs.RedgifsSearchExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.redgifs.com/gifs/jav?order=best&verified=1", + "#category": ("", "redgifs", "search"), + "#class": redgifs.RedgifsSearchExtractor, + }, + { + "#url": "https://www.redgifs.com/browse?type=i&verified=y&order=top7", + "#category": ("", "redgifs", "search"), + "#class": redgifs.RedgifsSearchExtractor, + }, + { + "#url": "https://v3.redgifs.com/browse?tags=JAV", + "#category": ("", "redgifs", "search"), + "#class": redgifs.RedgifsSearchExtractor, + }, + { + "#url": "https://redgifs.com/watch/foolishforkedabyssiniancat", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + "#pattern": r"https://\w+\.redgifs\.com/FoolishForkedAbyssiniancat\.mp4", + "#sha1_content": "f6e03f1df9a2ff2a74092f53ee7580d2fb943533", + }, + { + "#url": "https://www.redgifs.com/watch/desertedbaregraywolf", + "#comment": "gallery (#4021)", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.jpg", + "#count": 4, + "num": int, + "count": 4, + "gallery": "187ad979693-1922-fc66-0000-a96fb07b8a5d", + }, + { + "#url": "https://redgifs.com/ifr/FoolishForkedAbyssiniancat", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + }, + { + "#url": "https://i.redgifs.com/i/FoolishForkedAbyssiniancat", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + }, + { + "#url": "https://www.gifdeliverynetwork.com/foolishforkedabyssiniancat", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + }, + { + "#url": "https://v3.redgifs.com/watch/FoolishForkedAbyssiniancat", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + }, + { + "#url": "https://v3.redgifs.com/watch/605025947780972895", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + "id": "humblegrippingmole", + }, + { + "#url": "https://www.gfycat.com/foolishforkedabyssiniancat", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + }, ) diff --git a/test/results/rule34.py b/test/results/rule34.py index f6b9b2741..73b9025c3 100644 --- a/test/results/rule34.py +++ b/test/results/rule34.py @@ -1,84 +1,75 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v02 - __tests__ = ( -{ - "#url" : "https://rule34.xxx/index.php?page=post&s=list&tags=danraku", - "#category": ("gelbooru_v02", "rule34", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#pattern" : r"https?://.*rule34\.xxx/images/\d+/[0-9a-f]+\.jpg", - "#count" : 2, - "#sha1_content": [ - "5c6ae9ee13e6d4bc9cb8bdce224c84e67fbfa36c", - "622e80be3f496672c44aab5c47fbc6941c61bc79", - "1e0dced55bcb5eefe5cc32f69c7a8df35547b459", - ], -}, - -{ - "#url" : "https://rule34.xxx/index.php?page=pool&s=show&id=179", - "#category": ("gelbooru_v02", "rule34", "pool"), - "#class" : gelbooru_v02.GelbooruV02PoolExtractor, - "#count" : 3, -}, - -{ - "#url" : "https://rule34.xxx/index.php?page=favorites&s=view&id=1030218", - "#category": ("gelbooru_v02", "rule34", "favorite"), - "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, - "#count" : 3, -}, - -{ - "#url" : "https://www.rule34.xxx/index.php?page=post&s=view&id=863", - "#comment" : "www subdomain", - "#category": ("gelbooru_v02", "rule34", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, -}, - -{ - "#url" : "https://rule34.xxx/index.php?page=post&s=view&id=863", - "#category": ("gelbooru_v02", "rule34", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#options" : { - "tags" : True, - "notes": True, + { + "#url": "https://rule34.xxx/index.php?page=post&s=list&tags=danraku", + "#category": ("gelbooru_v02", "rule34", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#pattern": r"https?://.*rule34\.xxx/images/\d+/[0-9a-f]+\.jpg", + "#count": 2, + "#sha1_content": [ + "5c6ae9ee13e6d4bc9cb8bdce224c84e67fbfa36c", + "622e80be3f496672c44aab5c47fbc6941c61bc79", + "1e0dced55bcb5eefe5cc32f69c7a8df35547b459", + ], }, - "#pattern" : r"https://api-cdn\.rule34\.xxx/images/1/6aafbdb3e22f3f3b412ea2cf53321317a37063f3\.jpg", - "#sha1_content": [ - "a43f418aa350039af0d11cae501396a33bbe2201", - "67b516295950867e1c1ab6bc13b35d3b762ed2a3", - ], - - "tags_artist" : "reverse_noise yamu_(reverse_noise)", - "tags_character": "hong_meiling", - "tags_copyright": "team_shanghai_alice touhou", - "tags_general" : str, - "tags_metadata" : "censored translated", - "notes" : [ - { - "body" : "It feels angry, I'm losing myself... It won't calm down!", - "height": 65, - "id" : 93586, - "width" : 116, - "x" : 22, - "y" : 333, - }, - { - "body" : "REPUTATION OF RAGE", - "height": 272, - "id" : 93587, - "width" : 199, - "x" : 78, - "y" : 442, + { + "#url": "https://rule34.xxx/index.php?page=pool&s=show&id=179", + "#category": ("gelbooru_v02", "rule34", "pool"), + "#class": gelbooru_v02.GelbooruV02PoolExtractor, + "#count": 3, + }, + { + "#url": "https://rule34.xxx/index.php?page=favorites&s=view&id=1030218", + "#category": ("gelbooru_v02", "rule34", "favorite"), + "#class": gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count": 3, + }, + { + "#url": "https://www.rule34.xxx/index.php?page=post&s=view&id=863", + "#comment": "www subdomain", + "#category": ("gelbooru_v02", "rule34", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + }, + { + "#url": "https://rule34.xxx/index.php?page=post&s=view&id=863", + "#category": ("gelbooru_v02", "rule34", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#options": { + "tags": True, + "notes": True, }, - ], -}, - + "#pattern": r"https://api-cdn\.rule34\.xxx/images/1/6aafbdb3e22f3f3b412ea2cf53321317a37063f3\.jpg", + "#sha1_content": [ + "a43f418aa350039af0d11cae501396a33bbe2201", + "67b516295950867e1c1ab6bc13b35d3b762ed2a3", + ], + "tags_artist": "reverse_noise yamu_(reverse_noise)", + "tags_character": "hong_meiling", + "tags_copyright": "team_shanghai_alice touhou", + "tags_general": str, + "tags_metadata": "censored translated", + "notes": [ + { + "body": "It feels angry, I'm losing myself... It won't calm down!", + "height": 65, + "id": 93586, + "width": 116, + "x": 22, + "y": 333, + }, + { + "body": "REPUTATION OF RAGE", + "height": 272, + "id": 93587, + "width": 199, + "x": 78, + "y": 442, + }, + ], + }, ) diff --git a/test/results/rule34hentai.py b/test/results/rule34hentai.py index 1d3cb2912..f391db10e 100644 --- a/test/results/rule34hentai.py +++ b/test/results/rule34hentai.py @@ -1,51 +1,44 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shimmie2 - __tests__ = ( -{ - "#url" : "https://rule34hentai.net/post/list/mizuki_kotora/1", - "#category": ("shimmie2", "rule34hentai", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, - "#urls" : ( - "https://rule34hentai.net/_images/7f3a411263d0f6de936e47ae8f9d35fb/332%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", - "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", - "https://rule34hentai.net/_images/09511511c4c9e9e1f9b795e059a60832/259%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", - ), - - "extension" : "jpeg", - "file_url" : r"re:https://rule34hentai.net/_images/.+\.jpeg", - "filename" : r"re:\d+ - \w+", - "height" : range(496, 875), - "id" : range(259, 332), - "md5" : r"re:^[0-9a-f]{32}$", - "search_tags": "mizuki_kotora", - "size" : int, - "tags" : str, - "width" : range(500, 850), -}, - -{ - "#url" : "https://rule34hentai.net/post/view/264", - "#category": ("shimmie2", "rule34hentai", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, - "#urls" : "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpg", - "#sha1_content": "6c23780bb78673cbff1bca9accb77ea11ec734f3", - - "extension": "jpg", - "file_url" : "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpg", - "filename" : "264 - Darkstalkers Felicia mizuki_kotora", - "height" : 875, - "id" : 264, - "md5" : "1a8eca7c04f8bf325bc993c5751a91c4", - "size" : 0, - "tags" : "Darkstalkers Felicia mizuki_kotora", - "width" : 657, -}, - + { + "#url": "https://rule34hentai.net/post/list/mizuki_kotora/1", + "#category": ("shimmie2", "rule34hentai", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + "#urls": ( + "https://rule34hentai.net/_images/7f3a411263d0f6de936e47ae8f9d35fb/332%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", + "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", + "https://rule34hentai.net/_images/09511511c4c9e9e1f9b795e059a60832/259%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", + ), + "extension": "jpeg", + "file_url": r"re:https://rule34hentai.net/_images/.+\.jpeg", + "filename": r"re:\d+ - \w+", + "height": range(496, 875), + "id": range(259, 332), + "md5": r"re:^[0-9a-f]{32}$", + "search_tags": "mizuki_kotora", + "size": int, + "tags": str, + "width": range(500, 850), + }, + { + "#url": "https://rule34hentai.net/post/view/264", + "#category": ("shimmie2", "rule34hentai", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + "#urls": "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpg", + "#sha1_content": "6c23780bb78673cbff1bca9accb77ea11ec734f3", + "extension": "jpg", + "file_url": "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpg", + "filename": "264 - Darkstalkers Felicia mizuki_kotora", + "height": 875, + "id": 264, + "md5": "1a8eca7c04f8bf325bc993c5751a91c4", + "size": 0, + "tags": "Darkstalkers Felicia mizuki_kotora", + "width": 657, + }, ) diff --git a/test/results/rule34us.py b/test/results/rule34us.py index c3e193bab..2c620fa30 100644 --- a/test/results/rule34us.py +++ b/test/results/rule34us.py @@ -1,47 +1,40 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import rule34us - __tests__ = ( -{ - "#url" : "https://rule34.us/index.php?r=posts/index&q=[terios]_elysion", - "#category": ("booru", "rule34us", "tag"), - "#class" : rule34us.Rule34usTagExtractor, - "#pattern" : r"https://img\d*\.rule34\.us/images/../../[0-9a-f]{32}\.\w+", - "#count" : 10, -}, - -{ - "#url" : "https://rule34.us/index.php?r=posts/view&id=3709005", - "#category": ("booru", "rule34us", "post"), - "#class" : rule34us.Rule34usPostExtractor, - "#pattern" : r"https://img\d*\.rule34\.us/images/14/7b/147bee6fc2e13f73f5f9bac9d4930b13\.png", - "#sha1_content": "d714342ea84050f82dda5f0c194d677337abafc5", -}, - -{ - "#url" : "https://rule34.us/index.php?r=posts/view&id=4576310", - "#category": ("booru", "rule34us", "post"), - "#class" : rule34us.Rule34usPostExtractor, - "#pattern" : r"https://video-cdn\d\.rule34\.us/images/a2/94/a294ff8e1f8e0efa041e5dc9d1480011\.mp4", - - "extension" : "mp4", - "file_url" : str, - "filename" : "a294ff8e1f8e0efa041e5dc9d1480011", - "height" : "3982", - "id" : "4576310", - "md5" : "a294ff8e1f8e0efa041e5dc9d1480011", - "score" : r"re:\d+", - "tags" : "tagme, video", - "tags_general" : "video", - "tags_metadata": "tagme", - "uploader" : "Anonymous", - "width" : "3184", -}, - + { + "#url": "https://rule34.us/index.php?r=posts/index&q=[terios]_elysion", + "#category": ("booru", "rule34us", "tag"), + "#class": rule34us.Rule34usTagExtractor, + "#pattern": r"https://img\d*\.rule34\.us/images/../../[0-9a-f]{32}\.\w+", + "#count": 10, + }, + { + "#url": "https://rule34.us/index.php?r=posts/view&id=3709005", + "#category": ("booru", "rule34us", "post"), + "#class": rule34us.Rule34usPostExtractor, + "#pattern": r"https://img\d*\.rule34\.us/images/14/7b/147bee6fc2e13f73f5f9bac9d4930b13\.png", + "#sha1_content": "d714342ea84050f82dda5f0c194d677337abafc5", + }, + { + "#url": "https://rule34.us/index.php?r=posts/view&id=4576310", + "#category": ("booru", "rule34us", "post"), + "#class": rule34us.Rule34usPostExtractor, + "#pattern": r"https://video-cdn\d\.rule34\.us/images/a2/94/a294ff8e1f8e0efa041e5dc9d1480011\.mp4", + "extension": "mp4", + "file_url": str, + "filename": "a294ff8e1f8e0efa041e5dc9d1480011", + "height": "3982", + "id": "4576310", + "md5": "a294ff8e1f8e0efa041e5dc9d1480011", + "score": r"re:\d+", + "tags": "tagme, video", + "tags_general": "video", + "tags_metadata": "tagme", + "uploader": "Anonymous", + "width": "3184", + }, ) diff --git a/test/results/rule34vault.py b/test/results/rule34vault.py index 425ef7ed8..c52f14253 100644 --- a/test/results/rule34vault.py +++ b/test/results/rule34vault.py @@ -1,90 +1,82 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import rule34vault - __tests__ = ( -{ - "#url" : "https://rule34vault.com/sfw", - "#class": rule34vault.Rule34vaultTagExtractor, - "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://rule34vault.com/playlists/view/20164", - "#class": rule34vault.Rule34vaultPlaylistExtractor, - "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", - "#count" : 55, -}, - -{ - "#url" : "https://rule34vault.com/post/280517", - "#comment": "image", - "#class" : rule34vault.Rule34vaultPostExtractor, - "#options": {"tags": True}, - "#pattern" : "https://r34xyz.b-cdn.net/posts/280/280517/280517.jpg", - "#sha1_content": "1e19d601b4a79c06e6f885a83a5003e7e2a17057", - - "created" : "2023-09-01T11:57:57.317331Z", - "date" : "dt:2023-09-01 11:57:57", - "extension" : "jpg", - "file_url" : "https://r34xyz.b-cdn.net/posts/280/280517/280517.jpg", - "filename" : "280517", - "height" : 1152, - "id" : 280517, - "likes" : range(3, 100), - "posted" : "2023-09-01T12:01:41.008547Z", - "status" : 2, - "type" : 0, - "uploaderId": 20678, - "views" : range(90, 999), - "width" : 768, - "data": { - "sources": [ - "https://trynectar.ai/view/87c98fc8-e4f3-447c-a0d3-024b1890580a", + { + "#url": "https://rule34vault.com/sfw", + "#class": rule34vault.Rule34vaultTagExtractor, + "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://rule34vault.com/playlists/view/20164", + "#class": rule34vault.Rule34vaultPlaylistExtractor, + "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", + "#count": 55, + }, + { + "#url": "https://rule34vault.com/post/280517", + "#comment": "image", + "#class": rule34vault.Rule34vaultPostExtractor, + "#options": {"tags": True}, + "#pattern": "https://r34xyz.b-cdn.net/posts/280/280517/280517.jpg", + "#sha1_content": "1e19d601b4a79c06e6f885a83a5003e7e2a17057", + "created": "2023-09-01T11:57:57.317331Z", + "date": "dt:2023-09-01 11:57:57", + "extension": "jpg", + "file_url": "https://r34xyz.b-cdn.net/posts/280/280517/280517.jpg", + "filename": "280517", + "height": 1152, + "id": 280517, + "likes": range(3, 100), + "posted": "2023-09-01T12:01:41.008547Z", + "status": 2, + "type": 0, + "uploaderId": 20678, + "views": range(90, 999), + "width": 768, + "data": { + "sources": [ + "https://trynectar.ai/view/87c98fc8-e4f3-447c-a0d3-024b1890580a", + ], + }, + "tags": [ + "ai generated", + "demon slayer", + "kamado nezuko", + "school uniform", + "sfw", + ], + "tags_character": [ + "kamado nezuko", + ], + "tags_copyright": [ + "demon slayer", ], + "tags_general": [ + "ai generated", + "school uniform", + "sfw", + ], + "uploader": { + "created": "2023-07-24T04:33:36.734495Z", + "data": None, + "displayName": "quick1e", + "emailVerified": False, + "id": 20678, + "role": 1, + "userName": "quick1e", + }, }, - "tags": [ - "ai generated", - "demon slayer", - "kamado nezuko", - "school uniform", - "sfw", - ], - "tags_character": [ - "kamado nezuko", - ], - "tags_copyright": [ - "demon slayer", - ], - "tags_general": [ - "ai generated", - "school uniform", - "sfw", - ], - "uploader": { - "created" : "2023-07-24T04:33:36.734495Z", - "data" : None, - "displayName" : "quick1e", - "emailVerified": False, - "id" : 20678, - "role" : 1, - "userName" : "quick1e", + { + "#url": "https://rule34vault.com/post/382937", + "#comment": "video", + "#class": rule34vault.Rule34vaultPostExtractor, + "#urls": "https://r34xyz.b-cdn.net/posts/382/382937/382937.mp4", + "#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af", }, -}, - -{ - "#url" : "https://rule34vault.com/post/382937", - "#comment": "video", - "#class" : rule34vault.Rule34vaultPostExtractor, - "#urls" : "https://r34xyz.b-cdn.net/posts/382/382937/382937.mp4", - "#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af", -}, - ) diff --git a/test/results/rule34xyz.py b/test/results/rule34xyz.py index 8a07c5dc3..288df66c7 100644 --- a/test/results/rule34xyz.py +++ b/test/results/rule34xyz.py @@ -1,142 +1,129 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import rule34xyz - __tests__ = ( -{ - "#url" : "https://rule34.xyz/sfw", - "#class": rule34xyz.Rule34xyzTagExtractor, - "#pattern": r"https://rule34(\.xyz|xyz\.b-cdn\.net)/posts/\d+/\d+/\d+\.(pic|mov\d*)\.(jpg|mp4)", - "#range" : "1-150", - "#count" : 150, - - "search_tags": "sfw", -}, - -{ - "#url" : "https://rule34.xyz/playlists/view/119", - "#class": rule34xyz.Rule34xyzPlaylistExtractor, - "#pattern": r"https://rule34(\.xyz|xyz\.b-cdn\.net)/posts/\d+/\d+/\d+\.(pic|mov\d*)\.(jpg|mp4)", - "#count" : 64, - - "playlist_id": "119", -}, - -{ - "#url" : "https://rule34.xyz/post/3613851", - "#comment": "image", - "#class" : rule34xyz.Rule34xyzPostExtractor, - "#options" : {"tags": True}, - "#urls" : "https://rule34xyz.b-cdn.net/posts/3613/3613851/3613851.pic.jpg", - "#sha1_content": "4d7146db258fd5b1645a1a5fc01550d102f495e1", - - "attributes": 1, - "comments" : 0, - "created" : "2023-03-29T06:00:59.136819", - "date" : "dt:2023-03-29 06:00:59", - "duration" : None, - "error" : None, - "extension" : "jpg", - "file_url" : "https://rule34xyz.b-cdn.net/posts/3613/3613851/3613851.pic.jpg", - "filename" : "3613851.pic", - "format" : "pic", - "format_id" : "2", - "id" : 3613851, - "likes" : range(3, 100), - "posted" : "2023-03-29T06:01:07.900161", - "type" : 0, - "uploaderId": 9741, - "views" : range(200, 2000), - "status" : 2, - "files" : dict, - "sources": [ - "https://twitter.com/DesireDelta13/status/1636502494292373505?t=OrmlnC85cELyY5BPmBy9Hw&s=19", - ], - "tags": [ - "doki doki literature club", - "doki doki takeover", - "friday night funkin", - "friday night funkin mod", - "yuri (doki doki literature club)", - "desiredelta", - "1girls", - "big breasts", - "clothed", - "clothed female", - "female", - "female focus", - "female only", - "holding microphone", - "holding object", - "long hair", - "long purple hair", - "looking at viewer", - "microphone", - "open hand", - "open mouth", - "purple background", - "purple hair", - "solo", - "solo female", - "solo focus", - "sweater", - "white outline", - "jpeg", - "safe for work", - "sfw", - ], - "tags_artist": [ - "desiredelta", - ], - "tags_character": [ - "yuri (doki doki literature club)", - ], - "tags_copyright": [ - "doki doki literature club", - "friday night funkin", - "friday night funkin mod", - ], - "tags_general": list, - "uploader": { - "avatarUrl" : None, - "bookmarks" : 0, - "certified" : True, - "created" : "2021-04-03T08:29:51.373823", - "email" : "agent.rulexxx-uploader@z.com", - "id" : 9741, - "isSystemAccount": True, - "name" : "agent.rulexxx-uploader", - "role" : 2, - "uploadedPosts" : range(100000, 999999), - "webId" : None, + { + "#url": "https://rule34.xyz/sfw", + "#class": rule34xyz.Rule34xyzTagExtractor, + "#pattern": r"https://rule34(\.xyz|xyz\.b-cdn\.net)/posts/\d+/\d+/\d+\.(pic|mov\d*)\.(jpg|mp4)", + "#range": "1-150", + "#count": 150, + "search_tags": "sfw", + }, + { + "#url": "https://rule34.xyz/playlists/view/119", + "#class": rule34xyz.Rule34xyzPlaylistExtractor, + "#pattern": r"https://rule34(\.xyz|xyz\.b-cdn\.net)/posts/\d+/\d+/\d+\.(pic|mov\d*)\.(jpg|mp4)", + "#count": 64, + "playlist_id": "119", + }, + { + "#url": "https://rule34.xyz/post/3613851", + "#comment": "image", + "#class": rule34xyz.Rule34xyzPostExtractor, + "#options": {"tags": True}, + "#urls": "https://rule34xyz.b-cdn.net/posts/3613/3613851/3613851.pic.jpg", + "#sha1_content": "4d7146db258fd5b1645a1a5fc01550d102f495e1", + "attributes": 1, + "comments": 0, + "created": "2023-03-29T06:00:59.136819", + "date": "dt:2023-03-29 06:00:59", + "duration": None, + "error": None, + "extension": "jpg", + "file_url": "https://rule34xyz.b-cdn.net/posts/3613/3613851/3613851.pic.jpg", + "filename": "3613851.pic", + "format": "pic", + "format_id": "2", + "id": 3613851, + "likes": range(3, 100), + "posted": "2023-03-29T06:01:07.900161", + "type": 0, + "uploaderId": 9741, + "views": range(200, 2000), + "status": 2, + "files": dict, + "sources": [ + "https://twitter.com/DesireDelta13/status/1636502494292373505?t=OrmlnC85cELyY5BPmBy9Hw&s=19", + ], + "tags": [ + "doki doki literature club", + "doki doki takeover", + "friday night funkin", + "friday night funkin mod", + "yuri (doki doki literature club)", + "desiredelta", + "1girls", + "big breasts", + "clothed", + "clothed female", + "female", + "female focus", + "female only", + "holding microphone", + "holding object", + "long hair", + "long purple hair", + "looking at viewer", + "microphone", + "open hand", + "open mouth", + "purple background", + "purple hair", + "solo", + "solo female", + "solo focus", + "sweater", + "white outline", + "jpeg", + "safe for work", + "sfw", + ], + "tags_artist": [ + "desiredelta", + ], + "tags_character": [ + "yuri (doki doki literature club)", + ], + "tags_copyright": [ + "doki doki literature club", + "friday night funkin", + "friday night funkin mod", + ], + "tags_general": list, + "uploader": { + "avatarUrl": None, + "bookmarks": 0, + "certified": True, + "created": "2021-04-03T08:29:51.373823", + "email": "agent.rulexxx-uploader@z.com", + "id": 9741, + "isSystemAccount": True, + "name": "agent.rulexxx-uploader", + "role": 2, + "uploadedPosts": range(100000, 999999), + "webId": None, + }, + }, + { + "#url": "https://rule34.xyz/post/3571567", + "#comment": "video", + "#class": rule34xyz.Rule34xyzPostExtractor, + "#urls": "https://rule34xyz.b-cdn.net/posts/3571/3571567/3571567.mov720.mp4", + "#sha1_content": "c0a5e7e887774f91527f00e6142c435a3c482c1f", + "format": "mov720", + "format_id": "40", + }, + { + "#url": "https://rule34.xyz/post/3571567", + "#comment": "'format' option", + "#class": rule34xyz.Rule34xyzPostExtractor, + "#options": {"format": "10,33"}, + "#urls": "https://rule34xyz.b-cdn.net/posts/3571/3571567/3571567.pic256avif.avif", + "format": "pic256avif", + "format_id": "33", }, -}, - -{ - "#url" : "https://rule34.xyz/post/3571567", - "#comment": "video", - "#class" : rule34xyz.Rule34xyzPostExtractor, - "#urls" : "https://rule34xyz.b-cdn.net/posts/3571/3571567/3571567.mov720.mp4", - "#sha1_content": "c0a5e7e887774f91527f00e6142c435a3c482c1f", - - "format" : "mov720", - "format_id" : "40", -}, - -{ - "#url" : "https://rule34.xyz/post/3571567", - "#comment": "'format' option", - "#class" : rule34xyz.Rule34xyzPostExtractor, - "#options": {"format": "10,33"}, - "#urls" : "https://rule34xyz.b-cdn.net/posts/3571/3571567/3571567.pic256avif.avif", - - "format" : "pic256avif", - "format_id" : "33", -}, - ) diff --git a/test/results/safebooru.py b/test/results/safebooru.py index ec92cc073..662056eb6 100644 --- a/test/results/safebooru.py +++ b/test/results/safebooru.py @@ -1,61 +1,51 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v02 - __tests__ = ( -{ - "#url" : "https://safebooru.org/index.php?page=post&s=list&tags=bonocho", - "#category": ("gelbooru_v02", "safebooru", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#sha1_url" : "17c61b386530cf4c30842c9f580d15ef1cd09586", - "#sha1_content": "e5ad4c5bf241b1def154958535bef6c2f6b733eb", -}, - -{ - "#url" : "https://safebooru.org/index.php?page=post&s=list&tags=all", - "#category": ("gelbooru_v02", "safebooru", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#range" : "1-3", - "#count" : 3, -}, - -{ - "#url" : "https://safebooru.org/index.php?page=post&s=list&tags=", - "#category": ("gelbooru_v02", "safebooru", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, -}, - -{ - "#url" : "https://safebooru.org/index.php?page=pool&s=show&id=11", - "#category": ("gelbooru_v02", "safebooru", "pool"), - "#class" : gelbooru_v02.GelbooruV02PoolExtractor, - "#count" : 5, -}, - -{ - "#url" : "https://safebooru.org/index.php?page=favorites&s=view&id=17567", - "#category": ("gelbooru_v02", "safebooru", "favorite"), - "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://safebooru.org/index.php?page=post&s=view&id=1169132", - "#category": ("gelbooru_v02", "safebooru", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#options" : {"tags": True}, - "#sha1_url" : "cf05e37a3c62b2d55788e2080b8eabedb00f999b", - "#sha1_content": "93b293b27dabd198afafabbaf87c49863ac82f27", - - "tags_artist" : "kawanakajima", - "tags_character": "heath_ledger ronald_mcdonald the_joker", - "tags_copyright": "dc_comics mcdonald's the_dark_knight", - "tags_metadata" : "parody tagme", -}, - + { + "#url": "https://safebooru.org/index.php?page=post&s=list&tags=bonocho", + "#category": ("gelbooru_v02", "safebooru", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#sha1_url": "17c61b386530cf4c30842c9f580d15ef1cd09586", + "#sha1_content": "e5ad4c5bf241b1def154958535bef6c2f6b733eb", + }, + { + "#url": "https://safebooru.org/index.php?page=post&s=list&tags=all", + "#category": ("gelbooru_v02", "safebooru", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#range": "1-3", + "#count": 3, + }, + { + "#url": "https://safebooru.org/index.php?page=post&s=list&tags=", + "#category": ("gelbooru_v02", "safebooru", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + }, + { + "#url": "https://safebooru.org/index.php?page=pool&s=show&id=11", + "#category": ("gelbooru_v02", "safebooru", "pool"), + "#class": gelbooru_v02.GelbooruV02PoolExtractor, + "#count": 5, + }, + { + "#url": "https://safebooru.org/index.php?page=favorites&s=view&id=17567", + "#category": ("gelbooru_v02", "safebooru", "favorite"), + "#class": gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count": 2, + }, + { + "#url": "https://safebooru.org/index.php?page=post&s=view&id=1169132", + "#category": ("gelbooru_v02", "safebooru", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#options": {"tags": True}, + "#sha1_url": "cf05e37a3c62b2d55788e2080b8eabedb00f999b", + "#sha1_content": "93b293b27dabd198afafabbaf87c49863ac82f27", + "tags_artist": "kawanakajima", + "tags_character": "heath_ledger ronald_mcdonald the_joker", + "tags_copyright": "dc_comics mcdonald's the_dark_knight", + "tags_metadata": "parody tagme", + }, ) diff --git a/test/results/saint.py b/test/results/saint.py index 7f14b317e..0fef239a0 100644 --- a/test/results/saint.py +++ b/test/results/saint.py @@ -1,83 +1,73 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import saint - __tests__ = ( -{ - "#url" : "https://saint2.su/a/2c5iuWHTumH", - "#class": saint.SaintAlbumExtractor, - "#urls" : ( - "https://cold1.saint2.cr/videos/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", - "https://cold1.saint2.cr/videos/3b125e3fb4b98693f17d85cb53590215.mp4", - ), - - "album_id" : "2c5iuWHTumH", - "album_name" : "animations", - "album_size" : 37083862, - "count" : 2, - "date" : "type:datetime", - "description": "Descriptions can contain only alphanumeric ASCII characters", - "extension" : "mp4", - "file" : r"re:https://...", - "filename" : {"3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", - "3b125e3fb4b98693f17d85cb53590215-ze10Ohbpoy5"}, - "id" : {"6lC7mKrJst8", - "ze10Ohbpoy5"}, - "id2" : {"6712834015d67", - "671284a627e0e"}, - "id_dl" : {"M2IxY2NlYmYzNTc2ZjhkNWFhYzNlZTBlNWExMmRhOTUubXA0", - "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0"}, - "name" : {"3b1ccebf3576f8d5aac3ee0e5a12da95", - "3b125e3fb4b98693f17d85cb53590215"}, - "num" : {1, 2}, -}, - -{ - "#url" : "https://saint2.su/embed/6lC7mKrJst8", - "#class": saint.SaintMediaExtractor, - "#urls" : "https://cold1.saint2.cr/videos/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", - "#sha1_content": "39037a029b3fe96f838b4545316caaa545c84075", - - "count" : 1, - "date" : "dt:2024-10-18 15:48:16", - "extension": "mp4", - "file" : "https://cold1.saint2.cr/videos/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", - "filename" : "3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", - "id" : "6lC7mKrJst8", - "id2" : "6712834015d67", - "id_dl" : "M2IxY2NlYmYzNTc2ZjhkNWFhYzNlZTBlNWExMmRhOTUubXA0", - "name" : "3b1ccebf3576f8d5aac3ee0e5a12da95", - "num" : 1, -}, - -{ - "#url" : "https://saint2.su/d/M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "#class": saint.SaintMediaExtractor, - "#urls" : "https://cold1.saint2.cr/api/download.php?file=M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - - "count" : 1, - "extension": "mp4", - "file" : "https://cold1.saint2.cr/api/download.php?file=M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "filename" : "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "id" : "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "id_dl" : "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "name" : "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "num" : 1, -}, - -{ - "#url" : "https://saint2.pk/embed/6lC7mKrJst8", - "#class": saint.SaintMediaExtractor, -}, - -{ - "#url" : "https://saint.to/embed/6lC7mKrJst8", - "#class": saint.SaintMediaExtractor, -}, - + { + "#url": "https://saint2.su/a/2c5iuWHTumH", + "#class": saint.SaintAlbumExtractor, + "#urls": ( + "https://cold1.saint2.cr/videos/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", + "https://cold1.saint2.cr/videos/3b125e3fb4b98693f17d85cb53590215.mp4", + ), + "album_id": "2c5iuWHTumH", + "album_name": "animations", + "album_size": 37083862, + "count": 2, + "date": "type:datetime", + "description": "Descriptions can contain only alphanumeric ASCII characters", + "extension": "mp4", + "file": r"re:https://...", + "filename": { + "3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", + "3b125e3fb4b98693f17d85cb53590215-ze10Ohbpoy5", + }, + "id": {"6lC7mKrJst8", "ze10Ohbpoy5"}, + "id2": {"6712834015d67", "671284a627e0e"}, + "id_dl": { + "M2IxY2NlYmYzNTc2ZjhkNWFhYzNlZTBlNWExMmRhOTUubXA0", + "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + }, + "name": {"3b1ccebf3576f8d5aac3ee0e5a12da95", "3b125e3fb4b98693f17d85cb53590215"}, + "num": {1, 2}, + }, + { + "#url": "https://saint2.su/embed/6lC7mKrJst8", + "#class": saint.SaintMediaExtractor, + "#urls": "https://cold1.saint2.cr/videos/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", + "#sha1_content": "39037a029b3fe96f838b4545316caaa545c84075", + "count": 1, + "date": "dt:2024-10-18 15:48:16", + "extension": "mp4", + "file": "https://cold1.saint2.cr/videos/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", + "filename": "3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", + "id": "6lC7mKrJst8", + "id2": "6712834015d67", + "id_dl": "M2IxY2NlYmYzNTc2ZjhkNWFhYzNlZTBlNWExMmRhOTUubXA0", + "name": "3b1ccebf3576f8d5aac3ee0e5a12da95", + "num": 1, + }, + { + "#url": "https://saint2.su/d/M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "#class": saint.SaintMediaExtractor, + "#urls": "https://cold1.saint2.cr/api/download.php?file=M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "count": 1, + "extension": "mp4", + "file": "https://cold1.saint2.cr/api/download.php?file=M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "filename": "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "id": "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "id_dl": "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "name": "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "num": 1, + }, + { + "#url": "https://saint2.pk/embed/6lC7mKrJst8", + "#class": saint.SaintMediaExtractor, + }, + { + "#url": "https://saint.to/embed/6lC7mKrJst8", + "#class": saint.SaintMediaExtractor, + }, ) diff --git a/test/results/sakugabooru.py b/test/results/sakugabooru.py index 73d383168..5e6568bf0 100644 --- a/test/results/sakugabooru.py +++ b/test/results/sakugabooru.py @@ -1,35 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import moebooru - __tests__ = ( -{ - "#url" : "https://www.sakugabooru.com/post/show/125570", - "#category": ("moebooru", "sakugabooru", "post"), - "#class" : moebooru.MoebooruPostExtractor, -}, - -{ - "#url" : "https://www.sakugabooru.com/post?tags=nichijou", - "#category": ("moebooru", "sakugabooru", "tag"), - "#class" : moebooru.MoebooruTagExtractor, -}, - -{ - "#url" : "https://www.sakugabooru.com/pool/show/54", - "#category": ("moebooru", "sakugabooru", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, -}, - -{ - "#url" : "https://www.sakugabooru.com/post/popular_recent", - "#category": ("moebooru", "sakugabooru", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, -}, - + { + "#url": "https://www.sakugabooru.com/post/show/125570", + "#category": ("moebooru", "sakugabooru", "post"), + "#class": moebooru.MoebooruPostExtractor, + }, + { + "#url": "https://www.sakugabooru.com/post?tags=nichijou", + "#category": ("moebooru", "sakugabooru", "tag"), + "#class": moebooru.MoebooruTagExtractor, + }, + { + "#url": "https://www.sakugabooru.com/pool/show/54", + "#category": ("moebooru", "sakugabooru", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + }, + { + "#url": "https://www.sakugabooru.com/post/popular_recent", + "#category": ("moebooru", "sakugabooru", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + }, ) diff --git a/test/results/sankaku.py b/test/results/sankaku.py index e19ea67be..6f31b52ea 100644 --- a/test/results/sankaku.py +++ b/test/results/sankaku.py @@ -1,309 +1,266 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import sankaku from gallery_dl import exception - +from gallery_dl.extractor import sankaku __tests__ = ( -{ - "#url" : "https://sankaku.app/?tags=bonocho", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, - "#pattern" : r"https://s\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[0-9a-f]{32}\.\w+\?e=\d+&(expires=\d+&)?m=[^&#]+", - "#count" : 5, -}, - -{ - "#url" : "https://www.sankakucomplex.com/?tags=bonocho", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://beta.sankakucomplex.com/?tags=bonocho", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/?tags=bonocho", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://black.sankakucomplex.com/?tags=bonocho", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://white.sankakucomplex.com/?tags=bonocho", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://sankaku.app/ja?tags=order%3Apopularity", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://sankaku.app/no/?tags=order%3Apopularity", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/posts?tags=TAG", - "#comment" : "'/posts' in tag search URL (#4740)", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/ja/posts/?tags=あえいおう", - "#comment" : "'/posts' in tag search URL (#4740)", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/?tags=bonocho+a+b+c+d", - "#comment" : "error on five or more tags", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, - "#options" : {"username": None}, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/?tags=marie_rose&page=98&next=3874906&commit=Search", - "#comment" : "match arbitrary query parameters", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/?tags=date:2023-03-20", - "#comment" : "'date:' tags (#1790)", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, - "#range" : "1", - "#count" : 1, -}, - -{ - "#url" : "https://sankaku.app/books/90", - "#category": ("booru", "sankaku", "pool"), - "#class" : sankaku.SankakuPoolExtractor, - "#count" : 5, -}, - -{ - "#url" : "https://www.sankakucomplex.com/books/90", - "#category": ("booru", "sankaku", "pool"), - "#class" : sankaku.SankakuPoolExtractor, -}, - -{ - "#url" : "https://beta.sankakucomplex.com/books/90", - "#category": ("booru", "sankaku", "pool"), - "#class" : sankaku.SankakuPoolExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/pool/show/90", - "#category": ("booru", "sankaku", "pool"), - "#class" : sankaku.SankakuPoolExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/pools/show/90", - "#category": ("booru", "sankaku", "pool"), - "#class" : sankaku.SankakuPoolExtractor, -}, - -{ - "#url" : "https://sankaku.app/posts/y0abGlDOr2o", - "#comment" : "extended tag categories; alphanumeric ID (#5073)", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#options" : { - "tags" : True, - "notes" : True, - "id-format": "alphanumeric", - }, - "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", - - "id": "y0abGlDOr2o", - "notes": (), - "tags_artist": [ - "bonocho", - ], - "tags_character": [ - "batman", - "letty_whiterock", - "bruce_wayne", - "the_joker", - "heath_ledger", - ], - "tags_copyright": [ - "batman_(series)", - "the_dark_knight", - ], - "tags_studio": [ - "dc_comics", - ], - "tags_general": list, -}, - -{ - "#url" : "https://sankaku.app/posts/VAr2mjLJ2av", - "#comment" : "notes (#5073)", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#options" : {"notes": True}, - - "notes": [ - { - "body" : "A lonely person, is a lonely person, because he or she is lonely.", - "created_at": 1643733759, - "creator_id": 1370766, - "height" : 871, - "id" : 1832643, - "is_active" : True, - "post_id" : 23688624, - "updated_at": 1643733759, - "width" : 108, - "x" : 703, - "y" : 83, + { + "#url": "https://sankaku.app/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + "#pattern": r"https://s\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[0-9a-f]{32}\.\w+\?e=\d+&(expires=\d+&)?m=[^&#]+", + "#count": 5, + }, + { + "#url": "https://www.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://beta.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://black.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://white.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://sankaku.app/ja?tags=order%3Apopularity", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://sankaku.app/no/?tags=order%3Apopularity", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/posts?tags=TAG", + "#comment": "'/posts' in tag search URL (#4740)", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/ja/posts/?tags=あえいおう", + "#comment": "'/posts' in tag search URL (#4740)", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/?tags=bonocho+a+b+c+d", + "#comment": "error on five or more tags", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + "#options": {"username": None}, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://chan.sankakucomplex.com/?tags=marie_rose&page=98&next=3874906&commit=Search", + "#comment": "match arbitrary query parameters", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/?tags=date:2023-03-20", + "#comment": "'date:' tags (#1790)", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + "#range": "1", + "#count": 1, + }, + { + "#url": "https://sankaku.app/books/90", + "#category": ("booru", "sankaku", "pool"), + "#class": sankaku.SankakuPoolExtractor, + "#count": 5, + }, + { + "#url": "https://www.sankakucomplex.com/books/90", + "#category": ("booru", "sankaku", "pool"), + "#class": sankaku.SankakuPoolExtractor, + }, + { + "#url": "https://beta.sankakucomplex.com/books/90", + "#category": ("booru", "sankaku", "pool"), + "#class": sankaku.SankakuPoolExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/pool/show/90", + "#category": ("booru", "sankaku", "pool"), + "#class": sankaku.SankakuPoolExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/pools/show/90", + "#category": ("booru", "sankaku", "pool"), + "#class": sankaku.SankakuPoolExtractor, + }, + { + "#url": "https://sankaku.app/posts/y0abGlDOr2o", + "#comment": "extended tag categories; alphanumeric ID (#5073)", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#options": { + "tags": True, + "notes": True, + "id-format": "alphanumeric", }, - ], -}, - -{ - "#url" : "https://sankaku.app/post/show/360451", - "#comment" : "legacy post URL", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#pattern" : r"https://s\.sankakucomplex\.com/data/ac/8e/ac8e3b92ea328ce9cf7211e69c905bf9\.jpg\?e=.+", - - "id": 360451, -}, - -{ - "#url" : "https://sankaku.app/post/show/21418978", - "#comment" : "'contentious_content'", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#auth" : True, - "#pattern" : r"https://s\.sankakucomplex\.com/data/13/3c/133cda3bfde249c504284493903fb985\.jpg", -}, - -{ - "#url" : "https://sankaku.app/post/show/20758561", - "#comment" : "empty tags (#1617)", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#options" : {"tags": True}, - "#count" : 1, - - "id" : 20758561, - "tags" : list, - "tags_general": [ - "key(mangaka)", - "key(mangaka)", - "english_language", - "english_language", - "high_resolution", - "tagme", - "very_high_resolution", - "large_filesize", - ], -}, - -{ - "#url" : "https://chan.sankakucomplex.com/post/show/f8ba89043078f0e4be2d9c46550b840a", - "#comment" : "md5 hexdigest instead of ID (#3952)", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#pattern" : r"https://s\.sankakucomplex\.com/data/f8/ba/f8ba89043078f0e4be2d9c46550b840a\.jpg", - "#count" : 1, - - "id" : 33195194, - "md5": "f8ba89043078f0e4be2d9c46550b840a", -}, - -{ - "#url" : "https://chan.sankakucomplex.com/posts/f8ba89043078f0e4be2d9c46550b840a", - "#comment" : "/posts/ instead of /post/show/ (#4688)", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#pattern" : r"https://s\.sankakucomplex\.com/data/f8/ba/f8ba89043078f0e4be2d9c46550b840a\.jpg", - "#count" : 1, - - "id" : 33195194, - "md5": "f8ba89043078f0e4be2d9c46550b840a", -}, - -{ - "#url" : "https://chan.sankakucomplex.com/en/posts/show/ac8e3b92ea328ce9cf7211e69c905bf9", - "#comment" : "/en/posts/show/HEX", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - - "id" : 360451, - "md5": "ac8e3b92ea328ce9cf7211e69c905bf9", -}, - -{ - "#url" : "https://chan.sankakucomplex.com/post/show/360451", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/ja/post/show/360451", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, -}, - -{ - "#url" : "https://beta.sankakucomplex.com/post/show/360451", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, -}, - -{ - "#url" : "https://white.sankakucomplex.com/post/show/360451", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, -}, - -{ - "#url" : "https://black.sankakucomplex.com/post/show/360451", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, -}, - -{ - "#url" : "https://sankaku.app/books?tags=aiue_oka", - "#category": ("booru", "sankaku", "books"), - "#class" : sankaku.SankakuBooksExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://beta.sankakucomplex.com/books?tags=aiue_oka", - "#category": ("booru", "sankaku", "books"), - "#class" : sankaku.SankakuBooksExtractor, -}, - + "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", + "id": "y0abGlDOr2o", + "notes": (), + "tags_artist": [ + "bonocho", + ], + "tags_character": [ + "batman", + "letty_whiterock", + "bruce_wayne", + "the_joker", + "heath_ledger", + ], + "tags_copyright": [ + "batman_(series)", + "the_dark_knight", + ], + "tags_studio": [ + "dc_comics", + ], + "tags_general": list, + }, + { + "#url": "https://sankaku.app/posts/VAr2mjLJ2av", + "#comment": "notes (#5073)", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#options": {"notes": True}, + "notes": [ + { + "body": "A lonely person, is a lonely person, because he or she is lonely.", + "created_at": 1643733759, + "creator_id": 1370766, + "height": 871, + "id": 1832643, + "is_active": True, + "post_id": 23688624, + "updated_at": 1643733759, + "width": 108, + "x": 703, + "y": 83, + }, + ], + }, + { + "#url": "https://sankaku.app/post/show/360451", + "#comment": "legacy post URL", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#pattern": r"https://s\.sankakucomplex\.com/data/ac/8e/ac8e3b92ea328ce9cf7211e69c905bf9\.jpg\?e=.+", + "id": 360451, + }, + { + "#url": "https://sankaku.app/post/show/21418978", + "#comment": "'contentious_content'", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#auth": True, + "#pattern": r"https://s\.sankakucomplex\.com/data/13/3c/133cda3bfde249c504284493903fb985\.jpg", + }, + { + "#url": "https://sankaku.app/post/show/20758561", + "#comment": "empty tags (#1617)", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#options": {"tags": True}, + "#count": 1, + "id": 20758561, + "tags": list, + "tags_general": [ + "key(mangaka)", + "key(mangaka)", + "english_language", + "english_language", + "high_resolution", + "tagme", + "very_high_resolution", + "large_filesize", + ], + }, + { + "#url": "https://chan.sankakucomplex.com/post/show/f8ba89043078f0e4be2d9c46550b840a", + "#comment": "md5 hexdigest instead of ID (#3952)", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#pattern": r"https://s\.sankakucomplex\.com/data/f8/ba/f8ba89043078f0e4be2d9c46550b840a\.jpg", + "#count": 1, + "id": 33195194, + "md5": "f8ba89043078f0e4be2d9c46550b840a", + }, + { + "#url": "https://chan.sankakucomplex.com/posts/f8ba89043078f0e4be2d9c46550b840a", + "#comment": "/posts/ instead of /post/show/ (#4688)", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#pattern": r"https://s\.sankakucomplex\.com/data/f8/ba/f8ba89043078f0e4be2d9c46550b840a\.jpg", + "#count": 1, + "id": 33195194, + "md5": "f8ba89043078f0e4be2d9c46550b840a", + }, + { + "#url": "https://chan.sankakucomplex.com/en/posts/show/ac8e3b92ea328ce9cf7211e69c905bf9", + "#comment": "/en/posts/show/HEX", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "id": 360451, + "md5": "ac8e3b92ea328ce9cf7211e69c905bf9", + }, + { + "#url": "https://chan.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/ja/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + }, + { + "#url": "https://beta.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + }, + { + "#url": "https://white.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + }, + { + "#url": "https://black.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + }, + { + "#url": "https://sankaku.app/books?tags=aiue_oka", + "#category": ("booru", "sankaku", "books"), + "#class": sankaku.SankakuBooksExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://beta.sankakucomplex.com/books?tags=aiue_oka", + "#category": ("booru", "sankaku", "books"), + "#class": sankaku.SankakuBooksExtractor, + }, ) diff --git a/test/results/sankakucomplex.py b/test/results/sankakucomplex.py index bcf44d958..d67343ab6 100644 --- a/test/results/sankakucomplex.py +++ b/test/results/sankakucomplex.py @@ -1,69 +1,59 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import sankakucomplex - __tests__ = ( -{ - "#url" : "https://news.sankakucomplex.com/2019/05/11/twitter-cosplayers", - "#category": ("", "sankakucomplex", "article"), - "#class" : sankakucomplex.SankakucomplexArticleExtractor, - "#pattern" : r"https://news\.sankakucomplex\.com/wp-content/uploads/2019/05/maid-day-cosplay-\d+\.jpg", - "#sha1_metadata": "21bf106150913a1398860031f06d6e1e6423e518", -}, - -{ - "#url" : "https://www.sankakucomplex.com/2009/12/01/sexy-goddesses-of-2ch", - "#category": ("", "sankakucomplex", "article"), - "#class" : sankakucomplex.SankakucomplexArticleExtractor, - "#pattern" : r"https://news\.sankakucomplex\.com/wp-content/uploads/2009/12/Goddesses-of-2ch-amateur-internet-idol-\d+\.jpe?g", - "#sha1_metadata": "651e4ee79ecab1771b43df467b5ab32249d69b2a", -}, - -{ - "#url" : "https://www.sankakucomplex.com/2019/06/11/darling-ol-goddess-shows-off-her-plump-lower-area/", - "#comment" : "videos (#308)", - "#category": ("", "sankakucomplex", "article"), - "#class" : sankakucomplex.SankakucomplexArticleExtractor, - "#pattern" : r"/wp-content/uploads/2019/06/[^/]+\d\.mp4", - "#range" : "26-", - "#count" : 5, -}, - -{ - "#url" : "https://www.sankakucomplex.com/2015/02/12/snow-miku-2015-live-magical-indeed/", - "#comment" : "youtube embeds (#308)", - "#category": ("", "sankakucomplex", "article"), - "#class" : sankakucomplex.SankakucomplexArticleExtractor, - "#options" : {"embeds": True}, - "#pattern" : "https://www.youtube.com/embed/", - "#range" : "2-", - "#count" : 2, -}, - -{ - "#url" : "https://www.sankakucomplex.com/tag/cosplay/", - "#category": ("", "sankakucomplex", "tag"), - "#class" : sankakucomplex.SankakucomplexTagExtractor, - "#pattern" : sankakucomplex.SankakucomplexArticleExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://www.sankakucomplex.com/category/anime/", - "#category": ("", "sankakucomplex", "tag"), - "#class" : sankakucomplex.SankakucomplexTagExtractor, -}, - -{ - "#url" : "https://www.sankakucomplex.com/author/rift/page/5/", - "#category": ("", "sankakucomplex", "tag"), - "#class" : sankakucomplex.SankakucomplexTagExtractor, -}, - + { + "#url": "https://news.sankakucomplex.com/2019/05/11/twitter-cosplayers", + "#category": ("", "sankakucomplex", "article"), + "#class": sankakucomplex.SankakucomplexArticleExtractor, + "#pattern": r"https://news\.sankakucomplex\.com/wp-content/uploads/2019/05/maid-day-cosplay-\d+\.jpg", + "#sha1_metadata": "21bf106150913a1398860031f06d6e1e6423e518", + }, + { + "#url": "https://www.sankakucomplex.com/2009/12/01/sexy-goddesses-of-2ch", + "#category": ("", "sankakucomplex", "article"), + "#class": sankakucomplex.SankakucomplexArticleExtractor, + "#pattern": r"https://news\.sankakucomplex\.com/wp-content/uploads/2009/12/Goddesses-of-2ch-amateur-internet-idol-\d+\.jpe?g", + "#sha1_metadata": "651e4ee79ecab1771b43df467b5ab32249d69b2a", + }, + { + "#url": "https://www.sankakucomplex.com/2019/06/11/darling-ol-goddess-shows-off-her-plump-lower-area/", + "#comment": "videos (#308)", + "#category": ("", "sankakucomplex", "article"), + "#class": sankakucomplex.SankakucomplexArticleExtractor, + "#pattern": r"/wp-content/uploads/2019/06/[^/]+\d\.mp4", + "#range": "26-", + "#count": 5, + }, + { + "#url": "https://www.sankakucomplex.com/2015/02/12/snow-miku-2015-live-magical-indeed/", + "#comment": "youtube embeds (#308)", + "#category": ("", "sankakucomplex", "article"), + "#class": sankakucomplex.SankakucomplexArticleExtractor, + "#options": {"embeds": True}, + "#pattern": "https://www.youtube.com/embed/", + "#range": "2-", + "#count": 2, + }, + { + "#url": "https://www.sankakucomplex.com/tag/cosplay/", + "#category": ("", "sankakucomplex", "tag"), + "#class": sankakucomplex.SankakucomplexTagExtractor, + "#pattern": sankakucomplex.SankakucomplexArticleExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://www.sankakucomplex.com/category/anime/", + "#category": ("", "sankakucomplex", "tag"), + "#class": sankakucomplex.SankakucomplexTagExtractor, + }, + { + "#url": "https://www.sankakucomplex.com/author/rift/page/5/", + "#category": ("", "sankakucomplex", "tag"), + "#class": sankakucomplex.SankakucomplexTagExtractor, + }, ) diff --git a/test/results/scrolller.py b/test/results/scrolller.py index 6d34fa1f7..cc2a08d0e 100644 --- a/test/results/scrolller.py +++ b/test/results/scrolller.py @@ -1,81 +1,73 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import scrolller - __tests__ = ( -{ - "#url" : "https://scrolller.com/r/AmateurPhotography", - "#class" : scrolller.ScrolllerSubredditExtractor, - "#pattern": r"https://\w+\.scrolller\.com/(\w+/)?[\w-]+-\w+\.(jpg|png)", - "#range" : "1-100", - "#count" : 100, - - "albumUrl" : None, - "displayName" : None, - "fullLengthSource": None, - "gfycatSource" : None, - "hasAudio" : None, - "height" : int, - "id" : int, - "isFavorite" : False, - "isNsfw" : False, - "isOptimized" : bool, - "isPaid" : None, - "mediaSources" : list, - "ownerAvatar" : None, - "redditPath" : r"re:/r/AmateurPhotography/comments/...", - "redgifsSource" : None, - "subredditId" : {0, 413}, - "subredditTitle" : "AmateurPhotography", - "subredditUrl" : "/r/AmateurPhotography", - "tags" : None, - "title" : str, - "url" : str, - "username" : None, - "width" : int, -}, - -{ - "#url" : "https://scrolller.com/cabin-in-northern-finland-7nagf1929p", - "#class": scrolller.ScrolllerPostExtractor, - "#urls" : "https://yocto.scrolller.com/cabin-in-northern-finland-93vjsuxmcz.jpg", - - "albumUrl" : None, - "displayName" : None, - "extension" : "jpg", - "filename" : "cabin-in-northern-finland-93vjsuxmcz", - "fullLengthSource": None, - "gfycatSource" : None, - "hasAudio" : None, - "height" : 1350, - "id" : 10478722, - "isNsfw" : False, - "isOptimized" : False, - "isPaid" : None, - "mediaSources" : list, - "ownerAvatar" : None, - "redditPath" : "/r/AmateurPhotography/comments/jj048q/cabin_in_northern_finland/", - "redgifsSource" : None, - "subredditId" : 0, - "subredditTitle" : "AmateurPhotography", - "subredditUrl" : "/r/AmateurPhotography", - "tags" : None, - "title" : "Cabin in northern Finland", - "url" : "https://yocto.scrolller.com/cabin-in-northern-finland-93vjsuxmcz.jpg", - "username" : None, - "width" : 1080, -}, - -{ - "#url" : "https://scrolller.com/following", - "#class" : scrolller.ScrolllerFollowingExtractor, - "#pattern": scrolller.ScrolllerSubredditExtractor.pattern, - "#auth" : True, -}, - + { + "#url": "https://scrolller.com/r/AmateurPhotography", + "#class": scrolller.ScrolllerSubredditExtractor, + "#pattern": r"https://\w+\.scrolller\.com/(\w+/)?[\w-]+-\w+\.(jpg|png)", + "#range": "1-100", + "#count": 100, + "albumUrl": None, + "displayName": None, + "fullLengthSource": None, + "gfycatSource": None, + "hasAudio": None, + "height": int, + "id": int, + "isFavorite": False, + "isNsfw": False, + "isOptimized": bool, + "isPaid": None, + "mediaSources": list, + "ownerAvatar": None, + "redditPath": r"re:/r/AmateurPhotography/comments/...", + "redgifsSource": None, + "subredditId": {0, 413}, + "subredditTitle": "AmateurPhotography", + "subredditUrl": "/r/AmateurPhotography", + "tags": None, + "title": str, + "url": str, + "username": None, + "width": int, + }, + { + "#url": "https://scrolller.com/cabin-in-northern-finland-7nagf1929p", + "#class": scrolller.ScrolllerPostExtractor, + "#urls": "https://yocto.scrolller.com/cabin-in-northern-finland-93vjsuxmcz.jpg", + "albumUrl": None, + "displayName": None, + "extension": "jpg", + "filename": "cabin-in-northern-finland-93vjsuxmcz", + "fullLengthSource": None, + "gfycatSource": None, + "hasAudio": None, + "height": 1350, + "id": 10478722, + "isNsfw": False, + "isOptimized": False, + "isPaid": None, + "mediaSources": list, + "ownerAvatar": None, + "redditPath": "/r/AmateurPhotography/comments/jj048q/cabin_in_northern_finland/", + "redgifsSource": None, + "subredditId": 0, + "subredditTitle": "AmateurPhotography", + "subredditUrl": "/r/AmateurPhotography", + "tags": None, + "title": "Cabin in northern Finland", + "url": "https://yocto.scrolller.com/cabin-in-northern-finland-93vjsuxmcz.jpg", + "username": None, + "width": 1080, + }, + { + "#url": "https://scrolller.com/following", + "#class": scrolller.ScrolllerFollowingExtractor, + "#pattern": scrolller.ScrolllerSubredditExtractor.pattern, + "#auth": True, + }, ) diff --git a/test/results/seiga.py b/test/results/seiga.py index 3749adc4a..954dd7529 100644 --- a/test/results/seiga.py +++ b/test/results/seiga.py @@ -1,104 +1,88 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import seiga from gallery_dl import exception - +from gallery_dl.extractor import seiga __tests__ = ( -{ - "#url" : "https://seiga.nicovideo.jp/user/illust/39537793", - "#category": ("", "seiga", "user"), - "#class" : seiga.SeigaUserExtractor, - "#pattern" : r"https://lohas\.nicoseiga\.jp/priv/[0-9a-f]+/\d+/\d+", - "#count" : ">= 4", - - "user" : { - "id" : 39537793, - "message": str, - "name" : str, + { + "#url": "https://seiga.nicovideo.jp/user/illust/39537793", + "#category": ("", "seiga", "user"), + "#class": seiga.SeigaUserExtractor, + "#pattern": r"https://lohas\.nicoseiga\.jp/priv/[0-9a-f]+/\d+/\d+", + "#count": ">= 4", + "user": { + "id": 39537793, + "message": str, + "name": str, + }, + "clips": int, + "comments": int, + "count": int, + "extension": None, + "image_id": int, + "title": str, + "views": int, + }, + { + "#url": "https://seiga.nicovideo.jp/user/illust/79433", + "#category": ("", "seiga", "user"), + "#class": seiga.SeigaUserExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://seiga.nicovideo.jp/user/illust/39537793?sort=image_view&target=illust_all", + "#category": ("", "seiga", "user"), + "#class": seiga.SeigaUserExtractor, + }, + { + "#url": "https://sp.seiga.nicovideo.jp/user/illust/39537793", + "#category": ("", "seiga", "user"), + "#class": seiga.SeigaUserExtractor, + }, + { + "#url": "https://seiga.nicovideo.jp/seiga/im5977527", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + "#sha1_metadata": "c8339781da260f7fc44894ad9ada016f53e3b12a", + "#sha1_content": "d9202292012178374d57fb0126f6124387265297", + }, + { + "#url": "https://seiga.nicovideo.jp/seiga/im123", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://seiga.nicovideo.jp/seiga/im10877923", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + "#pattern": r"https://lohas\.nicoseiga\.jp/priv/5936a2a6c860a600e465e0411c0822e0b510e286/1688757110/10877923", + }, + { + "#url": "https://seiga.nicovideo.jp/image/source/5977527", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + }, + { + "#url": "https://sp.seiga.nicovideo.jp/seiga/#!/im5977527", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + }, + { + "#url": "https://lohas.nicoseiga.jp/thumb/5977527i", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + }, + { + "#url": "https://lohas.nicoseiga.jp/priv/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + }, + { + "#url": "https://lohas.nicoseiga.jp/o/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, }, - "clips" : int, - "comments" : int, - "count" : int, - "extension": None, - "image_id" : int, - "title" : str, - "views" : int, -}, - -{ - "#url" : "https://seiga.nicovideo.jp/user/illust/79433", - "#category": ("", "seiga", "user"), - "#class" : seiga.SeigaUserExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://seiga.nicovideo.jp/user/illust/39537793?sort=image_view&target=illust_all", - "#category": ("", "seiga", "user"), - "#class" : seiga.SeigaUserExtractor, -}, - -{ - "#url" : "https://sp.seiga.nicovideo.jp/user/illust/39537793", - "#category": ("", "seiga", "user"), - "#class" : seiga.SeigaUserExtractor, -}, - -{ - "#url" : "https://seiga.nicovideo.jp/seiga/im5977527", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, - "#sha1_metadata": "c8339781da260f7fc44894ad9ada016f53e3b12a", - "#sha1_content" : "d9202292012178374d57fb0126f6124387265297", -}, - -{ - "#url" : "https://seiga.nicovideo.jp/seiga/im123", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://seiga.nicovideo.jp/seiga/im10877923", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, - "#pattern" : r"https://lohas\.nicoseiga\.jp/priv/5936a2a6c860a600e465e0411c0822e0b510e286/1688757110/10877923", -}, - -{ - "#url" : "https://seiga.nicovideo.jp/image/source/5977527", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, -}, - -{ - "#url" : "https://sp.seiga.nicovideo.jp/seiga/#!/im5977527", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, -}, - -{ - "#url" : "https://lohas.nicoseiga.jp/thumb/5977527i", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, -}, - -{ - "#url" : "https://lohas.nicoseiga.jp/priv/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, -}, - -{ - "#url" : "https://lohas.nicoseiga.jp/o/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, -}, - ) diff --git a/test/results/senmanga.py b/test/results/senmanga.py index 8192cfad7..ef161aa33 100644 --- a/test/results/senmanga.py +++ b/test/results/senmanga.py @@ -1,65 +1,55 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import senmanga - __tests__ = ( -{ - "#url" : "https://raw.senmanga.com/Bokura-wa-Minna-Kawaisou/37A/1", - "#category": ("", "senmanga", "chapter"), - "#class" : senmanga.SenmangaChapterExtractor, - "#pattern" : r"https://raw\.senmanga\.com/viewer/Bokura-wa-Minna-Kawaisou/37A/[12]", - "#sha1_url" : "5f95140ff511d8497e2ec08fa7267c6bb231faec", - "#sha1_content": "556a16d5ca3441d7a5807b6b5ac06ec458a3e4ba", - - "chapter" : "37A", - "count" : 2, - "extension": "", - "filename" : r"re:[12]", - "lang" : "ja", - "language" : "Japanese", - "manga" : "Bokura wa Minna Kawaisou", - "page" : int, -}, - -{ - "#url" : "http://raw.senmanga.com/Love-Lab/2016-03/1", - "#category": ("", "senmanga", "chapter"), - "#class" : senmanga.SenmangaChapterExtractor, - "#pattern" : r"https://raw\.senmanga\.com/viewer/Love-Lab/2016-03/\d", - "#sha1_url": "8347b9f00c14b864dd3c19a1f5ae52adb2ef00de", - - "chapter" : "2016-03", - "count" : 9, - "extension": "", - "filename" : r"re:\d", - "manga" : "Renai Lab 恋愛ラボ", -}, - -{ - "#url" : "https://raw.senmanga.com/akabane-honeko-no-bodyguard/1", - "#category": ("", "senmanga", "chapter"), - "#class" : senmanga.SenmangaChapterExtractor, - "#pattern" : r"https://i\d\.wp\.com/kumacdn.club/image-new-2/a/akabane-honeko-no-bodyguard/chapter-1/\d+-[0-9a-f]{13}\.jpg", - - "chapter" : "1", - "count" : 65, - "extension": "jpg", - "filename" : r"re:\d+-\w+", - "manga" : "Akabane Honeko no Bodyguard", -}, - -{ - "#url" : "https://raw.senmanga.com/amama-cinderella/3", - "#comment" : "no http scheme ()", - "#category": ("", "senmanga", "chapter"), - "#class" : senmanga.SenmangaChapterExtractor, - "#pattern" : r"^https://kumacdn.club/image-new-2/a/amama-cinderella/chapter-3/.+\.jpg", - "#count" : 30, -}, - + { + "#url": "https://raw.senmanga.com/Bokura-wa-Minna-Kawaisou/37A/1", + "#category": ("", "senmanga", "chapter"), + "#class": senmanga.SenmangaChapterExtractor, + "#pattern": r"https://raw\.senmanga\.com/viewer/Bokura-wa-Minna-Kawaisou/37A/[12]", + "#sha1_url": "5f95140ff511d8497e2ec08fa7267c6bb231faec", + "#sha1_content": "556a16d5ca3441d7a5807b6b5ac06ec458a3e4ba", + "chapter": "37A", + "count": 2, + "extension": "", + "filename": r"re:[12]", + "lang": "ja", + "language": "Japanese", + "manga": "Bokura wa Minna Kawaisou", + "page": int, + }, + { + "#url": "http://raw.senmanga.com/Love-Lab/2016-03/1", + "#category": ("", "senmanga", "chapter"), + "#class": senmanga.SenmangaChapterExtractor, + "#pattern": r"https://raw\.senmanga\.com/viewer/Love-Lab/2016-03/\d", + "#sha1_url": "8347b9f00c14b864dd3c19a1f5ae52adb2ef00de", + "chapter": "2016-03", + "count": 9, + "extension": "", + "filename": r"re:\d", + "manga": "Renai Lab 恋愛ラボ", + }, + { + "#url": "https://raw.senmanga.com/akabane-honeko-no-bodyguard/1", + "#category": ("", "senmanga", "chapter"), + "#class": senmanga.SenmangaChapterExtractor, + "#pattern": r"https://i\d\.wp\.com/kumacdn.club/image-new-2/a/akabane-honeko-no-bodyguard/chapter-1/\d+-[0-9a-f]{13}\.jpg", + "chapter": "1", + "count": 65, + "extension": "jpg", + "filename": r"re:\d+-\w+", + "manga": "Akabane Honeko no Bodyguard", + }, + { + "#url": "https://raw.senmanga.com/amama-cinderella/3", + "#comment": "no http scheme ()", + "#category": ("", "senmanga", "chapter"), + "#class": senmanga.SenmangaChapterExtractor, + "#pattern": r"^https://kumacdn.club/image-new-2/a/amama-cinderella/chapter-3/.+\.jpg", + "#count": 30, + }, ) diff --git a/test/results/sexcom.py b/test/results/sexcom.py index e51f801c8..c1bd2ddea 100644 --- a/test/results/sexcom.py +++ b/test/results/sexcom.py @@ -1,110 +1,95 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import sexcom - __tests__ = ( -{ - "#url" : "https://www.sex.com/pin/21241874-sexy-ecchi-girls-166/", - "#comment" : "picture", - "#category": ("", "sexcom", "pin"), - "#class" : sexcom.SexcomPinExtractor, - "#pattern" : "https://cdn.sex.com/images/.+/2014/08/26/7637609.jpg", - "#sha1_content": "ebe1814dadfebf15d11c6af4f6afb1a50d6c2a1c", - - "comments" : int, - "date" : "dt:2014-10-19 15:45:44", - "extension": "jpg", - "filename" : "7637609", - "likes" : int, - "pin_id" : 21241874, - "repins" : int, - "tags" : list, - "thumbnail": str, - "title" : "Sexy Ecchi Girls 166", - "type" : "picture", - "uploader" : "mangazeta", - "url" : str, -}, - -{ - "#url" : "https://www.sex.com/pin/55435122-ecchi/", - "#comment" : "gif", - "#category": ("", "sexcom", "pin"), - "#class" : sexcom.SexcomPinExtractor, - "#pattern" : "https://cdn.sex.com/images/.+/2017/12/07/18760842.gif", - "#sha1_content": "176cc63fa05182cb0438c648230c0f324a5965fe", -}, - -{ - "#url" : "https://www.sex.com/pin/55748341/", - "#comment" : "video", - "#category": ("", "sexcom", "pin"), - "#class" : sexcom.SexcomPinExtractor, - "#pattern" : r"https://cdn\.sex\.com/videos/pinporn/2018/02/10/776229_hd\.mp4", - "#sha1_content": "e1a5834869163e2c4d1ca2677f5b7b367cf8cfff", -}, - -{ - "#url" : "https://www.sex.com/pin/55847384-very-nicely-animated/", - "#comment" : "pornhub embed (404 gone)", - "#category": ("", "sexcom", "pin"), - "#class" : sexcom.SexcomPinExtractor, -}, - -{ - "#url" : "https://www.sex.com/pin/21241874/#related", - "#category": ("", "sexcom", "related-pin"), - "#class" : sexcom.SexcomRelatedPinExtractor, - "#count" : ">= 20", -}, - -{ - "#url" : "https://www.sex.com/user/sirjuan79/pins/", - "#category": ("", "sexcom", "pins"), - "#class" : sexcom.SexcomPinsExtractor, - "#count" : ">= 4", -}, - -{ - "#url" : "https://www.sex.com/user/sirjuan79/likes/", - "#category": ("", "sexcom", "likes"), - "#class" : sexcom.SexcomLikesExtractor, - "#range" : "1-30", - "#count" : ">= 25", -}, - -{ - "#url" : "https://www.sex.com/user/ronin17/exciting-hentai/", - "#category": ("", "sexcom", "board"), - "#class" : sexcom.SexcomBoardExtractor, - "#count" : ">= 10", -}, - -{ - "#url" : "https://www.sex.com/search/pics?query=ecchi", - "#category": ("", "sexcom", "search"), - "#class" : sexcom.SexcomSearchExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.sex.com/videos/hentai/", - "#category": ("", "sexcom", "search"), - "#class" : sexcom.SexcomSearchExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.sex.com/pics/?sort=popular&sub=all&page=1", - "#category": ("", "sexcom", "search"), - "#class" : sexcom.SexcomSearchExtractor, -}, - + { + "#url": "https://www.sex.com/pin/21241874-sexy-ecchi-girls-166/", + "#comment": "picture", + "#category": ("", "sexcom", "pin"), + "#class": sexcom.SexcomPinExtractor, + "#pattern": "https://cdn.sex.com/images/.+/2014/08/26/7637609.jpg", + "#sha1_content": "ebe1814dadfebf15d11c6af4f6afb1a50d6c2a1c", + "comments": int, + "date": "dt:2014-10-19 15:45:44", + "extension": "jpg", + "filename": "7637609", + "likes": int, + "pin_id": 21241874, + "repins": int, + "tags": list, + "thumbnail": str, + "title": "Sexy Ecchi Girls 166", + "type": "picture", + "uploader": "mangazeta", + "url": str, + }, + { + "#url": "https://www.sex.com/pin/55435122-ecchi/", + "#comment": "gif", + "#category": ("", "sexcom", "pin"), + "#class": sexcom.SexcomPinExtractor, + "#pattern": "https://cdn.sex.com/images/.+/2017/12/07/18760842.gif", + "#sha1_content": "176cc63fa05182cb0438c648230c0f324a5965fe", + }, + { + "#url": "https://www.sex.com/pin/55748341/", + "#comment": "video", + "#category": ("", "sexcom", "pin"), + "#class": sexcom.SexcomPinExtractor, + "#pattern": r"https://cdn\.sex\.com/videos/pinporn/2018/02/10/776229_hd\.mp4", + "#sha1_content": "e1a5834869163e2c4d1ca2677f5b7b367cf8cfff", + }, + { + "#url": "https://www.sex.com/pin/55847384-very-nicely-animated/", + "#comment": "pornhub embed (404 gone)", + "#category": ("", "sexcom", "pin"), + "#class": sexcom.SexcomPinExtractor, + }, + { + "#url": "https://www.sex.com/pin/21241874/#related", + "#category": ("", "sexcom", "related-pin"), + "#class": sexcom.SexcomRelatedPinExtractor, + "#count": ">= 20", + }, + { + "#url": "https://www.sex.com/user/sirjuan79/pins/", + "#category": ("", "sexcom", "pins"), + "#class": sexcom.SexcomPinsExtractor, + "#count": ">= 4", + }, + { + "#url": "https://www.sex.com/user/sirjuan79/likes/", + "#category": ("", "sexcom", "likes"), + "#class": sexcom.SexcomLikesExtractor, + "#range": "1-30", + "#count": ">= 25", + }, + { + "#url": "https://www.sex.com/user/ronin17/exciting-hentai/", + "#category": ("", "sexcom", "board"), + "#class": sexcom.SexcomBoardExtractor, + "#count": ">= 10", + }, + { + "#url": "https://www.sex.com/search/pics?query=ecchi", + "#category": ("", "sexcom", "search"), + "#class": sexcom.SexcomSearchExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.sex.com/videos/hentai/", + "#category": ("", "sexcom", "search"), + "#class": sexcom.SexcomSearchExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.sex.com/pics/?sort=popular&sub=all&page=1", + "#category": ("", "sexcom", "search"), + "#class": sexcom.SexcomSearchExtractor, + }, ) diff --git a/test/results/simplyhentai.py b/test/results/simplyhentai.py index 561ce4264..db453c986 100644 --- a/test/results/simplyhentai.py +++ b/test/results/simplyhentai.py @@ -1,75 +1,64 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import simplyhentai from gallery_dl import exception - +from gallery_dl.extractor import simplyhentai __tests__ = ( -{ - "#url" : "https://original-work.simply-hentai.com/amazon-no-hiyaku-amazon-elixir", - "#category": ("", "simplyhentai", "gallery"), - "#class" : simplyhentai.SimplyhentaiGalleryExtractor, - "#sha1_url" : "21613585ae5ec2f69ea579e9713f536fceab5bd5", - "#sha1_metadata": "9e87a0973553b2922ddee37958b8f5d87910af72", -}, - -{ - "#url" : "https://www.simply-hentai.com/notfound", - "#category": ("", "simplyhentai", "gallery"), - "#class" : simplyhentai.SimplyhentaiGalleryExtractor, - "#exception": exception.GalleryDLException, -}, - -{ - "#url" : "https://pokemon.simply-hentai.com/mao-friends-9bc39", - "#comment" : "custom subdomain", - "#category": ("", "simplyhentai", "gallery"), - "#class" : simplyhentai.SimplyhentaiGalleryExtractor, -}, - -{ - "#url" : "https://www.simply-hentai.com/vocaloid/black-magnet", - "#comment" : "www subdomain, two path segments", - "#category": ("", "simplyhentai", "gallery"), - "#class" : simplyhentai.SimplyhentaiGalleryExtractor, -}, - -{ - "#url" : "https://www.simply-hentai.com/image/pheromomania-vol-1-kanzenban-isao-3949d8b3-400c-4b6", - "#category": ("", "simplyhentai", "image"), - "#class" : simplyhentai.SimplyhentaiImageExtractor, - "#sha1_url" : "3d8eb55240a960134891bd77fe1df7988fcdc455", - "#sha1_metadata": "e10e5588481cab68329ef6ec1e5325206b2079a2", -}, - -{ - "#url" : "https://www.simply-hentai.com/gif/8915dfcf-0b6a-47c", - "#category": ("", "simplyhentai", "image"), - "#class" : simplyhentai.SimplyhentaiImageExtractor, - "#sha1_url" : "f73916527211b4a40f26568ee26cd8999f5f4f30", - "#sha1_metadata": "f94d775177fed918759c8a78a50976f867425b48", -}, - -{ - "#url" : "https://videos.simply-hentai.com/creamy-pie-episode-02", - "#category": ("", "simplyhentai", "video"), - "#class" : simplyhentai.SimplyhentaiVideoExtractor, - "#options" : {"verify": False}, - "#pattern" : r"https://www\.googleapis\.com/drive/v3/files/0B1ecQ8ZVLm3JcHZzQzBnVy1ZUmc\?alt=media&key=[\w-]+", - "#count" : 1, - "#sha1_metadata": "706790708b14773efc1e075ddd3b738a375348a5", -}, - -{ - "#url" : "https://videos.simply-hentai.com/1715-tifa-in-hentai-gang-bang-3d-movie", - "#category": ("", "simplyhentai", "video"), - "#class" : simplyhentai.SimplyhentaiVideoExtractor, - "#sha1_url" : "ad9a36ae06c601b6490e3c401834b4949d947eb0", - "#sha1_metadata": "f9dad94fbde9c95859e631ff4f07297a9567b874", -}, - + { + "#url": "https://original-work.simply-hentai.com/amazon-no-hiyaku-amazon-elixir", + "#category": ("", "simplyhentai", "gallery"), + "#class": simplyhentai.SimplyhentaiGalleryExtractor, + "#sha1_url": "21613585ae5ec2f69ea579e9713f536fceab5bd5", + "#sha1_metadata": "9e87a0973553b2922ddee37958b8f5d87910af72", + }, + { + "#url": "https://www.simply-hentai.com/notfound", + "#category": ("", "simplyhentai", "gallery"), + "#class": simplyhentai.SimplyhentaiGalleryExtractor, + "#exception": exception.GalleryDLException, + }, + { + "#url": "https://pokemon.simply-hentai.com/mao-friends-9bc39", + "#comment": "custom subdomain", + "#category": ("", "simplyhentai", "gallery"), + "#class": simplyhentai.SimplyhentaiGalleryExtractor, + }, + { + "#url": "https://www.simply-hentai.com/vocaloid/black-magnet", + "#comment": "www subdomain, two path segments", + "#category": ("", "simplyhentai", "gallery"), + "#class": simplyhentai.SimplyhentaiGalleryExtractor, + }, + { + "#url": "https://www.simply-hentai.com/image/pheromomania-vol-1-kanzenban-isao-3949d8b3-400c-4b6", + "#category": ("", "simplyhentai", "image"), + "#class": simplyhentai.SimplyhentaiImageExtractor, + "#sha1_url": "3d8eb55240a960134891bd77fe1df7988fcdc455", + "#sha1_metadata": "e10e5588481cab68329ef6ec1e5325206b2079a2", + }, + { + "#url": "https://www.simply-hentai.com/gif/8915dfcf-0b6a-47c", + "#category": ("", "simplyhentai", "image"), + "#class": simplyhentai.SimplyhentaiImageExtractor, + "#sha1_url": "f73916527211b4a40f26568ee26cd8999f5f4f30", + "#sha1_metadata": "f94d775177fed918759c8a78a50976f867425b48", + }, + { + "#url": "https://videos.simply-hentai.com/creamy-pie-episode-02", + "#category": ("", "simplyhentai", "video"), + "#class": simplyhentai.SimplyhentaiVideoExtractor, + "#options": {"verify": False}, + "#pattern": r"https://www\.googleapis\.com/drive/v3/files/0B1ecQ8ZVLm3JcHZzQzBnVy1ZUmc\?alt=media&key=[\w-]+", + "#count": 1, + "#sha1_metadata": "706790708b14773efc1e075ddd3b738a375348a5", + }, + { + "#url": "https://videos.simply-hentai.com/1715-tifa-in-hentai-gang-bang-3d-movie", + "#category": ("", "simplyhentai", "video"), + "#class": simplyhentai.SimplyhentaiVideoExtractor, + "#sha1_url": "ad9a36ae06c601b6490e3c401834b4949d947eb0", + "#sha1_metadata": "f9dad94fbde9c95859e631ff4f07297a9567b874", + }, ) diff --git a/test/results/skeb.py b/test/results/skeb.py index 4aa8691d0..70c030da0 100644 --- a/test/results/skeb.py +++ b/test/results/skeb.py @@ -1,93 +1,82 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import skeb - __tests__ = ( -{ - "#url" : "https://skeb.jp/@kanade_cocotte/works/38", - "#category": ("", "skeb", "post"), - "#class" : skeb.SkebPostExtractor, - "#count" : 2, - - "anonymous" : False, - "body" : r"re:はじめまして。私はYouTubeにてVTuberとして活動をしている湊ラ", - "count" : 2, - "num" : range(1, 2), - "client" : { - "avatar_url" : r"re:https://pbs.twimg.com/profile_images/\d+/\w+\.jpg", - "header_url" : r"re:https://pbs.twimg.com/profile_banners/1375007870291300358/\d+/1500x500", - "id" : 1196514, - "name" : str, - "screen_name": "minato_ragi", + { + "#url": "https://skeb.jp/@kanade_cocotte/works/38", + "#category": ("", "skeb", "post"), + "#class": skeb.SkebPostExtractor, + "#count": 2, + "anonymous": False, + "body": r"re:はじめまして。私はYouTubeにてVTuberとして活動をしている湊ラ", + "count": 2, + "num": range(1, 2), + "client": { + "avatar_url": r"re:https://pbs.twimg.com/profile_images/\d+/\w+\.jpg", + "header_url": r"re:https://pbs.twimg.com/profile_banners/1375007870291300358/\d+/1500x500", + "id": 1196514, + "name": str, + "screen_name": "minato_ragi", + }, + "content_category": "preview", + "creator": { + "avatar_url": "https://pbs.twimg.com/profile_images/1225470417063645184/P8_SiB0V.jpg", + "header_url": "https://pbs.twimg.com/profile_banners/71243217/1647958329/1500x500", + "id": 159273, + "name": "イチノセ奏", + "screen_name": "kanade_cocotte", + }, + "file_id": int, + "file_url": str, + "genre": "art", + "nsfw": False, + "original": { + "byte_size": int, + "duration": None, + "extension": r"re:psd|png", + "frame_rate": None, + "height": 3727, + "is_movie": False, + "width": 2810, + }, + "post_num": "38", + "post_url": "https://skeb.jp/@kanade_cocotte/works/38", + "source_body": None, + "source_thanks": None, + "tags": list, + "thanks": None, + "translated_body": False, + "translated_thanks": None, }, - "content_category": "preview", - "creator" : { - "avatar_url" : "https://pbs.twimg.com/profile_images/1225470417063645184/P8_SiB0V.jpg", - "header_url" : "https://pbs.twimg.com/profile_banners/71243217/1647958329/1500x500", - "id" : 159273, - "name" : "イチノセ奏", - "screen_name": "kanade_cocotte", + { + "#url": "https://skeb.jp/@kanade_cocotte", + "#category": ("", "skeb", "user"), + "#class": skeb.SkebUserExtractor, + "#pattern": r"https://si\.imgix\.net/\w+/uploads/origins/[\w-]+", + "#range": "1-5", + "count": int, + "num": int, }, - "file_id" : int, - "file_url" : str, - "genre" : "art", - "nsfw" : False, - "original" : { - "byte_size" : int, - "duration" : None, - "extension" : r"re:psd|png", - "frame_rate": None, - "height" : 3727, - "is_movie" : False, - "width" : 2810, + { + "#url": "https://skeb.jp/search?q=bunny%20tree&t=works", + "#category": ("", "skeb", "search"), + "#class": skeb.SkebSearchExtractor, + "#count": ">= 18", + "search_tags": "bunny tree", + }, + { + "#url": "https://skeb.jp/@user/following_creators", + "#category": ("", "skeb", "following"), + "#class": skeb.SkebFollowingExtractor, + }, + { + "#url": "https://skeb.jp/following_users", + "#category": ("", "skeb", "following-users"), + "#class": skeb.SkebFollowingUsersExtractor, + "#pattern": skeb.SkebUserExtractor.pattern, + "#auth": True, }, - "post_num" : "38", - "post_url" : "https://skeb.jp/@kanade_cocotte/works/38", - "source_body" : None, - "source_thanks" : None, - "tags" : list, - "thanks" : None, - "translated_body" : False, - "translated_thanks": None, -}, - -{ - "#url" : "https://skeb.jp/@kanade_cocotte", - "#category": ("", "skeb", "user"), - "#class" : skeb.SkebUserExtractor, - "#pattern" : r"https://si\.imgix\.net/\w+/uploads/origins/[\w-]+", - "#range" : "1-5", - - "count": int, - "num" : int, -}, - -{ - "#url" : "https://skeb.jp/search?q=bunny%20tree&t=works", - "#category": ("", "skeb", "search"), - "#class" : skeb.SkebSearchExtractor, - "#count" : ">= 18", - - "search_tags": "bunny tree", -}, - -{ - "#url" : "https://skeb.jp/@user/following_creators", - "#category": ("", "skeb", "following"), - "#class" : skeb.SkebFollowingExtractor, -}, - -{ - "#url" : "https://skeb.jp/following_users", - "#category": ("", "skeb", "following-users"), - "#class" : skeb.SkebFollowingUsersExtractor, - "#pattern" : skeb.SkebUserExtractor.pattern, - "#auth" : True, -}, - ) diff --git a/test/results/slickpic.py b/test/results/slickpic.py index ff1eb1ff0..a9fdf2302 100644 --- a/test/results/slickpic.py +++ b/test/results/slickpic.py @@ -1,48 +1,41 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import slickpic - __tests__ = ( -{ - "#url" : "https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", - "#category": ("", "slickpic", "album"), - "#class" : slickpic.SlickpicAlbumExtractor, - "#pattern" : r"https://stored-cf\.slickpic\.com/NDk5MjNmYTc1MzU0MQ,,/20160807/\w+/p/o/JSBFSS-\d+\.jpg", - "#count" : 102, - "#sha1_metadata": "c37c4ce9c54c09abc6abdf295855d46f11529cbf", -}, - -{ - "#url" : "https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", - "#category": ("", "slickpic", "album"), - "#class" : slickpic.SlickpicAlbumExtractor, - "#range" : "34", - "#sha1_content": [ - "276eb2c902187bb177ae8013e310e1d6641fba9a", - "52b5a310587de1048030ab13a912f6a3a9cc7dab", - "cec6630e659dc72db1ee1a9a6f3b525189261988", - "6f81e1e74c6cd6db36844e7211eef8e7cd30055d", - "22e83645fc242bc3584eca7ec982c8a53a4d8a44", - ], -}, - -{ - "#url" : "https://mattcrandall.slickpic.com/gallery/", - "#category": ("", "slickpic", "user"), - "#class" : slickpic.SlickpicUserExtractor, - "#pattern" : slickpic.SlickpicAlbumExtractor.pattern, - "#count" : ">= 358", -}, - -{ - "#url" : "https://mattcrandall.slickpic.com/", - "#category": ("", "slickpic", "user"), - "#class" : slickpic.SlickpicUserExtractor, -}, - + { + "#url": "https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", + "#category": ("", "slickpic", "album"), + "#class": slickpic.SlickpicAlbumExtractor, + "#pattern": r"https://stored-cf\.slickpic\.com/NDk5MjNmYTc1MzU0MQ,,/20160807/\w+/p/o/JSBFSS-\d+\.jpg", + "#count": 102, + "#sha1_metadata": "c37c4ce9c54c09abc6abdf295855d46f11529cbf", + }, + { + "#url": "https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", + "#category": ("", "slickpic", "album"), + "#class": slickpic.SlickpicAlbumExtractor, + "#range": "34", + "#sha1_content": [ + "276eb2c902187bb177ae8013e310e1d6641fba9a", + "52b5a310587de1048030ab13a912f6a3a9cc7dab", + "cec6630e659dc72db1ee1a9a6f3b525189261988", + "6f81e1e74c6cd6db36844e7211eef8e7cd30055d", + "22e83645fc242bc3584eca7ec982c8a53a4d8a44", + ], + }, + { + "#url": "https://mattcrandall.slickpic.com/gallery/", + "#category": ("", "slickpic", "user"), + "#class": slickpic.SlickpicUserExtractor, + "#pattern": slickpic.SlickpicAlbumExtractor.pattern, + "#count": ">= 358", + }, + { + "#url": "https://mattcrandall.slickpic.com/", + "#category": ("", "slickpic", "user"), + "#class": slickpic.SlickpicUserExtractor, + }, ) diff --git a/test/results/slideshare.py b/test/results/slideshare.py index 66dd480be..1f58681e8 100644 --- a/test/results/slideshare.py +++ b/test/results/slideshare.py @@ -1,48 +1,40 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import slideshare - __tests__ = ( -{ - "#url" : "https://www.slideshare.net/Slideshare/get-started-with-slide-share", - "#category": ("", "slideshare", "presentation"), - "#class" : slideshare.SlidesharePresentationExtractor, - "#pattern" : r"https://image\.slidesharecdn\.com/getstartedwithslideshare-150520173821-lva1-app6892/95/Getting-Started-With-SlideShare-\d+-1024\.jpg", - "#count" : 19, - "#sha1_content": "2b6a191eab60b3978fdacfecf2da302dd45bc108", - - "description" : "Get Started with SlideShare - A Beginngers Guide for Creators", - "likes" : int, - "presentation": "get-started-with-slide-share", - "date" : "dt:2015-05-20 17:38:21", - "title" : "Getting Started With SlideShare", - "user" : "Slideshare", - "views" : int, -}, - -{ - "#url" : "https://www.slideshare.net/pragmaticsolutions/warum-sie-nicht-ihren-mitarbeitenden-ndern-sollten-sondern-ihr-managementsystem", - "#comment" : "long title and description", - "#category": ("", "slideshare", "presentation"), - "#class" : slideshare.SlidesharePresentationExtractor, - "#sha1_url": "c2d0079cc3b05de0fd93b0d0b1f47ff2a32119b7", - - "title" : "Warum Sie nicht Ihren Mitarbeitenden ändern sollten, sondern Ihr Managementsystem", - "description": "Mitarbeitende verhalten sich mehrheitlich so, wie das System es ihnen vorgibt. Welche Voraussetzungen es braucht, damit Ihre Mitarbeitenden ihr ganzes Herzblut einsetzen, bespricht Fredi Schmidli in diesem Referat.", -}, - -{ - "#url" : "https://www.slideshare.net/mobile/uqudent/introduction-to-fixed-prosthodontics", - "#comment" : "mobile URL", - "#category": ("", "slideshare", "presentation"), - "#class" : slideshare.SlidesharePresentationExtractor, - "#pattern" : r"https://image\.slidesharecdn\.com/introductiontofixedprosthodonticsfinal-110427200948-phpapp02/95/Introduction-to-fixed-prosthodontics-\d+-1024\.jpg", - "#count" : 27, -}, - + { + "#url": "https://www.slideshare.net/Slideshare/get-started-with-slide-share", + "#category": ("", "slideshare", "presentation"), + "#class": slideshare.SlidesharePresentationExtractor, + "#pattern": r"https://image\.slidesharecdn\.com/getstartedwithslideshare-150520173821-lva1-app6892/95/Getting-Started-With-SlideShare-\d+-1024\.jpg", + "#count": 19, + "#sha1_content": "2b6a191eab60b3978fdacfecf2da302dd45bc108", + "description": "Get Started with SlideShare - A Beginngers Guide for Creators", + "likes": int, + "presentation": "get-started-with-slide-share", + "date": "dt:2015-05-20 17:38:21", + "title": "Getting Started With SlideShare", + "user": "Slideshare", + "views": int, + }, + { + "#url": "https://www.slideshare.net/pragmaticsolutions/warum-sie-nicht-ihren-mitarbeitenden-ndern-sollten-sondern-ihr-managementsystem", + "#comment": "long title and description", + "#category": ("", "slideshare", "presentation"), + "#class": slideshare.SlidesharePresentationExtractor, + "#sha1_url": "c2d0079cc3b05de0fd93b0d0b1f47ff2a32119b7", + "title": "Warum Sie nicht Ihren Mitarbeitenden ändern sollten, sondern Ihr Managementsystem", + "description": "Mitarbeitende verhalten sich mehrheitlich so, wie das System es ihnen vorgibt. Welche Voraussetzungen es braucht, damit Ihre Mitarbeitenden ihr ganzes Herzblut einsetzen, bespricht Fredi Schmidli in diesem Referat.", + }, + { + "#url": "https://www.slideshare.net/mobile/uqudent/introduction-to-fixed-prosthodontics", + "#comment": "mobile URL", + "#category": ("", "slideshare", "presentation"), + "#class": slideshare.SlidesharePresentationExtractor, + "#pattern": r"https://image\.slidesharecdn\.com/introductiontofixedprosthodonticsfinal-110427200948-phpapp02/95/Introduction-to-fixed-prosthodontics-\d+-1024\.jpg", + "#count": 27, + }, ) diff --git a/test/results/smugloli.py b/test/results/smugloli.py index c71d054e4..ad5176171 100644 --- a/test/results/smugloli.py +++ b/test/results/smugloli.py @@ -1,49 +1,40 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import vichan - __tests__ = ( -{ - "#url" : "https://smuglo.li/a/res/1143245.html", - "#category": ("vichan", "smugloli", "thread"), - "#class" : vichan.VichanThreadExtractor, - "#pattern" : r"https://smug.+/a/src/\d+(-\d)?\.\w+", - "#count" : ">= 50", - - "board" : "a", - "thread": "1143245", - "title": "Rabbit Rabbit Thread #4", -}, - -{ - "#url" : "https://smugloli.net/a/res/1145409.html", - "#category": ("vichan", "smugloli", "thread"), - "#class" : vichan.VichanThreadExtractor, -}, - -{ - "#url" : "https://smuglo.li/a", - "#category": ("vichan", "smugloli", "board"), - "#class" : vichan.VichanBoardExtractor, - "#pattern" : vichan.VichanThreadExtractor.pattern, - "#count" : ">= 100", -}, - -{ - "#url" : "https://smuglo.li/a/1.html", - "#category": ("vichan", "smugloli", "board"), - "#class" : vichan.VichanBoardExtractor, -}, - -{ - "#url" : "https://smugloli.net/cute/catalog.html", - "#category": ("vichan", "smugloli", "board"), - "#class" : vichan.VichanBoardExtractor, -}, - + { + "#url": "https://smuglo.li/a/res/1143245.html", + "#category": ("vichan", "smugloli", "thread"), + "#class": vichan.VichanThreadExtractor, + "#pattern": r"https://smug.+/a/src/\d+(-\d)?\.\w+", + "#count": ">= 50", + "board": "a", + "thread": "1143245", + "title": "Rabbit Rabbit Thread #4", + }, + { + "#url": "https://smugloli.net/a/res/1145409.html", + "#category": ("vichan", "smugloli", "thread"), + "#class": vichan.VichanThreadExtractor, + }, + { + "#url": "https://smuglo.li/a", + "#category": ("vichan", "smugloli", "board"), + "#class": vichan.VichanBoardExtractor, + "#pattern": vichan.VichanThreadExtractor.pattern, + "#count": ">= 100", + }, + { + "#url": "https://smuglo.li/a/1.html", + "#category": ("vichan", "smugloli", "board"), + "#class": vichan.VichanBoardExtractor, + }, + { + "#url": "https://smugloli.net/cute/catalog.html", + "#category": ("vichan", "smugloli", "board"), + "#class": vichan.VichanBoardExtractor, + }, ) diff --git a/test/results/smugmug.py b/test/results/smugmug.py index dda8d3a28..08fc3fdde 100644 --- a/test/results/smugmug.py +++ b/test/results/smugmug.py @@ -1,175 +1,159 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import smugmug - __tests__ = ( -{ - "#url" : "smugmug:album:cr4C7f", - "#category": ("", "smugmug", "album"), - "#class" : smugmug.SmugmugAlbumExtractor, - "#urls": ( - "https://photos.smugmug.com/Nature/Dove/i-XvZFJFG/0/DMk7cm6qRBSFPvQgT9C4t4jtBJKF7JSK9jszgHZnr/O/Dual%20Suicide_20070721-DSC_4804.jpg", - "https://photos.smugmug.com/Nature/Dove/i-2wVPqHf/0/DBXmTSTqVWzTLZxL3JPVK7hGT9zp8tzsFdhtWm68v/O/Morning%20Dove2_20070621-DSC_3222.jpg", - "https://photos.smugmug.com/Nature/Dove/i-QHFnmb8/0/GKLvnm7zFQWX2G2VcJRprx8WZqTfFJkn8C5nRnCk/O/Speed%20Skater_03082008_POR7728.jpg", - "https://photos.smugmug.com/Nature/Dove/i-MXQZKws/0/D6XCS9xnncDVtZ9NtVq66ZK9xjL4D2H9KSbpFMjfM/O/Airing%20it%20Out0_5142008_DSC_8166.jpg", - "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/FfB6gSx8X6MS7Hvww7GK7tWsrfdtwCx79hCVzwSm/O/Fluff_20090521-_DSC1542.jpg", - "https://photos.smugmug.com/Nature/Dove/i-T9Qv5Pm/0/CFT4MB9hg7rKwWmbFhGQTCnmxdpnGBKPDbHTPLSgV/O/D2F_D300_20090827-_TDM5650.jpg", - ), -}, - -{ - "#url" : "smugmug:album:Fb7hMs", - "#comment" : "empty", - "#category": ("", "smugmug", "album"), - "#class" : smugmug.SmugmugAlbumExtractor, - "#count" : 0, -}, - -{ - "#url" : "smugmug:album:6VRT8G", - "#comment" : "no 'User'", - "#category": ("", "smugmug", "album"), - "#class" : smugmug.SmugmugAlbumExtractor, - "#sha1_url": "17837ff2c78a6e2335291666f43d620d82f2926a", - - "User": { - "Name" : "", - "NickName" : "", - "QuickShare" : False, - "RefTag" : "", - "ResponseLevel": "Public", - "Uri" : "", - "ViewPassHint" : "", - "WebUri" : "", + { + "#url": "smugmug:album:cr4C7f", + "#category": ("", "smugmug", "album"), + "#class": smugmug.SmugmugAlbumExtractor, + "#urls": ( + "https://photos.smugmug.com/Nature/Dove/i-XvZFJFG/0/DMk7cm6qRBSFPvQgT9C4t4jtBJKF7JSK9jszgHZnr/O/Dual%20Suicide_20070721-DSC_4804.jpg", + "https://photos.smugmug.com/Nature/Dove/i-2wVPqHf/0/DBXmTSTqVWzTLZxL3JPVK7hGT9zp8tzsFdhtWm68v/O/Morning%20Dove2_20070621-DSC_3222.jpg", + "https://photos.smugmug.com/Nature/Dove/i-QHFnmb8/0/GKLvnm7zFQWX2G2VcJRprx8WZqTfFJkn8C5nRnCk/O/Speed%20Skater_03082008_POR7728.jpg", + "https://photos.smugmug.com/Nature/Dove/i-MXQZKws/0/D6XCS9xnncDVtZ9NtVq66ZK9xjL4D2H9KSbpFMjfM/O/Airing%20it%20Out0_5142008_DSC_8166.jpg", + "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/FfB6gSx8X6MS7Hvww7GK7tWsrfdtwCx79hCVzwSm/O/Fluff_20090521-_DSC1542.jpg", + "https://photos.smugmug.com/Nature/Dove/i-T9Qv5Pm/0/CFT4MB9hg7rKwWmbFhGQTCnmxdpnGBKPDbHTPLSgV/O/D2F_D300_20090827-_TDM5650.jpg", + ), }, -}, - -{ - "#url" : "https://tdm.smugmug.com/Nature/Dove/i-kCsLJT6", - "#category": ("", "smugmug", "image"), - "#class" : smugmug.SmugmugImageExtractor, - "#urls" : "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/FfB6gSx8X6MS7Hvww7GK7tWsrfdtwCx79hCVzwSm/O/Fluff_20090521-_DSC1542.jpg", - "#sha1_content": "ecbd9d7b4f75a637abc8d35319be9ec065a44eb0", - - "Image": { - "Altitude" : 0, - "CanBuy" : True, - "CanEdit" : False, - "CanShare" : True, - "Caption" : "White Wing Dove", - "Collectable": False, - "Comments" : True, - "ComponentFileTypes": [], - "Date" : "2009-08-01T23:00:56+00:00", - "DateTimeOriginal": "2009-05-22T00:05:36+00:00", - "DateTimeUploaded": "2009-08-01T23:00:56+00:00", - "EZProject" : False, - "FileName" : "Fluff_20090521-_DSC1542.jpg", - "Format" : "JPG", - "FormattedValues": { - "Caption": { - "html": "White Wing Dove", - "text": "White Wing Dove", - }, - "FileName": { - "html": "Fluff_20090521-_DSC1542.jpg", - "text": "Fluff_20090521-_DSC1542.jpg", + { + "#url": "smugmug:album:Fb7hMs", + "#comment": "empty", + "#category": ("", "smugmug", "album"), + "#class": smugmug.SmugmugAlbumExtractor, + "#count": 0, + }, + { + "#url": "smugmug:album:6VRT8G", + "#comment": "no 'User'", + "#category": ("", "smugmug", "album"), + "#class": smugmug.SmugmugAlbumExtractor, + "#sha1_url": "17837ff2c78a6e2335291666f43d620d82f2926a", + "User": { + "Name": "", + "NickName": "", + "QuickShare": False, + "RefTag": "", + "ResponseLevel": "Public", + "Uri": "", + "ViewPassHint": "", + "WebUri": "", + }, + }, + { + "#url": "https://tdm.smugmug.com/Nature/Dove/i-kCsLJT6", + "#category": ("", "smugmug", "image"), + "#class": smugmug.SmugmugImageExtractor, + "#urls": "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/FfB6gSx8X6MS7Hvww7GK7tWsrfdtwCx79hCVzwSm/O/Fluff_20090521-_DSC1542.jpg", + "#sha1_content": "ecbd9d7b4f75a637abc8d35319be9ec065a44eb0", + "Image": { + "Altitude": 0, + "CanBuy": True, + "CanEdit": False, + "CanShare": True, + "Caption": "White Wing Dove", + "Collectable": False, + "Comments": True, + "ComponentFileTypes": [], + "Date": "2009-08-01T23:00:56+00:00", + "DateTimeOriginal": "2009-05-22T00:05:36+00:00", + "DateTimeUploaded": "2009-08-01T23:00:56+00:00", + "EZProject": False, + "FileName": "Fluff_20090521-_DSC1542.jpg", + "Format": "JPG", + "FormattedValues": { + "Caption": { + "html": "White Wing Dove", + "text": "White Wing Dove", + }, + "FileName": { + "html": "Fluff_20090521-_DSC1542.jpg", + "text": "Fluff_20090521-_DSC1542.jpg", + }, }, + "Height": 1008, + "Hidden": False, + "ImageKey": "kCsLJT6", + "IsArchive": False, + "IsVideo": False, + "KeywordArray": [ + "Birds", + "Dove", + "White Wing Dove", + ], + "Keywords": "Birds; Dove; White Wing Dove", + "LastUpdated": "2012-11-03T20:01:15+00:00", + "Latitude": "0", + "Longitude": "0", + "OriginalHeight": 1008, + "OriginalSize": 381297, + "OriginalWidth": 1024, + "PreferredDisplayFileExtension": "JPG", + "Processing": False, + "Protected": True, + "Serial": 0, + "ShowKeywords": True, + "Size": 381297, + "Status": "Open", + "SubStatus": "NFS", + "ThumbnailUrl": "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/Df2nQXwHWSmmh4W2CjhJJdxDcZWbhkKTG86JXp9x2/Th/Fluff_20090521-_DSC1542-Th.jpg", + "Title": "", + "UploadKey": "608043804", + "Uri": "/api/v2/image/kCsLJT6-0", + "Url": "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/FfB6gSx8X6MS7Hvww7GK7tWsrfdtwCx79hCVzwSm/O/Fluff_20090521-_DSC1542.jpg", + "Watermark": "No", + "Watermarked": False, + "Width": 1024, }, - "Height" : 1008, - "Hidden" : False, - "ImageKey" : "kCsLJT6", - "IsArchive" : False, - "IsVideo" : False, - "KeywordArray": [ - "Birds", - "Dove", - "White Wing Dove", - ], - "Keywords" : "Birds; Dove; White Wing Dove", - "LastUpdated": "2012-11-03T20:01:15+00:00", - "Latitude" : "0", - "Longitude" : "0", - "OriginalHeight": 1008, - "OriginalSize": 381297, - "OriginalWidth": 1024, - "PreferredDisplayFileExtension": "JPG", - "Processing" : False, - "Protected" : True, - "Serial" : 0, - "ShowKeywords": True, - "Size" : 381297, - "Status" : "Open", - "SubStatus" : "NFS", - "ThumbnailUrl": "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/Df2nQXwHWSmmh4W2CjhJJdxDcZWbhkKTG86JXp9x2/Th/Fluff_20090521-_DSC1542-Th.jpg", - "Title" : "", - "UploadKey" : "608043804", - "Uri" : "/api/v2/image/kCsLJT6-0", - "Url" : "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/FfB6gSx8X6MS7Hvww7GK7tWsrfdtwCx79hCVzwSm/O/Fluff_20090521-_DSC1542.jpg", - "Watermark" : "No", - "Watermarked": False, - "Width" : 1024, + "extension": "jpg", + "filename": "Fluff_20090521-_DSC1542", + }, + { + "#url": "https://tstravels.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB", + "#comment": "video", + "#category": ("", "smugmug", "image"), + "#class": smugmug.SmugmugImageExtractor, + "#urls": "https://photos.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB/0/Q4Qg6kt4SqVcKsSLWM4PnhMhSTS2r5BkmBMd9Dx4/1920/657%20WS3-1920.mp4", + }, + { + "#url": "https://tdm.smugmug.com/Nature/Dove", + "#category": ("", "smugmug", "path"), + "#class": smugmug.SmugmugPathExtractor, + "#pattern": "smugmug:album:cr4C7f$", + }, + { + "#url": "https://tdm.smugmug.com/", + "#category": ("", "smugmug", "path"), + "#class": smugmug.SmugmugPathExtractor, + "#pattern": smugmug.SmugmugAlbumExtractor.pattern, + "#sha1_url": "1640028712875b90974e5aecd91b60e6de6138c7", + }, + { + "#url": "https://www.smugmug.com/gallery/n-GLCjnD/", + "#comment": "gallery node without owner", + "#category": ("", "smugmug", "path"), + "#class": smugmug.SmugmugPathExtractor, + "#pattern": "smugmug:album:6VRT8G$", + }, + { + "#url": "smugmug:www.sitkapics.com/TREES-and-TRAILS/", + "#comment": "custom domain", + "#category": ("", "smugmug", "path"), + "#class": smugmug.SmugmugPathExtractor, + "#pattern": "smugmug:album:ct8Nds$", + }, + { + "#url": "smugmug:www.sitkapics.com/", + "#category": ("", "smugmug", "path"), + "#class": smugmug.SmugmugPathExtractor, + "#pattern": r"smugmug:album:\w{6}$", + "#count": ">= 14", + }, + { + "#url": "smugmug:https://www.sitkapics.com/", + "#category": ("", "smugmug", "path"), + "#class": smugmug.SmugmugPathExtractor, }, - "extension": "jpg", - "filename": "Fluff_20090521-_DSC1542", -}, - -{ - "#url" : "https://tstravels.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB", - "#comment" : "video", - "#category": ("", "smugmug", "image"), - "#class" : smugmug.SmugmugImageExtractor, - "#urls" : "https://photos.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB/0/Q4Qg6kt4SqVcKsSLWM4PnhMhSTS2r5BkmBMd9Dx4/1920/657%20WS3-1920.mp4", -}, - -{ - "#url" : "https://tdm.smugmug.com/Nature/Dove", - "#category": ("", "smugmug", "path"), - "#class" : smugmug.SmugmugPathExtractor, - "#pattern" : "smugmug:album:cr4C7f$", -}, - -{ - "#url" : "https://tdm.smugmug.com/", - "#category": ("", "smugmug", "path"), - "#class" : smugmug.SmugmugPathExtractor, - "#pattern" : smugmug.SmugmugAlbumExtractor.pattern, - "#sha1_url": "1640028712875b90974e5aecd91b60e6de6138c7", -}, - -{ - "#url" : "https://www.smugmug.com/gallery/n-GLCjnD/", - "#comment" : "gallery node without owner", - "#category": ("", "smugmug", "path"), - "#class" : smugmug.SmugmugPathExtractor, - "#pattern" : "smugmug:album:6VRT8G$", -}, - -{ - "#url" : "smugmug:www.sitkapics.com/TREES-and-TRAILS/", - "#comment" : "custom domain", - "#category": ("", "smugmug", "path"), - "#class" : smugmug.SmugmugPathExtractor, - "#pattern" : "smugmug:album:ct8Nds$", -}, - -{ - "#url" : "smugmug:www.sitkapics.com/", - "#category": ("", "smugmug", "path"), - "#class" : smugmug.SmugmugPathExtractor, - "#pattern" : r"smugmug:album:\w{6}$", - "#count" : ">= 14", -}, - -{ - "#url" : "smugmug:https://www.sitkapics.com/", - "#category": ("", "smugmug", "path"), - "#class" : smugmug.SmugmugPathExtractor, -}, - ) diff --git a/test/results/snootbooru.py b/test/results/snootbooru.py index 25a294ebb..4732ef26f 100644 --- a/test/results/snootbooru.py +++ b/test/results/snootbooru.py @@ -1,76 +1,64 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import szurubooru - __tests__ = ( -{ - "#url" : "https://snootbooru.com/posts/query=sport", - "#category": ("szurubooru", "snootbooru", "tag"), - "#class" : szurubooru.SzurubooruTagExtractor, - "#pattern" : r"https://snootbooru\.com/data/posts/\d+_[0-9a-f]{16}\.\w+", - "#count" : range(35, 50), -}, - -{ - "#url" : "https://snootbooru.com/post/14511", - "#category": ("szurubooru", "snootbooru", "post"), - "#class" : szurubooru.SzurubooruPostExtractor, - "#urls" : "https://snootbooru.com/data/posts/14511_e753313112755da6.png", - "#sha1_content": "e69e61e61c5372514808480aae3a8e355c9cd6fb", - - "canvasHeight" : 1000, - "canvasWidth" : 1414, - "checksum" : "e69e61e61c5372514808480aae3a8e355c9cd6fb", - "checksumMD5" : "f4f4ddfcbdf367f466ede0980acb3d7d", - "commentCount" : int, - "comments" : list, - "contentUrl" : "data/posts/14511_e753313112755da6.png", - "creationTime" : "2023-12-02T01:11:01.433664Z", - "date" : "dt:2023-12-02 01:11:01", - "extension" : "png", - "favoriteCount": int, - "favoritedBy" : list, - "featureCount" : int, - "fileSize" : 270639, - "filename" : "14511_e753313112755da6", - "flags" : [], - "hasCustomThumbnail": False, - "id" : 14511, - "lastEditTime" : "2023-12-02T01:12:09.500217Z", - "lastFeatureTime": None, - "mimeType" : "image/png", - "noteCount" : 0, - "notes" : [], - "ownFavorite" : False, - "ownScore" : 0, - "pools" : [], - "relationCount": 0, - "relations" : [], - "safety" : "safe", - "score" : 0, - "source" : None, - "tagCount" : 3, - "tags" : [ - "transparent", - "sport", - "text", - ], - "tags_default" : [ - "sport", - "text" - ], - "thumbnailUrl" : "data/generated-thumbnails/14511_e753313112755da6.jpg", - "type" : "image", - "user" : { - "avatarUrl": "data/avatars/komp.png", - "name": "komp" + { + "#url": "https://snootbooru.com/posts/query=sport", + "#category": ("szurubooru", "snootbooru", "tag"), + "#class": szurubooru.SzurubooruTagExtractor, + "#pattern": r"https://snootbooru\.com/data/posts/\d+_[0-9a-f]{16}\.\w+", + "#count": range(35, 50), + }, + { + "#url": "https://snootbooru.com/post/14511", + "#category": ("szurubooru", "snootbooru", "post"), + "#class": szurubooru.SzurubooruPostExtractor, + "#urls": "https://snootbooru.com/data/posts/14511_e753313112755da6.png", + "#sha1_content": "e69e61e61c5372514808480aae3a8e355c9cd6fb", + "canvasHeight": 1000, + "canvasWidth": 1414, + "checksum": "e69e61e61c5372514808480aae3a8e355c9cd6fb", + "checksumMD5": "f4f4ddfcbdf367f466ede0980acb3d7d", + "commentCount": int, + "comments": list, + "contentUrl": "data/posts/14511_e753313112755da6.png", + "creationTime": "2023-12-02T01:11:01.433664Z", + "date": "dt:2023-12-02 01:11:01", + "extension": "png", + "favoriteCount": int, + "favoritedBy": list, + "featureCount": int, + "fileSize": 270639, + "filename": "14511_e753313112755da6", + "flags": [], + "hasCustomThumbnail": False, + "id": 14511, + "lastEditTime": "2023-12-02T01:12:09.500217Z", + "lastFeatureTime": None, + "mimeType": "image/png", + "noteCount": 0, + "notes": [], + "ownFavorite": False, + "ownScore": 0, + "pools": [], + "relationCount": 0, + "relations": [], + "safety": "safe", + "score": 0, + "source": None, + "tagCount": 3, + "tags": [ + "transparent", + "sport", + "text", + ], + "tags_default": ["sport", "text"], + "thumbnailUrl": "data/generated-thumbnails/14511_e753313112755da6.jpg", + "type": "image", + "user": {"avatarUrl": "data/avatars/komp.png", "name": "komp"}, + "version": 2, }, - "version" : 2, -}, - ) diff --git a/test/results/soundgasm.py b/test/results/soundgasm.py index 16f0a17cc..19b96f6a7 100644 --- a/test/results/soundgasm.py +++ b/test/results/soundgasm.py @@ -1,47 +1,39 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import soundgasm - __tests__ = ( -{ - "#url" : "https://soundgasm.net/u/ClassWarAndPuppies2/687-Otto-von-Toontown-12822", - "#category": ("", "soundgasm", "audio"), - "#class" : soundgasm.SoundgasmAudioExtractor, - "#pattern" : r"https://media\.soundgasm\.net/sounds/26cb2b23b2f2c6094b40ee3a9167271e274b570a\.m4a", - - "description": "We celebrate today’s important prisoner swap, and finally bring the 2022 mid-terms to a close with Raphael Warnock’s defeat of Herschel Walker in Georgia. Then, we take a look at the Qanon-addled attempt to overthrow the German government and install Heinrich XIII Prince of Reuss as kaiser.", - "extension" : "m4a", - "filename" : "26cb2b23b2f2c6094b40ee3a9167271e274b570a", - "slug" : "687-Otto-von-Toontown-12822", - "title" : "687 - Otto von Toontown (12/8/22)", - "user" : "ClassWarAndPuppies2", -}, - -{ - "#url" : "https://www.soundgasm.net/user/ClassWarAndPuppies2/687-Otto-von-Toontown-12822", - "#category": ("", "soundgasm", "audio"), - "#class" : soundgasm.SoundgasmAudioExtractor, -}, - -{ - "#url" : "https://soundgasm.net/u/fierce-aphrodite", - "#category": ("", "soundgasm", "user"), - "#class" : soundgasm.SoundgasmUserExtractor, - "#pattern" : r"https://media\.soundgasm\.net/sounds/[0-9a-f]{40}\.m4a", - "#count" : ">= 15", - - "description": str, - "extension" : "m4a", - "filename" : r"re:^[0-9a-f]{40}$", - "slug" : str, - "title" : str, - "url" : str, - "user" : "fierce-aphrodite", -}, - + { + "#url": "https://soundgasm.net/u/ClassWarAndPuppies2/687-Otto-von-Toontown-12822", + "#category": ("", "soundgasm", "audio"), + "#class": soundgasm.SoundgasmAudioExtractor, + "#pattern": r"https://media\.soundgasm\.net/sounds/26cb2b23b2f2c6094b40ee3a9167271e274b570a\.m4a", + "description": "We celebrate today’s important prisoner swap, and finally bring the 2022 mid-terms to a close with Raphael Warnock’s defeat of Herschel Walker in Georgia. Then, we take a look at the Qanon-addled attempt to overthrow the German government and install Heinrich XIII Prince of Reuss as kaiser.", + "extension": "m4a", + "filename": "26cb2b23b2f2c6094b40ee3a9167271e274b570a", + "slug": "687-Otto-von-Toontown-12822", + "title": "687 - Otto von Toontown (12/8/22)", + "user": "ClassWarAndPuppies2", + }, + { + "#url": "https://www.soundgasm.net/user/ClassWarAndPuppies2/687-Otto-von-Toontown-12822", + "#category": ("", "soundgasm", "audio"), + "#class": soundgasm.SoundgasmAudioExtractor, + }, + { + "#url": "https://soundgasm.net/u/fierce-aphrodite", + "#category": ("", "soundgasm", "user"), + "#class": soundgasm.SoundgasmUserExtractor, + "#pattern": r"https://media\.soundgasm\.net/sounds/[0-9a-f]{40}\.m4a", + "#count": ">= 15", + "description": str, + "extension": "m4a", + "filename": r"re:^[0-9a-f]{40}$", + "slug": str, + "title": str, + "url": str, + "user": "fierce-aphrodite", + }, ) diff --git a/test/results/speakerdeck.py b/test/results/speakerdeck.py index b08932339..7baa7d43f 100644 --- a/test/results/speakerdeck.py +++ b/test/results/speakerdeck.py @@ -1,28 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import speakerdeck - __tests__ = ( -{ - "#url" : "https://speakerdeck.com/speakerdeck/introduction-to-speakerdeck", - "#category": ("", "speakerdeck", "presentation"), - "#class" : speakerdeck.SpeakerdeckPresentationExtractor, - "#pattern" : r"https://files.speakerdeck.com/presentations/50021f75cf1db900020005e7/slide_\d+.jpg", - "#count" : 6, - "#sha1_content": "75c7abf0969b0bcab23e0da9712c95ee5113db3a", - - "author" : "Speaker Deck", - "count" : 6, - "num" : range(1, 6), - "presentation" : "introduction-to-speakerdeck", - "presentation_id": "50021f75cf1db900020005e7", - "title" : "Introduction to SpeakerDeck", - "user" : "speakerdeck", -}, - + { + "#url": "https://speakerdeck.com/speakerdeck/introduction-to-speakerdeck", + "#category": ("", "speakerdeck", "presentation"), + "#class": speakerdeck.SpeakerdeckPresentationExtractor, + "#pattern": r"https://files.speakerdeck.com/presentations/50021f75cf1db900020005e7/slide_\d+.jpg", + "#count": 6, + "#sha1_content": "75c7abf0969b0bcab23e0da9712c95ee5113db3a", + "author": "Speaker Deck", + "count": 6, + "num": range(1, 6), + "presentation": "introduction-to-speakerdeck", + "presentation_id": "50021f75cf1db900020005e7", + "title": "Introduction to SpeakerDeck", + "user": "speakerdeck", + }, ) diff --git a/test/results/steamgriddb.py b/test/results/steamgriddb.py index 8cb39d172..8b2068619 100644 --- a/test/results/steamgriddb.py +++ b/test/results/steamgriddb.py @@ -1,118 +1,100 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import steamgriddb from gallery_dl import exception - +from gallery_dl.extractor import steamgriddb __tests__ = ( -{ - "#url" : "https://www.steamgriddb.com/grid/368023", - "#comment" : "deleted", - "#category": ("", "steamgriddb", "asset"), - "#class" : steamgriddb.SteamgriddbAssetExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.steamgriddb.com/grid/132605", - "#category": ("", "steamgriddb", "asset"), - "#class" : steamgriddb.SteamgriddbAssetExtractor, - "#count" : 2, - "#sha1_url" : "4ff9158c008a1f01921d7553bcabf5e6204cdc79", - "#sha1_content": "bc16c5eebf71463abdb33cfbf4b45a2fe092a2b2", - - "game": { - "id" : 5247997, - "name": "OMORI", + { + "#url": "https://www.steamgriddb.com/grid/368023", + "#comment": "deleted", + "#category": ("", "steamgriddb", "asset"), + "#class": steamgriddb.SteamgriddbAssetExtractor, + "#exception": exception.NotFoundError, }, -}, - -{ - "#url" : "https://www.steamgriddb.com/grid/132605", - "#category": ("", "steamgriddb", "asset"), - "#class" : steamgriddb.SteamgriddbAssetExtractor, - "#options" : {"download-fake-png": False}, - "#count" : 1, - "#sha1_url" : "f6819c593ff65f15864796fb89581f05d21adddb", - "#sha1_content": "0d9e6114dd8bb9699182fbb7c6bd9064d8b0b6cd", - - "game": { - "id" : 5247997, - "name": "OMORI", + { + "#url": "https://www.steamgriddb.com/grid/132605", + "#category": ("", "steamgriddb", "asset"), + "#class": steamgriddb.SteamgriddbAssetExtractor, + "#count": 2, + "#sha1_url": "4ff9158c008a1f01921d7553bcabf5e6204cdc79", + "#sha1_content": "bc16c5eebf71463abdb33cfbf4b45a2fe092a2b2", + "game": { + "id": 5247997, + "name": "OMORI", + }, + }, + { + "#url": "https://www.steamgriddb.com/grid/132605", + "#category": ("", "steamgriddb", "asset"), + "#class": steamgriddb.SteamgriddbAssetExtractor, + "#options": {"download-fake-png": False}, + "#count": 1, + "#sha1_url": "f6819c593ff65f15864796fb89581f05d21adddb", + "#sha1_content": "0d9e6114dd8bb9699182fbb7c6bd9064d8b0b6cd", + "game": { + "id": 5247997, + "name": "OMORI", + }, + }, + { + "#url": "https://www.steamgriddb.com/hero/61104", + "#category": ("", "steamgriddb", "asset"), + "#class": steamgriddb.SteamgriddbAssetExtractor, + }, + { + "#url": "https://www.steamgriddb.com/logo/9610", + "#category": ("", "steamgriddb", "asset"), + "#class": steamgriddb.SteamgriddbAssetExtractor, + }, + { + "#url": "https://www.steamgriddb.com/icon/173", + "#category": ("", "steamgriddb", "asset"), + "#class": steamgriddb.SteamgriddbAssetExtractor, + }, + { + "#url": "https://www.steamgriddb.com/game/5259324/grids", + "#category": ("", "steamgriddb", "grids"), + "#class": steamgriddb.SteamgriddbGridsExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.steamgriddb.com/game/5259324/grids", + "#category": ("", "steamgriddb", "grids"), + "#class": steamgriddb.SteamgriddbGridsExtractor, + "#options": {"humor": False, "epilepsy": False, "untagged": False}, + "#range": "1-33", + "#count": 33, + }, + { + "#url": "https://www.steamgriddb.com/game/5331605/heroes", + "#category": ("", "steamgriddb", "heroes"), + "#class": steamgriddb.SteamgriddbHeroesExtractor, + }, + { + "#url": "https://www.steamgriddb.com/game/5255394/logos", + "#category": ("", "steamgriddb", "logos"), + "#class": steamgriddb.SteamgriddbLogosExtractor, + }, + { + "#url": "https://www.steamgriddb.com/game/5279790/icons", + "#category": ("", "steamgriddb", "icons"), + "#class": steamgriddb.SteamgriddbIconsExtractor, + }, + { + "#url": "https://www.steamgriddb.com/collection/332/grids", + "#category": ("", "steamgriddb", "grids"), + "#class": steamgriddb.SteamgriddbGridsExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.steamgriddb.com/collection/332/heroes", + "#category": ("", "steamgriddb", "heroes"), + "#class": steamgriddb.SteamgriddbHeroesExtractor, + "#options": {"animated": False}, + "#count": 0, }, -}, - -{ - "#url" : "https://www.steamgriddb.com/hero/61104", - "#category": ("", "steamgriddb", "asset"), - "#class" : steamgriddb.SteamgriddbAssetExtractor, -}, - -{ - "#url" : "https://www.steamgriddb.com/logo/9610", - "#category": ("", "steamgriddb", "asset"), - "#class" : steamgriddb.SteamgriddbAssetExtractor, -}, - -{ - "#url" : "https://www.steamgriddb.com/icon/173", - "#category": ("", "steamgriddb", "asset"), - "#class" : steamgriddb.SteamgriddbAssetExtractor, -}, - -{ - "#url" : "https://www.steamgriddb.com/game/5259324/grids", - "#category": ("", "steamgriddb", "grids"), - "#class" : steamgriddb.SteamgriddbGridsExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.steamgriddb.com/game/5259324/grids", - "#category": ("", "steamgriddb", "grids"), - "#class" : steamgriddb.SteamgriddbGridsExtractor, - "#options" : {"humor": False, "epilepsy": False, "untagged": False}, - "#range" : "1-33", - "#count" : 33, -}, - -{ - "#url" : "https://www.steamgriddb.com/game/5331605/heroes", - "#category": ("", "steamgriddb", "heroes"), - "#class" : steamgriddb.SteamgriddbHeroesExtractor, -}, - -{ - "#url" : "https://www.steamgriddb.com/game/5255394/logos", - "#category": ("", "steamgriddb", "logos"), - "#class" : steamgriddb.SteamgriddbLogosExtractor, -}, - -{ - "#url" : "https://www.steamgriddb.com/game/5279790/icons", - "#category": ("", "steamgriddb", "icons"), - "#class" : steamgriddb.SteamgriddbIconsExtractor, -}, - -{ - "#url" : "https://www.steamgriddb.com/collection/332/grids", - "#category": ("", "steamgriddb", "grids"), - "#class" : steamgriddb.SteamgriddbGridsExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.steamgriddb.com/collection/332/heroes", - "#category": ("", "steamgriddb", "heroes"), - "#class" : steamgriddb.SteamgriddbHeroesExtractor, - "#options" : {"animated": False}, - "#count" : 0, -}, - ) diff --git a/test/results/subscribestar.py b/test/results/subscribestar.py index 8fe6d8dc8..48c2b3eca 100644 --- a/test/results/subscribestar.py +++ b/test/results/subscribestar.py @@ -1,82 +1,71 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import subscribestar import datetime +from gallery_dl.extractor import subscribestar __tests__ = ( -{ - "#url" : "https://www.subscribestar.com/subscribestar", - "#category": ("", "subscribestar", "user"), - "#class" : subscribestar.SubscribestarUserExtractor, - "#pattern" : r"https://(ss-uploads-prod\.b-cdn|\w+\.cloudfront)\.net/uploads(_v2)?/users/11/", - "#count" : ">= 20", - - "author_id" : 11, - "author_name": "subscribestar", - "author_nick": "SubscribeStar", - "content" : str, - "date" : datetime.datetime, - "id" : int, - "num" : int, - "post_id" : int, - "type" : r"re:image|video|attachment", - "url" : str, - "?pinned" : bool, -}, - -{ - "#url" : "https://www.subscribestar.com/subscribestar", - "#category": ("", "subscribestar", "user"), - "#class" : subscribestar.SubscribestarUserExtractor, - "#options" : {"metadata": True}, - "#range" : "1", - - "date": datetime.datetime, -}, - -{ - "#url" : "https://subscribestar.adult/kanashiipanda", - "#category": ("", "subscribestar", "user-adult"), - "#class" : subscribestar.SubscribestarUserExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.subscribestar.com/posts/102468", - "#category": ("", "subscribestar", "post"), - "#class" : subscribestar.SubscribestarPostExtractor, - "#count" : 1, - - "author_id" : 11, - "author_name": "subscribestar", - "author_nick": "SubscribeStar", - "content" : r"re:

          Brand Guidelines and Assets

          ", - "date" : "dt:2020-05-07 12:33:00", - "extension" : "jpg", - "filename" : "8ff61299-b249-47dc-880a-cdacc9081c62", - "group" : "imgs_and_videos", - "height" : 291, - "id" : 203885, - "num" : 1, - "pinned" : False, - "post_id" : 102468, - "type" : "image", - "width" : 700, -}, - -{ - "#url" : "https://subscribestar.adult/posts/22950", - "#category": ("", "subscribestar", "post-adult"), - "#class" : subscribestar.SubscribestarPostExtractor, - "#count" : 1, - - "date": "dt:2019-04-28 07:32:00", -}, - + { + "#url": "https://www.subscribestar.com/subscribestar", + "#category": ("", "subscribestar", "user"), + "#class": subscribestar.SubscribestarUserExtractor, + "#pattern": r"https://(ss-uploads-prod\.b-cdn|\w+\.cloudfront)\.net/uploads(_v2)?/users/11/", + "#count": ">= 20", + "author_id": 11, + "author_name": "subscribestar", + "author_nick": "SubscribeStar", + "content": str, + "date": datetime.datetime, + "id": int, + "num": int, + "post_id": int, + "type": r"re:image|video|attachment", + "url": str, + "?pinned": bool, + }, + { + "#url": "https://www.subscribestar.com/subscribestar", + "#category": ("", "subscribestar", "user"), + "#class": subscribestar.SubscribestarUserExtractor, + "#options": {"metadata": True}, + "#range": "1", + "date": datetime.datetime, + }, + { + "#url": "https://subscribestar.adult/kanashiipanda", + "#category": ("", "subscribestar", "user-adult"), + "#class": subscribestar.SubscribestarUserExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.subscribestar.com/posts/102468", + "#category": ("", "subscribestar", "post"), + "#class": subscribestar.SubscribestarPostExtractor, + "#count": 1, + "author_id": 11, + "author_name": "subscribestar", + "author_nick": "SubscribeStar", + "content": r"re:

          Brand Guidelines and Assets

          ", + "date": "dt:2020-05-07 12:33:00", + "extension": "jpg", + "filename": "8ff61299-b249-47dc-880a-cdacc9081c62", + "group": "imgs_and_videos", + "height": 291, + "id": 203885, + "num": 1, + "pinned": False, + "post_id": 102468, + "type": "image", + "width": 700, + }, + { + "#url": "https://subscribestar.adult/posts/22950", + "#category": ("", "subscribestar", "post-adult"), + "#class": subscribestar.SubscribestarPostExtractor, + "#count": 1, + "date": "dt:2019-04-28 07:32:00", + }, ) diff --git a/test/results/sushiski.py b/test/results/sushiski.py index e1bbd20a5..4e77ea5fd 100644 --- a/test/results/sushiski.py +++ b/test/results/sushiski.py @@ -1,37 +1,30 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import misskey - __tests__ = ( -{ - "#url" : "https://sushi.ski/@ui@misskey.04.si", - "#category": ("misskey", "sushi.ski", "user"), - "#class" : misskey.MisskeyUserExtractor, -}, - -{ - "#url" : "https://sushi.ski/@hatusimo_sigure/following", - "#category": ("misskey", "sushi.ski", "following"), - "#class" : misskey.MisskeyFollowingExtractor, -}, - -{ - "#url" : "https://sushi.ski/notes/9bm3x4ksqw", - "#category": ("misskey", "sushi.ski", "note"), - "#class" : misskey.MisskeyNoteExtractor, - "#pattern" : r"https://media\.sushi\.ski/files/[\w-]+\.png", - "#count" : 1, -}, - -{ - "#url" : "https://sushi.ski/my/favorites", - "#category": ("misskey", "sushi.ski", "favorite"), - "#class" : misskey.MisskeyFavoriteExtractor, -}, - + { + "#url": "https://sushi.ski/@ui@misskey.04.si", + "#category": ("misskey", "sushi.ski", "user"), + "#class": misskey.MisskeyUserExtractor, + }, + { + "#url": "https://sushi.ski/@hatusimo_sigure/following", + "#category": ("misskey", "sushi.ski", "following"), + "#class": misskey.MisskeyFollowingExtractor, + }, + { + "#url": "https://sushi.ski/notes/9bm3x4ksqw", + "#category": ("misskey", "sushi.ski", "note"), + "#class": misskey.MisskeyNoteExtractor, + "#pattern": r"https://media\.sushi\.ski/files/[\w-]+\.png", + "#count": 1, + }, + { + "#url": "https://sushi.ski/my/favorites", + "#category": ("misskey", "sushi.ski", "favorite"), + "#class": misskey.MisskeyFavoriteExtractor, + }, ) diff --git a/test/results/tapas.py b/test/results/tapas.py index d4289383f..0a608d5e1 100644 --- a/test/results/tapas.py +++ b/test/results/tapas.py @@ -1,89 +1,81 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import tapas - __tests__ = ( -{ - "#url" : "https://tapas.io/series/just-leave-me-be", - "#category": ("", "tapas", "series"), - "#class" : tapas.TapasSeriesExtractor, - "#pattern" : r"https://us-a\.tapas\.io/pc/\w\w/[0-9a-f-]+\.jpg", - "#count" : 132, -}, - -{ - "#url" : "https://tapas.io/series/yona", - "#comment" : "mature", - "#category": ("", "tapas", "series"), - "#class" : tapas.TapasSeriesExtractor, - "#count" : 26, -}, - -{ - "#url" : "https://tapas.io/episode/2068651", - "#category": ("", "tapas", "episode"), - "#class" : tapas.TapasEpisodeExtractor, - "#pattern" : "^text:", - "#sha1_url": "0b53644c864a0a097f65accea6bb620be9671078", - - "book" : True, - "comment_cnt" : int, - "date" : "dt:2021-02-23 16:02:07", - "early_access" : False, - "escape_title" : "You are a Tomb Raider (2)", - "free" : True, - "id" : 2068651, - "like_cnt" : int, - "liked" : bool, - "mature" : False, - "next_ep_id" : 2068652, - "nsfw" : False, - "nu" : False, - "num" : 1, - "open_comments" : True, - "pending_scene" : 2, - "prev_ep_id" : 2068650, - "publish_date" : "2021-02-23T16:02:07Z", - "read" : bool, - "related_ep_id" : None, - "relative_publish_date": "Feb 23, 2021", - "scene" : 2, - "scheduled" : False, - "title" : "You are a Tomb Raider (2)", - "unlock_cnt" : 0, - "unlocked" : False, - "view_cnt" : int, - "series" : { - "genre" : dict, - "has_book_cover": True, - "has_top_banner": True, - "id" : 199931, - "premium" : True, - "sale_type" : "WAIT_OR_MUST_PAY", - "subscribed" : bool, - "thumbsup_cnt" : int, - "title" : "Tomb Raider King", - "type" : "BOOKS", - "url" : "tomb-raider-king-novel", + { + "#url": "https://tapas.io/series/just-leave-me-be", + "#category": ("", "tapas", "series"), + "#class": tapas.TapasSeriesExtractor, + "#pattern": r"https://us-a\.tapas\.io/pc/\w\w/[0-9a-f-]+\.jpg", + "#count": 132, + }, + { + "#url": "https://tapas.io/series/yona", + "#comment": "mature", + "#category": ("", "tapas", "series"), + "#class": tapas.TapasSeriesExtractor, + "#count": 26, + }, + { + "#url": "https://tapas.io/episode/2068651", + "#category": ("", "tapas", "episode"), + "#class": tapas.TapasEpisodeExtractor, + "#pattern": "^text:", + "#sha1_url": "0b53644c864a0a097f65accea6bb620be9671078", + "book": True, + "comment_cnt": int, + "date": "dt:2021-02-23 16:02:07", + "early_access": False, + "escape_title": "You are a Tomb Raider (2)", + "free": True, + "id": 2068651, + "like_cnt": int, + "liked": bool, + "mature": False, + "next_ep_id": 2068652, + "nsfw": False, + "nu": False, + "num": 1, + "open_comments": True, + "pending_scene": 2, + "prev_ep_id": 2068650, + "publish_date": "2021-02-23T16:02:07Z", + "read": bool, + "related_ep_id": None, + "relative_publish_date": "Feb 23, 2021", + "scene": 2, + "scheduled": False, + "title": "You are a Tomb Raider (2)", + "unlock_cnt": 0, + "unlocked": False, + "view_cnt": int, + "series": { + "genre": dict, + "has_book_cover": True, + "has_top_banner": True, + "id": 199931, + "premium": True, + "sale_type": "WAIT_OR_MUST_PAY", + "subscribed": bool, + "thumbsup_cnt": int, + "title": "Tomb Raider King", + "type": "BOOKS", + "url": "tomb-raider-king-novel", + }, + }, + { + "#url": "https://tapas.io/SANG123/series", + "#comment": "#5306", + "#category": ("", "tapas", "creator"), + "#class": tapas.TapasCreatorExtractor, + "#urls": ( + "https://tapas.io/series/the-return-of-the-disaster-class-hero-novel", + "https://tapas.io/series/the-return-of-the-disaster-class-hero", + "https://tapas.io/series/tomb-raider-king", + "https://tapas.io/series/tomb-raider-king-novel", + ), }, -}, - -{ - "#url" : "https://tapas.io/SANG123/series", - "#comment" : "#5306", - "#category": ("", "tapas", "creator"), - "#class" : tapas.TapasCreatorExtractor, - "#urls" : ( - "https://tapas.io/series/the-return-of-the-disaster-class-hero-novel", - "https://tapas.io/series/the-return-of-the-disaster-class-hero", - "https://tapas.io/series/tomb-raider-king", - "https://tapas.io/series/tomb-raider-king-novel", - ), -}, - ) diff --git a/test/results/tbib.py b/test/results/tbib.py index e4481da5e..2faa53cd8 100644 --- a/test/results/tbib.py +++ b/test/results/tbib.py @@ -1,33 +1,27 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v02 - __tests__ = ( -{ - "#url" : "https://tbib.org/index.php?page=post&s=list&tags=yuyaiyaui", - "#category": ("gelbooru_v02", "tbib", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#count" : ">= 120", -}, - -{ - "#url" : "https://tbib.org/index.php?page=favorites&s=view&id=7881", - "#category": ("gelbooru_v02", "tbib", "favorite"), - "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, - "#count" : 3, -}, - -{ - "#url" : "https://tbib.org/index.php?page=post&s=view&id=9233957", - "#category": ("gelbooru_v02", "tbib", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#sha1_url" : "5a6ebe07bfff8e6d27f7c30b5480f27abcb577d2", - "#sha1_content": "1c3831b6fbaa4686e3c79035b5d98460b1c85c43", -}, - + { + "#url": "https://tbib.org/index.php?page=post&s=list&tags=yuyaiyaui", + "#category": ("gelbooru_v02", "tbib", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#count": ">= 120", + }, + { + "#url": "https://tbib.org/index.php?page=favorites&s=view&id=7881", + "#category": ("gelbooru_v02", "tbib", "favorite"), + "#class": gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count": 3, + }, + { + "#url": "https://tbib.org/index.php?page=post&s=view&id=9233957", + "#category": ("gelbooru_v02", "tbib", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#sha1_url": "5a6ebe07bfff8e6d27f7c30b5480f27abcb577d2", + "#sha1_content": "1c3831b6fbaa4686e3c79035b5d98460b1c85c43", + }, ) diff --git a/test/results/tcbscans.py b/test/results/tcbscans.py index b3d0cb545..96baed871 100644 --- a/test/results/tcbscans.py +++ b/test/results/tcbscans.py @@ -1,94 +1,80 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import tcbscans from gallery_dl import exception - +from gallery_dl.extractor import tcbscans __tests__ = ( -{ - "#url" : "https://tcbscans.com/chapters/4708/chainsaw-man-chapter-108", - "#category": ("", "tcbscans", "chapter"), - "#class" : tcbscans.TcbscansChapterExtractor, - "#pattern" : r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", - "#count" : 17, - - "manga" : "Chainsaw Man", - "chapter" : 108, - "chapter_minor": "", - "lang" : "en", - "language" : "English", -}, - -{ - "#url" : "https://onepiecechapters.com/chapters/4716/one-piece-chapter-1065", - "#category": ("", "tcbscans", "chapter"), - "#class" : tcbscans.TcbscansChapterExtractor, - "#pattern" : r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", - "#count" : 18, - - "manga" : "One Piece", - "chapter" : 1065, - "chapter_minor": "", - "lang" : "en", - "language" : "English", - "#exception" : exception.HttpError, -}, - -{ - "#url" : "https://onepiecechapters.com/chapters/44/ace-novel-manga-adaptation-chapter-1", - "#category": ("", "tcbscans", "chapter"), - "#class" : tcbscans.TcbscansChapterExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://tcb-backup.bihar-mirchi.com/chapters/7719/jujutsu-kaisen-chapter-258", - "#category": ("", "tcbscans", "chapter"), - "#class" : tcbscans.TcbscansChapterExtractor, - "#pattern" : r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", - "#count" : 15, - - "manga" : "Jujutsu Kaisen", - "chapter" : 258, - "chapter_minor": "", - "lang" : "en", - "language" : "English", -}, - -{ - "#url" : "https://tcbscans.com/mangas/13/chainsaw-man", - "#category": ("", "tcbscans", "manga"), - "#class" : tcbscans.TcbscansMangaExtractor, - "#pattern" : tcbscans.TcbscansChapterExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://onepiecechapters.com/mangas/4/jujutsu-kaisen", - "#category": ("", "tcbscans", "manga"), - "#class" : tcbscans.TcbscansMangaExtractor, - "#pattern" : tcbscans.TcbscansChapterExtractor.pattern, - "#range" : "1-50", - "#count" : 50, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://onepiecechapters.com/mangas/15/hunter-x-hunter", - "#category": ("", "tcbscans", "manga"), - "#class" : tcbscans.TcbscansMangaExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://tcbscans.me/mangas/4/jujutsu-kaisen", - "#category": ("", "tcbscans", "manga"), - "#class" : tcbscans.TcbscansMangaExtractor, -}, - + { + "#url": "https://tcbscans.com/chapters/4708/chainsaw-man-chapter-108", + "#category": ("", "tcbscans", "chapter"), + "#class": tcbscans.TcbscansChapterExtractor, + "#pattern": r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", + "#count": 17, + "manga": "Chainsaw Man", + "chapter": 108, + "chapter_minor": "", + "lang": "en", + "language": "English", + }, + { + "#url": "https://onepiecechapters.com/chapters/4716/one-piece-chapter-1065", + "#category": ("", "tcbscans", "chapter"), + "#class": tcbscans.TcbscansChapterExtractor, + "#pattern": r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", + "#count": 18, + "manga": "One Piece", + "chapter": 1065, + "chapter_minor": "", + "lang": "en", + "language": "English", + "#exception": exception.HttpError, + }, + { + "#url": "https://onepiecechapters.com/chapters/44/ace-novel-manga-adaptation-chapter-1", + "#category": ("", "tcbscans", "chapter"), + "#class": tcbscans.TcbscansChapterExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://tcb-backup.bihar-mirchi.com/chapters/7719/jujutsu-kaisen-chapter-258", + "#category": ("", "tcbscans", "chapter"), + "#class": tcbscans.TcbscansChapterExtractor, + "#pattern": r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", + "#count": 15, + "manga": "Jujutsu Kaisen", + "chapter": 258, + "chapter_minor": "", + "lang": "en", + "language": "English", + }, + { + "#url": "https://tcbscans.com/mangas/13/chainsaw-man", + "#category": ("", "tcbscans", "manga"), + "#class": tcbscans.TcbscansMangaExtractor, + "#pattern": tcbscans.TcbscansChapterExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://onepiecechapters.com/mangas/4/jujutsu-kaisen", + "#category": ("", "tcbscans", "manga"), + "#class": tcbscans.TcbscansMangaExtractor, + "#pattern": tcbscans.TcbscansChapterExtractor.pattern, + "#range": "1-50", + "#count": 50, + "#exception": exception.HttpError, + }, + { + "#url": "https://onepiecechapters.com/mangas/15/hunter-x-hunter", + "#category": ("", "tcbscans", "manga"), + "#class": tcbscans.TcbscansMangaExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://tcbscans.me/mangas/4/jujutsu-kaisen", + "#category": ("", "tcbscans", "manga"), + "#class": tcbscans.TcbscansMangaExtractor, + }, ) diff --git a/test/results/tco.py b/test/results/tco.py index 37a1b106a..1106af43e 100644 --- a/test/results/tco.py +++ b/test/results/tco.py @@ -1,27 +1,22 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import urlshortener from gallery_dl import exception - +from gallery_dl.extractor import urlshortener __tests__ = ( -{ - "#url" : "https://t.co/bCgBY8Iv5n", - "#category": ("urlshortener", "tco", "link"), - "#class" : urlshortener.UrlshortenerLinkExtractor, - "#pattern" : "^https://twitter.com/elonmusk/status/1421395561324896257/photo/1", - "#count" : 1, -}, - -{ - "#url" : "https://t.co/abcdefghij", - "#category": ("urlshortener", "tco", "link"), - "#class" : urlshortener.UrlshortenerLinkExtractor, - "#exception": exception.NotFoundError, -}, - + { + "#url": "https://t.co/bCgBY8Iv5n", + "#category": ("urlshortener", "tco", "link"), + "#class": urlshortener.UrlshortenerLinkExtractor, + "#pattern": "^https://twitter.com/elonmusk/status/1421395561324896257/photo/1", + "#count": 1, + }, + { + "#url": "https://t.co/abcdefghij", + "#category": ("urlshortener", "tco", "link"), + "#class": urlshortener.UrlshortenerLinkExtractor, + "#exception": exception.NotFoundError, + }, ) diff --git a/test/results/telegraph.py b/test/results/telegraph.py index 726981b79..ff1ae0d6d 100644 --- a/test/results/telegraph.py +++ b/test/results/telegraph.py @@ -1,84 +1,73 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import telegraph - __tests__ = ( -{ - "#url" : "https://telegra.ph/Telegraph-Test-03-28", - "#category": ("", "telegraph", "gallery"), - "#class" : telegraph.TelegraphGalleryExtractor, - "#pattern" : r"https://telegra\.ph/file/[0-9a-f]+\.png", - - "author" : "mikf", - "caption" : r"re:test|", - "count" : 2, - "date" : "dt:2022-03-28 16:01:36", - "description": "Just a test", - "post_url" : "https://telegra.ph/Telegraph-Test-03-28", - "slug" : "Telegraph-Test-03-28", - "title" : "Telegra.ph Test", -}, - -{ - "#url" : "https://telegra.ph/森-03-28", - "#category": ("", "telegraph", "gallery"), - "#class" : telegraph.TelegraphGalleryExtractor, - "#pattern" : "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", - "#count" : 1, - - "author" : "&", - "caption" : "kokiri", - "count" : 1, - "date" : "dt:2022-03-28 16:31:26", - "description" : "コキリの森", - "extension" : "jpg", - "filename" : "3ea79d23b0dd0889f215a", - "num" : 1, - "num_formatted": "1", - "post_url" : "https://telegra.ph/森-03-28", - "slug" : "森-03-28", - "title" : "\"森\"", - "url" : "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", -}, - -{ - "#url" : "https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", - "#category": ("", "telegraph", "gallery"), - "#class" : telegraph.TelegraphGalleryExtractor, - "#pattern" : r"^https://pith1\.ru/uploads/posts/2019-12/\d+_\d+\.jpg$", - "#sha1_url": "c1f3048e5d94bee53af30a8c27f70b0d3b15438e", - - "author" : "Shotacon - заходи сюда", - "caption" : "", - "count" : 19, - "date" : "dt:2022-05-27 16:17:27", - "description" : "", - "num_formatted": r"re:^\d{2}$", - "post_url" : "https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", - "slug" : "Vsyo-o-druzyah-moej-sestricy-05-27", - "title" : "Всё о друзьях моей сестрицы", -}, - -{ - "#url" : "https://telegra.ph/Disharmonica---Saber-Nero-02-21", - "#category": ("", "telegraph", "gallery"), - "#class" : telegraph.TelegraphGalleryExtractor, - "#pattern" : r"https://telegra\.ph/file/[0-9a-f]+\.(jpg|png)", - - "author" : "cosmos", - "caption" : "", - "count" : 89, - "date" : "dt:2022-02-21 05:57:39", - "description" : "", - "num_formatted": r"re:^\d{2}$", - "post_url" : "https://telegra.ph/Disharmonica---Saber-Nero-02-21", - "slug" : "Disharmonica---Saber-Nero-02-21", - "title" : "Disharmonica - Saber Nero", -}, - + { + "#url": "https://telegra.ph/Telegraph-Test-03-28", + "#category": ("", "telegraph", "gallery"), + "#class": telegraph.TelegraphGalleryExtractor, + "#pattern": r"https://telegra\.ph/file/[0-9a-f]+\.png", + "author": "mikf", + "caption": r"re:test|", + "count": 2, + "date": "dt:2022-03-28 16:01:36", + "description": "Just a test", + "post_url": "https://telegra.ph/Telegraph-Test-03-28", + "slug": "Telegraph-Test-03-28", + "title": "Telegra.ph Test", + }, + { + "#url": "https://telegra.ph/森-03-28", + "#category": ("", "telegraph", "gallery"), + "#class": telegraph.TelegraphGalleryExtractor, + "#pattern": "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", + "#count": 1, + "author": "&", + "caption": "kokiri", + "count": 1, + "date": "dt:2022-03-28 16:31:26", + "description": "コキリの森", + "extension": "jpg", + "filename": "3ea79d23b0dd0889f215a", + "num": 1, + "num_formatted": "1", + "post_url": "https://telegra.ph/森-03-28", + "slug": "森-03-28", + "title": '"森"', + "url": "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", + }, + { + "#url": "https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", + "#category": ("", "telegraph", "gallery"), + "#class": telegraph.TelegraphGalleryExtractor, + "#pattern": r"^https://pith1\.ru/uploads/posts/2019-12/\d+_\d+\.jpg$", + "#sha1_url": "c1f3048e5d94bee53af30a8c27f70b0d3b15438e", + "author": "Shotacon - заходи сюда", + "caption": "", + "count": 19, + "date": "dt:2022-05-27 16:17:27", + "description": "", + "num_formatted": r"re:^\d{2}$", + "post_url": "https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", + "slug": "Vsyo-o-druzyah-moej-sestricy-05-27", + "title": "Всё о друзьях моей сестрицы", + }, + { + "#url": "https://telegra.ph/Disharmonica---Saber-Nero-02-21", + "#category": ("", "telegraph", "gallery"), + "#class": telegraph.TelegraphGalleryExtractor, + "#pattern": r"https://telegra\.ph/file/[0-9a-f]+\.(jpg|png)", + "author": "cosmos", + "caption": "", + "count": 89, + "date": "dt:2022-02-21 05:57:39", + "description": "", + "num_formatted": r"re:^\d{2}$", + "post_url": "https://telegra.ph/Disharmonica---Saber-Nero-02-21", + "slug": "Disharmonica---Saber-Nero-02-21", + "title": "Disharmonica - Saber Nero", + }, ) diff --git a/test/results/tentaclerape.py b/test/results/tentaclerape.py index 247067f6d..45e7e712b 100644 --- a/test/results/tentaclerape.py +++ b/test/results/tentaclerape.py @@ -1,47 +1,40 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shimmie2 - __tests__ = ( -{ - "#url" : "https://tentaclerape.net/post/list/comic/1", - "#category": ("shimmie2", "tentaclerape", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, - "#pattern" : r"https://tentaclerape\.net/_images/[0-9a-f]{32}/\d+", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://tentaclerape.net/post/view/10", - "#category": ("shimmie2", "tentaclerape", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, - "#pattern" : r"https://tentaclerape\.net/\./index\.php\?q=/image/10\.jpg", - "#sha1_content": "d0fd8f0f6517a76cb5e23ba09f3844950bf2c516", - - "extension" : "jpg", - "file_url" : "https://tentaclerape.net/./index.php?q=/image/10.jpg", - "filename" : "10", - "height" : 427, - "id" : 10, - "md5" : "945db71eeccaef82ce44b77564260c0b", - "size" : 0, - "subcategory": "post", - "tags" : "Deviant_Art Pet Tentacle artist_sche blonde_hair blouse boots green_eyes highheels leash miniskirt octopus schoolgirl white_skin willing", - "width" : 300, -}, - -{ - "#url" : "https://tentaclerape.net/post/view/91267", - "#comment" : "video", - "#category": ("shimmie2", "tentaclerape", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, - "#pattern" : r"https://tentaclerape\.net/\./index\.php\?q=/image/91267\.mp4", -}, - + { + "#url": "https://tentaclerape.net/post/list/comic/1", + "#category": ("shimmie2", "tentaclerape", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + "#pattern": r"https://tentaclerape\.net/_images/[0-9a-f]{32}/\d+", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://tentaclerape.net/post/view/10", + "#category": ("shimmie2", "tentaclerape", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + "#pattern": r"https://tentaclerape\.net/\./index\.php\?q=/image/10\.jpg", + "#sha1_content": "d0fd8f0f6517a76cb5e23ba09f3844950bf2c516", + "extension": "jpg", + "file_url": "https://tentaclerape.net/./index.php?q=/image/10.jpg", + "filename": "10", + "height": 427, + "id": 10, + "md5": "945db71eeccaef82ce44b77564260c0b", + "size": 0, + "subcategory": "post", + "tags": "Deviant_Art Pet Tentacle artist_sche blonde_hair blouse boots green_eyes highheels leash miniskirt octopus schoolgirl white_skin willing", + "width": 300, + }, + { + "#url": "https://tentaclerape.net/post/view/91267", + "#comment": "video", + "#category": ("shimmie2", "tentaclerape", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + "#pattern": r"https://tentaclerape\.net/\./index\.php\?q=/image/91267\.mp4", + }, ) diff --git a/test/results/thebarchive.py b/test/results/thebarchive.py index 5fd6e2266..204405952 100644 --- a/test/results/thebarchive.py +++ b/test/results/thebarchive.py @@ -1,36 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://thebarchive.com/b/thread/739772332/", - "#category": ("foolfuuka", "thebarchive", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "e8b18001307d130d67db31740ce57c8561b5d80c", -}, - -{ - "#url" : "https://thebarchive.com/b/", - "#category": ("foolfuuka", "thebarchive", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://thebarchive.com/_/search/text/test/", - "#category": ("foolfuuka", "thebarchive", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://thebarchive.com/b/gallery/9", - "#category": ("foolfuuka", "thebarchive", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://thebarchive.com/b/thread/739772332/", + "#category": ("foolfuuka", "thebarchive", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "e8b18001307d130d67db31740ce57c8561b5d80c", + }, + { + "#url": "https://thebarchive.com/b/", + "#category": ("foolfuuka", "thebarchive", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://thebarchive.com/_/search/text/test/", + "#category": ("foolfuuka", "thebarchive", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://thebarchive.com/b/gallery/9", + "#category": ("foolfuuka", "thebarchive", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/thecollection.py b/test/results/thecollection.py index abe47380b..23e992503 100644 --- a/test/results/thecollection.py +++ b/test/results/thecollection.py @@ -1,34 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v01 - __tests__ = ( -{ - "#url" : "https://the-collection.booru.org/index.php?page=post&s=list&tags=parody", - "#category": ("gelbooru_v01", "thecollection", "tag"), - "#class" : gelbooru_v01.GelbooruV01TagExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://the-collection.booru.org/index.php?page=favorites&s=view&id=1166", - "#category": ("gelbooru_v01", "thecollection", "favorite"), - "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://the-collection.booru.org/index.php?page=post&s=view&id=100520", - "#category": ("gelbooru_v01", "thecollection", "post"), - "#class" : gelbooru_v01.GelbooruV01PostExtractor, - "#sha1_url" : "0329ac8588bb93cf242ca0edbe3e995b4ba554e8", - "#sha1_content": "1e585874e7b874f7937df1060dd1517fef2f4dfb", -}, - + { + "#url": "https://the-collection.booru.org/index.php?page=post&s=list&tags=parody", + "#category": ("gelbooru_v01", "thecollection", "tag"), + "#class": gelbooru_v01.GelbooruV01TagExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://the-collection.booru.org/index.php?page=favorites&s=view&id=1166", + "#category": ("gelbooru_v01", "thecollection", "favorite"), + "#class": gelbooru_v01.GelbooruV01FavoriteExtractor, + "#count": 2, + }, + { + "#url": "https://the-collection.booru.org/index.php?page=post&s=view&id=100520", + "#category": ("gelbooru_v01", "thecollection", "post"), + "#class": gelbooru_v01.GelbooruV01PostExtractor, + "#sha1_url": "0329ac8588bb93cf242ca0edbe3e995b4ba554e8", + "#sha1_content": "1e585874e7b874f7937df1060dd1517fef2f4dfb", + }, ) diff --git a/test/results/tmohentai.py b/test/results/tmohentai.py index b565ae5e4..a150bcf36 100644 --- a/test/results/tmohentai.py +++ b/test/results/tmohentai.py @@ -1,54 +1,48 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import tmohentai - __tests__ = ( -{ - "#url" : "https://tmohentai.com/contents/653c2aeaa693c", - "#category": ("", "tmohentai", "gallery"), - "#class" : tmohentai.TmohentaiGalleryExtractor, - "#pattern" : r"https://imgrojo\.tmohentai\.com/contents/653c2aeaa693c/\d\d\d\.webp", - "#count" : 46, - - "artists" : ["Andoryu"], - "genres" : [ - "Big Breasts", - "BlowJob", - "Cheating", - "Mature", - "Milf", - "Student", - ], - "count" : 46, - "extension" : "webp", - "gallery_id": "653c2aeaa693c", - "language" : "Español", - "num" : int, - "tags" : [ - "milf", - "Madre", - "enormes pechos", - "Peluda", - "nakadashi", - "cheating", - "madura", - "sexo a escondidas", - "Ama de casa", - "mamada", - ], - "title" : "La Mama de mi Novia es tan Pervertida que no Pude Soportarlo mas", - "uploader" : "NekoCreme Fansub", -}, - -{ - "#url" : "https://tmohentai.com/reader/653c2aeaa693c/paginated/1", - "#category": ("", "tmohentai", "gallery"), - "#class" : tmohentai.TmohentaiGalleryExtractor, -}, - + { + "#url": "https://tmohentai.com/contents/653c2aeaa693c", + "#category": ("", "tmohentai", "gallery"), + "#class": tmohentai.TmohentaiGalleryExtractor, + "#pattern": r"https://imgrojo\.tmohentai\.com/contents/653c2aeaa693c/\d\d\d\.webp", + "#count": 46, + "artists": ["Andoryu"], + "genres": [ + "Big Breasts", + "BlowJob", + "Cheating", + "Mature", + "Milf", + "Student", + ], + "count": 46, + "extension": "webp", + "gallery_id": "653c2aeaa693c", + "language": "Español", + "num": int, + "tags": [ + "milf", + "Madre", + "enormes pechos", + "Peluda", + "nakadashi", + "cheating", + "madura", + "sexo a escondidas", + "Ama de casa", + "mamada", + ], + "title": "La Mama de mi Novia es tan Pervertida que no Pude Soportarlo mas", + "uploader": "NekoCreme Fansub", + }, + { + "#url": "https://tmohentai.com/reader/653c2aeaa693c/paginated/1", + "#category": ("", "tmohentai", "gallery"), + "#class": tmohentai.TmohentaiGalleryExtractor, + }, ) diff --git a/test/results/toyhouse.py b/test/results/toyhouse.py index 21d13ee11..6b9fbfd2e 100644 --- a/test/results/toyhouse.py +++ b/test/results/toyhouse.py @@ -1,75 +1,65 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import toyhouse import datetime +from gallery_dl.extractor import toyhouse __tests__ = ( -{ - "#url" : "https://www.toyhou.se/d-floe/art", - "#category": ("", "toyhouse", "art"), - "#class" : toyhouse.ToyhouseArtExtractor, - "#pattern" : r"https://f\d+\.toyhou\.se/file/f\d+-toyhou-se/images/\d+_\w+\.\w+$", - "#range" : "1-30", - "#count" : 30, - - "artists" : list, - "characters": list, - "date" : datetime.datetime, - "hash" : r"re:\w+", - "id" : r"re:\d+", - "url" : str, - "user" : "d-floe", -}, - -{ - "#url" : "https://www.toyhou.se/kroksoc/art", - "#comment" : "protected by Content Warning", - "#category": ("", "toyhouse", "art"), - "#class" : toyhouse.ToyhouseArtExtractor, - "#count" : ">= 19", -}, - -{ - "#url" : "https://toyhou.se/~images/40587320", - "#category": ("", "toyhouse", "image"), - "#class" : toyhouse.ToyhouseImageExtractor, - "#sha1_content": "058ec8427977ab432c4cc5be5a6dd39ce18713ef", - - "artists" : ["d-floe"], - "characters": ["Sumi"], - "date" : "dt:2021-10-08 01:32:47", - "extension" : "png", - "filename" : "40587320_TT1NaBUr3FLkS1p", - "hash" : "TT1NaBUr3FLkS1p", - "id" : "40587320", - "url" : "https://f2.toyhou.se/file/f2-toyhou-se/images/40587320_TT1NaBUr3FLkS1p.png", -}, - -{ - "#url" : "https://f2.toyhou.se/file/f2-toyhou-se/watermarks/36817425_bqhGcwcnU.png?1625561467", - "#comment" : "direct link, multiple artists", - "#category": ("", "toyhouse", "image"), - "#class" : toyhouse.ToyhouseImageExtractor, - - "artists" : [ - "http://aminoapps.com/p/92sf3z", - "kroksoc (Color)", - ], - "characters": ["Reiichi❀"], - "date" : "dt:2021-07-03 20:02:02", - "hash" : "bqhGcwcnU", - "id" : "36817425", -}, - -{ - "#url" : "https://f2.toyhou.se/file/f2-toyhou-se/images/40587320_TT1NaBUr3FLkS1p.png", - "#category": ("", "toyhouse", "image"), - "#class" : toyhouse.ToyhouseImageExtractor, -}, - + { + "#url": "https://www.toyhou.se/d-floe/art", + "#category": ("", "toyhouse", "art"), + "#class": toyhouse.ToyhouseArtExtractor, + "#pattern": r"https://f\d+\.toyhou\.se/file/f\d+-toyhou-se/images/\d+_\w+\.\w+$", + "#range": "1-30", + "#count": 30, + "artists": list, + "characters": list, + "date": datetime.datetime, + "hash": r"re:\w+", + "id": r"re:\d+", + "url": str, + "user": "d-floe", + }, + { + "#url": "https://www.toyhou.se/kroksoc/art", + "#comment": "protected by Content Warning", + "#category": ("", "toyhouse", "art"), + "#class": toyhouse.ToyhouseArtExtractor, + "#count": ">= 19", + }, + { + "#url": "https://toyhou.se/~images/40587320", + "#category": ("", "toyhouse", "image"), + "#class": toyhouse.ToyhouseImageExtractor, + "#sha1_content": "058ec8427977ab432c4cc5be5a6dd39ce18713ef", + "artists": ["d-floe"], + "characters": ["Sumi"], + "date": "dt:2021-10-08 01:32:47", + "extension": "png", + "filename": "40587320_TT1NaBUr3FLkS1p", + "hash": "TT1NaBUr3FLkS1p", + "id": "40587320", + "url": "https://f2.toyhou.se/file/f2-toyhou-se/images/40587320_TT1NaBUr3FLkS1p.png", + }, + { + "#url": "https://f2.toyhou.se/file/f2-toyhou-se/watermarks/36817425_bqhGcwcnU.png?1625561467", + "#comment": "direct link, multiple artists", + "#category": ("", "toyhouse", "image"), + "#class": toyhouse.ToyhouseImageExtractor, + "artists": [ + "http://aminoapps.com/p/92sf3z", + "kroksoc (Color)", + ], + "characters": ["Reiichi❀"], + "date": "dt:2021-07-03 20:02:02", + "hash": "bqhGcwcnU", + "id": "36817425", + }, + { + "#url": "https://f2.toyhou.se/file/f2-toyhou-se/images/40587320_TT1NaBUr3FLkS1p.png", + "#category": ("", "toyhouse", "image"), + "#class": toyhouse.ToyhouseImageExtractor, + }, ) diff --git a/test/results/tsumino.py b/test/results/tsumino.py index cbe2c77a6..49a526577 100644 --- a/test/results/tsumino.py +++ b/test/results/tsumino.py @@ -1,72 +1,62 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import tsumino - __tests__ = ( -{ - "#url" : "https://www.tsumino.com/entry/40996", - "#category": ("", "tsumino", "gallery"), - "#class" : tsumino.TsuminoGalleryExtractor, - "#pattern" : r"https://content.tsumino.com/parts/40996/\d+\?key=\w+", - - "title" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", - "title_en" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", - "title_jp" : "シコシコ大好きナイチンゲール + 会場限定おまけ本", - "gallery_id": 40996, - "date" : "dt:2018-06-29 00:00:00", - "count" : 42, - "collection": "", - "artist" : ["Itou Life"], - "group" : ["Itou Life"], - "parody" : list, - "characters": list, - "tags" : list, - "type" : "Doujinshi", - "rating" : float, - "uploader" : "sehki", - "lang" : "en", - "language" : "English", - "thumbnail" : "https://content.tsumino.com/thumbs/40996/1", -}, - -{ - "#url" : "https://www.tsumino.com/Book/Info/40996", - "#category": ("", "tsumino", "gallery"), - "#class" : tsumino.TsuminoGalleryExtractor, -}, - -{ - "#url" : "https://www.tsumino.com/Read/View/45834", - "#category": ("", "tsumino", "gallery"), - "#class" : tsumino.TsuminoGalleryExtractor, -}, - -{ - "#url" : "https://www.tsumino.com/Read/Index/45834", - "#category": ("", "tsumino", "gallery"), - "#class" : tsumino.TsuminoGalleryExtractor, -}, - -{ - "#url" : "https://www.tsumino.com/Books#?Character=Reimu+Hakurei", - "#category": ("", "tsumino", "search"), - "#class" : tsumino.TsuminoSearchExtractor, - "#pattern" : tsumino.TsuminoGalleryExtractor.pattern, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "http://www.tsumino.com/Books#~(Tags~(~(Type~7~Text~'Reimu*20Hakurei~Exclude~false)~(Type~'1~Text~'Pantyhose~Exclude~false)))#", - "#category": ("", "tsumino", "search"), - "#class" : tsumino.TsuminoSearchExtractor, - "#pattern" : tsumino.TsuminoGalleryExtractor.pattern, - "#count" : ">= 3", -}, - + { + "#url": "https://www.tsumino.com/entry/40996", + "#category": ("", "tsumino", "gallery"), + "#class": tsumino.TsuminoGalleryExtractor, + "#pattern": r"https://content.tsumino.com/parts/40996/\d+\?key=\w+", + "title": r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", + "title_en": r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", + "title_jp": "シコシコ大好きナイチンゲール + 会場限定おまけ本", + "gallery_id": 40996, + "date": "dt:2018-06-29 00:00:00", + "count": 42, + "collection": "", + "artist": ["Itou Life"], + "group": ["Itou Life"], + "parody": list, + "characters": list, + "tags": list, + "type": "Doujinshi", + "rating": float, + "uploader": "sehki", + "lang": "en", + "language": "English", + "thumbnail": "https://content.tsumino.com/thumbs/40996/1", + }, + { + "#url": "https://www.tsumino.com/Book/Info/40996", + "#category": ("", "tsumino", "gallery"), + "#class": tsumino.TsuminoGalleryExtractor, + }, + { + "#url": "https://www.tsumino.com/Read/View/45834", + "#category": ("", "tsumino", "gallery"), + "#class": tsumino.TsuminoGalleryExtractor, + }, + { + "#url": "https://www.tsumino.com/Read/Index/45834", + "#category": ("", "tsumino", "gallery"), + "#class": tsumino.TsuminoGalleryExtractor, + }, + { + "#url": "https://www.tsumino.com/Books#?Character=Reimu+Hakurei", + "#category": ("", "tsumino", "search"), + "#class": tsumino.TsuminoSearchExtractor, + "#pattern": tsumino.TsuminoGalleryExtractor.pattern, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "http://www.tsumino.com/Books#~(Tags~(~(Type~7~Text~'Reimu*20Hakurei~Exclude~false)~(Type~'1~Text~'Pantyhose~Exclude~false)))#", + "#category": ("", "tsumino", "search"), + "#class": tsumino.TsuminoSearchExtractor, + "#pattern": tsumino.TsuminoGalleryExtractor.pattern, + "#count": ">= 3", + }, ) diff --git a/test/results/tumblr.py b/test/results/tumblr.py index 50b676761..72ac5d8e0 100644 --- a/test/results/tumblr.py +++ b/test/results/tumblr.py @@ -1,380 +1,326 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import tumblr from gallery_dl import exception - +from gallery_dl.extractor import tumblr __tests__ = ( -{ - "#url" : "http://demo.tumblr.com/", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#options" : {"posts": "photo"}, - "#pattern" : r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_\d+\.jpg", - "#count" : 1, -}, - -{ - "#url" : "http://demo.tumblr.com/", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#options" : { - "posts" : "all", - "external": True, - }, - "#pattern" : r"https?://(?:$|\d+\.media\.tumblr\.com/.+\.(jpg|png|gif|mp3|mp4)|v?a\.(media\.)?tumblr\.com/tumblr_\w+)", - "#count" : 27, -}, - -{ - "#url" : "https://mikf123-hidden.tumblr.com/", - "#comment" : "dashboard-only", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#options" : {"access-token": None}, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://mikf123-hidden.tumblr.com/", - "#comment" : "dashboard-only", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#count" : 2, - - "tags": [ - "test", - "hidden", - ], -}, - -{ - "#url" : "https://mikf123-private.tumblr.com/", - "#comment" : "password protected", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#count" : 2, - - "tags": [ - "test", - "private", - ], -}, - -{ - "#url" : "https://mikf123-private-hidden.tumblr.com/", - "#comment" : "dashboard-only & password protected", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#count" : 2, - - "tags": [ - "test", - "private", - "hidden", - ], -}, - -{ - "#url" : "https://mikf123.tumblr.com/", - "#comment" : "date-min/-max/-format (#337)", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#options" : { - "date-min" : "201804", - "date-max" : "201805", - "date-format": "%Y%m", - }, - "#count" : 4, -}, - -{ - "#url" : "https://donttrustthetits.tumblr.com/", - "#comment" : "pagination with 'date-max' (#2191) and 'api-key'", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#options" : { - "access-token": None, - "original" : False, - "date-max" : "2015-04-25T00:00:00", - "date-min" : "2015-04-01T00:00:00", - }, - "#count" : 192, -}, - -{ - "#url" : "https://demo.tumblr.com/page/2", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "https://demo.tumblr.com/archive", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "tumblr:http://www.b-authentique.com/", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "tumblr:www.b-authentique.com", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/view/smarties-art", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/smarties-art", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/smarties-art", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "http://demo.tumblr.com/post/459265350", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#pattern" : r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_1280.jpg", - "#count" : 1, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/167770226574/text-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/181022561719/quote-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 1, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/167623351559/link-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/167633596145/video-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/167770026604/audio-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/172687798174/photo-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 4, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/181022380064/chat-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://kichatundk.tumblr.com/post/654953419288821760", - "#comment" : "high-quality images (#1846)", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 2, - "#sha1_content": "d6fcc7b6f750d835d55c7f31fa3b63be26c9f89b", -}, - -{ - "#url" : "https://hameru-is-cool.tumblr.com/post/639261855227002880", - "#comment" : "high-quality images (#1344)", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#exception" : exception.NotFoundError, - "#count" : 2, - "#sha1_content": "6bc19a42787e46e1bba2ef4aeef5ca28fcd3cd34", -}, - -{ - "#url" : "https://k-eke.tumblr.com/post/185341184856", - "#comment" : "wrong extension returned by api (#3095)", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#options" : {"retries": 0}, - "#urls" : "https://64.media.tumblr.com/5e9d760aba24c65beaf0e72de5aae4dd/tumblr_psj5yaqV871t1ig6no1_1280.gif", - "#sha1_content": "3508d894b6cc25e364d182a8e1ff370d706965fb", -}, - -{ - "#url" : "https://mikf123.tumblr.com/image/689860196535762944", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#pattern" : r"^https://\d+\.media\.tumblr\.com/134791621559a79793563b636b5fe2c6/8f1131551cef6e74-bc/s99999x99999/188cf9b8915b0d0911c6c743d152fc62e8f38491\.png$", -}, - -{ - "#url" : "http://ziemniax.tumblr.com/post/109697912859/", - "#comment" : "HTML response (#297)", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "http://demo.tumblr.com/image/459265350", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/view/smarties-art/686047436641353728", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/smarties-art/686047436641353728", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/smarties-art/686047436641353728", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, -}, - -{ - "#url" : "http://demo.tumblr.com/tagged/Times%20Square", - "#category": ("", "tumblr", "tag"), - "#class" : tumblr.TumblrTagExtractor, - "#pattern" : r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_1280.jpg", - "#count" : 1, -}, - -{ - "#url" : "https://www.tumblr.com/blog/view/smarties-art/tagged/undertale", - "#category": ("", "tumblr", "tag"), - "#class" : tumblr.TumblrTagExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/smarties-art/tagged/undertale", - "#category": ("", "tumblr", "tag"), - "#class" : tumblr.TumblrTagExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/smarties-art/tagged/undertale", - "#category": ("", "tumblr", "tag"), - "#class" : tumblr.TumblrTagExtractor, -}, - -{ - "#url" : "https://mikf123.tumblr.com/day/2018/01/05", - "#category": ("", "tumblr", "day"), - "#class" : tumblr.TumblrDayExtractor, - "#pattern" : r"https://64\.media\.tumblr\.com/1a2be8c63f1df58abd2622861696c72a/tumblr_ozm9nqst9t1wgha4yo1_1280\.jpg", - "#count" : 1, - - "id": 169341068404, -}, - -{ - "#url" : "https://www.tumblr.com/blog/view/mikf123/day/2018/01/05", - "#category": ("", "tumblr", "day"), - "#class" : tumblr.TumblrDayExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/mikf123/day/2018/01/05", - "#category": ("", "tumblr", "day"), - "#class" : tumblr.TumblrDayExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/mikf123/day/2018/01/05", - "#category": ("", "tumblr", "day"), - "#class" : tumblr.TumblrDayExtractor, -}, - -{ - "#url" : "http://mikf123.tumblr.com/likes", - "#category": ("", "tumblr", "likes"), - "#class" : tumblr.TumblrLikesExtractor, - "#count" : 1, -}, - -{ - "#url" : "http://mikf123.tumblr.com/likes", - "#category": ("", "tumblr", "likes"), - "#class" : tumblr.TumblrLikesExtractor, - "#options" : {"api-secret": None}, - "#count" : 1, -}, - -{ - "#url" : "https://www.tumblr.com/blog/view/mikf123/likes", - "#category": ("", "tumblr", "likes"), - "#class" : tumblr.TumblrLikesExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/mikf123/likes", - "#category": ("", "tumblr", "likes"), - "#class" : tumblr.TumblrLikesExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/mikf123/likes", - "#category": ("", "tumblr", "likes"), - "#class" : tumblr.TumblrLikesExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/search/nathan fielder", - "#category": ("", "tumblr", "search"), - "#class" : tumblr.TumblrSearchExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/search/nathan fielder/recent/quote?src=typed_query", - "#category": ("", "tumblr", "search"), - "#class" : tumblr.TumblrSearchExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/search/nathan%20fielder?t=90", - "#category": ("", "tumblr", "search"), - "#class" : tumblr.TumblrSearchExtractor, -}, + { + "#url": "http://demo.tumblr.com/", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#options": {"posts": "photo"}, + "#pattern": r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_\d+\.jpg", + "#count": 1, + }, + { + "#url": "http://demo.tumblr.com/", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#options": { + "posts": "all", + "external": True, + }, + "#pattern": r"https?://(?:$|\d+\.media\.tumblr\.com/.+\.(jpg|png|gif|mp3|mp4)|v?a\.(media\.)?tumblr\.com/tumblr_\w+)", + "#count": 27, + }, + { + "#url": "https://mikf123-hidden.tumblr.com/", + "#comment": "dashboard-only", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#options": {"access-token": None}, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://mikf123-hidden.tumblr.com/", + "#comment": "dashboard-only", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#count": 2, + "tags": [ + "test", + "hidden", + ], + }, + { + "#url": "https://mikf123-private.tumblr.com/", + "#comment": "password protected", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#count": 2, + "tags": [ + "test", + "private", + ], + }, + { + "#url": "https://mikf123-private-hidden.tumblr.com/", + "#comment": "dashboard-only & password protected", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#count": 2, + "tags": [ + "test", + "private", + "hidden", + ], + }, + { + "#url": "https://mikf123.tumblr.com/", + "#comment": "date-min/-max/-format (#337)", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#options": { + "date-min": "201804", + "date-max": "201805", + "date-format": "%Y%m", + }, + "#count": 4, + }, + { + "#url": "https://donttrustthetits.tumblr.com/", + "#comment": "pagination with 'date-max' (#2191) and 'api-key'", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#options": { + "access-token": None, + "original": False, + "date-max": "2015-04-25T00:00:00", + "date-min": "2015-04-01T00:00:00", + }, + "#count": 192, + }, + { + "#url": "https://demo.tumblr.com/page/2", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "https://demo.tumblr.com/archive", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "tumblr:http://www.b-authentique.com/", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "tumblr:www.b-authentique.com", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/view/smarties-art", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/smarties-art", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "https://www.tumblr.com/smarties-art", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "http://demo.tumblr.com/post/459265350", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#pattern": r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_1280.jpg", + "#count": 1, + }, + { + "#url": "https://mikf123.tumblr.com/post/167770226574/text-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 2, + }, + { + "#url": "https://mikf123.tumblr.com/post/181022561719/quote-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 1, + }, + { + "#url": "https://mikf123.tumblr.com/post/167623351559/link-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 2, + }, + { + "#url": "https://mikf123.tumblr.com/post/167633596145/video-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 2, + }, + { + "#url": "https://mikf123.tumblr.com/post/167770026604/audio-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 2, + }, + { + "#url": "https://mikf123.tumblr.com/post/172687798174/photo-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 4, + }, + { + "#url": "https://mikf123.tumblr.com/post/181022380064/chat-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 0, + }, + { + "#url": "https://kichatundk.tumblr.com/post/654953419288821760", + "#comment": "high-quality images (#1846)", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 2, + "#sha1_content": "d6fcc7b6f750d835d55c7f31fa3b63be26c9f89b", + }, + { + "#url": "https://hameru-is-cool.tumblr.com/post/639261855227002880", + "#comment": "high-quality images (#1344)", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#exception": exception.NotFoundError, + "#count": 2, + "#sha1_content": "6bc19a42787e46e1bba2ef4aeef5ca28fcd3cd34", + }, + { + "#url": "https://k-eke.tumblr.com/post/185341184856", + "#comment": "wrong extension returned by api (#3095)", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#options": {"retries": 0}, + "#urls": "https://64.media.tumblr.com/5e9d760aba24c65beaf0e72de5aae4dd/tumblr_psj5yaqV871t1ig6no1_1280.gif", + "#sha1_content": "3508d894b6cc25e364d182a8e1ff370d706965fb", + }, + { + "#url": "https://mikf123.tumblr.com/image/689860196535762944", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#pattern": r"^https://\d+\.media\.tumblr\.com/134791621559a79793563b636b5fe2c6/8f1131551cef6e74-bc/s99999x99999/188cf9b8915b0d0911c6c743d152fc62e8f38491\.png$", + }, + { + "#url": "http://ziemniax.tumblr.com/post/109697912859/", + "#comment": "HTML response (#297)", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "http://demo.tumblr.com/image/459265350", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/view/smarties-art/686047436641353728", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/smarties-art/686047436641353728", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + }, + { + "#url": "https://www.tumblr.com/smarties-art/686047436641353728", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + }, + { + "#url": "http://demo.tumblr.com/tagged/Times%20Square", + "#category": ("", "tumblr", "tag"), + "#class": tumblr.TumblrTagExtractor, + "#pattern": r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_1280.jpg", + "#count": 1, + }, + { + "#url": "https://www.tumblr.com/blog/view/smarties-art/tagged/undertale", + "#category": ("", "tumblr", "tag"), + "#class": tumblr.TumblrTagExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/smarties-art/tagged/undertale", + "#category": ("", "tumblr", "tag"), + "#class": tumblr.TumblrTagExtractor, + }, + { + "#url": "https://www.tumblr.com/smarties-art/tagged/undertale", + "#category": ("", "tumblr", "tag"), + "#class": tumblr.TumblrTagExtractor, + }, + { + "#url": "https://mikf123.tumblr.com/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class": tumblr.TumblrDayExtractor, + "#pattern": r"https://64\.media\.tumblr\.com/1a2be8c63f1df58abd2622861696c72a/tumblr_ozm9nqst9t1wgha4yo1_1280\.jpg", + "#count": 1, + "id": 169341068404, + }, + { + "#url": "https://www.tumblr.com/blog/view/mikf123/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class": tumblr.TumblrDayExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/mikf123/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class": tumblr.TumblrDayExtractor, + }, + { + "#url": "https://www.tumblr.com/mikf123/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class": tumblr.TumblrDayExtractor, + }, + { + "#url": "http://mikf123.tumblr.com/likes", + "#category": ("", "tumblr", "likes"), + "#class": tumblr.TumblrLikesExtractor, + "#count": 1, + }, + { + "#url": "http://mikf123.tumblr.com/likes", + "#category": ("", "tumblr", "likes"), + "#class": tumblr.TumblrLikesExtractor, + "#options": {"api-secret": None}, + "#count": 1, + }, + { + "#url": "https://www.tumblr.com/blog/view/mikf123/likes", + "#category": ("", "tumblr", "likes"), + "#class": tumblr.TumblrLikesExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/mikf123/likes", + "#category": ("", "tumblr", "likes"), + "#class": tumblr.TumblrLikesExtractor, + }, + { + "#url": "https://www.tumblr.com/mikf123/likes", + "#category": ("", "tumblr", "likes"), + "#class": tumblr.TumblrLikesExtractor, + }, + { + "#url": "https://www.tumblr.com/search/nathan fielder", + "#category": ("", "tumblr", "search"), + "#class": tumblr.TumblrSearchExtractor, + }, + { + "#url": "https://www.tumblr.com/search/nathan fielder/recent/quote?src=typed_query", + "#category": ("", "tumblr", "search"), + "#class": tumblr.TumblrSearchExtractor, + }, + { + "#url": "https://www.tumblr.com/search/nathan%20fielder?t=90", + "#category": ("", "tumblr", "search"), + "#class": tumblr.TumblrSearchExtractor, + }, ) diff --git a/test/results/tumblrgallery.py b/test/results/tumblrgallery.py index a1a6eeb59..c3f881c20 100644 --- a/test/results/tumblrgallery.py +++ b/test/results/tumblrgallery.py @@ -1,33 +1,27 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import tumblrgallery - __tests__ = ( -{ - "#url" : "https://tumblrgallery.xyz/tumblrblog/gallery/103975.html", - "#category": ("", "tumblrgallery", "tumblrblog"), - "#class" : tumblrgallery.TumblrgalleryTumblrblogExtractor, -}, - -{ - "#url" : "https://tumblrgallery.xyz/post/405674.html", - "#category": ("", "tumblrgallery", "post"), - "#class" : tumblrgallery.TumblrgalleryPostExtractor, - "#pattern" : r"https://78\.media\.tumblr\.com/bec67072219c1f3bc04fd9711dec42ef/tumblr_p51qq1XCHS1txhgk3o1_1280\.jpg", - "#count" : 3, -}, - -{ - "#url" : "https://tumblrgallery.xyz/s.php?q=everyday-life", - "#category": ("", "tumblrgallery", "search"), - "#class" : tumblrgallery.TumblrgallerySearchExtractor, - "#pattern" : r"https://\d+\.media\.tumblr\.com/.+", - "#count" : "< 1000", -}, - + { + "#url": "https://tumblrgallery.xyz/tumblrblog/gallery/103975.html", + "#category": ("", "tumblrgallery", "tumblrblog"), + "#class": tumblrgallery.TumblrgalleryTumblrblogExtractor, + }, + { + "#url": "https://tumblrgallery.xyz/post/405674.html", + "#category": ("", "tumblrgallery", "post"), + "#class": tumblrgallery.TumblrgalleryPostExtractor, + "#pattern": r"https://78\.media\.tumblr\.com/bec67072219c1f3bc04fd9711dec42ef/tumblr_p51qq1XCHS1txhgk3o1_1280\.jpg", + "#count": 3, + }, + { + "#url": "https://tumblrgallery.xyz/s.php?q=everyday-life", + "#category": ("", "tumblrgallery", "search"), + "#class": tumblrgallery.TumblrgallerySearchExtractor, + "#pattern": r"https://\d+\.media\.tumblr\.com/.+", + "#count": "< 1000", + }, ) diff --git a/test/results/turboimagehost.py b/test/results/turboimagehost.py index 642d93219..b3f45d520 100644 --- a/test/results/turboimagehost.py +++ b/test/results/turboimagehost.py @@ -1,23 +1,19 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://www.turboimagehost.com/p/39078423/test--.png.html", - "#category": ("imagehost", "turboimagehost", "image"), - "#class" : imagehosts.TurboimagehostImageExtractor, - "#sha1_url" : "b94de43612318771ced924cb5085976f13b3b90e", - "#sha1_metadata": "704757ca8825f51cec516ec44c1e627c1f2058ca", - "#sha1_content" : ( - "f38b54b17cd7462e687b58d83f00fca88b1b105a", - "0c8768055e4e20e7c7259608b67799171b691140", - ), -}, - + { + "#url": "https://www.turboimagehost.com/p/39078423/test--.png.html", + "#category": ("imagehost", "turboimagehost", "image"), + "#class": imagehosts.TurboimagehostImageExtractor, + "#sha1_url": "b94de43612318771ced924cb5085976f13b3b90e", + "#sha1_metadata": "704757ca8825f51cec516ec44c1e627c1f2058ca", + "#sha1_content": ( + "f38b54b17cd7462e687b58d83f00fca88b1b105a", + "0c8768055e4e20e7c7259608b67799171b691140", + ), + }, ) diff --git a/test/results/twibooru.py b/test/results/twibooru.py index 4720309f1..89ef0c426 100644 --- a/test/results/twibooru.py +++ b/test/results/twibooru.py @@ -1,110 +1,97 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import twibooru - __tests__ = ( -{ - "#url" : "https://twibooru.org/1", - "#category": ("philomena", "twibooru", "post"), - "#class" : twibooru.TwibooruPostExtractor, - "#pattern" : "https://cdn.twibooru.org/img/2020/7/8/1/full.png", - "#sha1_content": "aac4d1dba611883ac701aaa8f0b2b322590517ae", - - "animated" : False, - "aspect_ratio" : 1.0, - "comment_count" : int, - "created_at" : "2020-07-08T22:26:55.743Z", - "date" : "dt:2020-07-08 22:26:55", - "description" : "Why have I done this?", - "downvotes" : 0, - "duration" : 0.0, - "faves" : int, - "first_seen_at" : "2020-07-08T22:26:55.743Z", - "format" : "png", - "height" : 576, - "hidden_from_users": False, - "id" : 1, - "intensities" : dict, - "locations" : [], - "media_type" : "image", - "mime_type" : "image/png", - "name" : "1676547__safe_artist-colon-scraggleman_oc_oc-colon-floor+bored_oc+only_bags+under+eyes_bust_earth+pony_female_goggles_helmet_mare_meme_neet_neet+home+g.png", - "orig_sha512_hash": r"re:8b4c00d2[0-9a-f]{120}", - "processed" : True, - "representations" : dict, - "score" : int, - "sha512_hash" : "8b4c00d2eff52d51ad9647e14738944ab306fd1d8e1bf634fbb181b32f44070aa588938e26c4eb072b1eb61489aaf3062fb644a76c79f936b97723a2c3e0e5d3", - "size" : 70910, - "source_url" : "", - "tag_ids" : list, - "tags" : list, - "thumbnails_generated": True, - "updated_at" : str, - "upvotes" : int, - "view_url" : "https://cdn.twibooru.org/img/2020/7/8/1/full.png", - "width" : 576, - "wilson_score" : float, -}, - -{ - "#url" : "https://twibooru.org/523964", - "#comment" : "svg (#5643)", - "#category": ("philomena", "twibooru", "post"), - "#class" : twibooru.TwibooruPostExtractor, - "#urls" : "https://cdn.twibooru.org/img/2020/7/13/523964/full.svg", - "#sha1_content": "15590fe151ff65ef767b409e46dfdf708b339f4d", - - "extension": "svg", - "format" : "svg", -}, - -{ - "#url" : "https://twibooru.org/523964", - "#comment" : "svg (#5643)", - "#category": ("philomena", "twibooru", "post"), - "#class" : twibooru.TwibooruPostExtractor, - "#options" : {"svg": False}, - "#urls" : "https://cdn.twibooru.org/img/2020/7/13/523964/full.png", - "#sha1_content": "f8ff78e6a929a024f8529199f9a600617898d03c", - - "extension": "png", - "format" : "svg", -}, - -{ - "#url" : "https://twibooru.org/search?q=cute", - "#category": ("philomena", "twibooru", "search"), - "#class" : twibooru.TwibooruSearchExtractor, - "#range" : "40-60", - "#count" : 21, -}, - -{ - "#url" : "https://twibooru.org/tags/cute", - "#category": ("philomena", "twibooru", "search"), - "#class" : twibooru.TwibooruSearchExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://twibooru.org/galleries/1", - "#category": ("philomena", "twibooru", "gallery"), - "#class" : twibooru.TwibooruGalleryExtractor, - "#range" : "1-20", - - "gallery": { - "description" : "Best nation pone and russian related pics.", - "id" : 1, - "spoiler_warning": "Russia", - "thumbnail_id" : 694923, - "title" : "Marussiaverse", + { + "#url": "https://twibooru.org/1", + "#category": ("philomena", "twibooru", "post"), + "#class": twibooru.TwibooruPostExtractor, + "#pattern": "https://cdn.twibooru.org/img/2020/7/8/1/full.png", + "#sha1_content": "aac4d1dba611883ac701aaa8f0b2b322590517ae", + "animated": False, + "aspect_ratio": 1.0, + "comment_count": int, + "created_at": "2020-07-08T22:26:55.743Z", + "date": "dt:2020-07-08 22:26:55", + "description": "Why have I done this?", + "downvotes": 0, + "duration": 0.0, + "faves": int, + "first_seen_at": "2020-07-08T22:26:55.743Z", + "format": "png", + "height": 576, + "hidden_from_users": False, + "id": 1, + "intensities": dict, + "locations": [], + "media_type": "image", + "mime_type": "image/png", + "name": "1676547__safe_artist-colon-scraggleman_oc_oc-colon-floor+bored_oc+only_bags+under+eyes_bust_earth+pony_female_goggles_helmet_mare_meme_neet_neet+home+g.png", + "orig_sha512_hash": r"re:8b4c00d2[0-9a-f]{120}", + "processed": True, + "representations": dict, + "score": int, + "sha512_hash": "8b4c00d2eff52d51ad9647e14738944ab306fd1d8e1bf634fbb181b32f44070aa588938e26c4eb072b1eb61489aaf3062fb644a76c79f936b97723a2c3e0e5d3", + "size": 70910, + "source_url": "", + "tag_ids": list, + "tags": list, + "thumbnails_generated": True, + "updated_at": str, + "upvotes": int, + "view_url": "https://cdn.twibooru.org/img/2020/7/8/1/full.png", + "width": 576, + "wilson_score": float, + }, + { + "#url": "https://twibooru.org/523964", + "#comment": "svg (#5643)", + "#category": ("philomena", "twibooru", "post"), + "#class": twibooru.TwibooruPostExtractor, + "#urls": "https://cdn.twibooru.org/img/2020/7/13/523964/full.svg", + "#sha1_content": "15590fe151ff65ef767b409e46dfdf708b339f4d", + "extension": "svg", + "format": "svg", + }, + { + "#url": "https://twibooru.org/523964", + "#comment": "svg (#5643)", + "#category": ("philomena", "twibooru", "post"), + "#class": twibooru.TwibooruPostExtractor, + "#options": {"svg": False}, + "#urls": "https://cdn.twibooru.org/img/2020/7/13/523964/full.png", + "#sha1_content": "f8ff78e6a929a024f8529199f9a600617898d03c", + "extension": "png", + "format": "svg", + }, + { + "#url": "https://twibooru.org/search?q=cute", + "#category": ("philomena", "twibooru", "search"), + "#class": twibooru.TwibooruSearchExtractor, + "#range": "40-60", + "#count": 21, + }, + { + "#url": "https://twibooru.org/tags/cute", + "#category": ("philomena", "twibooru", "search"), + "#class": twibooru.TwibooruSearchExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://twibooru.org/galleries/1", + "#category": ("philomena", "twibooru", "gallery"), + "#class": twibooru.TwibooruGalleryExtractor, + "#range": "1-20", + "gallery": { + "description": "Best nation pone and russian related pics.", + "id": 1, + "spoiler_warning": "Russia", + "thumbnail_id": 694923, + "title": "Marussiaverse", + }, }, -}, - ) diff --git a/test/results/twitter.py b/test/results/twitter.py index 6da5d68e4..0961c31ca 100644 --- a/test/results/twitter.py +++ b/test/results/twitter.py @@ -1,721 +1,628 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +from gallery_dl import exception +from gallery_dl import util from gallery_dl.extractor import twitter -from gallery_dl import util, exception - __tests__ = ( -{ - "#url" : "https://twitter.com/supernaturepics", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, - "#options" : {"include": "all"}, - "#urls" : [ - "https://x.com/supernaturepics/info", - "https://x.com/supernaturepics/photo", - "https://x.com/supernaturepics/header_photo", - "https://x.com/supernaturepics/timeline", - "https://x.com/supernaturepics/tweets", - "https://x.com/supernaturepics/media", - "https://x.com/supernaturepics/with_replies", - "https://x.com/supernaturepics/likes", - ], -}, - -{ - "#url" : "https://mobile.twitter.com/supernaturepics?p=i", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://www.twitter.com/id:2976459548", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://twitter.com/i/user/2976459548", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://twitter.com/intent/user?user_id=2976459548", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://fxtwitter.com/supernaturepics", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://vxtwitter.com/supernaturepics", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://fixupx.com/supernaturepics", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://fixvx.com/supernaturepics", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://x.com/supernaturepics", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/timeline", - "#category": ("", "twitter", "timeline"), - "#class" : twitter.TwitterTimelineExtractor, - "#range" : "1-40", - "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", - - "author": { - "date" : "dt:2015-01-12 10:25:22", - "description" : "The very best nature pictures.", - "favourites_count": int, - "followers_count" : int, - "friends_count" : int, - "listed_count" : int, - "media_count" : int, - "statuses_count" : int, - "id" : 2976459548, - "location" : "Earth", - "name" : "supernaturepics", - "nick" : "Nature Pictures", - "profile_banner" : "https://pbs.twimg.com/profile_banners/2976459548/1421058583", - "profile_image" : "https://pbs.twimg.com/profile_images/554585280938659841/FLVAlX18.jpeg", - "protected" : False, - "verified" : False, - }, - "user": { - "date" : "dt:2015-01-12 10:25:22", - "description" : "The very best nature pictures.", - "favourites_count": int, - "followers_count" : int, - "friends_count" : int, - "listed_count" : int, - "media_count" : int, - "statuses_count" : int, - "id" : 2976459548, - "location" : "Earth", - "name" : "supernaturepics", - "nick" : "Nature Pictures", - "profile_banner" : "https://pbs.twimg.com/profile_banners/2976459548/1421058583", - "profile_image" : "https://pbs.twimg.com/profile_images/554585280938659841/FLVAlX18.jpeg", - "protected" : False, - "verified" : False, - }, - "tweet_id" : range(400000000000000000, 800000000000000000), - "conversation_id": range(400000000000000000, 800000000000000000), - "quote_id" : 0, - "reply_id" : 0, - "retweet_id" : 0, - "count" : range(1, 4), - "num" : range(1, 4), - "favorite_count" : int, - "quote_count" : int, - "reply_count" : int, - "retweet_count" : int, - "content" : str, - "lang" : str, - "date" : "type:datetime", - "sensitive" : False, - "source" : "nature_pics", -}, - -{ - "#url" : "https://twitter.com/OptionalTypo/timeline", - "#comment" : "suspended account (#2216)", - "#category": ("", "twitter", "timeline"), - "#class" : twitter.TwitterTimelineExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://twitter.com/id:772949683521978368/timeline", - "#comment" : "suspended account user ID", - "#category": ("", "twitter", "timeline"), - "#class" : twitter.TwitterTimelineExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://mobile.twitter.com/supernaturepics/timeline#t", - "#category": ("", "twitter", "timeline"), - "#class" : twitter.TwitterTimelineExtractor, -}, - -{ - "#url" : "https://www.twitter.com/id:2976459548/timeline", - "#category": ("", "twitter", "timeline"), - "#class" : twitter.TwitterTimelineExtractor, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/tweets", - "#category": ("", "twitter", "tweets"), - "#class" : twitter.TwitterTweetsExtractor, - "#range" : "1-40", - "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", -}, - -{ - "#url" : "https://mobile.twitter.com/supernaturepics/tweets#t", - "#category": ("", "twitter", "tweets"), - "#class" : twitter.TwitterTweetsExtractor, -}, - -{ - "#url" : "https://www.twitter.com/id:2976459548/tweets", - "#category": ("", "twitter", "tweets"), - "#class" : twitter.TwitterTweetsExtractor, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/with_replies", - "#category": ("", "twitter", "replies"), - "#class" : twitter.TwitterRepliesExtractor, - "#range" : "1-40", - "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", -}, - -{ - "#url" : "https://mobile.twitter.com/supernaturepics/with_replies#t", - "#category": ("", "twitter", "replies"), - "#class" : twitter.TwitterRepliesExtractor, -}, - -{ - "#url" : "https://www.twitter.com/id:2976459548/with_replies", - "#category": ("", "twitter", "replies"), - "#class" : twitter.TwitterRepliesExtractor, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/media", - "#category": ("", "twitter", "media"), - "#class" : twitter.TwitterMediaExtractor, - "#range" : "1-40", - "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", -}, - -{ - "#url" : "https://mobile.twitter.com/supernaturepics/media#t", - "#category": ("", "twitter", "media"), - "#class" : twitter.TwitterMediaExtractor, -}, - -{ - "#url" : "https://www.twitter.com/id:2976459548/media", - "#category": ("", "twitter", "media"), - "#class" : twitter.TwitterMediaExtractor, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/likes", - "#category": ("", "twitter", "likes"), - "#class" : twitter.TwitterLikesExtractor, -}, - -{ - "#url" : "https://twitter.com/i/bookmarks", - "#category": ("", "twitter", "bookmark"), - "#class" : twitter.TwitterBookmarkExtractor, -}, - -{ - "#url" : "https://twitter.com/i/lists/784214683683127296", - "#category": ("", "twitter", "list"), - "#class" : twitter.TwitterListExtractor, - "#range" : "1-40", - "#count" : 40, - "#archive" : False, -}, - -{ - "#url" : "https://twitter.com/i/lists/784214683683127296/members", - "#category": ("", "twitter", "list-members"), - "#class" : twitter.TwitterListMembersExtractor, - "#pattern" : twitter.TwitterUserExtractor.pattern, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/following", - "#category": ("", "twitter", "following"), - "#class" : twitter.TwitterFollowingExtractor, -}, - -{ - "#url" : "https://www.twitter.com/id:2976459548/following", - "#category": ("", "twitter", "following"), - "#class" : twitter.TwitterFollowingExtractor, -}, - -{ - "#url" : "https://twitter.com/search?q=nature", - "#category": ("", "twitter", "search"), - "#class" : twitter.TwitterSearchExtractor, - "#range" : "1-20", - "#count" : 20, - "#archive" : False, -}, - -{ - "#url" : "https://twitter.com/hashtag/nature", - "#category": ("", "twitter", "hashtag"), - "#class" : twitter.TwitterHashtagExtractor, - "#pattern" : twitter.TwitterSearchExtractor.pattern, - "#urls" : "https://x.com/search?q=%23nature", -}, - -{ - "#url" : "https://twitter.com/i/events/1484669206993903616", - "#category": ("", "twitter", "event"), - "#class" : twitter.TwitterEventExtractor, - "#range" : "1-20", - "#count" : ">=1", -}, - -{ - "#url" : "https://twitter.com/i/communities", - "#category": ("", "twitter", "communities"), - "#class" : twitter.TwitterCommunitiesExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://twitter.com/i/communities/1651515740753735697", - "#category": ("", "twitter", "community"), - "#class" : twitter.TwitterCommunityExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/status/604341487988576256", - "#comment" : "all Tweets from a 'conversation' (#1319)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#sha1_url" : "88a40f7d25529c2501c46f2218f9e0de9aa634b4", - "#sha1_content": "ab05e1d8d21f8d43496df284d31e8b362cd3bcab", -}, - -{ - "#url" : "https://twitter.com/perrypumas/status/894001459754180609", - "#comment" : "4 images", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#sha1_url": "3a2a43dc5fb79dd5432c701d8e55e87c4e551f47", - - "type": "photo", -}, - -{ - "#url" : "https://twitter.com/perrypumas/status/1065692031626829824?s=20", - "#comment" : "video", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#pattern" : r"https://video.twimg.com/ext_tw_video/.+\.mp4\?tag=5", - - "type": "video", -}, - -{ - "#url" : "https://twitter.com/playpokemon/status/1263832915173048321/", - "#comment" : "content with emoji, newlines, hashtags (#338)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - - "source" : "Sprinklr", - "content": r"""re:Gear up for #PokemonSwordShieldEX with special Mystery Gifts! \n + { + "#url": "https://twitter.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + "#options": {"include": "all"}, + "#urls": [ + "https://x.com/supernaturepics/info", + "https://x.com/supernaturepics/photo", + "https://x.com/supernaturepics/header_photo", + "https://x.com/supernaturepics/timeline", + "https://x.com/supernaturepics/tweets", + "https://x.com/supernaturepics/media", + "https://x.com/supernaturepics/with_replies", + "https://x.com/supernaturepics/likes", + ], + }, + { + "#url": "https://mobile.twitter.com/supernaturepics?p=i", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://www.twitter.com/id:2976459548", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://twitter.com/i/user/2976459548", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://twitter.com/intent/user?user_id=2976459548", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://fxtwitter.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://vxtwitter.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://fixupx.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://fixvx.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://x.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://twitter.com/supernaturepics/timeline", + "#category": ("", "twitter", "timeline"), + "#class": twitter.TwitterTimelineExtractor, + "#range": "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", + "author": { + "date": "dt:2015-01-12 10:25:22", + "description": "The very best nature pictures.", + "favourites_count": int, + "followers_count": int, + "friends_count": int, + "listed_count": int, + "media_count": int, + "statuses_count": int, + "id": 2976459548, + "location": "Earth", + "name": "supernaturepics", + "nick": "Nature Pictures", + "profile_banner": "https://pbs.twimg.com/profile_banners/2976459548/1421058583", + "profile_image": "https://pbs.twimg.com/profile_images/554585280938659841/FLVAlX18.jpeg", + "protected": False, + "verified": False, + }, + "user": { + "date": "dt:2015-01-12 10:25:22", + "description": "The very best nature pictures.", + "favourites_count": int, + "followers_count": int, + "friends_count": int, + "listed_count": int, + "media_count": int, + "statuses_count": int, + "id": 2976459548, + "location": "Earth", + "name": "supernaturepics", + "nick": "Nature Pictures", + "profile_banner": "https://pbs.twimg.com/profile_banners/2976459548/1421058583", + "profile_image": "https://pbs.twimg.com/profile_images/554585280938659841/FLVAlX18.jpeg", + "protected": False, + "verified": False, + }, + "tweet_id": range(400000000000000000, 800000000000000000), + "conversation_id": range(400000000000000000, 800000000000000000), + "quote_id": 0, + "reply_id": 0, + "retweet_id": 0, + "count": range(1, 4), + "num": range(1, 4), + "favorite_count": int, + "quote_count": int, + "reply_count": int, + "retweet_count": int, + "content": str, + "lang": str, + "date": "type:datetime", + "sensitive": False, + "source": "nature_pics", + }, + { + "#url": "https://twitter.com/OptionalTypo/timeline", + "#comment": "suspended account (#2216)", + "#category": ("", "twitter", "timeline"), + "#class": twitter.TwitterTimelineExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://twitter.com/id:772949683521978368/timeline", + "#comment": "suspended account user ID", + "#category": ("", "twitter", "timeline"), + "#class": twitter.TwitterTimelineExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://mobile.twitter.com/supernaturepics/timeline#t", + "#category": ("", "twitter", "timeline"), + "#class": twitter.TwitterTimelineExtractor, + }, + { + "#url": "https://www.twitter.com/id:2976459548/timeline", + "#category": ("", "twitter", "timeline"), + "#class": twitter.TwitterTimelineExtractor, + }, + { + "#url": "https://twitter.com/supernaturepics/tweets", + "#category": ("", "twitter", "tweets"), + "#class": twitter.TwitterTweetsExtractor, + "#range": "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", + }, + { + "#url": "https://mobile.twitter.com/supernaturepics/tweets#t", + "#category": ("", "twitter", "tweets"), + "#class": twitter.TwitterTweetsExtractor, + }, + { + "#url": "https://www.twitter.com/id:2976459548/tweets", + "#category": ("", "twitter", "tweets"), + "#class": twitter.TwitterTweetsExtractor, + }, + { + "#url": "https://twitter.com/supernaturepics/with_replies", + "#category": ("", "twitter", "replies"), + "#class": twitter.TwitterRepliesExtractor, + "#range": "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", + }, + { + "#url": "https://mobile.twitter.com/supernaturepics/with_replies#t", + "#category": ("", "twitter", "replies"), + "#class": twitter.TwitterRepliesExtractor, + }, + { + "#url": "https://www.twitter.com/id:2976459548/with_replies", + "#category": ("", "twitter", "replies"), + "#class": twitter.TwitterRepliesExtractor, + }, + { + "#url": "https://twitter.com/supernaturepics/media", + "#category": ("", "twitter", "media"), + "#class": twitter.TwitterMediaExtractor, + "#range": "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", + }, + { + "#url": "https://mobile.twitter.com/supernaturepics/media#t", + "#category": ("", "twitter", "media"), + "#class": twitter.TwitterMediaExtractor, + }, + { + "#url": "https://www.twitter.com/id:2976459548/media", + "#category": ("", "twitter", "media"), + "#class": twitter.TwitterMediaExtractor, + }, + { + "#url": "https://twitter.com/supernaturepics/likes", + "#category": ("", "twitter", "likes"), + "#class": twitter.TwitterLikesExtractor, + }, + { + "#url": "https://twitter.com/i/bookmarks", + "#category": ("", "twitter", "bookmark"), + "#class": twitter.TwitterBookmarkExtractor, + }, + { + "#url": "https://twitter.com/i/lists/784214683683127296", + "#category": ("", "twitter", "list"), + "#class": twitter.TwitterListExtractor, + "#range": "1-40", + "#count": 40, + "#archive": False, + }, + { + "#url": "https://twitter.com/i/lists/784214683683127296/members", + "#category": ("", "twitter", "list-members"), + "#class": twitter.TwitterListMembersExtractor, + "#pattern": twitter.TwitterUserExtractor.pattern, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "https://twitter.com/supernaturepics/following", + "#category": ("", "twitter", "following"), + "#class": twitter.TwitterFollowingExtractor, + }, + { + "#url": "https://www.twitter.com/id:2976459548/following", + "#category": ("", "twitter", "following"), + "#class": twitter.TwitterFollowingExtractor, + }, + { + "#url": "https://twitter.com/search?q=nature", + "#category": ("", "twitter", "search"), + "#class": twitter.TwitterSearchExtractor, + "#range": "1-20", + "#count": 20, + "#archive": False, + }, + { + "#url": "https://twitter.com/hashtag/nature", + "#category": ("", "twitter", "hashtag"), + "#class": twitter.TwitterHashtagExtractor, + "#pattern": twitter.TwitterSearchExtractor.pattern, + "#urls": "https://x.com/search?q=%23nature", + }, + { + "#url": "https://twitter.com/i/events/1484669206993903616", + "#category": ("", "twitter", "event"), + "#class": twitter.TwitterEventExtractor, + "#range": "1-20", + "#count": ">=1", + }, + { + "#url": "https://twitter.com/i/communities", + "#category": ("", "twitter", "communities"), + "#class": twitter.TwitterCommunitiesExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://twitter.com/i/communities/1651515740753735697", + "#category": ("", "twitter", "community"), + "#class": twitter.TwitterCommunityExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://twitter.com/supernaturepics/status/604341487988576256", + "#comment": "all Tweets from a 'conversation' (#1319)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#sha1_url": "88a40f7d25529c2501c46f2218f9e0de9aa634b4", + "#sha1_content": "ab05e1d8d21f8d43496df284d31e8b362cd3bcab", + }, + { + "#url": "https://twitter.com/perrypumas/status/894001459754180609", + "#comment": "4 images", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#sha1_url": "3a2a43dc5fb79dd5432c701d8e55e87c4e551f47", + "type": "photo", + }, + { + "#url": "https://twitter.com/perrypumas/status/1065692031626829824?s=20", + "#comment": "video", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#pattern": r"https://video.twimg.com/ext_tw_video/.+\.mp4\?tag=5", + "type": "video", + }, + { + "#url": "https://twitter.com/playpokemon/status/1263832915173048321/", + "#comment": "content with emoji, newlines, hashtags (#338)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "source": "Sprinklr", + "content": r"""re:Gear up for #PokemonSwordShieldEX with special Mystery Gifts! \n You’ll be able to receive four Galarian form Pokémon with Hidden Abilities, plus some very useful items. It’s our \(Mystery\) Gift to you, Trainers! \n ❓🎁➡️ """, -}, - -{ - "#url" : "https://twitter.com/i/web/status/1170041925560258560", - "#comment" : "'replies' option (#705)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#pattern" : "https://pbs.twimg.com/media/EDzS7VrU0AAFL4_", -}, - -{ - "#url" : "https://twitter.com/i/web/status/1170041925560258560", - "#comment" : "'replies' option (#705)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"replies": False}, - "#count" : 0, -}, - -{ - "#url" : "https://twitter.com/i/web/status/1424882930803908612", - "#comment" : "'replies' to self (#1254)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"replies": "self"}, - "#count" : 4, - - "user": { - "description": r"re:business email-- rhettaro.bloom@gmail.com patreon- http://patreon.com/Princecanary", - "url" : "http://princecanary.tumblr.com", }, -}, - -{ - "#url" : "https://twitter.com/i/web/status/1424898916156284928", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"replies": "self"}, - "#count" : 1, -}, - -{ - "#url" : "https://twitter.com/StobiesGalaxy/status/1270755918330896395", - "#comment" : "quoted tweet (#526, #854)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"quoted": True}, - "#pattern" : r"https://pbs\.twimg\.com/media/Ea[KG].+=jpg", - "#count" : 8, -}, - -{ - "#url" : "https://twitter.com/StobiesGalaxy/status/1270755918330896395", - "#comment" : "quoted tweet (#526, #854)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#pattern" : r"https://pbs\.twimg\.com/media/EaK.+=jpg", - "#count" : 4, -}, - -{ - "#url" : "https://twitter.com/web/status/1644907989109751810", - "#comment" : "different 'user' and 'author' in quoted Tweet (#3922)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - - "author": { - "id" : 321629993, - "name": "Cakes_Comics", + { + "#url": "https://twitter.com/i/web/status/1170041925560258560", + "#comment": "'replies' option (#705)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#pattern": "https://pbs.twimg.com/media/EDzS7VrU0AAFL4_", }, - "user" : { - "id" : 718928225360080897, - "name": "StobiesGalaxy", + { + "#url": "https://twitter.com/i/web/status/1170041925560258560", + "#comment": "'replies' option (#705)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"replies": False}, + "#count": 0, }, -}, - -{ - "#url" : "https://twitter.com/i/web/status/112900228289540096", - "#comment" : "TwitPic embeds (#579)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : { - "twitpic": True, - "cards" : False, - }, - "#pattern" : r"https://\w+.cloudfront.net/photos/large/\d+.jpg", - "#count" : 2, -}, - -{ - "#url" : "https://twitter.com/shimoigusaP/status/8138669971", - "#comment" : "TwitPic URL not in 'urls' (#3792)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"twitpic": True}, - "#pattern" : r"https://\w+.cloudfront.net/photos/large/\d+.png", - "#count" : 1, -}, - -{ - "#url" : "https://twitter.com/billboard/status/1306599586602135555", - "#comment" : "Twitter card (#1005)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"cards": True}, - "#pattern" : r"https://pbs.twimg.com/card_img/\d+/", -}, - -{ - "#url" : "https://twitter.com/i/web/status/1561674543323910144", - "#comment" : "unified_card image_website (#2875)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"cards": True}, - "#pattern" : r"https://pbs\.twimg\.com/media/F.+=jpg", -}, - -{ - "#url" : "https://twitter.com/doax_vv_staff/status/1479438945662685184", - "#comment" : "unified_card image_carousel_website", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"cards": True}, - "#pattern" : r"https://pbs\.twimg\.com/media/F.+=png", - "#count" : 6, -}, - -{ - "#url" : "https://twitter.com/bang_dream_1242/status/1561548715348746241", - "#comment" : "unified_card video_website (#2875)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"cards": True}, - "#pattern" : r"https://video\.twimg\.com/amplify_video/1560607284333449216/vid/720x720/\w+\.mp4", -}, - -{ - "#url" : "https://twitter.com/i/web/status/1466183847628865544", - "#comment" : "unified_card without type", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://twitter.com/i/web/status/1571141912295243776", - "#comment" : "'cards-blacklist' option", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : { - "cards" : "ytdl", - "cards-blacklist": ("twitch.tv",), - }, - "#count" : 0, -}, - -{ - "#url" : "https://twitter.com/jessica_3978/status/1296304589591810048", - "#comment" : "original retweets (#1026)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"retweets": True}, - "#count" : 2, - - "tweet_id" : 1296304589591810048, - "retweet_id" : 1296296016002547713, - "date" : "dt:2020-08-20 04:34:32", - "date_original": "dt:2020-08-20 04:00:28", -}, - -{ - "#url" : "https://twitter.com/jessica_3978/status/1296304589591810048", - "#comment" : "original retweets (#1026)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"retweets": "original"}, - "#count" : 2, - - "tweet_id" : 1296296016002547713, - "retweet_id" : 1296296016002547713, - "date" : "dt:2020-08-20 04:00:28", - "date_original": "dt:2020-08-20 04:00:28", -}, - -{ - "#url" : "https://twitter.com/supernaturepics/status/604341487988576256", - "#comment" : "all Tweets from a 'conversation' (#1319)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"conversations": True}, - "#count" : 5, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/status/604341487988576256/photo/1", - "#comment" : "/photo/ URL (#5443)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, -}, - -{ - "#url" : "https://twitter.com/perrypumas/status/1065692031626829824/video/1", - "#comment" : "/video/ URL", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, -}, - -{ - "#url" : "https://twitter.com/morino_ya/status/1392763691599237121", - "#comment" : "retweet with missing media entities (#1555)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"retweets": True}, - "#count" : 4, -}, - -{ - "#url" : "https://twitter.com/i/web/status/1460044411165888515", - "#comment" : "deleted quote tweet (#2225)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://twitter.com/i/web/status/1486373748911575046", - "#comment" : "'Misleading' content", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#count" : 4, -}, - -{ - "#url" : "https://twitter.com/mightbecursed/status/1492954264909479936", - "#comment" : "age-restricted (#2354)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#auth" : False, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://twitter.com/my0nruri/status/1528379296041299968", - "#comment" : "media alt texts / descriptions (#2617)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - - "description": "oc", - "type" : "photo", -}, - -{ - "#url" : "https://twitter.com/poco_dandy/status/1150646424461176832", - "#comment" : "'?format=...&name=...'-style URLs", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"cards": True}, - "#pattern" : r"https://pbs.twimg.com/card_img/17\d+/[\w-]+\?format=(jpg|png)&name=orig$", - "#range" : "1,3", -}, - -{ - "#url" : "https://twitter.com/i/web/status/1629193457112686592", - "#comment" : "note tweet with long 'content'", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - - "content": """BREAKING - DEADLY LIES: Independent researchers at Texas A&M University have just contradicted federal government regulators, saying that toxic air pollutants in East Palestine, Ohio, could pose long-term risks. \n + { + "#url": "https://twitter.com/i/web/status/1424882930803908612", + "#comment": "'replies' to self (#1254)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"replies": "self"}, + "#count": 4, + "user": { + "description": r"re:business email-- rhettaro.bloom@gmail.com patreon- http://patreon.com/Princecanary", + "url": "http://princecanary.tumblr.com", + }, + }, + { + "#url": "https://twitter.com/i/web/status/1424898916156284928", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"replies": "self"}, + "#count": 1, + }, + { + "#url": "https://twitter.com/StobiesGalaxy/status/1270755918330896395", + "#comment": "quoted tweet (#526, #854)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"quoted": True}, + "#pattern": r"https://pbs\.twimg\.com/media/Ea[KG].+=jpg", + "#count": 8, + }, + { + "#url": "https://twitter.com/StobiesGalaxy/status/1270755918330896395", + "#comment": "quoted tweet (#526, #854)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#pattern": r"https://pbs\.twimg\.com/media/EaK.+=jpg", + "#count": 4, + }, + { + "#url": "https://twitter.com/web/status/1644907989109751810", + "#comment": "different 'user' and 'author' in quoted Tweet (#3922)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "author": { + "id": 321629993, + "name": "Cakes_Comics", + }, + "user": { + "id": 718928225360080897, + "name": "StobiesGalaxy", + }, + }, + { + "#url": "https://twitter.com/i/web/status/112900228289540096", + "#comment": "TwitPic embeds (#579)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": { + "twitpic": True, + "cards": False, + }, + "#pattern": r"https://\w+.cloudfront.net/photos/large/\d+.jpg", + "#count": 2, + }, + { + "#url": "https://twitter.com/shimoigusaP/status/8138669971", + "#comment": "TwitPic URL not in 'urls' (#3792)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"twitpic": True}, + "#pattern": r"https://\w+.cloudfront.net/photos/large/\d+.png", + "#count": 1, + }, + { + "#url": "https://twitter.com/billboard/status/1306599586602135555", + "#comment": "Twitter card (#1005)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"cards": True}, + "#pattern": r"https://pbs.twimg.com/card_img/\d+/", + }, + { + "#url": "https://twitter.com/i/web/status/1561674543323910144", + "#comment": "unified_card image_website (#2875)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"cards": True}, + "#pattern": r"https://pbs\.twimg\.com/media/F.+=jpg", + }, + { + "#url": "https://twitter.com/doax_vv_staff/status/1479438945662685184", + "#comment": "unified_card image_carousel_website", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"cards": True}, + "#pattern": r"https://pbs\.twimg\.com/media/F.+=png", + "#count": 6, + }, + { + "#url": "https://twitter.com/bang_dream_1242/status/1561548715348746241", + "#comment": "unified_card video_website (#2875)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"cards": True}, + "#pattern": r"https://video\.twimg\.com/amplify_video/1560607284333449216/vid/720x720/\w+\.mp4", + }, + { + "#url": "https://twitter.com/i/web/status/1466183847628865544", + "#comment": "unified_card without type", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#count": 0, + }, + { + "#url": "https://twitter.com/i/web/status/1571141912295243776", + "#comment": "'cards-blacklist' option", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": { + "cards": "ytdl", + "cards-blacklist": ("twitch.tv",), + }, + "#count": 0, + }, + { + "#url": "https://twitter.com/jessica_3978/status/1296304589591810048", + "#comment": "original retweets (#1026)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"retweets": True}, + "#count": 2, + "tweet_id": 1296304589591810048, + "retweet_id": 1296296016002547713, + "date": "dt:2020-08-20 04:34:32", + "date_original": "dt:2020-08-20 04:00:28", + }, + { + "#url": "https://twitter.com/jessica_3978/status/1296304589591810048", + "#comment": "original retweets (#1026)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"retweets": "original"}, + "#count": 2, + "tweet_id": 1296296016002547713, + "retweet_id": 1296296016002547713, + "date": "dt:2020-08-20 04:00:28", + "date_original": "dt:2020-08-20 04:00:28", + }, + { + "#url": "https://twitter.com/supernaturepics/status/604341487988576256", + "#comment": "all Tweets from a 'conversation' (#1319)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"conversations": True}, + "#count": 5, + }, + { + "#url": "https://twitter.com/supernaturepics/status/604341487988576256/photo/1", + "#comment": "/photo/ URL (#5443)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + }, + { + "#url": "https://twitter.com/perrypumas/status/1065692031626829824/video/1", + "#comment": "/video/ URL", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + }, + { + "#url": "https://twitter.com/morino_ya/status/1392763691599237121", + "#comment": "retweet with missing media entities (#1555)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"retweets": True}, + "#count": 4, + }, + { + "#url": "https://twitter.com/i/web/status/1460044411165888515", + "#comment": "deleted quote tweet (#2225)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#count": 0, + }, + { + "#url": "https://twitter.com/i/web/status/1486373748911575046", + "#comment": "'Misleading' content", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#count": 4, + }, + { + "#url": "https://twitter.com/mightbecursed/status/1492954264909479936", + "#comment": "age-restricted (#2354)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#auth": False, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://twitter.com/my0nruri/status/1528379296041299968", + "#comment": "media alt texts / descriptions (#2617)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "description": "oc", + "type": "photo", + }, + { + "#url": "https://twitter.com/poco_dandy/status/1150646424461176832", + "#comment": "'?format=...&name=...'-style URLs", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"cards": True}, + "#pattern": r"https://pbs.twimg.com/card_img/17\d+/[\w-]+\?format=(jpg|png)&name=orig$", + "#range": "1,3", + }, + { + "#url": "https://twitter.com/i/web/status/1629193457112686592", + "#comment": "note tweet with long 'content'", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "content": """BREAKING - DEADLY LIES: Independent researchers at Texas A&M University have just contradicted federal government regulators, saying that toxic air pollutants in East Palestine, Ohio, could pose long-term risks. \n The Washington Post writes, "Three weeks after the toxic train derailment in Ohio, an analysis of Environmental Protection Agency data has found nine air pollutants at levels that could raise long-term health concerns in and around East Palestine, according to an independent analysis. \n "The analysis by Texas A&M University seems to contradict statements by state and federal regulators that air near the crash site is completely safe, despite residents complaining about rashes, breathing problems and other health effects." Your reaction.""", -}, - -{ - "#url" : "https://twitter.com/KrisKobach1787/status/1765935595702919299", - "#comment" : "'birdwatch' note (#5317)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"text-tweets": True}, - - "birdwatch": "In addition to the known harm of lead exposure, especially to children, Mr. Kobach is incorrect when he states the mandate is unfunded. In fact, the BIPARTISAN Infrastructure Law Joe Biden signed into law in Nov 2021 provides $15B toward lead service line replacement projects. epa.gov/ground-water-a…", - "content" : "Biden wants to replace lead pipes. He failed to mention that the unfunded mandate sets an almost impossible timeline, will cost billions, infringe on the rights of the States and their residents – all for benefits that may be entirely speculative. #sotu https://ag.ks.gov/media-center/news-releases/2024/02/09/kobach-leads-coalition-demanding-biden-drop-unnecessary-epa-rule", -}, - -{ - "#url" : "https://x.com/jsports_motor/status/1801338077618524583", - "#comment" : "geo-restricted video (#5736)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://twitter.com/playpokemon/status/1263832915173048321/quotes", - "#category": ("", "twitter", "quotes"), - "#class" : twitter.TwitterQuotesExtractor, - "#pattern" : twitter.TwitterSearchExtractor.pattern, - "#urls" : "https://x.com/search?q=quoted_tweet_id:1263832915173048321", -}, - -{ - "#url" : "https://twitter.com/supernaturepics/info", - "#category": ("", "twitter", "info"), - "#class" : twitter.TwitterInfoExtractor, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/photo", - "#category": ("", "twitter", "avatar"), - "#class" : twitter.TwitterAvatarExtractor, - "#urls" : "https://pbs.twimg.com/profile_images/554585280938659841/FLVAlX18.jpeg", - - "date" : "dt:2015-01-12 10:26:49", - "extension": "jpeg", - "filename" : "FLVAlX18", - "tweet_id" : 554585280938659841, -}, - -{ - "#url" : "https://twitter.com/User16/photo", - "#category": ("", "twitter", "avatar"), - "#class" : twitter.TwitterAvatarExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://twitter.com/i_n_u/photo", - "#comment" : "old avatar with small ID and no valid 'date' (#4696)", - "#category": ("", "twitter", "avatar"), - "#class" : twitter.TwitterAvatarExtractor, - "#urls" : "https://pbs.twimg.com/profile_images/2946444489/32028c6affdab425e037ff5a6bf77c1d.jpeg", - - "date" : util.NONE, - "tweet_id" : 2946444489, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/header_photo", - "#category": ("", "twitter", "background"), - "#class" : twitter.TwitterBackgroundExtractor, - "#pattern" : r"https://pbs\.twimg\.com/profile_banners/2976459548/1421058583", - - "date" : "dt:2015-01-12 10:29:43", - "filename": "1421058583", - "tweet_id": 554586009367478272, -}, - -{ - "#url" : "https://twitter.com/User16/header_photo", - "#category": ("", "twitter", "background"), - "#class" : twitter.TwitterBackgroundExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://pbs.twimg.com/media/EqcpviCVoAAG-QG?format=jpg&name=orig", - "#category": ("", "twitter", "image"), - "#class" : twitter.TwitterImageExtractor, - "#options" : {"size": "4096x4096,orig"}, - "#sha1_url": "cb3042a6f6826923da98f0d2b66c427e9385114c", -}, - -{ - "#url" : "https://pbs.twimg.com/media/EqcpviCVoAAG-QG.jpg:orig", - "#category": ("", "twitter", "image"), - "#class" : twitter.TwitterImageExtractor, -}, - + }, + { + "#url": "https://twitter.com/KrisKobach1787/status/1765935595702919299", + "#comment": "'birdwatch' note (#5317)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"text-tweets": True}, + "birdwatch": "In addition to the known harm of lead exposure, especially to children, Mr. Kobach is incorrect when he states the mandate is unfunded. In fact, the BIPARTISAN Infrastructure Law Joe Biden signed into law in Nov 2021 provides $15B toward lead service line replacement projects. epa.gov/ground-water-a…", + "content": "Biden wants to replace lead pipes. He failed to mention that the unfunded mandate sets an almost impossible timeline, will cost billions, infringe on the rights of the States and their residents – all for benefits that may be entirely speculative. #sotu https://ag.ks.gov/media-center/news-releases/2024/02/09/kobach-leads-coalition-demanding-biden-drop-unnecessary-epa-rule", + }, + { + "#url": "https://x.com/jsports_motor/status/1801338077618524583", + "#comment": "geo-restricted video (#5736)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#count": 0, + }, + { + "#url": "https://twitter.com/playpokemon/status/1263832915173048321/quotes", + "#category": ("", "twitter", "quotes"), + "#class": twitter.TwitterQuotesExtractor, + "#pattern": twitter.TwitterSearchExtractor.pattern, + "#urls": "https://x.com/search?q=quoted_tweet_id:1263832915173048321", + }, + { + "#url": "https://twitter.com/supernaturepics/info", + "#category": ("", "twitter", "info"), + "#class": twitter.TwitterInfoExtractor, + }, + { + "#url": "https://twitter.com/supernaturepics/photo", + "#category": ("", "twitter", "avatar"), + "#class": twitter.TwitterAvatarExtractor, + "#urls": "https://pbs.twimg.com/profile_images/554585280938659841/FLVAlX18.jpeg", + "date": "dt:2015-01-12 10:26:49", + "extension": "jpeg", + "filename": "FLVAlX18", + "tweet_id": 554585280938659841, + }, + { + "#url": "https://twitter.com/User16/photo", + "#category": ("", "twitter", "avatar"), + "#class": twitter.TwitterAvatarExtractor, + "#count": 0, + }, + { + "#url": "https://twitter.com/i_n_u/photo", + "#comment": "old avatar with small ID and no valid 'date' (#4696)", + "#category": ("", "twitter", "avatar"), + "#class": twitter.TwitterAvatarExtractor, + "#urls": "https://pbs.twimg.com/profile_images/2946444489/32028c6affdab425e037ff5a6bf77c1d.jpeg", + "date": util.NONE, + "tweet_id": 2946444489, + }, + { + "#url": "https://twitter.com/supernaturepics/header_photo", + "#category": ("", "twitter", "background"), + "#class": twitter.TwitterBackgroundExtractor, + "#pattern": r"https://pbs\.twimg\.com/profile_banners/2976459548/1421058583", + "date": "dt:2015-01-12 10:29:43", + "filename": "1421058583", + "tweet_id": 554586009367478272, + }, + { + "#url": "https://twitter.com/User16/header_photo", + "#category": ("", "twitter", "background"), + "#class": twitter.TwitterBackgroundExtractor, + "#count": 0, + }, + { + "#url": "https://pbs.twimg.com/media/EqcpviCVoAAG-QG?format=jpg&name=orig", + "#category": ("", "twitter", "image"), + "#class": twitter.TwitterImageExtractor, + "#options": {"size": "4096x4096,orig"}, + "#sha1_url": "cb3042a6f6826923da98f0d2b66c427e9385114c", + }, + { + "#url": "https://pbs.twimg.com/media/EqcpviCVoAAG-QG.jpg:orig", + "#category": ("", "twitter", "image"), + "#class": twitter.TwitterImageExtractor, + }, ) diff --git a/test/results/unique-vintage.py b/test/results/unique-vintage.py index 8fe37a3be..a164be379 100644 --- a/test/results/unique-vintage.py +++ b/test/results/unique-vintage.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://www.unique-vintage.com/collections/flapper-1920s", - "#category": ("shopify", "unique-vintage", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.unique-vintage.com/collections/flapper-1920s/products/unique-vintage-plus-size-black-silver-beaded-troyes-flapper-dress", - "#category": ("shopify", "unique-vintage", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://www.unique-vintage.com/collections/flapper-1920s", + "#category": ("shopify", "unique-vintage", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.unique-vintage.com/collections/flapper-1920s/products/unique-vintage-plus-size-black-silver-beaded-troyes-flapper-dress", + "#category": ("shopify", "unique-vintage", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/unsplash.py b/test/results/unsplash.py index 8c5297430..f5f65a500 100644 --- a/test/results/unsplash.py +++ b/test/results/unsplash.py @@ -1,159 +1,147 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import unsplash - __tests__ = ( -{ - "#url" : "https://unsplash.com/photos/red-wooden-cross-on-gray-concrete-pathway-between-green-trees-during-daytime-kaoHI0iHJPM", - "#category": ("", "unsplash", "image"), - "#class" : unsplash.UnsplashImageExtractor, - "#pattern" : r"https://images\.unsplash\.com/photo-1601823984263-b87b59798b", - - "alt_description": "red wooden cross on gray concrete pathway between green trees during daytime", - "blur_hash" : "LIAwhq%e4TRjXAIBMyt89GRj%fj[", - "breadcrumbs": list, - "color" : "#0c2626", - "created_at" : "2020-10-04T15:13:59Z", - "date" : "dt:2020-10-04 15:13:59", - "description": None, - "downloads" : range(50000, 300000), - "exif" : { - "aperture" : "9", - "exposure_time": "1/125", - "focal_length" : "35.0", - "iso" : 800, - "make" : "SONY", - "model" : "ILCE-7M3", - "name" : "SONY, ILCE-7M3", - }, - "extension" : "jpg", - "filename" : "photo-1601823984263-b87b59798b70", - "height" : 5371, - "id" : "kaoHI0iHJPM", - "liked_by_user": False, - "likes" : range(1000, 10000), - "links" : dict, - "location" : { - "city" : "箱根町", - "country" : "日本", - "name" : "Hakone, 神奈川県 日本", - "position": { - "latitude" : 35.232383, - "longitude": 139.106936, + { + "#url": "https://unsplash.com/photos/red-wooden-cross-on-gray-concrete-pathway-between-green-trees-during-daytime-kaoHI0iHJPM", + "#category": ("", "unsplash", "image"), + "#class": unsplash.UnsplashImageExtractor, + "#pattern": r"https://images\.unsplash\.com/photo-1601823984263-b87b59798b", + "alt_description": "red wooden cross on gray concrete pathway between green trees during daytime", + "blur_hash": "LIAwhq%e4TRjXAIBMyt89GRj%fj[", + "breadcrumbs": list, + "color": "#0c2626", + "created_at": "2020-10-04T15:13:59Z", + "date": "dt:2020-10-04 15:13:59", + "description": None, + "downloads": range(50000, 300000), + "exif": { + "aperture": "9", + "exposure_time": "1/125", + "focal_length": "35.0", + "iso": 800, + "make": "SONY", + "model": "ILCE-7M3", + "name": "SONY, ILCE-7M3", }, - }, - "meta" : { - "index": True, - }, - "plus" : False, - "premium" : False, - "promoted_at": "2020-10-05T13:04:43Z", - "public_domain": False, - "slug" : "red-wooden-cross-on-gray-concrete-pathway-between-green-trees-during-daytime-kaoHI0iHJPM", - "sponsorship": None, - "subcategory": "image", - "tags" : list, - "tags_preview": list, - "topic_submissions": {}, - "topics" : [], - "updated_at" : str, - "urls": dict, - "user": { - "accepted_tos" : True, - "bio" : "Professional photographer.\r\nBased in Japan.", - "first_name" : "Syuhei", - "for_hire" : True, - "id" : "F4HO358YSeo", - "instagram_username": "_______life_", - "last_name" : "Inoue", - "links": { - "followers": "https://api.unsplash.com/users/_______life_/followers", - "following": "https://api.unsplash.com/users/_______life_/following", - "html" : "https://unsplash.com/@_______life_", - "likes" : "https://api.unsplash.com/users/_______life_/likes", - "photos" : "https://api.unsplash.com/users/_______life_/photos", - "portfolio": "https://api.unsplash.com/users/_______life_/portfolio", - "self" : "https://api.unsplash.com/users/_______life_", + "extension": "jpg", + "filename": "photo-1601823984263-b87b59798b70", + "height": 5371, + "id": "kaoHI0iHJPM", + "liked_by_user": False, + "likes": range(1000, 10000), + "links": dict, + "location": { + "city": "箱根町", + "country": "日本", + "name": "Hakone, 神奈川県 日本", + "position": { + "latitude": 35.232383, + "longitude": 139.106936, + }, }, - "location" : "Yokohama, Japan", - "name" : "Syuhei Inoue", - "portfolio_url" : "https://syuheiinoue.life/", - "profile_image" : { - "large" : "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=128&h=128", - "medium": "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=64&h=64", - "small" : "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=32&h=32", + "meta": { + "index": True, }, - "social" : { + "plus": False, + "premium": False, + "promoted_at": "2020-10-05T13:04:43Z", + "public_domain": False, + "slug": "red-wooden-cross-on-gray-concrete-pathway-between-green-trees-during-daytime-kaoHI0iHJPM", + "sponsorship": None, + "subcategory": "image", + "tags": list, + "tags_preview": list, + "topic_submissions": {}, + "topics": [], + "updated_at": str, + "urls": dict, + "user": { + "accepted_tos": True, + "bio": "Professional photographer.\r\nBased in Japan.", + "first_name": "Syuhei", + "for_hire": True, + "id": "F4HO358YSeo", "instagram_username": "_______life_", - "paypal_email" : None, - "portfolio_url" : "https://syuheiinoue.life/", - "twitter_username" : None, + "last_name": "Inoue", + "links": { + "followers": "https://api.unsplash.com/users/_______life_/followers", + "following": "https://api.unsplash.com/users/_______life_/following", + "html": "https://unsplash.com/@_______life_", + "likes": "https://api.unsplash.com/users/_______life_/likes", + "photos": "https://api.unsplash.com/users/_______life_/photos", + "portfolio": "https://api.unsplash.com/users/_______life_/portfolio", + "self": "https://api.unsplash.com/users/_______life_", + }, + "location": "Yokohama, Japan", + "name": "Syuhei Inoue", + "portfolio_url": "https://syuheiinoue.life/", + "profile_image": { + "large": "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=128&h=128", + "medium": "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=64&h=64", + "small": "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=32&h=32", + }, + "social": { + "instagram_username": "_______life_", + "paypal_email": None, + "portfolio_url": "https://syuheiinoue.life/", + "twitter_username": None, + }, + "total_collections": 2, + "total_likes": 32, + "total_photos": 86, + "total_promoted_photos": 24, + "twitter_username": None, + "updated_at": str, + "username": "_______life_", }, - "total_collections" : 2, - "total_likes" : 32, - "total_photos" : 86, - "total_promoted_photos": 24, - "twitter_username" : None, - "updated_at" : str, - "username" : "_______life_" + "views": range(2000000, 10000000), + "width": 3581, + }, + { + "#url": "https://unsplash.com/@_______life_", + "#category": ("", "unsplash", "user"), + "#class": unsplash.UnsplashUserExtractor, + "#pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://unsplash.com/@_______life_/likes", + "#category": ("", "unsplash", "favorite"), + "#class": unsplash.UnsplashFavoriteExtractor, + "#pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#count": range(25, 35), + }, + { + "#url": "https://unsplash.com/collections/3178572/winter", + "#category": ("", "unsplash", "collection"), + "#class": unsplash.UnsplashCollectionExtractor, + "#pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range": "1-30", + "#count": 30, + "collection_id": "3178572", + "collection_title": "winter", + }, + { + "#url": "https://unsplash.com/collections/3178572/", + "#category": ("", "unsplash", "collection"), + "#class": unsplash.UnsplashCollectionExtractor, + }, + { + "#url": "https://unsplash.com/collections/_8qJQ2bCMWE/2021.05", + "#category": ("", "unsplash", "collection"), + "#class": unsplash.UnsplashCollectionExtractor, + }, + { + "#url": "https://unsplash.com/s/photos/hair-style", + "#category": ("", "unsplash", "search"), + "#class": unsplash.UnsplashSearchExtractor, + "#pattern": r"https://(images|plus)\.unsplash\.com/((flagged/|premium_)?photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range": "1-30", + "#count": 30, }, - "views": range(2000000, 10000000), - "width": 3581, -}, - -{ - "#url" : "https://unsplash.com/@_______life_", - "#category": ("", "unsplash", "user"), - "#class" : unsplash.UnsplashUserExtractor, - "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://unsplash.com/@_______life_/likes", - "#category": ("", "unsplash", "favorite"), - "#class" : unsplash.UnsplashFavoriteExtractor, - "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "#count" : range(25, 35), -}, - -{ - "#url" : "https://unsplash.com/collections/3178572/winter", - "#category": ("", "unsplash", "collection"), - "#class" : unsplash.UnsplashCollectionExtractor, - "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "#range" : "1-30", - "#count" : 30, - - "collection_id" : "3178572", - "collection_title": "winter", -}, - -{ - "#url" : "https://unsplash.com/collections/3178572/", - "#category": ("", "unsplash", "collection"), - "#class" : unsplash.UnsplashCollectionExtractor, -}, - -{ - "#url" : "https://unsplash.com/collections/_8qJQ2bCMWE/2021.05", - "#category": ("", "unsplash", "collection"), - "#class" : unsplash.UnsplashCollectionExtractor, -}, - -{ - "#url" : "https://unsplash.com/s/photos/hair-style", - "#category": ("", "unsplash", "search"), - "#class" : unsplash.UnsplashSearchExtractor, - "#pattern" : r"https://(images|plus)\.unsplash\.com/((flagged/|premium_)?photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "#range" : "1-30", - "#count" : 30, -}, - ) diff --git a/test/results/uploadir.py b/test/results/uploadir.py index 18d6a204e..e0767d8ac 100644 --- a/test/results/uploadir.py +++ b/test/results/uploadir.py @@ -1,62 +1,51 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import uploadir - __tests__ = ( -{ - "#url" : "https://uploadir.com/u/rd3t46ry", - "#comment" : "image", - "#category": ("", "uploadir", "file"), - "#class" : uploadir.UploadirFileExtractor, - "#pattern" : r"https://uploadir\.com/u/rd3t46ry", - "#count" : 1, - - "extension": "jpg", - "filename" : "Chloe and Rachel 4K jpg", - "id" : "rd3t46ry", -}, - -{ - "#url" : "https://uploadir.com/uploads/gxe8ti9v/downloads/new", - "#comment" : "archive", - "#category": ("", "uploadir", "file"), - "#class" : uploadir.UploadirFileExtractor, - "#pattern" : r"https://uploadir\.com/uploads/gxe8ti9v/downloads", - "#count" : 1, - - "extension": "zip", - "filename" : "NYAN-Mods-Pack#1", - "id" : "gxe8ti9v", -}, - -{ - "#url" : "https://uploadir.com/u/fllda6xl", - "#comment" : "utf-8 filename", - "#category": ("", "uploadir", "file"), - "#class" : uploadir.UploadirFileExtractor, - "#pattern" : r"https://uploadir\.com/u/fllda6xl", - "#count" : 1, - - "extension": "png", - "filename" : "_圖片_🖼_image_", - "id" : "fllda6xl", -}, - -{ - "#url" : "https://uploadir.com/uploads/rd3t46ry", - "#category": ("", "uploadir", "file"), - "#class" : uploadir.UploadirFileExtractor, -}, - -{ - "#url" : "https://uploadir.com/user/uploads/rd3t46ry", - "#category": ("", "uploadir", "file"), - "#class" : uploadir.UploadirFileExtractor, -}, - + { + "#url": "https://uploadir.com/u/rd3t46ry", + "#comment": "image", + "#category": ("", "uploadir", "file"), + "#class": uploadir.UploadirFileExtractor, + "#pattern": r"https://uploadir\.com/u/rd3t46ry", + "#count": 1, + "extension": "jpg", + "filename": "Chloe and Rachel 4K jpg", + "id": "rd3t46ry", + }, + { + "#url": "https://uploadir.com/uploads/gxe8ti9v/downloads/new", + "#comment": "archive", + "#category": ("", "uploadir", "file"), + "#class": uploadir.UploadirFileExtractor, + "#pattern": r"https://uploadir\.com/uploads/gxe8ti9v/downloads", + "#count": 1, + "extension": "zip", + "filename": "NYAN-Mods-Pack#1", + "id": "gxe8ti9v", + }, + { + "#url": "https://uploadir.com/u/fllda6xl", + "#comment": "utf-8 filename", + "#category": ("", "uploadir", "file"), + "#class": uploadir.UploadirFileExtractor, + "#pattern": r"https://uploadir\.com/u/fllda6xl", + "#count": 1, + "extension": "png", + "filename": "_圖片_🖼_image_", + "id": "fllda6xl", + }, + { + "#url": "https://uploadir.com/uploads/rd3t46ry", + "#category": ("", "uploadir", "file"), + "#class": uploadir.UploadirFileExtractor, + }, + { + "#url": "https://uploadir.com/user/uploads/rd3t46ry", + "#category": ("", "uploadir", "file"), + "#class": uploadir.UploadirFileExtractor, + }, ) diff --git a/test/results/urlgalleries.py b/test/results/urlgalleries.py index 88a321e76..8811e52c6 100644 --- a/test/results/urlgalleries.py +++ b/test/results/urlgalleries.py @@ -1,49 +1,42 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import urlgalleries - __tests__ = ( -{ - "#url" : "https://photos2q.urlgalleries.net/porn-gallery-7851311/clarice-window-8", - "#category": ("", "urlgalleries", "gallery"), - "#class" : urlgalleries.UrlgalleriesGalleryExtractor, - "#range" : "1-3", - "#urls" : ( - "https://fappic.com/x207mqkn2463/4gq1yv.jpg", - "https://fappic.com/q684ua2rp0j9/4gq1xv.jpg", - "https://fappic.com/8vf3n8fgz9po/4gq1ya.jpg", - ), - - "blog" : "photos2q", - "count" : 39, - "date" : "dt:2023-12-08 13:59:00", - "gallery_id": "7851311", - "num" : range(1, 3), - "title" : "Clarice window 8", -}, - -{ - "#url" : "https://dreamer.urlgalleries.net/7645840", - "#category": ("", "urlgalleries", "gallery"), - "#class" : urlgalleries.UrlgalleriesGalleryExtractor, - "#range" : "1-3", - "#urls" : ( - "https://www.fappic.com/vj7up04ny487/AmourAngels-0001.jpg", - "https://www.fappic.com/zfgsmpm36iyv/AmourAngels-0002.jpg", - "https://www.fappic.com/rqpt37rdbwa5/AmourAngels-0003.jpg", - ), - - "blog" : "Dreamer", - "count" : 105, - "date" : "dt:2020-03-10 21:17:00", - "gallery_id": "7645840", - "num" : range(1, 3), - "title" : "Angelika - Rustic Charm - AmourAngels 2016-09-27", -}, - + { + "#url": "https://photos2q.urlgalleries.net/porn-gallery-7851311/clarice-window-8", + "#category": ("", "urlgalleries", "gallery"), + "#class": urlgalleries.UrlgalleriesGalleryExtractor, + "#range": "1-3", + "#urls": ( + "https://fappic.com/x207mqkn2463/4gq1yv.jpg", + "https://fappic.com/q684ua2rp0j9/4gq1xv.jpg", + "https://fappic.com/8vf3n8fgz9po/4gq1ya.jpg", + ), + "blog": "photos2q", + "count": 39, + "date": "dt:2023-12-08 13:59:00", + "gallery_id": "7851311", + "num": range(1, 3), + "title": "Clarice window 8", + }, + { + "#url": "https://dreamer.urlgalleries.net/7645840", + "#category": ("", "urlgalleries", "gallery"), + "#class": urlgalleries.UrlgalleriesGalleryExtractor, + "#range": "1-3", + "#urls": ( + "https://www.fappic.com/vj7up04ny487/AmourAngels-0001.jpg", + "https://www.fappic.com/zfgsmpm36iyv/AmourAngels-0002.jpg", + "https://www.fappic.com/rqpt37rdbwa5/AmourAngels-0003.jpg", + ), + "blog": "Dreamer", + "count": 105, + "date": "dt:2020-03-10 21:17:00", + "gallery_id": "7645840", + "num": range(1, 3), + "title": "Angelika - Rustic Charm - AmourAngels 2016-09-27", + }, ) diff --git a/test/results/vanillarock.py b/test/results/vanillarock.py index f0cf3e1c3..b1cb67477 100644 --- a/test/results/vanillarock.py +++ b/test/results/vanillarock.py @@ -1,35 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import vanillarock - __tests__ = ( -{ - "#url" : "https://vanilla-rock.com/mizuhashi_parsee-5", - "#category": ("", "vanillarock", "post"), - "#class" : vanillarock.VanillarockPostExtractor, - "#sha1_url" : "7fb9a4d18d9fa22d7295fee8d94ab5a7a52265dd", - "#sha1_metadata": "b91df99b714e1958d9636748b1c81a07c3ef52c9", -}, - -{ - "#url" : "https://vanilla-rock.com/tag/%e5%b0%84%e5%91%bd%e4%b8%b8%e6%96%87", - "#category": ("", "vanillarock", "tag"), - "#class" : vanillarock.VanillarockTagExtractor, - "#pattern" : vanillarock.VanillarockPostExtractor.pattern, - "#count" : ">= 12", -}, - -{ - "#url" : "https://vanilla-rock.com/category/%e4%ba%8c%e6%ac%a1%e3%82%a8%e3%83%ad%e7%94%bb%e5%83%8f/%e8%90%8c%e3%81%88%e3%83%bb%e3%82%bd%e3%83%95%e3%83%88%e3%82%a8%e3%83%ad", - "#category": ("", "vanillarock", "tag"), - "#class" : vanillarock.VanillarockTagExtractor, - "#pattern" : vanillarock.VanillarockPostExtractor.pattern, - "#count" : ">= 5", -}, - + { + "#url": "https://vanilla-rock.com/mizuhashi_parsee-5", + "#category": ("", "vanillarock", "post"), + "#class": vanillarock.VanillarockPostExtractor, + "#sha1_url": "7fb9a4d18d9fa22d7295fee8d94ab5a7a52265dd", + "#sha1_metadata": "b91df99b714e1958d9636748b1c81a07c3ef52c9", + }, + { + "#url": "https://vanilla-rock.com/tag/%e5%b0%84%e5%91%bd%e4%b8%b8%e6%96%87", + "#category": ("", "vanillarock", "tag"), + "#class": vanillarock.VanillarockTagExtractor, + "#pattern": vanillarock.VanillarockPostExtractor.pattern, + "#count": ">= 12", + }, + { + "#url": "https://vanilla-rock.com/category/%e4%ba%8c%e6%ac%a1%e3%82%a8%e3%83%ad%e7%94%bb%e5%83%8f/%e8%90%8c%e3%81%88%e3%83%bb%e3%82%bd%e3%83%95%e3%83%88%e3%82%a8%e3%83%ad", + "#category": ("", "vanillarock", "tag"), + "#class": vanillarock.VanillarockTagExtractor, + "#pattern": vanillarock.VanillarockPostExtractor.pattern, + "#count": ">= 5", + }, ) diff --git a/test/results/vidyapics.py b/test/results/vidyapics.py index bb81760a9..af0c64ad9 100644 --- a/test/results/vidyapics.py +++ b/test/results/vidyapics.py @@ -1,38 +1,32 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shimmie2 - __tests__ = ( -{ - "#url" : "https://vidya.pics/post/list/kirby/1", - "#category": ("shimmie2", "vidyapics", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, - "#pattern" : r"https://vidya.pics/_images/[0-9a-f]{32}/\d+", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://vidya.pics/post/view/108820", - "#category": ("shimmie2", "vidyapics", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, - "#pattern" : r"https://vidya\.pics/_images/277ecdb90285bfa6e0c4cd46d9515b11/108820.+\.png", - "#sha1_content": "7d2fe9327759c231ff17f6e341df749b70b191ce", - - "extension": "png", - "file_url" : "https://vidya.pics/_images/277ecdb90285bfa6e0c4cd46d9515b11/108820%20-%201boy%20artist%3Aunknown%20flag%20kirby%20kirby_%28series%29.png", - "filename" : "108820 - 1boy artist:unknown flag kirby kirby_(series)", - "height" : 700, - "id" : 108820, - "md5" : "277ecdb90285bfa6e0c4cd46d9515b11", - "size" : 0, - "tags" : "1boy artist:unknown flag kirby kirby_(series", - "width" : 700, -}, - + { + "#url": "https://vidya.pics/post/list/kirby/1", + "#category": ("shimmie2", "vidyapics", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + "#pattern": r"https://vidya.pics/_images/[0-9a-f]{32}/\d+", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://vidya.pics/post/view/108820", + "#category": ("shimmie2", "vidyapics", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + "#pattern": r"https://vidya\.pics/_images/277ecdb90285bfa6e0c4cd46d9515b11/108820.+\.png", + "#sha1_content": "7d2fe9327759c231ff17f6e341df749b70b191ce", + "extension": "png", + "file_url": "https://vidya.pics/_images/277ecdb90285bfa6e0c4cd46d9515b11/108820%20-%201boy%20artist%3Aunknown%20flag%20kirby%20kirby_%28series%29.png", + "filename": "108820 - 1boy artist:unknown flag kirby kirby_(series)", + "height": 700, + "id": 108820, + "md5": "277ecdb90285bfa6e0c4cd46d9515b11", + "size": 0, + "tags": "1boy artist:unknown flag kirby kirby_(series", + "width": 700, + }, ) diff --git a/test/results/vidyart2.py b/test/results/vidyart2.py index 6c11dcbd5..4ae419edc 100644 --- a/test/results/vidyart2.py +++ b/test/results/vidyart2.py @@ -1,29 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v01 - __tests__ = ( -{ - "#url" : "https://vidyart2.booru.org/index.php?page=post&s=list&tags=all", - "#category": ("gelbooru_v01", "vidyart2", "tag"), - "#class" : gelbooru_v01.GelbooruV01TagExtractor, -}, - -{ - "#url" : "https://vidyart2.booru.org/index.php?page=favorites&s=view&id=1", - "#category": ("gelbooru_v01", "vidyart2", "favorite"), - "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, -}, - -{ - "#url" : "https://vidyart2.booru.org/index.php?page=post&s=view&id=39168", - "#category": ("gelbooru_v01", "vidyart2", "post"), - "#class" : gelbooru_v01.GelbooruV01PostExtractor, -}, - + { + "#url": "https://vidyart2.booru.org/index.php?page=post&s=list&tags=all", + "#category": ("gelbooru_v01", "vidyart2", "tag"), + "#class": gelbooru_v01.GelbooruV01TagExtractor, + }, + { + "#url": "https://vidyart2.booru.org/index.php?page=favorites&s=view&id=1", + "#category": ("gelbooru_v01", "vidyart2", "favorite"), + "#class": gelbooru_v01.GelbooruV01FavoriteExtractor, + }, + { + "#url": "https://vidyart2.booru.org/index.php?page=post&s=view&id=39168", + "#category": ("gelbooru_v01", "vidyart2", "post"), + "#class": gelbooru_v01.GelbooruV01PostExtractor, + }, ) diff --git a/test/results/vipergirls.py b/test/results/vipergirls.py index cd6adac0f..95a679f38 100644 --- a/test/results/vipergirls.py +++ b/test/results/vipergirls.py @@ -1,59 +1,49 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import vipergirls - __tests__ = ( -{ - "#url" : "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000", - "#category": ("", "vipergirls", "thread"), - "#class" : vipergirls.VipergirlsThreadExtractor, - "#count" : 225, - "#sha1_url": "0d75cb42777f5bebc0d284d1d38cb90c750c61d9", -}, - -{ - "#url" : "https://vipergirls.to/threads/6858916-Karina/page4", - "#category": ("", "vipergirls", "thread"), - "#class" : vipergirls.VipergirlsThreadExtractor, - "#count" : 1279, -}, - -{ - "#url" : "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000?highlight=foobar", - "#category": ("", "vipergirls", "thread"), - "#class" : vipergirls.VipergirlsThreadExtractor, -}, - -{ - "#url" : "https://vipergirls.to/threads/4328304?foo=bar", - "#category": ("", "vipergirls", "thread"), - "#class" : vipergirls.VipergirlsThreadExtractor, -}, - -{ - "#url" : "https://vipergirls.to/threads/4328304", - "#category": ("", "vipergirls", "thread"), - "#class" : vipergirls.VipergirlsThreadExtractor, -}, - -{ - "#url" : "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000?p=116038081&viewfull=1#post116038081", - "#category": ("", "vipergirls", "post"), - "#class" : vipergirls.VipergirlsPostExtractor, - "#pattern" : r"https://vipr\.im/\w{12}$", - "#range" : "2-113", - "#count" : 112, - - "id" : "116038081", - "imagecount": "113", - "number" : "116038081", - "thread_id" : "4328304", - "title" : "FemJoy Danica - Simply Beautiful (x112) 3000x4500", -}, - + { + "#url": "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000", + "#category": ("", "vipergirls", "thread"), + "#class": vipergirls.VipergirlsThreadExtractor, + "#count": 225, + "#sha1_url": "0d75cb42777f5bebc0d284d1d38cb90c750c61d9", + }, + { + "#url": "https://vipergirls.to/threads/6858916-Karina/page4", + "#category": ("", "vipergirls", "thread"), + "#class": vipergirls.VipergirlsThreadExtractor, + "#count": 1279, + }, + { + "#url": "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000?highlight=foobar", + "#category": ("", "vipergirls", "thread"), + "#class": vipergirls.VipergirlsThreadExtractor, + }, + { + "#url": "https://vipergirls.to/threads/4328304?foo=bar", + "#category": ("", "vipergirls", "thread"), + "#class": vipergirls.VipergirlsThreadExtractor, + }, + { + "#url": "https://vipergirls.to/threads/4328304", + "#category": ("", "vipergirls", "thread"), + "#class": vipergirls.VipergirlsThreadExtractor, + }, + { + "#url": "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000?p=116038081&viewfull=1#post116038081", + "#category": ("", "vipergirls", "post"), + "#class": vipergirls.VipergirlsPostExtractor, + "#pattern": r"https://vipr\.im/\w{12}$", + "#range": "2-113", + "#count": 112, + "id": "116038081", + "imagecount": "113", + "number": "116038081", + "thread_id": "4328304", + "title": "FemJoy Danica - Simply Beautiful (x112) 3000x4500", + }, ) diff --git a/test/results/vipr.py b/test/results/vipr.py index f5e00269f..535eb1e18 100644 --- a/test/results/vipr.py +++ b/test/results/vipr.py @@ -1,19 +1,15 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://vipr.im/kcd5jcuhgs3v.html", - "#category": ("imagehost", "vipr", "image"), - "#class" : imagehosts.ViprImageExtractor, - "#sha1_url" : "88f6a3ecbf3356a11ae0868b518c60800e070202", - "#sha1_metadata": "c432e8a1836b0d97045195b745731c2b1bb0e771", -}, - + { + "#url": "https://vipr.im/kcd5jcuhgs3v.html", + "#category": ("imagehost", "vipr", "image"), + "#class": imagehosts.ViprImageExtractor, + "#sha1_url": "88f6a3ecbf3356a11ae0868b518c60800e070202", + "#sha1_metadata": "c432e8a1836b0d97045195b745731c2b1bb0e771", + }, ) diff --git a/test/results/vk.py b/test/results/vk.py index b681bdf95..4cc7442a7 100644 --- a/test/results/vk.py +++ b/test/results/vk.py @@ -1,100 +1,85 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import vk from gallery_dl import exception - +from gallery_dl.extractor import vk __tests__ = ( -{ - "#url" : "https://vk.com/id398982326", - "#category": ("", "vk", "photos"), - "#class" : vk.VkPhotosExtractor, - "#pattern" : r"https://sun\d+-\d+\.userapi\.com/s/v1/if1/[\w-]+\.jpg\?size=\d+x\d+&quality=96&type=album", - "#count" : ">= 35", - - "id" : r"re:\d+", - "user": { - "id" : "398982326", - "info": "Мы за Движуху! – m1ni SounD #4 [EROmusic]", - "name": "", - "nick": "Dobrov Kurva", + { + "#url": "https://vk.com/id398982326", + "#category": ("", "vk", "photos"), + "#class": vk.VkPhotosExtractor, + "#pattern": r"https://sun\d+-\d+\.userapi\.com/s/v1/if1/[\w-]+\.jpg\?size=\d+x\d+&quality=96&type=album", + "#count": ">= 35", + "id": r"re:\d+", + "user": { + "id": "398982326", + "info": "Мы за Движуху! – m1ni SounD #4 [EROmusic]", + "name": "", + "nick": "Dobrov Kurva", + }, }, -}, - -{ - "#url" : "https://vk.com/cosplayinrussia", - "#category": ("", "vk", "photos"), - "#class" : vk.VkPhotosExtractor, - "#range" : "15-25", - - "id" : r"re:\d+", - "user": { - "id" : "-165740836", - "info": str, - "name": "cosplayinrussia", - "nick": "Косплей | Cosplay 18+", + { + "#url": "https://vk.com/cosplayinrussia", + "#category": ("", "vk", "photos"), + "#class": vk.VkPhotosExtractor, + "#range": "15-25", + "id": r"re:\d+", + "user": { + "id": "-165740836", + "info": str, + "name": "cosplayinrussia", + "nick": "Косплей | Cosplay 18+", + }, + }, + { + "#url": "https://vk.com/id76957806", + "#comment": "photos without width/height (#2535)", + "#category": ("", "vk", "photos"), + "#class": vk.VkPhotosExtractor, + "#pattern": r"https://sun\d+-\d+\.userapi\.com/", + "#range": "1-9", + "#count": 9, + }, + { + "#url": "https://m.vk.com/albums398982326", + "#category": ("", "vk", "photos"), + "#class": vk.VkPhotosExtractor, + }, + { + "#url": "https://www.vk.com/id398982326?profile=1", + "#category": ("", "vk", "photos"), + "#class": vk.VkPhotosExtractor, + }, + { + "#url": "https://vk.com/albums-165740836", + "#category": ("", "vk", "photos"), + "#class": vk.VkPhotosExtractor, + }, + { + "#url": "https://vk.com/album-165740836_281339889", + "#category": ("", "vk", "album"), + "#class": vk.VkAlbumExtractor, + "#count": 12, + }, + { + "#url": "https://vk.com/album-53775183_00", + "#comment": "'Access denied' (#2556)", + "#category": ("", "vk", "album"), + "#class": vk.VkAlbumExtractor, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://vk.com/album232175027_00", + "#category": ("", "vk", "album"), + "#class": vk.VkAlbumExtractor, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://vk.com/tag304303884", + "#category": ("", "vk", "tagged"), + "#class": vk.VkTaggedExtractor, + "#count": 44, }, -}, - -{ - "#url" : "https://vk.com/id76957806", - "#comment" : "photos without width/height (#2535)", - "#category": ("", "vk", "photos"), - "#class" : vk.VkPhotosExtractor, - "#pattern" : r"https://sun\d+-\d+\.userapi\.com/", - "#range" : "1-9", - "#count" : 9, -}, - -{ - "#url" : "https://m.vk.com/albums398982326", - "#category": ("", "vk", "photos"), - "#class" : vk.VkPhotosExtractor, -}, - -{ - "#url" : "https://www.vk.com/id398982326?profile=1", - "#category": ("", "vk", "photos"), - "#class" : vk.VkPhotosExtractor, -}, - -{ - "#url" : "https://vk.com/albums-165740836", - "#category": ("", "vk", "photos"), - "#class" : vk.VkPhotosExtractor, -}, - -{ - "#url" : "https://vk.com/album-165740836_281339889", - "#category": ("", "vk", "album"), - "#class" : vk.VkAlbumExtractor, - "#count" : 12, -}, - -{ - "#url" : "https://vk.com/album-53775183_00", - "#comment" : "'Access denied' (#2556)", - "#category": ("", "vk", "album"), - "#class" : vk.VkAlbumExtractor, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://vk.com/album232175027_00", - "#category": ("", "vk", "album"), - "#class" : vk.VkAlbumExtractor, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://vk.com/tag304303884", - "#category": ("", "vk", "tagged"), - "#class" : vk.VkTaggedExtractor, - "#count" : 44, -}, - ) diff --git a/test/results/vsco.py b/test/results/vsco.py index c0b32c42b..ef0dbe56a 100644 --- a/test/results/vsco.py +++ b/test/results/vsco.py @@ -1,111 +1,93 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import vsco - __tests__ = ( -{ - "#url" : "https://vsco.co/missuri", - "#category": ("", "vsco", "user"), - "#class" : vsco.VscoUserExtractor, - "#urls" : "https://vsco.co/missuri/gallery", -}, - -{ - "#url" : "https://vsco.co/missuri", - "#category": ("", "vsco", "user"), - "#class" : vsco.VscoUserExtractor, - "#options" : {"include": "all"}, - "#urls" : [ - "https://vsco.co/missuri/avatar", - "https://vsco.co/missuri/gallery", - "https://vsco.co/missuri/spaces", - "https://vsco.co/missuri/collection", - ], -}, - -{ - "#url" : "https://vsco.co/missuri/gallery", - "#category": ("", "vsco", "gallery"), - "#class" : vsco.VscoGalleryExtractor, - "#pattern" : r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w-]+\.\w+", - "#range" : "1-80", - "#count" : 80, -}, - -{ - "#url" : "https://vsco.co/missuri/images/1", - "#category": ("", "vsco", "gallery"), - "#class" : vsco.VscoGalleryExtractor, -}, - -{ - "#url" : "https://vsco.co/vsco/collection/1", - "#category": ("", "vsco", "collection"), - "#class" : vsco.VscoCollectionExtractor, - "#pattern" : r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w\s-]+\.\w+", - "#range" : "1-80", - "#count" : 80, -}, - -{ - "#url" : "https://vsco.co/spaces/6320a3e1e0338d1350b33fea", - "#category": ("", "vsco", "space"), - "#class" : vsco.VscoSpaceExtractor, - "#pattern" : r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w\s-]+\.\w+", - "#count" : range(100, 150), -}, - -{ - "#url" : "https://vsco.co/missuri/spaces", - "#category": ("", "vsco", "spaces"), - "#class" : vsco.VscoSpacesExtractor, - "#urls" : ( - "https://vsco.co/spaces/62e4934e6920440801d19f05", - ), -}, - -{ - "#url" : "https://vsco.co/vsco/avatar", - "#category": ("", "vsco", "avatar"), - "#class" : vsco.VscoAvatarExtractor, - "#urls" : "https://image-aws-us-west-2.vsco.co/3c69ae/304128/652d9f3b39a6007526dda683/vscoprofile-avatar.jpg", - "#sha1_content" : "57cd648759e34a6daefc5c79542ddb4595b9b677", - - "id": "652d9f3b39a6007526dda683", -}, - -{ - "#url" : "https://vsco.co/erenyildiz/media/5d34b93ef632433030707ce2", - "#category": ("", "vsco", "image"), - "#class" : vsco.VscoImageExtractor, - "#sha1_url" : "a45f9712325b42742324b330c348b72477996031", - "#sha1_content": "1394d070828d82078035f19a92f404557b56b83f", - - "id" : "5d34b93ef632433030707ce2", - "user" : "erenyildiz", - "grid" : "erenyildiz", - "meta" : dict, - "tags" : list, - "date" : "dt:2019-07-21 19:12:11", - "video" : False, - "width" : 1537, - "height" : 1537, - "description": r"re:Ni seviyorum. #vsco #vscox #vscochallenges", -}, - -{ - "#url" : "https://vsco.co/jimenalazof/media/5b4feec558f6c45c18c040fd", - "#category": ("", "vsco", "image"), - "#class" : vsco.VscoImageExtractor, - "#sha1_url" : "08e7eef3301756ce81206c0b47c1e9373756a74a", - "#sha1_content": "e739f058d726ee42c51c180a505747972a7dfa47", - - "video": True, -}, - + { + "#url": "https://vsco.co/missuri", + "#category": ("", "vsco", "user"), + "#class": vsco.VscoUserExtractor, + "#urls": "https://vsco.co/missuri/gallery", + }, + { + "#url": "https://vsco.co/missuri", + "#category": ("", "vsco", "user"), + "#class": vsco.VscoUserExtractor, + "#options": {"include": "all"}, + "#urls": [ + "https://vsco.co/missuri/avatar", + "https://vsco.co/missuri/gallery", + "https://vsco.co/missuri/spaces", + "https://vsco.co/missuri/collection", + ], + }, + { + "#url": "https://vsco.co/missuri/gallery", + "#category": ("", "vsco", "gallery"), + "#class": vsco.VscoGalleryExtractor, + "#pattern": r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w-]+\.\w+", + "#range": "1-80", + "#count": 80, + }, + { + "#url": "https://vsco.co/missuri/images/1", + "#category": ("", "vsco", "gallery"), + "#class": vsco.VscoGalleryExtractor, + }, + { + "#url": "https://vsco.co/vsco/collection/1", + "#category": ("", "vsco", "collection"), + "#class": vsco.VscoCollectionExtractor, + "#pattern": r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w\s-]+\.\w+", + "#range": "1-80", + "#count": 80, + }, + { + "#url": "https://vsco.co/spaces/6320a3e1e0338d1350b33fea", + "#category": ("", "vsco", "space"), + "#class": vsco.VscoSpaceExtractor, + "#pattern": r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w\s-]+\.\w+", + "#count": range(100, 150), + }, + { + "#url": "https://vsco.co/missuri/spaces", + "#category": ("", "vsco", "spaces"), + "#class": vsco.VscoSpacesExtractor, + "#urls": ("https://vsco.co/spaces/62e4934e6920440801d19f05",), + }, + { + "#url": "https://vsco.co/vsco/avatar", + "#category": ("", "vsco", "avatar"), + "#class": vsco.VscoAvatarExtractor, + "#urls": "https://image-aws-us-west-2.vsco.co/3c69ae/304128/652d9f3b39a6007526dda683/vscoprofile-avatar.jpg", + "#sha1_content": "57cd648759e34a6daefc5c79542ddb4595b9b677", + "id": "652d9f3b39a6007526dda683", + }, + { + "#url": "https://vsco.co/erenyildiz/media/5d34b93ef632433030707ce2", + "#category": ("", "vsco", "image"), + "#class": vsco.VscoImageExtractor, + "#sha1_url": "a45f9712325b42742324b330c348b72477996031", + "#sha1_content": "1394d070828d82078035f19a92f404557b56b83f", + "id": "5d34b93ef632433030707ce2", + "user": "erenyildiz", + "grid": "erenyildiz", + "meta": dict, + "tags": list, + "date": "dt:2019-07-21 19:12:11", + "video": False, + "width": 1537, + "height": 1537, + "description": r"re:Ni seviyorum. #vsco #vscox #vscochallenges", + }, + { + "#url": "https://vsco.co/jimenalazof/media/5b4feec558f6c45c18c040fd", + "#category": ("", "vsco", "image"), + "#class": vsco.VscoImageExtractor, + "#sha1_url": "08e7eef3301756ce81206c0b47c1e9373756a74a", + "#sha1_content": "e739f058d726ee42c51c180a505747972a7dfa47", + "video": True, + }, ) diff --git a/test/results/wallhaven.py b/test/results/wallhaven.py index 47a8ba777..deb87fe27 100644 --- a/test/results/wallhaven.py +++ b/test/results/wallhaven.py @@ -1,104 +1,90 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wallhaven - __tests__ = ( -{ - "#url" : "https://wallhaven.cc/search?q=touhou", - "#category": ("", "wallhaven", "search"), - "#class" : wallhaven.WallhavenSearchExtractor, -}, - -{ - "#url" : "https://wallhaven.cc/search?q=id%3A87&categories=111&purity=100&sorting=date_added&order=asc&page=3", - "#category": ("", "wallhaven", "search"), - "#class" : wallhaven.WallhavenSearchExtractor, - "#pattern" : r"https://w\.wallhaven\.cc/full/\w\w/wallhaven-\w+\.\w+", - "#count" : "<= 30", -}, - -{ - "#url" : "https://wallhaven.cc/user/AksumkA/favorites/74", - "#category": ("", "wallhaven", "collection"), - "#class" : wallhaven.WallhavenCollectionExtractor, - "#count" : ">= 50", -}, - -{ - "#url" : "https://wallhaven.cc/user/AksumkA/", - "#category": ("", "wallhaven", "user"), - "#class" : wallhaven.WallhavenUserExtractor, -}, - -{ - "#url" : "https://wallhaven.cc/user/AksumkA/favorites", - "#category": ("", "wallhaven", "collections"), - "#class" : wallhaven.WallhavenCollectionsExtractor, - "#pattern" : wallhaven.WallhavenCollectionExtractor.pattern, - "#count" : 4, -}, - -{ - "#url" : "https://wallhaven.cc/user/AksumkA/uploads", - "#category": ("", "wallhaven", "uploads"), - "#class" : wallhaven.WallhavenUploadsExtractor, - "#pattern" : r"https://[^.]+\.wallhaven\.cc/full/\w\w/wallhaven-\w+\.\w+", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://wallhaven.cc/w/01w334", - "#category": ("", "wallhaven", "image"), - "#class" : wallhaven.WallhavenImageExtractor, - "#pattern" : r"https://[^.]+\.wallhaven\.cc/full/01/wallhaven-01w334\.jpg", - "#sha1_content": "497212679383a465da1e35bd75873240435085a2", - - "id" : "01w334", - "width" : 1920, - "height" : 1200, - "resolution" : "1920x1200", - "ratio" : "1.6", - "colors" : list, - "tags" : list, - "file_size" : 278799, - "file_type" : "image/jpeg", - "purity" : "sfw", - "short_url" : "https://whvn.cc/01w334", - "source" : str, - "uploader" : { - "group" : "Owner/Developer", - "username": "AksumkA", + { + "#url": "https://wallhaven.cc/search?q=touhou", + "#category": ("", "wallhaven", "search"), + "#class": wallhaven.WallhavenSearchExtractor, + }, + { + "#url": "https://wallhaven.cc/search?q=id%3A87&categories=111&purity=100&sorting=date_added&order=asc&page=3", + "#category": ("", "wallhaven", "search"), + "#class": wallhaven.WallhavenSearchExtractor, + "#pattern": r"https://w\.wallhaven\.cc/full/\w\w/wallhaven-\w+\.\w+", + "#count": "<= 30", + }, + { + "#url": "https://wallhaven.cc/user/AksumkA/favorites/74", + "#category": ("", "wallhaven", "collection"), + "#class": wallhaven.WallhavenCollectionExtractor, + "#count": ">= 50", + }, + { + "#url": "https://wallhaven.cc/user/AksumkA/", + "#category": ("", "wallhaven", "user"), + "#class": wallhaven.WallhavenUserExtractor, + }, + { + "#url": "https://wallhaven.cc/user/AksumkA/favorites", + "#category": ("", "wallhaven", "collections"), + "#class": wallhaven.WallhavenCollectionsExtractor, + "#pattern": wallhaven.WallhavenCollectionExtractor.pattern, + "#count": 4, + }, + { + "#url": "https://wallhaven.cc/user/AksumkA/uploads", + "#category": ("", "wallhaven", "uploads"), + "#class": wallhaven.WallhavenUploadsExtractor, + "#pattern": r"https://[^.]+\.wallhaven\.cc/full/\w\w/wallhaven-\w+\.\w+", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://wallhaven.cc/w/01w334", + "#category": ("", "wallhaven", "image"), + "#class": wallhaven.WallhavenImageExtractor, + "#pattern": r"https://[^.]+\.wallhaven\.cc/full/01/wallhaven-01w334\.jpg", + "#sha1_content": "497212679383a465da1e35bd75873240435085a2", + "id": "01w334", + "width": 1920, + "height": 1200, + "resolution": "1920x1200", + "ratio": "1.6", + "colors": list, + "tags": list, + "file_size": 278799, + "file_type": "image/jpeg", + "purity": "sfw", + "short_url": "https://whvn.cc/01w334", + "source": str, + "uploader": { + "group": "Owner/Developer", + "username": "AksumkA", + }, + "date": "dt:2014-08-31 06:17:19", + "wh_category": "anime", + "views": int, + "favorites": int, + }, + { + "#url": "https://wallhaven.cc/w/dge6v3", + "#comment": "NSFW", + "#category": ("", "wallhaven", "image"), + "#class": wallhaven.WallhavenImageExtractor, + "#sha1_url": "e4b802e70483f659d790ad5d0bd316245badf2ec", + }, + { + "#url": "https://whvn.cc/01w334", + "#category": ("", "wallhaven", "image"), + "#class": wallhaven.WallhavenImageExtractor, + }, + { + "#url": "https://w.wallhaven.cc/full/01/wallhaven-01w334.jpg", + "#category": ("", "wallhaven", "image"), + "#class": wallhaven.WallhavenImageExtractor, }, - "date" : "dt:2014-08-31 06:17:19", - "wh_category": "anime", - "views" : int, - "favorites" : int, -}, - -{ - "#url" : "https://wallhaven.cc/w/dge6v3", - "#comment" : "NSFW", - "#category": ("", "wallhaven", "image"), - "#class" : wallhaven.WallhavenImageExtractor, - "#sha1_url": "e4b802e70483f659d790ad5d0bd316245badf2ec", -}, - -{ - "#url" : "https://whvn.cc/01w334", - "#category": ("", "wallhaven", "image"), - "#class" : wallhaven.WallhavenImageExtractor, -}, - -{ - "#url" : "https://w.wallhaven.cc/full/01/wallhaven-01w334.jpg", - "#category": ("", "wallhaven", "image"), - "#class" : wallhaven.WallhavenImageExtractor, -}, - ) diff --git a/test/results/wallpapercave.py b/test/results/wallpapercave.py index 80ade86d5..e14c232bc 100644 --- a/test/results/wallpapercave.py +++ b/test/results/wallpapercave.py @@ -1,33 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wallpapercave - __tests__ = ( -{ - "#url" : "https://wallpapercave.com/w/wp10270355", - "#category": ("", "wallpapercave", "image"), - "#class" : wallpapercave.WallpapercaveImageExtractor, - "#urls" : "https://wallpapercave.com/download/sekai-saikou-no-ansatsusha-isekai-kizoku-ni-tensei-suru-wallpapers-wp10270355", - "#sha1_content": "58b088aaa1cf1a60e347015019eb0c5a22b263a6", -}, - -{ - "#url" : "https://wallpapercave.com/apple-wwdc-2024-wallpapers", - "#comment" : "album listing", - "#category": ("", "wallpapercave", "image"), - "#class" : wallpapercave.WallpapercaveImageExtractor, - "#archive" : False, - "#urls" : [ - "https://wallpapercave.com/wp/wp13775438.jpg", - "https://wallpapercave.com/wp/wp13775439.jpg", - "https://wallpapercave.com/wp/wp13775440.jpg", - "https://wallpapercave.com/wp/wp13775441.jpg", - ], -}, - + { + "#url": "https://wallpapercave.com/w/wp10270355", + "#category": ("", "wallpapercave", "image"), + "#class": wallpapercave.WallpapercaveImageExtractor, + "#urls": "https://wallpapercave.com/download/sekai-saikou-no-ansatsusha-isekai-kizoku-ni-tensei-suru-wallpapers-wp10270355", + "#sha1_content": "58b088aaa1cf1a60e347015019eb0c5a22b263a6", + }, + { + "#url": "https://wallpapercave.com/apple-wwdc-2024-wallpapers", + "#comment": "album listing", + "#category": ("", "wallpapercave", "image"), + "#class": wallpapercave.WallpapercaveImageExtractor, + "#archive": False, + "#urls": [ + "https://wallpapercave.com/wp/wp13775438.jpg", + "https://wallpapercave.com/wp/wp13775439.jpg", + "https://wallpapercave.com/wp/wp13775440.jpg", + "https://wallpapercave.com/wp/wp13775441.jpg", + ], + }, ) diff --git a/test/results/warosu.py b/test/results/warosu.py index fd095183d..72ff6b0bf 100644 --- a/test/results/warosu.py +++ b/test/results/warosu.py @@ -1,94 +1,84 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import warosu - __tests__ = ( -{ - "#url" : "https://warosu.org/jp/thread/16656025", - "#category": ("", "warosu", "thread"), - "#class" : warosu.WarosuThreadExtractor, - "#urls" : ( - "https://i.warosu.org/data/jp/img/0166/56/1488487280004.png", - "https://i.warosu.org/data/jp/img/0166/56/1488493239417.png", - "https://i.warosu.org/data/jp/img/0166/56/1488493636725.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1488493700040.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1488499585168.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1488530851199.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1488536072155.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1488603426484.png", - "https://i.warosu.org/data/jp/img/0166/56/1488647021253.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1488866825031.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1489094956868.jpg", - ), -}, - -{ - "#url" : "https://warosu.org/jp/thread/16658073", - "#category": ("", "warosu", "thread"), - "#class" : warosu.WarosuThreadExtractor, - "#sha1_content" : "d48df0a701e6599312bfff8674f4aa5d4fb8db1c", - "#urls" : "https://i.warosu.org/data/jp/img/0166/58/1488521824388.jpg", - "#count" : 1, - - "board" : "jp", - "board_name": "Otaku Culture", - "com" : "Is this canon?", - "ext" : ".jpg", - "extension" : "jpg", - "filename" : "sadako-vs-kayako-movie-review", - "fsize" : "55 KB", - "h" : 675, - "image" : "https://i.warosu.org/data/jp/img/0166/58/1488521824388.jpg", - "name" : "Anonymous", - "no" : 16658073, - "now" : "Fri Mar 3 01:17:04 2017", - "thread" : "16658073", - "tim" : 1488521824388, - "time" : 1488503824, - "title" : "Is this canon?", - "w" : 450, -}, - -{ - "#url" : "https://warosu.org/jp/thread/45886210", - "#comment" : "deleted post (#5289)", - "#category": ("", "warosu", "thread"), - "#class" : warosu.WarosuThreadExtractor, - "#count" : "> 150", - - "board" : "jp", - "board_name": "Otaku Culture", - "title" : "/07/th Expansion Thread", -}, - -{ - "#url" : "https://warosu.org/ic/thread/4604652", - "#category": ("", "warosu", "thread"), - "#class" : warosu.WarosuThreadExtractor, - "#pattern" : r"https://i.warosu\.org/data/ic/img/0046/04/1590\d{9}\.jpg", - "#count" : 133, - - "board" : "ic", - "board_name": "Artwork/Critique", - "com" : str, - "ext" : ".jpg", - "filename" : str, - "fsize" : str, - "h" : range(200, 3507), - "image" : r"re:https://i.warosu\.org/data/ic/img/0046/04/1590\d+\.jpg", - "name" : "re:Anonymous|Dhe Specky Spider-Man", - "no" : range(4604652, 4620000), - "now" : r"re:\w\w\w \w\w\w \d\d \d\d:\d\d:\d\d 2020", - "thread" : "4604652", - "tim" : range(1590430159651, 1590755510488), - "time" : range(1590415759, 1590755510), - "title" : "American Classic Comic Artists", - "w" : range(200, 3000), -}, - + { + "#url": "https://warosu.org/jp/thread/16656025", + "#category": ("", "warosu", "thread"), + "#class": warosu.WarosuThreadExtractor, + "#urls": ( + "https://i.warosu.org/data/jp/img/0166/56/1488487280004.png", + "https://i.warosu.org/data/jp/img/0166/56/1488493239417.png", + "https://i.warosu.org/data/jp/img/0166/56/1488493636725.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1488493700040.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1488499585168.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1488530851199.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1488536072155.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1488603426484.png", + "https://i.warosu.org/data/jp/img/0166/56/1488647021253.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1488866825031.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1489094956868.jpg", + ), + }, + { + "#url": "https://warosu.org/jp/thread/16658073", + "#category": ("", "warosu", "thread"), + "#class": warosu.WarosuThreadExtractor, + "#sha1_content": "d48df0a701e6599312bfff8674f4aa5d4fb8db1c", + "#urls": "https://i.warosu.org/data/jp/img/0166/58/1488521824388.jpg", + "#count": 1, + "board": "jp", + "board_name": "Otaku Culture", + "com": "Is this canon?", + "ext": ".jpg", + "extension": "jpg", + "filename": "sadako-vs-kayako-movie-review", + "fsize": "55 KB", + "h": 675, + "image": "https://i.warosu.org/data/jp/img/0166/58/1488521824388.jpg", + "name": "Anonymous", + "no": 16658073, + "now": "Fri Mar 3 01:17:04 2017", + "thread": "16658073", + "tim": 1488521824388, + "time": 1488503824, + "title": "Is this canon?", + "w": 450, + }, + { + "#url": "https://warosu.org/jp/thread/45886210", + "#comment": "deleted post (#5289)", + "#category": ("", "warosu", "thread"), + "#class": warosu.WarosuThreadExtractor, + "#count": "> 150", + "board": "jp", + "board_name": "Otaku Culture", + "title": "/07/th Expansion Thread", + }, + { + "#url": "https://warosu.org/ic/thread/4604652", + "#category": ("", "warosu", "thread"), + "#class": warosu.WarosuThreadExtractor, + "#pattern": r"https://i.warosu\.org/data/ic/img/0046/04/1590\d{9}\.jpg", + "#count": 133, + "board": "ic", + "board_name": "Artwork/Critique", + "com": str, + "ext": ".jpg", + "filename": str, + "fsize": str, + "h": range(200, 3507), + "image": r"re:https://i.warosu\.org/data/ic/img/0046/04/1590\d+\.jpg", + "name": "re:Anonymous|Dhe Specky Spider-Man", + "no": range(4604652, 4620000), + "now": r"re:\w\w\w \w\w\w \d\d \d\d:\d\d:\d\d 2020", + "thread": "4604652", + "tim": range(1590430159651, 1590755510488), + "time": range(1590415759, 1590755510), + "title": "American Classic Comic Artists", + "w": range(200, 3000), + }, ) diff --git a/test/results/weasyl.py b/test/results/weasyl.py index 0c652644a..dc8a932b0 100644 --- a/test/results/weasyl.py +++ b/test/results/weasyl.py @@ -1,102 +1,87 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import weasyl - __tests__ = ( -{ - "#url" : "https://www.weasyl.com/~fiz/submissions/2031/a-wesley", - "#category": ("", "weasyl", "submission"), - "#class" : weasyl.WeasylSubmissionExtractor, - "#pattern" : "https://cdn.weasyl.com/~fiz/submissions/2031/41ebc1c2940be928532785dfbf35c37622664d2fbb8114c3b063df969562fc51/fiz-a-wesley.png", - - "comments" : int, - "date" : "dt:2012-04-20 00:38:04", - "description" : """

          (flex)

          + { + "#url": "https://www.weasyl.com/~fiz/submissions/2031/a-wesley", + "#category": ("", "weasyl", "submission"), + "#class": weasyl.WeasylSubmissionExtractor, + "#pattern": "https://cdn.weasyl.com/~fiz/submissions/2031/41ebc1c2940be928532785dfbf35c37622664d2fbb8114c3b063df969562fc51/fiz-a-wesley.png", + "comments": int, + "date": "dt:2012-04-20 00:38:04", + "description": """

          (flex)

          """, - "favorites" : int, - "folder_name" : "Wesley Stuff", - "folderid" : 2081, - "friends_only": False, - "owner" : "Fiz", - "owner_login" : "fiz", - "rating" : "general", - "submitid" : 2031, - "subtype" : "visual", - "tags" : list, - "title" : "A Wesley!", - "type" : "submission", - "views" : int, -}, - -{ - "#url" : "https://www.weasyl.com/submission/2031/a-wesley", - "#category": ("", "weasyl", "submission"), - "#class" : weasyl.WeasylSubmissionExtractor, -}, - -{ - "#url" : "https://www.weasyl.com/~tanidareal", - "#category": ("", "weasyl", "submissions"), - "#class" : weasyl.WeasylSubmissionsExtractor, - "#count" : ">= 200", -}, - -{ - "#url" : "https://www.weasyl.com/submissions/tanidareal", - "#category": ("", "weasyl", "submissions"), - "#class" : weasyl.WeasylSubmissionsExtractor, -}, - -{ - "#url" : "https://www.weasyl.com/~aro~so", - "#category": ("", "weasyl", "submissions"), - "#class" : weasyl.WeasylSubmissionsExtractor, -}, - -{ - "#url" : "https://www.weasyl.com/submissions/tanidareal?folderid=7403", - "#category": ("", "weasyl", "folder"), - "#class" : weasyl.WeasylFolderExtractor, - "#count" : ">= 12", -}, - -{ - "#url" : "https://www.weasyl.com/journal/17647/bbcode", - "#category": ("", "weasyl", "journal"), - "#class" : weasyl.WeasylJournalExtractor, - - "title" : "BBCode", - "date" : "dt:2013-09-19 23:11:23", - "content": """

          javascript:alert(42);

          + "favorites": int, + "folder_name": "Wesley Stuff", + "folderid": 2081, + "friends_only": False, + "owner": "Fiz", + "owner_login": "fiz", + "rating": "general", + "submitid": 2031, + "subtype": "visual", + "tags": list, + "title": "A Wesley!", + "type": "submission", + "views": int, + }, + { + "#url": "https://www.weasyl.com/submission/2031/a-wesley", + "#category": ("", "weasyl", "submission"), + "#class": weasyl.WeasylSubmissionExtractor, + }, + { + "#url": "https://www.weasyl.com/~tanidareal", + "#category": ("", "weasyl", "submissions"), + "#class": weasyl.WeasylSubmissionsExtractor, + "#count": ">= 200", + }, + { + "#url": "https://www.weasyl.com/submissions/tanidareal", + "#category": ("", "weasyl", "submissions"), + "#class": weasyl.WeasylSubmissionsExtractor, + }, + { + "#url": "https://www.weasyl.com/~aro~so", + "#category": ("", "weasyl", "submissions"), + "#class": weasyl.WeasylSubmissionsExtractor, + }, + { + "#url": "https://www.weasyl.com/submissions/tanidareal?folderid=7403", + "#category": ("", "weasyl", "folder"), + "#class": weasyl.WeasylFolderExtractor, + "#count": ">= 12", + }, + { + "#url": "https://www.weasyl.com/journal/17647/bbcode", + "#category": ("", "weasyl", "journal"), + "#class": weasyl.WeasylJournalExtractor, + "title": "BBCode", + "date": "dt:2013-09-19 23:11:23", + "content": """

          javascript:alert(42);

          No more of that!

          """, -}, - -{ - "#url" : "https://www.weasyl.com/journals/charmander", - "#category": ("", "weasyl", "journals"), - "#class" : weasyl.WeasylJournalsExtractor, - "#count" : ">= 2", -}, - -{ - "#url" : "https://www.weasyl.com/favorites?userid=184616&feature=submit", - "#category": ("", "weasyl", "favorite"), - "#class" : weasyl.WeasylFavoriteExtractor, - "#count" : ">= 5", -}, - -{ - "#url" : "https://www.weasyl.com/favorites/furoferre", - "#category": ("", "weasyl", "favorite"), - "#class" : weasyl.WeasylFavoriteExtractor, - "#count" : ">= 5", -} - + }, + { + "#url": "https://www.weasyl.com/journals/charmander", + "#category": ("", "weasyl", "journals"), + "#class": weasyl.WeasylJournalsExtractor, + "#count": ">= 2", + }, + { + "#url": "https://www.weasyl.com/favorites?userid=184616&feature=submit", + "#category": ("", "weasyl", "favorite"), + "#class": weasyl.WeasylFavoriteExtractor, + "#count": ">= 5", + }, + { + "#url": "https://www.weasyl.com/favorites/furoferre", + "#category": ("", "weasyl", "favorite"), + "#class": weasyl.WeasylFavoriteExtractor, + "#count": ">= 5", + }, ) diff --git a/test/results/webmshare.py b/test/results/webmshare.py index 007a12b4b..7710e5e67 100644 --- a/test/results/webmshare.py +++ b/test/results/webmshare.py @@ -1,55 +1,46 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import webmshare - __tests__ = ( -{ - "#url" : "https://webmshare.com/O9mWY", - "#category": ("", "webmshare", "video"), - "#class" : webmshare.WebmshareVideoExtractor, - - "date" : "dt:2022-12-04 00:00:00", - "extension": "webm", - "filename" : "O9mWY", - "height" : 568, - "id" : "O9mWY", - "thumb" : "https://s1.webmshare.com/t/O9mWY.jpg", - "title" : "Yeah buddy over here", - "url" : "https://s1.webmshare.com/O9mWY.webm", - "views" : int, - "width" : 320, -}, - -{ - "#url" : "https://s1.webmshare.com/zBGAg.webm", - "#category": ("", "webmshare", "video"), - "#class" : webmshare.WebmshareVideoExtractor, - - "date" : "dt:2018-12-07 00:00:00", - "height": 1080, - "id" : "zBGAg", - "thumb" : "https://s1.webmshare.com/t/zBGAg.jpg", - "title" : "", - "url" : "https://s1.webmshare.com/zBGAg.webm", - "views" : int, - "width" : 1920, -}, - -{ - "#url" : "https://webmshare.com/play/zBGAg", - "#category": ("", "webmshare", "video"), - "#class" : webmshare.WebmshareVideoExtractor, -}, - -{ - "#url" : "https://webmshare.com/download-webm/zBGAg", - "#category": ("", "webmshare", "video"), - "#class" : webmshare.WebmshareVideoExtractor, -}, - + { + "#url": "https://webmshare.com/O9mWY", + "#category": ("", "webmshare", "video"), + "#class": webmshare.WebmshareVideoExtractor, + "date": "dt:2022-12-04 00:00:00", + "extension": "webm", + "filename": "O9mWY", + "height": 568, + "id": "O9mWY", + "thumb": "https://s1.webmshare.com/t/O9mWY.jpg", + "title": "Yeah buddy over here", + "url": "https://s1.webmshare.com/O9mWY.webm", + "views": int, + "width": 320, + }, + { + "#url": "https://s1.webmshare.com/zBGAg.webm", + "#category": ("", "webmshare", "video"), + "#class": webmshare.WebmshareVideoExtractor, + "date": "dt:2018-12-07 00:00:00", + "height": 1080, + "id": "zBGAg", + "thumb": "https://s1.webmshare.com/t/zBGAg.jpg", + "title": "", + "url": "https://s1.webmshare.com/zBGAg.webm", + "views": int, + "width": 1920, + }, + { + "#url": "https://webmshare.com/play/zBGAg", + "#category": ("", "webmshare", "video"), + "#class": webmshare.WebmshareVideoExtractor, + }, + { + "#url": "https://webmshare.com/download-webm/zBGAg", + "#category": ("", "webmshare", "video"), + "#class": webmshare.WebmshareVideoExtractor, + }, ) diff --git a/test/results/webtoons.py b/test/results/webtoons.py index 4574fd482..769dcf9c0 100644 --- a/test/results/webtoons.py +++ b/test/results/webtoons.py @@ -1,140 +1,120 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import webtoons from gallery_dl import exception - +from gallery_dl.extractor import webtoons __tests__ = ( -{ - "#url" : "https://www.webtoons.com/en/comedy/safely-endangered/ep-572-earth/viewer?title_no=352&episode_no=572", - "#category": ("", "webtoons", "episode"), - "#class" : webtoons.WebtoonsEpisodeExtractor, - "#count" : 5, - "#sha1_url" : "55bec5d7c42aba19e3d0d56db25fdf0b0b13be38", - "#sha1_content": [ - "1748c7e82b6db910fa179f6dc7c4281b0f680fa7", - "42055e44659f6ffc410b3fb6557346dfbb993df3", - "49e1f2def04c6f7a6a3dacf245a1cd9abe77a6a9", - ], - - "author_name" : "Chris McCoy", - "comic" : "safely-endangered", - "comic_name" : "Safely Endangered", - "count" : 5, - "description" : "Silly comics for silly people.", - "episode" : "572", - "episode_name": "Ep. 572 - Earth", - "episode_no" : "572", - "genre" : "comedy", - "lang" : "en", - "language" : "English", - "num" : range(1, 5), - "title" : "Safely Endangered - Ep. 572 - Earth", - "title_no" : "352", - "username" : "safelyendangered", -}, - -{ - "#url" : "https://www.webtoons.com/en/challenge/punderworld/happy-earth-day-/viewer?title_no=312584&episode_no=40", - "#category": ("", "webtoons", "episode"), - "#class" : webtoons.WebtoonsEpisodeExtractor, - "#exception": exception.NotFoundError, - - "comic" : "punderworld", - "description": str, - "episode" : "36", - "episode_no" : "40", - "genre" : "challenge", - "title" : r"re:^Punderworld - .+", - "title_no" : "312584", -}, - -{ - "#url" : "https://www.webtoons.com/en/canvas/i-want-to-be-a-cute-anime-girl/209-the-storys-story/viewer?title_no=349416&episode_no=214", - "#category": ("", "webtoons", "episode"), - "#class" : webtoons.WebtoonsEpisodeExtractor, - "#count" : 4, - - "comic_name" : "I want to be a cute anime girl", - "episode_name": "209 - The story's story", - "episode" : "214", - "username" : "m9huj", - "author_name" : "Azul Crescent", -}, - -{ - "#url" : "https://www.webtoons.com/en/canvas/i-want-to-be-a-cute-anime-girl/174-not-194-it-was-a-typo-later/viewer?title_no=349416&episode_no=179", - "#category": ("", "webtoons", "episode"), - "#class" : webtoons.WebtoonsEpisodeExtractor, - "#count" : 4, - - "comic_name" : "I want to be a cute anime girl", - "episode_name": "174 (not 194, it was a typo) - Later", - "episode" : "179", - "username" : "m9huj", - "author_name" : "Azul Crescent", -}, - -{ - "#url" : "https://www.webtoons.com/en/canvas/us-over-here/1-the-wheel/viewer?title_no=919536&episode_no=1", - "#category": ("", "webtoons", "episode"), - "#class" : webtoons.WebtoonsEpisodeExtractor, - "#count" : 60, - - "comic_name" : "Us, over here", - "episode_name": "1. The Wheel", - "episode" : "1", - "username" : "i94q8", - "author_name" : "spin.ani", -}, - -{ - "#url" : "https://www.webtoons.com/en/comedy/live-with-yourself/list?title_no=919", - "#comment" : "english", - "#category": ("", "webtoons", "comic"), - "#class" : webtoons.WebtoonsComicExtractor, - "#pattern" : webtoons.WebtoonsEpisodeExtractor.pattern, - "#range" : "1-15", - "#count" : ">= 14", - - "page" : range(1, 2), - "title_no" : 919, - "episode_no": range(1, 14), -}, - -{ - "#url" : "https://www.webtoons.com/fr/romance/subzero/list?title_no=1845&page=7", - "#comment" : "french", - "#category": ("", "webtoons", "comic"), - "#class" : webtoons.WebtoonsComicExtractor, - "#count" : ">= 15", - - "page" : range(7, 25), - "title_no" : 1845, - "episode_no": int, -}, - -{ - "#url" : "https://www.webtoons.com/en/challenge/scoob-and-shag/list?title_no=210827&page=9", - "#comment" : "(#820)", - "#category": ("", "webtoons", "comic"), - "#class" : webtoons.WebtoonsComicExtractor, - "#count" : ">= 18", - - "page" : int, - "title_no" : 210827, - "episode_no": int, -}, - -{ - "#url" : "https://www.webtoons.com/es/romance/lore-olympus/list?title_no=1725", - "#comment" : "(#1643)", - "#category": ("", "webtoons", "comic"), - "#class" : webtoons.WebtoonsComicExtractor, -}, - + { + "#url": "https://www.webtoons.com/en/comedy/safely-endangered/ep-572-earth/viewer?title_no=352&episode_no=572", + "#category": ("", "webtoons", "episode"), + "#class": webtoons.WebtoonsEpisodeExtractor, + "#count": 5, + "#sha1_url": "55bec5d7c42aba19e3d0d56db25fdf0b0b13be38", + "#sha1_content": [ + "1748c7e82b6db910fa179f6dc7c4281b0f680fa7", + "42055e44659f6ffc410b3fb6557346dfbb993df3", + "49e1f2def04c6f7a6a3dacf245a1cd9abe77a6a9", + ], + "author_name": "Chris McCoy", + "comic": "safely-endangered", + "comic_name": "Safely Endangered", + "count": 5, + "description": "Silly comics for silly people.", + "episode": "572", + "episode_name": "Ep. 572 - Earth", + "episode_no": "572", + "genre": "comedy", + "lang": "en", + "language": "English", + "num": range(1, 5), + "title": "Safely Endangered - Ep. 572 - Earth", + "title_no": "352", + "username": "safelyendangered", + }, + { + "#url": "https://www.webtoons.com/en/challenge/punderworld/happy-earth-day-/viewer?title_no=312584&episode_no=40", + "#category": ("", "webtoons", "episode"), + "#class": webtoons.WebtoonsEpisodeExtractor, + "#exception": exception.NotFoundError, + "comic": "punderworld", + "description": str, + "episode": "36", + "episode_no": "40", + "genre": "challenge", + "title": r"re:^Punderworld - .+", + "title_no": "312584", + }, + { + "#url": "https://www.webtoons.com/en/canvas/i-want-to-be-a-cute-anime-girl/209-the-storys-story/viewer?title_no=349416&episode_no=214", + "#category": ("", "webtoons", "episode"), + "#class": webtoons.WebtoonsEpisodeExtractor, + "#count": 4, + "comic_name": "I want to be a cute anime girl", + "episode_name": "209 - The story's story", + "episode": "214", + "username": "m9huj", + "author_name": "Azul Crescent", + }, + { + "#url": "https://www.webtoons.com/en/canvas/i-want-to-be-a-cute-anime-girl/174-not-194-it-was-a-typo-later/viewer?title_no=349416&episode_no=179", + "#category": ("", "webtoons", "episode"), + "#class": webtoons.WebtoonsEpisodeExtractor, + "#count": 4, + "comic_name": "I want to be a cute anime girl", + "episode_name": "174 (not 194, it was a typo) - Later", + "episode": "179", + "username": "m9huj", + "author_name": "Azul Crescent", + }, + { + "#url": "https://www.webtoons.com/en/canvas/us-over-here/1-the-wheel/viewer?title_no=919536&episode_no=1", + "#category": ("", "webtoons", "episode"), + "#class": webtoons.WebtoonsEpisodeExtractor, + "#count": 60, + "comic_name": "Us, over here", + "episode_name": "1. The Wheel", + "episode": "1", + "username": "i94q8", + "author_name": "spin.ani", + }, + { + "#url": "https://www.webtoons.com/en/comedy/live-with-yourself/list?title_no=919", + "#comment": "english", + "#category": ("", "webtoons", "comic"), + "#class": webtoons.WebtoonsComicExtractor, + "#pattern": webtoons.WebtoonsEpisodeExtractor.pattern, + "#range": "1-15", + "#count": ">= 14", + "page": range(1, 2), + "title_no": 919, + "episode_no": range(1, 14), + }, + { + "#url": "https://www.webtoons.com/fr/romance/subzero/list?title_no=1845&page=7", + "#comment": "french", + "#category": ("", "webtoons", "comic"), + "#class": webtoons.WebtoonsComicExtractor, + "#count": ">= 15", + "page": range(7, 25), + "title_no": 1845, + "episode_no": int, + }, + { + "#url": "https://www.webtoons.com/en/challenge/scoob-and-shag/list?title_no=210827&page=9", + "#comment": "(#820)", + "#category": ("", "webtoons", "comic"), + "#class": webtoons.WebtoonsComicExtractor, + "#count": ">= 18", + "page": int, + "title_no": 210827, + "episode_no": int, + }, + { + "#url": "https://www.webtoons.com/es/romance/lore-olympus/list?title_no=1725", + "#comment": "(#1643)", + "#category": ("", "webtoons", "comic"), + "#class": webtoons.WebtoonsComicExtractor, + }, ) diff --git a/test/results/weibo.py b/test/results/weibo.py index e9d8d6447..9c78ee6d2 100644 --- a/test/results/weibo.py +++ b/test/results/weibo.py @@ -1,297 +1,256 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import weibo from gallery_dl import exception - +from gallery_dl.extractor import weibo __tests__ = ( -{ - "#url" : "https://weibo.com/1758989602", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, - "#urls" : "https://weibo.com/u/1758989602?tabtype=feed", -}, - -{ - "#url" : "https://weibo.com/1758989602", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, - "#options" : {"include": "all"}, - "#urls" : ( - "https://weibo.com/u/1758989602?tabtype=home", - "https://weibo.com/u/1758989602?tabtype=feed", - "https://weibo.com/u/1758989602?tabtype=video", - "https://weibo.com/u/1758989602?tabtype=newVideo", - "https://weibo.com/u/1758989602?tabtype=album", - ), -}, - -{ - "#url" : "https://weibo.com/zhouyuxi77", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, - "#urls" : "https://weibo.com/u/7488709788?tabtype=feed", -}, - -{ - "#url" : "https://www.weibo.com/n/周于希Sally", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, - "#urls" : "https://weibo.com/u/7488709788?tabtype=feed", -}, - -{ - "#url" : "https://weibo.com/u/1758989602", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, -}, - -{ - "#url" : "https://weibo.com/p/1758989602", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, -}, - -{ - "#url" : "https://m.weibo.cn/profile/2314621010", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, -}, - -{ - "#url" : "https://m.weibo.cn/p/2304132314621010_-_WEIBO_SECOND_PROFILE_WEIBO", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, -}, - -{ - "#url" : "https://www.weibo.com/p/1003062314621010/home", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, -}, - -{ - "#url" : "https://weibo.com/1758989602?tabtype=home", - "#comment" : "'tabtype=home' is broken on website itself", - "#category": ("", "weibo", "home"), - "#class" : weibo.WeiboHomeExtractor, - "#range" : "1-30", - "#count" : 0, -}, - -{ - "#url" : "https://weibo.com/2553930725?tabtype=feed", - "#category": ("", "weibo", "feed"), - "#class" : weibo.WeiboFeedExtractor, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://weibo.com/zhouyuxi77?tabtype=feed", - "#category": ("", "weibo", "feed"), - "#class" : weibo.WeiboFeedExtractor, - "#range" : "1", - - "status": { - "user": { - "id": 7488709788, + { + "#url": "https://weibo.com/1758989602", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + "#urls": "https://weibo.com/u/1758989602?tabtype=feed", + }, + { + "#url": "https://weibo.com/1758989602", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + "#options": {"include": "all"}, + "#urls": ( + "https://weibo.com/u/1758989602?tabtype=home", + "https://weibo.com/u/1758989602?tabtype=feed", + "https://weibo.com/u/1758989602?tabtype=video", + "https://weibo.com/u/1758989602?tabtype=newVideo", + "https://weibo.com/u/1758989602?tabtype=album", + ), + }, + { + "#url": "https://weibo.com/zhouyuxi77", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + "#urls": "https://weibo.com/u/7488709788?tabtype=feed", + }, + { + "#url": "https://www.weibo.com/n/周于希Sally", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + "#urls": "https://weibo.com/u/7488709788?tabtype=feed", + }, + { + "#url": "https://weibo.com/u/1758989602", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + }, + { + "#url": "https://weibo.com/p/1758989602", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + }, + { + "#url": "https://m.weibo.cn/profile/2314621010", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + }, + { + "#url": "https://m.weibo.cn/p/2304132314621010_-_WEIBO_SECOND_PROFILE_WEIBO", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + }, + { + "#url": "https://www.weibo.com/p/1003062314621010/home", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + }, + { + "#url": "https://weibo.com/1758989602?tabtype=home", + "#comment": "'tabtype=home' is broken on website itself", + "#category": ("", "weibo", "home"), + "#class": weibo.WeiboHomeExtractor, + "#range": "1-30", + "#count": 0, + }, + { + "#url": "https://weibo.com/2553930725?tabtype=feed", + "#category": ("", "weibo", "feed"), + "#class": weibo.WeiboFeedExtractor, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://weibo.com/zhouyuxi77?tabtype=feed", + "#category": ("", "weibo", "feed"), + "#class": weibo.WeiboFeedExtractor, + "#range": "1", + "status": { + "user": { + "id": 7488709788, + }, }, }, -}, - -{ - "#url" : "https://www.weibo.com/n/周于希Sally?tabtype=feed", - "#category": ("", "weibo", "feed"), - "#class" : weibo.WeiboFeedExtractor, - "#range" : "1", - - - "status": { - "user": { - "id": 7488709788, + { + "#url": "https://www.weibo.com/n/周于希Sally?tabtype=feed", + "#category": ("", "weibo", "feed"), + "#class": weibo.WeiboFeedExtractor, + "#range": "1", + "status": { + "user": { + "id": 7488709788, + }, }, }, -}, - -{ - "#url" : "https://weibo.com/u/7500315942?tabtype=feed", - "#comment" : "deleted (#2521)", - "#category": ("", "weibo", "feed"), - "#class" : weibo.WeiboFeedExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://weibo.com/1758989602?tabtype=video", - "#category": ("", "weibo", "videos"), - "#class" : weibo.WeiboVideosExtractor, - "#pattern" : r"https://f\.(video\.weibocdn\.com|us\.sinaimg\.cn)/(../)?\w+\.mp4\?label=mp", - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://weibo.com/1758989602?tabtype=newVideo", - "#category": ("", "weibo", "newvideo"), - "#class" : weibo.WeiboNewvideoExtractor, - "#pattern" : r"https://f\.video\.weibocdn\.com/(../)?\w+\.mp4\?label=mp", - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://weibo.com/1758989602?tabtype=article", - "#category": ("", "weibo", "article"), - "#class" : weibo.WeiboArticleExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://weibo.com/1758989602?tabtype=album", - "#category": ("", "weibo", "album"), - "#class" : weibo.WeiboAlbumExtractor, - "#pattern" : r"https://(wx\d+\.sinaimg\.cn/large/\w{32}\.(jpg|png|gif)|g\.us\.sinaimg\.cn/../\w+\.mp4)", - "#range" : "1-3", - "#count" : 3, -}, - -{ - "#url" : "https://m.weibo.cn/detail/4323047042991618", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#pattern" : r"https?://wx\d+.sinaimg.cn/large/\w+.jpg", - - "status": { - "count": 1, - "date" : "dt:2018-12-30 13:56:36", + { + "#url": "https://weibo.com/u/7500315942?tabtype=feed", + "#comment": "deleted (#2521)", + "#category": ("", "weibo", "feed"), + "#class": weibo.WeiboFeedExtractor, + "#count": 0, }, -}, - -{ - "#url" : "https://m.weibo.cn/detail/4339748116375525", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#pattern" : r"https?://f.us.sinaimg.cn/\w+\.mp4\?label=mp4_1080p", -}, - -{ - "#url" : "https://m.weibo.cn/status/4268682979207023", - "#comment" : "unavailable video (#427)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://weibo.com/3314883543/Iy7fj4qVg", - "#comment" : "non-numeric status ID (#664)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, -}, - -{ - "#url" : "https://weibo.cn/detail/4600272267522211", - "#comment" : "retweet", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://weibo.cn/detail/4600272267522211", - "#comment" : "retweet", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#options" : {"retweets": True}, - "#count" : 2, - - "status": { - "id" : 4600272267522211, - "retweeted_status": {"id": 4600167083287033}, + { + "#url": "https://weibo.com/1758989602?tabtype=video", + "#category": ("", "weibo", "videos"), + "#class": weibo.WeiboVideosExtractor, + "#pattern": r"https://f\.(video\.weibocdn\.com|us\.sinaimg\.cn)/(../)?\w+\.mp4\?label=mp", + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://weibo.com/1758989602?tabtype=newVideo", + "#category": ("", "weibo", "newvideo"), + "#class": weibo.WeiboNewvideoExtractor, + "#pattern": r"https://f\.video\.weibocdn\.com/(../)?\w+\.mp4\?label=mp", + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://weibo.com/1758989602?tabtype=article", + "#category": ("", "weibo", "article"), + "#class": weibo.WeiboArticleExtractor, + "#count": 0, + }, + { + "#url": "https://weibo.com/1758989602?tabtype=album", + "#category": ("", "weibo", "album"), + "#class": weibo.WeiboAlbumExtractor, + "#pattern": r"https://(wx\d+\.sinaimg\.cn/large/\w{32}\.(jpg|png|gif)|g\.us\.sinaimg\.cn/../\w+\.mp4)", + "#range": "1-3", + "#count": 3, + }, + { + "#url": "https://m.weibo.cn/detail/4323047042991618", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#pattern": r"https?://wx\d+.sinaimg.cn/large/\w+.jpg", + "status": { + "count": 1, + "date": "dt:2018-12-30 13:56:36", + }, + }, + { + "#url": "https://m.weibo.cn/detail/4339748116375525", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#pattern": r"https?://f.us.sinaimg.cn/\w+\.mp4\?label=mp4_1080p", + }, + { + "#url": "https://m.weibo.cn/status/4268682979207023", + "#comment": "unavailable video (#427)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://weibo.com/3314883543/Iy7fj4qVg", + "#comment": "non-numeric status ID (#664)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + }, + { + "#url": "https://weibo.cn/detail/4600272267522211", + "#comment": "retweet", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#count": 0, + }, + { + "#url": "https://weibo.cn/detail/4600272267522211", + "#comment": "retweet", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#options": {"retweets": True}, + "#count": 2, + "status": { + "id": 4600272267522211, + "retweeted_status": {"id": 4600167083287033}, + }, + }, + { + "#url": "https://m.weibo.cn/detail/4600272267522211", + "#comment": "original retweets (#1542)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#options": {"retweets": "original"}, + "status": {"id": 4600167083287033}, + }, + { + "#url": "https://weibo.com/3194672795/OuxSwgUrC", + "#comment": "type == livephoto (#2146, #6471)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#pattern": r"https://livephoto\.us\.sinaimg\.cn/\w+\.mov\?Expires=\d+&ssig=[^&#]+&KID=unistore,video", + "#range": "2,4", + "filename": { + "000yfKhRjx08hBAXxdZ60f0f0100tBPr0k01", + "000GEYrCjx08hBAXUFo40f0f0100vS5G0k01", + }, + "extension": "mov", + }, + { + "#url": "https://weibo.com/1758989602/LvBhm5DiP", + "#comment": "type == gif", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#urls": "https://wx4.sinaimg.cn/large/68d80d22gy1h2ryfa8k0kg208w06o7wh.gif", + "extension": "gif", + }, + { + "#url": "https://weibo.com/1758989602/LvBhm5DiP", + "#comment": "type == gif as video (#5183)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#options": {"gifs": "video"}, + "#pattern": r"https://g\.us\.sinaimg.cn/o0/qNZcaAAglx07Wuf921CM0104120005tc0E010\.mp4\?label=gif_mp4", + }, + { + "#url": "https://weibo.com/2909128931/4409545658754086", + "#comment": "missing 'playback_list' (#2792)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#count": 10, + }, + { + "#url": "https://weibo.com/1501933722/4142890299009993", + "#comment": "empty 'playback_list' (#3301)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#pattern": r"https://f\.us\.sinaimg\.cn/004zstGKlx07dAHg4ZVu010f01000OOl0k01\.mp4\?label=mp4_hd&template=template_7&ori=0&ps=1CwnkDw1GXwCQx.+&KID=unistore,video", + "#count": 1, + }, + { + "#url": "https://weibo.com/2427303621/MxojLlLgQ", + "#comment": "mix_media_info (#3793)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#count": 9, + }, + { + "#url": "https://m.weibo.cn/status/4339748116375525", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + }, + { + "#url": "https://m.weibo.cn/5746766133/4339748116375525", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, }, -}, - -{ - "#url" : "https://m.weibo.cn/detail/4600272267522211", - "#comment" : "original retweets (#1542)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#options" : {"retweets": "original"}, - - "status": {"id": 4600167083287033}, -}, - -{ - "#url" : "https://weibo.com/3194672795/OuxSwgUrC", - "#comment" : "type == livephoto (#2146, #6471)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#pattern" : r"https://livephoto\.us\.sinaimg\.cn/\w+\.mov\?Expires=\d+&ssig=[^&#]+&KID=unistore,video", - "#range" : "2,4", - - "filename" : {"000yfKhRjx08hBAXxdZ60f0f0100tBPr0k01", "000GEYrCjx08hBAXUFo40f0f0100vS5G0k01"}, - "extension": "mov", -}, - -{ - "#url" : "https://weibo.com/1758989602/LvBhm5DiP", - "#comment" : "type == gif", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#urls" : "https://wx4.sinaimg.cn/large/68d80d22gy1h2ryfa8k0kg208w06o7wh.gif", - - "extension": "gif", -}, - -{ - "#url" : "https://weibo.com/1758989602/LvBhm5DiP", - "#comment" : "type == gif as video (#5183)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#options" : {"gifs": "video"}, - "#pattern" : r"https://g\.us\.sinaimg.cn/o0/qNZcaAAglx07Wuf921CM0104120005tc0E010\.mp4\?label=gif_mp4", -}, - -{ - "#url" : "https://weibo.com/2909128931/4409545658754086", - "#comment" : "missing 'playback_list' (#2792)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#count" : 10, -}, - -{ - "#url" : "https://weibo.com/1501933722/4142890299009993", - "#comment" : "empty 'playback_list' (#3301)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#pattern" : r"https://f\.us\.sinaimg\.cn/004zstGKlx07dAHg4ZVu010f01000OOl0k01\.mp4\?label=mp4_hd&template=template_7&ori=0&ps=1CwnkDw1GXwCQx.+&KID=unistore,video", - "#count" : 1, -}, - -{ - "#url" : "https://weibo.com/2427303621/MxojLlLgQ", - "#comment" : "mix_media_info (#3793)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#count" : 9, -}, - -{ - "#url" : "https://m.weibo.cn/status/4339748116375525", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, -}, - -{ - "#url" : "https://m.weibo.cn/5746766133/4339748116375525", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, -}, - ) diff --git a/test/results/wikiart.py b/test/results/wikiart.py index 9ab131030..10889e232 100644 --- a/test/results/wikiart.py +++ b/test/results/wikiart.py @@ -1,108 +1,97 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikiart - __tests__ = ( -{ - "#url" : "https://www.wikiart.org/en/thomas-cole", - "#category": ("", "wikiart", "artist"), - "#class" : wikiart.WikiartArtistExtractor, - "#pattern" : r"https://uploads\d+\.wikiart\.org/(\d+/)?images/thomas-cole/[\w()-]+\.(jpg|png)", - "#count" : "> 100", - - "albums" : None, - "artist" : { - "OriginalArtistName": "Thomas Cole", - "activeYearsCompletion": None, - "activeYearsStart" : None, - "artistName" : "Thomas Cole", - "biography" : "Thomas Cole inspired the generation of American [url href=https://www.wikiart.org/en/paintings-by-genre/landscape]landscape[/url] painters that came to be known as the [url href=https://www.wikiart.org/en/artists-by-painting-school/hudson-river-school]Hudson River School[/url]. Born in Bolton-le-Moors, Lancashire, England, in 1801, at the age of seventeen he emigrated with his family to the United States, first working as a wood engraver in Philadelphia before going to Steubenville, Ohio, where his father had established a wallpaper manufacturing business. \n\nCole received rudimentary instruction from an itinerant artist, began painting portraits, genre scenes, and a few landscapes, and set out to seek his fortune through Ohio and Pennsylvania. He soon moved on to Philadelphia to pursue his art, inspired by paintings he saw at the Pennsylvania Academy of the Fine Arts. Moving to New York City in spring 1825, Cole made a trip up the Hudson River to the eastern Catskill Mountains. Based on his sketches there, he executed three landscapes that a city bookseller agreed to display in his window. Colonel [url href=https://www.wikiart.org/en/john-trumbull]John Trumbull[/url], already renowned as the painter of the American Revolution, saw Cole’s pictures and instantly purchased one, recommending the other two to his colleagues William Dunlap and [url href=https://www.wikiart.org/en/asher-brown-durand]Asher B. Durand[/url]. \n\nWhat Trumbull recognized in the work of the young painter was the perception of wildness inherent in American scenery that landscape artists had theretofore ignored. Trumbull brought Cole to the attention of various patrons, who began eagerly buying his work. Dunlap publicized the discovery of the new talent, and Cole was welcomed into New York’s cultural community, which included the poet and editor William Cullen Bryant and the author James Fenimore Cooper. Cole became one of the founding members of the National Academy of Design in 1825. Even as Cole expanded his travels and subjects to include scenes in the White Mountains of New Hampshire, he aspired to what he termed a “higher style of a landscape” that included narrative—some of the paintings in paired series—including biblical and literary subjects, such as Cooper’s popular [url href=https://www.wikiart.org/en/thomas-cole/scene-from-the-last-of-the-mohicans-by-james-fenimore-cooper-1827][i]Last of the Mohicans[/i][/url]. \n\nBy 1829, his success enabled him to take the Grand Tour of Europe and especially Italy, where he remained in 1831–32, visiting Florence, Rome, and Naples. Thereafter he painted many Italian subjects, like [url href=https://www.wikiart.org/en/thomas-cole/a-view-near-tivoli-morning-1832][i]View near Tivoli. Morning[/i][/url] (1832). The region around Rome, along with the classical myth, also inspired [url href=https://www.wikiart.org/en/thomas-cole/the-titan-s-goblet-1833][i]The Titan’s Goblet[/i][/url] (1833). Cole’s travels and the encouragement and patronage of the New York merchant Luman Reed culminated in his most ambitious historical landscape series, [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-course-of-empire,resultType:masonry][i]The Course of Empire[/i][/url] (1833–1836), five pictures dramatizing the rise and fall of an ancient classical state. \n\nCole also continued to paint, with ever-rising technical assurance, sublime American scenes such as the [url href=https://www.wikiart.org/en/thomas-cole/view-from-mount-holyoke-1836][i]View from Mount Holyoke[/i][/url] (1836), [url href=https://www.wikiart.org/en/thomas-cole/the-oxbow-the-connecticut-river-near-northampton-1836][i]The Oxbow[/i][/url] (1836), in which he included a portrait of himself painting the vista and [url href=https://www.wikiart.org/en/thomas-cole/view-on-the-catskill-early-autunm-1837][i]View on the Catskill—Early Autumn[/i][/url] (1836-1837), in which he pastorally interpreted the prospect of his beloved Catskill Mountains from the village of Catskill, where he had moved the year before and met his wife-to-be, Maria Bartow. \n\nThe artist’s marriage brought with it increasing religious piety manifested in the four-part series [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-voyage-of-life,resultType:masonry][i]The Voyage of Life[/i][/url] (1840). In it, a river journey represents the human passage through life to eternal reward. Cole painted and exhibited a replica of the series in Rome, where he returned in 1841–42, traveling south to Sicily. After his return, he lived and worked chiefly in Catskill, keeping up with art activity in New York primarily through Durand. He continued to produce American and foreign landscape subjects of incredible beauty, including the [url href=https://www.wikiart.org/en/thomas-cole/the-mountain-ford-1846][i]Mountain Ford[/i][/url] (1846). \n\nIn 1844, Cole welcomed into his Catskill studio the young [url href=https://www.wikiart.org/en/frederic-edwin-church]Frederic Church[/url], who studied with him until 1846 and went on to become the most renowned exponent of the generation that followed Cole. By 1846, Cole was at work on his largest and most ambitious series, [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-cross-and-the-world,resultType:masonry][i]The Cross and the World[/i][/url], but in February 1848 contracted pleurisy and died before completing it. \n\nThe paintings of Thomas Cole, like the writings of his contemporary Ralph Waldo Emerson, stand as monuments to the dreams and anxieties of the fledgling American nation during the mid-19th century; and they are also euphoric celebrations of its natural landscapes. Cole is considered the first artist to bring the eye of a European [url href=https://www.wikiart.org/en/artists-by-art-movement/romanticism]Romantic[/url] landscape painter to those environments, but also a figure whose idealism and religious sensibilities expressed a uniquely American spirit. In his works, we find the dramatic splendor of [url href=https://www.wikiart.org/en/caspar-david-friedrich]Caspar David Freidrich[/url] or [url href=https://www.wikiart.org/en/william-turner]J.M.W Turner[/url] transposed onto the Catskill and Adirondack Mountains. But whereas younger American painters such as [url href=https://www.wikiart.org/en/albert-bierstadt]Albert Bierstadt[/url] had come into direct contact with [url href=https://www.wikiart.org/en/artists-by-art-institution/kunstakademie-dusseldorf-dusseldorf-germany#!#resultType:masonry]The Düsseldorf School of painting[/url], and thus with the tradition in which they placed themselves, Cole was largely self-tutored, representing something of the archetypal American figure of the auto-didact.\n\nIn many ways, Cole's art epitomizes all contradictions of European settler culture in America. He was in love with the sublime wildness of the American landscape and sought to preserve it with his art, but his very presence in that landscape, and the development of his career, depended on the processes of urbanization and civilization which threatened it. From a modern perspective, Cole's Eurocentric gaze on seemingly empty wildernesses which had, in fact, been populated for centuries, also seems troubling; where Native Americans do appear in his work, as in [url href=https://www.wikiart.org/en/thomas-cole/falls-of-the-kaaterskill-1826][i]Falls of the Kaaterskill[/i][/url] (1826), it is as picturesque flecks rather than characterized participants in the scene.\n\nCole's legacy is evident in the work of future American artists who advanced the Hudson River style, including his student Frederic Edwin Church, Albert Bierstadt, Jasper Cropsey, Asher B. Durand, [url href=https://www.wikiart.org/en/george-inness]George Inness[/url], [url href=https://www.wikiart.org/en/john-frederick-kensett]John Kensett[/url], and [url href=https://www.wikiart.org/en/thomas-moran]Thomas Moran[/url]. Speaking more broadly, a whole sweep of 20th-century North-American art, from [url href=https://www.wikiart.org/en/artists-by-art-movement/precisionism]Precisionism[/url] to [url href=https://www.wikiart.org/en/artists-by-art-movement/environmental-art]Land Art[/url], might be seen to have inherited something of the grand scale and ambition of Cole's work. In this sense, his paintings capture not only the character of American culture during the mid-19th century but perhaps something more enduring about the open and expansive quality of that culture.", - "birthDay" : "/Date(-5330448000000)/", - "birthDayAsString" : "February 1, 1801", - "contentId" : 254330, - "deathDay" : "/Date(-3846441600000)/", - "deathDayAsString" : "February 11, 1848", - "dictonaries" : [ - 1368, - 11415, - 310, - ], - "gender" : "male", - "image" : "https://uploads8.wikiart.org/temp/19f6a140-59d2-4959-8d11-fd4ca582b7f2.jpg!Portrait.jpg", - "lastNameFirst" : "Cole Thomas", - "periodsOfWork" : "", - "relatedArtistsIds" : [], - "series" : "The Cross and the World\r\nThe Course of Empire\r\nThe Voyage of Life", - "story" : "http://en.wikipedia.org/wiki/Thomas_Cole", - "themes" : "", - "url" : "thomas-cole", - "wikipediaUrl" : "http://en.wikipedia.org/wiki/Thomas_Cole" + { + "#url": "https://www.wikiart.org/en/thomas-cole", + "#category": ("", "wikiart", "artist"), + "#class": wikiart.WikiartArtistExtractor, + "#pattern": r"https://uploads\d+\.wikiart\.org/(\d+/)?images/thomas-cole/[\w()-]+\.(jpg|png)", + "#count": "> 100", + "albums": None, + "artist": { + "OriginalArtistName": "Thomas Cole", + "activeYearsCompletion": None, + "activeYearsStart": None, + "artistName": "Thomas Cole", + "biography": "Thomas Cole inspired the generation of American [url href=https://www.wikiart.org/en/paintings-by-genre/landscape]landscape[/url] painters that came to be known as the [url href=https://www.wikiart.org/en/artists-by-painting-school/hudson-river-school]Hudson River School[/url]. Born in Bolton-le-Moors, Lancashire, England, in 1801, at the age of seventeen he emigrated with his family to the United States, first working as a wood engraver in Philadelphia before going to Steubenville, Ohio, where his father had established a wallpaper manufacturing business. \n\nCole received rudimentary instruction from an itinerant artist, began painting portraits, genre scenes, and a few landscapes, and set out to seek his fortune through Ohio and Pennsylvania. He soon moved on to Philadelphia to pursue his art, inspired by paintings he saw at the Pennsylvania Academy of the Fine Arts. Moving to New York City in spring 1825, Cole made a trip up the Hudson River to the eastern Catskill Mountains. Based on his sketches there, he executed three landscapes that a city bookseller agreed to display in his window. Colonel [url href=https://www.wikiart.org/en/john-trumbull]John Trumbull[/url], already renowned as the painter of the American Revolution, saw Cole’s pictures and instantly purchased one, recommending the other two to his colleagues William Dunlap and [url href=https://www.wikiart.org/en/asher-brown-durand]Asher B. Durand[/url]. \n\nWhat Trumbull recognized in the work of the young painter was the perception of wildness inherent in American scenery that landscape artists had theretofore ignored. Trumbull brought Cole to the attention of various patrons, who began eagerly buying his work. Dunlap publicized the discovery of the new talent, and Cole was welcomed into New York’s cultural community, which included the poet and editor William Cullen Bryant and the author James Fenimore Cooper. Cole became one of the founding members of the National Academy of Design in 1825. Even as Cole expanded his travels and subjects to include scenes in the White Mountains of New Hampshire, he aspired to what he termed a “higher style of a landscape” that included narrative—some of the paintings in paired series—including biblical and literary subjects, such as Cooper’s popular [url href=https://www.wikiart.org/en/thomas-cole/scene-from-the-last-of-the-mohicans-by-james-fenimore-cooper-1827][i]Last of the Mohicans[/i][/url]. \n\nBy 1829, his success enabled him to take the Grand Tour of Europe and especially Italy, where he remained in 1831–32, visiting Florence, Rome, and Naples. Thereafter he painted many Italian subjects, like [url href=https://www.wikiart.org/en/thomas-cole/a-view-near-tivoli-morning-1832][i]View near Tivoli. Morning[/i][/url] (1832). The region around Rome, along with the classical myth, also inspired [url href=https://www.wikiart.org/en/thomas-cole/the-titan-s-goblet-1833][i]The Titan’s Goblet[/i][/url] (1833). Cole’s travels and the encouragement and patronage of the New York merchant Luman Reed culminated in his most ambitious historical landscape series, [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-course-of-empire,resultType:masonry][i]The Course of Empire[/i][/url] (1833–1836), five pictures dramatizing the rise and fall of an ancient classical state. \n\nCole also continued to paint, with ever-rising technical assurance, sublime American scenes such as the [url href=https://www.wikiart.org/en/thomas-cole/view-from-mount-holyoke-1836][i]View from Mount Holyoke[/i][/url] (1836), [url href=https://www.wikiart.org/en/thomas-cole/the-oxbow-the-connecticut-river-near-northampton-1836][i]The Oxbow[/i][/url] (1836), in which he included a portrait of himself painting the vista and [url href=https://www.wikiart.org/en/thomas-cole/view-on-the-catskill-early-autunm-1837][i]View on the Catskill—Early Autumn[/i][/url] (1836-1837), in which he pastorally interpreted the prospect of his beloved Catskill Mountains from the village of Catskill, where he had moved the year before and met his wife-to-be, Maria Bartow. \n\nThe artist’s marriage brought with it increasing religious piety manifested in the four-part series [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-voyage-of-life,resultType:masonry][i]The Voyage of Life[/i][/url] (1840). In it, a river journey represents the human passage through life to eternal reward. Cole painted and exhibited a replica of the series in Rome, where he returned in 1841–42, traveling south to Sicily. After his return, he lived and worked chiefly in Catskill, keeping up with art activity in New York primarily through Durand. He continued to produce American and foreign landscape subjects of incredible beauty, including the [url href=https://www.wikiart.org/en/thomas-cole/the-mountain-ford-1846][i]Mountain Ford[/i][/url] (1846). \n\nIn 1844, Cole welcomed into his Catskill studio the young [url href=https://www.wikiart.org/en/frederic-edwin-church]Frederic Church[/url], who studied with him until 1846 and went on to become the most renowned exponent of the generation that followed Cole. By 1846, Cole was at work on his largest and most ambitious series, [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-cross-and-the-world,resultType:masonry][i]The Cross and the World[/i][/url], but in February 1848 contracted pleurisy and died before completing it. \n\nThe paintings of Thomas Cole, like the writings of his contemporary Ralph Waldo Emerson, stand as monuments to the dreams and anxieties of the fledgling American nation during the mid-19th century; and they are also euphoric celebrations of its natural landscapes. Cole is considered the first artist to bring the eye of a European [url href=https://www.wikiart.org/en/artists-by-art-movement/romanticism]Romantic[/url] landscape painter to those environments, but also a figure whose idealism and religious sensibilities expressed a uniquely American spirit. In his works, we find the dramatic splendor of [url href=https://www.wikiart.org/en/caspar-david-friedrich]Caspar David Freidrich[/url] or [url href=https://www.wikiart.org/en/william-turner]J.M.W Turner[/url] transposed onto the Catskill and Adirondack Mountains. But whereas younger American painters such as [url href=https://www.wikiart.org/en/albert-bierstadt]Albert Bierstadt[/url] had come into direct contact with [url href=https://www.wikiart.org/en/artists-by-art-institution/kunstakademie-dusseldorf-dusseldorf-germany#!#resultType:masonry]The Düsseldorf School of painting[/url], and thus with the tradition in which they placed themselves, Cole was largely self-tutored, representing something of the archetypal American figure of the auto-didact.\n\nIn many ways, Cole's art epitomizes all contradictions of European settler culture in America. He was in love with the sublime wildness of the American landscape and sought to preserve it with his art, but his very presence in that landscape, and the development of his career, depended on the processes of urbanization and civilization which threatened it. From a modern perspective, Cole's Eurocentric gaze on seemingly empty wildernesses which had, in fact, been populated for centuries, also seems troubling; where Native Americans do appear in his work, as in [url href=https://www.wikiart.org/en/thomas-cole/falls-of-the-kaaterskill-1826][i]Falls of the Kaaterskill[/i][/url] (1826), it is as picturesque flecks rather than characterized participants in the scene.\n\nCole's legacy is evident in the work of future American artists who advanced the Hudson River style, including his student Frederic Edwin Church, Albert Bierstadt, Jasper Cropsey, Asher B. Durand, [url href=https://www.wikiart.org/en/george-inness]George Inness[/url], [url href=https://www.wikiart.org/en/john-frederick-kensett]John Kensett[/url], and [url href=https://www.wikiart.org/en/thomas-moran]Thomas Moran[/url]. Speaking more broadly, a whole sweep of 20th-century North-American art, from [url href=https://www.wikiart.org/en/artists-by-art-movement/precisionism]Precisionism[/url] to [url href=https://www.wikiart.org/en/artists-by-art-movement/environmental-art]Land Art[/url], might be seen to have inherited something of the grand scale and ambition of Cole's work. In this sense, his paintings capture not only the character of American culture during the mid-19th century but perhaps something more enduring about the open and expansive quality of that culture.", + "birthDay": "/Date(-5330448000000)/", + "birthDayAsString": "February 1, 1801", + "contentId": 254330, + "deathDay": "/Date(-3846441600000)/", + "deathDayAsString": "February 11, 1848", + "dictonaries": [ + 1368, + 11415, + 310, + ], + "gender": "male", + "image": "https://uploads8.wikiart.org/temp/19f6a140-59d2-4959-8d11-fd4ca582b7f2.jpg!Portrait.jpg", + "lastNameFirst": "Cole Thomas", + "periodsOfWork": "", + "relatedArtistsIds": [], + "series": "The Cross and the World\r\nThe Course of Empire\r\nThe Voyage of Life", + "story": "http://en.wikipedia.org/wiki/Thomas_Cole", + "themes": "", + "url": "thomas-cole", + "wikipediaUrl": "http://en.wikipedia.org/wiki/Thomas_Cole", + }, + "artistName": "Thomas Cole", + "artistUrl": "/en/thomas-cole", + "extension": str, + "filename": str, + "flags": int, + "height": int, + "id": r"re:[0-9a-f]+", + "image": str, + "map": str, + "paintingUrl": r"re:/en/thomas-cole/.+", + "title": str, + "width": int, + "year": str, + }, + { + "#url": "https://www.wikiart.org/en/thomas-cole/the-departure-1838", + "#category": ("", "wikiart", "image"), + "#class": wikiart.WikiartImageExtractor, + "#sha1_url": "976cc2545f308a650b5dbb35c29d3cee0f4673b3", + "#sha1_metadata": "8e80cdcb01c1fedb934633d1c4c3ab0419cfbedf", + }, + { + "#url": "https://www.wikiart.org/en/huang-shen/summer", + "#comment": "no year or '-' in slug", + "#category": ("", "wikiart", "image"), + "#class": wikiart.WikiartImageExtractor, + "#sha1_url": "d7f60118c34067b2b37d9577e412dc1477b94207", + "#urls": ("https://uploads5.wikiart.org/images/huang-shen/summer.jpg",), + }, + { + "#url": "https://www.wikiart.org/en/paintings-by-media/grisaille", + "#category": ("", "wikiart", "artworks"), + "#class": wikiart.WikiartArtworksExtractor, + "#sha1_url": "36e054fcb3363b7f085c81f4778e6db3994e56a3", + "#urls": ( + "https://uploads4.wikiart.org/images/hieronymus-bosch/triptych-of-last-judgement.jpg", + "https://uploads6.wikiart.org/images/hieronymus-bosch/triptych-of-last-judgement-1.jpg", + "https://uploads0.wikiart.org/images/hieronymus-bosch/tiptych-of-temptation-of-st-anthony-1506.jpg", + "https://uploads7.wikiart.org/images/matthias-grünewald/st-elizabeth-and-a-saint-woman-with-palm-1511.jpg", + "https://uploads2.wikiart.org/images/matthias-grünewald/st-lawrence-and-st-cyricus-1511.jpg", + "https://uploads0.wikiart.org/images/pieter-bruegel-the-elder/the-death-of-the-virgin.jpg", + "https://uploads4.wikiart.org/images/pieter-bruegel-the-elder/christ-and-the-woman-taken-in-adultery-1565-1.jpg", + "https://uploads6.wikiart.org/images/giovanni-battista-tiepolo/not_detected_241014.jpg", + "https://uploads4.wikiart.org/images/edgar-degas/interior-the-rape-1869.jpg", + "https://uploads3.wikiart.org/00265/images/john-singer-sargent/1396294310-dame-alice-ellen-terry-by-john-singer-sargent.jpg", + "https://uploads0.wikiart.org/00293/images/hryhorii-havrylenko/1954-18-5-32-5.jpg", + ), + }, + { + "#url": "https://www.wikiart.org/en/artists-by-century/12", + "#category": ("", "wikiart", "artists"), + "#class": wikiart.WikiartArtistsExtractor, + "#pattern": wikiart.WikiartArtistExtractor.pattern, + "#count": ">= 8", }, - "artistName" : "Thomas Cole", - "artistUrl" : "/en/thomas-cole", - "extension" : str, - "filename" : str, - "flags" : int, - "height" : int, - "id" : r"re:[0-9a-f]+", - "image" : str, - "map" : str, - "paintingUrl": r"re:/en/thomas-cole/.+", - "title" : str, - "width" : int, - "year" : str, -}, - -{ - "#url" : "https://www.wikiart.org/en/thomas-cole/the-departure-1838", - "#category": ("", "wikiart", "image"), - "#class" : wikiart.WikiartImageExtractor, - "#sha1_url" : "976cc2545f308a650b5dbb35c29d3cee0f4673b3", - "#sha1_metadata": "8e80cdcb01c1fedb934633d1c4c3ab0419cfbedf", -}, - -{ - "#url" : "https://www.wikiart.org/en/huang-shen/summer", - "#comment" : "no year or '-' in slug", - "#category": ("", "wikiart", "image"), - "#class" : wikiart.WikiartImageExtractor, - "#sha1_url": "d7f60118c34067b2b37d9577e412dc1477b94207", - "#urls" : ( - "https://uploads5.wikiart.org/images/huang-shen/summer.jpg", - ), -}, - -{ - "#url" : "https://www.wikiart.org/en/paintings-by-media/grisaille", - "#category": ("", "wikiart", "artworks"), - "#class" : wikiart.WikiartArtworksExtractor, - "#sha1_url": "36e054fcb3363b7f085c81f4778e6db3994e56a3", - "#urls" : ( - "https://uploads4.wikiart.org/images/hieronymus-bosch/triptych-of-last-judgement.jpg", - "https://uploads6.wikiart.org/images/hieronymus-bosch/triptych-of-last-judgement-1.jpg", - "https://uploads0.wikiart.org/images/hieronymus-bosch/tiptych-of-temptation-of-st-anthony-1506.jpg", - "https://uploads7.wikiart.org/images/matthias-grünewald/st-elizabeth-and-a-saint-woman-with-palm-1511.jpg", - "https://uploads2.wikiart.org/images/matthias-grünewald/st-lawrence-and-st-cyricus-1511.jpg", - "https://uploads0.wikiart.org/images/pieter-bruegel-the-elder/the-death-of-the-virgin.jpg", - "https://uploads4.wikiart.org/images/pieter-bruegel-the-elder/christ-and-the-woman-taken-in-adultery-1565-1.jpg", - "https://uploads6.wikiart.org/images/giovanni-battista-tiepolo/not_detected_241014.jpg", - "https://uploads4.wikiart.org/images/edgar-degas/interior-the-rape-1869.jpg", - "https://uploads3.wikiart.org/00265/images/john-singer-sargent/1396294310-dame-alice-ellen-terry-by-john-singer-sargent.jpg", - "https://uploads0.wikiart.org/00293/images/hryhorii-havrylenko/1954-18-5-32-5.jpg", - ), -}, - -{ - "#url" : "https://www.wikiart.org/en/artists-by-century/12", - "#category": ("", "wikiart", "artists"), - "#class" : wikiart.WikiartArtistsExtractor, - "#pattern" : wikiart.WikiartArtistExtractor.pattern, - "#count" : ">= 8", -}, - ) diff --git a/test/results/wikibooks.py b/test/results/wikibooks.py index da4d761db..15806977e 100644 --- a/test/results/wikibooks.py +++ b/test/results/wikibooks.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikibooks.org/wiki/Title", - "#category": ("wikimedia", "wikibooks", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikibooks.org/wiki/Category:Title", - "#category": ("wikimedia", "wikibooks", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikibooks.org/wiki/Title", + "#category": ("wikimedia", "wikibooks", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikibooks.org/wiki/Category:Title", + "#category": ("wikimedia", "wikibooks", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikidata.py b/test/results/wikidata.py index c0e2eb6ae..36c664c01 100644 --- a/test/results/wikidata.py +++ b/test/results/wikidata.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikidata.org/wiki/Title", - "#category": ("wikimedia", "wikidata", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikidata.org/wiki/Category:Title", - "#category": ("wikimedia", "wikidata", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikidata.org/wiki/Title", + "#category": ("wikimedia", "wikidata", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikidata.org/wiki/Category:Title", + "#category": ("wikimedia", "wikidata", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikifeet.py b/test/results/wikifeet.py index 2a8b849b3..dce6f40a7 100644 --- a/test/results/wikifeet.py +++ b/test/results/wikifeet.py @@ -1,51 +1,44 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikifeet - __tests__ = ( -{ - "#url" : "https://www.wikifeet.com/Madison_Beer", - "#category": ("", "wikifeet", "gallery"), - "#class" : wikifeet.WikifeetGalleryExtractor, - "#pattern" : r"https://pics\.wikifeet\.com/Madison_Beer-Feet-\d+\.jpg", - "#count" : ">= 352", - - "celeb" : "Madison_Beer", - "celebrity" : "Madison Beer", - "birthday" : "dt:1999-03-05 00:00:00", - "birthplace": "United States", - "rating" : float, - "pid" : int, - "width" : int, - "height" : int, - "shoesize" : r"re:\d+ US", - "type" : "women", - "tags" : list, -}, - -{ - "#url" : "https://men.wikifeet.com/Chris_Hemsworth", - "#category": ("", "wikifeet", "gallery"), - "#class" : wikifeet.WikifeetGalleryExtractor, - "#pattern" : r"https://pics\.wikifeet\.com/Chris_Hemsworth-Feet-\d+\.jpg", - "#count" : ">= 860", - - "celeb" : "Chris_Hemsworth", - "celebrity" : "Chris Hemsworth", - "birthday" : "dt:1983-08-11 00:00:00", - "birthplace": "Australia", - "rating" : float, - "pid" : int, - "width" : int, - "height" : int, - "shoesize" : "12.5 US", - "type" : "men", - "tags" : list, -}, - + { + "#url": "https://www.wikifeet.com/Madison_Beer", + "#category": ("", "wikifeet", "gallery"), + "#class": wikifeet.WikifeetGalleryExtractor, + "#pattern": r"https://pics\.wikifeet\.com/Madison_Beer-Feet-\d+\.jpg", + "#count": ">= 352", + "celeb": "Madison_Beer", + "celebrity": "Madison Beer", + "birthday": "dt:1999-03-05 00:00:00", + "birthplace": "United States", + "rating": float, + "pid": int, + "width": int, + "height": int, + "shoesize": r"re:\d+ US", + "type": "women", + "tags": list, + }, + { + "#url": "https://men.wikifeet.com/Chris_Hemsworth", + "#category": ("", "wikifeet", "gallery"), + "#class": wikifeet.WikifeetGalleryExtractor, + "#pattern": r"https://pics\.wikifeet\.com/Chris_Hemsworth-Feet-\d+\.jpg", + "#count": ">= 860", + "celeb": "Chris_Hemsworth", + "celebrity": "Chris Hemsworth", + "birthday": "dt:1983-08-11 00:00:00", + "birthplace": "Australia", + "rating": float, + "pid": int, + "width": int, + "height": int, + "shoesize": "12.5 US", + "type": "men", + "tags": list, + }, ) diff --git a/test/results/wikifeetx.py b/test/results/wikifeetx.py index a3d2be6ed..5e04d4c76 100644 --- a/test/results/wikifeetx.py +++ b/test/results/wikifeetx.py @@ -1,31 +1,26 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikifeet - __tests__ = ( -{ - "#url" : "https://www.wikifeetx.com/Tifa_Quinn", - "#category": ("", "wikifeetx", "gallery"), - "#class" : wikifeet.WikifeetGalleryExtractor, - "#pattern" : r"https://pics\.wikifeet\.com/Tifa_Quinn-Feet-\d+\.jpg", - "#count" : ">= 9", - - "celeb" : "Tifa_Quinn", - "celebrity" : "Tifa Quinn", - "birthday" : "[NOT SET]", - "birthplace": "United States", - "rating" : float, - "pid" : int, - "width" : int, - "height" : int, - "shoesize" : "4 US", - "type" : "women", - "tags" : list, -}, - + { + "#url": "https://www.wikifeetx.com/Tifa_Quinn", + "#category": ("", "wikifeetx", "gallery"), + "#class": wikifeet.WikifeetGalleryExtractor, + "#pattern": r"https://pics\.wikifeet\.com/Tifa_Quinn-Feet-\d+\.jpg", + "#count": ">= 9", + "celeb": "Tifa_Quinn", + "celebrity": "Tifa Quinn", + "birthday": "[NOT SET]", + "birthplace": "United States", + "rating": float, + "pid": int, + "width": int, + "height": int, + "shoesize": "4 US", + "type": "women", + "tags": list, + }, ) diff --git a/test/results/wikigg.py b/test/results/wikigg.py index f64259fea..ad6026006 100644 --- a/test/results/wikigg.py +++ b/test/results/wikigg.py @@ -1,33 +1,27 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wiki.gg/wiki/Title", - "#comment" : "for scripts/supportedsites.py", - "#category": ("wikimedia", "wikigg-www", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://hearthstone.wiki.gg/wiki/Flame_Juggler", - "#category": ("wikimedia", "wikigg-hearthstone", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://terraria.wiki.gg/de/wiki/Golem", - "#comment" : "non-English language prefix (#6370)", - "#category": ("wikimedia", "wikigg-terraria", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#count" : "> 45", - "#archive" : False, -}, - + { + "#url": "https://www.wiki.gg/wiki/Title", + "#comment": "for scripts/supportedsites.py", + "#category": ("wikimedia", "wikigg-www", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://hearthstone.wiki.gg/wiki/Flame_Juggler", + "#category": ("wikimedia", "wikigg-hearthstone", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://terraria.wiki.gg/de/wiki/Golem", + "#comment": "non-English language prefix (#6370)", + "#category": ("wikimedia", "wikigg-terraria", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#count": "> 45", + "#archive": False, + }, ) diff --git a/test/results/wikimediacommons.py b/test/results/wikimediacommons.py index b61a9061b..4fa308feb 100644 --- a/test/results/wikimediacommons.py +++ b/test/results/wikimediacommons.py @@ -1,24 +1,19 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://commons.wikimedia.org/wiki/File:Starr-050516-1367-Pimenta_dioica-flowers-Maunaloa-Molokai_(24762757525).jpg", - "#category": ("wikimedia", "wikimediacommons", "file"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#urls" : "https://upload.wikimedia.org/wikipedia/commons/f/fa/Starr-050516-1367-Pimenta_dioica-flowers-Maunaloa-Molokai_%2824762757525%29.jpg", -}, - -{ - "#url" : "https://commons.wikimedia.org/wiki/Category:Network_maps_of_the_Paris_Metro", - "#category": ("wikimedia", "wikimediacommons", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://commons.wikimedia.org/wiki/File:Starr-050516-1367-Pimenta_dioica-flowers-Maunaloa-Molokai_(24762757525).jpg", + "#category": ("wikimedia", "wikimediacommons", "file"), + "#class": wikimedia.WikimediaArticleExtractor, + "#urls": "https://upload.wikimedia.org/wikipedia/commons/f/fa/Starr-050516-1367-Pimenta_dioica-flowers-Maunaloa-Molokai_%2824762757525%29.jpg", + }, + { + "#url": "https://commons.wikimedia.org/wiki/Category:Network_maps_of_the_Paris_Metro", + "#category": ("wikimedia", "wikimediacommons", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikinews.py b/test/results/wikinews.py index 79817fdbf..3418a22de 100644 --- a/test/results/wikinews.py +++ b/test/results/wikinews.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikinews.org/wiki/Title", - "#category": ("wikimedia", "wikinews", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikinews.org/wiki/Category:Title", - "#category": ("wikimedia", "wikinews", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikinews.org/wiki/Title", + "#category": ("wikimedia", "wikinews", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikinews.org/wiki/Category:Title", + "#category": ("wikimedia", "wikinews", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikipedia.py b/test/results/wikipedia.py index f478a49d6..d67b22bf9 100644 --- a/test/results/wikipedia.py +++ b/test/results/wikipedia.py @@ -1,61 +1,53 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikipedia.org/wiki/Title", - "#category": ("wikimedia", "wikipedia", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikipedia.org/wiki/Athena", - "#category": ("wikimedia", "wikipedia", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#pattern" : r"https://upload.wikimedia.org/wikipedia/.+", - "#count" : range(50, 100), - - "bitdepth" : int, - "canonicaltitle": str, - "comment" : str, - "commonmetadata": dict, - "date" : "type:datetime", - "descriptionshorturl": str, - "descriptionurl": str, - "extension" : str, - "extmetadata" : dict, - "filename" : str, - "height" : int, - "metadata" : dict, - "mime" : r"re:image/\w+", - "page" : "Athena", - "sha1" : r"re:^[0-9a-f]{40}$", - "size" : int, - "timestamp" : str, - "url" : str, - "user" : str, - "userid" : int, - "width" : int, -}, - -{ - "#url" : "https://en.wikipedia.org/wiki/Category:Physics", - "#category": ("wikimedia", "wikipedia", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikipedia.org", - "#category": ("wikimedia", "wikipedia", "wiki"), - "#class" : wikimedia.WikimediaWikiExtractor, - "#range" : "1-10", - "#count" : 10, -}, - + { + "#url": "https://www.wikipedia.org/wiki/Title", + "#category": ("wikimedia", "wikipedia", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikipedia.org/wiki/Athena", + "#category": ("wikimedia", "wikipedia", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#pattern": r"https://upload.wikimedia.org/wikipedia/.+", + "#count": range(50, 100), + "bitdepth": int, + "canonicaltitle": str, + "comment": str, + "commonmetadata": dict, + "date": "type:datetime", + "descriptionshorturl": str, + "descriptionurl": str, + "extension": str, + "extmetadata": dict, + "filename": str, + "height": int, + "metadata": dict, + "mime": r"re:image/\w+", + "page": "Athena", + "sha1": r"re:^[0-9a-f]{40}$", + "size": int, + "timestamp": str, + "url": str, + "user": str, + "userid": int, + "width": int, + }, + { + "#url": "https://en.wikipedia.org/wiki/Category:Physics", + "#category": ("wikimedia", "wikipedia", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikipedia.org", + "#category": ("wikimedia", "wikipedia", "wiki"), + "#class": wikimedia.WikimediaWikiExtractor, + "#range": "1-10", + "#count": 10, + }, ) diff --git a/test/results/wikiquote.py b/test/results/wikiquote.py index 8365e3b7d..9e7dd5734 100644 --- a/test/results/wikiquote.py +++ b/test/results/wikiquote.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikiquote.org/wiki/Title", - "#category": ("wikimedia", "wikiquote", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikiquote.org/wiki/Category:Title", - "#category": ("wikimedia", "wikiquote", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikiquote.org/wiki/Title", + "#category": ("wikimedia", "wikiquote", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikiquote.org/wiki/Category:Title", + "#category": ("wikimedia", "wikiquote", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikisource.py b/test/results/wikisource.py index 0ac1bb0fa..1a172b3af 100644 --- a/test/results/wikisource.py +++ b/test/results/wikisource.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikisource.org/wiki/Title", - "#category": ("wikimedia", "wikisource", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikisource.org/wiki/Category:Title", - "#category": ("wikimedia", "wikisource", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikisource.org/wiki/Title", + "#category": ("wikimedia", "wikisource", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikisource.org/wiki/Category:Title", + "#category": ("wikimedia", "wikisource", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikispecies.py b/test/results/wikispecies.py index 26aca84b1..330f1079a 100644 --- a/test/results/wikispecies.py +++ b/test/results/wikispecies.py @@ -1,25 +1,20 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://species.wikimedia.org/wiki/Geranospiza", - "#category": ("wikimedia", "wikispecies", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#urls" : "https://upload.wikimedia.org/wikipedia/commons/0/01/Geranospiza_caerulescens.jpg", - "#sha1_content": "3a17c14b15489928e4154f826af1c42afb5a523e", -}, - -{ - "#url" : "https://species.wikimedia.org/wiki/Category:Names", - "#category": ("wikimedia", "wikispecies", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://species.wikimedia.org/wiki/Geranospiza", + "#category": ("wikimedia", "wikispecies", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#urls": "https://upload.wikimedia.org/wikipedia/commons/0/01/Geranospiza_caerulescens.jpg", + "#sha1_content": "3a17c14b15489928e4154f826af1c42afb5a523e", + }, + { + "#url": "https://species.wikimedia.org/wiki/Category:Names", + "#category": ("wikimedia", "wikispecies", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikiversity.py b/test/results/wikiversity.py index 2e64ca314..de37e1e29 100644 --- a/test/results/wikiversity.py +++ b/test/results/wikiversity.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikiversity.org/wiki/Title", - "#category": ("wikimedia", "wikiversity", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikiversity.org/wiki/Category:Title", - "#category": ("wikimedia", "wikiversity", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikiversity.org/wiki/Title", + "#category": ("wikimedia", "wikiversity", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikiversity.org/wiki/Category:Title", + "#category": ("wikimedia", "wikiversity", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikivoyage.py b/test/results/wikivoyage.py index 860e68dec..6d5ec9286 100644 --- a/test/results/wikivoyage.py +++ b/test/results/wikivoyage.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikivoyage.org/wiki/Title", - "#category": ("wikimedia", "wikivoyage", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikivoyage.org/wiki/Category:Title", - "#category": ("wikimedia", "wikivoyage", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikivoyage.org/wiki/Title", + "#category": ("wikimedia", "wikivoyage", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikivoyage.org/wiki/Category:Title", + "#category": ("wikimedia", "wikivoyage", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wiktionary.py b/test/results/wiktionary.py index 4a643ab57..d836aaf71 100644 --- a/test/results/wiktionary.py +++ b/test/results/wiktionary.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wiktionary.org/wiki/Word", - "#category": ("wikimedia", "wiktionary", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wiktionary.org/wiki/Category:Words", - "#category": ("wikimedia", "wiktionary", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wiktionary.org/wiki/Word", + "#category": ("wikimedia", "wiktionary", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wiktionary.org/wiki/Category:Words", + "#category": ("wikimedia", "wiktionary", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/windsorstore.py b/test/results/windsorstore.py index c7cfd6989..6b028ca7d 100644 --- a/test/results/windsorstore.py +++ b/test/results/windsorstore.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://www.windsorstore.com/collections/dresses-ball-gowns", - "#category": ("shopify", "windsorstore", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.windsorstore.com/collections/accessories-belts/products/rhine-buckle-dbl-o-ring-pu-strap-belt-073010158001", - "#category": ("shopify", "windsorstore", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://www.windsorstore.com/collections/dresses-ball-gowns", + "#category": ("shopify", "windsorstore", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.windsorstore.com/collections/accessories-belts/products/rhine-buckle-dbl-o-ring-pu-strap-belt-073010158001", + "#category": ("shopify", "windsorstore", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/xbooru.py b/test/results/xbooru.py index 84cd7c7c1..32ac176b1 100644 --- a/test/results/xbooru.py +++ b/test/results/xbooru.py @@ -1,46 +1,39 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v02 - __tests__ = ( -{ - "#url" : "https://xbooru.com/index.php?page=post&s=list&tags=konoyan", - "#category": ("gelbooru_v02", "xbooru", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#count" : range(28, 40), -}, - -{ - "#url" : "https://xbooru.com/index.php?page=pool&s=show&id=757", - "#category": ("gelbooru_v02", "xbooru", "pool"), - "#class" : gelbooru_v02.GelbooruV02PoolExtractor, - "#urls": ( - "https://img.xbooru.com/images/154/aeca160f8c7131f6a93033adac5416d7.jpeg", - "https://img.xbooru.com/images/278/6185a8a71547568020e45e8319c02978.jpeg", - "https://img.xbooru.com/images/524/0fc2b1e2e3cc8be259e9712ca3f48b0b.jpeg", - "https://img.xbooru.com/images/253/74412b59a60fac5040c6cfe8efe7a625.jpeg", - "https://img.xbooru.com/images/590/2eacd900958a467fb053b8a92145b55b.jpeg", - ), -}, - -{ - "#url" : "https://xbooru.com/index.php?page=favorites&s=view&id=45206", - "#category": ("gelbooru_v02", "xbooru", "favorite"), - "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, - "#count" : 4, -}, - -{ - "#url" : "https://xbooru.com/index.php?page=post&s=view&id=1025649", - "#category": ("gelbooru_v02", "xbooru", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#pattern" : r"https://img\.xbooru\.com/images/444/f3eda549ad8b9db244ac335c7406c92f\.jpeg", - "#sha1_content": "086668afd445438d491ecc11cee3ac69b4d65530", -}, - + { + "#url": "https://xbooru.com/index.php?page=post&s=list&tags=konoyan", + "#category": ("gelbooru_v02", "xbooru", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#count": range(28, 40), + }, + { + "#url": "https://xbooru.com/index.php?page=pool&s=show&id=757", + "#category": ("gelbooru_v02", "xbooru", "pool"), + "#class": gelbooru_v02.GelbooruV02PoolExtractor, + "#urls": ( + "https://img.xbooru.com/images/154/aeca160f8c7131f6a93033adac5416d7.jpeg", + "https://img.xbooru.com/images/278/6185a8a71547568020e45e8319c02978.jpeg", + "https://img.xbooru.com/images/524/0fc2b1e2e3cc8be259e9712ca3f48b0b.jpeg", + "https://img.xbooru.com/images/253/74412b59a60fac5040c6cfe8efe7a625.jpeg", + "https://img.xbooru.com/images/590/2eacd900958a467fb053b8a92145b55b.jpeg", + ), + }, + { + "#url": "https://xbooru.com/index.php?page=favorites&s=view&id=45206", + "#category": ("gelbooru_v02", "xbooru", "favorite"), + "#class": gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count": 4, + }, + { + "#url": "https://xbooru.com/index.php?page=post&s=view&id=1025649", + "#category": ("gelbooru_v02", "xbooru", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#pattern": r"https://img\.xbooru\.com/images/444/f3eda549ad8b9db244ac335c7406c92f\.jpeg", + "#sha1_content": "086668afd445438d491ecc11cee3ac69b4d65530", + }, ) diff --git a/test/results/xhamster.py b/test/results/xhamster.py index 72634e187..b84714a20 100644 --- a/test/results/xhamster.py +++ b/test/results/xhamster.py @@ -1,122 +1,106 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import xhamster - __tests__ = ( -{ - "#url" : "https://xhamster.com/photos/gallery/take-me-to-the-carwash-at-digitaldesire-15860946", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, - "#pattern" : r"https://ic-ph-\w+\.xhcdn\.com/a/\w+/webp/000/\d+/\d+/\d+_1000\.jpg$", - "#count" : 19, - - "comments": int, - "count" : int, - "favorite": bool, - "id" : int, - "num" : int, - "height" : int, - "width" : int, - "imageURL": str, - "pageURL" : str, - "thumbURL": str, - "gallery" : { - "date" : "dt:2022-02-02 06:30:09", - "description": "Alina Henessy loves to wash her car, and we love seeing every inch of her gorgeous body. More at DigitalDesire.com", - "dislikes" : int, - "id" : 15860946, - "likes" : int, - "tags" : [ - "Babe", - "Public Nudity", - "Take", - "Taking", - "Masturbation", - "Take Me", - ], - "thumbnail" : str, - "title" : "Take me to the carwash at DigitalDesire", - "views" : range(100000, 200000), - + { + "#url": "https://xhamster.com/photos/gallery/take-me-to-the-carwash-at-digitaldesire-15860946", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + "#pattern": r"https://ic-ph-\w+\.xhcdn\.com/a/\w+/webp/000/\d+/\d+/\d+_1000\.jpg$", + "#count": 19, + "comments": int, + "count": int, + "favorite": bool, + "id": int, + "num": int, + "height": int, + "width": int, + "imageURL": str, + "pageURL": str, + "thumbURL": str, + "gallery": { + "date": "dt:2022-02-02 06:30:09", + "description": "Alina Henessy loves to wash her car, and we love seeing every inch of her gorgeous body. More at DigitalDesire.com", + "dislikes": int, + "id": 15860946, + "likes": int, + "tags": [ + "Babe", + "Public Nudity", + "Take", + "Taking", + "Masturbation", + "Take Me", + ], + "thumbnail": str, + "title": "Take me to the carwash at DigitalDesire", + "views": range(100000, 200000), + }, + "user": { + "id": 4741860, + "name": "DaringSex", + "retired": False, + "subscribers": range(25000, 50000), + "url": "https://xhamster.com/users/daringsex", + "verified": False, + }, }, - "user" : { - "id" : 4741860, - "name" : "DaringSex", - "retired" : False, - "subscribers": range(25000, 50000), - "url" : "https://xhamster.com/users/daringsex", - "verified" : False, + { + "#url": "https://jp.xhamster2.com/photos/gallery/take-me-to-the-carwash-at-digitaldesire-15860946", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + "#pattern": r"https://ic-ph-\w+\.xhcdn\.com/a/\w+/webp/000/\d+/\d+/\d+_1000\.jpg$", + "#count": 19, + }, + { + "#url": "https://xhamster.com/photos/gallery/make-the-world-better-11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://xhamster.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://xhamster.one/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://xhamster.desi/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://xhamster2.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://en.xhamster.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://xhamster.porncache.net/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://xhamster.com/users/daringsex/photos", + "#category": ("", "xhamster", "user"), + "#class": xhamster.XhamsterUserExtractor, + "#pattern": xhamster.XhamsterGalleryExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://xhamster.com/users/nickname68", + "#category": ("", "xhamster", "user"), + "#class": xhamster.XhamsterUserExtractor, }, -}, - -{ - "#url" : "https://jp.xhamster2.com/photos/gallery/take-me-to-the-carwash-at-digitaldesire-15860946", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, - "#pattern" : r"https://ic-ph-\w+\.xhcdn\.com/a/\w+/webp/000/\d+/\d+/\d+_1000\.jpg$", - "#count" : 19, -}, - -{ - "#url" : "https://xhamster.com/photos/gallery/make-the-world-better-11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://xhamster.com/photos/gallery/11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://xhamster.one/photos/gallery/11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://xhamster.desi/photos/gallery/11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://xhamster2.com/photos/gallery/11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://en.xhamster.com/photos/gallery/11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://xhamster.porncache.net/photos/gallery/11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://xhamster.com/users/daringsex/photos", - "#category": ("", "xhamster", "user"), - "#class" : xhamster.XhamsterUserExtractor, - "#pattern" : xhamster.XhamsterGalleryExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://xhamster.com/users/nickname68", - "#category": ("", "xhamster", "user"), - "#class" : xhamster.XhamsterUserExtractor, -}, - ) diff --git a/test/results/xvideos.py b/test/results/xvideos.py index 7cfa1dc01..45b064cb3 100644 --- a/test/results/xvideos.py +++ b/test/results/xvideos.py @@ -1,83 +1,70 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import xvideos - __tests__ = ( -{ - "#url" : "https://www.xvideos.com/profiles/pervertedcouple/photos/751031", - "#category": ("", "xvideos", "gallery"), - "#class" : xvideos.XvideosGalleryExtractor, - "#pattern" : r"https://profile-pics-cdn\d+\.xvideos-cdn\.com/[^/]+\,\d+/videos/profiles/galleries/84/ca/37/pervertedcouple/gal751031/pic_\d+_big\.jpg", - "#count" : 8, - - "gallery": { - "id" : 751031, - "title": "Random Stuff", - "tags" : list, + { + "#url": "https://www.xvideos.com/profiles/pervertedcouple/photos/751031", + "#category": ("", "xvideos", "gallery"), + "#class": xvideos.XvideosGalleryExtractor, + "#pattern": r"https://profile-pics-cdn\d+\.xvideos-cdn\.com/[^/]+\,\d+/videos/profiles/galleries/84/ca/37/pervertedcouple/gal751031/pic_\d+_big\.jpg", + "#count": 8, + "gallery": { + "id": 751031, + "title": "Random Stuff", + "tags": list, + }, + "user": { + "id": 20245371, + "name": "pervertedcouple", + "display": "Pervertedcouple", + "sex": "Woman", + "description": str, + }, }, - "user" : { - "id" : 20245371, - "name" : "pervertedcouple", - "display" : "Pervertedcouple", - "sex" : "Woman", - "description": str, + { + "#url": "https://www.xvideos.com/amateur-channels/pervertedcouple/photos/12", + "#category": ("", "xvideos", "gallery"), + "#class": xvideos.XvideosGalleryExtractor, + }, + { + "#url": "https://www.xvideos.com/model-channels/pervertedcouple/photos/12", + "#category": ("", "xvideos", "gallery"), + "#class": xvideos.XvideosGalleryExtractor, + }, + { + "#url": "https://www.xvideos.com/channels/pervertedcouple/photos/12", + "#comment": "/channels/ URL (#5244)", + "#category": ("", "xvideos", "gallery"), + "#class": xvideos.XvideosGalleryExtractor, + }, + { + "#url": "https://www.xvideos.com/profiles/pervertedcouple", + "#category": ("", "xvideos", "user"), + "#class": xvideos.XvideosUserExtractor, + "#sha1_url": "a413f3e60d6d3a2de79bd44fa3b7a9c03db4336e", + "#sha1_metadata": "335a3304941ff2e666c0201e9122819b61b34adb", + }, + { + "#url": "https://www.xvideos.com/profiles/pervertedcouple#_tabPhotos", + "#category": ("", "xvideos", "user"), + "#class": xvideos.XvideosUserExtractor, + }, + { + "#url": "https://www.xvideos.com/channels/pervertedcouple", + "#category": ("", "xvideos", "user"), + "#class": xvideos.XvideosUserExtractor, + }, + { + "#url": "https://www.xvideos.com/amateur-channels/pervertedcouple", + "#category": ("", "xvideos", "user"), + "#class": xvideos.XvideosUserExtractor, + }, + { + "#url": "https://www.xvideos.com/model-channels/pervertedcouple", + "#category": ("", "xvideos", "user"), + "#class": xvideos.XvideosUserExtractor, }, -}, - -{ - "#url" : "https://www.xvideos.com/amateur-channels/pervertedcouple/photos/12", - "#category": ("", "xvideos", "gallery"), - "#class" : xvideos.XvideosGalleryExtractor, -}, - -{ - "#url" : "https://www.xvideos.com/model-channels/pervertedcouple/photos/12", - "#category": ("", "xvideos", "gallery"), - "#class" : xvideos.XvideosGalleryExtractor, -}, - -{ - "#url" : "https://www.xvideos.com/channels/pervertedcouple/photos/12", - "#comment" : "/channels/ URL (#5244)", - "#category": ("", "xvideos", "gallery"), - "#class" : xvideos.XvideosGalleryExtractor, -}, - -{ - "#url" : "https://www.xvideos.com/profiles/pervertedcouple", - "#category": ("", "xvideos", "user"), - "#class" : xvideos.XvideosUserExtractor, - "#sha1_url" : "a413f3e60d6d3a2de79bd44fa3b7a9c03db4336e", - "#sha1_metadata": "335a3304941ff2e666c0201e9122819b61b34adb", -}, - -{ - "#url" : "https://www.xvideos.com/profiles/pervertedcouple#_tabPhotos", - "#category": ("", "xvideos", "user"), - "#class" : xvideos.XvideosUserExtractor, -}, - -{ - "#url" : "https://www.xvideos.com/channels/pervertedcouple", - "#category": ("", "xvideos", "user"), - "#class" : xvideos.XvideosUserExtractor, -}, - -{ - "#url" : "https://www.xvideos.com/amateur-channels/pervertedcouple", - "#category": ("", "xvideos", "user"), - "#class" : xvideos.XvideosUserExtractor, -}, - -{ - "#url" : "https://www.xvideos.com/model-channels/pervertedcouple", - "#category": ("", "xvideos", "user"), - "#class" : xvideos.XvideosUserExtractor, -}, - ) diff --git a/test/results/yandere.py b/test/results/yandere.py index 74194bb4b..7220a9cbd 100644 --- a/test/results/yandere.py +++ b/test/results/yandere.py @@ -1,99 +1,85 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import moebooru - __tests__ = ( -{ - "#url" : "https://yande.re/post/show/51824", - "#category": ("moebooru", "yandere", "post"), - "#class" : moebooru.MoebooruPostExtractor, - "#options" : {"tags": True}, - "#sha1_content": "59201811c728096b2d95ce6896fd0009235fe683", - - "tags_artist" : "sasaki_tamaru", - "tags_circle" : "softhouse_chara", - "tags_copyright": "ouzoku", - "tags_general" : str, -}, - -{ - "#url" : "https://yande.re/post/show/993156", - "#category": ("moebooru", "yandere", "post"), - "#class" : moebooru.MoebooruPostExtractor, - "#options" : {"notes": True}, - "#sha1_content": "fed722bd90f48de41ec163692befc701056e2b1e", - - "notes": [ - { - "id" : 7096, - "x" : 90, - "y" : 626, - "width" : 283, - "height": 529, - "body" : "Please keep this as a secret for me!!", - }, - { - "id" : 7095, - "x" : 900, - "y" : 438, - "width" : 314, - "height": 588, - "body" : "The facts that I love playing games", + { + "#url": "https://yande.re/post/show/51824", + "#category": ("moebooru", "yandere", "post"), + "#class": moebooru.MoebooruPostExtractor, + "#options": {"tags": True}, + "#sha1_content": "59201811c728096b2d95ce6896fd0009235fe683", + "tags_artist": "sasaki_tamaru", + "tags_circle": "softhouse_chara", + "tags_copyright": "ouzoku", + "tags_general": str, + }, + { + "#url": "https://yande.re/post/show/993156", + "#category": ("moebooru", "yandere", "post"), + "#class": moebooru.MoebooruPostExtractor, + "#options": {"notes": True}, + "#sha1_content": "fed722bd90f48de41ec163692befc701056e2b1e", + "notes": [ + { + "id": 7096, + "x": 90, + "y": 626, + "width": 283, + "height": 529, + "body": "Please keep this as a secret for me!!", + }, + { + "id": 7095, + "x": 900, + "y": 438, + "width": 314, + "height": 588, + "body": "The facts that I love playing games", + }, + ], + }, + { + "#url": "https://yande.re/post?tags=ouzoku+armor", + "#category": ("moebooru", "yandere", "tag"), + "#class": moebooru.MoebooruTagExtractor, + "#sha1_content": "59201811c728096b2d95ce6896fd0009235fe683", + }, + { + "#url": "https://yande.re/pool/show/318", + "#category": ("moebooru", "yandere", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + "#sha1_content": "2a35b9d6edecce11cc2918c6dce4de2198342b68", + }, + { + "#url": "https://yande.re/pool/show/318", + "#comment": "'metadata' option (#4646)", + "#category": ("moebooru", "yandere", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + "#options": {"metadata": True}, + "#count": 3, + "pool": { + "created_at": "2008-12-13T15:56:10.728Z", + "description": "Dengeki Hime's posts are in pool #97.", + "id": 318, + "is_public": True, + "name": "Galgame_Mag_08", + "post_count": 3, + "updated_at": "2012-03-11T14:31:00.935Z", + "user_id": 1305, }, - ], -}, - -{ - "#url" : "https://yande.re/post?tags=ouzoku+armor", - "#category": ("moebooru", "yandere", "tag"), - "#class" : moebooru.MoebooruTagExtractor, - "#sha1_content": "59201811c728096b2d95ce6896fd0009235fe683", -}, - -{ - "#url" : "https://yande.re/pool/show/318", - "#category": ("moebooru", "yandere", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, - "#sha1_content": "2a35b9d6edecce11cc2918c6dce4de2198342b68", -}, - -{ - "#url" : "https://yande.re/pool/show/318", - "#comment" : "'metadata' option (#4646)", - "#category": ("moebooru", "yandere", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, - "#options" : {"metadata": True}, - "#count" : 3, - - "pool": { - "created_at" : "2008-12-13T15:56:10.728Z", - "description": "Dengeki Hime's posts are in pool #97.", - "id" : 318, - "is_public" : True, - "name" : "Galgame_Mag_08", - "post_count" : 3, - "updated_at" : "2012-03-11T14:31:00.935Z", - "user_id" : 1305, }, - -}, - -{ - "#url" : "https://yande.re/post/popular_by_month?month=6&year=2014", - "#category": ("moebooru", "yandere", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, - "#count" : 40, -}, - -{ - "#url" : "https://yande.re/post/popular_recent", - "#category": ("moebooru", "yandere", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, -}, - + { + "#url": "https://yande.re/post/popular_by_month?month=6&year=2014", + "#category": ("moebooru", "yandere", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + "#count": 40, + }, + { + "#url": "https://yande.re/post/popular_recent", + "#category": ("moebooru", "yandere", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + }, ) diff --git a/test/results/ytdl.py b/test/results/ytdl.py index 1aecee91f..e65171e23 100644 --- a/test/results/ytdl.py +++ b/test/results/ytdl.py @@ -1,17 +1,13 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import ytdl - __tests__ = ( -{ - "#url" : "ytdl:https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9", - "#category": ("", "ytdl", "Youtube"), - "#class" : ytdl.YoutubeDLExtractor, -}, - + { + "#url": "ytdl:https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9", + "#category": ("", "ytdl", "Youtube"), + "#class": ytdl.YoutubeDLExtractor, + }, ) diff --git a/test/results/zerochan.py b/test/results/zerochan.py index ec8eddf61..09e31aecf 100644 --- a/test/results/zerochan.py +++ b/test/results/zerochan.py @@ -1,191 +1,176 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import zerochan - __tests__ = ( -{ - "#url" : "https://www.zerochan.net/Perth+%28Kantai+Collection%29", - "#category": ("booru", "zerochan", "tag"), - "#class" : zerochan.ZerochanTagExtractor, - "#pattern" : r"https://static\.zerochan\.net/\.full\.\d+\.jpg", - "#count" : "> 50", - - "extension" : r"jpg", - "file_url" : r"re:https://static\.zerochan\.net/\.full\.\d+\.jpg", - "filename" : r"re:\.full\.\d+", - "height" : int, - "id" : int, - "search_tags": "Perth (Kantai Collection)", - "tag" : r"re:(Perth \(Kantai Collection\)|Kantai Collection)", - "tags" : list, - "width" : int, -}, - -{ - "#url" : "https://www.zerochan.net/Perth+%28Kantai+Collection%29", - "#category": ("booru", "zerochan", "tag"), - "#class" : zerochan.ZerochanTagExtractor, - "#options" : {"pagination": "html"}, - "#pattern" : r"https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", - "#count" : "> 45", - - "extension" : r"re:jpg|png", - "file_url" : r"re:https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", - "filename" : r"re:(Perth\.\(Kantai\.Collection\)|Kantai\.Collection)\.full\.\d+", - "height" : r"re:^\d+$", - "id" : r"re:^\d+$", - "name" : r"re:(Perth \(Kantai Collection\)|Kantai Collection)", - "search_tags": "Perth (Kantai Collection)", - "size" : r"re:^\d+k$", - "width" : r"re:^\d+$", -}, - -{ - "#url" : "https://www.zerochan.net/2920445", - "#category": ("booru", "zerochan", "image"), - "#class" : zerochan.ZerochanImageExtractor, - "#pattern" : r"https://static\.zerochan\.net/Perth\.%28Kantai\.Collection%29\.full.2920445\.jpg", - "#auth" : True, - - "author" : "YeFan 葉凡", - "date" : "dt:2020-04-24 21:33:44", - "file_url": "https://static.zerochan.net/Perth.%28Kantai.Collection%29.full.2920445.jpg", - "filename": "Perth.(Kantai.Collection).full.2920445", - "height" : 1366, - "id" : 2920445, - "path" : [ - "Kantai Collection", - "Perth (Kantai Collection)", - ], - "size" : 1975296, - "source" : "", - "tags" : [ - "Mangaka:YeFan 葉凡", - "Game:Kantai Collection", - "Character:Perth (Kantai Collection)", - "Theme:Blonde Hair", - "Theme:Braids", - "Theme:Coat", - "Theme:Female", - "Theme:Firefighter Outfit", - "Theme:Group", - "Theme:Long Sleeves", - "Theme:Personification", - "Theme:Pins", - "Theme:Ribbon", - "Theme:Short Hair", - "Theme:Top", - ], - "uploader": "YukinoTokisaki", - "width" : 1920, -}, - -{ - "#url" : "https://www.zerochan.net/2920445", - "#category": ("booru", "zerochan", "image"), - "#class" : zerochan.ZerochanImageExtractor, - "#pattern" : r"https://static\.zerochan\.net/Perth\.%28Kantai\.Collection%29\.full.2920445\.jpg", - "#auth" : False, - - "author" : "YeFan 葉凡", - "date" : "dt:2020-04-24 21:33:44", - "file_url": "https://static.zerochan.net/Perth.%28Kantai.Collection%29.full.2920445.jpg", - "filename": "Perth.(Kantai.Collection).full.2920445", - "height" : 1366, - "id" : 2920445, - "path" : [ - "Kantai Collection", - "Perth (Kantai Collection)", - ], - "size" : 1975296, - "source" : "", - "tags" : [ - "Mangaka:YeFan 葉凡", - "Game:Kantai Collection", - "Character:Perth (Kantai Collection)", - "Theme:Firefighter Outfit", - "Theme:Pins", - ], - "uploader": "YukinoTokisaki", - "width" : 1920, -}, - -{ - "#url" : "https://www.zerochan.net/4233756", - "#category": ("booru", "zerochan", "image"), - "#class" : zerochan.ZerochanImageExtractor, - "#urls" : "https://static.zerochan.net/DRAGON.BALL.full.4233756.jpg", - "#options" : {"tags": True}, - - "author" : "Raydash", - "date" : "dt:2024-07-23 00:10:51", - "extension": "jpg", - "file_url" : "https://static.zerochan.net/DRAGON.BALL.full.4233756.jpg", - "filename" : "DRAGON.BALL.full.4233756", - "height" : 1125, - "id" : 4233756, - "path" : [ - "Toriyama Akira", - "DRAGON BALL", - ], - "size" : 136192, - "source": "https://x.com/Raydash30/status/1766012730769862774", - "tags" : [ - "Mangaka:Raydash", - "Series:DRAGON BALL", - "Series:DRAGON BALL Z", - "Character:Piccolo", - "Character:Son Gohan", - "Theme:Duo", - "Theme:Green Skin", - "Theme:Male", - "Theme:Male Focus", - "Theme:Two Males", - "Source:Fanart", - "Source:Fanart from X (Twitter)", - "Source:X (Twitter)", - ], - "tags_character": [ - "Piccolo", - "Son Gohan", - ], - "tags_mangaka" : [ - "Raydash", - ], - "tags_series" : [ - "DRAGON BALL", - "DRAGON BALL Z", - ], - "tags_source" : [ - "Fanart", - "Fanart from X (Twitter)", - "X (Twitter)", - ], - "tags_theme" : [ - "Duo", - "Green Skin", - "Male", - "Male Focus", - "Two Males", - ], - "uploader" : "menotbug", - "width" : 750, -}, - -{ - "#url" : "https://www.zerochan.net/1395035", - "#comment" : "Invalid control character '\r' in 'source' field (#5892)", - "#category": ("booru", "zerochan", "image"), - "#class" : zerochan.ZerochanImageExtractor, - "#auth" : True, - "#options" : {"metadata": True}, - - "source": "http://www.youtube.com/watch?v=0vodqkGPxt8", -}, - + { + "#url": "https://www.zerochan.net/Perth+%28Kantai+Collection%29", + "#category": ("booru", "zerochan", "tag"), + "#class": zerochan.ZerochanTagExtractor, + "#pattern": r"https://static\.zerochan\.net/\.full\.\d+\.jpg", + "#count": "> 50", + "extension": r"jpg", + "file_url": r"re:https://static\.zerochan\.net/\.full\.\d+\.jpg", + "filename": r"re:\.full\.\d+", + "height": int, + "id": int, + "search_tags": "Perth (Kantai Collection)", + "tag": r"re:(Perth \(Kantai Collection\)|Kantai Collection)", + "tags": list, + "width": int, + }, + { + "#url": "https://www.zerochan.net/Perth+%28Kantai+Collection%29", + "#category": ("booru", "zerochan", "tag"), + "#class": zerochan.ZerochanTagExtractor, + "#options": {"pagination": "html"}, + "#pattern": r"https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", + "#count": "> 45", + "extension": r"re:jpg|png", + "file_url": r"re:https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", + "filename": r"re:(Perth\.\(Kantai\.Collection\)|Kantai\.Collection)\.full\.\d+", + "height": r"re:^\d+$", + "id": r"re:^\d+$", + "name": r"re:(Perth \(Kantai Collection\)|Kantai Collection)", + "search_tags": "Perth (Kantai Collection)", + "size": r"re:^\d+k$", + "width": r"re:^\d+$", + }, + { + "#url": "https://www.zerochan.net/2920445", + "#category": ("booru", "zerochan", "image"), + "#class": zerochan.ZerochanImageExtractor, + "#pattern": r"https://static\.zerochan\.net/Perth\.%28Kantai\.Collection%29\.full.2920445\.jpg", + "#auth": True, + "author": "YeFan 葉凡", + "date": "dt:2020-04-24 21:33:44", + "file_url": "https://static.zerochan.net/Perth.%28Kantai.Collection%29.full.2920445.jpg", + "filename": "Perth.(Kantai.Collection).full.2920445", + "height": 1366, + "id": 2920445, + "path": [ + "Kantai Collection", + "Perth (Kantai Collection)", + ], + "size": 1975296, + "source": "", + "tags": [ + "Mangaka:YeFan 葉凡", + "Game:Kantai Collection", + "Character:Perth (Kantai Collection)", + "Theme:Blonde Hair", + "Theme:Braids", + "Theme:Coat", + "Theme:Female", + "Theme:Firefighter Outfit", + "Theme:Group", + "Theme:Long Sleeves", + "Theme:Personification", + "Theme:Pins", + "Theme:Ribbon", + "Theme:Short Hair", + "Theme:Top", + ], + "uploader": "YukinoTokisaki", + "width": 1920, + }, + { + "#url": "https://www.zerochan.net/2920445", + "#category": ("booru", "zerochan", "image"), + "#class": zerochan.ZerochanImageExtractor, + "#pattern": r"https://static\.zerochan\.net/Perth\.%28Kantai\.Collection%29\.full.2920445\.jpg", + "#auth": False, + "author": "YeFan 葉凡", + "date": "dt:2020-04-24 21:33:44", + "file_url": "https://static.zerochan.net/Perth.%28Kantai.Collection%29.full.2920445.jpg", + "filename": "Perth.(Kantai.Collection).full.2920445", + "height": 1366, + "id": 2920445, + "path": [ + "Kantai Collection", + "Perth (Kantai Collection)", + ], + "size": 1975296, + "source": "", + "tags": [ + "Mangaka:YeFan 葉凡", + "Game:Kantai Collection", + "Character:Perth (Kantai Collection)", + "Theme:Firefighter Outfit", + "Theme:Pins", + ], + "uploader": "YukinoTokisaki", + "width": 1920, + }, + { + "#url": "https://www.zerochan.net/4233756", + "#category": ("booru", "zerochan", "image"), + "#class": zerochan.ZerochanImageExtractor, + "#urls": "https://static.zerochan.net/DRAGON.BALL.full.4233756.jpg", + "#options": {"tags": True}, + "author": "Raydash", + "date": "dt:2024-07-23 00:10:51", + "extension": "jpg", + "file_url": "https://static.zerochan.net/DRAGON.BALL.full.4233756.jpg", + "filename": "DRAGON.BALL.full.4233756", + "height": 1125, + "id": 4233756, + "path": [ + "Toriyama Akira", + "DRAGON BALL", + ], + "size": 136192, + "source": "https://x.com/Raydash30/status/1766012730769862774", + "tags": [ + "Mangaka:Raydash", + "Series:DRAGON BALL", + "Series:DRAGON BALL Z", + "Character:Piccolo", + "Character:Son Gohan", + "Theme:Duo", + "Theme:Green Skin", + "Theme:Male", + "Theme:Male Focus", + "Theme:Two Males", + "Source:Fanart", + "Source:Fanart from X (Twitter)", + "Source:X (Twitter)", + ], + "tags_character": [ + "Piccolo", + "Son Gohan", + ], + "tags_mangaka": [ + "Raydash", + ], + "tags_series": [ + "DRAGON BALL", + "DRAGON BALL Z", + ], + "tags_source": [ + "Fanart", + "Fanart from X (Twitter)", + "X (Twitter)", + ], + "tags_theme": [ + "Duo", + "Green Skin", + "Male", + "Male Focus", + "Two Males", + ], + "uploader": "menotbug", + "width": 750, + }, + { + "#url": "https://www.zerochan.net/1395035", + "#comment": "Invalid control character '\r' in 'source' field (#5892)", + "#category": ("booru", "zerochan", "image"), + "#class": zerochan.ZerochanImageExtractor, + "#auth": True, + "#options": {"metadata": True}, + "source": "http://www.youtube.com/watch?v=0vodqkGPxt8", + }, ) diff --git a/test/results/zzup.py b/test/results/zzup.py index 322a9601b..c60a51525 100644 --- a/test/results/zzup.py +++ b/test/results/zzup.py @@ -1,36 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import zzup - __tests__ = ( -{ - "#url" : "https://zzup.com/content/NjM=/MetArt_20080206_viki_c_sensazioni_by_ingret/OTE=/index.html", - "#category": ("", "zzup", "gallery"), - "#class" : zzup.ZzupGalleryExtractor, - "#pattern" : r"https://zzup\.com/MjAxNjc3OTIyMjE5Nzk=/showimage/zzup-8769086487/image00\d\d\d-5896498214-1-9689595623/MetArt-20080206_viki_c_sensazioni_by_ingret/9879560327/zzup.com.jpg", - - "slug" : "MetArt_20080206_viki_c_sensazioni_by_ingret", - "title" : "MetArt 20080206 viki c sensazioni by ingret", - "num" : int, - "count" : 135, -}, - -{ - "#url" : "https://zzup.com/content/MTc2MDYxMw==/Courtesan/NDA=/page-1.html", - "#category": ("", "zzup", "gallery"), - "#class" : zzup.ZzupGalleryExtractor, - "#pattern" : r"https://zzup.com/MjAxNjc3OTIyMjE5Nzk=/showimage/zzup-8769086487/image000\d\d-5896498214-40-9689595623/Courtesan/9879560327/zzup.com.jpg", -}, - -{ - "#url" : "https://up.zzup.com/viewalbum/TE9MQUxVWlogLSBMYWxsaSAtIFdhcm0gYW5kIENvenk=/NTM0MTk=/OTgz/index.html", - "#category": ("", "zzup", "gallery"), - "#class" : zzup.ZzupGalleryExtractor, -}, - + { + "#url": "https://zzup.com/content/NjM=/MetArt_20080206_viki_c_sensazioni_by_ingret/OTE=/index.html", + "#category": ("", "zzup", "gallery"), + "#class": zzup.ZzupGalleryExtractor, + "#pattern": r"https://zzup\.com/MjAxNjc3OTIyMjE5Nzk=/showimage/zzup-8769086487/image00\d\d\d-5896498214-1-9689595623/MetArt-20080206_viki_c_sensazioni_by_ingret/9879560327/zzup.com.jpg", + "slug": "MetArt_20080206_viki_c_sensazioni_by_ingret", + "title": "MetArt 20080206 viki c sensazioni by ingret", + "num": int, + "count": 135, + }, + { + "#url": "https://zzup.com/content/MTc2MDYxMw==/Courtesan/NDA=/page-1.html", + "#category": ("", "zzup", "gallery"), + "#class": zzup.ZzupGalleryExtractor, + "#pattern": r"https://zzup.com/MjAxNjc3OTIyMjE5Nzk=/showimage/zzup-8769086487/image000\d\d-5896498214-40-9689595623/Courtesan/9879560327/zzup.com.jpg", + }, + { + "#url": "https://up.zzup.com/viewalbum/TE9MQUxVWlogLSBMYWxsaSAtIFdhcm0gYW5kIENvenk=/NTM0MTk=/OTgz/index.html", + "#category": ("", "zzup", "gallery"), + "#class": zzup.ZzupGalleryExtractor, + }, ) diff --git a/test/test_cache.py b/test/test_cache.py index 9951ef203..dd639e324 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2020 Mike Fährmann # @@ -9,17 +8,17 @@ import os import sys +import tempfile import unittest from unittest.mock import patch -import tempfile - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import config, util # noqa E402 +from gallery_dl import config # noqa: E402 dbpath = tempfile.mkstemp()[1] config.set(("cache",), "file", dbpath) -from gallery_dl import cache # noqa E402 +from gallery_dl import cache # noqa: E402 + cache._init() @@ -28,9 +27,7 @@ class TestCache(unittest.TestCase): - def test_decorator(self): - @cache.memcache() def mc1(): pass @@ -50,7 +47,7 @@ def dbc(): def test_keyarg_mem_simple(self): @cache.memcache(keyarg=2) def ka(a, b, c): - return a+b+c + return a + b + c self.assertEqual(ka(1, 1, 1), 3) self.assertEqual(ka(2, 2, 2), 6) @@ -63,7 +60,7 @@ def ka(a, b, c): def test_keyarg_mem(self): @cache.memcache(keyarg=2, maxage=10) def ka(a, b, c): - return a+b+c + return a + b + c self.assertEqual(ka(1, 1, 1), 3) self.assertEqual(ka(2, 2, 2), 6) @@ -76,7 +73,7 @@ def ka(a, b, c): def test_keyarg_db(self): @cache.cache(keyarg=2, maxage=10) def ka(a, b, c): - return a+b+c + return a + b + c self.assertEqual(ka(1, 1, 1), 3) self.assertEqual(ka(2, 2, 2), 6) @@ -89,7 +86,7 @@ def ka(a, b, c): def test_expires_mem(self): @cache.memcache(maxage=2) def ex(a, b, c): - return a+b+c + return a + b + c with patch("time.time") as tmock: tmock.return_value = 0.001 @@ -112,7 +109,7 @@ def ex(a, b, c): def test_expires_db(self): @cache.cache(maxage=2) def ex(a, b, c): - return a+b+c + return a + b + c with patch("time.time") as tmock: tmock.return_value = 0.999 @@ -135,7 +132,7 @@ def ex(a, b, c): def test_update_mem_simple(self): @cache.memcache(keyarg=0) def up(a, b, c): - return a+b+c + return a + b + c self.assertEqual(up(1, 1, 1), 3) up.update(1, 0) @@ -146,7 +143,7 @@ def up(a, b, c): def test_update_mem(self): @cache.memcache(keyarg=0, maxage=10) def up(a, b, c): - return a+b+c + return a + b + c self.assertEqual(up(1, 1, 1), 3) up.update(1, 0) @@ -157,7 +154,7 @@ def up(a, b, c): def test_update_db(self): @cache.cache(keyarg=0, maxage=10) def up(a, b, c): - return a+b+c + return a + b + c self.assertEqual(up(1, 1, 1), 3) up.update(1, 0) @@ -168,7 +165,7 @@ def up(a, b, c): def test_invalidate_mem_simple(self): @cache.memcache(keyarg=0) def inv(a, b, c): - return a+b+c + return a + b + c self.assertEqual(inv(1, 1, 1), 3) inv.invalidate(1) @@ -179,7 +176,7 @@ def inv(a, b, c): def test_invalidate_mem(self): @cache.memcache(keyarg=0, maxage=10) def inv(a, b, c): - return a+b+c + return a + b + c self.assertEqual(inv(1, 1, 1), 3) inv.invalidate(1) @@ -190,7 +187,7 @@ def inv(a, b, c): def test_invalidate_db(self): @cache.cache(keyarg=0, maxage=10) def inv(a, b, c): - return a+b+c + return a + b + c self.assertEqual(inv(1, 1, 1), 3) inv.invalidate(1) @@ -201,7 +198,7 @@ def inv(a, b, c): def test_database_read(self): @cache.cache(keyarg=0, maxage=10) def db(a, b, c): - return a+b+c + return a + b + c # initialize cache self.assertEqual(db(1, 1, 1), 3) diff --git a/test/test_config.py b/test/test_config.py index bbe288ff3..14fc705be 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2015-2023 Mike Fährmann # @@ -9,73 +8,74 @@ import os import sys -import unittest - import tempfile +import unittest ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOTDIR) -from gallery_dl import config, util # noqa E402 +from gallery_dl import config # noqa: E402 +from gallery_dl import util # noqa: E402 class TestConfig(unittest.TestCase): - def setUp(self): - config.set(() , "a", 1) - config.set(("b",) , "a", 2) + config.set((), "a", 1) + config.set(("b",), "a", 2) config.set(("b", "b"), "a", 3) - config.set(("b",) , "c", "text") + config.set(("b",), "c", "text") config.set(("b", "b"), "c", [8, 9]) def tearDown(self): config.clear() def test_get(self): - self.assertEqual(config.get(() , "a") , 1) - self.assertEqual(config.get(("b",) , "a") , 2) - self.assertEqual(config.get(("b", "b"), "a") , 3) + self.assertEqual(config.get((), "a"), 1) + self.assertEqual(config.get(("b",), "a"), 2) + self.assertEqual(config.get(("b", "b"), "a"), 3) - self.assertEqual(config.get(() , "c") , None) - self.assertEqual(config.get(("b",) , "c") , "text") - self.assertEqual(config.get(("b", "b"), "c") , [8, 9]) + self.assertEqual(config.get((), "c"), None) + self.assertEqual(config.get(("b",), "c"), "text") + self.assertEqual(config.get(("b", "b"), "c"), [8, 9]) - self.assertEqual(config.get(("a",) , "g") , None) - self.assertEqual(config.get(("a", "a"), "g") , None) - self.assertEqual(config.get(("e", "f"), "g") , None) + self.assertEqual(config.get(("a",), "g"), None) + self.assertEqual(config.get(("a", "a"), "g"), None) + self.assertEqual(config.get(("e", "f"), "g"), None) self.assertEqual(config.get(("e", "f"), "g", 4), 4) def test_interpolate(self): - self.assertEqual(config.interpolate(() , "a"), 1) - self.assertEqual(config.interpolate(("b",) , "a"), 1) + self.assertEqual(config.interpolate((), "a"), 1) + self.assertEqual(config.interpolate(("b",), "a"), 1) self.assertEqual(config.interpolate(("b", "b"), "a"), 1) - self.assertEqual(config.interpolate(() , "c"), None) - self.assertEqual(config.interpolate(("b",) , "c"), "text") + self.assertEqual(config.interpolate((), "c"), None) + self.assertEqual(config.interpolate(("b",), "c"), "text") self.assertEqual(config.interpolate(("b", "b"), "c"), [8, 9]) - self.assertEqual(config.interpolate(("a",) , "g") , None) - self.assertEqual(config.interpolate(("a", "a"), "g") , None) - self.assertEqual(config.interpolate(("e", "f"), "g") , None) + self.assertEqual(config.interpolate(("a",), "g"), None) + self.assertEqual(config.interpolate(("a", "a"), "g"), None) + self.assertEqual(config.interpolate(("e", "f"), "g"), None) self.assertEqual(config.interpolate(("e", "f"), "g", 4), 4) - self.assertEqual(config.interpolate(("b",), "d", 1) , 1) - self.assertEqual(config.interpolate(("d",), "d", 1) , 1) - config.set(() , "d", 2) - self.assertEqual(config.interpolate(("b",), "d", 1) , 2) - self.assertEqual(config.interpolate(("d",), "d", 1) , 2) + self.assertEqual(config.interpolate(("b",), "d", 1), 1) + self.assertEqual(config.interpolate(("d",), "d", 1), 1) + config.set((), "d", 2) + self.assertEqual(config.interpolate(("b",), "d", 1), 2) + self.assertEqual(config.interpolate(("d",), "d", 1), 2) config.set(("b",), "d", 3) - self.assertEqual(config.interpolate(("b",), "d", 1) , 2) - self.assertEqual(config.interpolate(("d",), "d", 1) , 2) + self.assertEqual(config.interpolate(("b",), "d", 1), 2) + self.assertEqual(config.interpolate(("d",), "d", 1), 2) def test_interpolate_common(self): - def lookup(): return config.interpolate_common( - ("Z1", "Z2"), ( + ("Z1", "Z2"), + ( ("A1", "A2"), ("B1",), ("C1", "C2", "C3"), - ), "KEY", "DEFAULT", + ), + "KEY", + "DEFAULT", ) def test(path, value, expected=None): @@ -98,48 +98,42 @@ def test(path, value, expected=None): def test_accumulate(self): self.assertEqual(config.accumulate((), "l"), []) - config.set(() , "l", [5, 6]) - config.set(("c",) , "l", [3, 4]) + config.set((), "l", [5, 6]) + config.set(("c",), "l", [3, 4]) config.set(("c", "c"), "l", [1, 2]) - self.assertEqual( - config.accumulate((), "l") , [5, 6]) - self.assertEqual( - config.accumulate(("c",), "l") , [3, 4, 5, 6]) - self.assertEqual( - config.accumulate(("c", "c"), "l"), [1, 2, 3, 4, 5, 6]) + self.assertEqual(config.accumulate((), "l"), [5, 6]) + self.assertEqual(config.accumulate(("c",), "l"), [3, 4, 5, 6]) + self.assertEqual(config.accumulate(("c", "c"), "l"), [1, 2, 3, 4, 5, 6]) config.set(("c",), "l", None) config.unset(("c", "c"), "l") - self.assertEqual( - config.accumulate((), "l") , [5, 6]) - self.assertEqual( - config.accumulate(("c",), "l") , [5, 6]) - self.assertEqual( - config.accumulate(("c", "c"), "l"), [5, 6]) + self.assertEqual(config.accumulate((), "l"), [5, 6]) + self.assertEqual(config.accumulate(("c",), "l"), [5, 6]) + self.assertEqual(config.accumulate(("c", "c"), "l"), [5, 6]) def test_set(self): - config.set(() , "c", [1, 2, 3]) - config.set(("b",) , "c", [1, 2, 3]) + config.set((), "c", [1, 2, 3]) + config.set(("b",), "c", [1, 2, 3]) config.set(("e", "f"), "g", value=234) - self.assertEqual(config.get(() , "c"), [1, 2, 3]) - self.assertEqual(config.get(("b",) , "c"), [1, 2, 3]) + self.assertEqual(config.get((), "c"), [1, 2, 3]) + self.assertEqual(config.get(("b",), "c"), [1, 2, 3]) self.assertEqual(config.get(("e", "f"), "g"), 234) def test_setdefault(self): - config.setdefault(() , "c", [1, 2, 3]) - config.setdefault(("b",) , "c", [1, 2, 3]) + config.setdefault((), "c", [1, 2, 3]) + config.setdefault(("b",), "c", [1, 2, 3]) config.setdefault(("e", "f"), "g", value=234) - self.assertEqual(config.get(() , "c"), [1, 2, 3]) - self.assertEqual(config.get(("b",) , "c"), "text") + self.assertEqual(config.get((), "c"), [1, 2, 3]) + self.assertEqual(config.get(("b",), "c"), "text") self.assertEqual(config.get(("e", "f"), "g"), 234) def test_unset(self): - config.unset(() , "a") + config.unset((), "a") config.unset(("b",), "c") config.unset(("a",), "d") config.unset(("b",), "d") config.unset(("c",), "d") - self.assertEqual(config.get(() , "a"), None) + self.assertEqual(config.get((), "a"), None) self.assertEqual(config.get(("b",), "a"), 2) self.assertEqual(config.get(("b",), "c"), None) self.assertEqual(config.get(("a",), "d"), None) @@ -148,18 +142,18 @@ def test_unset(self): def test_apply(self): options = ( - (("b",) , "c", [1, 2, 3]), + (("b",), "c", [1, 2, 3]), (("e", "f"), "g", 234), ) - self.assertEqual(config.get(("b",) , "c"), "text") + self.assertEqual(config.get(("b",), "c"), "text") self.assertEqual(config.get(("e", "f"), "g"), None) with config.apply(options): - self.assertEqual(config.get(("b",) , "c"), [1, 2, 3]) + self.assertEqual(config.get(("b",), "c"), [1, 2, 3]) self.assertEqual(config.get(("e", "f"), "g"), 234) - self.assertEqual(config.get(("b",) , "c"), "text") + self.assertEqual(config.get(("b",), "c"), "text") self.assertEqual(config.get(("e", "f"), "g"), None) def test_load(self): @@ -174,26 +168,25 @@ def test_load(self): config.clear() config.load((path1,)) - self.assertEqual(config.get(() , "a"), 1) + self.assertEqual(config.get((), "a"), 1) self.assertEqual(config.get(("b",), "a"), 2) self.assertEqual(config.get(("b",), "c"), "text") config.load((path2,)) - self.assertEqual(config.get(() , "a"), 7) + self.assertEqual(config.get((), "a"), 7) self.assertEqual(config.get(("b",), "a"), 8) self.assertEqual(config.get(("b",), "c"), "text") self.assertEqual(config.get(("b",), "e"), "foo") config.clear() config.load((path1, path2)) - self.assertEqual(config.get(() , "a"), 7) + self.assertEqual(config.get((), "a"), 7) self.assertEqual(config.get(("b",), "a"), 8) self.assertEqual(config.get(("b",), "c"), "text") self.assertEqual(config.get(("b",), "e"), "foo") class TestConfigFiles(unittest.TestCase): - def test_default_config(self): cfg = self._load("gallery-dl.conf") self.assertIsInstance(cfg, dict) diff --git a/test/test_cookies.py b/test/test_cookies.py index 60c83ffb1..591b15161 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2017-2023 Mike Fährmann # @@ -7,22 +6,21 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import logging import os import sys -import unittest -from unittest import mock - -import time -import logging import tempfile +import time +import unittest from os.path import join +from unittest import mock sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import config, extractor # noqa E402 +from gallery_dl import config # noqa: E402 +from gallery_dl import extractor # noqa: E402 class TestCookiejar(unittest.TestCase): - @classmethod def setUpClass(cls): cls.path = tempfile.TemporaryDirectory() @@ -51,9 +49,9 @@ def test_cookiefile(self): cookie = next(iter(cookies)) self.assertEqual(cookie.domain, ".example.org") - self.assertEqual(cookie.path , "/") - self.assertEqual(cookie.name , "NAME") - self.assertEqual(cookie.value , "VALUE") + self.assertEqual(cookie.path, "/") + self.assertEqual(cookie.name, "NAME") + self.assertEqual(cookie.value, "VALUE") def test_invalid_cookiefile(self): self._test_warning(self.invalid_cookiefile, ValueError) @@ -75,7 +73,6 @@ def _test_warning(self, filename, exc): class TestCookiedict(unittest.TestCase): - def setUp(self): self.cdict = {"NAME1": "VALUE1", "NAME2": "VALUE2"} config.set((), "cookies", self.cdict) @@ -101,16 +98,15 @@ def test_domain(self): class TestCookieLogin(unittest.TestCase): - def tearDown(self): config.clear() def test_cookie_login(self): extr_cookies = { - "exhentai" : ("ipb_member_id", "ipb_pass_hash"), + "exhentai": ("ipb_member_id", "ipb_pass_hash"), "idolcomplex": ("login", "pass_hash"), - "nijie" : ("nijie_tok",), - "horne" : ("horne_tok",), + "nijie": ("nijie_tok",), + "horne": ("horne_tok",), } for category, cookienames in extr_cookies.items(): cookies = {name: "value" for name in cookienames} @@ -122,7 +118,6 @@ def test_cookie_login(self): class TestCookieUtils(unittest.TestCase): - def test_check_cookies(self): extr = _get_extractor("test") self.assertFalse(extr.cookies, "empty") @@ -175,27 +170,29 @@ def test_check_cookies_expires(self): now = int(time.time()) log = logging.getLogger("generic") - extr.cookies.set("a", "1", expires=now-100) + extr.cookies.set("a", "1", expires=now - 100) with mock.patch.object(log, "warning") as mw: self.assertFalse(extr.cookies_check(("a",))) self.assertEqual(mw.call_count, 1) self.assertEqual(mw.call_args[0], ("Cookie '%s' has expired", "a")) - extr.cookies.set("a", "1", expires=now+100) + extr.cookies.set("a", "1", expires=now + 100) with mock.patch.object(log, "warning") as mw: self.assertTrue(extr.cookies_check(("a",))) self.assertEqual(mw.call_count, 1) - self.assertEqual(mw.call_args[0], ( - "Cookie '%s' will expire in less than %s hour%s", "a", 1, "")) + self.assertEqual( + mw.call_args[0], ("Cookie '%s' will expire in less than %s hour%s", "a", 1, "") + ) - extr.cookies.set("a", "1", expires=now+100+7200) + extr.cookies.set("a", "1", expires=now + 100 + 7200) with mock.patch.object(log, "warning") as mw: self.assertTrue(extr.cookies_check(("a",))) self.assertEqual(mw.call_count, 1) - self.assertEqual(mw.call_args[0], ( - "Cookie '%s' will expire in less than %s hour%s", "a", 3, "s")) + self.assertEqual( + mw.call_args[0], ("Cookie '%s' will expire in less than %s hour%s", "a", 3, "s") + ) - extr.cookies.set("a", "1", expires=now+100+24*3600) + extr.cookies.set("a", "1", expires=now + 100 + 24 * 3600) with mock.patch.object(log, "warning") as mw: self.assertTrue(extr.cookies_check(("a",))) self.assertEqual(mw.call_count, 0) @@ -208,11 +205,11 @@ def _get_extractor(category): URLS = { - "exhentai" : "https://exhentai.org/g/1200119/d55c44d3d0/", + "exhentai": "https://exhentai.org/g/1200119/d55c44d3d0/", "idolcomplex": "https://idol.sankakucomplex.com/post/show/1", - "nijie" : "https://nijie.info/view.php?id=1", - "horne" : "https://horne.red/view.php?id=1", - "test" : "generic:https://example.org/", + "nijie": "https://nijie.info/view.php?id=1", + "horne": "https://horne.red/view.php?id=1", + "test": "generic:https://example.org/", } diff --git a/test/test_downloader.py b/test/test_downloader.py index 35cccc402..6b0563011 100644 --- a/test/test_downloader.py +++ b/test/test_downloader.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2018-2022 Mike Fährmann # @@ -7,31 +6,34 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import os -import sys -import unittest -from unittest.mock import Mock, MagicMock, patch - -import re +import binascii +import http.server import logging +import os import os.path -import binascii +import re +import sys import tempfile import threading -import http.server - +import unittest +from unittest.mock import MagicMock +from unittest.mock import Mock +from unittest.mock import patch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import downloader, extractor, output, config, path # noqa E402 -from gallery_dl.downloader.http import MIME_TYPES, SIGNATURE_CHECKS # noqa E402 +from gallery_dl import config # noqa: E402 +from gallery_dl import downloader # noqa: E402 +from gallery_dl import extractor # noqa: E402 +from gallery_dl import output # noqa: E402 +from gallery_dl import path # noqa: E402 +from gallery_dl.downloader.http import MIME_TYPES # noqa: E402 class MockDownloaderModule(Mock): __downloader__ = "mock" -class FakeJob(): - +class FakeJob: def __init__(self): self.extractor = extractor.find("generic:https://example.org/") self.extractor.initialize() @@ -41,7 +43,6 @@ def __init__(self): class TestDownloaderModule(unittest.TestCase): - @classmethod def setUpClass(cls): # allow import of ytdl downloader module without youtube_dl installed @@ -61,24 +62,24 @@ def tearDown(self): def test_find(self): cls = downloader.find("http") self.assertEqual(cls.__name__, "HttpDownloader") - self.assertEqual(cls.scheme , "http") + self.assertEqual(cls.scheme, "http") cls = downloader.find("https") self.assertEqual(cls.__name__, "HttpDownloader") - self.assertEqual(cls.scheme , "http") + self.assertEqual(cls.scheme, "http") cls = downloader.find("text") self.assertEqual(cls.__name__, "TextDownloader") - self.assertEqual(cls.scheme , "text") + self.assertEqual(cls.scheme, "text") cls = downloader.find("ytdl") self.assertEqual(cls.__name__, "YoutubeDLDownloader") - self.assertEqual(cls.scheme , "ytdl") + self.assertEqual(cls.scheme, "ytdl") self.assertEqual(downloader.find("ftp"), None) self.assertEqual(downloader.find("foo"), None) - self.assertEqual(downloader.find(1234) , None) - self.assertEqual(downloader.find(None) , None) + self.assertEqual(downloader.find(1234), None) + self.assertEqual(downloader.find(None), None) @patch("builtins.__import__") def test_cache(self, import_module): @@ -108,7 +109,6 @@ def test_cache_https(self, import_module): class TestDownloaderBase(unittest.TestCase): - @classmethod def setUpClass(cls): cls.dir = tempfile.TemporaryDirectory() @@ -123,14 +123,14 @@ def tearDownClass(cls): @classmethod def _prepare_destination(cls, content=None, part=True, extension=None): - name = "file-{}".format(cls.fnum) + name = f"file-{cls.fnum}" cls.fnum += 1 kwdict = { - "category" : "test", + "category": "test", "subcategory": "test", - "filename" : name, - "extension" : extension, + "filename": name, + "extension": extension, } pathfmt = cls.job.pathfmt @@ -145,13 +145,12 @@ def _prepare_destination(cls, content=None, part=True, extension=None): return pathfmt - def _run_test(self, url, input, output, - extension, expected_extension=None): + def _run_test(self, url, input, output, extension, expected_extension=None): pathfmt = self._prepare_destination(input, extension=extension) success = self.downloader.download(url, pathfmt) # test successful download - self.assertTrue(success, "downloading '{}' failed".format(url)) + self.assertTrue(success, f"downloading '{url}' failed") # test content mode = "r" + ("b" if isinstance(output, bytes) else "") @@ -172,7 +171,6 @@ def _run_test(self, url, input, output, class TestHTTPDownloader(TestDownloaderBase): - @classmethod def setUpClass(cls): TestDownloaderBase.setUpClass() @@ -184,18 +182,16 @@ def setUpClass(cls): try: server = http.server.HTTPServer((host, port), HttpRequestHandler) except OSError as exc: - raise unittest.SkipTest( - "cannot spawn local HTTP server ({})".format(exc)) + raise unittest.SkipTest(f"cannot spawn local HTTP server ({exc})") host, port = server.server_address - cls.address = "http://{}:{}".format(host, port) + cls.address = f"http://{host}:{port}" threading.Thread(target=server.serve_forever, daemon=True).start() - def _run_test(self, ext, input, output, - extension, expected_extension=None): + def _run_test(self, ext, input, output, extension, expected_extension=None): TestDownloaderBase._run_test( - self, self.address + "/" + ext, input, output, - extension, expected_extension) + self, self.address + "/" + ext, input, output, extension, expected_extension + ) def tearDown(self): self.downloader.minsize = self.downloader.maxsize = None @@ -207,8 +203,8 @@ def test_http_download(self): def test_http_offset(self): self._run_test("jpg", DATA["jpg"][:123], DATA["jpg"], "jpg", "jpg") - self._run_test("png", DATA["png"][:12] , DATA["png"], "png", "png") - self._run_test("gif", DATA["gif"][:1] , DATA["gif"], "gif", "gif") + self._run_test("png", DATA["png"][:12], DATA["png"], "png", "png") + self._run_test("gif", DATA["gif"][:1], DATA["gif"], "gif", "gif") def test_http_extension(self): self._run_test("jpg", None, DATA["jpg"], None, "jpg") @@ -240,7 +236,6 @@ def test_http_filesize_max(self): class TestTextDownloader(TestDownloaderBase): - @classmethod def setUpClass(cls): TestDownloaderBase.setUpClass() @@ -257,7 +252,6 @@ def test_text_empty(self): class HttpRequestHandler(http.server.BaseHTTPRequestHandler): - def do_GET(self): try: output = DATA[self.path[1:]] @@ -274,8 +268,7 @@ def do_GET(self): match = re.match(r"bytes=(\d+)-", self.headers["Range"]) start = int(match.group(1)) - headers["Content-Range"] = "bytes {}-{}/{}".format( - start, len(output)-1, len(output)) + headers["Content-Range"] = f"bytes {start}-{len(output) - 1}/{len(output)}" output = output[start:] else: status = 200 @@ -288,19 +281,26 @@ def do_GET(self): SAMPLES = { - ("jpg" , binascii.a2b_base64( - "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB" - "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEB" - "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB" - "AQEBAQEBAQEBAQEBAQH/wAARCAABAAEDAREAAhEBAxEB/8QAFAABAAAAAAAAAAAA" - "AAAAAAAACv/EABQQAQAAAAAAAAAAAAAAAAAAAAD/xAAUAQEAAAAAAAAAAAAAAAAA" - "AAAA/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAwDAQACEQMRAD8AfwD/2Q==")), - ("png" , binascii.a2b_base64( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVQIHWP4DwAB" - "AQEANl9ngAAAAABJRU5ErkJggg==")), - ("gif" , binascii.a2b_base64( - "R0lGODdhAQABAIAAAP///////ywAAAAAAQABAAACAkQBADs=")), - ("bmp" , b"BM"), + ( + "jpg", + binascii.a2b_base64( + "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB" + "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEB" + "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB" + "AQEBAQEBAQEBAQEBAQH/wAARCAABAAEDAREAAhEBAxEB/8QAFAABAAAAAAAAAAAA" + "AAAAAAAACv/EABQQAQAAAAAAAAAAAAAAAAAAAAD/xAAUAQEAAAAAAAAAAAAAAAAA" + "AAAA/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAwDAQACEQMRAD8AfwD/2Q==" + ), + ), + ( + "png", + binascii.a2b_base64( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVQIHWP4DwAB" + "AQEANl9ngAAAAABJRU5ErkJggg==" + ), + ), + ("gif", binascii.a2b_base64("R0lGODdhAQABAIAAAP///////ywAAAAAAQABAAACAkQBADs=")), + ("bmp", b"BM"), ("webp", b"RIFF????WEBP"), ("avif", b"????ftypavif"), ("avif", b"????ftypavis"), @@ -308,33 +308,33 @@ def do_GET(self): ("heic", b"????ftypheim"), ("heic", b"????ftypheis"), ("heic", b"????ftypheix"), - ("svg" , b"02}".format(idx)] = content + DATA[f"S{idx:>02}"] = content # reverse mime types mapping -MIME_TYPES = { - ext: mtype - for mtype, ext in MIME_TYPES.items() -} +MIME_TYPES = {ext: mtype for mtype, ext in MIME_TYPES.items()} def generate_tests(): def generate_test(idx, ext, content): def test(self): - self._run_test("S{:>02}".format(idx), None, content, "bin", ext) - test.__name__ = "test_http_ext_{:>02}_{}".format(idx, ext) + self._run_test(f"S{idx:>02}", None, content, "bin", ext) + + test.__name__ = f"test_http_ext_{idx:>02}_{ext}" return test for idx, (ext, content) in enumerate(SAMPLES): diff --git a/test/test_extractor.py b/test/test_extractor.py index cc85fb2b2..181861091 100644 --- a/test/test_extractor.py +++ b/test/test_extractor.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2018-2023 Mike Fährmann # @@ -8,19 +7,20 @@ # published by the Free Software Foundation. import os +import string import sys +import time import unittest +from datetime import datetime +from datetime import timedelta from unittest.mock import patch -import time -import string -from datetime import datetime, timedelta - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import extractor, util # noqa E402 -from gallery_dl.extractor import mastodon # noqa E402 -from gallery_dl.extractor.common import Extractor, Message # noqa E402 -from gallery_dl.extractor.directlink import DirectlinkExtractor # noqa E402 +from gallery_dl import extractor # noqa: E402 +from gallery_dl import util # noqa: E402 +from gallery_dl.extractor.common import Extractor +from gallery_dl.extractor.common import Message +from gallery_dl.extractor.directlink import DirectlinkExtractor _list_classes = extractor._list_classes @@ -110,7 +110,7 @@ def test_categories(self): extr = cls.from_url(url) except ImportError as exc: if exc.name in ("youtube_dl", "yt_dlp"): - print("Skipping '{}' category checks".format(cls.category)) + print(f"Skipping '{cls.category}' category checks") continue raise self.assertTrue(extr, url) @@ -141,10 +141,8 @@ def test_unique_pattern_matches(self): # ... and apply all regex patterns to each one for extr2 in _list_classes(): - # skip DirectlinkExtractor pattern if it isn't tested - if extr1 != DirectlinkExtractor and \ - extr2 == DirectlinkExtractor: + if extr1 != DirectlinkExtractor and extr2 == DirectlinkExtractor: continue match = extr2.pattern.match(url) @@ -153,14 +151,13 @@ def test_unique_pattern_matches(self): # fail if more or less than 1 match happened if len(matches) > 1: - msg = "'{}' gets matched by more than one pattern:".format(url) + msg = f"'{url}' gets matched by more than one pattern:" for match, extr in matches: - msg += "\n\n- {}:\n{}".format( - extr.__name__, match.re.pattern) + msg += f"\n\n- {extr.__name__}:\n{match.re.pattern}" self.fail(msg) elif len(matches) < 1: - msg = "'{}' isn't matched by any pattern".format(url) + msg = f"'{url}' isn't matched by any pattern" self.fail(msg) else: @@ -168,6 +165,7 @@ def test_unique_pattern_matches(self): def test_init(self): """Test for exceptions in Extractor.initialize() and .finalize()""" + def fail_request(*args, **kwargs): self.fail("called 'request() during initialization") @@ -190,8 +188,7 @@ def test_init_ytdl(self): extr.finalize() except ImportError as exc: if exc.name in ("youtube_dl", "yt_dlp"): - raise unittest.SkipTest("cannot import module '{}'".format( - exc.name)) + raise unittest.SkipTest(f"cannot import module '{exc.name}'") raise def test_docstrings(self): @@ -202,11 +199,12 @@ def test_docstrings(self): self.assertNotEqual( extr1.__doc__, extr2.__doc__, - "{} <-> {}".format(extr1, extr2), + f"{extr1} <-> {extr2}", ) def test_names(self): """Ensure extractor classes are named CategorySubcategoryExtractor""" + def capitalize(c): if "-" in c: return string.capwords(c.replace("-", " ")).replace(" ", "") @@ -214,17 +212,13 @@ def capitalize(c): for extr in extractor.extractors(): if extr.category not in ("", "oauth", "ytdl"): - expected = "{}{}Extractor".format( - capitalize(extr.category), - capitalize(extr.subcategory), - ) + expected = f"{capitalize(extr.category)}{capitalize(extr.subcategory)}Extractor" if expected[0].isdigit(): expected = "_" + expected self.assertEqual(expected, extr.__name__) class TestExtractorWait(unittest.TestCase): - def test_wait_seconds(self): extr = extractor.find("generic:https://example.org/") seconds = 5 @@ -278,7 +272,7 @@ def _assert_isotime(self, output, until): until = datetime.fromtimestamp(until) o = self._isotime_to_seconds(output) u = self._isotime_to_seconds(until.time().isoformat()[:8]) - self.assertLessEqual(o-u, 1.0) + self.assertLessEqual(o - u, 1.0) @staticmethod def _isotime_to_seconds(isotime): @@ -287,7 +281,6 @@ def _isotime_to_seconds(isotime): class TextExtractorOAuth(unittest.TestCase): - def test_oauth1(self): for category in ("flickr", "smugmug", "tumblr"): extr = extractor.find("oauth:" + category) @@ -309,8 +302,10 @@ def test_oauth2(self): def test_oauth2_mastodon(self): extr = extractor.find("oauth:mastodon:pawoo.net") - with patch.object(extr, "_oauth2_authorization_code_grant") as m, \ - patch.object(extr, "_register") as r: + with ( + patch.object(extr, "_oauth2_authorization_code_grant") as m, + patch.object(extr, "_register") as r, + ): for msg in extr: pass self.assertEqual(len(r.mock_calls), 0) @@ -319,10 +314,12 @@ def test_oauth2_mastodon(self): def test_oauth2_mastodon_unknown(self): extr = extractor.find("oauth:mastodon:example.com") - with patch.object(extr, "_oauth2_authorization_code_grant") as m, \ - patch.object(extr, "_register") as r: + with ( + patch.object(extr, "_oauth2_authorization_code_grant") as m, + patch.object(extr, "_register") as r, + ): r.return_value = { - "client-id" : "foo", + "client-id": "foo", "client-secret": "bar", } diff --git a/test/test_formatter.py b/test/test_formatter.py index c0b504da8..587f4f136 100644 --- a/test/test_formatter.py +++ b/test/test_formatter.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2021-2023 Mike Fährmann # @@ -7,19 +6,21 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import datetime import os import sys +import tempfile import time import unittest -import datetime -import tempfile +from time import sleep sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import formatter, text, util # noqa E402 +from gallery_dl import formatter # noqa: E402 +from gallery_dl import text # noqa: E402 +from gallery_dl import util # noqa: E402 class TestFormatter(unittest.TestCase): - kwdict = { "a": "hElLo wOrLd", "b": "äöü", @@ -106,28 +107,27 @@ def test_missing(self): def test_missing_custom_default(self): replacement = default = "foobar" - self._run_test("{missing}" , replacement, default) + self._run_test("{missing}", replacement, default) self._run_test("{missing.attr}", replacement, default) self._run_test("{missing[key]}", replacement, default) self._run_test("{missing:?a//}", "a" + default, default) def test_fmt_func(self): - self._run_test("{t}" , self.kwdict["t"] , None, int) - self._run_test("{t}" , self.kwdict["t"] , None, util.identity) + self._run_test("{t}", self.kwdict["t"], None, int) + self._run_test("{t}", self.kwdict["t"], None, util.identity) self._run_test("{dt}", self.kwdict["dt"], None, util.identity) self._run_test("{ds}", self.kwdict["dt"], None, text.parse_datetime) - self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", self.kwdict["dt"], - None, util.identity) + self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", self.kwdict["dt"], None, util.identity) def test_alternative(self): - self._run_test("{a|z}" , "hElLo wOrLd") - self._run_test("{z|a}" , "hElLo wOrLd") - self._run_test("{z|y|a}" , "hElLo wOrLd") + self._run_test("{a|z}", "hElLo wOrLd") + self._run_test("{z|a}", "hElLo wOrLd") + self._run_test("{z|y|a}", "hElLo wOrLd") self._run_test("{z|y|x|a}", "hElLo wOrLd") self._run_test("{z|n|a|y}", "hElLo wOrLd") - self._run_test("{z|a!C}" , "Hello World") - self._run_test("{z|a:Rh/C/}" , "CElLo wOrLd") + self._run_test("{z|a!C}", "Hello World") + self._run_test("{z|a:Rh/C/}", "CElLo wOrLd") self._run_test("{z|a!C:RH/C/}", "Cello World") self._run_test("{z|y|x:?/}", "") @@ -136,93 +136,93 @@ def test_alternative(self): self._run_test("{d[z]|d[y]|d[x]}", "None") def test_indexing(self): - self._run_test("{l[0]}" , "a") - self._run_test("{a[6]}" , "w") + self._run_test("{l[0]}", "a") + self._run_test("{a[6]}", "w") def test_dict_access(self): - self._run_test("{d[a]}" , "foo") + self._run_test("{d[a]}", "foo") self._run_test("{d['a']}", "foo") self._run_test('{d["a"]}', "foo") def test_slice_str(self): v = self.kwdict["a"] - self._run_test("{a[1:10]}" , v[1:10]) + self._run_test("{a[1:10]}", v[1:10]) self._run_test("{a[-10:-1]}", v[-10:-1]) - self._run_test("{a[5:]}" , v[5:]) + self._run_test("{a[5:]}", v[5:]) self._run_test("{a[50:]}", v[50:]) - self._run_test("{a[:5]}" , v[:5]) + self._run_test("{a[:5]}", v[:5]) self._run_test("{a[:50]}", v[:50]) - self._run_test("{a[:]}" , v) - self._run_test("{a[1:10:2]}" , v[1:10:2]) + self._run_test("{a[:]}", v) + self._run_test("{a[1:10:2]}", v[1:10:2]) self._run_test("{a[-10:-1:2]}", v[-10:-1:2]) - self._run_test("{a[5::2]}" , v[5::2]) + self._run_test("{a[5::2]}", v[5::2]) self._run_test("{a[50::2]}", v[50::2]) - self._run_test("{a[:5:2]}" , v[:5:2]) + self._run_test("{a[:5:2]}", v[:5:2]) self._run_test("{a[:50:2]}", v[:50:2]) - self._run_test("{a[::]}" , v) + self._run_test("{a[::]}", v) - self._run_test("{a:[1:10]}" , v[1:10]) + self._run_test("{a:[1:10]}", v[1:10]) self._run_test("{a:[-10:-1]}", v[-10:-1]) - self._run_test("{a:[5:]}" , v[5:]) + self._run_test("{a:[5:]}", v[5:]) self._run_test("{a:[50:]}", v[50:]) - self._run_test("{a:[:5]}" , v[:5]) + self._run_test("{a:[:5]}", v[:5]) self._run_test("{a:[:50]}", v[:50]) - self._run_test("{a:[:]}" , v) - self._run_test("{a:[1:10:2]}" , v[1:10:2]) + self._run_test("{a:[:]}", v) + self._run_test("{a:[1:10:2]}", v[1:10:2]) self._run_test("{a:[-10:-1:2]}", v[-10:-1:2]) - self._run_test("{a:[5::2]}" , v[5::2]) + self._run_test("{a:[5::2]}", v[5::2]) self._run_test("{a:[50::2]}", v[50::2]) - self._run_test("{a:[:5:2]}" , v[:5:2]) + self._run_test("{a:[:5:2]}", v[:5:2]) self._run_test("{a:[:50:2]}", v[:50:2]) - self._run_test("{a:[::]}" , v) + self._run_test("{a:[::]}", v) def test_slice_bytes(self): v = self.kwdict["j"] - self._run_test("{j[b1:10]}" , v[1:3]) + self._run_test("{j[b1:10]}", v[1:3]) self._run_test("{j[b-10:-1]}", v[-3:-1]) - self._run_test("{j[b5:]}" , v[2:]) - self._run_test("{j[b50:]}" , v[50:]) - self._run_test("{j[b:5]}" , v[:1]) - self._run_test("{j[b:50]}" , v[:50]) - self._run_test("{j[b:]}" , v) - self._run_test("{j[b::]}" , v) - - self._run_test("{j:[b1:10]}" , v[1:3]) + self._run_test("{j[b5:]}", v[2:]) + self._run_test("{j[b50:]}", v[50:]) + self._run_test("{j[b:5]}", v[:1]) + self._run_test("{j[b:50]}", v[:50]) + self._run_test("{j[b:]}", v) + self._run_test("{j[b::]}", v) + + self._run_test("{j:[b1:10]}", v[1:3]) self._run_test("{j:[b-10:-1]}", v[-3:-1]) - self._run_test("{j:[b5:]}" , v[2:]) - self._run_test("{j:[b50:]}" , v[50:]) - self._run_test("{j:[b:5]}" , v[:1]) - self._run_test("{j:[b:50]}" , v[:50]) - self._run_test("{j:[b:]}" , v) - self._run_test("{j:[b::]}" , v) + self._run_test("{j:[b5:]}", v[2:]) + self._run_test("{j:[b50:]}", v[50:]) + self._run_test("{j:[b:5]}", v[:1]) + self._run_test("{j:[b:50]}", v[:50]) + self._run_test("{j:[b:]}", v) + self._run_test("{j:[b::]}", v) def test_maxlen(self): v = self.kwdict["a"] - self._run_test("{a:L5/foo/}" , "foo") + self._run_test("{a:L5/foo/}", "foo") self._run_test("{a:L50/foo/}", v) self._run_test("{a:L50/foo/>50}", " " * 39 + v) self._run_test("{a:L50/foo/>51}", "foo") self._run_test("{a:Lab/foo/}", "foo") def test_join(self): - self._run_test("{l:J}" , "abc") - self._run_test("{l:J,}" , "a,b,c") - self._run_test("{l:J,/}" , "a,b,c") - self._run_test("{l:J,/>20}" , " a,b,c") - self._run_test("{l:J - }" , "a - b - c") - self._run_test("{l:J - /}" , "a - b - c") + self._run_test("{l:J}", "abc") + self._run_test("{l:J,}", "a,b,c") + self._run_test("{l:J,/}", "a,b,c") + self._run_test("{l:J,/>20}", " a,b,c") + self._run_test("{l:J - }", "a - b - c") + self._run_test("{l:J - /}", "a - b - c") self._run_test("{l:J - />20}", " a - b - c") - self._run_test("{a:J/}" , self.kwdict["a"]) - self._run_test("{a:J, /}" , self.kwdict["a"]) + self._run_test("{a:J/}", self.kwdict["a"]) + self._run_test("{a:J, /}", self.kwdict["a"]) def test_replace(self): - self._run_test("{a:Rh/C/}" , "CElLo wOrLd") + self._run_test("{a:Rh/C/}", "CElLo wOrLd") self._run_test("{a!l:Rh/C/}", "Cello world") self._run_test("{a!u:Rh/C/}", "HELLO WORLD") self._run_test("{a!l:Rl/_/}", "he__o wor_d") - self._run_test("{a!l:Rl//}" , "heo word") + self._run_test("{a!l:Rl//}", "heo word") self._run_test("{name:Rame/othing/}", "Nothing") def test_datetime(self): @@ -242,34 +242,33 @@ def test_offset(self): self._run_test("{t!d:O2}", "2010-01-01 02:00:00") def test_offset_local(self): - ts = self.kwdict["dt"].replace( - tzinfo=datetime.timezone.utc).timestamp() + ts = self.kwdict["dt"].replace(tzinfo=datetime.timezone.utc).timestamp() offset = time.localtime(ts).tm_gmtoff dt = self.kwdict["dt"] + datetime.timedelta(seconds=offset) self._run_test("{dt:O}", str(dt)) self._run_test("{dt:Olocal}", str(dt)) - ts = self.kwdict["dt_dst"].replace( - tzinfo=datetime.timezone.utc).timestamp() + ts = self.kwdict["dt_dst"].replace(tzinfo=datetime.timezone.utc).timestamp() offset = time.localtime(ts).tm_gmtoff dt = self.kwdict["dt_dst"] + datetime.timedelta(seconds=offset) self._run_test("{dt_dst:O}", str(dt)) self._run_test("{dt_dst:Olocal}", str(dt)) def test_sort(self): - self._run_test("{l:S}" , "['a', 'b', 'c']") + self._run_test("{l:S}", "['a', 'b', 'c']") self._run_test("{l:Sa}", "['a', 'b', 'c']") self._run_test("{l:Sd}", "['c', 'b', 'a']") self._run_test("{l:Sr}", "['c', 'b', 'a']") - self._run_test( - "{a:S}", "[' ', 'E', 'L', 'L', 'O', 'd', 'h', 'l', 'o', 'r', 'w']") + self._run_test("{a:S}", "[' ', 'E', 'L', 'L', 'O', 'd', 'h', 'l', 'o', 'r', 'w']") self._run_test( "{a:S-asc}", # starts with 'S', contains 'a' - "[' ', 'E', 'L', 'L', 'O', 'd', 'h', 'l', 'o', 'r', 'w']") + "[' ', 'E', 'L', 'L', 'O', 'd', 'h', 'l', 'o', 'r', 'w']", + ) self._run_test( "{a:Sort-reverse}", # starts with 'S', contains 'r' - "['w', 'r', 'o', 'l', 'h', 'd', 'O', 'L', 'L', 'E', ' ']") + "['w', 'r', 'o', 'l', 'h', 'd', 'O', 'L', 'L', 'E', ' ']", + ) def test_specifier_arithmetic(self): self._run_test("{i:A+1}", "3") @@ -277,8 +276,8 @@ def test_specifier_arithmetic(self): self._run_test("{i:A*3}", "6") def test_specifier_conversions(self): - self._run_test("{a:Cl}" , "hello world") - self._run_test("{h:CHC}" , "Foo & Bar") + self._run_test("{a:Cl}", "hello world") + self._run_test("{h:CHC}", "Foo & Bar") self._run_test("{l:CSulc}", "A, b, c") def test_specifier_limit(self): @@ -332,7 +331,7 @@ def test_separator(self): def test_globals_env(self): os.environ["FORMATTER_TEST"] = value = self.kwdict["a"] - self._run_test("{_env[FORMATTER_TEST]}" , value) + self._run_test("{_env[FORMATTER_TEST]}", value) self._run_test("{_env[FORMATTER_TEST]!l}", value.lower()) self._run_test("{z|_env[FORMATTER_TEST]}", value) @@ -350,6 +349,10 @@ def test_globals_now(self): self.assertRegex(out, r"^\d{4}$") self.assertEqual(out, format(now, "%Y")) + # Sleep briefly to ensure `now` is actually different + # Without this, the following assertion can fail if the instructions are executed quickly enough + sleep(0.01) + out2 = fmt.format_map(self.kwdict) self.assertRegex(out1, r"^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(\.\d+)?$") self.assertNotEqual(out1, out2) @@ -357,38 +360,38 @@ def test_globals_now(self): def test_globals_nul(self): value = "None" - self._run_test("{_nul}" , value) - self._run_test("{_nul[key]}" , value) - self._run_test("{z|_nul}" , value) + self._run_test("{_nul}", value) + self._run_test("{_nul[key]}", value) + self._run_test("{z|_nul}", value) self._run_test("{z|_nul:%Y%m%s}", value) def test_literals(self): value = "foo" - self._run_test("{'foo'}" , value) - self._run_test("{'foo'!u}" , value.upper()) + self._run_test("{'foo'}", value) + self._run_test("{'foo'!u}", value.upper()) self._run_test("{'f00':R0/o/}", value) - self._run_test("{z|'foo'}" , value) - self._run_test("{z|''|'foo'}" , value) - self._run_test("{z|'foo'!u}" , value.upper()) + self._run_test("{z|'foo'}", value) + self._run_test("{z|''|'foo'}", value) + self._run_test("{z|'foo'!u}", value.upper()) self._run_test("{z|'f00':R0/o/}", value) - self._run_test("{_lit[foo]}" , value) - self._run_test("{_lit[foo]!u}" , value.upper()) - self._run_test("{_lit[f00]:R0/o/}" , value) + self._run_test("{_lit[foo]}", value) + self._run_test("{_lit[foo]!u}", value.upper()) + self._run_test("{_lit[f00]:R0/o/}", value) self._run_test("{_lit[foobar][:3]}", value) - self._run_test("{z|_lit[foo]}" , value) + self._run_test("{z|_lit[foo]}", value) # empty (#4492) - self._run_test("{z|''}" , "") + self._run_test("{z|''}", "") self._run_test("{''|''}", "") # special characters (dots, brackets, singlee quotes) (#5539) - self._run_test("{'f.o.o'}" , "f.o.o") + self._run_test("{'f.o.o'}", "f.o.o") self._run_test("{_lit[f.o.o]}", "f.o.o") self._run_test("{_lit[f'o'o]}", "f'o'o") - self._run_test("{'f.[].[]'}" , "f.[].[]") + self._run_test("{'f.[].[]'}", "f.[].[]") self._run_test("{z|'f.[].[]'}", "f.[].[]") def test_template(self): @@ -412,16 +415,21 @@ def test_template(self): def test_expression(self): self._run_test("\fE a", self.kwdict["a"]) - self._run_test("\fE name * 2 + ' ' + a", "{}{} {}".format( - self.kwdict["name"], self.kwdict["name"], self.kwdict["a"])) + self._run_test( + "\fE name * 2 + ' ' + a", + "{}{} {}".format(self.kwdict["name"], self.kwdict["name"], self.kwdict["a"]), + ) @unittest.skipIf(sys.hexversion < 0x3060000, "no fstring support") def test_fstring(self): self._run_test("\fF {a}", self.kwdict["a"]) - self._run_test("\fF {name}{name} {a}", "{}{} {}".format( - self.kwdict["name"], self.kwdict["name"], self.kwdict["a"])) - self._run_test("\fF foo-'\"{a.upper()}\"'-bar", - """foo-'"{}"'-bar""".format(self.kwdict["a"].upper())) + self._run_test( + "\fF {name}{name} {a}", + "{}{} {}".format(self.kwdict["name"], self.kwdict["name"], self.kwdict["a"]), + ) + self._run_test( + "\fF foo-'\"{a.upper()}\"'-bar", """foo-'"{}"'-bar""".format(self.kwdict["a"].upper()) + ) @unittest.skipIf(sys.hexversion < 0x3060000, "no fstring support") def test_template_fstring(self): @@ -438,8 +446,9 @@ def test_template_fstring(self): fmt2 = formatter.parse("\fTF " + path2) self.assertEqual(fmt1.format_map(self.kwdict), self.kwdict["a"]) - self.assertEqual(fmt2.format_map(self.kwdict), - """foo-'"{}"'-bar""".format(self.kwdict["a"].upper())) + self.assertEqual( + fmt2.format_map(self.kwdict), """foo-'"{}"'-bar""".format(self.kwdict["a"].upper()) + ) with self.assertRaises(OSError): formatter.parse("\fTF /") diff --git a/test/test_job.py b/test/test_job.py index 3e6f85be6..b987b2c1c 100644 --- a/test/test_job.py +++ b/test/test_job.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2021-2023 Mike Fährmann # @@ -7,20 +6,21 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import io import os import sys import unittest from unittest.mock import patch -import io - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import job, config, text # noqa E402 -from gallery_dl.extractor.common import Extractor, Message # noqa E402 +from gallery_dl import config # noqa: E402 +from gallery_dl import job # noqa: E402 +from gallery_dl import text # noqa: E402 +from gallery_dl.extractor.common import Extractor +from gallery_dl.extractor.common import Message class TestJob(unittest.TestCase): - def tearDown(self): config.clear() @@ -49,21 +49,21 @@ def test_extractor_filter(self): tjob = self.jobclass(extr) func = tjob._build_extractor_filter() - self.assertEqual(func(TestExtractor) , False) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), False) - self.assertEqual(func(TestExtractorAlt) , True) + self.assertEqual(func(TestExtractorAlt), True) config.set((), "blacklist", ":test_subcategory") func = tjob._build_extractor_filter() - self.assertEqual(func(TestExtractor) , False) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), True) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) config.set((), "whitelist", "test_category:test_subcategory") func = tjob._build_extractor_filter() - self.assertEqual(func(TestExtractor) , True) + self.assertEqual(func(TestExtractor), True) self.assertEqual(func(TestExtractorParent), False) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) class TestKeywordJob(TestJob): @@ -72,7 +72,9 @@ class TestKeywordJob(TestJob): def test_default(self): self.maxDiff = None extr = TestExtractor.from_url("test:self") - self.assertEqual(self._capture_stdout(extr), """\ + self.assertEqual( + self._capture_stdout(extr), + """\ Keywords for directory names: ----------------------------- author['id'] @@ -120,7 +122,8 @@ def test_default(self): test user['self'] -""") +""", + ) class TestUrlJob(TestJob): @@ -128,42 +131,55 @@ class TestUrlJob(TestJob): def test_default(self): extr = TestExtractor.from_url("test:") - self.assertEqual(self._capture_stdout(extr), """\ + self.assertEqual( + self._capture_stdout(extr), + """\ https://example.org/1.jpg https://example.org/2.jpg https://example.org/3.jpg -""") +""", + ) def test_fallback(self): extr = TestExtractor.from_url("test:") tjob = self.jobclass(extr) tjob.handle_url = tjob.handle_url_fallback - self.assertEqual(self._capture_stdout(tjob), """\ + self.assertEqual( + self._capture_stdout(tjob), + """\ https://example.org/1.jpg | https://example.org/alt/1.jpg https://example.org/2.jpg | https://example.org/alt/2.jpg https://example.org/3.jpg | https://example.org/alt/3.jpg -""") +""", + ) def test_parent(self): extr = TestExtractorParent.from_url("test:parent") - self.assertEqual(self._capture_stdout(extr), """\ + self.assertEqual( + self._capture_stdout(extr), + """\ test:child test:child test:child -""") +""", + ) def test_child(self): extr = TestExtractorParent.from_url("test:parent") tjob = job.UrlJob(extr, depth=0) - self.assertEqual(self._capture_stdout(tjob), 3 * """\ + self.assertEqual( + self._capture_stdout(tjob), + 3 + * """\ https://example.org/1.jpg https://example.org/2.jpg https://example.org/3.jpg -""") +""", + ) class TestInfoJob(TestJob): @@ -171,7 +187,9 @@ class TestInfoJob(TestJob): def test_default(self): extr = TestExtractor.from_url("test:") - self.assertEqual(self._capture_stdout(extr), """\ + self.assertEqual( + self._capture_stdout(extr), + """\ Category / Subcategory "test_category" / "test_subcategory" @@ -181,7 +199,8 @@ def test_default(self): Directory format (default): ["{category}"] -""") +""", + ) def test_custom(self): config.set((), "filename", "custom") @@ -190,7 +209,9 @@ def test_custom(self): extr = TestExtractor.from_url("test:") extr.request_interval = 123.456 - self.assertEqual(self._capture_stdout(extr), """\ + self.assertEqual( + self._capture_stdout(extr), + """\ Category / Subcategory "test_category" / "test_subcategory" @@ -209,13 +230,16 @@ def test_custom(self): Request interval (default): 123.456 -""") +""", + ) def test_base_category(self): extr = TestExtractor.from_url("test:") extr.basecategory = "test_basecategory" - self.assertEqual(self._capture_stdout(extr), """\ + self.assertEqual( + self._capture_stdout(extr), + """\ Category / Subcategory / Basecategory "test_category" / "test_subcategory" / "test_basecategory" @@ -225,7 +249,8 @@ def test_base_category(self): Directory format (default): ["{category}"] -""") +""", + ) class TestDataJob(TestJob): @@ -238,51 +263,68 @@ def test_default(self): tjob.run() - self.assertEqual(tjob.data, [ - (Message.Directory, { - "category" : "test_category", - "subcategory": "test_subcategory", - "user" : user, - "author" : user, - }), - (Message.Url, "https://example.org/1.jpg", { - "category" : "test_category", - "subcategory": "test_subcategory", - "filename" : "1", - "extension" : "jpg", - "num" : 1, - "tags" : ["foo", "bar", "テスト"], - "user" : user, - "author" : user, - }), - (Message.Url, "https://example.org/2.jpg", { - "category" : "test_category", - "subcategory": "test_subcategory", - "filename" : "2", - "extension" : "jpg", - "num" : 2, - "tags" : ["foo", "bar", "テスト"], - "user" : user, - "author" : user, - }), - (Message.Url, "https://example.org/3.jpg", { - "category" : "test_category", - "subcategory": "test_subcategory", - "filename" : "3", - "extension" : "jpg", - "num" : 3, - "tags" : ["foo", "bar", "テスト"], - "user" : user, - "author" : user, - }), - ]) + self.assertEqual( + tjob.data, + [ + ( + Message.Directory, + { + "category": "test_category", + "subcategory": "test_subcategory", + "user": user, + "author": user, + }, + ), + ( + Message.Url, + "https://example.org/1.jpg", + { + "category": "test_category", + "subcategory": "test_subcategory", + "filename": "1", + "extension": "jpg", + "num": 1, + "tags": ["foo", "bar", "テスト"], + "user": user, + "author": user, + }, + ), + ( + Message.Url, + "https://example.org/2.jpg", + { + "category": "test_category", + "subcategory": "test_subcategory", + "filename": "2", + "extension": "jpg", + "num": 2, + "tags": ["foo", "bar", "テスト"], + "user": user, + "author": user, + }, + ), + ( + Message.Url, + "https://example.org/3.jpg", + { + "category": "test_category", + "subcategory": "test_subcategory", + "filename": "3", + "extension": "jpg", + "num": 3, + "tags": ["foo", "bar", "テスト"], + "user": user, + "author": user, + }, + ), + ], + ) def test_exception(self): extr = TestExtractorException.from_url("test:exception") tjob = self.jobclass(extr, file=io.StringIO()) tjob.run() - self.assertEqual( - tjob.data[-1], ("ZeroDivisionError", "division by zero")) + self.assertEqual(tjob.data[-1], ("ZeroDivisionError", "division by zero")) def test_private(self): config.set(("output",), "private", True) @@ -294,7 +336,7 @@ def test_private(self): for i in range(1, 4): self.assertEqual( tjob.data[i][2]["_fallback"], - ("https://example.org/alt/{}.jpg".format(i),), + (f"https://example.org/alt/{i}.jpg",), ) def test_sleep(self): @@ -317,24 +359,30 @@ def test_ascii(self): tjob.file = buffer = io.StringIO() tjob.run() - self.assertIn("""\ + self.assertIn( + """\ "tags": [ "foo", "bar", "\\u30c6\\u30b9\\u30c8" ], -""", buffer.getvalue()) +""", + buffer.getvalue(), + ) tjob.file = buffer = io.StringIO() tjob.ascii = False tjob.run() - self.assertIn("""\ + self.assertIn( + """\ "tags": [ "foo", "bar", "テスト" ], -""", buffer.getvalue()) +""", + buffer.getvalue(), + ) def test_num_string(self): extr = TestExtractor.from_url("test:") @@ -371,20 +419,30 @@ def items(self): root = "https://example.org" user = self.user - yield Message.Directory, { - "user": user, - "author": user, - } - - for i in range(1, 4): - url = "{}/{}.jpg".format(root, i) - yield Message.Url, url, text.nameext_from_url(url, { - "num" : i, - "tags": ["foo", "bar", "テスト"], + yield ( + Message.Directory, + { "user": user, "author": user, - "_fallback": ("{}/alt/{}.jpg".format(root, i),), - }) + }, + ) + + for i in range(1, 4): + url = f"{root}/{i}.jpg" + yield ( + Message.Url, + url, + text.nameext_from_url( + url, + { + "num": i, + "tags": ["foo", "bar", "テスト"], + "user": user, + "author": user, + "_fallback": (f"{root}/alt/{i}.jpg",), + }, + ), + ) class TestExtractorParent(Extractor): @@ -396,11 +454,15 @@ def items(self): url = "test:child" for i in range(11, 14): - yield Message.Queue, url, { - "num" : i, - "tags": ["abc", "def"], - "_extractor": TestExtractor, - } + yield ( + Message.Queue, + url, + { + "num": i, + "tags": ["abc", "def"], + "_extractor": TestExtractor, + }, + ) class TestExtractorException(Extractor): @@ -409,7 +471,7 @@ class TestExtractorException(Extractor): pattern = r"test:exception$" def items(self): - return 1/0 + return 1 / 0 class TestExtractorAlt(Extractor): diff --git a/test/test_oauth.py b/test/test_oauth.py index 0082419d3..65a1772d4 100644 --- a/test/test_oauth.py +++ b/test/test_oauth.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2018-2023 Mike Fährmann # @@ -13,7 +12,8 @@ from unittest.mock import patch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import oauth, text # noqa E402 +from gallery_dl import oauth # noqa: E402 +from gallery_dl import text # noqa: E402 TESTSERVER = "http://term.ie/oauth/example" CONSUMER_KEY = "key" @@ -25,7 +25,6 @@ class TestOAuthSession(unittest.TestCase): - def test_concat(self): concat = oauth.concat @@ -36,11 +35,11 @@ def test_concat(self): self.assertEqual(concat("&", "?/"), "%26&%3F%2F") self.assertEqual( concat("GET", "http://example.org/", "foo=bar&baz=a"), - "GET&http%3A%2F%2Fexample.org%2F&foo%3Dbar%26baz%3Da" + "GET&http%3A%2F%2Fexample.org%2F&foo%3Dbar%26baz%3Da", ) def test_nonce(self, size=16): - nonce_values = set(oauth.nonce(size) for _ in range(size)) + nonce_values = {oauth.nonce(size) for _ in range(size)} # uniqueness self.assertEqual(len(nonce_values), size) @@ -53,9 +52,7 @@ def test_quote(self): quote = oauth.quote reserved = ",;:!\"§$%&/(){}[]=?`´+*'äöü" - unreserved = ("ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789-._~") + unreserved = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789-._~" for char in unreserved: self.assertEqual(quote(char), char) @@ -69,34 +66,36 @@ def test_quote(self): def test_generate_signature(self): client = oauth.OAuth1Client( - CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET) + CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET + ) request = MockRequest() params = [] self.assertEqual( - client.generate_signature(request, params), - "Wt2xo49dM5pkL4gsnCakNdHaVUo%3D") + client.generate_signature(request, params), "Wt2xo49dM5pkL4gsnCakNdHaVUo%3D" + ) request = MockRequest("https://example.org/") params = [("hello", "world"), ("foo", "bar")] self.assertEqual( - client.generate_signature(request, params), - "ay2269%2F8uKpZqKJR1doTtpv%2Bzn0%3D") + client.generate_signature(request, params), "ay2269%2F8uKpZqKJR1doTtpv%2Bzn0%3D" + ) - request = MockRequest("https://example.org/index.html" - "?hello=world&foo=bar", method="POST") + request = MockRequest( + "https://example.org/index.html" "?hello=world&foo=bar", method="POST" + ) params = [("oauth_signature_method", "HMAC-SHA1")] self.assertEqual( - client.generate_signature(request, params), - "yVZWb1ts4smdMmXxMlhaXrkoOng%3D") + client.generate_signature(request, params), "yVZWb1ts4smdMmXxMlhaXrkoOng%3D" + ) def test_dunder_call(self): client = oauth.OAuth1Client( - CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET) + CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET + ) request = MockRequest("https://example.org/") - with patch("time.time") as tmock, \ - patch("gallery_dl.oauth.nonce") as nmock: + with patch("time.time") as tmock, patch("gallery_dl.oauth.nonce") as nmock: tmock.return_value = 123456789.123 nmock.return_value = "abcdefghijklmno" @@ -112,11 +111,11 @@ def test_dunder_call(self): oauth_version="1.0",\ oauth_token="accesskey",\ oauth_signature="DjtTk5j5P3BDZFnstZ%2FtEYcwD6c%3D"\ -""") +""", + ) def test_request_token(self): - response = self._oauth_request( - "/request_token.php", {}) + response = self._oauth_request("/request_token.php", {}) expected = "oauth_token=requestkey&oauth_token_secret=requestsecret" self.assertEqual(response, expected, msg=response) @@ -125,8 +124,7 @@ def test_request_token(self): self.assertTrue(data["oauth_token_secret"], REQUEST_TOKEN_SECRET) def test_access_token(self): - response = self._oauth_request( - "/access_token.php", {}, REQUEST_TOKEN, REQUEST_TOKEN_SECRET) + response = self._oauth_request("/access_token.php", {}, REQUEST_TOKEN, REQUEST_TOKEN_SECRET) expected = "oauth_token=accesskey&oauth_token_secret=accesssecret" self.assertEqual(response, expected, msg=response) @@ -136,19 +134,19 @@ def test_access_token(self): def test_authenticated_call(self): params = {"method": "foo", "a": "äöüß/?&#", "äöüß/?&#": "a"} - response = self._oauth_request( - "/echo_api.php", params, ACCESS_TOKEN, ACCESS_TOKEN_SECRET) + response = self._oauth_request("/echo_api.php", params, ACCESS_TOKEN, ACCESS_TOKEN_SECRET) self.assertEqual(text.parse_query(response), params) - def _oauth_request(self, endpoint, params=None, - oauth_token=None, oauth_token_secret=None): + def _oauth_request(self, endpoint, params=None, oauth_token=None, oauth_token_secret=None): # the test server at 'term.ie' is unreachable raise unittest.SkipTest() session = oauth.OAuth1Session( - CONSUMER_KEY, CONSUMER_SECRET, - oauth_token, oauth_token_secret, + CONSUMER_KEY, + CONSUMER_SECRET, + oauth_token, + oauth_token_secret, ) try: response = session.get(TESTSERVER + endpoint, params=params) @@ -158,8 +156,7 @@ def _oauth_request(self, endpoint, params=None, raise unittest.SkipTest() -class MockRequest(): - +class MockRequest: def __init__(self, url="", method="GET"): self.url = url self.method = method diff --git a/test/test_output.py b/test/test_output.py index e81f7681c..83e41876d 100644 --- a/test/test_output.py +++ b/test/test_output.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2021 Mike Fährmann # @@ -12,13 +11,12 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import output # noqa E402 +from gallery_dl import output class TestShorten(unittest.TestCase): - def test_shorten_noop(self, f=output.shorten_string): - self.assertEqual(f("" , 10), "") + self.assertEqual(f("", 10), "") self.assertEqual(f("foobar", 10), "foobar") def test_shorten(self, f=output.shorten_string): @@ -36,9 +34,9 @@ def test_shorten(self, f=output.shorten_string): self.assertEqual(f(s, 12), "01234…456789") self.assertEqual(f(s, 11), "01234…56789") self.assertEqual(f(s, 10), "0123…56789") - self.assertEqual(f(s, 9) , "0123…6789") - self.assertEqual(f(s, 3) , "0…9") - self.assertEqual(f(s, 2) , "…9") + self.assertEqual(f(s, 9), "0123…6789") + self.assertEqual(f(s, 3), "0…9") + self.assertEqual(f(s, 2), "…9") def test_shorten_separator(self, f=output.shorten_string): s = "01234567890123456789" # string of length 20 @@ -48,15 +46,14 @@ def test_shorten_separator(self, f=output.shorten_string): self.assertEqual(f(s, 10, "|---|"), "01|---|789") self.assertEqual(f(s, 19, "..."), "01234567...23456789") - self.assertEqual(f(s, 19, "..") , "01234567..123456789") - self.assertEqual(f(s, 19, ".") , "012345678.123456789") - self.assertEqual(f(s, 19, "") , "0123456780123456789") + self.assertEqual(f(s, 19, ".."), "01234567..123456789") + self.assertEqual(f(s, 19, "."), "012345678.123456789") + self.assertEqual(f(s, 19, ""), "0123456780123456789") class TestShortenEAW(unittest.TestCase): - def test_shorten_eaw_noop(self, f=output.shorten_string_eaw): - self.assertEqual(f("" , 10), "") + self.assertEqual(f("", 10), "") self.assertEqual(f("foobar", 10), "foobar") def test_shorten_eaw(self, f=output.shorten_string_eaw): @@ -74,9 +71,9 @@ def test_shorten_eaw(self, f=output.shorten_string_eaw): self.assertEqual(f(s, 12), "01234…456789") self.assertEqual(f(s, 11), "01234…56789") self.assertEqual(f(s, 10), "0123…56789") - self.assertEqual(f(s, 9) , "0123…6789") - self.assertEqual(f(s, 3) , "0…9") - self.assertEqual(f(s, 2) , "…9") + self.assertEqual(f(s, 9), "0123…6789") + self.assertEqual(f(s, 3), "0…9") + self.assertEqual(f(s, 2), "…9") def test_shorten_eaw_wide(self, f=output.shorten_string_eaw): s = "幻想郷幻想郷幻想郷幻想郷" # 12 wide characters @@ -93,8 +90,8 @@ def test_shorten_eaw_wide(self, f=output.shorten_string_eaw): self.assertEqual(f(s, 12), "幻想…幻想郷") self.assertEqual(f(s, 11), "幻想…幻想郷") self.assertEqual(f(s, 10), "幻想…想郷") - self.assertEqual(f(s, 9) , "幻想…想郷") - self.assertEqual(f(s, 3) , "…郷") + self.assertEqual(f(s, 9), "幻想…想郷") + self.assertEqual(f(s, 3), "…郷") def test_shorten_eaw_mix(self, f=output.shorten_string_eaw): s = "幻-想-郷##幻-想-郷##幻-想-郷" # mixed characters @@ -112,8 +109,8 @@ def test_shorten_eaw_mix(self, f=output.shorten_string_eaw): self.assertEqual(f(s, 12), "幻-想…-想-郷") self.assertEqual(f(s, 11), "幻-想…想-郷") self.assertEqual(f(s, 10), "幻-…-想-郷") - self.assertEqual(f(s, 9) , "幻-…想-郷") - self.assertEqual(f(s, 3) , "…郷") + self.assertEqual(f(s, 9), "幻-…想-郷") + self.assertEqual(f(s, 3), "…郷") def test_shorten_eaw_separator(self, f=output.shorten_string_eaw): s = "01234567890123456789" # 20 ascii characters @@ -123,9 +120,9 @@ def test_shorten_eaw_separator(self, f=output.shorten_string_eaw): self.assertEqual(f(s, 10, "|---|"), "01|---|789") self.assertEqual(f(s, 19, "..."), "01234567...23456789") - self.assertEqual(f(s, 19, "..") , "01234567..123456789") - self.assertEqual(f(s, 19, ".") , "012345678.123456789") - self.assertEqual(f(s, 19, "") , "0123456780123456789") + self.assertEqual(f(s, 19, ".."), "01234567..123456789") + self.assertEqual(f(s, 19, "."), "012345678.123456789") + self.assertEqual(f(s, 19, ""), "0123456780123456789") def test_shorten_eaw_separator_wide(self, f=output.shorten_string_eaw): s = "幻想郷幻想郷幻想郷幻想郷" # 12 wide characters @@ -135,9 +132,9 @@ def test_shorten_eaw_separator_wide(self, f=output.shorten_string_eaw): self.assertEqual(f(s, 10, "|---|"), "幻|---|郷") self.assertEqual(f(s, 19, "..."), "幻想郷幻...郷幻想郷") - self.assertEqual(f(s, 19, "..") , "幻想郷幻..郷幻想郷") - self.assertEqual(f(s, 19, ".") , "幻想郷幻.想郷幻想郷") - self.assertEqual(f(s, 19, "") , "幻想郷幻想郷幻想郷") + self.assertEqual(f(s, 19, ".."), "幻想郷幻..郷幻想郷") + self.assertEqual(f(s, 19, "."), "幻想郷幻.想郷幻想郷") + self.assertEqual(f(s, 19, ""), "幻想郷幻想郷幻想郷") def test_shorten_eaw_separator_mix_(self, f=output.shorten_string_eaw): s = "幻-想-郷##幻-想-郷##幻-想-郷" # mixed characters @@ -147,9 +144,9 @@ def test_shorten_eaw_separator_mix_(self, f=output.shorten_string_eaw): self.assertEqual(f(s, 10, "|---|"), "幻|---|-郷") self.assertEqual(f(s, 19, "..."), "幻-想-郷...幻-想-郷") - self.assertEqual(f(s, 19, "..") , "幻-想-郷..#幻-想-郷") - self.assertEqual(f(s, 19, ".") , "幻-想-郷#.#幻-想-郷") - self.assertEqual(f(s, 19, "") , "幻-想-郷###幻-想-郷") + self.assertEqual(f(s, 19, ".."), "幻-想-郷..#幻-想-郷") + self.assertEqual(f(s, 19, "."), "幻-想-郷#.#幻-想-郷") + self.assertEqual(f(s, 19, ""), "幻-想-郷###幻-想-郷") if __name__ == "__main__": diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index dd53803fe..02ee5dcd7 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019-2023 Mike Fährmann # @@ -7,30 +6,34 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import collections +import logging import os +import shutil import sys +import tempfile import unittest -from unittest.mock import Mock, mock_open, patch - -import shutil -import logging import zipfile -import tempfile -import collections from datetime import datetime +from pathlib import Path +from unittest.mock import Mock +from unittest.mock import mock_open +from unittest.mock import patch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import extractor, output, path # noqa E402 -from gallery_dl import postprocessor, config # noqa E402 -from gallery_dl.postprocessor.common import PostProcessor # noqa E402 +from gallery_dl import config # noqa: E402 +from gallery_dl import extractor # noqa: E402 +from gallery_dl import output # noqa: E402 +from gallery_dl import path # noqa: E402 +from gallery_dl import postprocessor # noqa: E402 +from gallery_dl.postprocessor.common import PostProcessor class MockPostprocessorModule(Mock): __postprocessor__ = "mock" -class FakeJob(): - +class FakeJob: def __init__(self, extr=extractor.find("generic:https://example.org/")): extr.directory_fmt = ("{category}",) self.extractor = extr @@ -45,36 +48,34 @@ def register_hooks(self, hooks, options): class TestPostprocessorModule(unittest.TestCase): - def setUp(self): postprocessor._cache.clear() def test_find(self): - for name in (postprocessor.modules): + for name in postprocessor.modules: cls = postprocessor.find(name) self.assertEqual(cls.__name__, name.capitalize() + "PP") self.assertIs(cls.__base__, PostProcessor) self.assertEqual(postprocessor.find("foo"), None) - self.assertEqual(postprocessor.find(1234) , None) - self.assertEqual(postprocessor.find(None) , None) + self.assertEqual(postprocessor.find(1234), None) + self.assertEqual(postprocessor.find(None), None) @patch("builtins.__import__") def test_cache(self, import_module): import_module.return_value = MockPostprocessorModule() - for name in (postprocessor.modules): + for name in postprocessor.modules: postprocessor.find(name) self.assertEqual(import_module.call_count, len(postprocessor.modules)) # no new calls to import_module - for name in (postprocessor.modules): + for name in postprocessor.modules: postprocessor.find(name) self.assertEqual(import_module.call_count, len(postprocessor.modules)) class BasePostprocessorTest(unittest.TestCase): - @classmethod def setUpClass(cls): cls.dir = tempfile.TemporaryDirectory() @@ -105,21 +106,19 @@ def _create(self, options=None, data=None): return pp(self.job, options) def _trigger(self, events=None): - for event in (events or ("prepare", "file")): + for event in events or ("prepare", "file"): for callback in self.job.hooks[event]: callback(self.pathfmt) class ClassifyTest(BasePostprocessorTest): - def test_classify_default(self): pp = self._create() - self.assertEqual(pp.mapping, { - ext: directory - for directory, exts in pp.DEFAULT_MAPPING.items() - for ext in exts - }) + self.assertEqual( + pp.mapping, + {ext: directory for directory, exts in pp.DEFAULT_MAPPING.items() for ext in exts}, + ) self.pathfmt.set_extension("jpg") self.pathfmt.build_path() @@ -145,33 +144,41 @@ def test_classify_noop(self): self.assertEqual(mkdirs.call_count, 0) def test_classify_custom(self): - pp = self._create({"mapping": { - "foo/bar": ["foo", "bar"], - }}) - - self.assertEqual(pp.mapping, { - "foo": "foo/bar", - "bar": "foo/bar", - }) + pp = self._create( + { + "mapping": { + "foo/bar": ["foo", "bar"], + } + } + ) + + self.assertEqual( + pp.mapping, + { + "foo": "foo/bar", + "bar": "foo/bar", + }, + ) self.pathfmt.set_extension("foo") self.pathfmt.build_path() pp.prepare(self.pathfmt) - path = os.path.join(self.dir.name, "test", "foo", "bar") - self.assertEqual(self.pathfmt.path, path + "/file.foo") - self.assertEqual(self.pathfmt.realpath, path + "/file.foo") + path = Path(self.dir.name) / "test" / "foo" / "bar" + self.assertEqual(self.pathfmt.path, str(path / "file.foo")) + self.assertEqual(self.pathfmt.realpath, str(path / "file.foo")) with patch("os.makedirs") as mkdirs: self._trigger() - mkdirs.assert_called_once_with(path, exist_ok=True) + mkdirs.assert_called_once_with(str(path), exist_ok=True) class ExecTest(BasePostprocessorTest): - def test_command_string(self): - self._create({ - "command": "echo {} {_path} {_directory} {_filename} && rm {};", - }) + self._create( + { + "command": "echo {} {_path} {_directory} {_filename} && rm {};", + } + ) with patch("gallery_dl.util.Popen") as p: i = Mock() @@ -180,18 +187,17 @@ def test_command_string(self): self._trigger(("after",)) p.assert_called_once_with( - "echo {0} {0} {1} {2} && rm {0};".format( - self.pathfmt.realpath, - self.pathfmt.realdirectory, - self.pathfmt.filename), - shell=True) + f"echo {self.pathfmt.realpath} {self.pathfmt.realpath} {self.pathfmt.realdirectory} {self.pathfmt.filename} && rm {self.pathfmt.realpath};", + shell=True, + ) i.wait.assert_called_once_with() def test_command_list(self): - self._create({ - "command": ["~/script.sh", "{category}", - "\fE _directory.upper()"], - }) + self._create( + { + "command": ["~/script.sh", "{category}", "\fE _directory.upper()"], + } + ) with patch("gallery_dl.util.Popen") as p: i = Mock() @@ -209,9 +215,11 @@ def test_command_list(self): ) def test_command_returncode(self): - self._create({ - "command": "echo {}", - }) + self._create( + { + "command": "echo {}", + } + ) with patch("gallery_dl.util.Popen") as p: i = Mock() @@ -221,15 +229,19 @@ def test_command_returncode(self): with self.assertLogs() as log: self._trigger(("after",)) - msg = ("WARNING:postprocessor.exec:'echo {}' returned with " - "non-zero exit status (123)".format(self.pathfmt.realpath)) + msg = ( + f"WARNING:postprocessor.exec:'echo {self.pathfmt.realpath}' returned with " + "non-zero exit status (123)" + ) self.assertEqual(log.output[0], msg) def test_async(self): - self._create({ - "async" : True, - "command": "echo {}", - }) + self._create( + { + "async": True, + "command": "echo {}", + } + ) with patch("gallery_dl.util.Popen") as p: i = Mock() @@ -241,7 +253,6 @@ def test_async(self): class HashTest(BasePostprocessorTest): - def test_default(self): self._create({}) @@ -251,10 +262,8 @@ def test_default(self): self._trigger() kwdict = self.pathfmt.kwdict - self.assertEqual( - "35c9c9c7c90ad764bae9e2623f522c24", kwdict["md5"], "md5") - self.assertEqual( - "14d3d804494ef4e57d72de63e4cfee761240471a", kwdict["sha1"], "sha1") + self.assertEqual("35c9c9c7c90ad764bae9e2623f522c24", kwdict["md5"], "md5") + self.assertEqual("14d3d804494ef4e57d72de63e4cfee761240471a", kwdict["sha1"], "sha1") def test_custom_hashes(self): self._create({"hashes": "sha256:a,sha512:b"}) @@ -267,11 +276,15 @@ def test_custom_hashes(self): kwdict = self.pathfmt.kwdict self.assertEqual( "4775b55be17206445d7015a5fc7656f38a74b880670523c3b175455f885f2395", - kwdict["a"], "sha256") + kwdict["a"], + "sha256", + ) self.assertEqual( "6028f9e6957f4ca929941318c4bba6258713fd5162f9e33bd10e1c456d252700" "3e1095b50736c4fd1e2deea152e3c8ecd5993462a747208e4d842659935a1c62", - kwdict["b"], "sha512") + kwdict["b"], + "sha512", + ) def test_custom_hashes_dict(self): self._create({"hashes": {"a": "sha256", "b": "sha512"}}) @@ -284,33 +297,39 @@ def test_custom_hashes_dict(self): kwdict = self.pathfmt.kwdict self.assertEqual( "4775b55be17206445d7015a5fc7656f38a74b880670523c3b175455f885f2395", - kwdict["a"], "sha256") + kwdict["a"], + "sha256", + ) self.assertEqual( "6028f9e6957f4ca929941318c4bba6258713fd5162f9e33bd10e1c456d252700" "3e1095b50736c4fd1e2deea152e3c8ecd5993462a747208e4d842659935a1c62", - kwdict["b"], "sha512") + kwdict["b"], + "sha512", + ) class MetadataTest(BasePostprocessorTest): - def test_metadata_default(self): pp = self._create() # default arguments - self.assertEqual(pp.write , pp._write_json) + self.assertEqual(pp.write, pp._write_json) self.assertEqual(pp.extension, "json") self.assertTrue(callable(pp._json_encode)) def test_metadata_json(self): - pp = self._create({ - "mode" : "json", - "extension" : "JSON", - }, { - "public" : "hello ワールド", - "_private" : "foo バー", - }) - - self.assertEqual(pp.write , pp._write_json) + pp = self._create( + { + "mode": "json", + "extension": "JSON", + }, + { + "public": "hello ワールド", + "_private": "foo バー", + }, + ) + + self.assertEqual(pp.write, pp._write_json) self.assertEqual(pp.extension, "JSON") self.assertTrue(callable(pp._json_encode)) @@ -322,31 +341,37 @@ def test_metadata_json(self): if sys.hexversion >= 0x3060000: # python 3.4 & 3.5 have random order without 'sort: True' - self.assertEqual(self._output(m), """{ + self.assertEqual( + self._output(m), + """{ "category": "test", "filename": "file", "extension": "ext", "public": "hello ワールド" } -""") +""", + ) def test_metadata_json_options(self): - pp = self._create({ - "mode" : "json", - "ascii" : True, - "sort" : True, - "separators": [",", " : "], - "private" : True, - "indent" : None, - "open" : "a", - "encoding" : "UTF-8", - "extension" : "JSON", - }, { - "public" : "hello ワールド", - "_private" : "foo バー", - }) - - self.assertEqual(pp.write , pp._write_json) + pp = self._create( + { + "mode": "json", + "ascii": True, + "sort": True, + "separators": [",", " : "], + "private": True, + "indent": None, + "open": "a", + "encoding": "UTF-8", + "extension": "JSON", + }, + { + "public": "hello ワールド", + "_private": "foo バー", + }, + ) + + self.assertEqual(pp.write, pp._write_json) self.assertEqual(pp.extension, "JSON") self.assertTrue(callable(pp._json_encode)) @@ -355,13 +380,16 @@ def test_metadata_json_options(self): path = self.pathfmt.realpath + ".JSON" m.assert_called_once_with(path, "a", encoding="UTF-8") - self.assertEqual(self._output(m), """{\ + self.assertEqual( + self._output(m), + """{\ "_private" : "foo \\u30d0\\u30fc",\ "category" : "test",\ "extension" : "ext",\ "filename" : "file",\ "public" : "hello \\u30ef\\u30fc\\u30eb\\u30c9"} -""") +""", + ) def test_metadata_tags(self): pp = self._create( @@ -417,10 +445,12 @@ def test_metadata_tags_dict(self): def test_metadata_tags_list_of_dict(self): self._create( {"mode": "tags"}, - {"tags": [ - {"g": "foobar1", "m": "foobar2", "u": True}, - {"g": None, "m": "foobarbaz", "u": [3, 4]}, - ]}, + { + "tags": [ + {"g": "foobar1", "m": "foobar2", "u": True}, + {"g": None, "m": "foobarbaz", "u": [3, 4]}, + ] + }, ) with patch("builtins.open", mock_open()) as m: self._trigger() @@ -444,10 +474,12 @@ def test(pp_info): test({"format": "{foo}\n{missing}\n"}) def test_metadata_extfmt(self): - pp = self._create({ - "extension" : "ignored", - "extension-format": "json", - }) + pp = self._create( + { + "extension": "ignored", + "extension-format": "json", + } + ) self.assertEqual(pp._filename, pp._filename_extfmt) @@ -458,9 +490,11 @@ def test_metadata_extfmt(self): m.assert_called_once_with(path, "w", encoding="utf-8") def test_metadata_extfmt_2(self): - self._create({ - "extension-format": "{extension!u}-data:{category:Res/ES/}", - }) + self._create( + { + "extension-format": "{extension!u}-data:{category:Res/ES/}", + } + ) self.pathfmt.prefix = "2." with patch("builtins.open", mock_open()) as m: @@ -470,9 +504,11 @@ def test_metadata_extfmt_2(self): m.assert_called_once_with(path, "w", encoding="utf-8") def test_metadata_directory(self): - self._create({ - "directory": "metadata", - }) + self._create( + { + "directory": "metadata", + } + ) with patch("builtins.open", mock_open()) as m: self._trigger() @@ -481,10 +517,12 @@ def test_metadata_directory(self): m.assert_called_once_with(path, "w", encoding="utf-8") def test_metadata_directory_2(self): - self._create({ - "directory" : "metadata////", - "extension-format": "json", - }) + self._create( + { + "directory": "metadata////", + "extension-format": "json", + } + ) with patch("builtins.open", mock_open()) as m: self._trigger() @@ -514,10 +552,12 @@ def test_metadata_basedirectory(self): m.assert_called_once_with(path, "w", encoding="utf-8") def test_metadata_basedirectory_custom(self): - self._create({ - "base-directory": "/home/test", - "directory": "meta", - }) + self._create( + { + "base-directory": "/home/test", + "directory": "meta", + } + ) with patch("builtins.open", mock_open()) as m: self._trigger() @@ -526,10 +566,12 @@ def test_metadata_basedirectory_custom(self): m.assert_called_once_with(path, "w", encoding="utf-8") def test_metadata_filename(self): - self._create({ - "filename" : "{category}_{filename}_/meta/\n\r.data", - "extension-format": "json", - }) + self._create( + { + "filename": "{category}_{filename}_/meta/\n\r.data", + "extension-format": "json", + } + ) with patch("builtins.open", mock_open()) as m: self._trigger() @@ -543,21 +585,27 @@ def test_metadata_stdout(self): with patch("sys.stdout", Mock()) as m: self._trigger() - self.assertEqual(self._output(m), """\ + self.assertEqual( + self._output(m), + """\ {"category": "test", "extension": "ext", "filename": "file"} -""") +""", + ) def test_metadata_modify(self): kwdict = {"foo": 0, "bar": {"bax": 1, "bay": 2, "baz": 3, "ba2": {}}} - self._create({ - "mode": "modify", - "fields": { - "foo" : "{filename}-{foo!s}", - "foo2" : "\fE bar['bax'] + 122", - "bar[\"baz\"]" : "{_now}", - "bar['ba2'][a]": "test", + self._create( + { + "mode": "modify", + "fields": { + "foo": "{filename}-{foo!s}", + "foo2": "\fE bar['bax'] + 122", + 'bar["baz"]': "{_now}", + "bar['ba2'][a]": "test", + }, }, - }, kwdict) + kwdict, + ) pdict = self.pathfmt.kwdict self.assertIsNot(kwdict, pdict) @@ -566,7 +614,7 @@ def test_metadata_modify(self): self._trigger() - self.assertEqual(pdict["foo"] , "file-0") + self.assertEqual(pdict["foo"], "file-0") self.assertEqual(pdict["foo2"], 123) self.assertEqual(pdict["bar"]["ba2"]["a"], "test") self.assertIsInstance(pdict["bar"]["baz"], datetime) @@ -580,10 +628,13 @@ def test_metadata_delete(self): "baz": {"a": 3, "b": 4}, }, } - self._create({ - "mode": "delete", - "fields": ["foo", "bar['bax']", "bar[\"baz\"][a]"], - }, kwdict) + self._create( + { + "mode": "delete", + "fields": ["foo", "bar['bax']", 'bar["baz"][a]'], + }, + kwdict, + ) pdict = self.pathfmt.kwdict self.assertIsNot(kwdict, pdict) @@ -606,8 +657,7 @@ def test_metadata_delete(self): def test_metadata_option_skip(self): self._create({"skip": True}) - with patch("builtins.open", mock_open()) as m, \ - patch("os.path.exists") as e: + with patch("builtins.open", mock_open()) as m, patch("os.path.exists") as e: e.return_value = True self._trigger() @@ -615,8 +665,7 @@ def test_metadata_option_skip(self): self.assertTrue(not m.called) self.assertTrue(not len(self._output(m))) - with patch("builtins.open", mock_open()) as m, \ - patch("os.path.exists") as e: + with patch("builtins.open", mock_open()) as m, patch("os.path.exists") as e: e.return_value = False self._trigger() @@ -630,8 +679,7 @@ def test_metadata_option_skip(self): def test_metadata_option_skip_false(self): self._create({"skip": False}) - with patch("builtins.open", mock_open()) as m, \ - patch("os.path.exists") as e: + with patch("builtins.open", mock_open()) as m, patch("os.path.exists") as e: self._trigger() self.assertTrue(not e.called) @@ -646,11 +694,14 @@ def test_metadata_option_include(self): with patch("builtins.open", mock_open()) as m: self._trigger() - self.assertEqual(self._output(m), """{ + self.assertEqual( + self._output(m), + """{ "_private": "foo バー", "filename": "file" } -""") +""", + ) def test_metadata_option_exclude(self): self._create( @@ -661,23 +712,21 @@ def test_metadata_option_exclude(self): with patch("builtins.open", mock_open()) as m: self._trigger() - self.assertEqual(self._output(m), """{ + self.assertEqual( + self._output(m), + """{ "extension": "ext", "public": "hello ワールド" } -""") +""", + ) @staticmethod def _output(mock): - return "".join( - call[1][0] - for call in mock.mock_calls - if call[0].endswith("write") - ) + return "".join(call[1][0] for call in mock.mock_calls if call[0].endswith("write")) class MtimeTest(BasePostprocessorTest): - def test_mtime_datetime(self): self._create(None, {"date": datetime(1980, 1, 1)}) self._trigger() @@ -710,7 +759,6 @@ def test_mtime_value(self): class PythonTest(BasePostprocessorTest): - def test_module(self): path = os.path.join(self.dir.name, "module.py") self._write_module(path) @@ -744,7 +792,6 @@ def calc(kwdict): class RenameTest(BasePostprocessorTest): - def _prepare(self, filename): path = self.pathfmt.realdirectory shutil.rmtree(path, ignore_errors=True) @@ -791,22 +838,29 @@ def test_rename_skip(self): with self.assertLogs("postprocessor.rename", level="WARNING") as cm: self._trigger() - self.assertTrue(cm.output[0].startswith( - "WARNING:postprocessor.rename:Not renaming " - "'12345.ext' to 'file.ext'")) + self.assertTrue( + cm.output[0].startswith( + "WARNING:postprocessor.rename:Not renaming " "'12345.ext' to 'file.ext'" + ) + ) self.assertEqual(sorted(os.listdir(path)), ["12345.ext", "file.ext"]) class ZipTest(BasePostprocessorTest): - def test_zip_default(self): pp = self._create() self.assertEqual(self.job.hooks["file"][0], pp.write_fast) self.assertEqual(pp.path, self.pathfmt.realdirectory[:-1]) self.assertEqual(pp.delete, True) - self.assertEqual(pp.args, ( - pp.path + ".zip", "a", zipfile.ZIP_STORED, True, - )) + self.assertEqual( + pp.args, + ( + pp.path + ".zip", + "a", + zipfile.ZIP_STORED, + True, + ), + ) self.assertTrue(pp.args[0].endswith("/test.zip")) def test_zip_safe(self): @@ -814,41 +868,54 @@ def test_zip_safe(self): self.assertEqual(self.job.hooks["file"][0], pp.write_safe) self.assertEqual(pp.path, self.pathfmt.realdirectory[:-1]) self.assertEqual(pp.delete, True) - self.assertEqual(pp.args, ( - pp.path + ".zip", "a", zipfile.ZIP_STORED, True, - )) + self.assertEqual( + pp.args, + ( + pp.path + ".zip", + "a", + zipfile.ZIP_STORED, + True, + ), + ) self.assertTrue(pp.args[0].endswith("/test.zip")) def test_zip_options(self): - pp = self._create({ - "keep-files": True, - "compression": "zip", - "extension": "cbz", - }) + pp = self._create( + { + "keep-files": True, + "compression": "zip", + "extension": "cbz", + } + ) self.assertEqual(pp.delete, False) - self.assertEqual(pp.args, ( - pp.path + ".cbz", "a", zipfile.ZIP_DEFLATED, True, - )) + self.assertEqual( + pp.args, + ( + pp.path + ".cbz", + "a", + zipfile.ZIP_DEFLATED, + True, + ), + ) self.assertTrue(pp.args[0].endswith("/test.cbz")) def test_zip_write(self): with tempfile.NamedTemporaryFile("w", dir=self.dir.name) as file: - pp = self._create({"files": [file.name, "_info_.json"], - "keep-files": True}) + pp = self._create({"files": [file.name, "_info_.json"], "keep-files": True}) filename = os.path.basename(file.name) file.write("foobar\n") # write dummy file with 3 different names for i in range(3): - name = "file{}.ext".format(i) + name = f"file{i}.ext" self.pathfmt.temppath = file.name self.pathfmt.filename = name self._trigger() nti = pp.zfile.NameToInfo - self.assertEqual(len(nti), i+2) + self.assertEqual(len(nti), i + 2) self.assertIn(name, nti) # check file contents @@ -877,7 +944,6 @@ def test_zip_write(self): os.unlink(pp.zfile.filename) def test_zip_write_mock(self): - def side_effect(_, name): pp.zfile.NameToInfo.add(name) @@ -889,7 +955,7 @@ def side_effect(_, name): # write 3 files for i in range(3): self.pathfmt.temppath = self.pathfmt.realdirectory + "file.ext" - self.pathfmt.filename = "file{}.ext".format(i) + self.pathfmt.filename = f"file{i}.ext" self._trigger() # write the last file a second time (should be skipped) diff --git a/test/test_results.py b/test/test_results.py index f36f7984a..b2bb40109 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2015-2023 Mike Fährmann # @@ -7,20 +6,22 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import collections +import datetime +import hashlib +import json import os +import re import sys import unittest -import re -import json -import hashlib -import datetime -import collections - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import \ - extractor, util, job, config, exception, formatter # noqa E402 - +from gallery_dl import config # noqa: E402 +from gallery_dl import exception # noqa: E402 +from gallery_dl import extractor # noqa: E402 +from gallery_dl import formatter # noqa: E402 +from gallery_dl import job # noqa: E402 +from gallery_dl import util # noqa: E402 RESULTS = os.environ.get("GDL_TEST_RESULTS") if RESULTS: @@ -30,8 +31,7 @@ # temporary issues, etc. -BROKEN = { -} +BROKEN = {} CONFIG = { "cache": { @@ -66,7 +66,6 @@ class TestExtractorResults(unittest.TestCase): - def setUp(self): setup_test_config() @@ -82,7 +81,7 @@ def tearDownClass(cls): if cls._skipped: print("\n\nSkipped tests:") for url, exc in cls._skipped: - print('- {} ("{}")'.format(url, exc)) + print(f'- {url} ("{exc}")') def assertRange(self, value, range, msg=None): if range.step > 1: @@ -104,10 +103,10 @@ def _run_test(self, result): self.assertIs(extr_url.__class__, extr_cls.__class__) if len(result) <= 2: - return # only matching + return None # only matching if auth is None: - auth = (cat in AUTH_REQUIRED) + auth = cat in AUTH_REQUIRED elif not auth: # auth explicitly disabled for key in AUTH_KEYS: @@ -121,7 +120,7 @@ def _run_test(self, result): key = key.split(".") config.set(key[:-1], key[-1], value) if "#range" in result: - config.set((), "image-range" , result["#range"]) + config.set((), "image-range", result["#range"]) config.set((), "chapter-range", result["#range"]) tjob = ResultJob(extr, content=("#sha1_content" in result)) @@ -129,7 +128,7 @@ def _run_test(self, result): if "#exception" in result: with self.assertRaises(result["#exception"], msg="#exception"): tjob.run() - return + return None try: tjob.run() @@ -137,17 +136,15 @@ def _run_test(self, result): pass except exception.HttpError as exc: exc = str(exc) - if re.match(r"'5\d\d ", exc) or \ - re.search(r"\bRead timed out\b", exc): + if re.match(r"'5\d\d ", exc) or re.search(r"\bRead timed out\b", exc): self._skipped.append((result["#url"], exc)) self.skipTest(exc) raise if result.get("#archive", True): self.assertEqual( - len(set(tjob.archive_list)), - len(tjob.archive_list), - msg="archive-id uniqueness") + len(set(tjob.archive_list)), len(tjob.archive_list), msg="archive-id uniqueness" + ) if tjob.queue: # test '_extractor' entries @@ -165,10 +162,7 @@ def _run_test(self, result): # test extraction results if "#sha1_url" in result: - self.assertEqual( - result["#sha1_url"], - tjob.url_hash.hexdigest(), - msg="#sha1_url") + self.assertEqual(result["#sha1_url"], tjob.url_hash.hexdigest(), msg="#sha1_url") if "#sha1_content" in result: expected = result["#sha1_content"] @@ -180,17 +174,15 @@ def _run_test(self, result): if "#sha1_metadata" in result: self.assertEqual( - result["#sha1_metadata"], - tjob.kwdict_hash.hexdigest(), - "#sha1_metadata") + result["#sha1_metadata"], tjob.kwdict_hash.hexdigest(), "#sha1_metadata" + ) if "#count" in result: count = result["#count"] len_urls = len(tjob.url_list) if isinstance(count, str): - self.assertRegex( - count, r"^ *(==|!=|<|<=|>|>=) *\d+ *$", msg="#count") - expr = "{} {}".format(len_urls, count) + self.assertRegex(count, r"^ *(==|!=|<|<=|>|>=) *\d+ *$", msg="#count") + expr = f"{len_urls} {count}" self.assertTrue(eval(expr), msg=expr) elif isinstance(count, range): self.assertRange(len_urls, count, msg="#count") @@ -226,7 +218,7 @@ def _test_kwdict(self, kwdict, tests, parent=None): key = key[1:] if key not in kwdict: continue - path = "{}.{}".format(parent, key) if parent else key + path = f"{parent}.{key}" if parent else key self.assertIn(key, kwdict, msg=path) value = kwdict[key] @@ -243,7 +235,7 @@ def _test_kwdict(self, kwdict, tests, parent=None): for idx, item in enumerate(test): if isinstance(item, dict): subtest = True - subpath = "{}[{}]".format(path, idx) + subpath = f"{path}[{idx}]" self._test_kwdict(value[idx], item, subpath) if not subtest: self.assertEqual(test, value, msg=path) @@ -285,12 +277,9 @@ def __init__(self, url, parent=None, content=False): else: self._update_content = lambda url, kwdict: None - self.format_directory = TestFormatter( - "".join(self.extractor.directory_fmt)).format_map - self.format_filename = TestFormatter( - self.extractor.filename_fmt).format_map - self.format_archive = TestFormatter( - self.extractor.archive_fmt).format_map + self.format_directory = TestFormatter("".join(self.extractor.directory_fmt)).format_map + self.format_filename = TestFormatter(self.extractor.filename_fmt).format_map + self.format_archive = TestFormatter(self.extractor.archive_fmt).format_map def run(self): self._init() @@ -324,8 +313,7 @@ def _update_kwdict(self, kwdict, to_list=True): if to_list: self.kwdict_list.append(kwdict.copy()) kwdict = util.filter_dict(kwdict) - self.kwdict_hash.update( - json.dumps(kwdict, sort_keys=True, default=str).encode()) + self.kwdict_hash.update(json.dumps(kwdict, sort_keys=True, default=str).encode()) def _update_archive(self, kwdict): archive_id = self.format_archive(kwdict) @@ -346,8 +334,7 @@ def _update_content(self, url, kwdict): return -class TestPathfmt(): - +class TestPathfmt: def __init__(self, hashobj): self.hashobj = hashobj self.path = "" @@ -378,7 +365,6 @@ def part_size(self): class TestFormatter(formatter.StringFormatter): - @staticmethod def _noop(_): return "" @@ -389,6 +375,7 @@ def _apply_simple(self, key, fmt): def wrap(obj): return fmt(obj[key]) + return wrap def _apply(self, key, funcs, fmt): @@ -400,6 +387,7 @@ def wrap(obj): for func in funcs: obj = func(obj) return fmt(obj) + return wrap @@ -409,16 +397,13 @@ def setup_test_config(): def load_test_config(): try: - path = os.path.join( - os.path.dirname(os.path.dirname(__file__)), - "archive", "config.json") + path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "archive", "config.json") with open(path) as fp: CONFIG.update(json.loads(fp.read())) except FileNotFoundError: pass except Exception as exc: - sys.exit("Error when loading {}: {}: {}".format( - path, exc.__class__.__name__, exc)) + sys.exit(f"Error when loading {path}: {exc.__class__.__name__}: {exc}") def result_categories(result): @@ -432,6 +417,7 @@ def result_categories(result): def generate_tests(): """Dynamically generate extractor unittests""" + def _generate_method(result): def test(self): print("\n" + result["#url"]) @@ -446,6 +432,7 @@ def test(self): else: self._skipped.append((result["#url"], "manual skip")) self.skipTest(exc) + return test # enable selective testing for direct calls @@ -455,8 +442,7 @@ def test(self): if category.startswith("+"): basecategory = category[1:].lower() - tests = [t for t in results.all() - if result_categories(t)[0].lower() == basecategory] + tests = [t for t in results.all() if result_categories(t)[0].lower() == basecategory] else: tests = results.category(category) @@ -466,11 +452,9 @@ def test(self): tests = [t for t in tests if url in t["#url"]] elif subcategory.startswith("~"): com = subcategory[1:] - tests = [t for t in tests - if "#comment" in t and com in t["#comment"].lower()] + tests = [t for t in tests if "#comment" in t and com in t["#comment"].lower()] else: - tests = [t for t in tests - if result_categories(t)[-1] == subcategory] + tests = [t for t in tests if result_categories(t)[-1] == subcategory] else: tests = results.all() @@ -478,12 +462,12 @@ def test(self): enum = collections.defaultdict(int) for result in tests: base, cat, sub = result_categories(result) - name = "{}_{}".format(cat, sub) + name = f"{cat}_{sub}" enum[name] += 1 method = _generate_method(result) method.__doc__ = result["#url"] - method.__name__ = "test_{}_{}".format(name, enum[name]) + method.__name__ = f"test_{name}_{enum[name]}" setattr(TestExtractorResults, method.__name__, method) diff --git a/test/test_text.py b/test/test_text.py index 1b19c4742..cee7c0cf2 100644 --- a/test/test_text.py +++ b/test/test_text.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2015-2022 Mike Fährmann # @@ -7,22 +6,20 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import datetime import os import sys import unittest -import datetime - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import text, util # noqa E402 - +from gallery_dl import text # noqa: E402 +from gallery_dl import util # noqa: E402 INVALID = ((), [], {}, None, 1, 2.3) INVALID_ALT = ((), [], {}, None, "") class TestText(unittest.TestCase): - def test_remove_html(self, f=text.remove_html): result = "Hello World." @@ -31,8 +28,7 @@ def test_remove_html(self, f=text.remove_html): self.assertEqual(f("Hello World."), result) self.assertEqual(f(" Hello World. "), result) self.assertEqual(f("Hello
          World."), result) - self.assertEqual( - f("
          HelloWorld.
          "), result) + self.assertEqual(f("
          HelloWorld.
          "), result) # empty HTML self.assertEqual(f("
          "), "") @@ -56,12 +52,10 @@ def test_split_html(self, f=text.split_html): self.assertEqual(f(" Hello World. "), ["Hello World."]) self.assertEqual(f("Hello
          World."), result) self.assertEqual(f(" Hello
          World. "), result) - self.assertEqual( - f("
          HelloWorld.
          "), result) + self.assertEqual(f("
          HelloWorld.
          "), result) # escaped HTML entities - self.assertEqual( - f("<foo> <bar> "), ["", ""]) + self.assertEqual(f("<foo> <bar> "), ["", ""]) # empty HTML self.assertEqual(f("
          "), empty) @@ -121,17 +115,17 @@ def test_ensure_http_scheme(self, f=text.ensure_http_scheme): def test_root_from_url(self, f=text.root_from_url): result = "https://example.org" - self.assertEqual(f("https://example.org") , result) - self.assertEqual(f("https://example.org/") , result) + self.assertEqual(f("https://example.org"), result) + self.assertEqual(f("https://example.org/"), result) self.assertEqual(f("https://example.org/path"), result) - self.assertEqual(f("example.org/") , result) - self.assertEqual(f("example.org/path/") , result) + self.assertEqual(f("example.org/"), result) + self.assertEqual(f("example.org/path/"), result) result = "http://example.org" - self.assertEqual(f("http://example.org") , result) - self.assertEqual(f("http://example.org/") , result) + self.assertEqual(f("http://example.org"), result) + self.assertEqual(f("http://example.org/"), result) self.assertEqual(f("http://example.org/path/"), result) - self.assertEqual(f("example.org/", "http://") , result) + self.assertEqual(f("example.org/", "http://"), result) def test_filename_from_url(self, f=text.filename_from_url): result = "filename.ext" @@ -142,8 +136,7 @@ def test_filename_from_url(self, f=text.filename_from_url): self.assertEqual(f("/filename.ext"), result) self.assertEqual(f("example.org/filename.ext"), result) self.assertEqual(f("http://example.org/v2/filename.ext"), result) - self.assertEqual( - f("http://example.org/v2/filename.ext?param=value#frag"), result) + self.assertEqual(f("http://example.org/v2/filename.ext?param=value#frag"), result) # invalid arguments for value in INVALID: @@ -159,8 +152,7 @@ def test_ext_from_url(self, f=text.ext_from_url): self.assertEqual(f("/filename.ExT"), result) self.assertEqual(f("example.org/filename.ext"), result) self.assertEqual(f("http://example.org/v2/filename.ext"), result) - self.assertEqual( - f("http://example.org/v2/filename.ext?param=value#frag"), result) + self.assertEqual(f("http://example.org/v2/filename.ext?param=value#frag"), result) # invalid arguments for value in INVALID: @@ -176,8 +168,7 @@ def test_nameext_from_url(self, f=text.nameext_from_url): self.assertEqual(f("/filename.ExT"), result) self.assertEqual(f("example.org/filename.ext"), result) self.assertEqual(f("http://example.org/v2/filename.ext"), result) - self.assertEqual( - f("http://example.org/v2/filename.ext?param=value#frag"), result) + self.assertEqual(f("http://example.org/v2/filename.ext?param=value#frag"), result) # long "extension" fn = "httpswww.example.orgpath-path-path-path-path-path-path-path" @@ -189,7 +180,7 @@ def test_nameext_from_url(self, f=text.nameext_from_url): def test_extract(self, f=text.extract): txt = "" - self.assertEqual(f(txt, "<", ">"), ("a" , 3)) + self.assertEqual(f(txt, "<", ">"), ("a", 3)) self.assertEqual(f(txt, "X", ">"), (None, 0)) self.assertEqual(f(txt, "<", "X"), (None, 0)) @@ -201,9 +192,9 @@ def test_extract(self, f=text.extract): # invalid arguments for value in INVALID: - self.assertEqual(f(value, "<" , ">") , (None, 0)) - self.assertEqual(f(txt , value, ">") , (None, 0)) - self.assertEqual(f(txt , "<" , value), (None, 0)) + self.assertEqual(f(value, "<", ">"), (None, 0)) + self.assertEqual(f(txt, value, ">"), (None, 0)) + self.assertEqual(f(txt, "<", value), (None, 0)) def test_extr(self, f=text.extr): txt = "" @@ -219,13 +210,13 @@ def test_extr(self, f=text.extr): # invalid arguments for value in INVALID: - self.assertEqual(f(value, "<" , ">") , "") - self.assertEqual(f(txt , value, ">") , "") - self.assertEqual(f(txt , "<" , value), "") + self.assertEqual(f(value, "<", ">"), "") + self.assertEqual(f(txt, value, ">"), "") + self.assertEqual(f(txt, "<", value), "") def test_rextract(self, f=text.rextract): txt = "" - self.assertEqual(f(txt, "<", ">"), ("b" , 3)) + self.assertEqual(f(txt, "<", ">"), ("b", 3)) self.assertEqual(f(txt, "X", ">"), (None, -1)) self.assertEqual(f(txt, "<", "X"), (None, -1)) @@ -237,15 +228,14 @@ def test_rextract(self, f=text.rextract): # invalid arguments for value in INVALID: - self.assertEqual(f(value, "<" , ">") , (None, -1)) - self.assertEqual(f(txt , value, ">") , (None, -1)) - self.assertEqual(f(txt , "<" , value), (None, -1)) + self.assertEqual(f(value, "<", ">"), (None, -1)) + self.assertEqual(f(txt, value, ">"), (None, -1)) + self.assertEqual(f(txt, "<", value), (None, -1)) def test_extract_all(self, f=text.extract_all): txt = "[c][b][a]: xyz! [d][e" - self.assertEqual( - f(txt, ()), ({}, 0)) + self.assertEqual(f(txt, ()), ({}, 0)) self.assertEqual( f(txt, (("C", "[", "]"), ("B", "[", "]"), ("A", "[", "]"))), ({"A": "a", "B": "b", "C": "c"}, 9), @@ -289,16 +279,11 @@ def test_extract_iter(self, f=text.extract_iter): def g(*args): return list(f(*args)) - self.assertEqual( - g("", "[", "]"), []) - self.assertEqual( - g("[a]", "[", "]"), ["a"]) - self.assertEqual( - g(txt, "[", "]"), ["c", "b", "a", "d"]) - self.assertEqual( - g(txt, "X", "X"), []) - self.assertEqual( - g(txt, "[", "]", 6), ["a", "d"]) + self.assertEqual(g("", "[", "]"), []) + self.assertEqual(g("[a]", "[", "]"), ["a"]) + self.assertEqual(g(txt, "[", "]"), ["c", "b", "a", "d"]) + self.assertEqual(g(txt, "X", "X"), []) + self.assertEqual(g(txt, "[", "]", 6), ["a", "d"]) def test_extract_from(self, f=text.extract_from): txt = "[c][b][a]: xyz! [d][e" @@ -439,9 +424,9 @@ def test_parse_timestamp(self, f=text.parse_timestamp): null = util.datetime_utcfromtimestamp(0) value = util.datetime_utcfromtimestamp(1555816235) - self.assertEqual(f(0) , null) - self.assertEqual(f("0") , null) - self.assertEqual(f(1555816235) , value) + self.assertEqual(f(0), null) + self.assertEqual(f("0"), null) + self.assertEqual(f(1555816235), value) self.assertEqual(f("1555816235"), value) for value in INVALID_ALT: @@ -452,8 +437,8 @@ def test_parse_datetime(self, f=text.parse_datetime): null = util.datetime_utcfromtimestamp(0) self.assertEqual(f("1970-01-01T00:00:00+00:00"), null) - self.assertEqual(f("1970-01-01T00:00:00+0000") , null) - self.assertEqual(f("1970.01.01", "%Y.%m.%d") , null) + self.assertEqual(f("1970-01-01T00:00:00+0000"), null) + self.assertEqual(f("1970.01.01", "%Y.%m.%d"), null) self.assertEqual( f("2019-05-07T21:25:02+09:00"), diff --git a/test/test_util.py b/test/test_util.py index 0a9ff423d..d7e90d3e7 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2015-2023 Mike Fährmann # @@ -7,26 +6,26 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import os -import sys -import unittest - +import datetime +import http.cookiejar import io -import time +import itertools +import os +import platform import random import string -import datetime -import platform +import sys import tempfile -import itertools -import http.cookiejar +import time +import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import util, text, exception # noqa E402 +from gallery_dl import exception # noqa: E402 +from gallery_dl import text # noqa: E402 +from gallery_dl import util # noqa: E402 class TestRange(unittest.TestCase): - def test_parse_empty(self, f=util.RangePredicate._parse): self.assertEqual(f(""), []) self.assertEqual(f([]), []) @@ -36,9 +35,7 @@ def test_parse_digit(self, f=util.RangePredicate._parse): self.assertEqual( f("2, 3, 4"), - [range(2, 3), - range(3, 4), - range(4, 5)], + [range(2, 3), range(3, 4), range(4, 5)], ) def test_parse_range(self, f=util.RangePredicate._parse): @@ -49,44 +46,38 @@ def test_parse_range(self, f=util.RangePredicate._parse): self.assertEqual( f("-2,4,6-8,10-"), - [range(1, 3), - range(4, 5), - range(6, 9), - range(10, sys.maxsize)], + [range(1, 3), range(4, 5), range(6, 9), range(10, sys.maxsize)], ) self.assertEqual( f(" - 3 , 4- 4, 2-6"), - [range(1, 4), - range(4, 5), - range(2, 7)], + [range(1, 4), range(4, 5), range(2, 7)], ) def test_parse_slice(self, f=util.RangePredicate._parse): - self.assertEqual(f("2:4") , [range(2, 4)]) - self.assertEqual(f("3::") , [range(3, sys.maxsize)]) - self.assertEqual(f(":4:") , [range(1, 4)]) - self.assertEqual(f("::5") , [range(1, sys.maxsize, 5)]) - self.assertEqual(f("::") , [range(1, sys.maxsize)]) + self.assertEqual(f("2:4"), [range(2, 4)]) + self.assertEqual(f("3::"), [range(3, sys.maxsize)]) + self.assertEqual(f(":4:"), [range(1, 4)]) + self.assertEqual(f("::5"), [range(1, sys.maxsize, 5)]) + self.assertEqual(f("::"), [range(1, sys.maxsize)]) self.assertEqual(f("2:3:4"), [range(2, 3, 4)]) self.assertEqual( f("2:4, 4:, :4, :4:, ::4"), - [range(2, 4), - range(4, sys.maxsize), - range(1, 4), - range(1, 4), - range(1, sys.maxsize, 4)], + [ + range(2, 4), + range(4, sys.maxsize), + range(1, 4), + range(1, 4), + range(1, sys.maxsize, 4), + ], ) self.assertEqual( f(" : 3 , 4: 4, 2:6"), - [range(1, 3), - range(4, 4), - range(2, 6)], + [range(1, 3), range(4, 4), range(2, 6)], ) class TestPredicate(unittest.TestCase): - def test_range_predicate(self): dummy = None @@ -136,10 +127,8 @@ def test_filter_predicate(self): with self.assertRaises(SyntaxError): util.FilterPredicate("(") - self.assertFalse( - util.FilterPredicate("a > 1")(url, {"a": None})) - self.assertFalse( - util.FilterPredicate("b > 1")(url, {"a": 2})) + self.assertFalse(util.FilterPredicate("a > 1")(url, {"a": None})) + self.assertFalse(util.FilterPredicate("b > 1")(url, {"a": 2})) pred = util.FilterPredicate(["a < 3", "b < 4", "c < 5"]) self.assertTrue(pred(url, {"a": 2, "b": 3, "c": 4})) @@ -156,44 +145,48 @@ def test_build_predicate(self): pred = util.build_predicate([util.UniquePredicate()]) self.assertIsInstance(pred, util.UniquePredicate) - pred = util.build_predicate([util.UniquePredicate(), - util.UniquePredicate()]) + pred = util.build_predicate([util.UniquePredicate(), util.UniquePredicate()]) self.assertIs(pred.func, util.chain_predicates) class TestISO639_1(unittest.TestCase): - def test_code_to_language(self): d = "default" - self._run_test(util.code_to_language, { - ("en",): "English", - ("FR",): "French", - ("ja",): "Japanese", - ("xx",): None, - ("" ,): None, - (None,): None, - ("en", d): "English", - ("FR", d): "French", - ("xx", d): d, - ("" , d): d, - (None, d): d, - }) + self._run_test( + util.code_to_language, + { + ("en",): "English", + ("FR",): "French", + ("ja",): "Japanese", + ("xx",): None, + ("",): None, + (None,): None, + ("en", d): "English", + ("FR", d): "French", + ("xx", d): d, + ("", d): d, + (None, d): d, + }, + ) def test_language_to_code(self): d = "default" - self._run_test(util.language_to_code, { - ("English",): "en", - ("fRENch",): "fr", - ("Japanese",): "ja", - ("xx",): None, - ("" ,): None, - (None,): None, - ("English", d): "en", - ("fRENch", d): "fr", - ("xx", d): d, - ("" , d): d, - (None, d): d, - }) + self._run_test( + util.language_to_code, + { + ("English",): "en", + ("fRENch",): "fr", + ("Japanese",): "ja", + ("xx",): None, + ("",): None, + (None,): None, + ("English", d): "en", + ("fRENch", d): "fr", + ("xx", d): d, + ("", d): d, + (None, d): d, + }, + ) def _run_test(self, func, tests): for args, result in tests.items(): @@ -201,9 +194,7 @@ def _run_test(self, func, tests): class TestCookiesTxt(unittest.TestCase): - def test_cookiestxt_load(self): - def _assert(content, expected): cookies = util.cookiestxt_load(io.StringIO(content, None)) for c, e in zip(cookies, expected): @@ -238,16 +229,11 @@ def _assert(content, expected): "www.example.org FALSE / FALSE n4 \n" "www.example.org FALSE /path FALSE 100 n5 v5\n", [ - self._cookie( - "n1", "v1", ".example.org", True, "/", False), - self._cookie( - "n2", "v2", ".example.org", True, "/", True, 2145945600), - self._cookie( - "n3", None, ".example.org", True, "/path", False), - self._cookie( - "n4", "" , "www.example.org", False, "/", False), - self._cookie( - "n5", "v5", "www.example.org", False, "/path", False, 100), + self._cookie("n1", "v1", ".example.org", True, "/", False), + self._cookie("n2", "v2", ".example.org", True, "/", True, 2145945600), + self._cookie("n3", None, ".example.org", True, "/path", False), + self._cookie("n4", "", "www.example.org", False, "/", False), + self._cookie("n5", "v5", "www.example.org", False, "/path", False, 100), ], ) @@ -255,7 +241,6 @@ def _assert(content, expected): util.cookiestxt_load("example.org\tTRUE\t/\tTRUE\t0\tname") def test_cookiestxt_store(self): - def _assert(cookies, expected): fp = io.StringIO(newline=None) util.cookiestxt_store(fp, cookies) @@ -264,23 +249,16 @@ def _assert(cookies, expected): _assert([], "# Netscape HTTP Cookie File\n\n") _assert( [self._cookie("name", "value", ".example.org")], - "# Netscape HTTP Cookie File\n\n" - ".example.org\tTRUE\t/\tTRUE\t0\tname\tvalue\n", + "# Netscape HTTP Cookie File\n\n" ".example.org\tTRUE\t/\tTRUE\t0\tname\tvalue\n", ) _assert( [ - self._cookie( - "n1", "v1", ".example.org", True, "/", False), - self._cookie( - "n2", "v2", ".example.org", True, "/", True, 2145945600), - self._cookie( - "n3", None, ".example.org", True, "/path", False), - self._cookie( - "n4", "" , "www.example.org", False, "/", False), - self._cookie( - "n5", "v5", "www.example.org", False, "/path", False, 100), - self._cookie( - "n6", "v6", "", False), + self._cookie("n1", "v1", ".example.org", True, "/", False), + self._cookie("n2", "v2", ".example.org", True, "/", True, 2145945600), + self._cookie("n3", None, ".example.org", True, "/path", False), + self._cookie("n4", "", "www.example.org", False, "/", False), + self._cookie("n5", "v5", "www.example.org", False, "/path", False, 100), + self._cookie("n6", "v6", "", False), ], "# Netscape HTTP Cookie File\n" "\n" @@ -291,17 +269,30 @@ def _assert(cookies, expected): "www.example.org FALSE /path FALSE 100 n5 v5\n", ) - def _cookie(self, name, value, domain, domain_specified=True, - path="/", secure=True, expires=None): + def _cookie( + self, name, value, domain, domain_specified=True, path="/", secure=True, expires=None + ): return http.cookiejar.Cookie( - 0, name, value, None, False, - domain, domain_specified, domain.startswith("."), - path, False, secure, expires, False, None, None, {}, + 0, + name, + value, + None, + False, + domain, + domain_specified, + domain.startswith("."), + path, + False, + secure, + expires, + False, + None, + None, + {}, ) class TestCompileExpression(unittest.TestCase): - def test_compile_expression(self): expr = util.compile_expression("1 + 2 * 3") self.assertEqual(expr(), 7) @@ -391,7 +382,6 @@ def hash(value): class TestOther(unittest.TestCase): - def test_bencode(self): self.assertEqual(util.bencode(0), "") self.assertEqual(util.bencode(123), "123") @@ -414,34 +404,22 @@ def test_bencode_bdecode(self): def test_advance(self): items = range(5) - self.assertCountEqual( - util.advance(items, 0), items) - self.assertCountEqual( - util.advance(items, 3), range(3, 5)) - self.assertCountEqual( - util.advance(items, 9), []) - self.assertCountEqual( - util.advance(util.advance(items, 1), 2), range(3, 5)) + self.assertCountEqual(util.advance(items, 0), items) + self.assertCountEqual(util.advance(items, 3), range(3, 5)) + self.assertCountEqual(util.advance(items, 9), []) + self.assertCountEqual(util.advance(util.advance(items, 1), 2), range(3, 5)) def test_unique(self): - self.assertSequenceEqual( - list(util.unique("")), "") - self.assertSequenceEqual( - list(util.unique("AABBCC")), "ABC") - self.assertSequenceEqual( - list(util.unique("ABABABCAABBCC")), "ABC") - self.assertSequenceEqual( - list(util.unique([1, 2, 1, 3, 2, 1])), [1, 2, 3]) + self.assertSequenceEqual(list(util.unique("")), "") + self.assertSequenceEqual(list(util.unique("AABBCC")), "ABC") + self.assertSequenceEqual(list(util.unique("ABABABCAABBCC")), "ABC") + self.assertSequenceEqual(list(util.unique([1, 2, 1, 3, 2, 1])), [1, 2, 3]) def test_unique_sequence(self): - self.assertSequenceEqual( - list(util.unique_sequence("")), "") - self.assertSequenceEqual( - list(util.unique_sequence("AABBCC")), "ABC") - self.assertSequenceEqual( - list(util.unique_sequence("ABABABCAABBCC")), "ABABABCABC") - self.assertSequenceEqual( - list(util.unique_sequence([1, 2, 1, 3, 2, 1])), [1, 2, 1, 3, 2, 1]) + self.assertSequenceEqual(list(util.unique_sequence("")), "") + self.assertSequenceEqual(list(util.unique_sequence("AABBCC")), "ABC") + self.assertSequenceEqual(list(util.unique_sequence("ABABABCAABBCC")), "ABABABCABC") + self.assertSequenceEqual(list(util.unique_sequence([1, 2, 1, 3, 2, 1])), [1, 2, 1, 3, 2, 1]) def test_contains(self): c = [1, "2", 3, 4, "5", "foo"] @@ -485,44 +463,28 @@ def test_noop(self): self.assertEqual(util.noop(), None) def test_md5(self): - self.assertEqual(util.md5(b""), - "d41d8cd98f00b204e9800998ecf8427e") - self.assertEqual(util.md5(b"hello"), - "5d41402abc4b2a76b9719d911017c592") - - self.assertEqual(util.md5(""), - "d41d8cd98f00b204e9800998ecf8427e") - self.assertEqual(util.md5("hello"), - "5d41402abc4b2a76b9719d911017c592") - self.assertEqual(util.md5("ワルド"), - "051f29cd6c942cf110a0ccc5729871d2") - - self.assertEqual(util.md5(0), - "d41d8cd98f00b204e9800998ecf8427e") - self.assertEqual(util.md5(()), - "d41d8cd98f00b204e9800998ecf8427e") - self.assertEqual(util.md5(None), - "d41d8cd98f00b204e9800998ecf8427e") + self.assertEqual(util.md5(b""), "d41d8cd98f00b204e9800998ecf8427e") + self.assertEqual(util.md5(b"hello"), "5d41402abc4b2a76b9719d911017c592") + + self.assertEqual(util.md5(""), "d41d8cd98f00b204e9800998ecf8427e") + self.assertEqual(util.md5("hello"), "5d41402abc4b2a76b9719d911017c592") + self.assertEqual(util.md5("ワルド"), "051f29cd6c942cf110a0ccc5729871d2") + + self.assertEqual(util.md5(0), "d41d8cd98f00b204e9800998ecf8427e") + self.assertEqual(util.md5(()), "d41d8cd98f00b204e9800998ecf8427e") + self.assertEqual(util.md5(None), "d41d8cd98f00b204e9800998ecf8427e") def test_sha1(self): - self.assertEqual(util.sha1(b""), - "da39a3ee5e6b4b0d3255bfef95601890afd80709") - self.assertEqual(util.sha1(b"hello"), - "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d") - - self.assertEqual(util.sha1(""), - "da39a3ee5e6b4b0d3255bfef95601890afd80709") - self.assertEqual(util.sha1("hello"), - "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d") - self.assertEqual(util.sha1("ワルド"), - "0cbe319081aa0e9298448ec2bb16df8c494aa04e") - - self.assertEqual(util.sha1(0), - "da39a3ee5e6b4b0d3255bfef95601890afd80709") - self.assertEqual(util.sha1(()), - "da39a3ee5e6b4b0d3255bfef95601890afd80709") - self.assertEqual(util.sha1(None), - "da39a3ee5e6b4b0d3255bfef95601890afd80709") + self.assertEqual(util.sha1(b""), "da39a3ee5e6b4b0d3255bfef95601890afd80709") + self.assertEqual(util.sha1(b"hello"), "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d") + + self.assertEqual(util.sha1(""), "da39a3ee5e6b4b0d3255bfef95601890afd80709") + self.assertEqual(util.sha1("hello"), "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d") + self.assertEqual(util.sha1("ワルド"), "0cbe319081aa0e9298448ec2bb16df8c494aa04e") + + self.assertEqual(util.sha1(0), "da39a3ee5e6b4b0d3255bfef95601890afd80709") + self.assertEqual(util.sha1(()), "da39a3ee5e6b4b0d3255bfef95601890afd80709") + self.assertEqual(util.sha1(None), "da39a3ee5e6b4b0d3255bfef95601890afd80709") def test_import_file(self): module = util.import_file("datetime") @@ -544,7 +506,6 @@ def test_import_file(self): self.assertIs(module.datetime, datetime) def test_build_duration_func(self, f=util.build_duration_func): - def test_single(df, v): for _ in range(10): self.assertEqual(df(), v) @@ -575,54 +536,52 @@ def test_range(df, lower, upper): def test_extractor_filter(self): # empty func = util.build_extractor_filter("") - self.assertEqual(func(TestExtractor) , True) + self.assertEqual(func(TestExtractor), True) self.assertEqual(func(TestExtractorParent), True) - self.assertEqual(func(TestExtractorAlt) , True) + self.assertEqual(func(TestExtractorAlt), True) # category func = util.build_extractor_filter("test_category") - self.assertEqual(func(TestExtractor) , False) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), False) - self.assertEqual(func(TestExtractorAlt) , True) + self.assertEqual(func(TestExtractorAlt), True) # subcategory func = util.build_extractor_filter("*:test_subcategory") - self.assertEqual(func(TestExtractor) , False) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), True) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) # basecategory func = util.build_extractor_filter("test_basecategory") - self.assertEqual(func(TestExtractor) , False) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), False) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) # category-subcategory pair func = util.build_extractor_filter("test_category:test_subcategory") - self.assertEqual(func(TestExtractor) , False) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), True) - self.assertEqual(func(TestExtractorAlt) , True) + self.assertEqual(func(TestExtractorAlt), True) # combination - func = util.build_extractor_filter( - ["test_category", "*:test_subcategory"]) - self.assertEqual(func(TestExtractor) , False) + func = util.build_extractor_filter(["test_category", "*:test_subcategory"]) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), False) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) # whitelist - func = util.build_extractor_filter( - "test_category:test_subcategory", negate=False) - self.assertEqual(func(TestExtractor) , True) + func = util.build_extractor_filter("test_category:test_subcategory", negate=False) + self.assertEqual(func(TestExtractor), True) self.assertEqual(func(TestExtractorParent), False) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) func = util.build_extractor_filter( - ["test_category:test_subcategory", "*:test_subcategory_parent"], - negate=False) - self.assertEqual(func(TestExtractor) , True) + ["test_category:test_subcategory", "*:test_subcategory_parent"], negate=False + ) + self.assertEqual(func(TestExtractor), True) self.assertEqual(func(TestExtractorParent), True) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) def test_generate_token(self): tokens = set() @@ -638,35 +597,33 @@ def test_generate_token(self): self.assertRegex(token, r"^[0-9a-f]+$") def test_format_value(self): - self.assertEqual(util.format_value(0) , "0") - self.assertEqual(util.format_value(1) , "1") - self.assertEqual(util.format_value(12) , "12") - self.assertEqual(util.format_value(123) , "123") - self.assertEqual(util.format_value(1234) , "1.23k") - self.assertEqual(util.format_value(12345) , "12.34k") - self.assertEqual(util.format_value(123456) , "123.45k") - self.assertEqual(util.format_value(1234567) , "1.23M") - self.assertEqual(util.format_value(12345678) , "12.34M") - self.assertEqual(util.format_value(123456789) , "123.45M") + self.assertEqual(util.format_value(0), "0") + self.assertEqual(util.format_value(1), "1") + self.assertEqual(util.format_value(12), "12") + self.assertEqual(util.format_value(123), "123") + self.assertEqual(util.format_value(1234), "1.23k") + self.assertEqual(util.format_value(12345), "12.34k") + self.assertEqual(util.format_value(123456), "123.45k") + self.assertEqual(util.format_value(1234567), "1.23M") + self.assertEqual(util.format_value(12345678), "12.34M") + self.assertEqual(util.format_value(123456789), "123.45M") self.assertEqual(util.format_value(1234567890), "1.23G") def test_combine_dict(self): - self.assertEqual( - util.combine_dict({}, {}), - {}) - self.assertEqual( - util.combine_dict({1: 1, 2: 2}, {2: 4, 4: 8}), - {1: 1, 2: 4, 4: 8}) + self.assertEqual(util.combine_dict({}, {}), {}) + self.assertEqual(util.combine_dict({1: 1, 2: 2}, {2: 4, 4: 8}), {1: 1, 2: 4, 4: 8}) self.assertEqual( util.combine_dict( - {1: {11: 22, 12: 24}, 2: {13: 26, 14: 28}}, - {1: {11: 33, 13: 39}, 2: "str"}), - {1: {11: 33, 12: 24, 13: 39}, 2: "str"}) + {1: {11: 22, 12: 24}, 2: {13: 26, 14: 28}}, {1: {11: 33, 13: 39}, 2: "str"} + ), + {1: {11: 33, 12: 24, 13: 39}, 2: "str"}, + ) self.assertEqual( util.combine_dict( - {1: {2: {3: {4: {"1": "a", "2": "b"}}}}}, - {1: {2: {3: {4: {"1": "A", "3": "C"}}}}}), - {1: {2: {3: {4: {"1": "A", "2": "b", "3": "C"}}}}}) + {1: {2: {3: {4: {"1": "a", "2": "b"}}}}}, {1: {2: {3: {4: {"1": "A", "3": "C"}}}}} + ), + {1: {2: {3: {4: {"1": "A", "2": "b", "3": "C"}}}}}, + ) def test_transform_dict(self): d = {} @@ -675,13 +632,11 @@ def test_transform_dict(self): d = {1: 123, 2: "123", 3: True, 4: None} util.transform_dict(d, str) - self.assertEqual( - d, {1: "123", 2: "123", 3: "True", 4: "None"}) + self.assertEqual(d, {1: "123", 2: "123", 3: "True", 4: "None"}) d = {1: 123, 2: "123", 3: "foo", 4: {11: 321, 12: "321", 13: "bar"}} util.transform_dict(d, text.parse_int) - self.assertEqual( - d, {1: 123, 2: 123, 3: 0, 4: {11: 321, 12: 321, 13: 0}}) + self.assertEqual(d, {1: 123, 2: 123, 3: 0, 4: {11: 321, 12: 321, 13: 0}}) def test_filter_dict(self): d = {} @@ -699,13 +654,11 @@ def test_filter_dict(self): self.assertEqual(r, {"foo": 123}) def test_enumerate_reversed(self): - seq = [11, 22, 33] result = [(3, 33), (2, 22), (1, 11)] def gen(): - for i in seq: - yield i + yield from seq def gen_2(): yield from seq @@ -715,75 +668,62 @@ def assertEqual(it1, it2): for i1, i2 in itertools.zip_longest(it1, it2): ae(i1, i2) - assertEqual( - util.enumerate_reversed(seq), [(2, 33), (1, 22), (0, 11)]) - assertEqual( - util.enumerate_reversed(seq, 1), result) - assertEqual( - util.enumerate_reversed(seq, 2), [(4, 33), (3, 22), (2, 11)]) - - assertEqual( - util.enumerate_reversed(gen(), 0, len(seq)), - [(2, 33), (1, 22), (0, 11)]) - assertEqual( - util.enumerate_reversed(gen(), 1, len(seq)), result) - assertEqual( - util.enumerate_reversed(gen_2(), 1, len(seq)), result) - assertEqual( - util.enumerate_reversed(gen_2(), 2, len(seq)), - [(4, 33), (3, 22), (2, 11)]) + assertEqual(util.enumerate_reversed(seq), [(2, 33), (1, 22), (0, 11)]) + assertEqual(util.enumerate_reversed(seq, 1), result) + assertEqual(util.enumerate_reversed(seq, 2), [(4, 33), (3, 22), (2, 11)]) + + assertEqual(util.enumerate_reversed(gen(), 0, len(seq)), [(2, 33), (1, 22), (0, 11)]) + assertEqual(util.enumerate_reversed(gen(), 1, len(seq)), result) + assertEqual(util.enumerate_reversed(gen_2(), 1, len(seq)), result) + assertEqual(util.enumerate_reversed(gen_2(), 2, len(seq)), [(4, 33), (3, 22), (2, 11)]) def test_number_to_string(self, f=util.number_to_string): - self.assertEqual(f(1) , "1") - self.assertEqual(f(1.0) , "1.0") - self.assertEqual(f("1.0") , "1.0") - self.assertEqual(f([1]) , [1]) + self.assertEqual(f(1), "1") + self.assertEqual(f(1.0), "1.0") + self.assertEqual(f("1.0"), "1.0") + self.assertEqual(f([1]), [1]) self.assertEqual(f({1: 2}), {1: 2}) - self.assertEqual(f(True) , True) - self.assertEqual(f(None) , None) + self.assertEqual(f(True), True) + self.assertEqual(f(None), None) def test_to_string(self, f=util.to_string): - self.assertEqual(f(1) , "1") - self.assertEqual(f(1.0) , "1.0") + self.assertEqual(f(1), "1") + self.assertEqual(f(1.0), "1.0") self.assertEqual(f("1.0"), "1.0") - self.assertEqual(f("") , "") - self.assertEqual(f(None) , "") - self.assertEqual(f(0) , "") + self.assertEqual(f(""), "") + self.assertEqual(f(None), "") + self.assertEqual(f(0), "") self.assertEqual(f(["a"]), "a") - self.assertEqual(f([1]) , "1") + self.assertEqual(f([1]), "1") self.assertEqual(f(["a", "b", "c"]), "a, b, c") self.assertEqual(f([1, 2, 3]), "1, 2, 3") def test_datetime_to_timestamp(self, f=util.datetime_to_timestamp): self.assertEqual(f(util.EPOCH), 0.0) self.assertEqual(f(datetime.datetime(2010, 1, 1)), 1262304000.0) - self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)), - 1262304000.128000) + self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)), 1262304000.128000) with self.assertRaises(TypeError): f(None) - def test_datetime_to_timestamp_string( - self, f=util.datetime_to_timestamp_string): + def test_datetime_to_timestamp_string(self, f=util.datetime_to_timestamp_string): self.assertEqual(f(util.EPOCH), "0") self.assertEqual(f(datetime.datetime(2010, 1, 1)), "1262304000") self.assertEqual(f(None), "") - def test_datetime_from_timestamp( - self, f=util.datetime_from_timestamp): + def test_datetime_from_timestamp(self, f=util.datetime_from_timestamp): self.assertEqual(f(0.0), util.EPOCH) self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) - self.assertEqual(f(1262304000.128000).replace(microsecond=0), - datetime.datetime(2010, 1, 1, 0, 0, 0)) + self.assertEqual( + f(1262304000.128000).replace(microsecond=0), datetime.datetime(2010, 1, 1, 0, 0, 0) + ) - def test_datetime_utcfromtimestamp( - self, f=util.datetime_utcfromtimestamp): + def test_datetime_utcfromtimestamp(self, f=util.datetime_utcfromtimestamp): self.assertEqual(f(0.0), util.EPOCH) self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) - def test_datetime_utcnow( - self, f=util.datetime_utcnow): + def test_datetime_utcnow(self, f=util.datetime_utcnow): self.assertIsInstance(f(), datetime.datetime) def test_universal_none(self): @@ -893,7 +833,7 @@ def test_null_context(self): self.assertIs(exc, exc_orig) -class TestExtractor(): +class TestExtractor: category = "test_category" subcategory = "test_subcategory" basecategory = "test_basecategory" diff --git a/test/test_ytdl.py b/test/test_ytdl.py index f7eb67127..ebdb307ee 100644 --- a/test/test_ytdl.py +++ b/test/test_ytdl.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2022-2023 Mike Fährmann # @@ -12,7 +11,8 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import ytdl, util, config # noqa E402 +from gallery_dl import util # noqa: E402 +from gallery_dl import ytdl # noqa: E402 class Test_CommandlineArguments(unittest.TestCase): @@ -23,37 +23,35 @@ def setUpClass(cls): try: cls.module = __import__(cls.module_name) except (ImportError, SyntaxError): - raise unittest.SkipTest("cannot import module '{}'".format( - cls.module_name)) + raise unittest.SkipTest(f"cannot import module '{cls.module_name}'") cls.default = ytdl.parse_command_line(cls.module, []) def test_ignore_errors(self): - self._("--ignore-errors" , "ignoreerrors", True) + self._("--ignore-errors", "ignoreerrors", True) self._("--abort-on-error", "ignoreerrors", False) def test_default_search(self): - self._(["--default-search", "foo"] , "default_search", "foo") + self._(["--default-search", "foo"], "default_search", "foo") def test_mark_watched(self): - self._("--mark-watched" , "mark_watched", True) + self._("--mark-watched", "mark_watched", True) self._("--no-mark-watched", "mark_watched", False) def test_proxy(self): - self._(["--proxy", "socks5://127.0.0.1:1080/"], - "proxy", "socks5://127.0.0.1:1080/") - self._(["--cn-verification-proxy", "https://127.0.0.1"], - "cn_verification_proxy", "https://127.0.0.1") - self._(["--geo-verification-proxy", "127.0.0.1"], - "geo_verification_proxy", "127.0.0.1") + self._(["--proxy", "socks5://127.0.0.1:1080/"], "proxy", "socks5://127.0.0.1:1080/") + self._( + ["--cn-verification-proxy", "https://127.0.0.1"], + "cn_verification_proxy", + "https://127.0.0.1", + ) + self._(["--geo-verification-proxy", "127.0.0.1"], "geo_verification_proxy", "127.0.0.1") def test_network_options(self): - self._(["--socket-timeout", "3.5"], - "socket_timeout", 3.5) - self._(["--source-address", "127.0.0.1"], - "source_address", "127.0.0.1") - self._("-4" , "source_address", "0.0.0.0") + self._(["--socket-timeout", "3.5"], "socket_timeout", 3.5) + self._(["--source-address", "127.0.0.1"], "source_address", "127.0.0.1") + self._("-4", "source_address", "0.0.0.0") self._("--force-ipv4", "source_address", "0.0.0.0") - self._("-6" , "source_address", "::") + self._("-6", "source_address", "::") self._("--force-ipv6", "source_address", "::") def test_thumbnail_options(self): @@ -61,26 +59,26 @@ def test_thumbnail_options(self): self._("--write-all-thumbnails", "write_all_thumbnails", True) def test_authentication_options(self): - self._(["-u" , "foo"], "username", "foo") + self._(["-u", "foo"], "username", "foo") self._(["--username", "foo"], "username", "foo") - self._(["-p" , "bar"], "password", "bar") + self._(["-p", "bar"], "password", "bar") self._(["--password", "bar"], "password", "bar") - self._(["--ap-mso" , "mso"], "ap_mso", "mso") + self._(["--ap-mso", "mso"], "ap_mso", "mso") self._(["--ap-username", "foo"], "ap_username", "foo") self._(["--ap-password", "bar"], "ap_password", "bar") - self._(["-2" , "pass"], "twofactor", "pass") + self._(["-2", "pass"], "twofactor", "pass") self._(["--twofactor", "pass"], "twofactor", "pass") self._(["--video-password", "pass"], "videopassword", "pass") - self._("-n" , "usenetrc", True) + self._("-n", "usenetrc", True) self._("--netrc", "usenetrc", True) def test_subtitle_options(self): - self._("--write-sub" , "writesubtitles" , True) + self._("--write-sub", "writesubtitles", True) self._("--write-auto-sub", "writeautomaticsub", True) self._(["--sub-format", "best"], "subtitlesformat", "best") @@ -100,8 +98,9 @@ def test_geo_bypass(self): self._("--geo-bypass", "geo_bypass", True) self._("--no-geo-bypass", "geo_bypass", False) self._(["--geo-bypass-country", "EN"], "geo_bypass_country", "EN") - self._(["--geo-bypass-ip-block", "198.51.100.14/24"], - "geo_bypass_ip_block", "198.51.100.14/24") + self._( + ["--geo-bypass-ip-block", "198.51.100.14/24"], "geo_bypass_ip_block", "198.51.100.14/24" + ) def test_headers(self): headers = self.module.std_headers @@ -116,41 +115,58 @@ def test_headers(self): self.assertNotEqual(headers["Accept"], "*/*") self.assertNotIn("DNT", headers) - self._([ - "--add-header", "accept:*/*", - "--add-header", "dnt:1", - ]) + self._( + [ + "--add-header", + "accept:*/*", + "--add-header", + "dnt:1", + ] + ) self.assertEqual(headers["accept"], "*/*") self.assertEqual(headers["dnt"], "1") def test_extract_audio(self): opts = self._(["--extract-audio"]) - self.assertEqual(opts["postprocessors"][0], { - "key": "FFmpegExtractAudio", - "preferredcodec": "best", - "preferredquality": "5", - "nopostoverwrites": False, - }) - - opts = self._([ - "--extract-audio", - "--audio-format", "opus", - "--audio-quality", "9", - "--no-post-overwrites", - ]) - self.assertEqual(opts["postprocessors"][0], { - "key": "FFmpegExtractAudio", - "preferredcodec": "opus", - "preferredquality": "9", - "nopostoverwrites": True, - }) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "FFmpegExtractAudio", + "preferredcodec": "best", + "preferredquality": "5", + "nopostoverwrites": False, + }, + ) + + opts = self._( + [ + "--extract-audio", + "--audio-format", + "opus", + "--audio-quality", + "9", + "--no-post-overwrites", + ] + ) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "FFmpegExtractAudio", + "preferredcodec": "opus", + "preferredquality": "9", + "nopostoverwrites": True, + }, + ) def test_recode_video(self): opts = self._(["--recode-video", " mkv "]) - self.assertEqual(opts["postprocessors"][0], { - "key": "FFmpegVideoConvertor", - "preferedformat": "mkv", - }) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "FFmpegVideoConvertor", + "preferedformat": "mkv", + }, + ) def test_subs(self): opts = self._(["--convert-subs", "srt"]) @@ -173,12 +189,14 @@ def test_embed(self): subs["already_have_subtitle"] = True thumb["already_have_thumbnail"] = "all" - opts = self._([ - "--embed-thumbnail", - "--embed-subs", - "--write-sub", - "--write-all-thumbnails", - ]) + opts = self._( + [ + "--embed-thumbnail", + "--embed-subs", + "--write-sub", + "--write-all-thumbnails", + ] + ) self.assertEqual(opts["postprocessors"][:2], [subs, thumb]) def test_metadata(self): @@ -187,10 +205,13 @@ def test_metadata(self): def test_metadata_from_title(self): opts = self._(["--metadata-from-title", "%(artist)s - %(title)s"]) - self.assertEqual(opts["postprocessors"][0], { - "key": "MetadataFromTitle", - "titleformat": "%(artist)s - %(title)s", - }) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "MetadataFromTitle", + "titleformat": "%(artist)s - %(title)s", + }, + ) def test_xattr(self): self._("--xattr-set-filesize", "xattr_set_filesize", True) @@ -213,11 +234,14 @@ def test_noop(self): ] if self.module_name != "yt_dlp": - cmdline.extend(( - "--dump-json", - "--dump-single-json", - "--config-location", "~", - )) + cmdline.extend( + ( + "--dump-json", + "--dump-single-json", + "--config-location", + "~", + ) + ) result = self._(cmdline) result["daterange"] = self.default["daterange"] @@ -244,30 +268,38 @@ def test_retries_extractor(self): def test_remuxs_video(self): opts = self._(["--remux-video", " mkv "]) - self.assertEqual(opts["postprocessors"][0], { - "key": "FFmpegVideoRemuxer", - "preferedformat": "mkv", - }) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "FFmpegVideoRemuxer", + "preferedformat": "mkv", + }, + ) def test_metadata(self): - opts = self._(["--embed-metadata", - "--no-embed-chapters", - "--embed-info-json"]) - self.assertEqual(opts["postprocessors"][0], { - "key": "FFmpegMetadata", - "add_chapters": False, - "add_metadata": True, - "add_infojson": True, - }) + opts = self._(["--embed-metadata", "--no-embed-chapters", "--embed-info-json"]) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "FFmpegMetadata", + "add_chapters": False, + "add_metadata": True, + "add_infojson": True, + }, + ) def test_metadata_from_title(self): opts = self._(["--metadata-from-title", "%(artist)s - %(title)s"]) - self.assertEqual(opts["postprocessors"][0], { - "key" : "MetadataParser", - "when" : "pre_process", - "actions": [self.module.MetadataFromFieldPP.to_action( - "title:%(artist)s - %(title)s")], - }) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "MetadataParser", + "when": "pre_process", + "actions": [ + self.module.MetadataFromFieldPP.to_action("title:%(artist)s - %(title)s") + ], + }, + ) def test_geo_bypass(self): try: @@ -276,37 +308,42 @@ def test_geo_bypass(self): # before --xff (c16644642) return Test_CommandlineArguments.test_geo_bypass(self) - self._(["--xff", "default"], - "geo_bypass", "default") - self._(["--xff", "never"], - "geo_bypass", "never") - self._(["--xff", "EN"], - "geo_bypass", "EN") - self._(["--xff", "198.51.100.14/24"], - "geo_bypass", "198.51.100.14/24") - - self._("--geo-bypass", - "geo_bypass", "default") - self._("--no-geo-bypass", - "geo_bypass", "never") - self._(["--geo-bypass-country", "EN"], - "geo_bypass", "EN") - self._(["--geo-bypass-ip-block", "198.51.100.14/24"], - "geo_bypass", "198.51.100.14/24") + self._(["--xff", "default"], "geo_bypass", "default") + self._(["--xff", "never"], "geo_bypass", "never") + self._(["--xff", "EN"], "geo_bypass", "EN") + self._(["--xff", "198.51.100.14/24"], "geo_bypass", "198.51.100.14/24") + + self._("--geo-bypass", "geo_bypass", "default") + self._("--no-geo-bypass", "geo_bypass", "never") + self._(["--geo-bypass-country", "EN"], "geo_bypass", "EN") + self._(["--geo-bypass-ip-block", "198.51.100.14/24"], "geo_bypass", "198.51.100.14/24") def test_cookiesfrombrowser(self): - self._(["--cookies-from-browser", "firefox"], - "cookiesfrombrowser", ("firefox", None, None, None)) - self._(["--cookies-from-browser", "firefox:profile"], - "cookiesfrombrowser", ("firefox", "profile", None, None)) - self._(["--cookies-from-browser", "firefox+keyring"], - "cookiesfrombrowser", ("firefox", None, "KEYRING", None)) - self._(["--cookies-from-browser", "firefox::container"], - "cookiesfrombrowser", ("firefox", None, None, "container")) - self._(["--cookies-from-browser", - "firefox+keyring:profile::container"], - "cookiesfrombrowser", - ("firefox", "profile", "KEYRING", "container")) + self._( + ["--cookies-from-browser", "firefox"], + "cookiesfrombrowser", + ("firefox", None, None, None), + ) + self._( + ["--cookies-from-browser", "firefox:profile"], + "cookiesfrombrowser", + ("firefox", "profile", None, None), + ) + self._( + ["--cookies-from-browser", "firefox+keyring"], + "cookiesfrombrowser", + ("firefox", None, "KEYRING", None), + ) + self._( + ["--cookies-from-browser", "firefox::container"], + "cookiesfrombrowser", + ("firefox", None, None, "container"), + ) + self._( + ["--cookies-from-browser", "firefox+keyring:profile::container"], + "cookiesfrombrowser", + ("firefox", "profile", "KEYRING", "container"), + ) if __name__ == "__main__":