From c7da679454de42ce17afb8000a016993d346ef48 Mon Sep 17 00:00:00 2001 From: breakid Date: Sat, 16 Nov 2024 14:31:30 -0500 Subject: [PATCH 1/9] Added Ruff config to pyproject.toml; fixed linting errors --- .editorconfig | 2 +- .gitignore | 1 + README.rst | 2 +- gallery_dl/__init__.py | 151 +- gallery_dl/__main__.py | 2 +- gallery_dl/actions.py | 44 +- gallery_dl/aes.py | 1299 +++++++++++++--- gallery_dl/archive.py | 39 +- gallery_dl/cache.py | 58 +- gallery_dl/config.py | 36 +- gallery_dl/cookies.py | 469 +++--- gallery_dl/downloader/__init__.py | 2 - gallery_dl/downloader/common.py | 9 +- gallery_dl/downloader/http.py | 204 ++- gallery_dl/downloader/text.py | 2 - gallery_dl/downloader/ytdl.py | 52 +- gallery_dl/exception.py | 23 +- gallery_dl/extractor/2ch.py | 29 +- gallery_dl/extractor/2chan.py | 54 +- gallery_dl/extractor/2chen.py | 40 +- gallery_dl/extractor/35photo.py | 73 +- gallery_dl/extractor/3dbooru.py | 25 +- gallery_dl/extractor/4archive.py | 57 +- gallery_dl/extractor/4chan.py | 25 +- gallery_dl/extractor/4chanarchives.py | 34 +- gallery_dl/extractor/500px.py | 100 +- gallery_dl/extractor/8chan.py | 39 +- gallery_dl/extractor/8muses.py | 80 +- gallery_dl/extractor/__init__.py | 23 +- gallery_dl/extractor/adultempire.py | 20 +- gallery_dl/extractor/agnph.py | 25 +- gallery_dl/extractor/ao3.py | 148 +- gallery_dl/extractor/architizer.py | 46 +- gallery_dl/extractor/artstation.py | 215 +-- gallery_dl/extractor/aryion.py | 78 +- gallery_dl/extractor/batoto.py | 78 +- gallery_dl/extractor/bbc.py | 34 +- gallery_dl/extractor/behance.py | 79 +- gallery_dl/extractor/bilibili.py | 33 +- gallery_dl/extractor/blogger.py | 66 +- gallery_dl/extractor/bluesky.py | 161 +- gallery_dl/extractor/booru.py | 16 +- gallery_dl/extractor/boosty.py | 142 +- gallery_dl/extractor/bunkr.py | 59 +- gallery_dl/extractor/catbox.py | 18 +- gallery_dl/extractor/chevereto.py | 51 +- gallery_dl/extractor/cien.py | 66 +- gallery_dl/extractor/civitai.py | 284 ++-- gallery_dl/extractor/cohost.py | 53 +- gallery_dl/extractor/comicvine.py | 22 +- gallery_dl/extractor/common.py | 268 ++-- gallery_dl/extractor/cyberdrop.py | 33 +- gallery_dl/extractor/danbooru.py | 112 +- gallery_dl/extractor/desktopography.py | 36 +- gallery_dl/extractor/deviantart.py | 658 ++++---- gallery_dl/extractor/directlink.py | 19 +- gallery_dl/extractor/dynastyscans.py | 66 +- gallery_dl/extractor/e621.py | 69 +- gallery_dl/extractor/erome.py | 44 +- gallery_dl/extractor/everia.py | 22 +- gallery_dl/extractor/exhentai.py | 177 ++- gallery_dl/extractor/fanbox.py | 96 +- gallery_dl/extractor/fanleaks.py | 33 +- gallery_dl/extractor/fantia.py | 73 +- gallery_dl/extractor/fapachi.py | 23 +- gallery_dl/extractor/fapello.py | 49 +- gallery_dl/extractor/flickr.py | 166 +- gallery_dl/extractor/foolfuuka.py | 113 +- gallery_dl/extractor/foolslide.py | 62 +- gallery_dl/extractor/furaffinity.py | 144 +- gallery_dl/extractor/fuskator.py | 47 +- gallery_dl/extractor/gelbooru.py | 86 +- gallery_dl/extractor/gelbooru_v01.py | 83 +- gallery_dl/extractor/gelbooru_v02.py | 162 +- gallery_dl/extractor/generic.py | 107 +- gallery_dl/extractor/gofile.py | 28 +- gallery_dl/extractor/hatenablog.py | 52 +- gallery_dl/extractor/hentai2read.py | 64 +- gallery_dl/extractor/hentaicosplays.py | 22 +- gallery_dl/extractor/hentaifoundry.py | 194 +-- gallery_dl/extractor/hentaifox.py | 69 +- gallery_dl/extractor/hentaihand.py | 60 +- gallery_dl/extractor/hentaihere.py | 66 +- gallery_dl/extractor/hentainexus.py | 94 +- gallery_dl/extractor/hiperdex.py | 63 +- gallery_dl/extractor/hitomi.py | 121 +- gallery_dl/extractor/hotleak.py | 60 +- gallery_dl/extractor/idolcomplex.py | 89 +- gallery_dl/extractor/imagebam.py | 30 +- gallery_dl/extractor/imagechest.py | 46 +- gallery_dl/extractor/imagefap.py | 110 +- gallery_dl/extractor/imagehosts.py | 98 +- gallery_dl/extractor/imgbb.py | 96 +- gallery_dl/extractor/imgbox.py | 26 +- gallery_dl/extractor/imgth.py | 28 +- gallery_dl/extractor/imgur.py | 58 +- gallery_dl/extractor/inkbunny.py | 123 +- gallery_dl/extractor/instagram.py | 379 ++--- gallery_dl/extractor/issuu.py | 34 +- gallery_dl/extractor/itaku.py | 42 +- gallery_dl/extractor/itchio.py | 17 +- gallery_dl/extractor/jschan.py | 35 +- gallery_dl/extractor/kabeuchi.py | 29 +- gallery_dl/extractor/keenspot.py | 14 +- gallery_dl/extractor/kemonoparty.py | 173 ++- gallery_dl/extractor/khinsider.py | 46 +- gallery_dl/extractor/koharu.py | 75 +- gallery_dl/extractor/komikcast.py | 26 +- gallery_dl/extractor/lensdump.py | 83 +- gallery_dl/extractor/lexica.py | 29 +- gallery_dl/extractor/lightroom.py | 15 +- gallery_dl/extractor/livedoor.py | 55 +- gallery_dl/extractor/lolisafe.py | 18 +- gallery_dl/extractor/luscious.py | 56 +- gallery_dl/extractor/lynxchan.py | 53 +- gallery_dl/extractor/mangadex.py | 108 +- gallery_dl/extractor/mangafox.py | 64 +- gallery_dl/extractor/mangahere.py | 64 +- gallery_dl/extractor/mangakakalot.py | 52 +- gallery_dl/extractor/manganelo.py | 51 +- gallery_dl/extractor/mangapark.py | 108 +- gallery_dl/extractor/mangaread.py | 76 +- gallery_dl/extractor/mangasee.py | 47 +- gallery_dl/extractor/mangoxo.py | 41 +- gallery_dl/extractor/mastodon.py | 138 +- gallery_dl/extractor/message.py | 4 +- gallery_dl/extractor/misskey.py | 62 +- gallery_dl/extractor/moebooru.py | 79 +- gallery_dl/extractor/myhentaigallery.py | 28 +- gallery_dl/extractor/myportfolio.py | 39 +- gallery_dl/extractor/naver.py | 82 +- gallery_dl/extractor/naverwebtoon.py | 44 +- gallery_dl/extractor/newgrounds.py | 208 ++- gallery_dl/extractor/nhentai.py | 66 +- gallery_dl/extractor/nijie.py | 131 +- gallery_dl/extractor/nitter.py | 193 +-- gallery_dl/extractor/noop.py | 5 +- gallery_dl/extractor/nozomi.py | 40 +- gallery_dl/extractor/nsfwalbum.py | 47 +- gallery_dl/extractor/oauth.py | 170 +- gallery_dl/extractor/paheal.py | 70 +- gallery_dl/extractor/patreon.py | 160 +- gallery_dl/extractor/philomena.py | 64 +- gallery_dl/extractor/photovogue.py | 8 +- gallery_dl/extractor/picarto.py | 9 +- gallery_dl/extractor/piczel.py | 23 +- gallery_dl/extractor/pillowfort.py | 50 +- gallery_dl/extractor/pinterest.py | 143 +- gallery_dl/extractor/pixeldrain.py | 26 +- gallery_dl/extractor/pixiv.py | 468 +++--- gallery_dl/extractor/pixnet.py | 56 +- gallery_dl/extractor/plurk.py | 28 +- gallery_dl/extractor/poipiku.py | 60 +- gallery_dl/extractor/poringa.py | 38 +- gallery_dl/extractor/pornhub.py | 93 +- gallery_dl/extractor/pornpics.py | 44 +- gallery_dl/extractor/postmill.py | 111 +- gallery_dl/extractor/reactor.py | 62 +- gallery_dl/extractor/readcomiconline.py | 55 +- gallery_dl/extractor/recursive.py | 7 +- gallery_dl/extractor/reddit.py | 192 ++- gallery_dl/extractor/redgifs.py | 81 +- gallery_dl/extractor/rule34us.py | 29 +- gallery_dl/extractor/rule34vault.py | 19 +- gallery_dl/extractor/rule34xyz.py | 19 +- gallery_dl/extractor/saint.py | 61 +- gallery_dl/extractor/sankaku.py | 83 +- gallery_dl/extractor/sankakucomplex.py | 49 +- gallery_dl/extractor/scrolller.py | 49 +- gallery_dl/extractor/seiga.py | 125 +- gallery_dl/extractor/senmanga.py | 16 +- gallery_dl/extractor/sexcom.py | 76 +- gallery_dl/extractor/shimmie2.py | 154 +- gallery_dl/extractor/shopify.py | 102 +- gallery_dl/extractor/simplyhentai.py | 82 +- gallery_dl/extractor/skeb.py | 157 +- gallery_dl/extractor/slickpic.py | 86 +- gallery_dl/extractor/slideshare.py | 35 +- gallery_dl/extractor/smugmug.py | 34 +- gallery_dl/extractor/soundgasm.py | 24 +- gallery_dl/extractor/speakerdeck.py | 20 +- gallery_dl/extractor/steamgriddb.py | 311 +++- gallery_dl/extractor/subscribestar.py | 76 +- gallery_dl/extractor/szurubooru.py | 55 +- gallery_dl/extractor/tapas.py | 72 +- gallery_dl/extractor/tcbscans.py | 31 +- gallery_dl/extractor/telegraph.py | 47 +- gallery_dl/extractor/tmohentai.py | 27 +- gallery_dl/extractor/toyhouse.py | 55 +- gallery_dl/extractor/tsumino.py | 101 +- gallery_dl/extractor/tumblr.py | 131 +- gallery_dl/extractor/tumblrgallery.py | 33 +- gallery_dl/extractor/twibooru.py | 36 +- gallery_dl/extractor/twitter.py | 645 ++++---- gallery_dl/extractor/unsplash.py | 27 +- gallery_dl/extractor/uploadir.py | 26 +- gallery_dl/extractor/urlgalleries.py | 22 +- gallery_dl/extractor/urlshortener.py | 41 +- gallery_dl/extractor/vanillarock.py | 26 +- gallery_dl/extractor/vichan.py | 60 +- gallery_dl/extractor/vipergirls.py | 45 +- gallery_dl/extractor/vk.py | 67 +- gallery_dl/extractor/vsco.py | 149 +- gallery_dl/extractor/wallhaven.py | 51 +- gallery_dl/extractor/wallpapercave.py | 12 +- gallery_dl/extractor/warosu.py | 27 +- gallery_dl/extractor/weasyl.py | 29 +- gallery_dl/extractor/webmshare.py | 31 +- gallery_dl/extractor/webtoons.py | 158 +- gallery_dl/extractor/weibo.py | 94 +- gallery_dl/extractor/wikiart.py | 29 +- gallery_dl/extractor/wikifeet.py | 42 +- gallery_dl/extractor/wikimedia.py | 158 +- gallery_dl/extractor/xhamster.py | 56 +- gallery_dl/extractor/xvideos.py | 56 +- gallery_dl/extractor/ytdl.py | 47 +- gallery_dl/extractor/zerochan.py | 88 +- gallery_dl/extractor/zzup.py | 30 +- gallery_dl/formatter.py | 90 +- gallery_dl/job.py | 211 ++- gallery_dl/oauth.py | 33 +- gallery_dl/option.py | 831 ++++++---- gallery_dl/output.py | 161 +- gallery_dl/path.py | 64 +- gallery_dl/postprocessor/__init__.py | 2 - gallery_dl/postprocessor/classify.py | 24 +- gallery_dl/postprocessor/common.py | 32 +- gallery_dl/postprocessor/compare.py | 19 +- gallery_dl/postprocessor/exec.py | 12 +- gallery_dl/postprocessor/hash.py | 11 +- gallery_dl/postprocessor/metadata.py | 23 +- gallery_dl/postprocessor/mtime.py | 14 +- gallery_dl/postprocessor/python.py | 5 +- gallery_dl/postprocessor/rename.py | 40 +- gallery_dl/postprocessor/ugoira.py | 139 +- gallery_dl/postprocessor/zip.py | 37 +- gallery_dl/text.py | 26 +- gallery_dl/update.py | 62 +- gallery_dl/util.py | 266 ++-- gallery_dl/version.py | 2 - gallery_dl/ytdl.py | 302 ++-- pyproject.toml | 129 +- scripts/build_testresult_db.py | 18 +- scripts/completion_bash.py | 17 +- scripts/completion_fish.py | 3 +- scripts/completion_zsh.py | 18 +- scripts/create_test_data.py | 19 +- scripts/export_tests.py | 80 +- scripts/hook-gallery_dl.py | 6 +- scripts/man.py | 53 +- scripts/options.py | 18 +- scripts/pyinstaller.py | 36 +- scripts/pyprint.py | 43 +- scripts/run_tests.py | 11 +- scripts/supportedsites.py | 520 +++---- scripts/util.py | 7 +- setup.py | 56 +- test/results/2ch.py | 103 +- test/results/2chan.py | 44 +- test/results/2chen.py | 105 +- test/results/35photo.py | 124 +- test/results/3dbooru.py | 73 +- test/results/4archive.py | 92 +- test/results/4chan.py | 49 +- test/results/4chanarchives.py | 71 +- test/results/4plebs.py | 51 +- test/results/500px.py | 175 +-- test/results/8chan.py | 147 +- test/results/8kun.py | 55 +- test/results/8muses.py | 110 +- test/results/94chan.py | 66 +- test/results/__init__.py | 6 +- test/results/acidimg.py | 20 +- test/results/adultempire.py | 35 +- test/results/agnph.py | 85 +- test/results/aibooru.py | 64 +- test/results/allgirlbooru.py | 71 +- test/results/animereactor.py | 36 +- test/results/ao3.py | 460 +++--- test/results/architizer.py | 58 +- test/results/archivedmoe.py | 87 +- test/results/archiveofsins.py | 62 +- test/results/artstation.py | 417 +++-- test/results/aryion.py | 165 +- test/results/atfbooru.py | 55 +- test/results/azurlanewiki.py | 29 +- test/results/b4k.py | 38 +- test/results/baraag.py | 49 +- test/results/batoto.py | 532 +++---- test/results/bbc.py | 74 +- test/results/bbw-chan.py | 55 +- test/results/bcbnsfw.py | 31 +- test/results/behance.py | 169 +- test/results/bilibili.py | 65 +- test/results/bitly.py | 18 +- test/results/blogger.py | 51 +- test/results/blogspot.py | 156 +- test/results/bluesky.py | 726 +++++---- test/results/booruvar.py | 57 +- test/results/boosty.py | 230 ++- test/results/bulbapedia.py | 35 +- test/results/bunkr.py | 371 ++--- test/results/catbox.py | 92 +- test/results/cavemanon.py | 65 +- test/results/chelseacrew.py | 25 +- test/results/cien.py | 155 +- test/results/civitai.py | 426 +++-- test/results/cohost.py | 31 +- test/results/comicvine.py | 33 +- test/results/coomerparty.py | 31 +- test/results/cyberdrop.py | 99 +- test/results/danbooru.py | 434 +++--- test/results/derpibooru.py | 223 ++- test/results/desktopography.py | 36 +- test/results/desuarchive.py | 86 +- test/results/deviantart.py | 1881 +++++++++++------------ test/results/directlink.py | 322 ++-- test/results/drawfriends.py | 36 +- test/results/dynastyscans.py | 93 +- test/results/e621.py | 227 ++- test/results/e6ai.py | 108 +- test/results/e926.py | 144 +- test/results/endchan.py | 53 +- test/results/erome.py | 89 +- test/results/everia.py | 74 +- test/results/exhentai.py | 241 ++- test/results/fanbox.py | 317 ++-- test/results/fandom.py | 197 ++- test/results/fanleaks.py | 107 +- test/results/fantia.py | 109 +- test/results/fapachi.py | 64 +- test/results/fapello.py | 244 ++- test/results/fappic.py | 31 +- test/results/fashionnova.py | 66 +- test/results/fireden.py | 49 +- test/results/flickr.py | 300 ++-- test/results/foalcon.py | 59 +- test/results/furaffinity.py | 452 +++--- test/results/furbooru.py | 44 +- test/results/fuskator.py | 70 +- test/results/gelbooru.py | 341 ++-- test/results/generic.py | 108 +- test/results/giantessbooru.py | 98 +- test/results/gofile.py | 90 +- test/results/hatenablog.py | 251 ++- test/results/hentai2read.py | 122 +- test/results/hentaicosplays.py | 87 +- test/results/hentaifoundry.py | 349 ++--- test/results/hentaifox.py | 185 +-- test/results/hentaihand.py | 89 +- test/results/hentaihere.py | 105 +- test/results/hentainexus.py | 137 +- test/results/hiperdex.py | 257 ++-- test/results/hitomi.py | 425 +++-- test/results/horne.py | 233 ++- test/results/hotleak.py | 176 +-- test/results/hypnohub.py | 126 +- test/results/idolcomplex.py | 210 ++- test/results/illusioncardsbooru.py | 46 +- test/results/imagebam.py | 129 +- test/results/imagechest.py | 80 +- test/results/imagefap.py | 377 +++-- test/results/imagetwist.py | 87 +- test/results/imagevenue.py | 46 +- test/results/imgbb.py | 156 +- test/results/imgbox.py | 80 +- test/results/imgclick.py | 20 +- test/results/imgkiwi.py | 78 +- test/results/imgspice.py | 25 +- test/results/imgth.py | 46 +- test/results/imgur.py | 755 +++++---- test/results/imxto.py | 103 +- test/results/inkbunny.py | 322 ++-- test/results/instagram.py | 508 +++--- test/results/issuu.py | 66 +- test/results/itaku.py | 137 +- test/results/itchio.py | 43 +- test/results/joyreactor.py | 185 +-- test/results/jpgfish.py | 268 ++-- test/results/kabeuchi.py | 33 +- test/results/keenspot.py | 86 +- test/results/kemonoparty.py | 746 +++++---- test/results/khinsider.py | 37 +- test/results/koharu.py | 284 ++-- test/results/kohlchan.py | 55 +- test/results/komikcast.py | 135 +- test/results/konachan.py | 153 +- test/results/lensdump.py | 171 +-- test/results/lesbianenergy.py | 66 +- test/results/lexica.py | 61 +- test/results/lightroom.py | 39 +- test/results/livedoor.py | 105 +- test/results/lolibooru.py | 65 +- test/results/loungeunderwear.py | 25 +- test/results/luscious.py | 199 ++- test/results/mangadex.py | 291 ++-- test/results/mangafox.py | 115 +- test/results/mangahere.py | 126 +- test/results/mangakakalot.py | 57 +- test/results/mangalife.py | 114 +- test/results/manganelo.py | 151 +- test/results/mangapark.py | 244 ++- test/results/mangaread.py | 213 ++- test/results/mangasee.py | 114 +- test/results/mangoxo.py | 58 +- test/results/mariowiki.py | 18 +- test/results/mastodon.py | 41 +- test/results/mastodonsocial.py | 355 ++--- test/results/mediawiki.py | 28 +- test/results/michaelscameras.py | 25 +- test/results/misskeydesign.py | 81 +- test/results/misskeyio.py | 98 +- test/results/modcloth.py | 25 +- test/results/myhentaigallery.py | 53 +- test/results/myportfolio.py | 78 +- test/results/naver.py | 137 +- test/results/naverwebtoon.py | 216 ++- test/results/newgrounds.py | 743 +++++---- test/results/nhentai.py | 189 ++- test/results/nijie.py | 346 ++--- test/results/noop.py | 34 +- test/results/nozomi.py | 181 +-- test/results/nsfwalbum.py | 51 +- test/results/ohpolly.py | 25 +- test/results/omgmiamiswimwear.py | 29 +- test/results/paheal.py | 190 ++- test/results/palanq.py | 49 +- test/results/patreon.py | 210 ++- test/results/pawoo.py | 53 +- test/results/photovogue.py | 75 +- test/results/picarto.py | 22 +- test/results/piczel.py | 91 +- test/results/pidgiwiki.py | 27 +- test/results/pillowfort.py | 340 ++-- test/results/pinterest.py | 360 ++--- test/results/pinupgirlclothing.py | 25 +- test/results/pixeldrain.py | 174 +-- test/results/pixhost.py | 35 +- test/results/pixiv.py | 1252 +++++++-------- test/results/pixnet.py | 143 +- test/results/plurk.py | 48 +- test/results/poipiku.py | 125 +- test/results/ponybooru.py | 55 +- test/results/poringa.py | 83 +- test/results/pornhub.py | 234 ++- test/results/pornpics.py | 257 ++-- test/results/pornreactor.py | 105 +- test/results/postimg.py | 68 +- test/results/raddle.py | 191 ++- test/results/raidlondon.py | 25 +- test/results/rbt.py | 62 +- test/results/reactor.py | 49 +- test/results/readcomiconline.py | 50 +- test/results/realbooru.py | 145 +- test/results/recursive.py | 16 +- test/results/reddit.py | 519 +++---- test/results/redgifs.py | 322 ++-- test/results/rule34.py | 139 +- test/results/rule34hentai.py | 79 +- test/results/rule34us.py | 71 +- test/results/rule34vault.py | 150 +- test/results/rule34xyz.py | 250 ++- test/results/safebooru.py | 96 +- test/results/saint.py | 140 +- test/results/sakugabooru.py | 47 +- test/results/sankaku.py | 557 ++++--- test/results/sankakucomplex.py | 112 +- test/results/scrolller.py | 138 +- test/results/seiga.py | 174 +-- test/results/senmanga.py | 104 +- test/results/sexcom.py | 189 ++- test/results/simplyhentai.py | 123 +- test/results/skeb.py | 153 +- test/results/slickpic.py | 73 +- test/results/slideshare.py | 72 +- test/results/smugloli.py | 73 +- test/results/smugmug.py | 310 ++-- test/results/snootbooru.py | 122 +- test/results/soundgasm.py | 70 +- test/results/speakerdeck.py | 35 +- test/results/steamgriddb.py | 198 ++- test/results/subscribestar.py | 135 +- test/results/sushiski.py | 51 +- test/results/tapas.py | 152 +- test/results/tbib.py | 44 +- test/results/tcbscans.py | 158 +- test/results/tco.py | 33 +- test/results/telegraph.py | 141 +- test/results/tentaclerape.py | 71 +- test/results/thebarchive.py | 49 +- test/results/thecollection.py | 46 +- test/results/tmohentai.py | 86 +- test/results/toyhouse.py | 122 +- test/results/tsumino.py | 118 +- test/results/tumblr.py | 688 ++++----- test/results/tumblrgallery.py | 44 +- test/results/turboimagehost.py | 26 +- test/results/twibooru.py | 189 ++- test/results/twitter.py | 1319 ++++++++-------- test/results/unique-vintage.py | 25 +- test/results/unsplash.py | 278 ++-- test/results/uploadir.py | 97 +- test/results/urlgalleries.py | 75 +- test/results/vanillarock.py | 48 +- test/results/vidyapics.py | 54 +- test/results/vidyart2.py | 36 +- test/results/vipergirls.py | 92 +- test/results/vipr.py | 18 +- test/results/vk.py | 165 +- test/results/vsco.py | 188 +-- test/results/wallhaven.py | 176 +-- test/results/wallpapercave.py | 45 +- test/results/warosu.py | 162 +- test/results/weasyl.py | 165 +- test/results/webmshare.py | 85 +- test/results/webtoons.py | 244 ++- test/results/weibo.py | 523 +++---- test/results/wikiart.py | 187 ++- test/results/wikibooks.py | 25 +- test/results/wikidata.py | 25 +- test/results/wikifeet.py | 79 +- test/results/wikifeetx.py | 41 +- test/results/wikigg.py | 44 +- test/results/wikimediacommons.py | 27 +- test/results/wikinews.py | 25 +- test/results/wikipedia.py | 98 +- test/results/wikiquote.py | 25 +- test/results/wikisource.py | 25 +- test/results/wikispecies.py | 29 +- test/results/wikiversity.py | 25 +- test/results/wikivoyage.py | 25 +- test/results/wiktionary.py | 25 +- test/results/windsorstore.py | 25 +- test/results/xbooru.py | 69 +- test/results/xhamster.py | 208 ++- test/results/xvideos.py | 133 +- test/results/yandere.py | 164 +- test/results/ytdl.py | 14 +- test/results/zerochan.py | 351 ++--- test/results/zzup.py | 49 +- test/test_cache.py | 33 +- test/test_config.py | 122 +- test/test_cookies.py | 52 +- test/test_downloader.py | 170 +- test/test_extractor.py | 56 +- test/test_formatter.py | 195 +-- test/test_job.py | 239 +-- test/test_oauth.py | 58 +- test/test_output.py | 53 +- test/test_postprocessor.py | 426 ++--- test/test_results.py | 96 +- test/test_text.py | 88 +- test/test_util.py | 447 +++--- test/test_ytdl.py | 268 ++-- 554 files changed, 32575 insertions(+), 34034 deletions(-) diff --git a/.editorconfig b/.editorconfig index 05dc0a91d0..dbf1990df9 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,7 +9,7 @@ trim_trailing_whitespace = true [**.py] indent_size = 4 indent_style = space -max_line_length = 79 +max_line_length = 100 [Makefile] indent_style = tab diff --git a/.gitignore b/.gitignore index af2643ae97..e1039685b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.devcontainer/ archive/ # Byte-compiled / optimized / DLL files diff --git a/README.rst b/README.rst index 335101cb22..76b7402f10 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,7 @@ and powerful `filenaming capabilities 1: import itertools + extractor._module_iter = itertools.chain(*modules) elif not modules: extractor._module_iter = () @@ -201,54 +220,57 @@ def main(): if args.update: from . import update + extr = update.UpdateExtractor.from_url("update:" + args.update) ujob = update.UpdateJob(extr) return ujob.run() - elif args.list_modules: + if args.list_modules: extractor.modules.append("") sys.stdout.write("\n".join(extractor.modules)) elif args.list_extractors is not None: write = sys.stdout.write - fmt = ("{}{}\nCategory: {} - Subcategory: {}" - "\nExample : {}\n\n").format + fmt = ("{}{}\nCategory: {} - Subcategory: {}" "\nExample : {}\n\n").format extractors = extractor.extractors() if args.list_extractors: - fltr = util.build_extractor_filter( - args.list_extractors, negate=False) + fltr = util.build_extractor_filter(args.list_extractors, negate=False) extractors = filter(fltr, extractors) for extr in extractors: - write(fmt( - extr.__name__, - "\n" + extr.__doc__ if extr.__doc__ else "", - extr.category, extr.subcategory, - extr.example, - )) + write( + fmt( + extr.__name__, + "\n" + extr.__doc__ if extr.__doc__ else "", + extr.category, + extr.subcategory, + extr.example, + ) + ) elif args.clear_cache: from . import cache + log = logging.getLogger("cache") cnt = cache.clear(args.clear_cache) if cnt is None: log.error("Database file not available") return 1 - else: - log.info( - "Deleted %d %s from '%s'", - cnt, "entry" if cnt == 1 else "entries", cache._path(), - ) + log.info( + "Deleted %d %s from '%s'", + cnt, + "entry" if cnt == 1 else "entries", + cache._path(), + ) elif args.config: if args.config == "init": return config.initialize() - elif args.config == "status": + if args.config == "status": return config.status() - else: - return config.open_extern() + return config.open_extern() else: input_files = config.get((), "input-files") @@ -259,20 +281,19 @@ def main(): args.input_files.append(input_file) if not args.urls and not args.input_files: - if args.cookies_from_browser or config.interpolate( - ("extractor",), "cookies"): + if args.cookies_from_browser or config.interpolate(("extractor",), "cookies"): args.urls.append("noop") else: parser.error( "The following arguments are required: URL\nUse " - "'gallery-dl --help' to get a list of all options.") + "'gallery-dl --help' to get a list of all options." + ) if args.list_urls: jobtype = job.UrlJob jobtype.maxdepth = args.list_urls if config.get(("output",), "fallback", True): - jobtype.handle_url = \ - staticmethod(jobtype.handle_url_fallback) + jobtype.handle_url = staticmethod(jobtype.handle_url_fallback) elif args.dump_json: jobtype = job.DataJob jobtype.resolve = args.dump_json - 1 @@ -283,16 +304,14 @@ def main(): input_manager.log = input_log = logging.getLogger("inputfile") # unsupported file logging handler - handler = output.setup_logging_handler( - "unsupportedfile", fmt="{message}") + handler = output.setup_logging_handler("unsupportedfile", fmt="{message}") if handler: ulog = job.Job.ulog = logging.getLogger("unsupported") ulog.addHandler(handler) ulog.propagate = False # error file logging handler - handler = output.setup_logging_handler( - "errorfile", fmt="{message}", mode="a") + handler = output.setup_logging_handler("errorfile", fmt="{message}", mode="a") if handler: elog = input_manager.err = logging.getLogger("errorfile") elog.addHandler(handler) @@ -311,8 +330,7 @@ def main(): return getattr(exc, "code", 128) pformat = config.get(("output",), "progress", True) - if pformat and len(input_manager.urls) > 1 and \ - args.loglevel < logging.ERROR: + if pformat and len(input_manager.urls) > 1 and args.loglevel < logging.ERROR: input_manager.progress(pformat) # process input URLs @@ -357,13 +375,13 @@ def main(): pass except OSError as exc: import errno + if exc.errno != errno.EPIPE: raise return 1 -class InputManager(): - +class InputManager: def __init__(self): self.urls = [] self.files = () @@ -448,7 +466,7 @@ def add_file(self, path, action=None): # empty line or comment continue - elif line[0] == "-": + if line[0] == "-": # config spec if len(line) >= 2 and line[1] == "G": conf = gconf @@ -463,15 +481,18 @@ def add_file(self, path, action=None): if not sep: raise exception.InputFileError( "Invalid KEY=VALUE pair '%s' on line %s in %s", - line, n+1, path) + line, + n + 1, + path, + ) try: value = util.json_loads(value.strip()) except ValueError as exc: self.log.debug("%s: %s", exc.__class__.__name__, exc) raise exception.InputFileError( - "Unable to parse '%s' on line %s in %s", - value, n+1, path) + "Unable to parse '%s' on line %s in %s", value, n + 1, path + ) key = key.strip().split(".") conf.append((key[:-1], key[-1], value)) @@ -481,6 +502,7 @@ def add_file(self, path, action=None): if " #" in line or "\t#" in line: if strip_comment is None: import re + strip_comment = re.compile(r"\s+#.*").sub line = strip_comment("", line) if gconf or lconf: @@ -532,9 +554,7 @@ def _rewrite(self): with open(path, "w", encoding="utf-8") as fp: fp.writelines(lines) except Exception as exc: - self.log.warning( - "Unable to update '%s' (%s: %s)", - path, exc.__class__.__name__, exc) + self.log.warning("Unable to update '%s' (%s: %s)", path, exc.__class__.__name__, exc) @staticmethod def _action_comment(lines, indicies): @@ -564,16 +584,21 @@ def __next__(self): self._url = url if self._pformat: - output.stderr_write(self._pformat({ - "total" : len(self.urls), - "current": self._index + 1, - "url" : url, - })) + output.stderr_write( + self._pformat( + { + "total": len(self.urls), + "current": self._index + 1, + "url": url, + } + ) + ) return url -class ExtendedUrl(): +class ExtendedUrl: """URL with attached config key-value pairs""" + __slots__ = ("value", "gconfig", "lconfig") def __init__(self, url, gconf, lconf): diff --git a/gallery_dl/__main__.py b/gallery_dl/__main__.py index db3a34e811..3601adc85a 100644 --- a/gallery_dl/__main__.py +++ b/gallery_dl/__main__.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2017-2023 Mike Fährmann # @@ -11,6 +10,7 @@ if not __package__ and not hasattr(sys, "frozen"): import os.path + path = os.path.realpath(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(os.path.dirname(path))) diff --git a/gallery_dl/actions.py b/gallery_dl/actions.py index 668032d5de..2a6de0e346 100644 --- a/gallery_dl/actions.py +++ b/gallery_dl/actions.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,15 @@ """ """ -import re -import time +import functools import logging import operator -import functools -from . import util, exception +import re +import time +from contextlib import suppress + +from . import exception +from . import util def parse(actionspec): @@ -74,8 +75,7 @@ def parse(actionspec): return actions -class LoggerAdapter(): - +class LoggerAdapter: def __init__(self, logger, job): self.logger = logger self.extra = job._logger_extra @@ -126,14 +126,17 @@ def _chain_actions(actions): def _chain(args): for action in actions: action(args) + return _chain # -------------------------------------------------------------------- + def action_print(opts): def _print(_): print(opts) + return None, _print @@ -151,6 +154,7 @@ def action_status(opts): def _status(args): args["job"].status = op(args["job"].status, value) + return _status, None @@ -159,12 +163,14 @@ def action_level(opts): def _level(args): args["level"] = level + return _level, None def action_exec(opts): def _exec(_): util.Popen(opts, shell=True).wait() + return None, _exec @@ -175,6 +181,7 @@ def action_wait(opts): def _wait(args): time.sleep(seconds()) else: + def _wait(args): input("Press Enter to continue") @@ -194,24 +201,23 @@ def action_restart(opts): def action_exit(opts): - try: + with suppress(ValueError): opts = int(opts) - except ValueError: - pass def _exit(args): raise SystemExit(opts) + return None, _exit ACTIONS = { - "abort" : action_abort, - "exec" : action_exec, - "exit" : action_exit, - "level" : action_level, - "print" : action_print, - "restart" : action_restart, - "status" : action_status, + "abort": action_abort, + "exec": action_exec, + "exit": action_exit, + "level": action_level, + "print": action_print, + "restart": action_restart, + "status": action_status, "terminate": action_terminate, - "wait" : action_wait, + "wait": action_wait, } diff --git a/gallery_dl/aes.py b/gallery_dl/aes.py index 891104abff..0b55894da8 100644 --- a/gallery_dl/aes.py +++ b/gallery_dl/aes.py @@ -1,10 +1,8 @@ -# -*- coding: utf-8 -*- - # This is a slightly modified version of yt-dlp's aes module. # https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/aes.py -import struct import binascii +import struct from math import ceil try: @@ -17,32 +15,36 @@ if Cryptodome_AES: + def aes_cbc_decrypt_bytes(data, key, iv): """Decrypt bytes with AES-CBC using pycryptodome""" - return Cryptodome_AES.new( - key, Cryptodome_AES.MODE_CBC, iv).decrypt(data) + return Cryptodome_AES.new(key, Cryptodome_AES.MODE_CBC, iv).decrypt(data) def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce): """Decrypt bytes with AES-GCM using pycryptodome""" - return Cryptodome_AES.new( - key, Cryptodome_AES.MODE_GCM, nonce).decrypt_and_verify(data, tag) + return Cryptodome_AES.new(key, Cryptodome_AES.MODE_GCM, nonce).decrypt_and_verify(data, tag) else: + def aes_cbc_decrypt_bytes(data, key, iv): """Decrypt bytes with AES-CBC using native implementation""" - return intlist_to_bytes(aes_cbc_decrypt( - bytes_to_intlist(data), - bytes_to_intlist(key), - bytes_to_intlist(iv), - )) + return intlist_to_bytes( + aes_cbc_decrypt( + bytes_to_intlist(data), + bytes_to_intlist(key), + bytes_to_intlist(iv), + ) + ) def aes_gcm_decrypt_and_verify_bytes(data, key, tag, nonce): """Decrypt bytes with AES-GCM using native implementation""" - return intlist_to_bytes(aes_gcm_decrypt_and_verify( - bytes_to_intlist(data), - bytes_to_intlist(key), - bytes_to_intlist(tag), - bytes_to_intlist(nonce), - )) + return intlist_to_bytes( + aes_gcm_decrypt_and_verify( + bytes_to_intlist(data), + bytes_to_intlist(key), + bytes_to_intlist(tag), + bytes_to_intlist(nonce), + ) + ) bytes_to_intlist = list @@ -55,7 +57,7 @@ def intlist_to_bytes(xs): def unpad_pkcs7(data): - return data[:-data[-1]] + return data[: -data[-1]] BLOCK_SIZE_BYTES = 16 @@ -75,9 +77,9 @@ def aes_ecb_encrypt(data, key, iv=None): encrypted_data = [] for i in range(block_count): - block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES] + block = data[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES] encrypted_data += aes_encrypt(block, expanded_key) - encrypted_data = encrypted_data[:len(data)] + encrypted_data = encrypted_data[: len(data)] return encrypted_data @@ -96,9 +98,9 @@ def aes_ecb_decrypt(data, key, iv=None): encrypted_data = [] for i in range(block_count): - block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES] + block = data[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES] encrypted_data += aes_decrypt(block, expanded_key) - encrypted_data = encrypted_data[:len(data)] + encrypted_data = encrypted_data[: len(data)] return encrypted_data @@ -131,12 +133,12 @@ def aes_ctr_encrypt(data, key, iv): encrypted_data = [] for i in range(block_count): counter_block = next(counter) - block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES] + block = data[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES] block += [0] * (BLOCK_SIZE_BYTES - len(block)) cipher_counter_block = aes_encrypt(counter_block, expanded_key) encrypted_data += xor(block, cipher_counter_block) - encrypted_data = encrypted_data[:len(data)] + encrypted_data = encrypted_data[: len(data)] return encrypted_data @@ -156,13 +158,13 @@ def aes_cbc_decrypt(data, key, iv): decrypted_data = [] previous_cipher_block = iv for i in range(block_count): - block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES] + block = data[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES] block += [0] * (BLOCK_SIZE_BYTES - len(block)) decrypted_block = aes_decrypt(block, expanded_key) decrypted_data += xor(decrypted_block, previous_cipher_block) previous_cipher_block = block - decrypted_data = decrypted_data[:len(data)] + decrypted_data = decrypted_data[: len(data)] return decrypted_data @@ -182,7 +184,7 @@ def aes_cbc_encrypt(data, key, iv): encrypted_data = [] previous_cipher_block = iv for i in range(block_count): - block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES] + block = data[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES] remaining_length = BLOCK_SIZE_BYTES - len(block) block += [remaining_length] * remaining_length mixed_block = xor(block, previous_cipher_block) @@ -213,10 +215,8 @@ def aes_gcm_decrypt_and_verify(data, key, tag, nonce): if len(nonce) == 12: j0 = nonce + [0, 0, 0, 1] else: - fill = (BLOCK_SIZE_BYTES - (len(nonce) % BLOCK_SIZE_BYTES)) % \ - BLOCK_SIZE_BYTES + 8 - ghash_in = nonce + [0] * fill + bytes_to_intlist( - (8 * len(nonce)).to_bytes(8, "big")) + fill = (BLOCK_SIZE_BYTES - (len(nonce) % BLOCK_SIZE_BYTES)) % BLOCK_SIZE_BYTES + 8 + ghash_in = nonce + [0] * fill + bytes_to_intlist((8 * len(nonce)).to_bytes(8, "big")) j0 = ghash(hash_subkey, ghash_in) # TODO: add nonce support to aes_ctr_decrypt @@ -224,19 +224,17 @@ def aes_gcm_decrypt_and_verify(data, key, tag, nonce): # nonce_ctr = j0[:12] iv_ctr = inc(j0) - decrypted_data = aes_ctr_decrypt( - data, key, iv_ctr + [0] * (BLOCK_SIZE_BYTES - len(iv_ctr))) + decrypted_data = aes_ctr_decrypt(data, key, iv_ctr + [0] * (BLOCK_SIZE_BYTES - len(iv_ctr))) - pad_len = ( - (BLOCK_SIZE_BYTES - (len(data) % BLOCK_SIZE_BYTES)) % BLOCK_SIZE_BYTES) + pad_len = (BLOCK_SIZE_BYTES - (len(data) % BLOCK_SIZE_BYTES)) % BLOCK_SIZE_BYTES s_tag = ghash( hash_subkey, - data + - [0] * pad_len + # pad - bytes_to_intlist( - (0 * 8).to_bytes(8, "big") + # length of associated data - ((len(data) * 8).to_bytes(8, "big")) # length of data - ) + data + + [0] * pad_len # pad + + bytes_to_intlist( + (0 * 8).to_bytes(8, "big") # length of associated data + + ((len(data) * 8).to_bytes(8, "big")) # length of data + ), ) if tag != aes_ctr_encrypt(s_tag, key, j0): @@ -261,8 +259,7 @@ def aes_encrypt(data, expanded_key): data = shift_rows(data) if i != rounds: data = list(iter_mix_columns(data, MIX_COLUMN_MATRIX)) - data = xor(data, expanded_key[ - i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]) + data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES]) return data @@ -278,8 +275,7 @@ def aes_decrypt(data, expanded_key): rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1 for i in range(rounds, 0, -1): - data = xor(data, expanded_key[ - i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]) + data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES : (i + 1) * BLOCK_SIZE_BYTES]) if i != rounds: data = list(iter_mix_columns(data, MIX_COLUMN_MATRIX_INV)) data = shift_rows_inv(data) @@ -311,89 +307,548 @@ def aes_decrypt_text(data, password, key_size_bytes): password = bytes_to_intlist(password.encode("utf-8")) key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password)) - key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * \ - (key_size_bytes // BLOCK_SIZE_BYTES) + key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * ( + key_size_bytes // BLOCK_SIZE_BYTES + ) nonce = data[:NONCE_LENGTH_BYTES] cipher = data[NONCE_LENGTH_BYTES:] - return intlist_to_bytes(aes_ctr_decrypt( - cipher, key, nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES) - )) + return intlist_to_bytes( + aes_ctr_decrypt(cipher, key, nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)) + ) RCON = ( - 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, + 0x8D, + 0x01, + 0x02, + 0x04, + 0x08, + 0x10, + 0x20, + 0x40, + 0x80, + 0x1B, + 0x36, ) SBOX = ( - 0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, - 0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76, - 0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0, - 0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0, - 0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC, - 0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15, - 0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A, - 0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75, - 0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0, - 0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84, - 0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B, - 0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF, - 0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85, - 0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8, - 0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5, - 0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2, - 0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17, - 0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73, - 0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88, - 0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB, - 0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C, - 0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79, - 0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9, - 0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08, - 0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6, - 0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A, - 0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, - 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E, - 0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, - 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF, - 0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, - 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16, + 0x63, + 0x7C, + 0x77, + 0x7B, + 0xF2, + 0x6B, + 0x6F, + 0xC5, + 0x30, + 0x01, + 0x67, + 0x2B, + 0xFE, + 0xD7, + 0xAB, + 0x76, + 0xCA, + 0x82, + 0xC9, + 0x7D, + 0xFA, + 0x59, + 0x47, + 0xF0, + 0xAD, + 0xD4, + 0xA2, + 0xAF, + 0x9C, + 0xA4, + 0x72, + 0xC0, + 0xB7, + 0xFD, + 0x93, + 0x26, + 0x36, + 0x3F, + 0xF7, + 0xCC, + 0x34, + 0xA5, + 0xE5, + 0xF1, + 0x71, + 0xD8, + 0x31, + 0x15, + 0x04, + 0xC7, + 0x23, + 0xC3, + 0x18, + 0x96, + 0x05, + 0x9A, + 0x07, + 0x12, + 0x80, + 0xE2, + 0xEB, + 0x27, + 0xB2, + 0x75, + 0x09, + 0x83, + 0x2C, + 0x1A, + 0x1B, + 0x6E, + 0x5A, + 0xA0, + 0x52, + 0x3B, + 0xD6, + 0xB3, + 0x29, + 0xE3, + 0x2F, + 0x84, + 0x53, + 0xD1, + 0x00, + 0xED, + 0x20, + 0xFC, + 0xB1, + 0x5B, + 0x6A, + 0xCB, + 0xBE, + 0x39, + 0x4A, + 0x4C, + 0x58, + 0xCF, + 0xD0, + 0xEF, + 0xAA, + 0xFB, + 0x43, + 0x4D, + 0x33, + 0x85, + 0x45, + 0xF9, + 0x02, + 0x7F, + 0x50, + 0x3C, + 0x9F, + 0xA8, + 0x51, + 0xA3, + 0x40, + 0x8F, + 0x92, + 0x9D, + 0x38, + 0xF5, + 0xBC, + 0xB6, + 0xDA, + 0x21, + 0x10, + 0xFF, + 0xF3, + 0xD2, + 0xCD, + 0x0C, + 0x13, + 0xEC, + 0x5F, + 0x97, + 0x44, + 0x17, + 0xC4, + 0xA7, + 0x7E, + 0x3D, + 0x64, + 0x5D, + 0x19, + 0x73, + 0x60, + 0x81, + 0x4F, + 0xDC, + 0x22, + 0x2A, + 0x90, + 0x88, + 0x46, + 0xEE, + 0xB8, + 0x14, + 0xDE, + 0x5E, + 0x0B, + 0xDB, + 0xE0, + 0x32, + 0x3A, + 0x0A, + 0x49, + 0x06, + 0x24, + 0x5C, + 0xC2, + 0xD3, + 0xAC, + 0x62, + 0x91, + 0x95, + 0xE4, + 0x79, + 0xE7, + 0xC8, + 0x37, + 0x6D, + 0x8D, + 0xD5, + 0x4E, + 0xA9, + 0x6C, + 0x56, + 0xF4, + 0xEA, + 0x65, + 0x7A, + 0xAE, + 0x08, + 0xBA, + 0x78, + 0x25, + 0x2E, + 0x1C, + 0xA6, + 0xB4, + 0xC6, + 0xE8, + 0xDD, + 0x74, + 0x1F, + 0x4B, + 0xBD, + 0x8B, + 0x8A, + 0x70, + 0x3E, + 0xB5, + 0x66, + 0x48, + 0x03, + 0xF6, + 0x0E, + 0x61, + 0x35, + 0x57, + 0xB9, + 0x86, + 0xC1, + 0x1D, + 0x9E, + 0xE1, + 0xF8, + 0x98, + 0x11, + 0x69, + 0xD9, + 0x8E, + 0x94, + 0x9B, + 0x1E, + 0x87, + 0xE9, + 0xCE, + 0x55, + 0x28, + 0xDF, + 0x8C, + 0xA1, + 0x89, + 0x0D, + 0xBF, + 0xE6, + 0x42, + 0x68, + 0x41, + 0x99, + 0x2D, + 0x0F, + 0xB0, + 0x54, + 0xBB, + 0x16, ) SBOX_INV = ( - 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, - 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, - 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, - 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, - 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, - 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, - 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, - 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, - 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, - 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, - 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, - 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, - 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, - 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, - 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, - 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, - 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, - 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, - 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, - 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, - 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, - 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, - 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, - 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, - 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, - 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, - 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, - 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, - 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, - 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, - 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, - 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d + 0x52, + 0x09, + 0x6A, + 0xD5, + 0x30, + 0x36, + 0xA5, + 0x38, + 0xBF, + 0x40, + 0xA3, + 0x9E, + 0x81, + 0xF3, + 0xD7, + 0xFB, + 0x7C, + 0xE3, + 0x39, + 0x82, + 0x9B, + 0x2F, + 0xFF, + 0x87, + 0x34, + 0x8E, + 0x43, + 0x44, + 0xC4, + 0xDE, + 0xE9, + 0xCB, + 0x54, + 0x7B, + 0x94, + 0x32, + 0xA6, + 0xC2, + 0x23, + 0x3D, + 0xEE, + 0x4C, + 0x95, + 0x0B, + 0x42, + 0xFA, + 0xC3, + 0x4E, + 0x08, + 0x2E, + 0xA1, + 0x66, + 0x28, + 0xD9, + 0x24, + 0xB2, + 0x76, + 0x5B, + 0xA2, + 0x49, + 0x6D, + 0x8B, + 0xD1, + 0x25, + 0x72, + 0xF8, + 0xF6, + 0x64, + 0x86, + 0x68, + 0x98, + 0x16, + 0xD4, + 0xA4, + 0x5C, + 0xCC, + 0x5D, + 0x65, + 0xB6, + 0x92, + 0x6C, + 0x70, + 0x48, + 0x50, + 0xFD, + 0xED, + 0xB9, + 0xDA, + 0x5E, + 0x15, + 0x46, + 0x57, + 0xA7, + 0x8D, + 0x9D, + 0x84, + 0x90, + 0xD8, + 0xAB, + 0x00, + 0x8C, + 0xBC, + 0xD3, + 0x0A, + 0xF7, + 0xE4, + 0x58, + 0x05, + 0xB8, + 0xB3, + 0x45, + 0x06, + 0xD0, + 0x2C, + 0x1E, + 0x8F, + 0xCA, + 0x3F, + 0x0F, + 0x02, + 0xC1, + 0xAF, + 0xBD, + 0x03, + 0x01, + 0x13, + 0x8A, + 0x6B, + 0x3A, + 0x91, + 0x11, + 0x41, + 0x4F, + 0x67, + 0xDC, + 0xEA, + 0x97, + 0xF2, + 0xCF, + 0xCE, + 0xF0, + 0xB4, + 0xE6, + 0x73, + 0x96, + 0xAC, + 0x74, + 0x22, + 0xE7, + 0xAD, + 0x35, + 0x85, + 0xE2, + 0xF9, + 0x37, + 0xE8, + 0x1C, + 0x75, + 0xDF, + 0x6E, + 0x47, + 0xF1, + 0x1A, + 0x71, + 0x1D, + 0x29, + 0xC5, + 0x89, + 0x6F, + 0xB7, + 0x62, + 0x0E, + 0xAA, + 0x18, + 0xBE, + 0x1B, + 0xFC, + 0x56, + 0x3E, + 0x4B, + 0xC6, + 0xD2, + 0x79, + 0x20, + 0x9A, + 0xDB, + 0xC0, + 0xFE, + 0x78, + 0xCD, + 0x5A, + 0xF4, + 0x1F, + 0xDD, + 0xA8, + 0x33, + 0x88, + 0x07, + 0xC7, + 0x31, + 0xB1, + 0x12, + 0x10, + 0x59, + 0x27, + 0x80, + 0xEC, + 0x5F, + 0x60, + 0x51, + 0x7F, + 0xA9, + 0x19, + 0xB5, + 0x4A, + 0x0D, + 0x2D, + 0xE5, + 0x7A, + 0x9F, + 0x93, + 0xC9, + 0x9C, + 0xEF, + 0xA0, + 0xE0, + 0x3B, + 0x4D, + 0xAE, + 0x2A, + 0xF5, + 0xB0, + 0xC8, + 0xEB, + 0xBB, + 0x3C, + 0x83, + 0x53, + 0x99, + 0x61, + 0x17, + 0x2B, + 0x04, + 0x7E, + 0xBA, + 0x77, + 0xD6, + 0x26, + 0xE1, + 0x69, + 0x14, + 0x63, + 0x55, + 0x21, + 0x0C, + 0x7D, ) MIX_COLUMN_MATRIX = ( @@ -411,73 +866,521 @@ def aes_decrypt_text(data, password, key_size_bytes): ) RIJNDAEL_EXP_TABLE = ( - 0x01, 0x03, 0x05, 0x0F, 0x11, 0x33, 0x55, 0xFF, - 0x1A, 0x2E, 0x72, 0x96, 0xA1, 0xF8, 0x13, 0x35, - 0x5F, 0xE1, 0x38, 0x48, 0xD8, 0x73, 0x95, 0xA4, - 0xF7, 0x02, 0x06, 0x0A, 0x1E, 0x22, 0x66, 0xAA, - 0xE5, 0x34, 0x5C, 0xE4, 0x37, 0x59, 0xEB, 0x26, - 0x6A, 0xBE, 0xD9, 0x70, 0x90, 0xAB, 0xE6, 0x31, - 0x53, 0xF5, 0x04, 0x0C, 0x14, 0x3C, 0x44, 0xCC, - 0x4F, 0xD1, 0x68, 0xB8, 0xD3, 0x6E, 0xB2, 0xCD, - 0x4C, 0xD4, 0x67, 0xA9, 0xE0, 0x3B, 0x4D, 0xD7, - 0x62, 0xA6, 0xF1, 0x08, 0x18, 0x28, 0x78, 0x88, - 0x83, 0x9E, 0xB9, 0xD0, 0x6B, 0xBD, 0xDC, 0x7F, - 0x81, 0x98, 0xB3, 0xCE, 0x49, 0xDB, 0x76, 0x9A, - 0xB5, 0xC4, 0x57, 0xF9, 0x10, 0x30, 0x50, 0xF0, - 0x0B, 0x1D, 0x27, 0x69, 0xBB, 0xD6, 0x61, 0xA3, - 0xFE, 0x19, 0x2B, 0x7D, 0x87, 0x92, 0xAD, 0xEC, - 0x2F, 0x71, 0x93, 0xAE, 0xE9, 0x20, 0x60, 0xA0, - 0xFB, 0x16, 0x3A, 0x4E, 0xD2, 0x6D, 0xB7, 0xC2, - 0x5D, 0xE7, 0x32, 0x56, 0xFA, 0x15, 0x3F, 0x41, - 0xC3, 0x5E, 0xE2, 0x3D, 0x47, 0xC9, 0x40, 0xC0, - 0x5B, 0xED, 0x2C, 0x74, 0x9C, 0xBF, 0xDA, 0x75, - 0x9F, 0xBA, 0xD5, 0x64, 0xAC, 0xEF, 0x2A, 0x7E, - 0x82, 0x9D, 0xBC, 0xDF, 0x7A, 0x8E, 0x89, 0x80, - 0x9B, 0xB6, 0xC1, 0x58, 0xE8, 0x23, 0x65, 0xAF, - 0xEA, 0x25, 0x6F, 0xB1, 0xC8, 0x43, 0xC5, 0x54, - 0xFC, 0x1F, 0x21, 0x63, 0xA5, 0xF4, 0x07, 0x09, - 0x1B, 0x2D, 0x77, 0x99, 0xB0, 0xCB, 0x46, 0xCA, - 0x45, 0xCF, 0x4A, 0xDE, 0x79, 0x8B, 0x86, 0x91, - 0xA8, 0xE3, 0x3E, 0x42, 0xC6, 0x51, 0xF3, 0x0E, - 0x12, 0x36, 0x5A, 0xEE, 0x29, 0x7B, 0x8D, 0x8C, - 0x8F, 0x8A, 0x85, 0x94, 0xA7, 0xF2, 0x0D, 0x17, - 0x39, 0x4B, 0xDD, 0x7C, 0x84, 0x97, 0xA2, 0xFD, - 0x1C, 0x24, 0x6C, 0xB4, 0xC7, 0x52, 0xF6, 0x01, + 0x01, + 0x03, + 0x05, + 0x0F, + 0x11, + 0x33, + 0x55, + 0xFF, + 0x1A, + 0x2E, + 0x72, + 0x96, + 0xA1, + 0xF8, + 0x13, + 0x35, + 0x5F, + 0xE1, + 0x38, + 0x48, + 0xD8, + 0x73, + 0x95, + 0xA4, + 0xF7, + 0x02, + 0x06, + 0x0A, + 0x1E, + 0x22, + 0x66, + 0xAA, + 0xE5, + 0x34, + 0x5C, + 0xE4, + 0x37, + 0x59, + 0xEB, + 0x26, + 0x6A, + 0xBE, + 0xD9, + 0x70, + 0x90, + 0xAB, + 0xE6, + 0x31, + 0x53, + 0xF5, + 0x04, + 0x0C, + 0x14, + 0x3C, + 0x44, + 0xCC, + 0x4F, + 0xD1, + 0x68, + 0xB8, + 0xD3, + 0x6E, + 0xB2, + 0xCD, + 0x4C, + 0xD4, + 0x67, + 0xA9, + 0xE0, + 0x3B, + 0x4D, + 0xD7, + 0x62, + 0xA6, + 0xF1, + 0x08, + 0x18, + 0x28, + 0x78, + 0x88, + 0x83, + 0x9E, + 0xB9, + 0xD0, + 0x6B, + 0xBD, + 0xDC, + 0x7F, + 0x81, + 0x98, + 0xB3, + 0xCE, + 0x49, + 0xDB, + 0x76, + 0x9A, + 0xB5, + 0xC4, + 0x57, + 0xF9, + 0x10, + 0x30, + 0x50, + 0xF0, + 0x0B, + 0x1D, + 0x27, + 0x69, + 0xBB, + 0xD6, + 0x61, + 0xA3, + 0xFE, + 0x19, + 0x2B, + 0x7D, + 0x87, + 0x92, + 0xAD, + 0xEC, + 0x2F, + 0x71, + 0x93, + 0xAE, + 0xE9, + 0x20, + 0x60, + 0xA0, + 0xFB, + 0x16, + 0x3A, + 0x4E, + 0xD2, + 0x6D, + 0xB7, + 0xC2, + 0x5D, + 0xE7, + 0x32, + 0x56, + 0xFA, + 0x15, + 0x3F, + 0x41, + 0xC3, + 0x5E, + 0xE2, + 0x3D, + 0x47, + 0xC9, + 0x40, + 0xC0, + 0x5B, + 0xED, + 0x2C, + 0x74, + 0x9C, + 0xBF, + 0xDA, + 0x75, + 0x9F, + 0xBA, + 0xD5, + 0x64, + 0xAC, + 0xEF, + 0x2A, + 0x7E, + 0x82, + 0x9D, + 0xBC, + 0xDF, + 0x7A, + 0x8E, + 0x89, + 0x80, + 0x9B, + 0xB6, + 0xC1, + 0x58, + 0xE8, + 0x23, + 0x65, + 0xAF, + 0xEA, + 0x25, + 0x6F, + 0xB1, + 0xC8, + 0x43, + 0xC5, + 0x54, + 0xFC, + 0x1F, + 0x21, + 0x63, + 0xA5, + 0xF4, + 0x07, + 0x09, + 0x1B, + 0x2D, + 0x77, + 0x99, + 0xB0, + 0xCB, + 0x46, + 0xCA, + 0x45, + 0xCF, + 0x4A, + 0xDE, + 0x79, + 0x8B, + 0x86, + 0x91, + 0xA8, + 0xE3, + 0x3E, + 0x42, + 0xC6, + 0x51, + 0xF3, + 0x0E, + 0x12, + 0x36, + 0x5A, + 0xEE, + 0x29, + 0x7B, + 0x8D, + 0x8C, + 0x8F, + 0x8A, + 0x85, + 0x94, + 0xA7, + 0xF2, + 0x0D, + 0x17, + 0x39, + 0x4B, + 0xDD, + 0x7C, + 0x84, + 0x97, + 0xA2, + 0xFD, + 0x1C, + 0x24, + 0x6C, + 0xB4, + 0xC7, + 0x52, + 0xF6, + 0x01, ) RIJNDAEL_LOG_TABLE = ( - 0x00, 0x00, 0x19, 0x01, 0x32, 0x02, 0x1a, 0xc6, - 0x4b, 0xc7, 0x1b, 0x68, 0x33, 0xee, 0xdf, 0x03, - 0x64, 0x04, 0xe0, 0x0e, 0x34, 0x8d, 0x81, 0xef, - 0x4c, 0x71, 0x08, 0xc8, 0xf8, 0x69, 0x1c, 0xc1, - 0x7d, 0xc2, 0x1d, 0xb5, 0xf9, 0xb9, 0x27, 0x6a, - 0x4d, 0xe4, 0xa6, 0x72, 0x9a, 0xc9, 0x09, 0x78, - 0x65, 0x2f, 0x8a, 0x05, 0x21, 0x0f, 0xe1, 0x24, - 0x12, 0xf0, 0x82, 0x45, 0x35, 0x93, 0xda, 0x8e, - 0x96, 0x8f, 0xdb, 0xbd, 0x36, 0xd0, 0xce, 0x94, - 0x13, 0x5c, 0xd2, 0xf1, 0x40, 0x46, 0x83, 0x38, - 0x66, 0xdd, 0xfd, 0x30, 0xbf, 0x06, 0x8b, 0x62, - 0xb3, 0x25, 0xe2, 0x98, 0x22, 0x88, 0x91, 0x10, - 0x7e, 0x6e, 0x48, 0xc3, 0xa3, 0xb6, 0x1e, 0x42, - 0x3a, 0x6b, 0x28, 0x54, 0xfa, 0x85, 0x3d, 0xba, - 0x2b, 0x79, 0x0a, 0x15, 0x9b, 0x9f, 0x5e, 0xca, - 0x4e, 0xd4, 0xac, 0xe5, 0xf3, 0x73, 0xa7, 0x57, - 0xaf, 0x58, 0xa8, 0x50, 0xf4, 0xea, 0xd6, 0x74, - 0x4f, 0xae, 0xe9, 0xd5, 0xe7, 0xe6, 0xad, 0xe8, - 0x2c, 0xd7, 0x75, 0x7a, 0xeb, 0x16, 0x0b, 0xf5, - 0x59, 0xcb, 0x5f, 0xb0, 0x9c, 0xa9, 0x51, 0xa0, - 0x7f, 0x0c, 0xf6, 0x6f, 0x17, 0xc4, 0x49, 0xec, - 0xd8, 0x43, 0x1f, 0x2d, 0xa4, 0x76, 0x7b, 0xb7, - 0xcc, 0xbb, 0x3e, 0x5a, 0xfb, 0x60, 0xb1, 0x86, - 0x3b, 0x52, 0xa1, 0x6c, 0xaa, 0x55, 0x29, 0x9d, - 0x97, 0xb2, 0x87, 0x90, 0x61, 0xbe, 0xdc, 0xfc, - 0xbc, 0x95, 0xcf, 0xcd, 0x37, 0x3f, 0x5b, 0xd1, - 0x53, 0x39, 0x84, 0x3c, 0x41, 0xa2, 0x6d, 0x47, - 0x14, 0x2a, 0x9e, 0x5d, 0x56, 0xf2, 0xd3, 0xab, - 0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, - 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5, - 0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, - 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07, + 0x00, + 0x00, + 0x19, + 0x01, + 0x32, + 0x02, + 0x1A, + 0xC6, + 0x4B, + 0xC7, + 0x1B, + 0x68, + 0x33, + 0xEE, + 0xDF, + 0x03, + 0x64, + 0x04, + 0xE0, + 0x0E, + 0x34, + 0x8D, + 0x81, + 0xEF, + 0x4C, + 0x71, + 0x08, + 0xC8, + 0xF8, + 0x69, + 0x1C, + 0xC1, + 0x7D, + 0xC2, + 0x1D, + 0xB5, + 0xF9, + 0xB9, + 0x27, + 0x6A, + 0x4D, + 0xE4, + 0xA6, + 0x72, + 0x9A, + 0xC9, + 0x09, + 0x78, + 0x65, + 0x2F, + 0x8A, + 0x05, + 0x21, + 0x0F, + 0xE1, + 0x24, + 0x12, + 0xF0, + 0x82, + 0x45, + 0x35, + 0x93, + 0xDA, + 0x8E, + 0x96, + 0x8F, + 0xDB, + 0xBD, + 0x36, + 0xD0, + 0xCE, + 0x94, + 0x13, + 0x5C, + 0xD2, + 0xF1, + 0x40, + 0x46, + 0x83, + 0x38, + 0x66, + 0xDD, + 0xFD, + 0x30, + 0xBF, + 0x06, + 0x8B, + 0x62, + 0xB3, + 0x25, + 0xE2, + 0x98, + 0x22, + 0x88, + 0x91, + 0x10, + 0x7E, + 0x6E, + 0x48, + 0xC3, + 0xA3, + 0xB6, + 0x1E, + 0x42, + 0x3A, + 0x6B, + 0x28, + 0x54, + 0xFA, + 0x85, + 0x3D, + 0xBA, + 0x2B, + 0x79, + 0x0A, + 0x15, + 0x9B, + 0x9F, + 0x5E, + 0xCA, + 0x4E, + 0xD4, + 0xAC, + 0xE5, + 0xF3, + 0x73, + 0xA7, + 0x57, + 0xAF, + 0x58, + 0xA8, + 0x50, + 0xF4, + 0xEA, + 0xD6, + 0x74, + 0x4F, + 0xAE, + 0xE9, + 0xD5, + 0xE7, + 0xE6, + 0xAD, + 0xE8, + 0x2C, + 0xD7, + 0x75, + 0x7A, + 0xEB, + 0x16, + 0x0B, + 0xF5, + 0x59, + 0xCB, + 0x5F, + 0xB0, + 0x9C, + 0xA9, + 0x51, + 0xA0, + 0x7F, + 0x0C, + 0xF6, + 0x6F, + 0x17, + 0xC4, + 0x49, + 0xEC, + 0xD8, + 0x43, + 0x1F, + 0x2D, + 0xA4, + 0x76, + 0x7B, + 0xB7, + 0xCC, + 0xBB, + 0x3E, + 0x5A, + 0xFB, + 0x60, + 0xB1, + 0x86, + 0x3B, + 0x52, + 0xA1, + 0x6C, + 0xAA, + 0x55, + 0x29, + 0x9D, + 0x97, + 0xB2, + 0x87, + 0x90, + 0x61, + 0xBE, + 0xDC, + 0xFC, + 0xBC, + 0x95, + 0xCF, + 0xCD, + 0x37, + 0x3F, + 0x5B, + 0xD1, + 0x53, + 0x39, + 0x84, + 0x3C, + 0x41, + 0xA2, + 0x6D, + 0x47, + 0x14, + 0x2A, + 0x9E, + 0x5D, + 0x56, + 0xF2, + 0xD3, + 0xAB, + 0x44, + 0x11, + 0x92, + 0xD9, + 0x23, + 0x20, + 0x2E, + 0x89, + 0xB4, + 0x7C, + 0xB8, + 0x26, + 0x77, + 0x99, + 0xE3, + 0xA5, + 0x67, + 0x4A, + 0xED, + 0xDE, + 0xC5, + 0x31, + 0xFE, + 0x18, + 0x0D, + 0x63, + 0x8C, + 0x80, + 0xC0, + 0xF7, + 0x70, + 0x07, ) @@ -497,21 +1400,20 @@ def key_expansion(data): temp = data[-4:] temp = key_schedule_core(temp, rcon_iteration) rcon_iteration += 1 - data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes]) + data += xor(temp, data[-key_size_bytes : 4 - key_size_bytes]) for _ in range(3): temp = data[-4:] - data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes]) + data += xor(temp, data[-key_size_bytes : 4 - key_size_bytes]) if key_size_bytes == 32: temp = data[-4:] temp = sub_bytes(temp) - data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes]) + data += xor(temp, data[-key_size_bytes : 4 - key_size_bytes]) - for _ in range(3 if key_size_bytes == 32 else - 2 if key_size_bytes == 24 else 0): + for _ in range(3 if key_size_bytes == 32 else 2 if key_size_bytes == 24 else 0): temp = data[-4:] - data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes]) + data += xor(temp, data[-key_size_bytes : 4 - key_size_bytes]) data = data[:expanded_key_size_bytes] return data @@ -552,30 +1454,21 @@ def iter_mix_columns(data, matrix): for row in matrix: mixed = 0 for j in range(4): - if data[i:i + 4][j] == 0 or row[j] == 0: + if data[i : i + 4][j] == 0 or row[j] == 0: mixed ^= 0 else: mixed ^= RIJNDAEL_EXP_TABLE[ - (RIJNDAEL_LOG_TABLE[data[i + j]] + - RIJNDAEL_LOG_TABLE[row[j]]) % 0xFF + (RIJNDAEL_LOG_TABLE[data[i + j]] + RIJNDAEL_LOG_TABLE[row[j]]) % 0xFF ] yield mixed def shift_rows(data): - return [ - data[((column + row) & 0b11) * 4 + row] - for column in range(4) - for row in range(4) - ] + return [data[((column + row) & 0b11) * 4 + row] for column in range(4) for row in range(4)] def shift_rows_inv(data): - return [ - data[((column - row) & 0b11) * 4 + row] - for column in range(4) - for row in range(4) - ] + return [data[((column - row) & 0b11) * 4 + row] for column in range(4) for row in range(4)] def shift_block(data): @@ -607,8 +1500,7 @@ def block_product(block_x, block_y): # NIST SP 800-38D, Algorithm 1 if len(block_x) != BLOCK_SIZE_BYTES or len(block_y) != BLOCK_SIZE_BYTES: - raise ValueError( - "Length of blocks need to be %d bytes" % BLOCK_SIZE_BYTES) + raise ValueError("Length of blocks need to be %d bytes" % BLOCK_SIZE_BYTES) block_r = [0xE1] + [0] * (BLOCK_SIZE_BYTES - 1) block_v = block_y[:] @@ -631,12 +1523,11 @@ def ghash(subkey, data): # NIST SP 800-38D, Algorithm 2 if len(data) % BLOCK_SIZE_BYTES: - raise ValueError( - "Length of data should be %d bytes" % BLOCK_SIZE_BYTES) + raise ValueError("Length of data should be %d bytes" % BLOCK_SIZE_BYTES) last_y = [0] * BLOCK_SIZE_BYTES for i in range(0, len(data), BLOCK_SIZE_BYTES): - block = data[i: i + BLOCK_SIZE_BYTES] + block = data[i : i + BLOCK_SIZE_BYTES] last_y = block_product(xor(last_y, block), subkey) return last_y diff --git a/gallery_dl/archive.py b/gallery_dl/archive.py index 5f05bbfd8c..607f39e935 100644 --- a/gallery_dl/archive.py +++ b/gallery_dl/archive.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -10,13 +8,13 @@ import os import sqlite3 -from . import formatter +from contextlib import suppress +from . import formatter -class DownloadArchive(): - def __init__(self, path, format_string, pragma=None, - cache_key="_archive_key"): +class DownloadArchive: + def __init__(self, path, format_string, pragma=None, cache_key="_archive_key"): try: con = sqlite3.connect(path, timeout=60, check_same_thread=False) except sqlite3.OperationalError: @@ -35,24 +33,22 @@ def __init__(self, path, format_string, pragma=None, cursor.execute("PRAGMA " + stmt) try: - cursor.execute("CREATE TABLE IF NOT EXISTS archive " - "(entry TEXT PRIMARY KEY) WITHOUT ROWID") + cursor.execute( + "CREATE TABLE IF NOT EXISTS archive " "(entry TEXT PRIMARY KEY) WITHOUT ROWID" + ) except sqlite3.OperationalError: # fallback for missing WITHOUT ROWID support (#553) - cursor.execute("CREATE TABLE IF NOT EXISTS archive " - "(entry TEXT PRIMARY KEY)") + cursor.execute("CREATE TABLE IF NOT EXISTS archive " "(entry TEXT PRIMARY KEY)") def add(self, kwdict): """Add item described by 'kwdict' to archive""" key = kwdict.get(self._cache_key) or self.keygen(kwdict) - self.cursor.execute( - "INSERT OR IGNORE INTO archive (entry) VALUES (?)", (key,)) + self.cursor.execute("INSERT OR IGNORE INTO archive (entry) VALUES (?)", (key,)) def check(self, kwdict): """Return True if the item described by 'kwdict' exists in archive""" key = kwdict[self._cache_key] = self.keygen(kwdict) - self.cursor.execute( - "SELECT 1 FROM archive WHERE entry=? LIMIT 1", (key,)) + self.cursor.execute("SELECT 1 FROM archive WHERE entry=? LIMIT 1", (key,)) return self.cursor.fetchone() def finalize(self): @@ -60,23 +56,18 @@ def finalize(self): class DownloadArchiveMemory(DownloadArchive): - - def __init__(self, path, format_string, pragma=None, - cache_key="_archive_key"): + def __init__(self, path, format_string, pragma=None, cache_key="_archive_key"): DownloadArchive.__init__(self, path, format_string, pragma, cache_key) self.keys = set() def add(self, kwdict): - self.keys.add( - kwdict.get(self._cache_key) or - self.keygen(kwdict)) + self.keys.add(kwdict.get(self._cache_key) or self.keygen(kwdict)) def check(self, kwdict): key = kwdict[self._cache_key] = self.keygen(kwdict) if key in self.keys: return True - self.cursor.execute( - "SELECT 1 FROM archive WHERE entry=? LIMIT 1", (key,)) + self.cursor.execute("SELECT 1 FROM archive WHERE entry=? LIMIT 1", (key,)) return self.cursor.fetchone() def finalize(self): @@ -85,10 +76,8 @@ def finalize(self): cursor = self.cursor with self.connection: - try: + with suppress(sqlite3.OperationalError): cursor.execute("BEGIN") - except sqlite3.OperationalError: - pass stmt = "INSERT OR IGNORE INTO archive (entry) VALUES (?)" if len(self.keys) < 100: diff --git a/gallery_dl/cache.py b/gallery_dl/cache.py index 923ed32c95..f45997d4e4 100644 --- a/gallery_dl/cache.py +++ b/gallery_dl/cache.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2016-2021 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,16 +6,20 @@ """Decorators to keep function results in an in-memory and database cache""" -import sqlite3 +import functools +import os import pickle +import sqlite3 import time -import os -import functools -from . import config, util +from contextlib import suppress + +from . import config +from . import util -class CacheDecorator(): +class CacheDecorator: """Simplified in-memory cache""" + def __init__(self, func, keyarg): self.func = func self.cache = {} @@ -38,14 +40,13 @@ def update(self, key, value): self.cache[key] = value def invalidate(self, key=""): - try: + with suppress(KeyError): del self.cache[key] - except KeyError: - pass class MemoryCacheDecorator(CacheDecorator): """In-memory cache""" + def __init__(self, func, keyarg, maxage): CacheDecorator.__init__(self, func, keyarg) self.maxage = maxage @@ -67,13 +68,14 @@ def update(self, key, value): self.cache[key] = value, int(time.time()) + self.maxage -class DatabaseCacheDecorator(): +class DatabaseCacheDecorator: """Database cache""" + db = None _init = True def __init__(self, func, keyarg, maxage): - self.key = "%s.%s" % (func.__module__, func.__name__) + self.key = f"{func.__module__}.{func.__name__}" self.func = func self.cache = {} self.keyarg = keyarg @@ -87,21 +89,19 @@ def __call__(self, *args, **kwargs): timestamp = int(time.time()) # in-memory cache lookup - try: + with suppress(KeyError): value, expires = self.cache[key] + if expires > timestamp: return value - except KeyError: - pass # database lookup - fullkey = "%s-%s" % (self.key, key) + fullkey = f"{self.key}-{key}" with self.database() as db: cursor = db.cursor() - try: + # Silently swallow exception - workaround for Python 3.6 + with suppress(sqlite3.OperationalError): cursor.execute("BEGIN EXCLUSIVE") - except sqlite3.OperationalError: - pass # Silently swallow exception - workaround for Python 3.6 cursor.execute( "SELECT value, expires FROM data WHERE key=? LIMIT 1", (fullkey,), @@ -128,18 +128,16 @@ def update(self, key, value): with self.database() as db: db.execute( "INSERT OR REPLACE INTO data VALUES (?,?,?)", - ("%s-%s" % (self.key, key), pickle.dumps(value), expires), + (f"{self.key}-{key}", pickle.dumps(value), expires), ) def invalidate(self, key): - try: + with suppress(KeyError): del self.cache[key] - except KeyError: - pass with self.database() as db: db.execute( "DELETE FROM data WHERE key=?", - ("%s-%s" % (self.key, key),), + (f"{self.key}-{key}",), ) def database(self): @@ -154,17 +152,21 @@ def database(self): def memcache(maxage=None, keyarg=None): if maxage: + def wrap(func): return MemoryCacheDecorator(func, keyarg, maxage) else: + def wrap(func): return CacheDecorator(func, keyarg) + return wrap def cache(maxage=3600, keyarg=None): def wrap(func): return DatabaseCacheDecorator(func, keyarg, maxage) + return wrap @@ -182,9 +184,8 @@ def clear(module): cursor.execute("DELETE FROM data") else: cursor.execute( - "DELETE FROM data " - "WHERE key LIKE 'gallery_dl.extractor.' || ? || '.%'", - (module.lower(),) + "DELETE FROM data " "WHERE key LIKE 'gallery_dl.extractor.' || ? || '.%'", + (module.lower(),), ) except sqlite3.OperationalError: pass # database not initialized, cannot be modified, etc. @@ -218,8 +219,7 @@ def _init(): # restrict access permissions for new db files os.close(os.open(dbfile, os.O_CREAT | os.O_RDONLY, 0o600)) - DatabaseCacheDecorator.db = sqlite3.connect( - dbfile, timeout=60, check_same_thread=False) + DatabaseCacheDecorator.db = sqlite3.connect(dbfile, timeout=60, check_same_thread=False) except (OSError, TypeError, sqlite3.OperationalError): global cache cache = memcache diff --git a/gallery_dl/config.py b/gallery_dl/config.py index 855fb4f7b7..ff906a98d7 100644 --- a/gallery_dl/config.py +++ b/gallery_dl/config.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,9 +6,10 @@ """Global configuration module""" -import sys -import os.path import logging +import os.path +import sys + from . import util log = logging.getLogger("config") @@ -32,18 +31,20 @@ _default_configs = [ "/etc/gallery-dl.conf", "${XDG_CONFIG_HOME}/gallery-dl/config.json" - if os.environ.get("XDG_CONFIG_HOME") else - "${HOME}/.config/gallery-dl/config.json", + if os.environ.get("XDG_CONFIG_HOME") + else "${HOME}/.config/gallery-dl/config.json", "${HOME}/.gallery-dl.conf", ] if util.EXECUTABLE: # look for config file in PyInstaller executable directory (#682) - _default_configs.append(os.path.join( - os.path.dirname(sys.executable), - "gallery-dl.conf", - )) + _default_configs.append( + os.path.join( + os.path.dirname(sys.executable), + "gallery-dl.conf", + ) + ) # -------------------------------------------------------------------- @@ -82,8 +83,7 @@ def initialize(): except OSError as exc: log.debug("%s: %s", exc.__class__.__name__, exc) else: - log.error("Unable to create a new configuration file " - "at any of the default paths") + log.error("Unable to create a new configuration file " "at any of the default paths") return 1 log.info("Created a basic configuration file at '%s'", path) @@ -108,6 +108,7 @@ def open_extern(): openers = (editor,) + openers import shutil + for opener in openers: opener = shutil.which(opener) if opener: @@ -124,8 +125,7 @@ def open_extern(): with open(path, encoding="utf-8") as fp: util.json_loads(fp.read()) except Exception as exc: - log.warning("%s when parsing '%s': %s", - exc.__class__.__name__, path, exc) + log.warning("%s when parsing '%s': %s", exc.__class__.__name__, path, exc) return 2 return retcode @@ -155,8 +155,7 @@ def status(): paths.append((path, status)) - fmt = "{{:<{}}} : {{}}\n".format( - max(len(p[0]) for p in paths)).format + fmt = f"{{:<{max(len(p[0]) for p in paths)}}} : {{}}\n".format for path, status in paths: stdout_write(fmt(path, status)) @@ -174,8 +173,7 @@ def load(files=None, strict=False, loads=util.json_loads): log.error(exc) raise SystemExit(1) except Exception as exc: - log.error("%s when loading '%s': %s", - exc.__class__.__name__, path, exc) + log.error("%s when loading '%s': %s", exc.__class__.__name__, path, exc) if strict: raise SystemExit(2) else: @@ -303,7 +301,7 @@ def unset(path, key, conf=_config): pass -class apply(): +class apply: """Context Manager: apply a collection of key-value pairs""" def __init__(self, kvlist): diff --git a/gallery_dl/cookies.py b/gallery_dl/cookies.py index 71b0b6b0a4..f3cb2c9086 100644 --- a/gallery_dl/cookies.py +++ b/gallery_dl/cookies.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -21,41 +19,41 @@ import tempfile from hashlib import pbkdf2_hmac from http.cookiejar import Cookie -from . import aes, text, util +from . import aes +from . import text +from . import util -SUPPORTED_BROWSERS_CHROMIUM = { - "brave", "chrome", "chromium", "edge", "opera", "thorium", "vivaldi"} +SUPPORTED_BROWSERS_CHROMIUM = {"brave", "chrome", "chromium", "edge", "opera", "thorium", "vivaldi"} SUPPORTED_BROWSERS = SUPPORTED_BROWSERS_CHROMIUM | {"firefox", "safari"} logger = logging.getLogger("cookies") def load_cookies(browser_specification): - browser_name, profile, keyring, container, domain = \ - _parse_browser_specification(*browser_specification) + browser_name, profile, keyring, container, domain = _parse_browser_specification( + *browser_specification + ) if browser_name == "firefox": return load_cookies_firefox(profile, container, domain) - elif browser_name == "safari": + if browser_name == "safari": return load_cookies_safari(profile, domain) - elif browser_name in SUPPORTED_BROWSERS_CHROMIUM: + if browser_name in SUPPORTED_BROWSERS_CHROMIUM: return load_cookies_chromium(browser_name, profile, keyring, domain) - else: - raise ValueError("unknown browser '{}'".format(browser_name)) + raise ValueError(f"unknown browser '{browser_name}'") def load_cookies_firefox(profile=None, container=None, domain=None): path, container_id = _firefox_cookies_database(profile, container) - sql = ("SELECT name, value, host, path, isSecure, expiry " - "FROM moz_cookies") + sql = "SELECT name, value, host, path, isSecure, expiry " "FROM moz_cookies" conditions = [] parameters = [] if container_id is False: conditions.append("NOT INSTR(originAttributes,'userContextId=')") elif container_id: - uid = "%userContextId={}".format(container_id) + uid = f"%userContextId={container_id}" conditions.append("originAttributes LIKE ? OR originAttributes LIKE ?") parameters += (uid, uid + "&%") @@ -73,14 +71,24 @@ def load_cookies_firefox(profile=None, container=None, domain=None): with DatabaseConnection(path) as db: cookies = [ Cookie( - 0, name, value, None, False, - domain, True if domain else False, + 0, + name, + value, + None, + False, + domain, + bool(domain), domain[0] == "." if domain else False, - path, True if path else False, secure, expires, - False, None, None, {}, + path, + bool(path), + secure, + expires, + False, + None, + None, + {}, ) - for name, value, domain, path, secure, expires in db.execute( - sql, parameters) + for name, value, domain, path, secure, expires in db.execute(sql, parameters) ] _log_info("Extracted %s cookies from Firefox", len(cookies)) @@ -107,8 +115,7 @@ def load_cookies_safari(profile=None, domain=None): return cookies -def load_cookies_chromium(browser_name, profile=None, - keyring=None, domain=None): +def load_cookies_chromium(browser_name, profile=None, keyring=None, domain=None): config = _chromium_browser_settings(browser_name) path = _chromium_cookies_database(profile, config) _log_debug("Extracting cookies from %s", path) @@ -129,30 +136,36 @@ def load_cookies_chromium(browser_name, profile=None, cursor = db.cursor() try: - meta_version = int(cursor.execute( - "SELECT value FROM meta WHERE key = 'version'").fetchone()[0]) + meta_version = int( + cursor.execute("SELECT value FROM meta WHERE key = 'version'").fetchone()[0] + ) except Exception as exc: - _log_warning("Failed to get cookie database meta version (%s: %s)", - exc.__class__.__name__, exc) + _log_warning( + "Failed to get cookie database meta version (%s: %s)", exc.__class__.__name__, exc + ) meta_version = 0 try: rows = cursor.execute( "SELECT host_key, name, value, encrypted_value, path, " - "expires_utc, is_secure FROM cookies" + condition, parameters) + "expires_utc, is_secure FROM cookies" + condition, + parameters, + ) except sqlite3.OperationalError: rows = cursor.execute( "SELECT host_key, name, value, encrypted_value, path, " - "expires_utc, secure FROM cookies" + condition, parameters) + "expires_utc, secure FROM cookies" + condition, + parameters, + ) failed_cookies = 0 unencrypted_cookies = 0 decryptor = _chromium_cookie_decryptor( - config["directory"], config["keyring"], keyring, meta_version) + config["directory"], config["keyring"], keyring, meta_version + ) cookies = [] for domain, name, value, enc_value, path, expires, secure in rows: - if not value and enc_value: # encrypted value = decryptor.decrypt(enc_value) if value is None: @@ -162,31 +175,39 @@ def load_cookies_chromium(browser_name, profile=None, value = value.decode() unencrypted_cookies += 1 - if expires: - # https://stackoverflow.com/a/43520042 - expires = int(expires) // 1000000 - 11644473600 - else: - expires = None + # https://stackoverflow.com/a/43520042 + expires = int(expires) // 1000000 - 11644473600 if expires else None domain = domain.decode() path = path.decode() name = name.decode() - cookies.append(Cookie( - 0, name, value, None, False, - domain, True if domain else False, - domain[0] == "." if domain else False, - path, True if path else False, secure, expires, - False, None, None, {}, - )) + cookies.append( + Cookie( + 0, + name, + value, + None, + False, + domain, + bool(domain), + domain[0] == "." if domain else False, + path, + bool(path), + secure, + expires, + False, + None, + None, + {}, + ) + ) - if failed_cookies > 0: - failed_message = " ({} could not be decrypted)".format(failed_cookies) - else: - failed_message = "" + failed_message = f" ({failed_cookies} could not be decrypted)" if failed_cookies > 0 else "" - _log_info("Extracted %s cookies from %s%s", - len(cookies), browser_name.capitalize(), failed_message) + _log_info( + "Extracted %s cookies from %s%s", len(cookies), browser_name.capitalize(), failed_message + ) counts = decryptor.cookie_counts counts["unencrypted"] = unencrypted_cookies _log_debug("version breakdown: %s", counts) @@ -196,6 +217,7 @@ def load_cookies_chromium(browser_name, profile=None, # -------------------------------------------------------------------- # firefox + def _firefox_cookies_database(profile=None, container=None): if not profile: search_root = _firefox_browser_directory() @@ -206,8 +228,7 @@ def _firefox_cookies_database(profile=None, container=None): path = _find_most_recently_used_file(search_root, "cookies.sqlite") if path is None: - raise FileNotFoundError("Unable to find Firefox cookies database in " - "{}".format(search_root)) + raise FileNotFoundError("Unable to find Firefox cookies database in " f"{search_root}") _log_debug("Extracting cookies from %s", path) if not container or container == "none": @@ -218,54 +239,51 @@ def _firefox_cookies_database(profile=None, container=None): container_id = None else: - containers_path = os.path.join( - os.path.dirname(path), "containers.json") + containers_path = os.path.join(os.path.dirname(path), "containers.json") try: with open(containers_path) as fp: identities = util.json_loads(fp.read())["identities"] except OSError: - _log_error("Unable to read Firefox container database at '%s'", - containers_path) + _log_error("Unable to read Firefox container database at '%s'", containers_path) raise except KeyError: identities = () for context in identities: if container == context.get("name") or container == text.extr( - context.get("l10nID", ""), "userContext", ".label"): + context.get("l10nID", ""), "userContext", ".label" + ): container_id = context["userContextId"] break else: - raise ValueError("Unable to find Firefox container '{}'".format( - container)) - _log_debug("Only loading cookies from container '%s' (ID %s)", - container, container_id) + raise ValueError(f"Unable to find Firefox container '{container}'") + _log_debug("Only loading cookies from container '%s' (ID %s)", container, container_id) return path, container_id def _firefox_browser_directory(): if sys.platform in ("win32", "cygwin"): - return os.path.expandvars( - r"%APPDATA%\Mozilla\Firefox\Profiles") + return os.path.expandvars(r"%APPDATA%\Mozilla\Firefox\Profiles") if sys.platform == "darwin": - return os.path.expanduser( - "~/Library/Application Support/Firefox/Profiles") + return os.path.expanduser("~/Library/Application Support/Firefox/Profiles") return os.path.expanduser("~/.mozilla/firefox") # -------------------------------------------------------------------- # safari + def _safari_cookies_database(): try: path = os.path.expanduser("~/Library/Cookies/Cookies.binarycookies") return open(path, "rb") except FileNotFoundError: _log_debug("Trying secondary cookie location") - path = os.path.expanduser("~/Library/Containers/com.apple.Safari/Data" - "/Library/Cookies/Cookies.binarycookies") + path = os.path.expanduser( + "~/Library/Containers/com.apple.Safari/Data" "/Library/Cookies/Cookies.binarycookies" + ) return open(path, "rb") @@ -273,8 +291,7 @@ def _safari_parse_cookies_header(data): p = DataParser(data) p.expect_bytes(b"cook", "database signature") number_of_pages = p.read_uint(big_endian=True) - page_sizes = [p.read_uint(big_endian=True) - for _ in range(number_of_pages)] + page_sizes = [p.read_uint(big_endian=True) for _ in range(number_of_pages)] return page_sizes, p.cursor @@ -291,8 +308,7 @@ def _safari_parse_cookies_page(data, cookies, domain=None): for i, record_offset in enumerate(record_offsets): p.skip_to(record_offset, "space between records") - record_length = _safari_parse_cookies_record( - data[record_offset:], cookies, domain) + record_length = _safari_parse_cookies_record(data[record_offset:], cookies, domain) p.read_bytes(record_length) p.skip_to_end("space in between pages") @@ -302,7 +318,7 @@ def _safari_parse_cookies_record(data, cookies, host=None): record_size = p.read_uint() p.skip(4, "unknown record field 1") flags = p.read_uint() - is_secure = True if (flags & 0x0001) else False + is_secure = bool(flags & 0x0001) p.skip(4, "unknown record field 2") domain_offset = p.read_uint() name_offset = p.read_uint() @@ -310,7 +326,7 @@ def _safari_parse_cookies_record(data, cookies, host=None): value_offset = p.read_uint() p.skip(8, "unknown record field 3") expiration_date = _mac_absolute_time_to_posix(p.read_double()) - _creation_date = _mac_absolute_time_to_posix(p.read_double()) # noqa: F841 + _creation_date = _mac_absolute_time_to_posix(p.read_double()) try: p.skip_to(domain_offset) @@ -320,9 +336,8 @@ def _safari_parse_cookies_record(data, cookies, host=None): if host[0] == ".": if host[1:] != domain and not domain.endswith(host): return record_size - else: - if host != domain and ("." + host) != domain: - return record_size + elif host != domain and ("." + host) != domain: + return record_size p.skip_to(name_offset) name = p.read_cstring() @@ -338,13 +353,26 @@ def _safari_parse_cookies_record(data, cookies, host=None): p.skip_to(record_size, "space at the end of the record") - cookies.append(Cookie( - 0, name, value, None, False, - domain, True if domain else False, - domain[0] == "." if domain else False, - path, True if path else False, is_secure, expiration_date, - False, None, None, {}, - )) + cookies.append( + Cookie( + 0, + name, + value, + None, + False, + domain, + bool(domain), + domain[0] == "." if domain else False, + path, + bool(path), + is_secure, + expiration_date, + False, + None, + None, + {}, + ) + ) return record_size @@ -352,13 +380,13 @@ def _safari_parse_cookies_record(data, cookies, host=None): # -------------------------------------------------------------------- # chromium + def _chromium_cookies_database(profile, config): if profile is None: search_root = config["directory"] elif _is_path(profile): search_root = profile - config["directory"] = (os.path.dirname(profile) - if config["profiles"] else profile) + config["directory"] = os.path.dirname(profile) if config["profiles"] else profile elif config["profiles"]: search_root = os.path.join(config["directory"], profile) else: @@ -367,8 +395,9 @@ def _chromium_cookies_database(profile, config): path = _find_most_recently_used_file(search_root, "Cookies") if path is None: - raise FileNotFoundError("Unable to find {} cookies database in " - "'{}'".format(config["browser"], search_root)) + raise FileNotFoundError( + "Unable to find {} cookies database in " "'{}'".format(config["browser"], search_root) + ) return path @@ -381,76 +410,69 @@ def _chromium_browser_settings(browser_name): appdata_local = os.path.expandvars("%LOCALAPPDATA%") appdata_roaming = os.path.expandvars("%APPDATA%") browser_dir = { - "brave" : join(appdata_local, - R"BraveSoftware\Brave-Browser\User Data"), - "chrome" : join(appdata_local, R"Google\Chrome\User Data"), + "brave": join(appdata_local, R"BraveSoftware\Brave-Browser\User Data"), + "chrome": join(appdata_local, R"Google\Chrome\User Data"), "chromium": join(appdata_local, R"Chromium\User Data"), - "edge" : join(appdata_local, R"Microsoft\Edge\User Data"), - "opera" : join(appdata_roaming, R"Opera Software\Opera Stable"), - "thorium" : join(appdata_local, R"Thorium\User Data"), - "vivaldi" : join(appdata_local, R"Vivaldi\User Data"), + "edge": join(appdata_local, R"Microsoft\Edge\User Data"), + "opera": join(appdata_roaming, R"Opera Software\Opera Stable"), + "thorium": join(appdata_local, R"Thorium\User Data"), + "vivaldi": join(appdata_local, R"Vivaldi\User Data"), }[browser_name] elif sys.platform == "darwin": appdata = os.path.expanduser("~/Library/Application Support") browser_dir = { - "brave" : join(appdata, "BraveSoftware/Brave-Browser"), - "chrome" : join(appdata, "Google/Chrome"), + "brave": join(appdata, "BraveSoftware/Brave-Browser"), + "chrome": join(appdata, "Google/Chrome"), "chromium": join(appdata, "Chromium"), - "edge" : join(appdata, "Microsoft Edge"), - "opera" : join(appdata, "com.operasoftware.Opera"), - "thorium" : join(appdata, "Thorium"), - "vivaldi" : join(appdata, "Vivaldi"), + "edge": join(appdata, "Microsoft Edge"), + "opera": join(appdata, "com.operasoftware.Opera"), + "thorium": join(appdata, "Thorium"), + "vivaldi": join(appdata, "Vivaldi"), }[browser_name] else: - config = (os.environ.get("XDG_CONFIG_HOME") or - os.path.expanduser("~/.config")) + config = os.environ.get("XDG_CONFIG_HOME") or os.path.expanduser("~/.config") browser_dir = { - "brave" : join(config, "BraveSoftware/Brave-Browser"), - "chrome" : join(config, "google-chrome"), + "brave": join(config, "BraveSoftware/Brave-Browser"), + "chrome": join(config, "google-chrome"), "chromium": join(config, "chromium"), - "edge" : join(config, "microsoft-edge"), - "opera" : join(config, "opera"), - "thorium" : join(config, "Thorium"), - "vivaldi" : join(config, "vivaldi"), + "edge": join(config, "microsoft-edge"), + "opera": join(config, "opera"), + "thorium": join(config, "Thorium"), + "vivaldi": join(config, "vivaldi"), }[browser_name] # Linux keyring names can be determined by snooping on dbus # while opening the browser in KDE: # dbus-monitor "interface="org.kde.KWallet"" "type=method_return" keyring_name = { - "brave" : "Brave", - "chrome" : "Chrome", + "brave": "Brave", + "chrome": "Chrome", "chromium": "Chromium", - "edge" : "Microsoft Edge" if sys.platform == "darwin" else - "Chromium", - "opera" : "Opera" if sys.platform == "darwin" else "Chromium", - "thorium" : "Thorium", - "vivaldi" : "Vivaldi" if sys.platform == "darwin" else "Chrome", + "edge": "Microsoft Edge" if sys.platform == "darwin" else "Chromium", + "opera": "Opera" if sys.platform == "darwin" else "Chromium", + "thorium": "Thorium", + "vivaldi": "Vivaldi" if sys.platform == "darwin" else "Chrome", }[browser_name] browsers_without_profiles = {"opera"} return { - "browser" : browser_name, + "browser": browser_name, "directory": browser_dir, - "keyring" : keyring_name, - "profiles" : browser_name not in browsers_without_profiles + "keyring": keyring_name, + "profiles": browser_name not in browsers_without_profiles, } -def _chromium_cookie_decryptor( - browser_root, browser_keyring_name, keyring=None, meta_version=0): +def _chromium_cookie_decryptor(browser_root, browser_keyring_name, keyring=None, meta_version=0): if sys.platform in ("win32", "cygwin"): - return WindowsChromiumCookieDecryptor( - browser_root, meta_version) + return WindowsChromiumCookieDecryptor(browser_root, meta_version) elif sys.platform == "darwin": - return MacChromiumCookieDecryptor( - browser_keyring_name, meta_version) + return MacChromiumCookieDecryptor(browser_keyring_name, meta_version) else: - return LinuxChromiumCookieDecryptor( - browser_keyring_name, keyring, meta_version) + return LinuxChromiumCookieDecryptor(browser_keyring_name, keyring, meta_version) class ChromiumCookieDecryptor: @@ -498,15 +520,14 @@ def __init__(self, browser_keyring_name, keyring=None, meta_version=0): self._v10_key = self.derive_key(b"peanuts") self._v11_key = None if password is None else self.derive_key(password) self._cookie_counts = {"v10": 0, "v11": 0, "other": 0} - self._offset = (32 if meta_version >= 24 else 0) + self._offset = 32 if meta_version >= 24 else 0 @staticmethod def derive_key(password): # values from # https://chromium.googlesource.com/chromium/src/+/refs/heads # /main/components/os_crypt/os_crypt_linux.cc - return pbkdf2_sha1(password, salt=b"saltysalt", - iterations=1, key_length=16) + return pbkdf2_sha1(password, salt=b"saltysalt", iterations=1, key_length=16) @property def cookie_counts(self): @@ -520,16 +541,15 @@ def decrypt(self, encrypted_value): self._cookie_counts["v10"] += 1 value = _decrypt_aes_cbc(ciphertext, self._v10_key, self._offset) - elif version == b"v11": + if version == b"v11": self._cookie_counts["v11"] += 1 if self._v11_key is None: _log_warning("Unable to decrypt v11 cookies: no key found") return None value = _decrypt_aes_cbc(ciphertext, self._v11_key, self._offset) - else: - self._cookie_counts["other"] += 1 - return None + self._cookie_counts["other"] += 1 + return None if value is None: value = _decrypt_aes_cbc(ciphertext, self._empty_key, self._offset) @@ -543,15 +563,14 @@ def __init__(self, browser_keyring_name, meta_version=0): password = _get_mac_keyring_password(browser_keyring_name) self._v10_key = None if password is None else self.derive_key(password) self._cookie_counts = {"v10": 0, "other": 0} - self._offset = (32 if meta_version >= 24 else 0) + self._offset = 32 if meta_version >= 24 else 0 @staticmethod def derive_key(password): # values from # https://chromium.googlesource.com/chromium/src/+/refs/heads # /main/components/os_crypt/os_crypt_mac.mm - return pbkdf2_sha1(password, salt=b"saltysalt", - iterations=1003, key_length=16) + return pbkdf2_sha1(password, salt=b"saltysalt", iterations=1003, key_length=16) @property def cookie_counts(self): @@ -568,20 +587,19 @@ def decrypt(self, encrypted_value): return None return _decrypt_aes_cbc(ciphertext, self._v10_key, self._offset) - else: - self._cookie_counts["other"] += 1 - # other prefixes are considered "old data", - # which were stored as plaintext - # https://chromium.googlesource.com/chromium/src/+/refs/heads - # /main/components/os_crypt/os_crypt_mac.mm - return encrypted_value + self._cookie_counts["other"] += 1 + # other prefixes are considered "old data", + # which were stored as plaintext + # https://chromium.googlesource.com/chromium/src/+/refs/heads + # /main/components/os_crypt/os_crypt_mac.mm + return encrypted_value class WindowsChromiumCookieDecryptor(ChromiumCookieDecryptor): def __init__(self, browser_root, meta_version=0): self._v10_key = _get_windows_v10_key(browser_root) self._cookie_counts = {"v10": 0, "other": 0} - self._offset = (32 if meta_version >= 24 else 0) + self._offset = 32 if meta_version >= 24 else 0 @property def cookie_counts(self): @@ -607,25 +625,24 @@ def decrypt(self, encrypted_value): raw_ciphertext = ciphertext nonce = raw_ciphertext[:nonce_length] - ciphertext = raw_ciphertext[ - nonce_length:-authentication_tag_length] + ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length] authentication_tag = raw_ciphertext[-authentication_tag_length:] return _decrypt_aes_gcm( - ciphertext, self._v10_key, nonce, authentication_tag, - self._offset) + ciphertext, self._v10_key, nonce, authentication_tag, self._offset + ) - else: - self._cookie_counts["other"] += 1 - # any other prefix means the data is DPAPI encrypted - # https://chromium.googlesource.com/chromium/src/+/refs/heads - # /main/components/os_crypt/os_crypt_win.cc - return _decrypt_windows_dpapi(encrypted_value).decode() + self._cookie_counts["other"] += 1 + # any other prefix means the data is DPAPI encrypted + # https://chromium.googlesource.com/chromium/src/+/refs/heads + # /main/components/os_crypt/os_crypt_win.cc + return _decrypt_windows_dpapi(encrypted_value).decode() # -------------------------------------------------------------------- # keyring + def _choose_linux_keyring(): """ https://chromium.googlesource.com/chromium/src/+/refs/heads @@ -642,7 +659,7 @@ def _choose_linux_keyring(): def _get_kwallet_network_wallet(): - """ The name of the wallet used to store network passwords. + """The name of the wallet used to store network passwords. https://chromium.googlesource.com/chromium/src/+/refs/heads /main/components/os_crypt/kwallet_dbus.cc @@ -654,22 +671,22 @@ def _get_kwallet_network_wallet(): default_wallet = "kdewallet" try: proc, stdout = Popen_communicate( - "dbus-send", "--session", "--print-reply=literal", + "dbus-send", + "--session", + "--print-reply=literal", "--dest=org.kde.kwalletd5", "/modules/kwalletd5", - "org.kde.KWallet.networkWallet" + "org.kde.KWallet.networkWallet", ) if proc.returncode != 0: _log_warning("Failed to read NetworkWallet") return default_wallet - else: - network_wallet = stdout.decode().strip() - _log_debug("NetworkWallet = '%s'", network_wallet) - return network_wallet + network_wallet = stdout.decode().strip() + _log_debug("NetworkWallet = '%s'", network_wallet) + return network_wallet except Exception as exc: - _log_warning("Error while obtaining NetworkWallet (%s: %s)", - exc.__class__.__name__, exc) + _log_warning("Error while obtaining NetworkWallet (%s: %s)", exc.__class__.__name__, exc) return default_wallet @@ -680,7 +697,8 @@ def _get_kwallet_password(browser_keyring_name): _log_error( "kwallet-query command not found. KWallet and kwallet-query " "must be installed to read from KWallet. kwallet-query should be " - "included in the kwallet package for your distribution") + "included in the kwallet package for your distribution" + ) return b"" network_wallet = _get_kwallet_network_wallet() @@ -688,20 +706,23 @@ def _get_kwallet_password(browser_keyring_name): try: proc, stdout = Popen_communicate( "kwallet-query", - "--read-password", browser_keyring_name + " Safe Storage", - "--folder", browser_keyring_name + " Keys", + "--read-password", + browser_keyring_name + " Safe Storage", + "--folder", + browser_keyring_name + " Keys", network_wallet, ) if proc.returncode != 0: - _log_error("kwallet-query failed with return code {}. " - "Please consult the kwallet-query man page " - "for details".format(proc.returncode)) + _log_error( + f"kwallet-query failed with return code {proc.returncode}. " + "Please consult the kwallet-query man page " + "for details" + ) return b"" if stdout.lower().startswith(b"failed to read"): - _log_debug("Failed to read password from kwallet. " - "Using empty string instead") + _log_debug("Failed to read password from kwallet. " "Using empty string instead") # This sometimes occurs in KDE because chrome does not check # hasEntry and instead just tries to read the value (which # kwallet returns "") whereas kwallet-query checks hasEntry. @@ -711,13 +732,11 @@ def _get_kwallet_password(browser_keyring_name): # This may be a bug, as the intended behaviour is to generate a # random password and store it, but that doesn't matter here. return b"" - else: - if stdout[-1:] == b"\n": - stdout = stdout[:-1] - return stdout + if stdout[-1:] == b"\n": + stdout = stdout[:-1] + return stdout except Exception as exc: - _log_warning("Error when running kwallet-query (%s: %s)", - exc.__class__.__name__, exc) + _log_warning("Error when running kwallet-query (%s: %s)", exc.__class__.__name__, exc) return b"" @@ -740,9 +759,8 @@ def _get_gnome_keyring_password(browser_keyring_name): for item in col.get_all_items(): if item.get_label() == label: return item.get_secret() - else: - _log_error("Failed to read from GNOME keyring") - return b"" + _log_error("Failed to read from GNOME keyring") + return b"" finally: con.close() @@ -761,9 +779,9 @@ def _get_linux_keyring_password(browser_keyring_name, keyring): if keyring == KEYRING_KWALLET: return _get_kwallet_password(browser_keyring_name) - elif keyring == KEYRING_GNOMEKEYRING: + if keyring == KEYRING_GNOMEKEYRING: return _get_gnome_keyring_password(browser_keyring_name) - elif keyring == KEYRING_BASICTEXT: + if keyring == KEYRING_BASICTEXT: # when basic text is chosen, all cookies are stored as v10 # so no keyring password is required return None @@ -771,22 +789,23 @@ def _get_linux_keyring_password(browser_keyring_name, keyring): def _get_mac_keyring_password(browser_keyring_name): - _log_debug("Using find-generic-password to obtain " - "password from OSX keychain") + _log_debug("Using find-generic-password to obtain " "password from OSX keychain") try: proc, stdout = Popen_communicate( - "security", "find-generic-password", + "security", + "find-generic-password", "-w", # write password to stdout - "-a", browser_keyring_name, # match "account" - "-s", browser_keyring_name + " Safe Storage", # match "service" + "-a", + browser_keyring_name, # match "account" + "-s", + browser_keyring_name + " Safe Storage", # match "service" ) if stdout[-1:] == b"\n": stdout = stdout[:-1] return stdout except Exception as exc: - _log_warning("Error when using find-generic-password (%s: %s)", - exc.__class__.__name__, exc) + _log_warning("Error when using find-generic-password (%s: %s)", exc.__class__.__name__, exc) return None @@ -808,12 +827,13 @@ def _get_windows_v10_key(browser_root): if not encrypted_key.startswith(prefix): _log_error("Invalid Local State key") return None - return _decrypt_windows_dpapi(encrypted_key[len(prefix):]) + return _decrypt_windows_dpapi(encrypted_key[len(prefix) :]) # -------------------------------------------------------------------- # utility + class ParserError(Exception): pass @@ -825,19 +845,18 @@ def __init__(self, data): def read_bytes(self, num_bytes): if num_bytes < 0: - raise ParserError("invalid read of {} bytes".format(num_bytes)) + raise ParserError(f"invalid read of {num_bytes} bytes") end = self.cursor + num_bytes if end > len(self._data): raise ParserError("reached end of input") - data = self._data[self.cursor:end] + data = self._data[self.cursor : end] self.cursor = end return data def expect_bytes(self, expected_value, message): value = self.read_bytes(len(expected_value)) if value != expected_value: - raise ParserError("unexpected value: {} != {} ({})".format( - value, expected_value, message)) + raise ParserError(f"unexpected value: {value} != {expected_value} ({message})") def read_uint(self, big_endian=False): data_format = ">I" if big_endian else " 0: - _log_debug("Skipping {} bytes ({}): {!r}".format( - num_bytes, description, self.read_bytes(num_bytes))) + _log_debug( + f"Skipping {num_bytes} bytes ({description}): {self.read_bytes(num_bytes)!r}" + ) elif num_bytes < 0: - raise ParserError("Invalid skip of {} bytes".format(num_bytes)) + raise ParserError(f"Invalid skip of {num_bytes} bytes") def skip_to(self, offset, description="unknown"): self.skip(offset - self.cursor, description) @@ -870,8 +889,7 @@ def skip_to_end(self, description="unknown"): self.skip_to(len(self._data), description) -class DatabaseConnection(): - +class DatabaseConnection: def __init__(self, path): self.path = path self.database = None @@ -884,20 +902,23 @@ def __enter__(self): if util.WINDOWS: path = "/" + os.path.abspath(path) - uri = "file:{}?mode=ro&immutable=1".format(path) + uri = f"file:{path}?mode=ro&immutable=1" self.database = sqlite3.connect( - uri, uri=True, isolation_level=None, check_same_thread=False) + uri, uri=True, isolation_level=None, check_same_thread=False + ) return self.database except Exception as exc: - _log_debug("Falling back to temporary database copy (%s: %s)", - exc.__class__.__name__, exc) + _log_debug( + "Falling back to temporary database copy (%s: %s)", exc.__class__.__name__, exc + ) try: self.directory = tempfile.TemporaryDirectory(prefix="gallery-dl-") path_copy = os.path.join(self.directory.name, "copy.sqlite") shutil.copyfile(self.path, path_copy) self.database = sqlite3.connect( - path_copy, isolation_level=None, check_same_thread=False) + path_copy, isolation_level=None, check_same_thread=False + ) return self.database except BaseException: if self.directory: @@ -911,8 +932,7 @@ def __exit__(self, exc_type, exc_value, traceback): def Popen_communicate(*args): - proc = util.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + proc = util.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) try: stdout, stderr = proc.communicate() except BaseException: # Including KeyboardInterrupt @@ -954,23 +974,21 @@ def _get_linux_desktop_environment(env): desktop_session = env.get("DESKTOP_SESSION") if xdg_current_desktop: - xdg_current_desktop = (xdg_current_desktop.partition(":")[0] - .strip().lower()) + xdg_current_desktop = xdg_current_desktop.partition(":")[0].strip().lower() if xdg_current_desktop == "unity": if desktop_session and "gnome-fallback" in desktop_session: return DE_GNOME - else: - return DE_UNITY - elif xdg_current_desktop == "gnome": + return DE_UNITY + if xdg_current_desktop == "gnome": return DE_GNOME - elif xdg_current_desktop == "x-cinnamon": + if xdg_current_desktop == "x-cinnamon": return DE_CINNAMON - elif xdg_current_desktop == "kde": + if xdg_current_desktop == "kde": return DE_KDE - elif xdg_current_desktop == "pantheon": + if xdg_current_desktop == "pantheon": return DE_PANTHEON - elif xdg_current_desktop == "xfce": + if xdg_current_desktop == "xfce": return DE_XFCE if desktop_session: @@ -997,10 +1015,8 @@ def pbkdf2_sha1(password, salt, iterations, key_length): return pbkdf2_hmac("sha1", password, salt, iterations, key_length) -def _decrypt_aes_cbc(ciphertext, key, offset=0, - initialization_vector=b" " * 16): - plaintext = aes.unpad_pkcs7(aes.aes_cbc_decrypt_bytes( - ciphertext, key, initialization_vector)) +def _decrypt_aes_cbc(ciphertext, key, offset=0, initialization_vector=b" " * 16): + plaintext = aes.unpad_pkcs7(aes.aes_cbc_decrypt_bytes(ciphertext, key, initialization_vector)) if offset: plaintext = plaintext[offset:] try: @@ -1011,8 +1027,7 @@ def _decrypt_aes_cbc(ciphertext, key, offset=0, def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, offset=0): try: - plaintext = aes.aes_gcm_decrypt_and_verify_bytes( - ciphertext, key, authentication_tag, nonce) + plaintext = aes.aes_gcm_decrypt_and_verify_bytes(ciphertext, key, authentication_tag, nonce) if offset: plaintext = plaintext[offset:] return plaintext.decode() @@ -1032,8 +1047,7 @@ def _decrypt_windows_dpapi(ciphertext): from ctypes.wintypes import DWORD class DATA_BLOB(ctypes.Structure): - _fields_ = [("cbData", DWORD), - ("pbData", ctypes.POINTER(ctypes.c_char))] + _fields_ = [("cbData", DWORD), ("pbData", ctypes.POINTER(ctypes.c_char))] buffer = ctypes.create_string_buffer(ciphertext) blob_in = DATA_BLOB(ctypes.sizeof(buffer), buffer) @@ -1045,7 +1059,7 @@ class DATA_BLOB(ctypes.Structure): None, # pvReserved: must be NULL None, # pPromptStruct: information about prompts to display 0, # dwFlags - ctypes.byref(blob_out) # pDataOut + ctypes.byref(blob_out), # pDataOut ) if not ret: _log_warning("Failed to decrypt cookie (DPAPI)") @@ -1078,13 +1092,12 @@ def _is_path(value): return os.path.sep in value -def _parse_browser_specification( - browser, profile=None, keyring=None, container=None, domain=None): +def _parse_browser_specification(browser, profile=None, keyring=None, container=None, domain=None): browser = browser.lower() if browser not in SUPPORTED_BROWSERS: - raise ValueError("Unsupported browser '{}'".format(browser)) + raise ValueError(f"Unsupported browser '{browser}'") if keyring and keyring not in SUPPORTED_KEYRINGS: - raise ValueError("Unsupported keyring '{}'".format(keyring)) + raise ValueError(f"Unsupported keyring '{keyring}'") if profile and _is_path(profile): profile = os.path.expanduser(profile) return browser, profile, keyring, container, domain diff --git a/gallery_dl/downloader/__init__.py b/gallery_dl/downloader/__init__.py index e1b936e465..4f0b424851 100644 --- a/gallery_dl/downloader/__init__.py +++ b/gallery_dl/downloader/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2021 Mike Fährmann # # This program is free software; you can redistribute it and/or modify diff --git a/gallery_dl/downloader/common.py b/gallery_dl/downloader/common.py index 1168d83df2..46c8977f5f 100644 --- a/gallery_dl/downloader/common.py +++ b/gallery_dl/downloader/common.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -9,11 +7,14 @@ """Common classes and constants used by downloader modules.""" import os -from .. import config, util + +from .. import config +from .. import util -class DownloaderBase(): +class DownloaderBase: """Base class for downloaders""" + scheme = "" def __init__(self, job): diff --git a/gallery_dl/downloader/http.py b/gallery_dl/downloader/http.py index 54750ac733..53af50d751 100644 --- a/gallery_dl/downloader/http.py +++ b/gallery_dl/downloader/http.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,18 @@ """Downloader module for http:// and https:// URLs""" -import time import mimetypes -from requests.exceptions import RequestException, ConnectionError, Timeout -from .common import DownloaderBase -from .. import text, util +import time from ssl import SSLError +from requests.exceptions import ConnectionError +from requests.exceptions import RequestException +from requests.exceptions import Timeout + +from .. import text +from .. import util +from .common import DownloaderBase + class HttpDownloader(DownloaderBase): scheme = "http" @@ -50,35 +53,30 @@ def __init__(self, job): if self.minsize: minsize = text.parse_bytes(self.minsize) if not minsize: - self.log.warning( - "Invalid minimum file size (%r)", self.minsize) + self.log.warning("Invalid minimum file size (%r)", self.minsize) self.minsize = minsize if self.maxsize: maxsize = text.parse_bytes(self.maxsize) if not maxsize: - self.log.warning( - "Invalid maximum file size (%r)", self.maxsize) + self.log.warning("Invalid maximum file size (%r)", self.maxsize) self.maxsize = maxsize if isinstance(self.chunk_size, str): chunk_size = text.parse_bytes(self.chunk_size) if not chunk_size: - self.log.warning( - "Invalid chunk size (%r)", self.chunk_size) + self.log.warning("Invalid chunk size (%r)", self.chunk_size) chunk_size = 32768 self.chunk_size = chunk_size if self.rate: rate = text.parse_bytes(self.rate) if rate: - if rate < self.chunk_size: - self.chunk_size = rate + self.chunk_size = min(rate, self.chunk_size) self.rate = rate self.receive = self._receive_rate else: self.log.warning("Invalid rate limit (%r)", self.rate) if self.progress is not None: self.receive = self._receive_rate - if self.progress < 0.0: - self.progress = 0.0 + self.progress = max(self.progress, 0.0) def download(self, url, pathfmt): try: @@ -98,10 +96,8 @@ def _download_impl(self, url, pathfmt): metadata = self.metadata kwdict = pathfmt.kwdict - expected_status = kwdict.get( - "_http_expected_status", ()) - adjust_extension = kwdict.get( - "_http_adjust_extension", self.adjust_extension) + expected_status = kwdict.get("_http_expected_status", ()) + adjust_extension = kwdict.get("_http_adjust_extension", self.adjust_extension) if self.part and not metadata: pathfmt.part_enable(self.partdir) @@ -111,7 +107,7 @@ def _download_impl(self, url, pathfmt): if response: self.release_conn(response) response = None - self.log.warning("%s (%s/%s)", msg, tries, self.retries+1) + self.log.warning("%s (%s/%s)", msg, tries, self.retries + 1) if tries > self.retries: return False time.sleep(tries) @@ -131,12 +127,13 @@ def _download_impl(self, url, pathfmt): # partial content file_size = pathfmt.part_size() if file_size: - headers["Range"] = "bytes={}-".format(file_size) + headers["Range"] = f"bytes={file_size}-" # connect to (remote) source try: response = self.session.request( - kwdict.get("_http_method", "GET"), url, + kwdict.get("_http_method", "GET"), + url, stream=True, headers=headers, data=kwdict.get("_http_data"), @@ -162,7 +159,7 @@ def _download_impl(self, url, pathfmt): elif code == 416 and file_size: # Requested Range Not Satisfiable break else: - msg = "'{} {}' for '{}'".format(code, response.reason, url) + msg = f"'{code} {response.reason}' for '{url}'" if code in self.retry_codes or 500 <= code < 600: continue retry = kwdict.get("_http_retry") @@ -195,15 +192,15 @@ def _download_impl(self, url, pathfmt): if self.minsize and size < self.minsize: self.release_conn(response) self.log.warning( - "File size smaller than allowed minimum (%s < %s)", - size, self.minsize) + "File size smaller than allowed minimum (%s < %s)", size, self.minsize + ) pathfmt.temppath = "" return True if self.maxsize and size > self.maxsize: self.release_conn(response) self.log.warning( - "File size larger than allowed maximum (%s > %s)", - size, self.maxsize) + "File size larger than allowed maximum (%s > %s)", size, self.maxsize + ) pathfmt.temppath = "" return True @@ -240,18 +237,16 @@ def _download_impl(self, url, pathfmt): content = response.iter_content(self.chunk_size) # check filename extension against file header - if adjust_extension and not offset and \ - pathfmt.extension in SIGNATURE_CHECKS: + if adjust_extension and not offset and pathfmt.extension in SIGNATURE_CHECKS: try: file_header = next( - content if response.raw.chunked - else response.iter_content(16), b"") + content if response.raw.chunked else response.iter_content(16), b"" + ) except (RequestException, SSLError) as exc: msg = str(exc) print() continue - if self._adjust_extension(pathfmt, file_header) and \ - pathfmt.exists(): + if self._adjust_extension(pathfmt, file_header) and pathfmt.exists(): pathfmt.temppath = "" response.close() return True @@ -272,8 +267,7 @@ def _download_impl(self, url, pathfmt): fp.write(file_header) offset += len(file_header) elif offset: - if adjust_extension and \ - pathfmt.extension in SIGNATURE_CHECKS: + if adjust_extension and pathfmt.extension in SIGNATURE_CHECKS: self._adjust_extension(pathfmt, fp.read(16)) fp.seek(offset) @@ -287,8 +281,7 @@ def _download_impl(self, url, pathfmt): # check file size if size and fp.tell() < size: - msg = "file size mismatch ({} < {})".format( - fp.tell(), size) + msg = f"file size mismatch ({fp.tell()} < {size})" print() continue @@ -310,8 +303,10 @@ def release_conn(self, response): except (RequestException, SSLError) as exc: print() self.log.debug( - "Unable to consume response body (%s: %s); " - "closing the connection anyway", exc.__class__.__name__, exc) + "Unable to consume response body (%s: %s); " "closing the connection anyway", + exc.__class__.__name__, + exc, + ) response.close() @staticmethod @@ -334,13 +329,12 @@ def _receive_rate(self, fp, content, bytes_total, bytes_start): write(data) - if progress is not None: - if time_elapsed > progress: - self.out.progress( - bytes_total, - bytes_start + bytes_downloaded, - int(bytes_downloaded / time_elapsed), - ) + if progress is not None and time_elapsed > progress: + self.out.progress( + bytes_total, + bytes_start + bytes_downloaded, + int(bytes_downloaded / time_elapsed), + ) if rate: time_expected = bytes_downloaded / rate @@ -378,51 +372,46 @@ def _adjust_extension(pathfmt, file_header): MIME_TYPES = { - "image/jpeg" : "jpg", - "image/jpg" : "jpg", - "image/png" : "png", - "image/gif" : "gif", - "image/bmp" : "bmp", - "image/x-bmp" : "bmp", + "image/jpeg": "jpg", + "image/jpg": "jpg", + "image/png": "png", + "image/gif": "gif", + "image/bmp": "bmp", + "image/x-bmp": "bmp", "image/x-ms-bmp": "bmp", - "image/webp" : "webp", - "image/avif" : "avif", - "image/heic" : "heic", - "image/heif" : "heif", - "image/svg+xml" : "svg", - "image/ico" : "ico", - "image/icon" : "ico", - "image/x-icon" : "ico", - "image/vnd.microsoft.icon" : "ico", - "image/x-photoshop" : "psd", - "application/x-photoshop" : "psd", + "image/webp": "webp", + "image/avif": "avif", + "image/heic": "heic", + "image/heif": "heif", + "image/svg+xml": "svg", + "image/ico": "ico", + "image/icon": "ico", + "image/x-icon": "ico", + "image/vnd.microsoft.icon": "ico", + "image/x-photoshop": "psd", + "application/x-photoshop": "psd", "image/vnd.adobe.photoshop": "psd", - "video/webm": "webm", - "video/ogg" : "ogg", - "video/mp4" : "mp4", - "video/m4v" : "m4v", + "video/ogg": "ogg", + "video/mp4": "mp4", + "video/m4v": "m4v", "video/x-m4v": "m4v", "video/quicktime": "mov", - - "audio/wav" : "wav", + "audio/wav": "wav", "audio/x-wav": "wav", - "audio/webm" : "webm", - "audio/ogg" : "ogg", - "audio/mpeg" : "mp3", - - "application/zip" : "zip", + "audio/webm": "webm", + "audio/ogg": "ogg", + "audio/mpeg": "mp3", + "application/zip": "zip", "application/x-zip": "zip", "application/x-zip-compressed": "zip", - "application/rar" : "rar", + "application/rar": "rar", "application/x-rar": "rar", "application/x-rar-compressed": "rar", - "application/x-7z-compressed" : "7z", - - "application/pdf" : "pdf", + "application/x-7z-compressed": "7z", + "application/pdf": "pdf", "application/x-pdf": "pdf", "application/x-shockwave-flash": "swf", - "application/ogg": "ogg", # https://www.iana.org/assignments/media-types/model/obj "model/obj": "obj", @@ -431,43 +420,40 @@ def _adjust_extension(pathfmt, file_header): # https://en.wikipedia.org/wiki/List_of_file_signatures SIGNATURE_CHECKS = { - "jpg" : lambda s: s[0:3] == b"\xFF\xD8\xFF", - "png" : lambda s: s[0:8] == b"\x89PNG\r\n\x1A\n", - "gif" : lambda s: s[0:6] in (b"GIF87a", b"GIF89a"), - "bmp" : lambda s: s[0:2] == b"BM", - "webp": lambda s: (s[0:4] == b"RIFF" and - s[8:12] == b"WEBP"), + "jpg": lambda s: s[0:3] == b"\xff\xd8\xff", + "png": lambda s: s[0:8] == b"\x89PNG\r\n\x1a\n", + "gif": lambda s: s[0:6] in (b"GIF87a", b"GIF89a"), + "bmp": lambda s: s[0:2] == b"BM", + "webp": lambda s: (s[0:4] == b"RIFF" and s[8:12] == b"WEBP"), "avif": lambda s: s[4:11] == b"ftypavi" and s[11] in b"fs", - "heic": lambda s: (s[4:10] == b"ftyphe" and s[10:12] in ( - b"ic", b"im", b"is", b"ix", b"vc", b"vm", b"vs")), - "svg" : lambda s: s[0:5] == b"= 0 else float("inf"), + "retries": retries + 1 if retries >= 0 else float("inf"), "socket_timeout": self.config("timeout", extractor._timeout), "nocheckcertificate": not self.config("verify", extractor._verify), "proxy": self.proxies.get("http") if self.proxies else None, @@ -43,18 +43,17 @@ def download(self, url, pathfmt): try: module = ytdl.import_module(self.config("module")) except (ImportError, SyntaxError) as exc: - self.log.error("Cannot import module '%s'", - getattr(exc, "name", "")) + self.log.error("Cannot import module '%s'", getattr(exc, "name", "")) self.log.debug("", exc_info=exc) self.download = lambda u, p: False return False self.ytdl_instance = ytdl_instance = ytdl.construct_YoutubeDL( - module, self, self.ytdl_opts) + module, self, self.ytdl_opts + ) if self.outtmpl == "default": self.outtmpl = module.DEFAULT_OUTTMPL if self.forward_cookies: - self.log.debug("Forwarding cookies to %s", - ytdl_instance.__module__) + self.log.debug("Forwarding cookies to %s", ytdl_instance.__module__) set_cookie = ytdl_instance.cookiejar.set_cookie for cookie in self.session.cookies: set_cookie(cookie) @@ -68,8 +67,7 @@ def download(self, url, pathfmt): try: manifest = kwdict.pop("_ytdl_manifest", None) if manifest: - info_dict = self._extract_manifest( - ytdl_instance, url, manifest) + info_dict = self._extract_manifest(ytdl_instance, url, manifest) else: info_dict = self._extract_info(ytdl_instance, url) except Exception as exc: @@ -82,10 +80,8 @@ def download(self, url, pathfmt): if "entries" in info_dict: index = kwdict.get("_ytdl_index") if index is None: - return self._download_playlist( - ytdl_instance, pathfmt, info_dict) - else: - info_dict = info_dict["entries"][index] + return self._download_playlist(ytdl_instance, pathfmt, info_dict) + info_dict = info_dict["entries"][index] extra = kwdict.get("_ytdl_extra") if extra: @@ -108,12 +104,10 @@ def _download_video(self, ytdl_instance, pathfmt, info_dict): if self.outtmpl: self._set_outtmpl(ytdl_instance, self.outtmpl) - pathfmt.filename = filename = \ - ytdl_instance.prepare_filename(info_dict) + pathfmt.filename = filename = ytdl_instance.prepare_filename(info_dict) pathfmt.extension = info_dict["ext"] pathfmt.path = pathfmt.directory + filename - pathfmt.realpath = pathfmt.temppath = ( - pathfmt.realdirectory + filename) + pathfmt.realpath = pathfmt.temppath = pathfmt.realdirectory + filename else: pathfmt.set_extension(info_dict["ext"]) pathfmt.build_path() @@ -122,8 +116,7 @@ def _download_video(self, ytdl_instance, pathfmt, info_dict): pathfmt.temppath = "" return True if self.part and self.partdir: - pathfmt.temppath = os.path.join( - self.partdir, pathfmt.filename) + pathfmt.temppath = os.path.join(self.partdir, pathfmt.filename) self._set_outtmpl(ytdl_instance, pathfmt.temppath.replace("%", "%%")) @@ -153,16 +146,14 @@ def _extract_manifest(self, ytdl, url, manifest): if manifest == "hls": try: - formats, subtitles = extr._extract_m3u8_formats_and_subtitles( - url, video_id, "mp4") + formats, subtitles = extr._extract_m3u8_formats_and_subtitles(url, video_id, "mp4") except AttributeError: formats = extr._extract_m3u8_formats(url, video_id, "mp4") subtitles = None elif manifest == "dash": try: - formats, subtitles = extr._extract_mpd_formats_and_subtitles( - url, video_id) + formats, subtitles = extr._extract_mpd_formats_and_subtitles(url, video_id) except AttributeError: formats = extr._extract_mpd_formats(url, video_id) subtitles = None @@ -172,17 +163,16 @@ def _extract_manifest(self, ytdl, url, manifest): return None info_dict = { - "id" : video_id, - "title" : video_id, - "formats" : formats, + "id": video_id, + "title": video_id, + "formats": formats, "subtitles": subtitles, } # extr._extra_manifest_info(info_dict, url) return ytdl.process_ie_result(info_dict, download=False) def _progress_hook(self, info): - if info["status"] == "downloading" and \ - info["elapsed"] >= self.progress: + if info["status"] == "downloading" and info["elapsed"] >= self.progress: total = info.get("total_bytes") or info.get("total_bytes_estimate") speed = info.get("speed") self.out.progress( diff --git a/gallery_dl/exception.py b/gallery_dl/exception.py index 6b2ce3a8e9..328fb6b4cc 100644 --- a/gallery_dl/exception.py +++ b/gallery_dl/exception.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -31,6 +29,7 @@ class GalleryDLException(Exception): """Base class for GalleryDL exceptions""" + default = None msgfmt = None code = 1 @@ -39,7 +38,7 @@ def __init__(self, message=None, fmt=True): if not message: message = self.default elif isinstance(message, Exception): - message = "{}: {}".format(message.__class__.__name__, message) + message = f"{message.__class__.__name__}: {message}" if self.msgfmt and fmt: message = self.msgfmt.format(message) Exception.__init__(self, message) @@ -51,6 +50,7 @@ class ExtractionError(GalleryDLException): class HttpError(ExtractionError): """HTTP request during data extraction failed""" + default = "HTTP request failed" code = 4 @@ -61,13 +61,13 @@ def __init__(self, message="", response=None): else: self.status = response.status_code if not message: - message = "'{} {}' for '{}'".format( - response.status_code, response.reason, response.url) + message = f"'{response.status_code} {response.reason}' for '{response.url}'" ExtractionError.__init__(self, message) class NotFoundError(ExtractionError): """Requested resource (gallery/image) could not be found""" + msgfmt = "Requested {} could not be found" default = "resource (gallery/image)" code = 8 @@ -75,48 +75,55 @@ class NotFoundError(ExtractionError): class AuthenticationError(ExtractionError): """Invalid or missing login credentials""" + default = "Invalid or missing login credentials" code = 16 class AuthorizationError(ExtractionError): """Insufficient privileges to access a resource""" + default = "Insufficient privileges to access the specified resource" code = 16 class FormatError(GalleryDLException): """Error while building output paths""" + code = 32 class FilenameFormatError(FormatError): """Error while building output filenames""" + msgfmt = "Applying filename format string failed ({})" class DirectoryFormatError(FormatError): """Error while building output directory paths""" + msgfmt = "Applying directory format string failed ({})" class FilterError(GalleryDLException): """Error while evaluating a filter expression""" + msgfmt = "Evaluating filter expression failed ({})" code = 32 class InputFileError(GalleryDLException): """Error when parsing input file""" + code = 32 def __init__(self, message, *args): - GalleryDLException.__init__( - self, message % args if args else message) + GalleryDLException.__init__(self, message % args if args else message) class NoExtractorError(GalleryDLException): """No extractor can handle the given URL""" + code = 64 @@ -131,9 +138,11 @@ def __init__(self, message=None, *args): class TerminateExtraction(GalleryDLException): """Terminate data extraction""" + code = 0 class RestartExtraction(GalleryDLException): """Restart data extraction""" + code = 0 diff --git a/gallery_dl/extractor/2ch.py b/gallery_dl/extractor/2ch.py index dbbf21b635..2dc76c2227 100644 --- a/gallery_dl/extractor/2ch.py +++ b/gallery_dl/extractor/2ch.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://2ch.hk/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message class _2chThreadExtractor(Extractor): """Extractor for 2ch threads""" + category = "2ch" subcategory = "thread" root = "https://2ch.hk" @@ -26,16 +27,16 @@ def __init__(self, match): self.board, self.thread = match.groups() def items(self): - url = "{}/{}/res/{}.json".format(self.root, self.board, self.thread) + url = f"{self.root}/{self.board}/res/{self.thread}.json" posts = self.request(url).json()["threads"][0]["posts"] op = posts[0] title = op.get("subject") or text.remove_html(op["comment"]) thread = { - "board" : self.board, + "board": self.board, "thread": self.thread, - "title" : text.unescape(title)[:50], + "title": text.unescape(title)[:50], } yield Message.Directory, thread @@ -52,14 +53,14 @@ def items(self): file.update(post) file["filename"] = file["fullname"].rpartition(".")[0] - file["tim"], _, file["extension"] = \ - file["name"].rpartition(".") + file["tim"], _, file["extension"] = file["name"].rpartition(".") yield Message.Url, self.root + file["path"], file class _2chBoardExtractor(Extractor): """Extractor for 2ch boards""" + category = "2ch" subcategory = "board" root = "https://2ch.hk" @@ -72,20 +73,18 @@ def __init__(self, match): def items(self): # index page - url = "{}/{}/index.json".format(self.root, self.board) + url = f"{self.root}/{self.board}/index.json" index = self.request(url).json() index["_extractor"] = _2chThreadExtractor for thread in index["threads"]: - url = "{}/{}/res/{}.html".format( - self.root, self.board, thread["thread_num"]) + url = "{}/{}/res/{}.html".format(self.root, self.board, thread["thread_num"]) yield Message.Queue, url, index # pages 1..n for n in util.advance(index["pages"], 1): - url = "{}/{}/{}.json".format(self.root, self.board, n) + url = f"{self.root}/{self.board}/{n}.json" page = self.request(url).json() page["_extractor"] = _2chThreadExtractor for thread in page["threads"]: - url = "{}/{}/res/{}.html".format( - self.root, self.board, thread["thread_num"]) + url = "{}/{}/res/{}.html".format(self.root, self.board, thread["thread_num"]) yield Message.Queue, url, page diff --git a/gallery_dl/extractor/2chan.py b/gallery_dl/extractor/2chan.py index 337ba48047..fec036ccd2 100644 --- a/gallery_dl/extractor/2chan.py +++ b/gallery_dl/extractor/2chan.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,14 @@ """Extractors for https://www.2chan.net/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class _2chanThreadExtractor(Extractor): """Extractor for 2chan threads""" + category = "2chan" subcategory = "thread" directory_fmt = ("{category}", "{board_name}", "{thread}") @@ -28,8 +28,7 @@ def __init__(self, match): self.server, self.board, self.thread = match.groups() def items(self): - url = "https://{}.2chan.net/{}/res/{}.htm".format( - self.server, self.board, self.thread) + url = f"https://{self.server}.2chan.net/{self.board}/res/{self.thread}.htm" page = self.request(url).text data = self.metadata(page) yield Message.Directory, data @@ -42,8 +41,7 @@ def items(self): def metadata(self, page): """Collect metadata for extractor-job""" - title, _, boardname = text.extr( - page, "", "").rpartition(" - ") + title, _, boardname = text.extr(page, "", "").rpartition(" - ") return { "server": self.server, "title": title, @@ -54,12 +52,8 @@ def metadata(self, page): def posts(self, page): """Build a list of all post-objects""" - page = text.extr( - page, '
') - return [ - self.parse(post) - for post in page.split('') - ] + page = text.extr(page, '
') + return [self.parse(post) for post in page.split("
")] def parse(self, post): """Build post-object by extracting data from an HTML post""" @@ -76,19 +70,27 @@ def parse(self, post): @staticmethod def _extract_post(post): - return text.extract_all(post, ( - ("post", 'class="csb">' , '<'), - ("name", 'class="cnm">' , '<'), - ("now" , 'class="cnw">' , '<'), - ("no" , 'class="cno">No.', '<'), - (None , '', ''), - ))[0] + return text.extract_all( + post, + ( + ("post", 'class="csb">', "<"), + ("name", 'class="cnm">', "<"), + ("now", 'class="cnw">', "<"), + ("no", 'class="cno">No.', "<"), + (None, "", ""), + ), + )[0] @staticmethod def _extract_image(post, data): - text.extract_all(post, ( - (None , '_blank', ''), - ("filename", '>', '<'), - ("fsize" , '(', ' '), - ), 0, data) + text.extract_all( + post, + ( + (None, "_blank", ""), + ("filename", ">", "<"), + ("fsize", "(", " "), + ), + 0, + data, + ) diff --git a/gallery_dl/extractor/2chen.py b/gallery_dl/extractor/2chen.py index 0c978897a0..002f5ea954 100644 --- a/gallery_dl/extractor/2chen.py +++ b/gallery_dl/extractor/2chen.py @@ -1,19 +1,19 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://sturdychan.help/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:sturdychan.help|2chen\.(?:moe|club))" class _2chenThreadExtractor(Extractor): """Extractor for 2chen threads""" + category = "2chen" subcategory = "thread" root = "https://sturdychan.help" @@ -28,13 +28,12 @@ def __init__(self, match): self.board, self.thread = match.groups() def items(self): - url = "{}/{}/{}".format(self.root, self.board, self.thread) + url = f"{self.root}/{self.board}/{self.thread}" page = self.request(url, encoding="utf-8", notfound="thread").text data = self.metadata(page) yield Message.Directory, data for post in self.posts(page): - url = post["url"] if not url: continue @@ -44,40 +43,38 @@ def items(self): post.update(data) post["time"] = text.parse_int(post["date"].timestamp()) - yield Message.Url, url, text.nameext_from_url( - post["filename"], post) + yield Message.Url, url, text.nameext_from_url(post["filename"], post) def metadata(self, page): - board, pos = text.extract(page, 'class="board">/', '/<') + board, pos = text.extract(page, 'class="board">/', "/<") title = text.extract(page, "

", "

", pos)[0] return { - "board" : board, + "board": board, "thread": self.thread, - "title" : text.unescape(title), + "title": text.unescape(title), } def posts(self, page): """Return iterable with relevant posts""" - return map(self.parse, text.extract_iter( - page, 'class="glass media', '')) + return map(self.parse, text.extract_iter(page, 'class="glass media', "")) def parse(self, post): extr = text.extract_from(post) return { - "name" : text.unescape(extr("", "")), - "date" : text.parse_datetime( - extr("")[2], - "%d %b %Y (%a) %H:%M:%S" + "name": text.unescape(extr("", "")), + "date": text.parse_datetime( + extr("")[2], "%d %b %Y (%a) %H:%M:%S" ), - "no" : extr('href="#p', '"'), - "url" : extr('[^&#]+)") + + pattern = ( + r"(?:https?://)?(?:www\.)?behoimi\.org/post" r"(?:/(?:index)?)?\?tags=(?P[^&#]+)" + ) example = "http://behoimi.org/post?tags=TAG" def posts(self): @@ -36,6 +37,7 @@ def posts(self): class _3dbooruPoolExtractor(_3dbooruBase, moebooru.MoebooruPoolExtractor): """Extractor for image-pools from behoimi.org""" + pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/pool/show/(?P\d+)" example = "http://behoimi.org/pool/show/12345" @@ -46,6 +48,7 @@ def posts(self): class _3dbooruPostExtractor(_3dbooruBase, moebooru.MoebooruPostExtractor): """Extractor for single images from behoimi.org""" + pattern = r"(?:https?://)?(?:www\.)?behoimi\.org/post/show/(?P\d+)" example = "http://behoimi.org/post/show/12345" @@ -54,10 +57,12 @@ def posts(self): return self._pagination(self.root + "/post/index.json", params) -class _3dbooruPopularExtractor( - _3dbooruBase, moebooru.MoebooruPopularExtractor): +class _3dbooruPopularExtractor(_3dbooruBase, moebooru.MoebooruPopularExtractor): """Extractor for popular images from behoimi.org""" - pattern = (r"(?:https?://)?(?:www\.)?behoimi\.org" - r"/post/popular_(?Pby_(?:day|week|month)|recent)" - r"(?:\?(?P[^#]*))?") + + pattern = ( + r"(?:https?://)?(?:www\.)?behoimi\.org" + r"/post/popular_(?Pby_(?:day|week|month)|recent)" + r"(?:\?(?P[^#]*))?" + ) example = "http://behoimi.org/post/popular_by_month" diff --git a/gallery_dl/extractor/4archive.py b/gallery_dl/extractor/4archive.py index 948a605460..3af8af62d8 100644 --- a/gallery_dl/extractor/4archive.py +++ b/gallery_dl/extractor/4archive.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://4archive.org/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message class _4archiveThreadExtractor(Extractor): """Extractor for 4archive threads""" + category = "4archive" subcategory = "thread" directory_fmt = ("{category}", "{board}", "{thread} {title}") @@ -27,8 +28,7 @@ def __init__(self, match): self.board, self.thread = match.groups() def items(self): - url = "{}/board/{}/thread/{}".format( - self.root, self.board, self.thread) + url = f"{self.root}/board/{self.board}/thread/{self.thread}" page = self.request(url).text data = self.metadata(page) posts = self.posts(page) @@ -41,22 +41,17 @@ def items(self): post["time"] = int(util.datetime_to_timestamp(post["date"])) yield Message.Directory, post if "url" in post: - yield Message.Url, post["url"], text.nameext_from_url( - post["filename"], post) + yield Message.Url, post["url"], text.nameext_from_url(post["filename"], post) def metadata(self, page): return { - "board" : self.board, + "board": self.board, "thread": text.parse_int(self.thread), - "title" : text.unescape(text.extr( - page, 'class="subject">', "")) + "title": text.unescape(text.extr(page, 'class="subject">', "")), } def posts(self, page): - return [ - self.parse(post) - for post in page.split('class="postContainer')[1:] - ] + return [self.parse(post) for post in page.split('class="postContainer')[1:]] @staticmethod def parse(post): @@ -64,28 +59,29 @@ def parse(post): data = { "name": extr('class="name">', ""), "date": text.parse_datetime( - extr('class="dateTime postNum">', "<").strip(), - "%Y-%m-%d %H:%M:%S"), - "no" : text.parse_int(extr('href="#p', '"')), + extr('class="dateTime postNum">', "<").strip(), "%Y-%m-%d %H:%M:%S" + ), + "no": text.parse_int(extr('href="#p', '"')), } if 'class="file"' in post: extr('class="fileText"', ">File: ").strip()[1:], - "size" : text.parse_bytes(extr(" (", ", ")[:-1]), - "width" : text.parse_int(extr("", "x")), - "height" : text.parse_int(extr("", "px")), - }) + data.update( + { + "url": extr('href="', '"'), + "filename": extr('rel="noreferrer noopener"', "").strip()[1:], + "size": text.parse_bytes(extr(" (", ", ")[:-1]), + "width": text.parse_int(extr("", "x")), + "height": text.parse_int(extr("", "px")), + } + ) extr("
", "
"))) + data["com"] = text.unescape(text.remove_html(extr(">", ""))) return data class _4archiveBoardExtractor(Extractor): """Extractor for 4archive boards""" + category = "4archive" subcategory = "board" root = "https://4archive.org" @@ -100,12 +96,11 @@ def __init__(self, match): def items(self): data = {"_extractor": _4archiveThreadExtractor} while True: - url = "{}/board/{}/{}".format(self.root, self.board, self.num) + url = f"{self.root}/board/{self.board}/{self.num}" page = self.request(url).text if 'class="thread"' not in page: return for thread in text.extract_iter(page, 'class="thread" id="t', '"'): - url = "{}/board/{}/thread/{}".format( - self.root, self.board, thread) + url = f"{self.root}/board/{self.board}/thread/{thread}" yield Message.Queue, url, data self.num += 1 diff --git a/gallery_dl/extractor/4chan.py b/gallery_dl/extractor/4chan.py index 2db60422ed..73631ab416 100644 --- a/gallery_dl/extractor/4chan.py +++ b/gallery_dl/extractor/4chan.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,19 +6,20 @@ """Extractors for https://www.4chan.org/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class _4chanThreadExtractor(Extractor): """Extractor for 4chan threads""" + category = "4chan" subcategory = "thread" directory_fmt = ("{category}", "{board}", "{thread} {title}") filename_fmt = "{tim} {filename}.{extension}" archive_fmt = "{board}_{thread}_{tim}" - pattern = (r"(?:https?://)?boards\.4chan(?:nel)?\.org" - r"/([^/]+)/thread/(\d+)") + pattern = r"(?:https?://)?boards\.4chan(?:nel)?\.org" r"/([^/]+)/thread/(\d+)" example = "https://boards.4channel.org/a/thread/12345/" def __init__(self, match): @@ -28,15 +27,14 @@ def __init__(self, match): self.board, self.thread = match.groups() def items(self): - url = "https://a.4cdn.org/{}/thread/{}.json".format( - self.board, self.thread) + url = f"https://a.4cdn.org/{self.board}/thread/{self.thread}.json" posts = self.request(url).json()["posts"] title = posts[0].get("sub") or text.remove_html(posts[0]["com"]) data = { - "board" : self.board, + "board": self.board, "thread": self.thread, - "title" : text.unescape(title)[:50], + "title": text.unescape(title)[:50], } yield Message.Directory, data @@ -45,13 +43,13 @@ def items(self): post.update(data) post["extension"] = post["ext"][1:] post["filename"] = text.unescape(post["filename"]) - url = "https://i.4cdn.org/{}/{}{}".format( - post["board"], post["tim"], post["ext"]) + url = "https://i.4cdn.org/{}/{}{}".format(post["board"], post["tim"], post["ext"]) yield Message.Url, url, post class _4chanBoardExtractor(Extractor): """Extractor for 4chan boards""" + category = "4chan" subcategory = "board" pattern = r"(?:https?://)?boards\.4chan(?:nel)?\.org/([^/?#]+)/\d*$" @@ -62,13 +60,12 @@ def __init__(self, match): self.board = match.group(1) def items(self): - url = "https://a.4cdn.org/{}/threads.json".format(self.board) + url = f"https://a.4cdn.org/{self.board}/threads.json" threads = self.request(url).json() for page in threads: for thread in page["threads"]: - url = "https://boards.4chan.org/{}/thread/{}/".format( - self.board, thread["no"]) + url = "https://boards.4chan.org/{}/thread/{}/".format(self.board, thread["no"]) thread["page"] = page["page"] thread["_extractor"] = _4chanThreadExtractor yield Message.Queue, url, thread diff --git a/gallery_dl/extractor/4chanarchives.py b/gallery_dl/extractor/4chanarchives.py index 27ac7c5567..ec188bc51d 100644 --- a/gallery_dl/extractor/4chanarchives.py +++ b/gallery_dl/extractor/4chanarchives.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,14 @@ """Extractors for https://4chanarchives.com/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class _4chanarchivesThreadExtractor(Extractor): """Extractor for threads on 4chanarchives.com""" + category = "4chanarchives" subcategory = "thread" root = "https://4chanarchives.com" @@ -29,15 +29,13 @@ def __init__(self, match): self.board, self.thread = match.groups() def items(self): - url = "{}/board/{}/thread/{}".format( - self.root, self.board, self.thread) + url = f"{self.root}/board/{self.board}/thread/{self.thread}" page = self.request(url).text data = self.metadata(page) posts = self.posts(page) if not data["title"]: - data["title"] = text.unescape(text.remove_html( - posts[0]["com"]))[:50] + data["title"] = text.unescape(text.remove_html(posts[0]["com"]))[:50] for post in posts: post.update(data) @@ -47,16 +45,14 @@ def items(self): def metadata(self, page): return { - "board" : self.board, - "thread" : self.thread, - "title" : text.unescape(text.extr( - page, 'property="og:title" content="', '"')), + "board": self.board, + "thread": self.thread, + "title": text.unescape(text.extr(page, 'property="og:title" content="', '"')), } def posts(self, page): """Build a list of all post objects""" - return [self.parse(html) for html in text.extract_iter( - page, 'id="pc', '')] + return [self.parse(html) for html in text.extract_iter(page, 'id="pc', "")] def parse(self, html): """Build post object by extracting data from an HTML post""" @@ -70,11 +66,10 @@ def parse(self, html): def _extract_post(html): extr = text.extract_from(html) return { - "no" : text.parse_int(extr('', '"')), - "name": extr('class="name">', '<'), - "time": extr('class="dateTime postNum" >', '<').rstrip(), - "com" : text.unescape( - html[html.find('")[2]), + "no": text.parse_int(extr("", '"')), + "name": extr('class="name">', "<"), + "time": extr('class="dateTime postNum" >', "<").rstrip(), + "com": text.unescape(html[html.find("")[2]), } @staticmethod @@ -89,6 +84,7 @@ def _extract_file(html, post): class _4chanarchivesBoardExtractor(Extractor): """Extractor for boards on 4chanarchives.com""" + category = "4chanarchives" subcategory = "board" root = "https://4chanarchives.com" @@ -106,7 +102,7 @@ def items(self): data["pageCount"]: return - url = "{}/{}/{}.json".format(self.root, board, pnum) + url = f"{self.root}/{board}/{pnum}.json" threads = self.request(url).json()["threads"] diff --git a/gallery_dl/extractor/8muses.py b/gallery_dl/extractor/8muses.py index f88a0c6745..1e4004ef4d 100644 --- a/gallery_dl/extractor/8muses.py +++ b/gallery_dl/extractor/8muses.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,20 +6,22 @@ """Extractors for https://comics.8muses.com/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message class _8musesAlbumExtractor(Extractor): """Extractor for image albums on comics.8muses.com""" + category = "8muses" subcategory = "album" directory_fmt = ("{category}", "{album[path]}") filename_fmt = "{page:>03}.{extension}" archive_fmt = "{hash}" root = "https://comics.8muses.com" - pattern = (r"(?:https?://)?(?:comics\.|www\.)?8muses\.com" - r"(/comics/album/[^?#]+)(\?[^#]+)?") + pattern = r"(?:https?://)?(?:comics\.|www\.)?8muses\.com" r"(/comics/album/[^?#]+)(\?[^#]+)?" example = "https://comics.8muses.com/comics/album/PATH/TITLE" def __init__(self, match): @@ -33,9 +33,11 @@ def items(self): url = self.root + self.path + self.params while True: - data = self._unobfuscate(text.extr( - self.request(url).text, - 'id="ractive-public" type="text/plain">', '')) + data = self._unobfuscate( + text.extr( + self.request(url).text, 'id="ractive-public" type="text/plain">', "" + ) + ) images = data.get("pictures") if images: @@ -45,11 +47,11 @@ def items(self): for num, image in enumerate(images, 1): url = self.root + "/image/fl/" + image["publicUri"] img = { - "url" : url, - "page" : num, - "hash" : image["publicUri"], - "count" : count, - "album" : album, + "url": url, + "page": num, + "hash": image["publicUri"], + "count": count, + "album": album, "extension": "jpg", } yield Message.Url, url, img @@ -58,38 +60,44 @@ def items(self): if albums: for album in albums: url = self.root + "/comics/album/" + album["permalink"] - yield Message.Queue, url, { - "url" : url, - "name" : album["name"], - "private" : album["isPrivate"], - "_extractor": _8musesAlbumExtractor, - } + yield ( + Message.Queue, + url, + { + "url": url, + "name": album["name"], + "private": album["isPrivate"], + "_extractor": _8musesAlbumExtractor, + }, + ) if data["page"] >= data["pages"]: return path, _, num = self.path.rstrip("/").rpartition("/") path = path if num.isdecimal() else self.path - url = "{}{}/{}{}".format( - self.root, path, data["page"] + 1, self.params) + url = "{}{}/{}{}".format(self.root, path, data["page"] + 1, self.params) def _make_album(self, album): return { - "id" : album["id"], - "path" : album["path"], - "parts" : album["path"].split("/"), - "title" : album["name"], + "id": album["id"], + "path": album["path"], + "parts": album["path"].split("/"), + "title": album["name"], "private": album["isPrivate"], - "url" : self.root + "/comics/album/" + album["permalink"], - "parent" : text.parse_int(album["parentId"]), - "views" : text.parse_int(album["numberViews"]), - "likes" : text.parse_int(album["numberLikes"]), - "date" : text.parse_datetime( - album["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ"), + "url": self.root + "/comics/album/" + album["permalink"], + "parent": text.parse_int(album["parentId"]), + "views": text.parse_int(album["numberViews"]), + "likes": text.parse_int(album["numberLikes"]), + "date": text.parse_datetime(album["updatedAt"], "%Y-%m-%dT%H:%M:%S.%fZ"), } @staticmethod def _unobfuscate(data): - return util.json_loads("".join([ - chr(33 + (ord(c) + 14) % 94) if "!" <= c <= "~" else c - for c in text.unescape(data.strip("\t\n\r !")) - ])) + return util.json_loads( + "".join( + [ + chr(33 + (ord(c) + 14) % 94) if "!" <= c <= "~" else c + for c in text.unescape(data.strip("\t\n\r !")) + ] + ) + ) diff --git a/gallery_dl/extractor/__init__.py b/gallery_dl/extractor/__init__.py index 903370213f..330116fe81 100644 --- a/gallery_dl/extractor/__init__.py +++ b/gallery_dl/extractor/__init__.py @@ -1,13 +1,11 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import sys import re +import sys modules = [ "2ch", @@ -238,10 +236,7 @@ def add_module(module): def extractors(): """Yield all available extractor classes""" - return sorted( - _list_classes(), - key=lambda x: x.__name__ - ) + return sorted(_list_classes(), key=lambda x: x.__name__) # -------------------------------------------------------------------- @@ -255,7 +250,7 @@ def _list_classes(): for module in _module_iter: yield from add_module(module) - globals()["_list_classes"] = lambda : _cache + globals()["_list_classes"] = lambda: _cache def _modules_internal(): @@ -267,11 +262,7 @@ def _modules_internal(): def _modules_path(path, files): sys.path.insert(0, path) try: - return [ - __import__(name[:-3]) - for name in files - if name.endswith(".py") - ] + return [__import__(name[:-3]) for name in files if name.endswith(".py")] finally: del sys.path[0] @@ -279,9 +270,9 @@ def _modules_path(path, files): def _get_classes(module): """Return a list of all extractor classes in a module""" return [ - cls for cls in module.__dict__.values() if ( - hasattr(cls, "pattern") and cls.__module__ == module.__name__ - ) + cls + for cls in module.__dict__.values() + if (hasattr(cls, "pattern") and cls.__module__ == module.__name__) ] diff --git a/gallery_dl/extractor/adultempire.py b/gallery_dl/extractor/adultempire.py index 1617414a73..d309b46e2f 100644 --- a/gallery_dl/extractor/adultempire.py +++ b/gallery_dl/extractor/adultempire.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,16 +6,16 @@ """Extractors for https://www.adultempire.com/""" -from .common import GalleryExtractor from .. import text +from .common import GalleryExtractor class AdultempireGalleryExtractor(GalleryExtractor): """Extractor for image galleries from www.adultempire.com""" + category = "adultempire" root = "https://www.adultempire.com" - pattern = (r"(?:https?://)?(?:www\.)?adult(?:dvd)?empire\.com" - r"(/(\d+)/gallery\.html)") + pattern = r"(?:https?://)?(?:www\.)?adult(?:dvd)?empire\.com" r"(/(\d+)/gallery\.html)" example = "https://www.adultempire.com/12345/gallery.html" def __init__(self, match): @@ -28,12 +26,12 @@ def metadata(self, page): extr = text.extract_from(page, page.index('
')) return { "gallery_id": text.parse_int(self.gallery_id), - "title" : text.unescape(extr('title="', '"')), - "studio" : extr(">studio", "<").strip(), - "date" : text.parse_datetime(extr( - ">released", "<").strip(), "%m/%d/%Y"), - "actors" : sorted(text.split_html(extr( - '
    studio", "<").strip(), + "date": text.parse_datetime(extr(">released", "<").strip(), "%m/%d/%Y"), + "actors": sorted( + text.split_html(extr('
      "))[1:] + ), } def images(self, page): diff --git a/gallery_dl/extractor/agnph.py b/gallery_dl/extractor/agnph.py index 653b73f1b0..29b526892f 100644 --- a/gallery_dl/extractor/agnph.py +++ b/gallery_dl/extractor/agnph.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,12 @@ """Extractors for https://agn.ph/""" -from . import booru -from .. import text - -from xml.etree import ElementTree import collections import re +from xml.etree import ElementTree as ET + +from .. import text +from . import booru BASE_PATTERN = r"(?:https?://)?agn\.ph" @@ -38,7 +36,7 @@ def _init(self): def _prepare(self, post): post["date"] = text.parse_timestamp(post["created_at"]) post["status"] = post["status"].strip() - post["has_children"] = ("true" in post["has_children"]) + post["has_children"] = "true" in post["has_children"] def _xml_to_dict(self, xml): return {element.tag: element.text for element in xml} @@ -46,14 +44,13 @@ def _xml_to_dict(self, xml): def _pagination(self, url, params): params["api"] = "xml" if "page" in params: - params["page"] = \ - self.page_start + text.parse_int(params["page"]) - 1 + params["page"] = self.page_start + text.parse_int(params["page"]) - 1 else: params["page"] = self.page_start while True: data = self.request(url, params=params).text - root = ElementTree.fromstring(data) + root = ET.fromstring(data) yield from map(self._xml_to_dict, root) @@ -68,8 +65,7 @@ def _html(self, post): return self.request(url).text def _tags(self, post, page): - tag_container = text.extr( - page, '
        ', '

        Statistics

        ') + tag_container = text.extr(page, '
          ', "

          Statistics

          ") if not tag_container: return @@ -107,7 +103,6 @@ class AgnphPostExtractor(AgnphExtractor): example = "https://agn.ph/gallery/post/show/12345/" def posts(self): - url = "{}/gallery/post/show/{}/?api=xml".format( - self.root, self.groups[0]) - post = ElementTree.fromstring(self.request(url).text) + url = f"{self.root}/gallery/post/show/{self.groups[0]}/?api=xml" + post = ET.fromstring(self.request(url).text) return (self._xml_to_dict(post),) diff --git a/gallery_dl/extractor/ao3.py b/gallery_dl/extractor/ao3.py index d3ab846888..1445839c2c 100644 --- a/gallery_dl/extractor/ao3.py +++ b/gallery_dl/extractor/ao3.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,16 +6,19 @@ """Extractors for https://archiveofourown.org/""" -from .common import Extractor, Message -from .. import text, util, exception +from .. import exception +from .. import text +from .. import util from ..cache import cache +from .common import Extractor +from .common import Message -BASE_PATTERN = (r"(?:https?://)?(?:www\.)?" - r"a(?:rchiveofourown|o3)\.(?:org|com|net)") +BASE_PATTERN = r"(?:https?://)?(?:www\.)?" r"a(?:rchiveofourown|o3)\.(?:org|com|net)" class Ao3Extractor(Extractor): """Base class for ao3 extractors""" + category = "ao3" root = "https://archiveofourown.org" categorytransfer = True @@ -59,13 +60,13 @@ def works(self): def login(self): if self.cookies_check(self.cookies_names): - return + return None username, password = self._get_auth_info() if username: return self.cookies_update(self._login_impl(username, password)) - @cache(maxage=90*86400, keyarg=1) + @cache(maxage=90 * 86400, keyarg=1) def _login_impl(self, username, password): self.log.info("Logging in as %s", username) @@ -73,17 +74,16 @@ def _login_impl(self, username, password): page = self.request(url).text pos = page.find('id="loginform"') - token = text.extract( - page, ' name="authenticity_token" value="', '"', pos)[0] + token = text.extract(page, ' name="authenticity_token" value="', '"', pos)[0] if not token: self.log.error("Unable to extract 'authenticity_token'") data = { "authenticity_token": text.unescape(token), - "user[login]" : username, - "user[password]" : password, - "user[remember_me]" : "1", - "commit" : "Log In", + "user[login]": username, + "user[password]": password, + "user[remember_me]": "1", + "commit": "Log In", } response = self.request(url, method="POST", data=data) @@ -96,7 +96,7 @@ def _login_impl(self, username, password): return { "remember_user_token": remember, - "user_credentials" : "1", + "user_credentials": "1", } def _pagination(self, path, needle='
        • Adult Content WarningAdult Content Warning', "")), - "warnings" : text.split_html( - extr('
          ', "
          ")), - "categories" : text.split_html( - extr('
          ', "
          ")), - "fandom" : text.split_html( - extr('
          ', "
          ")), - "relationships": text.split_html( - extr('
          ', "
          ")), - "characters" : text.split_html( - extr('
          ', "
          ")), - "tags" : text.split_html( - extr('
          ', "
          ")), - "lang" : extr('
          ', "
          "), - "date" : text.parse_datetime( - extr('
          ', "<"), "%Y-%m-%d"), + "id": text.parse_int(work_id), + "rating": text.split_html(extr('
          ', "
          ")), + "warnings": text.split_html(extr('
          ', "
          ")), + "categories": text.split_html(extr('
          ', "
          ")), + "fandom": text.split_html(extr('
          ', "
          ")), + "relationships": text.split_html(extr('
          ', "
          ")), + "characters": text.split_html(extr('
          ', "
          ")), + "tags": text.split_html(extr('
          ', "
          ")), + "lang": extr('
          ', "
          "), + "date": text.parse_datetime(extr('
          ', "<"), "%Y-%m-%d"), "date_completed": text.parse_datetime( - extr('>Completed:
          ', "<"), "%Y-%m-%d"), - "date_updated" : text.parse_timestamp( - path.rpartition("updated_at=")[2]), - "words" : text.parse_int( - extr('
          ', "<").replace(",", "")), - "chapters" : chapters, - "comments" : text.parse_int( - extr('
          ', "<").replace(",", "")), - "likes" : text.parse_int( - extr('
          ', "<").replace(",", "")), - "bookmarks" : text.parse_int(text.remove_html( - extr('
          ', "
          ")).replace(",", "")), - "views" : text.parse_int( - extr('
          ', "<").replace(",", "")), - "title" : text.unescape(text.remove_html( - extr(' class="title heading">', "")).strip()), - "author" : text.unescape(text.remove_html( - extr(' class="byline heading">', ""))), - "summary" : text.split_html( - extr(' class="heading">Summary:', "
")), + extr('>Completed:
', "<"), "%Y-%m-%d" + ), + "date_updated": text.parse_timestamp(path.rpartition("updated_at=")[2]), + "words": text.parse_int(extr('
', "<").replace(",", "")), + "chapters": chapters, + "comments": text.parse_int(extr('
', "<").replace(",", "")), + "likes": text.parse_int(extr('
', "<").replace(",", "")), + "bookmarks": text.parse_int( + text.remove_html(extr('
', "
")).replace(",", "") + ), + "views": text.parse_int(extr('
', "<").replace(",", "")), + "title": text.unescape( + text.remove_html(extr(' class="title heading">', "")).strip() + ), + "author": text.unescape(text.remove_html(extr(' class="byline heading">', ""))), + "summary": text.split_html(extr(' class="heading">Summary:', "")), } data["language"] = util.code_to_language(data["lang"]) @@ -209,11 +196,11 @@ def items(self): if series: extr = text.extract_from(series) data["series"] = { - "prev" : extr(' class="previous" href="/works/', '"'), + "prev": extr(' class="previous" href="/works/', '"'), "index": extr(' class="position">Part ', " "), - "id" : extr(' href="/series/', '"'), - "name" : text.unescape(extr(">", "<")), - "next" : extr(' class="next" href="/works/', '"'), + "id": extr(' href="/series/', '"'), + "name": text.unescape(extr(">", "<")), + "next": extr(' class="next" href="/works/', '"'), } else: data["series"] = None @@ -230,6 +217,7 @@ def items(self): class Ao3SeriesExtractor(Ao3Extractor): """Extractor for AO3 works of a series""" + subcategory = "series" pattern = BASE_PATTERN + r"(/series/(\d+))" example = "https://archiveofourown.org/series/12345" @@ -237,6 +225,7 @@ class Ao3SeriesExtractor(Ao3Extractor): class Ao3TagExtractor(Ao3Extractor): """Extractor for AO3 works by tag""" + subcategory = "tag" pattern = BASE_PATTERN + r"(/tags/([^/?#]+)/works(?:/?\?.+)?)" example = "https://archiveofourown.org/tags/TAG/works" @@ -244,6 +233,7 @@ class Ao3TagExtractor(Ao3Extractor): class Ao3SearchExtractor(Ao3Extractor): """Extractor for AO3 search results""" + subcategory = "search" pattern = BASE_PATTERN + r"(/works/search/?\?.+)" example = "https://archiveofourown.org/works/search?work_search[query]=air" @@ -251,36 +241,39 @@ class Ao3SearchExtractor(Ao3Extractor): class Ao3UserExtractor(Ao3Extractor): """Extractor for an AO3 user profile""" + subcategory = "user" - pattern = (BASE_PATTERN + r"/users/([^/?#]+(?:/pseuds/[^/?#]+)?)" - r"(?:/profile)?/?(?:$|\?|#)") + pattern = BASE_PATTERN + r"/users/([^/?#]+(?:/pseuds/[^/?#]+)?)" r"(?:/profile)?/?(?:$|\?|#)" example = "https://archiveofourown.org/users/USER" def initialize(self): pass def items(self): - base = "{}/users/{}/".format(self.root, self.groups[0]) - return self._dispatch_extractors(( - (Ao3UserWorksExtractor , base + "works"), - (Ao3UserSeriesExtractor , base + "series"), - (Ao3UserBookmarkExtractor, base + "bookmarks"), - ), ("user-works", "user-series")) + base = f"{self.root}/users/{self.groups[0]}/" + return self._dispatch_extractors( + ( + (Ao3UserWorksExtractor, base + "works"), + (Ao3UserSeriesExtractor, base + "series"), + (Ao3UserBookmarkExtractor, base + "bookmarks"), + ), + ("user-works", "user-series"), + ) class Ao3UserWorksExtractor(Ao3Extractor): """Extractor for works of an AO3 user""" + subcategory = "user-works" - pattern = (BASE_PATTERN + r"(/users/([^/?#]+)/(?:pseuds/([^/?#]+)/)?" - r"works(?:/?\?.+)?)") + pattern = BASE_PATTERN + r"(/users/([^/?#]+)/(?:pseuds/([^/?#]+)/)?" r"works(?:/?\?.+)?)" example = "https://archiveofourown.org/users/USER/works" class Ao3UserSeriesExtractor(Ao3Extractor): """Extractor for series of an AO3 user""" + subcategory = "user-series" - pattern = (BASE_PATTERN + r"(/users/([^/?#]+)/(?:pseuds/([^/?#]+)/)?" - r"series(?:/?\?.+)?)") + pattern = BASE_PATTERN + r"(/users/([^/?#]+)/(?:pseuds/([^/?#]+)/)?" r"series(?:/?\?.+)?)" example = "https://archiveofourown.org/users/USER/series" def items(self): @@ -298,9 +291,9 @@ def series(self): class Ao3UserBookmarkExtractor(Ao3Extractor): """Extractor for bookmarked works of an AO3 user""" + subcategory = "user-bookmark" - pattern = (BASE_PATTERN + r"(/users/([^/?#]+)/(?:pseuds/([^/?#]+)/)?" - r"bookmarks(?:/?\?.+)?)") + pattern = BASE_PATTERN + r"(/users/([^/?#]+)/(?:pseuds/([^/?#]+)/)?" r"bookmarks(?:/?\?.+)?)" example = "https://archiveofourown.org/users/USER/bookmarks" def items(self): @@ -309,6 +302,7 @@ def items(self): class Ao3SubscriptionsExtractor(Ao3Extractor): """Extractor for your AO3 account's subscriptions""" + subcategory = "subscriptions" pattern = BASE_PATTERN + r"(/users/([^/?#]+)/subscriptions(?:/?\?.+)?)" example = "https://archiveofourown.org/users/USER/subscriptions" diff --git a/gallery_dl/extractor/architizer.py b/gallery_dl/extractor/architizer.py index 8064e7898f..720f4b0f0c 100644 --- a/gallery_dl/extractor/architizer.py +++ b/gallery_dl/extractor/architizer.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,15 @@ """Extractors for https://architizer.com/""" -from .common import GalleryExtractor, Extractor, Message from .. import text +from .common import Extractor +from .common import GalleryExtractor +from .common import Message class ArchitizerProjectExtractor(GalleryExtractor): """Extractor for project pages on architizer.com""" + category = "architizer" subcategory = "project" root = "https://architizer.com" @@ -24,7 +25,7 @@ class ArchitizerProjectExtractor(GalleryExtractor): example = "https://architizer.com/projects/NAME/" def __init__(self, match): - url = "{}/projects/{}/".format(self.root, match.group(1)) + url = f"{self.root}/projects/{match.group(1)}/" GalleryExtractor.__init__(self, match, url) def metadata(self, page): @@ -32,34 +33,30 @@ def metadata(self, page): extr('id="Pages"', "") return { - "title" : extr('data-name="', '"'), - "slug" : extr('data-slug="', '"'), - "gid" : extr('data-gid="', '"').rpartition(".")[2], - "firm" : extr('data-firm-leaders-str="', '"'), - "location" : extr("

", "<").strip(), - "type" : text.unescape(text.remove_html(extr( - '
Type
', 'STATUS', 'YEAR', 'SIZE', '', '') - .replace("
", "\n")), + "title": extr('data-name="', '"'), + "slug": extr('data-slug="', '"'), + "gid": extr('data-gid="', '"').rpartition(".")[2], + "firm": extr('data-firm-leaders-str="', '"'), + "location": extr("

", "<").strip(), + "type": text.unescape(text.remove_html(extr('
Type
', "STATUS', "YEAR', "SIZE', "', "").replace("
", "\n") + ), } def images(self, page): return [ (url, None) - for url in text.extract_iter( - page, 'property="og:image:secure_url" content="', "?") + for url in text.extract_iter(page, 'property="og:image:secure_url" content="', "?") ] class ArchitizerFirmExtractor(Extractor): """Extractor for all projects of a firm""" + category = "architizer" subcategory = "firm" root = "https://architizer.com" @@ -71,12 +68,11 @@ def __init__(self, match): self.firm = match.group(1) def items(self): - url = url = "{}/firms/{}/?requesting_merlin=pages".format( - self.root, self.firm) + url = url = f"{self.root}/firms/{self.firm}/?requesting_merlin=pages" page = self.request(url).text data = {"_extractor": ArchitizerProjectExtractor} for project in text.extract_iter(page, '= 400: self.log.warning( "Unable to fetch post %s ('%s %s')", - post_id, response.status_code, response.reason) + post_id, + response.status_code, + response.reason, + ) return None headers = response.headers @@ -141,40 +146,38 @@ def _parse_post(self, post_id): # fix 'Last-Modified' header lmod = headers["last-modified"] if lmod[22] != ":": - lmod = "{}:{} GMT".format(lmod[:22], lmod[22:24]) + lmod = f"{lmod[:22]}:{lmod[22:24]} GMT" - post_url = "{}/g4/view/{}".format(self.root, post_id) + post_url = f"{self.root}/g4/view/{post_id}" extr = text.extract_from(self.request(post_url).text) - title, _, artist = text.unescape(extr( - "g4 :: ", "<")).rpartition(" by ") + title, _, artist = text.unescape(extr("<title>g4 :: ", "<")).rpartition(" by ") return { - "id" : text.parse_int(post_id), - "url" : url, - "user" : self.user or artist, - "title" : title, + "id": text.parse_int(post_id), + "url": url, + "user": self.user or artist, + "title": title, "artist": artist, - "path" : text.split_html(extr( - "cookiecrumb'>", '</span'))[4:-1:2], - "date" : datetime(*parsedate_tz(lmod)[:6]), - "size" : text.parse_int(clen), - "views" : text.parse_int(extr("Views</b>:", "<").replace(",", "")), - "width" : text.parse_int(extr("Resolution</b>:", "x")), + "path": text.split_html(extr("cookiecrumb'>", "</span"))[4:-1:2], + "date": datetime(*parsedate_tz(lmod)[:6]), + "size": text.parse_int(clen), + "views": text.parse_int(extr("Views</b>:", "<").replace(",", "")), + "width": text.parse_int(extr("Resolution</b>:", "x")), "height": text.parse_int(extr("", "<")), - "comments" : text.parse_int(extr("Comments</b>:", "<")), + "comments": text.parse_int(extr("Comments</b>:", "<")), "favorites": text.parse_int(extr("Favorites</b>:", "<")), - "tags" : text.split_html(extr("class='taglist'>", "</span>")), - "description": text.unescape(text.remove_html(extr( - "<p>", "</p>"), "", "")), - "filename" : fname, + "tags": text.split_html(extr("class='taglist'>", "</span>")), + "description": text.unescape(text.remove_html(extr("<p>", "</p>"), "", "")), + "filename": fname, "extension": ext, - "_mtime" : lmod, + "_mtime": lmod, } class AryionGalleryExtractor(AryionExtractor): """Extractor for a user's gallery on eka's portal""" + subcategory = "gallery" categorytransfer = True pattern = BASE_PATTERN + r"/(?:gallery/|user/|latest.php\?name=)([^/?#]+)" @@ -195,15 +198,15 @@ def skip(self, num): def posts(self): if self.recursive: - url = "{}/g4/gallery/{}".format(self.root, self.user) + url = f"{self.root}/g4/gallery/{self.user}" return self._pagination_params(url) - else: - url = "{}/g4/latest.php?name={}".format(self.root, self.user) - return util.advance(self._pagination_next(url), self.offset) + url = f"{self.root}/g4/latest.php?name={self.user}" + return util.advance(self._pagination_next(url), self.offset) class AryionFavoriteExtractor(AryionExtractor): """Extractor for a user's favorites gallery""" + subcategory = "favorite" directory_fmt = ("{category}", "{user!l}", "favorites") archive_fmt = "f_{user}_{id}" @@ -212,13 +215,13 @@ class AryionFavoriteExtractor(AryionExtractor): example = "https://aryion.com/g4/favorites/USER" def posts(self): - url = "{}/g4/favorites/{}".format(self.root, self.user) - return self._pagination_params( - url, None, "class='gallery-item favorite' id='") + url = f"{self.root}/g4/favorites/{self.user}" + return self._pagination_params(url, None, "class='gallery-item favorite' id='") class AryionTagExtractor(AryionExtractor): """Extractor for tag searches on eka's portal""" + subcategory = "tag" directory_fmt = ("{category}", "tags", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" @@ -239,6 +242,7 @@ def posts(self): class AryionPostExtractor(AryionExtractor): """Extractor for individual posts on eka's portal""" + subcategory = "post" pattern = BASE_PATTERN + r"/view/(\d+)" example = "https://aryion.com/g4/view/12345" diff --git a/gallery_dl/extractor/batoto.py b/gallery_dl/extractor/batoto.py index 786acd9fec..00dea83718 100644 --- a/gallery_dl/extractor/batoto.py +++ b/gallery_dl/extractor/batoto.py @@ -1,24 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://bato.to/""" -from .common import Extractor, ChapterExtractor, MangaExtractor -from .. import text, exception import re -BASE_PATTERN = (r"(?:https?://)?(?:" - r"(?:ba|d|h|m|w)to\.to|" - r"(?:(?:manga|read)toto|batocomic|[xz]bato)\.(?:com|net|org)|" - r"comiko\.(?:net|org)|" - r"bat(?:otoo|o?two)\.com)") +from .. import exception +from .. import text +from .common import ChapterExtractor +from .common import Extractor +from .common import MangaExtractor + +BASE_PATTERN = ( + r"(?:https?://)?(?:" + r"(?:ba|d|h|m|w)to\.to|" + r"(?:(?:manga|read)toto|batocomic|[xz]bato)\.(?:com|net|org)|" + r"comiko\.(?:net|org)|" + r"bat(?:otoo|o?two)\.com)" +) -class BatotoBase(): +class BatotoBase: """Base class for batoto extractors""" + category = "batoto" root = "https://bato.to" @@ -29,13 +34,14 @@ def request(self, url, **kwargs): class BatotoChapterExtractor(BatotoBase, ChapterExtractor): """Extractor for bato.to manga chapters""" + pattern = BASE_PATTERN + r"/(?:title/[^/?#]+|chapter)/(\d+)" example = "https://bato.to/title/12345-MANGA/54321" def __init__(self, match): self.root = text.root_from_url(match.group(0)) self.chapter_id = match.group(1) - url = "{}/title/0/{}".format(self.root, self.chapter_id) + url = f"{self.root}/title/0/{self.chapter_id}" ChapterExtractor.__init__(self, match, url) def metadata(self, page): @@ -45,8 +51,7 @@ def metadata(self, page): except ValueError: manga = info = None - manga_id = text.extr( - extr('rel="canonical" href="', '"'), "/title/", "/") + manga_id = text.extr(extr('rel="canonical" href="', '"'), "/title/", "/") if not manga: manga = extr('link-hover">', "<") @@ -54,8 +59,9 @@ def metadata(self, page): info = text.unescape(info) match = re.match( - r"(?i)(?:(?:Volume|S(?:eason)?)\s*(\d+)\s+)?" - r"(?:Chapter|Episode)\s*(\d+)([\w.]*)", info) + r"(?i)(?:(?:Volume|S(?:eason)?)\s*(\d+)\s+)?" r"(?:Chapter|Episode)\s*(\d+)([\w.]*)", + info, + ) if match: volume, chapter, minor = match.groups() else: @@ -63,40 +69,38 @@ def metadata(self, page): minor = "" return { - "manga" : text.unescape(manga), - "manga_id" : text.parse_int(manga_id), - "chapter_url" : extr(self.chapter_id + "-ch_", '"'), - "title" : text.unescape(text.remove_html(extr( - "selected>", "</option")).partition(" : ")[2]), - "volume" : text.parse_int(volume), - "chapter" : text.parse_int(chapter), - "chapter_minor" : minor, + "manga": text.unescape(manga), + "manga_id": text.parse_int(manga_id), + "chapter_url": extr(self.chapter_id + "-ch_", '"'), + "title": text.unescape( + text.remove_html(extr("selected>", "</option")).partition(" : ")[2] + ), + "volume": text.parse_int(volume), + "chapter": text.parse_int(chapter), + "chapter_minor": minor, "chapter_string": info, - "chapter_id" : text.parse_int(self.chapter_id), - "date" : text.parse_timestamp(extr(' time="', '"')[:-3]), + "chapter_id": text.parse_int(self.chapter_id), + "date": text.parse_timestamp(extr(' time="', '"')[:-3]), } def images(self, page): - images_container = text.extr(page, 'pageOpts', ':[0,0]}"') + images_container = text.extr(page, "pageOpts", ':[0,0]}"') images_container = text.unescape(images_container) - return [ - (url, None) - for url in text.extract_iter(images_container, r"\"", r"\"") - ] + return [(url, None) for url in text.extract_iter(images_container, r"\"", r"\"")] class BatotoMangaExtractor(BatotoBase, MangaExtractor): """Extractor for bato.to manga""" + reverse = False chapterclass = BatotoChapterExtractor - pattern = (BASE_PATTERN + - r"/(?:title/(\d+)[^/?#]*|series/(\d+)(?:/[^/?#]*)?)/?$") + pattern = BASE_PATTERN + r"/(?:title/(\d+)[^/?#]*|series/(\d+)(?:/[^/?#]*)?)/?$" example = "https://bato.to/title/12345-MANGA/" def __init__(self, match): self.root = text.root_from_url(match.group(0)) self.manga_id = match.group(1) or match.group(2) - url = "{}/title/{}".format(self.root, self.manga_id) + url = f"{self.root}/title/{self.manga_id}" MangaExtractor.__init__(self, match, url) def chapters(self, page): @@ -108,8 +112,7 @@ def chapters(self, page): data = { "manga_id": text.parse_int(self.manga_id), - "manga" : text.unescape(extr( - "<title>", "<").rpartition(" - ")[0]), + "manga": text.unescape(extr("<title>", "<").rpartition(" - ")[0]), } extr('<div data-hk="0-0-0-0"', "") @@ -124,9 +127,8 @@ def chapters(self, page): data["chapter"] = text.parse_int(chapter) data["chapter_minor"] = sep + minor - data["date"] = text.parse_datetime( - extr('time="', '"'), "%Y-%m-%dT%H:%M:%S.%fZ") + data["date"] = text.parse_datetime(extr('time="', '"'), "%Y-%m-%dT%H:%M:%S.%fZ") - url = "{}/title/{}".format(self.root, href) + url = f"{self.root}/title/{href}" results.append((url, data.copy())) return results diff --git a/gallery_dl/extractor/bbc.py b/gallery_dl/extractor/bbc.py index 54aaac4e30..07d9a695d6 100644 --- a/gallery_dl/extractor/bbc.py +++ b/gallery_dl/extractor/bbc.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,42 +6,42 @@ """Extractors for https://bbc.co.uk/""" -from .common import GalleryExtractor, Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import GalleryExtractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?bbc\.co\.uk(/programmes/" class BbcGalleryExtractor(GalleryExtractor): """Extractor for a programme gallery on bbc.co.uk""" + category = "bbc" root = "https://www.bbc.co.uk" - directory_fmt = ("{category}", "{path[0]}", "{path[1]}", "{path[2]}", - "{path[3:]:J - /}") + directory_fmt = ("{category}", "{path[0]}", "{path[1]}", "{path[2]}", "{path[3:]:J - /}") filename_fmt = "{num:>02}.{extension}" archive_fmt = "{programme}_{num}" pattern = BASE_PATTERN + r"[^/?#]+(?!/galleries)(?:/[^/?#]+)?)$" example = "https://www.bbc.co.uk/programmes/PATH" def metadata(self, page): - data = util.json_loads(text.extr( - page, '<script type="application/ld+json">', '</script>')) + data = util.json_loads(text.extr(page, '<script type="application/ld+json">', "</script>")) return { "programme": self.gallery_url.split("/")[4], - "path": list(util.unique_sequence( - element["name"] - for element in data["itemListElement"] - )), + "path": list( + util.unique_sequence(element["name"] for element in data["itemListElement"]) + ), } def images(self, page): width = self.config("width") width = width - width % 16 if width else 1920 - dimensions = "/{}xn/".format(width) + dimensions = f"/{width}xn/" return [ - (src.replace("/320x180_b/", dimensions), - {"_fallback": self._fallback_urls(src, width)}) + (src.replace("/320x180_b/", dimensions), {"_fallback": self._fallback_urls(src, width)}) for src in text.extract_iter(page, 'data-image-src="', '"') ] @@ -52,11 +50,12 @@ def _fallback_urls(src, max_width): front, _, back = src.partition("/320x180_b/") for width in (1920, 1600, 1280, 976): if width < max_width: - yield "{}/{}xn/{}".format(front, width, back) + yield f"{front}/{width}xn/{back}" class BbcProgrammeExtractor(Extractor): """Extractor for all galleries of a bbc programme""" + category = "bbc" subcategory = "programme" root = "https://www.bbc.co.uk" @@ -75,7 +74,8 @@ def items(self): while True: page = self.request(galleries_url, params=params).text for programme_id in text.extract_iter( - page, '<a href="https://www.bbc.co.uk/programmes/', '"'): + page, '<a href="https://www.bbc.co.uk/programmes/', '"' + ): url = "https://www.bbc.co.uk/programmes/" + programme_id yield Message.Queue, url, data if 'rel="next"' not in page: diff --git a/gallery_dl/extractor/behance.py b/gallery_dl/extractor/behance.py index 14598b7a09..3384e1a32e 100644 --- a/gallery_dl/extractor/behance.py +++ b/gallery_dl/extractor/behance.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,16 @@ """Extractors for https://www.behance.net/""" -from .common import Extractor, Message -from .. import text, util, exception +from .. import exception +from .. import text +from .. import util +from .common import Extractor +from .common import Message class BehanceExtractor(Extractor): """Base class for behance extractors""" + category = "behance" root = "https://www.behance.net" request_interval = (2.0, 4.0) @@ -36,28 +38,23 @@ def _request_graphql(self, endpoint, variables): url = self.root + "/v3/graphql" headers = { "Origin": self.root, - "X-BCP" : self._bcp, + "X-BCP": self._bcp, "X-Requested-With": "XMLHttpRequest", } data = { - "query" : GRAPHQL_QUERIES[endpoint], + "query": GRAPHQL_QUERIES[endpoint], "variables": variables, } - return self.request(url, method="POST", headers=headers, - json=data).json()["data"] + return self.request(url, method="POST", headers=headers, json=data).json()["data"] def _update(self, data): # compress data to simple lists if data.get("fields") and isinstance(data["fields"][0], dict): - data["fields"] = [ - field.get("name") or field.get("label") - for field in data["fields"] - ] + data["fields"] = [field.get("name") or field.get("label") for field in data["fields"]] data["owners"] = [ - owner.get("display_name") or owner.get("displayName") - for owner in data["owners"] + owner.get("display_name") or owner.get("displayName") for owner in data["owners"] ] tags = data.get("tags") or () @@ -66,7 +63,8 @@ def _update(self, data): data["tags"] = tags data["date"] = text.parse_timestamp( - data.get("publishedOn") or data.get("conceived_on") or 0) + data.get("publishedOn") or data.get("conceived_on") or 0 + ) # backwards compatibility data["gallery_id"] = data["id"] @@ -78,6 +76,7 @@ def _update(self, data): class BehanceGalleryExtractor(BehanceExtractor): """Extractor for image galleries from www.behance.net""" + subcategory = "gallery" directory_fmt = ("{category}", "{owners:J, }", "{id} {name}") filename_fmt = "{category}_{id}_{num:>02}.{extension}" @@ -108,23 +107,21 @@ def items(self): yield Message.Directory, data for data["num"], (url, module) in enumerate(imgs, 1): data["module"] = module - data["extension"] = (module.get("extension") or - text.ext_from_url(url)) + data["extension"] = module.get("extension") or text.ext_from_url(url) yield Message.Url, url, data def get_gallery_data(self): """Collect gallery info dict""" - url = "{}/gallery/{}/a".format(self.root, self.gallery_id) + url = f"{self.root}/gallery/{self.gallery_id}/a" cookies = { "gki": '{"feature_project_view":false,' - '"feature_discover_login_prompt":false,' - '"feature_project_login_prompt":false}', + '"feature_discover_login_prompt":false,' + '"feature_project_login_prompt":false}', "ilo0": "true", } page = self.request(url, cookies=cookies).text - data = util.json_loads(text.extr( - page, 'id="beconfig-store_state">', '</script>')) + data = util.json_loads(text.extr(page, 'id="beconfig-store_state">', "</script>")) return self._update(data["project"]["project"]) def get_images(self, data): @@ -133,10 +130,10 @@ def get_images(self, data): access = data.get("matureAccess") if access == "logged-out": raise exception.AuthorizationError( - "Mature content galleries require logged-in cookies") + "Mature content galleries require logged-in cookies" + ) if access == "restricted-safe": - raise exception.AuthorizationError( - "Mature content blocked in account settings") + raise exception.AuthorizationError("Mature content blocked in account settings") if access and access != "allowed": raise exception.AuthorizationError() return () @@ -156,11 +153,13 @@ def get_images(self, data): size["url"].rsplit("/", 2)[1]: size for size in module["imageSizes"]["allAvailable"] } - size = (sizes.get("source") or - sizes.get("max_3840") or - sizes.get("fs") or - sizes.get("hd") or - sizes.get("disp")) + size = ( + sizes.get("source") + or sizes.get("max_3840") + or sizes.get("fs") + or sizes.get("hd") + or sizes.get("disp") + ) append((size["url"], module)) elif mtype == "video": @@ -181,15 +180,13 @@ def get_images(self, data): try: renditions = module["videoData"]["renditions"] except Exception: - self.log.warning("No download URLs for video %s", - module.get("id") or "???") + self.log.warning("No download URLs for video %s", module.get("id") or "???") continue try: - url = [ - r["url"] for r in renditions - if text.ext_from_url(r["url"]) != "m3u8" - ][-1] + url = [r["url"] for r in renditions if text.ext_from_url(r["url"]) != "m3u8"][ + -1 + ] except Exception as exc: self.log.debug("%s: %s", exc.__class__.__name__, exc) url = "ytdl:" + renditions[-1]["url"] @@ -221,6 +218,7 @@ def get_images(self, data): class BehanceUserExtractor(BehanceExtractor): """Extractor for a user's galleries from www.behance.net""" + subcategory = "user" categorytransfer = True pattern = r"(?:https?://)?(?:www\.)?behance\.net/([^/?#]+)/?$" @@ -234,7 +232,7 @@ def galleries(self): endpoint = "GetProfileProjects" variables = { "username": self.user, - "after" : "MAo=", # "0" in base64 + "after": "MAo=", # "0" in base64 } while True: @@ -249,6 +247,7 @@ def galleries(self): class BehanceCollectionExtractor(BehanceExtractor): """Extractor for a collection's galleries from www.behance.net""" + subcategory = "collection" categorytransfer = True pattern = r"(?:https?://)?(?:www\.)?behance\.net/collection/(\d+)" @@ -263,8 +262,8 @@ def galleries(self): variables = { "afterItem": "MAo=", # "0" in base64 "firstItem": 40, - "id" : int(self.collection_id), - "shouldGetItems" : True, + "id": int(self.collection_id), + "shouldGetItems": True, "shouldGetMoodboardFields": False, "shouldGetRecommendations": False, } @@ -437,7 +436,6 @@ def galleries(self): } } """, - "GetMoodboardItemsAndRecommendations": """\ query GetMoodboardItemsAndRecommendations( $id: Int! @@ -689,5 +687,4 @@ def galleries(self): } } """, - } diff --git a/gallery_dl/extractor/bilibili.py b/gallery_dl/extractor/bilibili.py index d5c419eb5b..3594b2e41a 100644 --- a/gallery_dl/extractor/bilibili.py +++ b/gallery_dl/extractor/bilibili.py @@ -1,17 +1,21 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://www.bilibili.com/""" -from .common import Extractor, Message -from .. import text, util, exception +from contextlib import suppress + +from .. import exception +from .. import text +from .. import util +from .common import Extractor +from .common import Message class BilibiliExtractor(Extractor): """Base class for bilibili extractors""" + category = "bilibili" root = "https://www.bilibili.com" request_interval = (3.0, 6.0) @@ -22,6 +26,7 @@ def _init(self): class BilibiliUserArticlesExtractor(BilibiliExtractor): """Extractor for a bilibili user's articles""" + subcategory = "user-articles" pattern = r"(?:https?://)?space\.bilibili\.com/(\d+)/article" example = "https://space.bilibili.com/12345/article" @@ -35,9 +40,9 @@ def items(self): class BilibiliArticleExtractor(BilibiliExtractor): """Extractor for a bilibili article""" + subcategory = "article" - pattern = (r"(?:https?://)?" - r"(?:t\.bilibili\.com|(?:www\.)?bilibili.com/opus)/(\d+)") + pattern = r"(?:https?://)?" r"(?:t\.bilibili\.com|(?:www\.)?bilibili.com/opus)/(\d+)" example = "https://www.bilibili.com/opus/12345" directory_fmt = ("{category}", "{username}") filename_fmt = "{id}_{num}.{extension}" @@ -49,21 +54,19 @@ def items(self): # Flatten modules list modules = {} for module in article["detail"]["modules"]: - del module['module_type'] + del module["module_type"] modules.update(module) article["detail"]["modules"] = modules article["username"] = modules["module_author"]["name"] pics = [] - for paragraph in modules['module_content']['paragraphs']: + for paragraph in modules["module_content"]["paragraphs"]: if "pic" not in paragraph: continue - try: + with suppress(Exception): pics.extend(paragraph["pic"]["pics"]) - except Exception: - pass article["count"] = len(pics) yield Message.Directory, article @@ -73,7 +76,7 @@ def items(self): yield Message.Url, url, text.nameext_from_url(url, article) -class BilibiliAPI(): +class BilibiliAPI: def __init__(self, extractor): self.extractor = extractor @@ -107,10 +110,10 @@ def article(self, article_id): while True: page = self.extractor.request(url).text try: - return util.json_loads(text.extr( - page, "window.__INITIAL_STATE__=", "};") + "}") + return util.json_loads(text.extr(page, "window.__INITIAL_STATE__=", "};") + "}") except Exception: if "window._riskdata_" not in page: raise exception.StopExtraction( - "%s: Unable to extract INITIAL_STATE data", article_id) + "%s: Unable to extract INITIAL_STATE data", article_id + ) self.extractor.wait(seconds=300) diff --git a/gallery_dl/extractor/blogger.py b/gallery_dl/extractor/blogger.py index 37075eaa34..980ca772aa 100644 --- a/gallery_dl/extractor/blogger.py +++ b/gallery_dl/extractor/blogger.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,16 +6,19 @@ """Extractors for Blogger blogs""" -from .common import BaseExtractor, Message -from .. import text, util import re +from .. import text +from .. import util +from .common import BaseExtractor +from .common import Message + class BloggerExtractor(BaseExtractor): """Base class for blogger extractors""" + basecategory = "blogger" - directory_fmt = ("blogger", "{blog[name]}", - "{post[date]:%Y-%m-%d} {post[title]}") + directory_fmt = ("blogger", "{blog[name]}", "{post[date]:%Y-%m-%d} {post[title]}") filename_fmt = "{num:>03}.{extension}" archive_fmt = "{post[id]}_{num}" @@ -36,11 +37,13 @@ def items(self): sub = re.compile(r"(/|=)(?:[sw]\d+|w\d+-h\d+)(?=/|$)").sub findall_image = re.compile( r'src="(https?://(?:' - r'blogger\.googleusercontent\.com/img|' - r'lh\d+(?:-\w+)?\.googleusercontent\.com|' - r'\d+\.bp\.blogspot\.com)/[^"]+)').findall + r"blogger\.googleusercontent\.com/img|" + r"lh\d+(?:-\w+)?\.googleusercontent\.com|" + r'\d+\.bp\.blogspot\.com)/[^"]+)' + ).findall findall_video = re.compile( - r'src="(https?://www\.blogger\.com/video\.g\?token=[^"]+)').findall + r'src="(https?://www\.blogger\.com/video\.g\?token=[^"]+)' + ).findall metadata = self.metadata() for post in self.posts(blog): @@ -54,12 +57,13 @@ def items(self): page = self.request(post["url"]).text for url in findall_video(page): page = self.request(url).text - video_config = util.json_loads(text.extr( - page, 'var VIDEO_CONFIG =', '\n')) - files.append(max( - video_config["streams"], - key=lambda x: x["format_id"], - )["play_url"]) + video_config = util.json_loads(text.extr(page, "var VIDEO_CONFIG =", "\n")) + files.append( + max( + video_config["streams"], + key=lambda x: x["format_id"], + )["play_url"] + ) post["author"] = post["author"]["displayName"] post["replies"] = post["replies"]["totalItems"] @@ -84,16 +88,19 @@ def metadata(self): """Return additional metadata""" -BASE_PATTERN = BloggerExtractor.update({ - "blogspot": { - "root": None, - "pattern": r"[\w-]+\.blogspot\.com", - }, -}) +BASE_PATTERN = BloggerExtractor.update( + { + "blogspot": { + "root": None, + "pattern": r"[\w-]+\.blogspot\.com", + }, + } +) class BloggerPostExtractor(BloggerExtractor): """Extractor for a single blog post""" + subcategory = "post" pattern = BASE_PATTERN + r"(/\d\d\d\d/\d\d/[^/?#]+\.html)" example = "https://BLOG.blogspot.com/1970/01/TITLE.html" @@ -108,6 +115,7 @@ def posts(self, blog): class BloggerBlogExtractor(BloggerExtractor): """Extractor for an entire Blogger blog""" + subcategory = "blog" pattern = BASE_PATTERN + r"/?$" example = "https://BLOG.blogspot.com/" @@ -118,6 +126,7 @@ def posts(self, blog): class BloggerSearchExtractor(BloggerExtractor): """Extractor for Blogger search resuls""" + subcategory = "search" pattern = BASE_PATTERN + r"/search/?\?q=([^&#]+)" example = "https://BLOG.blogspot.com/search?q=QUERY" @@ -135,6 +144,7 @@ def metadata(self): class BloggerLabelExtractor(BloggerExtractor): """Extractor for Blogger posts by label""" + subcategory = "label" pattern = BASE_PATTERN + r"/search/label/([^/?#]+)" example = "https://BLOG.blogspot.com/search/label/LABEL" @@ -150,11 +160,12 @@ def metadata(self): return {"label": self.label} -class BloggerAPI(): +class BloggerAPI: """Minimal interface for the Blogger v3 API Ref: https://developers.google.com/blogger """ + API_KEY = "AIzaSyCN9ax34oMMyM07g_M-5pjeDp_312eITK8" def __init__(self, extractor): @@ -165,24 +176,23 @@ def blog_by_url(self, url): return self._call("blogs/byurl", {"url": url}, "blog") def blog_posts(self, blog_id, label=None): - endpoint = "blogs/{}/posts".format(blog_id) + endpoint = f"blogs/{blog_id}/posts" params = {"labels": label} return self._pagination(endpoint, params) def blog_search(self, blog_id, query): - endpoint = "blogs/{}/posts/search".format(blog_id) + endpoint = f"blogs/{blog_id}/posts/search" params = {"q": query} return self._pagination(endpoint, params) def post_by_path(self, blog_id, path): - endpoint = "blogs/{}/posts/bypath".format(blog_id) + endpoint = f"blogs/{blog_id}/posts/bypath" return self._call(endpoint, {"path": path}, "post") def _call(self, endpoint, params, notfound=None): url = "https://www.googleapis.com/blogger/v3/" + endpoint params["key"] = self.api_key - return self.extractor.request( - url, params=params, notfound=notfound).json() + return self.extractor.request(url, params=params, notfound=notfound).json() def _pagination(self, endpoint, params): while True: diff --git a/gallery_dl/extractor/bluesky.py b/gallery_dl/extractor/bluesky.py index 18a504732a..2f977911d2 100644 --- a/gallery_dl/extractor/bluesky.py +++ b/gallery_dl/extractor/bluesky.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,17 +6,21 @@ """Extractors for https://bsky.app/""" -from .common import Extractor, Message -from .. import text, util, exception -from ..cache import cache, memcache +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from ..cache import memcache +from .common import Extractor +from .common import Message -BASE_PATTERN = (r"(?:https?://)?" - r"(?:(?:www\.)?(?:c|[fv]x)?bs[ky]y[ex]?\.app|main\.bsky\.dev)") +BASE_PATTERN = r"(?:https?://)?" r"(?:(?:www\.)?(?:c|[fv]x)?bs[ky]y[ex]?\.app|main\.bsky\.dev)" USER_PATTERN = BASE_PATTERN + r"/profile/([^/?#]+)" class BlueskyExtractor(Extractor): """Base class for bluesky extractors""" + category = "bluesky" directory_fmt = ("{category}", "{author[handle]}") filename_fmt = "{createdAt[:19]}_{post_id}_{num}.{extension}" @@ -36,8 +38,8 @@ def _init(self): meta = meta.replace(" ", "").split(",") elif not isinstance(meta, (list, tuple)): meta = ("user", "facets") - self._metadata_user = ("user" in meta) - self._metadata_facets = ("facets" in meta) + self._metadata_user = "user" in meta + self._metadata_facets = "facets" in meta self.api = BlueskyAPI(self) self._user = self._user_did = None @@ -62,9 +64,7 @@ def items(self): yield Message.Directory, post if files: did = post["author"]["did"] - base = ( - "{}/xrpc/com.atproto.sync.getBlob?did={}&cid=".format( - self.api.get_service_endpoint(did), did)) + base = f"{self.api.get_service_endpoint(did)}/xrpc/com.atproto.sync.getBlob?did={did}&cid=" # noqa: E501 for post["num"], file in enumerate(files, 1): post.update(file) yield Message.Url, base + file["filename"], post @@ -116,8 +116,7 @@ def _prepare(self, post): post["instance"] = self.instance post["post_id"] = self._pid(post) - post["date"] = text.parse_datetime( - post["createdAt"][:19], "%Y-%m-%dT%H:%M:%S") + post["date"] = text.parse_datetime(post["createdAt"][:19], "%Y-%m-%dT%H:%M:%S") def _extract_files(self, post): if "embed" not in post: @@ -154,10 +153,10 @@ def _extract_media(self, media, key): return { "description": media.get("alt") or "", - "width" : width, - "height" : height, - "filename" : cid, - "extension" : data["mimeType"].rpartition("/")[2], + "width": width, + "height": height, + "filename": cid, + "extension": data["mimeType"].rpartition("/")[2], } def _make_post(self, actor, kind): @@ -168,27 +167,33 @@ def _make_post(self, actor, kind): return () cid = profile[kind].rpartition("/")[2].partition("@")[0] - return ({ - "post": { - "embed": {"images": [{ - "alt": kind, - "image": { - "$type" : "blob", - "ref" : {"$link": cid}, - "mimeType": "image/jpeg", - "size" : 0, + return ( + { + "post": { + "embed": { + "images": [ + { + "alt": kind, + "image": { + "$type": "blob", + "ref": {"$link": cid}, + "mimeType": "image/jpeg", + "size": 0, + }, + "aspectRatio": { + "width": 1000, + "height": 1000, + }, + } + ] }, - "aspectRatio": { - "width" : 1000, - "height": 1000, - }, - }]}, - "author" : profile, - "record" : (), - "createdAt": "", - "uri" : cid, + "author": profile, + "record": (), + "createdAt": "", + "uri": cid, + }, }, - },) + ) class BlueskyUserExtractor(BlueskyExtractor): @@ -200,15 +205,18 @@ def initialize(self): pass def items(self): - base = "{}/profile/{}/".format(self.root, self.user) - return self._dispatch_extractors(( - (BlueskyAvatarExtractor , base + "avatar"), - (BlueskyBackgroundExtractor, base + "banner"), - (BlueskyPostsExtractor , base + "posts"), - (BlueskyRepliesExtractor , base + "replies"), - (BlueskyMediaExtractor , base + "media"), - (BlueskyLikesExtractor , base + "likes"), - ), ("media",)) + base = f"{self.root}/profile/{self.user}/" + return self._dispatch_extractors( + ( + (BlueskyAvatarExtractor, base + "avatar"), + (BlueskyBackgroundExtractor, base + "banner"), + (BlueskyPostsExtractor, base + "posts"), + (BlueskyRepliesExtractor, base + "replies"), + (BlueskyMediaExtractor, base + "media"), + (BlueskyLikesExtractor, base + "likes"), + ), + ("media",), + ) class BlueskyPostsExtractor(BlueskyExtractor): @@ -333,10 +341,10 @@ class BlueskyHashtagExtractor(BlueskyExtractor): example = "https://bsky.app/hashtag/NAME" def posts(self): - return self.api.search_posts("#"+self.user, self.groups[1]) + return self.api.search_posts("#" + self.user, self.groups[1]) -class BlueskyAPI(): +class BlueskyAPI: """Interface for the Bluesky API https://www.docs.bsky.app/docs/category/http-reference @@ -365,17 +373,16 @@ def get_actor_likes(self, actor): def get_author_feed(self, actor, filter="posts_and_author_threads"): endpoint = "app.bsky.feed.getAuthorFeed" params = { - "actor" : self._did_from_actor(actor, True), + "actor": self._did_from_actor(actor, True), "filter": filter, - "limit" : "100", + "limit": "100", } return self._pagination(endpoint, params) def get_feed(self, actor, feed): endpoint = "app.bsky.feed.getFeed" params = { - "feed" : "at://{}/app.bsky.feed.generator/{}".format( - self._did_from_actor(actor), feed), + "feed": f"at://{self._did_from_actor(actor)}/app.bsky.feed.generator/{feed}", "limit": "100", } return self._pagination(endpoint, params) @@ -391,8 +398,7 @@ def get_follows(self, actor): def get_list_feed(self, actor, list): endpoint = "app.bsky.feed.getListFeed" params = { - "list" : "at://{}/app.bsky.graph.list/{}".format( - self._did_from_actor(actor), list), + "list": f"at://{self._did_from_actor(actor)}/app.bsky.graph.list/{list}", "limit": "100", } return self._pagination(endpoint, params) @@ -400,9 +406,8 @@ def get_list_feed(self, actor, list): def get_post_thread(self, actor, post_id): endpoint = "app.bsky.feed.getPostThread" params = { - "uri": "at://{}/app.bsky.feed.post/{}".format( - self._did_from_actor(actor), post_id), - "depth" : self.extractor.config("depth", "0"), + "uri": f"at://{self._did_from_actor(actor)}/app.bsky.feed.post/{post_id}", + "depth": self.extractor.config("depth", "0"), "parentHeight": "0", } @@ -433,9 +438,8 @@ def resolve_handle(self, handle): @memcache(keyarg=1) def get_service_endpoint(self, did): - if did.startswith('did:web:'): - url = "https://{}/.well-known/did.json".format( - did.rpartition(":")[2]) + if did.startswith("did:web:"): + url = "https://{}/.well-known/did.json".format(did.rpartition(":")[2]) else: url = "https://plc.directory/" + did @@ -451,21 +455,19 @@ def get_service_endpoint(self, did): def search_posts(self, query, sort=None): endpoint = "app.bsky.feed.searchPosts" params = { - "q" : query, + "q": query, "limit": "100", - "sort" : sort, + "sort": sort, } return self._pagination(endpoint, params, "posts") def _did_from_actor(self, actor, user_did=False): - if actor.startswith("did:"): - did = actor - else: - did = self.resolve_handle(actor) - + did = actor if actor.startswith("did:") else self.resolve_handle(actor) extr = self.extractor + if user_did and not extr.config("reposts", False): extr._user_did = did + if extr._metadata_user: extr._user = user = self.get_profile(did) user["instance"] = extr._instance(user["handle"]) @@ -490,29 +492,30 @@ def _authenticate_impl(self, username): headers = None data = { "identifier": username, - "password" : self.password, + "password": self.password, } - url = "{}/xrpc/{}".format(self.root, endpoint) + url = f"{self.root}/xrpc/{endpoint}" response = self.extractor.request( - url, method="POST", headers=headers, json=data, fatal=None) + url, method="POST", headers=headers, json=data, fatal=None + ) data = response.json() if response.status_code != 200: self.log.debug("Server response: %s", data) - raise exception.AuthenticationError('"{}: {}"'.format( - data.get("error"), data.get("message"))) + raise exception.AuthenticationError( + '"{}: {}"'.format(data.get("error"), data.get("message")) + ) _refresh_token_cache.update(self.username, data["refreshJwt"]) return "Bearer " + data["accessJwt"] def _call(self, endpoint, params): - url = "{}/xrpc/{}".format(self.root, endpoint) + url = f"{self.root}/xrpc/{endpoint}" while True: self.authenticate() - response = self.extractor.request( - url, params=params, headers=self.headers, fatal=None) + response = self.extractor.request(url, params=params, headers=self.headers, fatal=None) if response.status_code < 400: return response.json() @@ -523,11 +526,9 @@ def _call(self, endpoint, params): try: data = response.json() - msg = "API request failed ('{}: {}')".format( - data["error"], data["message"]) + msg = "API request failed ('{}: {}')".format(data["error"], data["message"]) except Exception: - msg = "API request failed ({} {})".format( - response.status_code, response.reason) + msg = f"API request failed ({response.status_code} {response.reason})" self.extractor.log.debug("Server response: %s", response.text) raise exception.StopExtraction(msg) @@ -543,6 +544,6 @@ def _pagination(self, endpoint, params, key="feed"): params["cursor"] = cursor -@cache(maxage=84*86400, keyarg=0) +@cache(maxage=84 * 86400, keyarg=0) def _refresh_token_cache(username): return None diff --git a/gallery_dl/extractor/booru.py b/gallery_dl/extractor/booru.py index 7e26f38b37..35a72327b3 100644 --- a/gallery_dl/extractor/booru.py +++ b/gallery_dl/extractor/booru.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,16 @@ """Extractors for *booru sites""" -from .common import BaseExtractor, Message -from .. import text import operator +from .. import text +from .common import BaseExtractor +from .common import Message + class BooruExtractor(BaseExtractor): """Base class for *booru extractors""" + basecategory = "booru" filename_fmt = "{category}_{id}_{md5}.{extension}" page_start = 0 @@ -42,8 +43,11 @@ def items(self): url = self.root + url except Exception as exc: self.log.debug("%s: %s", exc.__class__.__name__, exc) - self.log.warning("Unable to fetch download URL for post %s " - "(md5: %s)", post.get("id"), post.get("md5")) + self.log.warning( + "Unable to fetch download URL for post %s " "(md5: %s)", + post.get("id"), + post.get("md5"), + ) continue if fetch_html: diff --git a/gallery_dl/extractor/boosty.py b/gallery_dl/extractor/boosty.py index 33823be15f..1d80e2b427 100644 --- a/gallery_dl/extractor/boosty.py +++ b/gallery_dl/extractor/boosty.py @@ -1,23 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://www.boosty.to/""" -from .common import Extractor, Message -from .. import text, util, exception +from .. import exception +from .. import text +from .. import util +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?boosty\.to" class BoostyExtractor(Extractor): """Base class for boosty extractors""" + category = "boosty" root = "https://www.boosty.to" - directory_fmt = ("{category}", "{user[blogUrl]} ({user[id]})", - "{post[date]:%Y-%m-%d} {post[int_id]}") + directory_fmt = ( + "{category}", + "{user[blogUrl]} ({user[id]})", + "{post[date]:%Y-%m-%d} {post[int_id]}", + ) filename_fmt = "{num:>02} {file[id]}.{extension}" archive_fmt = "{file[id]}" cookies_domain = ".boosty.to" @@ -43,8 +48,16 @@ def _init(self): # low: 360p # lowest: 240p # tiny: 144p - videos = ("ultra_hd", "quad_hd", "full_hd", - "high", "medium", "low", "lowest", "tiny") + videos = ( + "ultra_hd", + "quad_hd", + "full_hd", + "high", + "medium", + "low", + "lowest", + "tiny", + ) self.videos = videos def items(self): @@ -55,8 +68,8 @@ def items(self): files = self._process_post(post) data = { - "post" : post, - "user" : post.pop("user", None), + "post": post, + "user": post.pop("user", None), "count": len(files), } @@ -93,19 +106,10 @@ def _process_post(self, post): elif type == "ok_video": if not self.videos: - self.log.debug("%s: Skipping video %s", - post["int_id"], block["id"]) + self.log.debug("%s: Skipping video %s", post["int_id"], block["id"]) continue - fmts = { - fmt["type"]: fmt["url"] - for fmt in block["playerUrls"] - if fmt["url"] - } - formats = [ - fmts[fmt] - for fmt in self.videos - if fmt in fmts - ] + fmts = {fmt["type"]: fmt["url"] for fmt in block["playerUrls"] if fmt["url"]} + formats = [fmts[fmt] for fmt in self.videos if fmt in fmts] if formats: formats = iter(formats) block["url"] = next(formats) @@ -113,8 +117,8 @@ def _process_post(self, post): files.append(block) else: self.log.warning( - "%s: Found no suitable video format for %s", - post["int_id"], block["id"]) + "%s: Found no suitable video format for %s", post["int_id"], block["id"] + ) elif type == "link": url = block["url"] @@ -125,8 +129,7 @@ def _process_post(self, post): files.append(self._update_url(post, block)) else: - self.log.debug("%s: Unsupported data type '%s'", - post["int_id"], type) + self.log.debug("%s: Unsupported data type '%s'", post["int_id"], type) except Exception as exc: self.log.debug("%s: %s", exc.__class__.__name__, exc) @@ -152,6 +155,7 @@ def _update_url(self, post, block): class BoostyUserExtractor(BoostyExtractor): """Extractor for boosty.to user profiles""" + subcategory = "user" pattern = BASE_PATTERN + r"/([^/?#]+)(?:\?([^#]+))?$" example = "https://boosty.to/USER" @@ -166,6 +170,7 @@ def posts(self): class BoostyMediaExtractor(BoostyExtractor): """Extractor for boosty.to user media""" + subcategory = "media" directory_fmt = "{category}", "{user[blogUrl]} ({user[id]})", "media" filename_fmt = "{post[id]}_{num}.{extension}" @@ -181,6 +186,7 @@ def posts(self): class BoostyFeedExtractor(BoostyExtractor): """Extractor for your boosty.to subscription feed""" + subcategory = "feed" pattern = BASE_PATTERN + r"/(?:\?([^#]+))?(?:$|#)" example = "https://boosty.to/" @@ -192,6 +198,7 @@ def posts(self): class BoostyPostExtractor(BoostyExtractor): """Extractor for boosty.to posts""" + subcategory = "post" pattern = BASE_PATTERN + r"/([^/?#]+)/posts/([0-9a-f-]+)" example = "https://boosty.to/USER/posts/01234567-89ab-cdef-0123-456789abcd" @@ -205,6 +212,7 @@ def posts(self): class BoostyFollowingExtractor(BoostyExtractor): """Extractor for your boosty.to subscribed users""" + subcategory = "following" pattern = BASE_PATTERN + r"/app/settings/subscriptions" example = "https://boosty.to/app/settings/subscriptions" @@ -216,8 +224,9 @@ def items(self): yield Message.Queue, url, user -class BoostyAPI(): +class BoostyAPI: """Interface for the Boosty API""" + root = "https://api.boosty.to" def __init__(self, extractor, access_token=None): @@ -230,29 +239,34 @@ def __init__(self, extractor, access_token=None): if not access_token: auth = self.extractor.cookies.get("auth", domain=".boosty.to") if auth: - access_token = text.extr( - auth, "%22accessToken%22%3A%22", "%22") + access_token = text.extr(auth, "%22accessToken%22%3A%22", "%22") if access_token: self.headers["Authorization"] = "Bearer " + access_token def blog_posts(self, username, params): - endpoint = "/v1/blog/{}/post/".format(username) - params = self._merge_params(params, { - "limit" : "5", - "offset" : None, - "comments_limit": "2", - "reply_limit" : "1", - }) + endpoint = f"/v1/blog/{username}/post/" + params = self._merge_params( + params, + { + "limit": "5", + "offset": None, + "comments_limit": "2", + "reply_limit": "1", + }, + ) return self._pagination(endpoint, params) def blog_media_album(self, username, type="all", params=()): - endpoint = "/v1/blog/{}/media_album/".format(username) - params = self._merge_params(params, { - "type" : type.rstrip("s"), - "limit" : "15", - "limit_by": "media", - "offset" : None, - }) + endpoint = f"/v1/blog/{username}/media_album/" + params = self._merge_params( + params, + { + "type": type.rstrip("s"), + "limit": "15", + "limit_by": "media", + "offset": None, + }, + ) return self._pagination(endpoint, params, self._transform_media_posts) def _transform_media_posts(self, data): @@ -266,16 +280,19 @@ def _transform_media_posts(self, data): return posts def post(self, username, post_id): - endpoint = "/v1/blog/{}/post/{}".format(username, post_id) + endpoint = f"/v1/blog/{username}/post/{post_id}" return self._call(endpoint) def feed_posts(self, params=None): endpoint = "/v1/feed/post/" - params = self._merge_params(params, { - "limit" : "5", - "offset" : None, - "comments_limit": "2", - }) + params = self._merge_params( + params, + { + "limit": "5", + "offset": None, + "comments_limit": "2", + }, + ) if "only_allowed" not in params and self.extractor.only_allowed: params["only_allowed"] = "true" if "only_bought" not in params and self.extractor.only_bought: @@ -290,20 +307,23 @@ def user(self, username): def user_subscriptions(self, params=None): endpoint = "/v1/user/subscriptions" - params = self._merge_params(params, { - "limit" : "30", - "with_follow": "true", - "offset" : None, - }) + params = self._merge_params( + params, + { + "limit": "30", + "with_follow": "true", + "offset": None, + }, + ) return self._pagination_users(endpoint, params) def _merge_params(self, params_web, params_api): if params_web: web_to_api = { "isOnlyAllowedPosts": "is_only_allowed", - "postsTagsIds" : "tags_ids", - "postsFrom" : "from_ts", - "postsTo" : "to_ts", + "postsTagsIds": "tags_ids", + "postsFrom": "from_ts", + "postsTo": "to_ts", } for name, value in params_web.items(): name = web_to_api.get(name, name) @@ -315,16 +335,16 @@ def _call(self, endpoint, params=None): while True: response = self.extractor.request( - url, params=params, headers=self.headers, - fatal=None, allow_redirects=False) + url, params=params, headers=self.headers, fatal=None, allow_redirects=False + ) if response.status_code < 300: return response.json() - elif response.status_code < 400: + if response.status_code < 400: raise exception.AuthenticationError("Invalid API access token") - elif response.status_code == 429: + if response.status_code == 429: self.extractor.wait(seconds=600) else: diff --git a/gallery_dl/extractor/bunkr.py b/gallery_dl/extractor/bunkr.py index 3e12452545..961f920757 100644 --- a/gallery_dl/extractor/bunkr.py +++ b/gallery_dl/extractor/bunkr.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,16 +6,16 @@ """Extractors for https://bunkr.si/""" +import random + +from .. import config +from .. import exception +from .. import text from .common import Extractor from .lolisafe import LolisafeAlbumExtractor -from .. import text, config, exception -import random if config.get(("extractor", "bunkr"), "tlds"): - BASE_PATTERN = ( - r"(?:bunkr:(?:https?://)?([^/?#]+)|" - r"(?:https?://)?(?:app\.)?(bunkr+\.\w+))" - ) + BASE_PATTERN = r"(?:bunkr:(?:https?://)?([^/?#]+)|" r"(?:https?://)?(?:app\.)?(bunkr+\.\w+))" else: BASE_PATTERN = ( r"(?:bunkr:(?:https?://)?([^/?#]+)|" @@ -58,6 +56,7 @@ class BunkrAlbumExtractor(LolisafeAlbumExtractor): """Extractor for bunkr.si albums""" + category = "bunkr" root = "https://bunkr.si" pattern = BASE_PATTERN + r"/a/([^/?#]+)" @@ -83,8 +82,7 @@ def request(self, url, **kwargs): root, path = self._split(url) if root not in CF_DOMAINS: continue - self.log.debug("Redirect to known CF challenge domain '%s'", - root) + self.log.debug("Redirect to known CF challenge domain '%s'", root) except exception.HttpError as exc: if exc.status != 403: @@ -102,7 +100,8 @@ def request(self, url, **kwargs): else: if not DOMAINS: raise exception.StopExtraction( - "All Bunkr domains require solving a CF challenge") + "All Bunkr domains require solving a CF challenge" + ) # select alternative domain root = "https://" + random.choice(DOMAINS) @@ -112,19 +111,17 @@ def request(self, url, **kwargs): def fetch_album(self, album_id): # album metadata page = self.request(self.root + "/a/" + album_id).text - title, size = text.split_html(text.extr( - page, "<h1", "</span>").partition(">")[2]) + title, size = text.split_html(text.extr(page, "<h1", "</span>").partition(">")[2]) if "&" in title: - title = title.replace( - "<", "<").replace(">", ">").replace("&", "&") + title = title.replace("<", "<").replace(">", ">").replace("&", "&") # files items = list(text.extract_iter(page, "<!-- item -->", "<!-- -->")) return self._extract_files(items), { - "album_id" : album_id, - "album_name" : title, - "album_size" : text.extr(size, "(", ")"), - "count" : len(items), + "album_id": album_id, + "album_name": title, + "album_size": text.extr(size, "(", ")"), + "count": len(items), } def _extract_files(self, items): @@ -136,8 +133,7 @@ def _extract_files(self, items): info = text.split_html(item) file["name"] = info[0] file["size"] = info[2] - file["date"] = text.parse_datetime( - info[-1], "%H:%M:%S %d/%m/%Y") + file["date"] = text.parse_datetime(info[-1], "%H:%M:%S %d/%m/%Y") yield file except exception.StopExtraction: @@ -149,18 +145,18 @@ def _extract_files(self, items): def _extract_file(self, webpage_url): response = self.request(webpage_url) page = response.text - file_url = (text.extr(page, '<source src="', '"') or - text.extr(page, '<img src="', '"')) + file_url = text.extr(page, '<source src="', '"') or text.extr(page, '<img src="', '"') if not file_url: - webpage_url = text.unescape(text.rextract( - page, ' href="', '"', page.rindex("Download"))[0]) + webpage_url = text.unescape( + text.rextract(page, ' href="', '"', page.rindex("Download"))[0] + ) response = self.request(webpage_url) file_url = text.rextract(response.text, ' href="', '"')[0] return { - "file" : text.unescape(file_url), - "_http_headers" : {"Referer": response.url}, + "file": text.unescape(file_url), + "_http_headers": {"Referer": response.url}, "_http_validate": self._validate, } @@ -177,6 +173,7 @@ def _split(self, url): class BunkrMediaExtractor(BunkrAlbumExtractor): """Extractor for bunkr.si media links""" + subcategory = "media" directory_fmt = ("{category}",) pattern = BASE_PATTERN + r"(/[vid]/[^/?#]+)" @@ -190,9 +187,9 @@ def fetch_album(self, album_id): return (), {} return (file,), { - "album_id" : "", - "album_name" : "", - "album_size" : -1, + "album_id": "", + "album_name": "", + "album_size": -1, "description": "", - "count" : 1, + "count": 1, } diff --git a/gallery_dl/extractor/catbox.py b/gallery_dl/extractor/catbox.py index 6c81f5394a..0f3890000e 100644 --- a/gallery_dl/extractor/catbox.py +++ b/gallery_dl/extractor/catbox.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,15 @@ """Extractors for https://catbox.moe/""" -from .common import GalleryExtractor, Extractor, Message from .. import text +from .common import Extractor +from .common import GalleryExtractor +from .common import Message class CatboxAlbumExtractor(GalleryExtractor): """Extractor for catbox albums""" + category = "catbox" subcategory = "album" root = "https://catbox.moe" @@ -26,23 +27,22 @@ class CatboxAlbumExtractor(GalleryExtractor): def metadata(self, page): extr = text.extract_from(page) return { - "album_id" : self.gallery_url.rpartition("/")[2], - "album_name" : text.unescape(extr("<h1>", "<")), - "date" : text.parse_datetime(extr( - "<p>Created ", "<"), "%B %d %Y"), + "album_id": self.gallery_url.rpartition("/")[2], + "album_name": text.unescape(extr("<h1>", "<")), + "date": text.parse_datetime(extr("<p>Created ", "<"), "%B %d %Y"), "description": text.unescape(extr("<p>", "<")), } def images(self, page): return [ ("https://files.catbox.moe/" + path, None) - for path in text.extract_iter( - page, ">https://files.catbox.moe/", "<") + for path in text.extract_iter(page, ">https://files.catbox.moe/", "<") ] class CatboxFileExtractor(Extractor): """Extractor for catbox files""" + category = "catbox" subcategory = "file" archive_fmt = "{filename}" diff --git a/gallery_dl/extractor/chevereto.py b/gallery_dl/extractor/chevereto.py index aedcea4574..67e1939888 100644 --- a/gallery_dl/extractor/chevereto.py +++ b/gallery_dl/extractor/chevereto.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,14 +6,20 @@ """Extractors for Chevereto galleries""" -from .common import BaseExtractor, Message from .. import text +from .common import BaseExtractor +from .common import Message class CheveretoExtractor(BaseExtractor): """Base class for chevereto extractors""" + basecategory = "chevereto" - directory_fmt = ("{category}", "{user}", "{album}",) + directory_fmt = ( + "{category}", + "{user}", + "{album}", + ) archive_fmt = "{id}" def __init__(self, match): @@ -26,27 +30,29 @@ def _pagination(self, url): while url: page = self.request(url).text - for item in text.extract_iter( - page, '<div class="list-item-image ', 'image-container'): + for item in text.extract_iter(page, '<div class="list-item-image ', "image-container"): yield text.extr(item, '<a href="', '"') url = text.extr(page, '<a data-pagination="next" href="', '" ><') -BASE_PATTERN = CheveretoExtractor.update({ - "jpgfish": { - "root": "https://jpg5.su", - "pattern": r"jpe?g\d?\.(?:su|pet|fish(?:ing)?|church)", - }, - "imgkiwi": { - "root": "https://img.kiwi", - "pattern": r"img\.kiwi", - }, -}) +BASE_PATTERN = CheveretoExtractor.update( + { + "jpgfish": { + "root": "https://jpg5.su", + "pattern": r"jpe?g\d?\.(?:su|pet|fish(?:ing)?|church)", + }, + "imgkiwi": { + "root": "https://img.kiwi", + "pattern": r"img\.kiwi", + }, + } +) class CheveretoImageExtractor(CheveretoExtractor): """Extractor for chevereto Images""" + subcategory = "image" pattern = BASE_PATTERN + r"(/im(?:g|age)/[^/?#]+)" example = "https://jpg2.su/img/TITLE.ID" @@ -56,10 +62,10 @@ def items(self): extr = text.extract_from(self.request(url).text) image = { - "id" : self.path.rpartition(".")[2], - "url" : extr('<meta property="og:image" content="', '"'), + "id": self.path.rpartition(".")[2], + "url": extr('<meta property="og:image" content="', '"'), "album": text.extr(extr("Added to <a", "/a>"), ">", "<"), - "user" : extr('username: "', '"'), + "user": extr('username: "', '"'), } text.nameext_from_url(image["url"], image) @@ -69,6 +75,7 @@ def items(self): class CheveretoAlbumExtractor(CheveretoExtractor): """Extractor for chevereto Albums""" + subcategory = "album" pattern = BASE_PATTERN + r"(/a(?:lbum)?/[^/?#]+(?:/sub)?)" example = "https://jpg2.su/album/TITLE.ID" @@ -77,10 +84,7 @@ def items(self): url = self.root + self.path data = {"_extractor": CheveretoImageExtractor} - if self.path.endswith("/sub"): - albums = self._pagination(url) - else: - albums = (url,) + albums = self._pagination(url) if self.path.endswith("/sub") else (url,) for album in albums: for image in self._pagination(album): @@ -89,6 +93,7 @@ def items(self): class CheveretoUserExtractor(CheveretoExtractor): """Extractor for chevereto Users""" + subcategory = "user" pattern = BASE_PATTERN + r"(/(?!img|image|a(?:lbum)?)[^/?#]+(?:/albums)?)" example = "https://jpg2.su/USER" diff --git a/gallery_dl/extractor/cien.py b/gallery_dl/extractor/cien.py index 378365eb78..83e680611c 100644 --- a/gallery_dl/extractor/cien.py +++ b/gallery_dl/extractor/cien.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,8 +6,10 @@ """Extractors for https://ci-en.net/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?ci-en\.(?:net|dlsite\.com)" @@ -33,8 +33,7 @@ def _pagination_articles(self, url, params): while True: page = self.request(url, params=params).text - for card in text.extract_iter( - page, ' class="c-cardCase-item', '</div>'): + for card in text.extract_iter(page, ' class="c-cardCase-item', "</div>"): article_url = text.extr(card, ' href="', '"') yield Message.Queue, article_url, data @@ -52,12 +51,12 @@ class CienArticleExtractor(CienExtractor): example = "https://ci-en.net/creator/123/article/12345" def items(self): - url = "{}/creator/{}/article/{}".format( - self.root, self.groups[0], self.groups[1]) + url = f"{self.root}/creator/{self.groups[0]}/article/{self.groups[1]}" page = self.request(url, notfound="article").text - post = util.json_loads(text.extr( - page, '<script type="application/ld+json">', '</script>'))[0] + post = util.json_loads(text.extr(page, '<script type="application/ld+json">', "</script>"))[ + 0 + ] files = self._extract_files(page) @@ -90,10 +89,10 @@ def _extract_files(self, page): self._extract_files_gallery(page, files) else: generators = { - "image" : self._extract_files_image, - "video" : self._extract_files_video, + "image": self._extract_files_image, + "video": self._extract_files_video, "download": self._extract_files_download, - "gallery" : self._extract_files_gallery, + "gallery": self._extract_files_gallery, "gallerie": self._extract_files_gallery, } if isinstance(filetypes, str): @@ -104,33 +103,32 @@ def _extract_files(self, page): return files def _extract_files_image(self, page, files): - for image in text.extract_iter( - page, 'class="file-player-image"', "</figure>"): + for image in text.extract_iter(page, 'class="file-player-image"', "</figure>"): size = text.extr(image, ' data-size="', '"') w, _, h = size.partition("x") - files.append({ - "url" : text.extr(image, ' data-raw="', '"'), - "width" : text.parse_int(w), - "height": text.parse_int(h), - "type" : "image", - }) + files.append( + { + "url": text.extr(image, ' data-raw="', '"'), + "width": text.parse_int(w), + "height": text.parse_int(h), + "type": "image", + } + ) def _extract_files_video(self, page, files): - for video in text.extract_iter( - page, "<vue-file-player", "</vue-file-player>"): + for video in text.extract_iter(page, "<vue-file-player", "</vue-file-player>"): path = text.extr(video, ' base-path="', '"') name = text.extr(video, ' file-name="', '"') auth = text.extr(video, ' auth-key="', '"') file = text.nameext_from_url(name) - file["url"] = "{}video-web.mp4?{}".format(path, auth) + file["url"] = f"{path}video-web.mp4?{auth}" file["type"] = "video" files.append(file) def _extract_files_download(self, page, files): - for download in text.extract_iter( - page, 'class="downloadBlock', "</div>"): + for download in text.extract_iter(page, 'class="downloadBlock', "</div>"): name = text.extr(download, "<p>", "<") file = text.nameext_from_url(name.rpartition(" ")[0]) @@ -139,20 +137,17 @@ def _extract_files_download(self, page, files): files.append(file) def _extract_files_gallery(self, page, files): - for gallery in text.extract_iter( - page, "<vue-image-gallery", "</vue-image-gallery>"): - + for gallery in text.extract_iter(page, "<vue-image-gallery", "</vue-image-gallery>"): url = self.root + "/api/creator/gallery/images" params = { - "hash" : text.extr(gallery, ' hash="', '"'), + "hash": text.extr(gallery, ' hash="', '"'), "gallery_id": text.extr(gallery, ' gallery-id="', '"'), - "time" : text.extr(gallery, ' time="', '"'), + "time": text.extr(gallery, ' time="', '"'), } data = self.request(url, params=params).json() url = self.root + "/api/creator/gallery/imagePath" - for params["page"], params["file_id"] in enumerate( - data["imgList"]): + for params["page"], params["file_id"] in enumerate(data["imgList"]): path = self.request(url, params=params).json()["path"] file = params.copy() @@ -166,7 +161,7 @@ class CienCreatorExtractor(CienExtractor): example = "https://ci-en.net/creator/123" def items(self): - url = "{}/creator/{}/article".format(self.root, self.groups[0]) + url = f"{self.root}/creator/{self.groups[0]}/article" params = text.parse_query(self.groups[1]) params["mode"] = "list" return self._pagination_articles(url, params) @@ -193,7 +188,6 @@ def items(self): page = self.request(url).text data = {"_extractor": CienCreatorExtractor} - for subscription in text.extract_iter( - page, 'class="c-grid-subscriptionInfo', '</figure>'): + for subscription in text.extract_iter(page, 'class="c-grid-subscriptionInfo', "</figure>"): url = text.extr(subscription, ' href="', '"') yield Message.Queue, url, data diff --git a/gallery_dl/extractor/civitai.py b/gallery_dl/extractor/civitai.py index 1e8cb4241a..4da5964eba 100644 --- a/gallery_dl/extractor/civitai.py +++ b/gallery_dl/extractor/civitai.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,17 +6,22 @@ """Extractors for https://www.civitai.com/""" -from .common import Extractor, Message -from .. import text, util, exception import itertools import time +from .. import exception +from .. import text +from .. import util +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?civitai\.com" USER_PATTERN = BASE_PATTERN + r"/user/([^/?#]+)" class CivitaiExtractor(Extractor): """Base class for civitai extractors""" + category = "civitai" root = "https://civitai.com" directory_fmt = ("{category}", "{username|user[username]}", "images") @@ -39,7 +42,7 @@ def _init(self): if not isinstance(quality, str): quality = ",".join(quality) self._image_quality = quality - self._image_ext = ("png" if quality == "original=true" else "jpg") + self._image_ext = "png" if quality == "original=true" else "jpg" else: self._image_quality = "original=true" self._image_ext = "png" @@ -50,7 +53,7 @@ def _init(self): metadata = metadata.split(",") elif not isinstance(metadata, (list, tuple)): metadata = ("generation",) - self._meta_generation = ("generation" in metadata) + self._meta_generation = "generation" in metadata else: self._meta_generation = False @@ -66,15 +69,10 @@ def items(self): posts = self.posts() if posts: for post in posts: - - if "images" in post: - images = post["images"] - else: - images = self.api.images_post(post["id"]) + images = post.get("images", self.api.images_post(post["id"])) post = self.api.post(post["id"]) - post["date"] = text.parse_datetime( - post["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date"] = text.parse_datetime(post["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") data = { "post": post, "user": post["user"], @@ -92,10 +90,8 @@ def items(self): for image in images: url = self._url(image) if self._meta_generation: - image["generation"] = self.api.image_generationdata( - image["id"]) - image["date"] = text.parse_datetime( - image["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + image["generation"] = self.api.image_generationdata(image["id"]) + image["date"] = text.parse_datetime(image["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") text.nameext_from_url(url, image) image["extension"] = self._image_ext yield Message.Directory, image @@ -125,17 +121,19 @@ def _url(self, image): mime = image.get("mimeType") or self._image_ext name = "{}.{}".format(image.get("id"), mime.rpartition("/")[2]) return ( - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/{}/{}/{}".format( - url, self._image_quality, name) + f"https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/{url}/{self._image_quality}/{name}" ) def _image_results(self, images): for num, file in enumerate(images, 1): - data = text.nameext_from_url(file["url"], { - "num" : num, - "file": file, - "url" : self._url(file), - }) + data = text.nameext_from_url( + file["url"], + { + "num": num, + "file": file, + "url": self._url(file), + }, + ) if not data["extension"]: data["extension"] = self._image_ext if "id" not in file and data["filename"].isdecimal(): @@ -147,9 +145,12 @@ def _image_results(self, images): class CivitaiModelExtractor(CivitaiExtractor): subcategory = "model" - directory_fmt = ("{category}", "{user[username]}", - "{model[id]}{model[name]:? //}", - "{version[id]}{version[name]:? //}") + directory_fmt = ( + "{category}", + "{user[username]}", + "{model[id]}{model[name]:? //}", + "{version[id]}{version[name]:? //}", + ) pattern = BASE_PATTERN + r"/models/(\d+)(?:/?\?modelVersionId=(\d+))?" example = "https://civitai.com/models/12345/TITLE" @@ -176,13 +177,12 @@ def items(self): versions = (version,) for version in versions: - version["date"] = text.parse_datetime( - version["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + version["date"] = text.parse_datetime(version["createdAt"], "%Y-%m-%dT%H:%M:%S.%fZ") data = { - "model" : model, + "model": model, "version": version, - "user" : user, + "user": user, } yield Message.Directory, data @@ -196,37 +196,35 @@ def _extract_files(self, model, version, user): return self._extract_files_image(model, version, user) generators = { - "model" : self._extract_files_model, - "image" : self._extract_files_image, - "gallery" : self._extract_files_gallery, + "model": self._extract_files_model, + "image": self._extract_files_image, + "gallery": self._extract_files_gallery, "gallerie": self._extract_files_gallery, } if isinstance(filetypes, str): filetypes = filetypes.split(",") return itertools.chain.from_iterable( - generators[ft.rstrip("s")](model, version, user) - for ft in filetypes + generators[ft.rstrip("s")](model, version, user) for ft in filetypes ) def _extract_files_model(self, model, version, user): files = [] for num, file in enumerate(version["files"], 1): - file["uuid"] = "model-{}-{}-{}".format( - model["id"], version["id"], file["id"]) - files.append({ - "num" : num, - "file" : file, - "filename" : file["name"], - "extension": "bin", - "url" : file.get("downloadUrl") or - "{}/api/download/models/{}".format( - self.root, version["id"]), - "_http_headers" : { - "Authorization": self.api.headers.get("Authorization")}, - "_http_validate": self._validate_file_model, - }) + file["uuid"] = "model-{}-{}-{}".format(model["id"], version["id"], file["id"]) + files.append( + { + "num": num, + "file": file, + "filename": file["name"], + "extension": "bin", + "url": file.get("downloadUrl") + or "{}/api/download/models/{}".format(self.root, version["id"]), + "_http_headers": {"Authorization": self.api.headers.get("Authorization")}, + "_http_validate": self._validate_file_model, + } + ) return files @@ -252,11 +250,9 @@ def _extract_files_gallery(self, model, version, user): def _validate_file_model(self, response): if response.headers.get("Content-Type", "").startswith("text/html"): - alert = text.extr( - response.text, 'mantine-Alert-message">', "</div></div></div>") + alert = text.extr(response.text, 'mantine-Alert-message">', "</div></div></div>") if alert: - msg = "\"{}\" - 'api-key' required".format( - text.remove_html(alert)) + msg = f"\"{text.remove_html(alert)}\" - 'api-key' required" else: msg = "'api-key' required to download this file" self.log.warning(msg) @@ -275,8 +271,12 @@ def images(self): class CivitaiPostExtractor(CivitaiExtractor): subcategory = "post" - directory_fmt = ("{category}", "{username|user[username]}", "posts", - "{post[id]}{post[title]:? //}") + directory_fmt = ( + "{category}", + "{username|user[username]}", + "posts", + "{post[id]}{post[title]:? //}", + ) pattern = BASE_PATTERN + r"/posts/(\d+)" example = "https://civitai.com/posts/12345" @@ -333,12 +333,15 @@ def initialize(self): pass def items(self): - base = "{}/user/{}/".format(self.root, self.groups[0]) - return self._dispatch_extractors(( - (CivitaiUserModelsExtractor, base + "models"), - (CivitaiUserPostsExtractor , base + "posts"), - (CivitaiUserImagesExtractor, base + "images"), - ), ("user-models", "user-posts")) + base = f"{self.root}/user/{self.groups[0]}/" + return self._dispatch_extractors( + ( + (CivitaiUserModelsExtractor, base + "models"), + (CivitaiUserPostsExtractor, base + "posts"), + (CivitaiUserImagesExtractor, base + "images"), + ), + ("user-models", "user-posts"), + ) class CivitaiUserModelsExtractor(CivitaiExtractor): @@ -354,8 +357,12 @@ def models(self): class CivitaiUserPostsExtractor(CivitaiExtractor): subcategory = "user-posts" - directory_fmt = ("{category}", "{username|user[username]}", "posts", - "{post[id]}{post[title]:? //}") + directory_fmt = ( + "{category}", + "{username|user[username]}", + "posts", + "{post[id]}{post[title]:? //}", + ) pattern = USER_PATTERN + r"/posts/?(?:\?([^#]+))?" example = "https://civitai.com/user/USER/posts" @@ -383,9 +390,9 @@ def images(self): return self.api.images(params) def images_reactions(self): - if "Authorization" not in self.api.headers and \ - not self.cookies.get( - "__Secure-civitai-token", domain=".civitai.com"): + if "Authorization" not in self.api.headers and not self.cookies.get( + "__Secure-civitai-token", domain=".civitai.com" + ): raise exception.AuthorizationError("api-key or cookies required") params = self.params @@ -395,12 +402,11 @@ def images_reactions(self): if isinstance(params["reactions"], str): params["reactions"] = (params["reactions"],) else: - params["reactions"] = ( - "Like", "Dislike", "Heart", "Laugh", "Cry") + params["reactions"] = ("Like", "Dislike", "Heart", "Laugh", "Cry") return self.api.images(params) -class CivitaiRestAPI(): +class CivitaiRestAPI: """Interface for the Civitai Public REST API https://developer.civitai.com/docs/api/public-rest @@ -424,9 +430,11 @@ def __init__(self, extractor): self.nsfw = nsfw def image(self, image_id): - return self.images({ - "imageId": image_id, - }) + return self.images( + { + "imageId": image_id, + } + ) def images(self, params): endpoint = "/v1/images" @@ -435,17 +443,19 @@ def images(self, params): return self._pagination(endpoint, params) def images_gallery(self, model, version, user): - return self.images({ - "modelId" : model["id"], - "modelVersionId": version["id"], - }) + return self.images( + { + "modelId": model["id"], + "modelVersionId": version["id"], + } + ) def model(self, model_id): - endpoint = "/v1/models/{}".format(model_id) + endpoint = f"/v1/models/{model_id}" return self._call(endpoint) def model_version(self, model_version_id): - endpoint = "/v1/model-versions/{}".format(model_version_id) + endpoint = f"/v1/model-versions/{model_version_id}" return self._call(endpoint) def models(self, params): @@ -455,13 +465,9 @@ def models_tag(self, tag): return self.models({"tag": tag}) def _call(self, endpoint, params=None): - if endpoint[0] == "/": - url = self.root + endpoint - else: - url = endpoint + url = self.root + endpoint if endpoint[0] == "/" else endpoint - response = self.extractor.request( - url, params=params, headers=self.headers) + response = self.extractor.request(url, params=params, headers=self.headers) return response.json() def _pagination(self, endpoint, params): @@ -476,18 +482,18 @@ def _pagination(self, endpoint, params): params = None -class CivitaiTrpcAPI(): +class CivitaiTrpcAPI: """Interface for the Civitai tRPC API""" def __init__(self, extractor): self.extractor = extractor self.root = extractor.root + "/api/trpc/" self.headers = { - "content-type" : "application/json", + "content-type": "application/json", "x-client-version": "5.0.211", - "x-client-date" : "", - "x-client" : "web", - "x-fingerprint" : "undefined", + "x-client-date": "", + "x-client": "web", + "x-fingerprint": "undefined", } api_key = extractor.config("api-key") if api_key: @@ -515,16 +521,19 @@ def images(self, params, defaults=True): endpoint = "image.getInfinite" if defaults: - params = self._merge_params(params, { - "useIndex" : True, - "period" : "AllTime", - "sort" : "Newest", - "types" : ["image"], - "withMeta" : False, # Metadata Only - "fromPlatform" : False, # Made On-Site - "browsingLevel": self.nsfw, - "include" : ["cosmetics"], - }) + params = self._merge_params( + params, + { + "useIndex": True, + "period": "AllTime", + "sort": "Newest", + "types": ["image"], + "withMeta": False, # Metadata Only + "fromPlatform": False, # Made On-Site + "browsingLevel": self.nsfw, + "include": ["cosmetics"], + }, + ) params = self._type_params(params) return self._pagination(endpoint, params) @@ -532,13 +541,13 @@ def images(self, params, defaults=True): def images_gallery(self, model, version, user): endpoint = "image.getImagesAsPostsInfinite" params = { - "period" : "AllTime", - "sort" : "Newest", + "period": "AllTime", + "sort": "Newest", "modelVersionId": version["id"], - "modelId" : model["id"], - "hidden" : False, - "limit" : 50, - "browsingLevel" : self.nsfw, + "modelId": model["id"], + "hidden": False, + "limit": 50, + "browsingLevel": self.nsfw, } for post in self._pagination(endpoint, params): @@ -546,7 +555,7 @@ def images_gallery(self, model, version, user): def images_post(self, post_id): params = { - "postId" : int(post_id), + "postId": int(post_id), "pending": True, } return self.images(params) @@ -565,18 +574,21 @@ def models(self, params, defaults=True): endpoint = "model.getAll" if defaults: - params = self._merge_params(params, { - "period" : "AllTime", - "periodMode" : "published", - "sort" : "Newest", - "pending" : False, - "hidden" : False, - "followed" : False, - "earlyAccess" : False, - "fromPlatform" : False, - "supportsGeneration": False, - "browsingLevel": self.nsfw, - }) + params = self._merge_params( + params, + { + "period": "AllTime", + "periodMode": "published", + "sort": "Newest", + "pending": False, + "hidden": False, + "followed": False, + "earlyAccess": False, + "fromPlatform": False, + "supportsGeneration": False, + "browsingLevel": self.nsfw, + }, + ) return self._pagination(endpoint, params) @@ -593,16 +605,19 @@ def posts(self, params, defaults=True): meta = {"cursor": ("Date",)} if defaults: - params = self._merge_params(params, { - "browsingLevel": self.nsfw, - "period" : "AllTime", - "periodMode" : "published", - "sort" : "Newest", - "followed" : False, - "draftOnly" : False, - "pending" : True, - "include" : ["cosmetics"], - }) + params = self._merge_params( + params, + { + "browsingLevel": self.nsfw, + "period": "AllTime", + "periodMode": "published", + "sort": "Newest", + "followed": False, + "draftOnly": False, + "pending": True, + "include": ["cosmetics"], + }, + ) return self._pagination(endpoint, params, meta) @@ -615,12 +630,9 @@ def _call(self, endpoint, params, meta=None): url = self.root + endpoint headers = self.headers - if meta: - input = {"json": params, "meta": {"values": meta}} - else: - input = {"json": params} + input_ = {"json": params, "meta": {"values": meta}} if meta else {"json": params} - params = {"input": util.json_dumps(input)} + params = {"input": util.json_dumps(input_)} headers["x-client-date"] = str(int(time.time() * 1000)) response = self.extractor.request(url, params=params, headers=headers) @@ -650,8 +662,8 @@ def _merge_params(self, params_user, params_default): def _type_params(self, params): for key, type in ( - ("tags" , int), - ("modelId" , int), + ("tags", int), + ("modelId", int), ("modelVersionId", int), ): if key in params: diff --git a/gallery_dl/extractor/cohost.py b/gallery_dl/extractor/cohost.py index 0524239b4f..69a2239799 100644 --- a/gallery_dl/extractor/cohost.py +++ b/gallery_dl/extractor/cohost.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,18 +6,21 @@ """Extractors for https://cohost.org/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?cohost\.org" class CohostExtractor(Extractor): """Base class for cohost extractors""" + category = "cohost" root = "https://cohost.org" directory_fmt = ("{category}", "{postingProject[handle]}") - filename_fmt = ("{postId}_{headline:?/_/[b:200]}{num}.{extension}") + filename_fmt = "{postId}_{headline:?/_/[b:200]}{num}.{extension}" archive_fmt = "{postId}_{num}" def _init(self): @@ -33,14 +34,12 @@ def items(self): reason = post.get("limitedVisibilityReason") if reason and reason != "none": if reason == "log-in-first": - reason = ("This page's posts are visible only to users " - "who are logged in.") + reason = "This page's posts are visible only to users " "who are logged in." self.log.warning('%s: "%s"', post["postId"], reason) files = self._extract_files(post) post["count"] = len(files) - post["date"] = text.parse_datetime( - post["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") + post["date"] = text.parse_datetime(post["publishedAt"], "%Y-%m-%dT%H:%M:%S.%fZ") yield Message.Directory, post for post["num"], file in enumerate(files, 1): @@ -53,7 +52,7 @@ def posts(self): return () def _request_api(self, endpoint, input): - url = "{}/api/v1/trpc/{}".format(self.root, endpoint) + url = f"{self.root}/api/v1/trpc/{endpoint}" params = {"batch": "1", "input": util.json_dumps({"0": input})} headers = {"content-type": "application/json"} @@ -91,14 +90,14 @@ def _extract_blocks(self, post, files, shared=None): elif type == "ask": post["ask"] = block["ask"] else: - self.log.debug("%s: Unsupported block type '%s'", - post["postId"], type) + self.log.debug("%s: Unsupported block type '%s'", post["postId"], type) except Exception as exc: self.log.debug("%s: %s", exc.__class__.__name__, exc) class CohostUserExtractor(CohostExtractor): """Extractor for media from a cohost user""" + subcategory = "user" pattern = BASE_PATTERN + r"/([^/?#]+)/?(?:$|\?|#)" example = "https://cohost.org/USER" @@ -109,10 +108,10 @@ def posts(self): "projectHandle": self.groups[0], "page": 0, "options": { - "pinnedPostsAtTop" : True if self.pinned else False, - "hideReplies" : not self.replies, - "hideShares" : not self.shares, - "hideAsks" : not self.asks, + "pinnedPostsAtTop": bool(self.pinned), + "hideReplies": not self.replies, + "hideShares": not self.shares, + "hideAsks": not self.asks, "viewingOnProjectPage": True, }, } @@ -137,6 +136,7 @@ def posts(self): class CohostPostExtractor(CohostExtractor): """Extractor for media from a single cohost post""" + subcategory = "post" pattern = BASE_PATTERN + r"/([^/?#]+)/post/(\d+)" example = "https://cohost.org/USER/post/12345" @@ -161,21 +161,20 @@ def posts(self): class CohostTagExtractor(CohostExtractor): """Extractor for tagged posts""" + subcategory = "tag" pattern = BASE_PATTERN + r"/([^/?#]+)/tagged/([^/?#]+)(?:\?([^#]+))?" example = "https://cohost.org/USER/tagged/TAG" def posts(self): user, tag, query = self.groups - url = "{}/{}/tagged/{}".format(self.root, user, tag) + url = f"{self.root}/{user}/tagged/{tag}" params = text.parse_query(query) - post_feed_key = ("tagged-post-feed" if user == "rc" else - "project-tagged-post-feed") + post_feed_key = "tagged-post-feed" if user == "rc" else "project-tagged-post-feed" while True: page = self.request(url, params=params).text - data = util.json_loads(text.extr( - page, 'id="__COHOST_LOADER_STATE__">', '</script>')) + data = util.json_loads(text.extr(page, 'id="__COHOST_LOADER_STATE__">', "</script>")) try: feed = data[post_feed_key] @@ -188,24 +187,23 @@ def posts(self): if not pagination.get("morePagesForward"): return params["refTimestamp"] = pagination["refTimestamp"] - params["skipPosts"] = \ - pagination["currentSkip"] + pagination["idealPageStride"] + params["skipPosts"] = pagination["currentSkip"] + pagination["idealPageStride"] class CohostLikesExtractor(CohostExtractor): """Extractor for liked posts""" + subcategory = "likes" pattern = BASE_PATTERN + r"/rc/liked-posts" example = "https://cohost.org/rc/liked-posts" def posts(self): - url = "{}/rc/liked-posts".format(self.root) + url = f"{self.root}/rc/liked-posts" params = {} while True: page = self.request(url, params=params).text - data = util.json_loads(text.extr( - page, 'id="__COHOST_LOADER_STATE__">', '</script>')) + data = util.json_loads(text.extr(page, 'id="__COHOST_LOADER_STATE__">', "</script>")) try: feed = data["liked-posts-feed"] @@ -218,5 +216,4 @@ def posts(self): if not pagination.get("morePagesForward"): return params["refTimestamp"] = pagination["refTimestamp"] - params["skipPosts"] = \ - pagination["currentSkip"] + pagination["idealPageStride"] + params["skipPosts"] = pagination["currentSkip"] + pagination["idealPageStride"] diff --git a/gallery_dl/extractor/comicvine.py b/gallery_dl/extractor/comicvine.py index d076795c13..bbcc2bbcfd 100644 --- a/gallery_dl/extractor/comicvine.py +++ b/gallery_dl/extractor/comicvine.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,15 @@ """Extractors for https://comicvine.gamespot.com/""" -from .booru import BooruExtractor -from .. import text import operator +from .. import text +from .booru import BooruExtractor + class ComicvineTagExtractor(BooruExtractor): """Extractor for a gallery on comicvine.gamespot.com""" + category = "comicvine" subcategory = "tag" basecategory = "" @@ -23,8 +23,7 @@ class ComicvineTagExtractor(BooruExtractor): directory_fmt = ("{category}", "{tag}") filename_fmt = "{filename}.{extension}" archive_fmt = "{id}" - pattern = (r"(?:https?://)?comicvine\.gamespot\.com" - r"(/([^/?#]+)/(\d+-\d+)/images/.*)") + pattern = r"(?:https?://)?comicvine\.gamespot\.com" r"(/([^/?#]+)/(\d+-\d+)/images/.*)" example = "https://comicvine.gamespot.com/TAG/123-45/images/" def __init__(self, match): @@ -38,10 +37,10 @@ def posts(self): url = self.root + "/js/image-data.json" params = { "images": text.extract( - self.request(self.root + self.path).text, - 'data-gallery-id="', '"')[0], - "start" : self.page_start, - "count" : self.per_page, + self.request(self.root + self.path).text, 'data-gallery-id="', '"' + )[0], + "start": self.page_start, + "count": self.per_page, "object": self.object_id, } @@ -61,6 +60,5 @@ def skip(self, num): @staticmethod def _prepare(post): - post["date"] = text.parse_datetime( - post["dateCreated"], "%a, %b %d %Y") + post["date"] = text.parse_datetime(post["dateCreated"], "%a, %b %d %Y") post["tags"] = [tag["name"] for tag in post["tags"] if tag["name"]] diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index 090823e12f..d385196b5c 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,26 +6,44 @@ """Common classes and constants used by extractor modules.""" +from __future__ import annotations + +import datetime +import getpass +import logging +import netrc import os +import queue +import random import re import ssl +import threading import time -import netrc -import queue -import random -import getpass -import logging -import datetime +from contextlib import suppress +from typing import TYPE_CHECKING +from typing import Any +from typing import AnyStr +from typing import Literal +from typing import Self + import requests -import threading from requests.adapters import HTTPAdapter + +from .. import cache +from .. import config +from .. import exception +from .. import output +from .. import text +from .. import util from .message import Message -from .. import config, output, text, util, cache, exception -urllib3 = requests.packages.urllib3 +if TYPE_CHECKING: + from collections.abc import Generator -class Extractor(): +urllib3 = requests.packages.urllib3 + +class Extractor: category = "" subcategory = "" basecategory = "" @@ -47,8 +63,8 @@ class Extractor(): request_interval_429 = 60.0 request_timestamp = 0.0 - def __init__(self, match): - self.log = logging.getLogger(self.category) + def __init__(self, match: re.Match[AnyStr]) -> None: + self.log: logging.Logger = logging.getLogger(self.category) self.url = match.string self.match = match self.groups = match.groups() @@ -56,7 +72,7 @@ def __init__(self, match): self._parentdir = "" @classmethod - def from_url(cls, url): + def from_url(cls, url) -> Self | None: if isinstance(cls.pattern, str): cls.pattern = re.compile(cls.pattern) match = cls.pattern.match(url) @@ -66,20 +82,20 @@ def __iter__(self): self.initialize() return self.items() - def initialize(self): + def initialize(self) -> None: self._init_options() self._init_session() self._init_cookies() self._init() self.initialize = util.noop - def finalize(self): + def finalize(self) -> None: pass - def items(self): + def items(self) -> Generator[tuple[int, Literal[1]], Any, None]: yield Message.Version, 1 - def skip(self, num): + def skip(self, num) -> Literal[0]: return 0 def config(self, key, default=None): @@ -91,14 +107,14 @@ def config2(self, key, key2, default=None, sentinel=util.SENTINEL): return value return self.config(key2, default) - def config_deprecated(self, key, deprecated, default=None, - sentinel=util.SENTINEL, history=set()): + def config_deprecated( + self, key, deprecated, default=None, sentinel=util.SENTINEL, history=set() + ): value = self.config(deprecated, sentinel) if value is not sentinel: if deprecated not in history: history.add(deprecated) - self.log.warning("'%s' is deprecated. Use '%s' instead.", - deprecated, key) + self.log.warning("'%s' is deprecated. Use '%s' instead.", deprecated, key) default = value value = self.config(key, sentinel) @@ -113,8 +129,7 @@ def config_instance(self, key, default=None): return default def _config_shared(self, key, default=None): - return config.interpolate_common( - ("extractor",), self._cfgpath, key, default) + return config.interpolate_common(("extractor",), self._cfgpath, key, default) def _config_shared_accumulate(self, key): first = True @@ -127,13 +142,21 @@ def _config_shared_accumulate(self, key): else: conf = config.get(extr, path[0]) if conf: - values[:0] = config.accumulate( - (self.subcategory,), key, conf=conf) + values[:0] = config.accumulate((self.subcategory,), key, conf=conf) return values - def request(self, url, method="GET", session=None, - retries=None, retry_codes=None, encoding=None, - fatal=True, notfound=None, **kwargs): + def request( + self, + url, + method="GET", + session=None, + retries=None, + retry_codes=None, + encoding=None, + fatal=True, + notfound=None, + **kwargs, + ): if session is None: session = self.session if retries is None: @@ -162,31 +185,32 @@ def request(self, url, method="GET", session=None, tries = 1 if self._interval: - seconds = (self._interval() - - (time.time() - Extractor.request_timestamp)) + seconds = self._interval() - (time.time() - Extractor.request_timestamp) if seconds > 0.0: self.sleep(seconds, "request") while True: try: response = session.request(method, url, **kwargs) - except (requests.exceptions.ConnectionError, - requests.exceptions.Timeout, - requests.exceptions.ChunkedEncodingError, - requests.exceptions.ContentDecodingError) as exc: + except ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.ChunkedEncodingError, + requests.exceptions.ContentDecodingError, + ) as exc: msg = exc code = 0 - except (requests.exceptions.RequestException) as exc: + except requests.exceptions.RequestException as exc: raise exception.HttpError(exc) else: code = response.status_code if self._write_pages: self._dump_response(response) if ( - code < 400 or - code < 500 and ( - not fatal and code != 429 or fatal is None) or - fatal is ... + code < 400 + or code < 500 + and (not fatal and code != 429 or fatal is None) + or fatal is ... ): if encoding: response.encoding = encoding @@ -194,11 +218,9 @@ def request(self, url, method="GET", session=None, if notfound and code == 404: raise exception.NotFoundError(notfound) - msg = "'{} {}' for '{}'".format( - code, response.reason, response.url) + msg = f"'{code} {response.reason}' for '{response.url}'" server = response.headers.get("Server") - if server and server.startswith("cloudflare") and \ - code in (403, 503): + if server and server.startswith("cloudflare") and code in (403, 503): mitigated = response.headers.get("cf-mitigated") if mitigated and mitigated.lower() == "challenge": self.log.warning("Cloudflare challenge") @@ -213,7 +235,7 @@ def request(self, url, method="GET", session=None, if code == 429 and self._handle_429(response): continue - elif code == 429 and self._interval_429: + if code == 429 and self._interval_429: pass elif code not in retry_codes and code < 500: break @@ -221,19 +243,17 @@ def request(self, url, method="GET", session=None, finally: Extractor.request_timestamp = time.time() - self.log.debug("%s (%s/%s)", msg, tries, retries+1) + self.log.debug("%s (%s/%s)", msg, tries, retries + 1) if tries > retries: break seconds = tries if self._interval: s = self._interval() - if seconds < s: - seconds = s + seconds = max(seconds, s) if code == 429 and self._interval_429: s = self._interval_429() - if seconds < s: - seconds = s + seconds = max(seconds, s) self.wait(seconds=seconds, reason="429 Too Many Requests") else: self.sleep(seconds, "retry") @@ -243,8 +263,7 @@ def request(self, url, method="GET", session=None, _handle_429 = util.false - def wait(self, seconds=None, until=None, adjust=1.0, - reason="rate limit"): + def wait(self, seconds=None, until=None, adjust=1.0, reason="rate limit"): now = time.time() if seconds: @@ -266,13 +285,12 @@ def wait(self, seconds=None, until=None, adjust=1.0, if reason: t = datetime.datetime.fromtimestamp(until).time() - isotime = "{:02}:{:02}:{:02}".format(t.hour, t.minute, t.second) + isotime = f"{t.hour:02}:{t.minute:02}:{t.second:02}" self.log.info("Waiting until %s (%s)", isotime, reason) time.sleep(seconds) def sleep(self, seconds, reason): - self.log.debug("Sleeping %.2f seconds (%s)", - seconds, reason) + self.log.debug("Sleeping %.2f seconds (%s)", seconds, reason) time.sleep(seconds) def input(self, prompt, echo=True): @@ -291,8 +309,7 @@ def _check_input_allowed(self, prompt=""): if input is None: input = output.TTY_STDIN if not input: - raise exception.StopExtraction( - "User input required (%s)", prompt.strip(" :")) + raise exception.StopExtraction("User input required (%s)", prompt.strip(" :")) def _get_auth_info(self): """Return authentication information as (username, password) tuple""" @@ -346,7 +363,7 @@ def _init_session(self): ssl_options = ssl_ciphers = 0 # .netrc Authorization headers are alwsays disabled - session.trust_env = True if self.config("proxy-env", False) else False + session.trust_env = bool(self.config("proxy-env", False)) browser = self.config("browser") if browser is None: @@ -355,8 +372,7 @@ def _init_session(self): browser, _, platform = browser.lower().partition(":") if not platform or platform == "auto": - platform = ("Windows NT 10.0; Win64; x64" - if util.WINDOWS else "X11; Linux x86_64") + platform = "Windows NT 10.0; Win64; x64" if util.WINDOWS else "X11; Linux x86_64" elif platform == "windows": platform = "Windows NT 10.0; Win64; x64" elif platform == "linux": @@ -376,14 +392,15 @@ def _init_session(self): else: headers[key] = value - ssl_options |= (ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | - ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1) + ssl_options |= ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3 | ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 ssl_ciphers = SSL_CIPHERS[browser] else: useragent = self.config("user-agent") if useragent is None: - useragent = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64; " - "rv:128.0) Gecko/20100101 Firefox/128.0") + useragent = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; " + "rv:128.0) Gecko/20100101 Firefox/128.0" + ) elif useragent == "browser": useragent = _browser_useragent() headers["User-Agent"] = useragent @@ -430,8 +447,7 @@ def _init_session(self): ssl_options |= ssl.OP_NO_TLSv1_2 self.log.debug("TLS 1.2 disabled.") - adapter = _build_requests_adapter( - ssl_options, ssl_ciphers, source_address) + adapter = _build_requests_adapter(ssl_options, ssl_ciphers, source_address) session.mount("https://", adapter) session.mount("http://", adapter) @@ -477,6 +493,7 @@ def cookies_load(self, cookies_source): if cookies is None: from ..cookies import load_cookies + try: cookies = load_cookies(cookies_source) except Exception as exc: @@ -493,9 +510,10 @@ def cookies_load(self, cookies_source): else: self.log.warning( - "Expected 'dict', 'list', or 'str' value for 'cookies' " - "option, got '%s' (%s)", - cookies_source.__class__.__name__, cookies_source) + "Expected 'dict', 'list', or 'str' value for 'cookies' " "option, got '%s' (%s)", + cookies_source.__class__.__name__, + cookies_source, + ) def cookies_store(self): """Store the session's cookies in a cookies.txt file""" @@ -549,22 +567,22 @@ def cookies_check(self, cookies_names, domain=None): now = time.time() for cookie in self.cookies: - if cookie.name in names and ( - not domain or cookie.domain == domain): - + if cookie.name in names and (not domain or cookie.domain == domain): if cookie.expires: diff = int(cookie.expires - now) if diff <= 0: - self.log.warning( - "Cookie '%s' has expired", cookie.name) + self.log.warning("Cookie '%s' has expired", cookie.name) continue - elif diff <= 86400: + if diff <= 86400: hours = diff // 3600 self.log.warning( "Cookie '%s' will expire in less than %s hour%s", - cookie.name, hours + 1, "s" if hours else "") + cookie.name, + hours + 1, + "s" if hours else "", + ) names.discard(cookie.name) if not names: @@ -573,8 +591,7 @@ def cookies_check(self, cookies_names, domain=None): def _prepare_ddosguard_cookies(self): if not self.cookies.get("__ddg2", domain=self.cookies_domain): - self.cookies.set( - "__ddg2", util.generate_token(), domain=self.cookies_domain) + self.cookies.set("__ddg2", util.generate_token(), domain=self.cookies_domain) def _cache(self, func, maxage, keyarg=None): # return cache.DatabaseCacheDecorator(func, maxage, keyarg) @@ -585,6 +602,7 @@ def _cache_memory(self, func, maxage=None, keyarg=None): def _get_date_min_max(self, dmin=None, dmax=None): """Retrieve and parse 'date-min' and 'date-max' config values""" + def get(key, default): ts = self.config(key, default) if isinstance(ts, str): @@ -594,15 +612,13 @@ def get(key, default): self.log.warning("Unable to parse '%s': %s", key, exc) ts = default return ts + fmt = self.config("date-format", "%Y-%m-%dT%H:%M:%S") return get("date-min", dmin), get("date-max", dmax) def _dispatch_extractors(self, extractor_data, default=()): """ """ - extractors = { - data[0].subcategory: data - for data in extractor_data - } + extractors = {data[0].subcategory: data for data in extractor_data} include = self.config("include", default) or () if include == "all": @@ -642,30 +658,25 @@ def _dump_response(self, response, history=True): fname = "{:>02}_{}".format( Extractor._dump_index, - Extractor._dump_sanitize('_', response.url), + Extractor._dump_sanitize("_", response.url), ) - if util.WINDOWS: - path = os.path.abspath(fname)[:255] - else: - path = fname[:251] + path = os.path.abspath(fname)[:255] if util.WINDOWS else fname[:251] try: - with open(path + ".txt", 'wb') as fp: + with open(path + ".txt", "wb") as fp: util.dump_response( - response, fp, + response, + fp, headers=(self._write_pages in ("all", "ALL")), - hide_auth=(self._write_pages != "ALL") + hide_auth=(self._write_pages != "ALL"), ) - self.log.info("Writing '%s' response to '%s'", - response.url, path + ".txt") + self.log.info("Writing '%s' response to '%s'", response.url, path + ".txt") except Exception as e: - self.log.warning("Failed to dump HTTP request (%s: %s)", - e.__class__.__name__, e) + self.log.warning("Failed to dump HTTP request (%s: %s)", e.__class__.__name__, e) class GalleryExtractor(Extractor): - subcategory = "gallery" filename_fmt = "{category}_{gallery_id}_{num:>03}.{extension}" directory_fmt = ("{category}", "{gallery_id} {title}") @@ -680,8 +691,7 @@ def items(self): self.login() if self.gallery_url: - page = self.request( - self.gallery_url, notfound=self.subcategory).text + page = self.request(self.gallery_url, notfound=self.subcategory).text else: page = None @@ -693,7 +703,7 @@ def items(self): images = util.enumerate_reversed(imgs, 1, data["count"]) else: images = zip( - range(1, data["count"]+1), + range(1, data["count"] + 1), imgs, ) else: @@ -728,20 +738,18 @@ def images(self, page): class ChapterExtractor(GalleryExtractor): - subcategory = "chapter" directory_fmt = ( - "{category}", "{manga}", - "{volume:?v/ />02}c{chapter:>03}{chapter_minor:?//}{title:?: //}") - filename_fmt = ( - "{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}") - archive_fmt = ( - "{manga}_{chapter}{chapter_minor}_{page}") + "{category}", + "{manga}", + "{volume:?v/ />02}c{chapter:>03}{chapter_minor:?//}{title:?: //}", + ) + filename_fmt = "{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}" + archive_fmt = "{manga}_{chapter}{chapter_minor}_{page}" enum = "page" class MangaExtractor(Extractor): - subcategory = "manga" categorytransfer = True chapterclass = None @@ -773,7 +781,7 @@ def chapters(self, page): """Return a list of all (chapter-url, metadata)-tuples""" -class AsynchronousMixin(): +class AsynchronousMixin: """Run info extraction in a separate thread""" def __iter__(self): @@ -821,7 +829,7 @@ def _init_category(self): for index, group in enumerate(self.groups): if group is not None: if index: - self.category, self.root, info = self.instances[index-1] + self.category, self.root, info = self.instances[index - 1] if not self.root: self.root = text.root_from_url(self.match.group(0)) self.config_instance = info.get @@ -848,7 +856,7 @@ def update(cls, instances): pattern = info.get("pattern") if not pattern: - pattern = re.escape(root[root.index(":") + 3:]) + pattern = re.escape(root[root.index(":") + 3 :]) pattern_list.append(pattern + "()") return ( @@ -858,7 +866,6 @@ def update(cls, instances): class RequestsAdapter(HTTPAdapter): - def __init__(self, ssl_context=None, source_address=None): self.ssl_context = ssl_context self.source_address = source_address @@ -884,7 +891,8 @@ def _build_requests_adapter(ssl_options, ssl_ciphers, source_address): if ssl_options or ssl_ciphers: ssl_context = urllib3.connection.create_urllib3_context( - options=ssl_options or None, ciphers=ssl_ciphers) + options=ssl_options or None, ciphers=ssl_ciphers + ) if not requests.__version__ < "2.32": # https://github.com/psf/requests/pull/6731 ssl_context.load_default_certs() @@ -892,16 +900,15 @@ def _build_requests_adapter(ssl_options, ssl_ciphers, source_address): else: ssl_context = None - adapter = _adapter_cache[key] = RequestsAdapter( - ssl_context, source_address) + adapter = _adapter_cache[key] = RequestsAdapter(ssl_context, source_address) return adapter @cache.cache(maxage=86400) def _browser_useragent(): """Get User-Agent header from default browser""" - import webbrowser import socket + import webbrowser server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -933,10 +940,12 @@ def _browser_useragent(): HTTP_HEADERS = { "firefox": ( - ("User-Agent", "Mozilla/5.0 ({}; " - "rv:128.0) Gecko/20100101 Firefox/128.0"), - ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9," - "image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8"), + ("User-Agent", "Mozilla/5.0 ({}; " "rv:128.0) Gecko/20100101 Firefox/128.0"), + ( + "Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9," + "image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8", + ), ("Accept-Language", "en-US,en;q=0.5"), ("Accept-Encoding", None), ("Referer", None), @@ -951,11 +960,17 @@ def _browser_useragent(): "chrome": ( ("Connection", "keep-alive"), ("Upgrade-Insecure-Requests", "1"), - ("User-Agent", "Mozilla/5.0 ({}) AppleWebKit/537.36 (KHTML, " - "like Gecko) Chrome/111.0.0.0 Safari/537.36"), - ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9," - "image/avif,image/webp,image/apng,*/*;q=0.8," - "application/signed-exchange;v=b3;q=0.7"), + ( + "User-Agent", + "Mozilla/5.0 ({}) AppleWebKit/537.36 (KHTML, " + "like Gecko) Chrome/111.0.0.0 Safari/537.36", + ), + ( + "Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9," + "image/avif,image/webp,image/apng,*/*;q=0.8," + "application/signed-exchange;v=b3;q=0.7", + ), ("Referer", None), ("Sec-Fetch-Site", "same-origin"), ("Sec-Fetch-Mode", "no-cors"), @@ -1008,10 +1023,8 @@ def _browser_useragent(): # disable Basic Authorization header injection from .netrc data -try: +with suppress(Exception): requests.sessions.get_netrc_auth = lambda _: None -except Exception: - pass # detect brotli support try: @@ -1030,6 +1043,7 @@ def _browser_useragent(): if action: try: import warnings + warnings.simplefilter(action, urllib3.exceptions.HTTPWarning) except Exception: pass diff --git a/gallery_dl/extractor/cyberdrop.py b/gallery_dl/extractor/cyberdrop.py index a514696b0b..f3126aefd7 100644 --- a/gallery_dl/extractor/cyberdrop.py +++ b/gallery_dl/extractor/cyberdrop.py @@ -1,14 +1,12 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://cyberdrop.me/""" +from .. import text from . import lolisafe from .common import Message -from .. import text class CyberdropAlbumExtractor(lolisafe.LolisafeAlbumExtractor): @@ -29,25 +27,27 @@ def items(self): yield Message.Url, file["url"], file def fetch_album(self, album_id): - url = "{}/a/{}".format(self.root, album_id) + url = f"{self.root}/a/{album_id}" page = self.request(url).text extr = text.extract_from(page) desc = extr('property="og:description" content="', '"') - if desc.startswith("A privacy-focused censorship-resistant file " - "sharing platform free for everyone."): + if desc.startswith( + "A privacy-focused censorship-resistant file " "sharing platform free for everyone." + ): desc = "" extr('id="title"', "") album = { - "album_id" : self.album_id, - "album_name" : text.unescape(extr('title="', '"')), - "album_size" : text.parse_bytes(extr( - '<p class="title">', "B")), - "date" : text.parse_datetime(extr( - '<p class="title">', '<'), "%d.%m.%Y"), - "description": text.unescape(text.unescape( # double - desc.rpartition(" [R")[0])), + "album_id": self.album_id, + "album_name": text.unescape(extr('title="', '"')), + "album_size": text.parse_bytes(extr('<p class="title">', "B")), + "date": text.parse_datetime(extr('<p class="title">', "<"), "%d.%m.%Y"), + "description": text.unescape( + text.unescape( # double + desc.rpartition(" [R")[0] + ) + ), } file_ids = list(text.extract_iter(page, 'id="file" href="/f/', '"')) @@ -57,13 +57,12 @@ def fetch_album(self, album_id): def _extract_files(self, file_ids): for file_id in file_ids: try: - url = "{}/api/file/info/{}".format(self.root_api, file_id) + url = f"{self.root_api}/api/file/info/{file_id}" file = self.request(url).json() auth = self.request(file["auth_url"]).json() file["url"] = auth["url"] except Exception as exc: - self.log.warning("%s (%s: %s)", - file_id, exc.__class__.__name__, exc) + self.log.warning("%s (%s: %s)", file_id, exc.__class__.__name__, exc) continue yield file diff --git a/gallery_dl/extractor/danbooru.py b/gallery_dl/extractor/danbooru.py index 1746647e9b..b8ae5f01ee 100644 --- a/gallery_dl/extractor/danbooru.py +++ b/gallery_dl/extractor/danbooru.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,17 @@ """Extractors for https://danbooru.donmai.us/ and other Danbooru instances""" -from .common import BaseExtractor, Message -from .. import text, util import datetime +from .. import text +from .. import util +from .common import BaseExtractor +from .common import Message + class DanbooruExtractor(BaseExtractor): """Base class for danbooru extractors""" + basecategory = "Danbooru" filename_fmt = "{category}_{id}_{filename}.{extension}" page_limit = 1000 @@ -29,7 +31,7 @@ def _init(self): threshold = self.config("threshold") if isinstance(threshold, int): - self.threshold = 1 if threshold < 1 else threshold + self.threshold = max(threshold, 1) else: self.threshold = self.per_page @@ -58,7 +60,6 @@ def items(self): data = self.metadata() for post in self.posts(): - try: url = post["file_url"] except KeyError: @@ -69,33 +70,29 @@ def items(self): continue text.nameext_from_url(url, post) - post["date"] = text.parse_datetime( - post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + post["date"] = text.parse_datetime(post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") - post["tags"] = ( - post["tag_string"].split(" ") - if post["tag_string"] else ()) + post["tags"] = post["tag_string"].split(" ") if post["tag_string"] else () post["tags_artist"] = ( - post["tag_string_artist"].split(" ") - if post["tag_string_artist"] else ()) + post["tag_string_artist"].split(" ") if post["tag_string_artist"] else () + ) post["tags_character"] = ( - post["tag_string_character"].split(" ") - if post["tag_string_character"] else ()) + post["tag_string_character"].split(" ") if post["tag_string_character"] else () + ) post["tags_copyright"] = ( - post["tag_string_copyright"].split(" ") - if post["tag_string_copyright"] else ()) + post["tag_string_copyright"].split(" ") if post["tag_string_copyright"] else () + ) post["tags_general"] = ( - post["tag_string_general"].split(" ") - if post["tag_string_general"] else ()) + post["tag_string_general"].split(" ") if post["tag_string_general"] else () + ) post["tags_meta"] = ( - post["tag_string_meta"].split(" ") - if post["tag_string_meta"] else ()) + post["tag_string_meta"].split(" ") if post["tag_string_meta"] else () + ) if post["extension"] == "zip": if self.ugoira: post["_ugoira_original"] = False - post["_ugoira_frame_data"] = post["frames"] = \ - self._ugoira_frames(post) + post["_ugoira_frame_data"] = post["frames"] = self._ugoira_frames(post) post["_http_adjust_extension"] = False else: url = post["large_file_url"] @@ -128,14 +125,12 @@ def _pagination(self, endpoint, params, prefix=None): if posts: if self.includes: params_meta = { - "only" : self.includes, + "only": self.includes, "limit": len(posts), - "tags" : "id:" + ",".join(str(p["id"]) for p in posts), + "tags": "id:" + ",".join(str(p["id"]) for p in posts), } data = { - meta["id"]: meta - for meta in self.request( - url, params=params_meta).json() + meta["id"]: meta for meta in self.request(url, params=params_meta).json() } for post in posts: post.update(data[post["id"]]) @@ -157,40 +152,41 @@ def _pagination(self, endpoint, params, prefix=None): first = False def _ugoira_frames(self, post): - data = self.request("{}/posts/{}.json?only=media_metadata".format( - self.root, post["id"]) + data = self.request( + "{}/posts/{}.json?only=media_metadata".format(self.root, post["id"]) ).json()["media_metadata"]["metadata"] ext = data["ZIP:ZipFileName"].rpartition(".")[2] fmt = ("{:>06}." + ext).format delays = data["Ugoira:FrameDelays"] - return [{"file": fmt(index), "delay": delay} - for index, delay in enumerate(delays)] - - -BASE_PATTERN = DanbooruExtractor.update({ - "danbooru": { - "root": None, - "pattern": r"(?:(?:danbooru|hijiribe|sonohara|safebooru)\.donmai\.us" - r"|donmai\.moe)", - }, - "atfbooru": { - "root": "https://booru.allthefallen.moe", - "pattern": r"booru\.allthefallen\.moe", - }, - "aibooru": { - "root": None, - "pattern": r"(?:safe\.)?aibooru\.online", - }, - "booruvar": { - "root": "https://booru.borvar.art", - "pattern": r"booru\.borvar\.art", - }, -}) + return [{"file": fmt(index), "delay": delay} for index, delay in enumerate(delays)] + + +BASE_PATTERN = DanbooruExtractor.update( + { + "danbooru": { + "root": None, + "pattern": r"(?:(?:danbooru|hijiribe|sonohara|safebooru)\.donmai\.us" r"|donmai\.moe)", + }, + "atfbooru": { + "root": "https://booru.allthefallen.moe", + "pattern": r"booru\.allthefallen\.moe", + }, + "aibooru": { + "root": None, + "pattern": r"(?:safe\.)?aibooru\.online", + }, + "booruvar": { + "root": "https://booru.borvar.art", + "pattern": r"booru\.borvar\.art", + }, + } +) class DanbooruTagExtractor(DanbooruExtractor): """Extractor for danbooru posts from tag searches""" + subcategory = "tag" directory_fmt = ("{category}", "{search_tags}") archive_fmt = "t_{search_tags}_{id}" @@ -215,8 +211,7 @@ def posts(self): prefix = "b" else: prefix = None - elif tag.startswith( - ("id:", "md5", "ordfav:", "ordfavgroup:", "ordpool:")): + elif tag.startswith(("id:", "md5", "ordfav:", "ordfavgroup:", "ordpool:")): prefix = None break @@ -225,6 +220,7 @@ def posts(self): class DanbooruPoolExtractor(DanbooruExtractor): """Extractor for posts from danbooru pools""" + subcategory = "pool" directory_fmt = ("{category}", "pool", "{pool[id]} {pool[name]}") archive_fmt = "p_{pool[id]}_{id}" @@ -236,7 +232,7 @@ def __init__(self, match): self.pool_id = match.group(match.lastindex) def metadata(self): - url = "{}/pools/{}.json".format(self.root, self.pool_id) + url = f"{self.root}/pools/{self.pool_id}.json" pool = self.request(url).json() pool["name"] = pool["name"].replace("_", " ") self.post_ids = pool.pop("post_ids", ()) @@ -249,6 +245,7 @@ def posts(self): class DanbooruPostExtractor(DanbooruExtractor): """Extractor for single danbooru posts""" + subcategory = "post" archive_fmt = "{id}" pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)" @@ -259,7 +256,7 @@ def __init__(self, match): self.post_id = match.group(match.lastindex) def posts(self): - url = "{}/posts/{}.json".format(self.root, self.post_id) + url = f"{self.root}/posts/{self.post_id}.json" post = self.request(url).json() if self.includes: params = {"only": self.includes} @@ -269,6 +266,7 @@ def posts(self): class DanbooruPopularExtractor(DanbooruExtractor): """Extractor for popular images from danbooru""" + subcategory = "popular" directory_fmt = ("{category}", "popular", "{scale}", "{date}") archive_fmt = "P_{scale[0]}_{date}_{id}" diff --git a/gallery_dl/extractor/desktopography.py b/gallery_dl/extractor/desktopography.py index 35bb2992f2..eb684a4d95 100644 --- a/gallery_dl/extractor/desktopography.py +++ b/gallery_dl/extractor/desktopography.py @@ -1,26 +1,27 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://desktopography.net/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?desktopography\.net" class DesktopographyExtractor(Extractor): """Base class for desktopography extractors""" + category = "desktopography" archive_fmt = "{filename}" root = "https://desktopography.net" class DesktopographySiteExtractor(DesktopographyExtractor): - """Extractor for all desktopography exhibitions """ + """Extractor for all desktopography exhibitions""" + subcategory = "site" pattern = BASE_PATTERN + r"/$" example = "https://desktopography.net/" @@ -30,16 +31,15 @@ def items(self): data = {"_extractor": DesktopographyExhibitionExtractor} for exhibition_year in text.extract_iter( - page, - '<a href="https://desktopography.net/exhibition-', - '/">'): - + page, '<a href="https://desktopography.net/exhibition-', '/">' + ): url = self.root + "/exhibition-" + exhibition_year + "/" yield Message.Queue, url, data class DesktopographyExhibitionExtractor(DesktopographyExtractor): """Extractor for a yearly desktopography exhibition""" + subcategory = "exhibition" pattern = BASE_PATTERN + r"/exhibition-([^/?#]+)/" example = "https://desktopography.net/exhibition-2020/" @@ -49,7 +49,7 @@ def __init__(self, match): self.year = match.group(1) def items(self): - url = "{}/exhibition-{}/".format(self.root, self.year) + url = f"{self.root}/exhibition-{self.year}/" base_entry_url = "https://desktopography.net/portfolios/" page = self.request(url).text @@ -59,16 +59,15 @@ def items(self): } for entry_url in text.extract_iter( - page, - '<a class="overlay-background" href="' + base_entry_url, - '">'): - + page, '<a class="overlay-background" href="' + base_entry_url, '">' + ): url = base_entry_url + entry_url yield Message.Queue, url, data class DesktopographyEntryExtractor(DesktopographyExtractor): """Extractor for all resolutions of a desktopography wallpaper""" + subcategory = "entry" pattern = BASE_PATTERN + r"/portfolios/([\w-]+)" example = "https://desktopography.net/portfolios/NAME/" @@ -78,18 +77,15 @@ def __init__(self, match): self.entry = match.group(1) def items(self): - url = "{}/portfolios/{}".format(self.root, self.entry) + url = f"{self.root}/portfolios/{self.entry}" page = self.request(url).text entry_data = {"entry": self.entry} yield Message.Directory, entry_data for image_data in text.extract_iter( - page, - '<a target="_blank" href="https://desktopography.net', - '">'): - - path, _, filename = image_data.partition( - '" class="wallpaper-button" download="') + page, '<a target="_blank" href="https://desktopography.net', '">' + ): + path, _, filename = image_data.partition('" class="wallpaper-button" download="') text.nameext_from_url(filename, entry_data) yield Message.Url, self.root + path, entry_data diff --git a/gallery_dl/extractor/deviantart.py b/gallery_dl/extractor/deviantart.py index ea3f13df32..4f4cc27538 100644 --- a/gallery_dl/extractor/deviantart.py +++ b/gallery_dl/extractor/deviantart.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,14 +6,20 @@ """Extractors for https://www.deviantart.com/""" -from .common import Extractor, Message -from .. import text, util, exception -from ..cache import cache, memcache +import binascii import collections import mimetypes -import binascii -import time import re +import time +from typing import AnyStr + +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from ..cache import memcache +from .common import Extractor +from .common import Message BASE_PATTERN = ( r"(?:https?://)?(?:" @@ -27,6 +31,7 @@ class DeviantartExtractor(Extractor): """Base class for deviantart extractors""" + category = "deviantart" root = "https://www.deviantart.com" directory_fmt = ("{category}", "{username}") @@ -35,12 +40,12 @@ class DeviantartExtractor(Extractor): cookies_names = ("auth", "auth_secure", "userinfo") _last_request = 0 - def __init__(self, match): + def __init__(self, match: re.Match[AnyStr]) -> None: Extractor.__init__(self, match) - self.user = (match.group(1) or match.group(2) or "").lower() + self.user: str = str(match.group(1) or match.group(2) or "").lower() self.offset = 0 - def _init(self): + def _init(self) -> None: self.jwt = self.config("jwt", False) self.flat = self.config("flat", True) self.extra = self.config("extra", False) @@ -68,11 +73,10 @@ def _init(self): self.quality = "-fullview.png?" self.quality_sub = re.compile(r"-fullview\.[a-z0-9]+\?").sub else: - self.quality = ",q_{}".format(self.quality) + self.quality = f",q_{self.quality}" self.quality_sub = re.compile(r",q_\d+").sub - if isinstance(self.original, str) and \ - self.original.lower().startswith("image"): + if isinstance(self.original, str) and self.original.lower().startswith("image"): self.original = True self._update_content = self._update_content_image else: @@ -96,8 +100,7 @@ def request(self, url, **kwargs): kwargs["fatal"] = False while True: response = Extractor.request(self, url, **kwargs) - if response.status_code != 403 or \ - b"Request blocked." not in response.content: + if response.status_code != 403 or b"Request blocked." not in response.content: return response self.wait(seconds=300, reason="CloudFront block") @@ -138,14 +141,12 @@ def items(self): if deviation["is_deleted"]: # prevent crashing in case the deviation really is # deleted - self.log.debug( - "Skipping %s (deleted)", deviation["deviationid"]) + self.log.debug("Skipping %s (deleted)", deviation["deviationid"]) continue tier_access = deviation.get("tier_access") if tier_access == "locked": - self.log.debug( - "Skipping %s (access locked)", deviation["deviationid"]) + self.log.debug("Skipping %s (access locked)", deviation["deviationid"]) continue if "premium_folder_data" in deviation: @@ -166,9 +167,8 @@ def items(self): deviation["is_original"] = True yield self.commit(deviation, content) - if "videos" in deviation and deviation["videos"]: - video = max(deviation["videos"], - key=lambda x: text.parse_int(x["quality"][:-1])) + if deviation.get("videos"): + video = max(deviation["videos"], key=lambda x: text.parse_int(x["quality"][:-1])) deviation["is_original"] = False yield self.commit(deviation, video) @@ -189,12 +189,11 @@ def items(self): user = comment["user"] name = user["username"].lower() if user["usericon"] == DEFAULT_AVATAR: - self.log.debug( - "Skipping avatar of '%s' (default)", name) + self.log.debug("Skipping avatar of '%s' (default)", name) continue _user_details.update(name, user) - url = "{}/{}/avatar/".format(self.root, name) + url = f"{self.root}/{name}/avatar/" comment["_extractor"] = DeviantartAvatarExtractor yield Message.Queue, url, comment @@ -204,8 +203,7 @@ def items(self): if self.previews_images: yield self.commit(deviation, preview) else: - mtype = mimetypes.guess_type( - "a." + deviation["extension"], False)[0] + mtype = mimetypes.guess_type("a." + deviation["extension"], False)[0] if mtype and not mtype.startswith("image/"): yield self.commit(deviation, preview) del deviation["is_preview"] @@ -217,11 +215,16 @@ def items(self): # /developers/http/v1/20210526/object/editor_text # the value of "features" is a JSON string with forward # slashes escaped - text_content = \ - deviation["text_content"]["body"]["features"].replace( - "\\/", "/") if "text_content" in deviation else None - for txt in (text_content, deviation.get("description"), - deviation.get("_journal")): + text_content = ( + deviation["text_content"]["body"]["features"].replace("\\/", "/") + if "text_content" in deviation + else None + ) + for txt in ( + text_content, + deviation.get("description"), + deviation.get("_journal"), + ): if txt is None: continue for match in DeviantartStashExtractor.pattern.finditer(txt): @@ -236,16 +239,17 @@ def prepare(self, deviation): """Adjust the contents of a Deviation-object""" if "index" not in deviation: try: - if deviation["url"].startswith(( - "https://www.deviantart.com/stash/", "https://sta.sh", - )): + if deviation["url"].startswith( + ( + "https://www.deviantart.com/stash/", + "https://sta.sh", + ) + ): filename = deviation["content"]["src"].split("/")[5] deviation["index_base36"] = filename.partition("-")[0][1:] - deviation["index"] = id_from_base36( - deviation["index_base36"]) + deviation["index"] = id_from_base36(deviation["index_base36"]) else: - deviation["index"] = text.parse_int( - deviation["url"].rpartition("-")[2]) + deviation["index"] = text.parse_int(deviation["url"].rpartition("-")[2]) except KeyError: deviation["index"] = 0 deviation["index_base36"] = "0" @@ -259,24 +263,27 @@ def prepare(self, deviation): deviation["username"] = deviation["author"]["username"] deviation["_username"] = deviation["username"].lower() - deviation["published_time"] = text.parse_int( - deviation["published_time"]) - deviation["date"] = text.parse_timestamp( - deviation["published_time"]) + deviation["published_time"] = text.parse_int(deviation["published_time"]) + deviation["date"] = text.parse_timestamp(deviation["published_time"]) if self.comments: deviation["comments"] = ( self._extract_comments(deviation["deviationid"], "deviation") - if deviation["stats"]["comments"] else () + if deviation["stats"]["comments"] + else () ) # filename metadata sub = re.compile(r"\W").sub - deviation["filename"] = "".join(( - sub("_", deviation["title"].lower()), "_by_", - sub("_", deviation["author"]["username"].lower()), "-d", - deviation["index_base36"], - )) + deviation["filename"] = "".join( + ( + sub("_", deviation["title"].lower()), + "_by_", + sub("_", deviation["author"]["username"].lower()), + "-d", + deviation["index_base36"], + ) + ) @staticmethod def commit(deviation, target): @@ -287,7 +294,7 @@ def commit(deviation, target): deviation["target"] = target deviation["extension"] = target["extension"] = text.ext_from_url(name) if "is_original" not in deviation: - deviation["is_original"] = ("/v1/" not in url) + deviation["is_original"] = "/v1/" not in url return Message.Url, url, deviation def _commit_journal_html(self, deviation, journal): @@ -312,7 +319,9 @@ def _commit_journal_html(self, deviation, journal): if html.find('<div class="boxtop journaltop">', 0, 250) != -1: needle = '<div class="boxtop journaltop">' header = HEADER_CUSTOM_TEMPLATE.format( - title=title, url=url, date=deviation["date"], + title=title, + url=url, + date=deviation["date"], ) else: needle = '<div usr class="gr">' @@ -321,7 +330,7 @@ def _commit_journal_html(self, deviation, journal): header = HEADER_TEMPLATE.format( title=title, url=url, - userurl="{}/{}/".format(self.root, urlname), + userurl=f"{self.root}/{urlname}/", username=username, date=deviation["date"], ) @@ -331,8 +340,7 @@ def _commit_journal_html(self, deviation, journal): else: html = JOURNAL_TEMPLATE_HTML_EXTRA.format(header, html) - html = JOURNAL_TEMPLATE_HTML.format( - title=title, html=html, shadow=shadow, css=css, cls=cls) + html = JOURNAL_TEMPLATE_HTML.format(title=title, html=html, shadow=shadow, css=css, cls=cls) deviation["extension"] = "htm" return Message.Url, html, deviation @@ -345,8 +353,7 @@ def _commit_journal_text(self, deviation, journal): html = html.partition("</style>")[2] head, _, tail = html.rpartition("<script") content = "\n".join( - text.unescape(text.remove_html(txt)) - for txt in (head or tail).split("<br />") + text.unescape(text.remove_html(txt)) for txt in (head or tail).split("<br />") ) txt = JOURNAL_TEMPLATE_TEXT.format( title=deviation["title"], @@ -373,18 +380,24 @@ def _extract_journal(self, deviation): html = text.extr( page, "<h2>Literature Text</h2></span><div>", - "</div></section></div></div>") + "</div></section></div></div>", + ) if html: return {"html": html} - self.log.debug("%s: Failed to extract journal HTML from webpage. " - "Falling back to __INITIAL_STATE__ markup.", - deviation["index"]) + self.log.debug( + "%s: Failed to extract journal HTML from webpage. " + "Falling back to __INITIAL_STATE__ markup.", + deviation["index"], + ) # parse __INITIAL_STATE__ as fallback - state = util.json_loads(text.extr( - page, 'window.__INITIAL_STATE__ = JSON.parse("', '");') - .replace("\\\\", "\\").replace("\\'", "'").replace('\\"', '"')) + state = util.json_loads( + text.extr(page, 'window.__INITIAL_STATE__ = JSON.parse("', '");') + .replace("\\\\", "\\") + .replace("\\'", "'") + .replace('\\"', '"') + ) deviations = state["@@entities"]["deviation"] content = deviations.popitem()[1]["textContent"] @@ -409,17 +422,14 @@ def _textcontent_to_html(self, deviation, content): return self._tiptap_to_html(markup) except Exception as exc: self.log.debug("", exc_info=exc) - self.log.error("%s: '%s: %s'", deviation["index"], - exc.__class__.__name__, exc) + self.log.error("%s: '%s: %s'", deviation["index"], exc.__class__.__name__, exc) - self.log.warning("%s: Unsupported '%s' markup.", - deviation["index"], html["type"]) + self.log.warning("%s: Unsupported '%s' markup.", deviation["index"], html["type"]) def _tiptap_to_html(self, markup): html = [] - html.append('<div data-editor-viewer="1" ' - 'class="_83r8m _2CKTq _3NjDa mDnFl">') + html.append('<div data-editor-viewer="1" ' 'class="_83r8m _2CKTq _3NjDa mDnFl">') data = util.json_loads(markup) for block in data["document"]["content"]: self._tiptap_process_content(html, block) @@ -466,7 +476,7 @@ def _tiptap_process_content(self, html, content): html.append(user.lower()) html.append('" data-da-type="da-mention" data-user="">@<!-- -->') html.append(user) - html.append('</a>') + html.append("</a>") else: self.log.warning("Unsupported content type '%s'", type) @@ -506,9 +516,11 @@ def _tiptap_process_deviation(self, html, content): media = dev.get("media") or () html.append('<div class="jjNX2">') - html.append('<figure class="Qf-HY" data-da-type="da-deviation" ' - 'data-deviation="" ' - 'data-width="" data-link="" data-alignment="center">') + html.append( + '<figure class="Qf-HY" data-da-type="da-deviation" ' + 'data-deviation="" ' + 'data-width="" data-link="" data-alignment="center">' + ) if "baseUri" in media: url, formats = self._eclipse_media(media) @@ -516,9 +528,11 @@ def _tiptap_process_deviation(self, html, content): html.append('<a href="') html.append(text.escape(dev["url"])) - html.append('" class="_3ouD5" style="margin:0 auto;display:flex;' - 'align-items:center;justify-content:center;' - 'overflow:hidden;width:780px;height:') + html.append( + '" class="_3ouD5" style="margin:0 auto;display:flex;' + "align-items:center;justify-content:center;" + "overflow:hidden;width:780px;height:" + ) html.append(str(780 * full["h"] / full["w"])) html.append('px">') @@ -536,7 +550,7 @@ def _tiptap_process_deviation(self, html, content): html.append(text.escape(dev["url"])) html.append('" class="_3ouD5">') - html.append('''\ + html.append("""\ <section class="Q91qI aG7Yi" style="width:350px;height:313px">\ <div class="_16ECM _1xMkk" aria-hidden="true">\ <svg height="100%" viewBox="0 0 15 12" preserveAspectRatio="xMidYMin slice" \ @@ -548,13 +562,13 @@ def _tiptap_process_deviation(self, html, content): </linearGradient>\ <text class="_2uqbc" fill="url(#app-root-3)" text-anchor="end" x="15" y="11">J\ </text></svg></div><div class="_1xz9u">Literature</div><h3 class="_2WvKD">\ -''') +""") html.append(text.escape(dev["title"])) html.append('</h3><div class="_2CPLm">') html.append(text.escape(dev["textContent"]["excerpt"])) - html.append('</div></section></a></div>') + html.append("</div></section></a></div>") - html.append('</figure></div>') + html.append("</figure></div>") def _extract_content(self, deviation): content = deviation["content"] @@ -571,23 +585,21 @@ def _extract_content(self, deviation): if self.intermediary and deviation["index"] <= 790677560: # https://github.com/r888888888/danbooru/issues/4069 intermediary, count = re.subn( - r"(/f/[^/]+/[^/]+)/v\d+/.*", - r"/intermediary\1", content["src"], 1) + r"(/f/[^/]+/[^/]+)/v\d+/.*", r"/intermediary\1", content["src"], 1 + ) if count: deviation["is_original"] = False deviation["_fallback"] = (content["src"],) content["src"] = intermediary if self.quality: - content["src"] = self.quality_sub( - self.quality, content["src"], 1) + content["src"] = self.quality_sub(self.quality, content["src"], 1) return content @staticmethod def _find_folder(folders, name, uuid): if uuid.isdecimal(): - match = re.compile(name.replace( - "-", r"[^a-z0-9]+") + "$", re.IGNORECASE).match + match = re.compile(name.replace("-", r"[^a-z0-9]+") + "$", re.IGNORECASE).match for folder in folders: if match(folder["name"]): return folder @@ -598,17 +610,14 @@ def _find_folder(folders, name, uuid): raise exception.NotFoundError("folder") def _folder_urls(self, folders, category, extractor): - base = "{}/{}/{}/".format(self.root, self.user, category) + base = f"{self.root}/{self.user}/{category}/" for folder in folders: folder["_extractor"] = extractor url = "{}{}/{}".format(base, folder["folderid"], folder["name"]) yield url, folder def _update_content_default(self, deviation, content): - if "premium_folder_data" in deviation or deviation.get("is_mature"): - public = False - else: - public = None + public = False if "premium_folder_data" in deviation or deviation.get("is_mature") else None data = self.api.deviation_download(deviation["deviationid"], public) content.update(data) @@ -637,19 +646,18 @@ def _update_token(self, deviation, content): # header = b'{"typ":"JWT","alg":"none"}' payload = ( - b'{"sub":"urn:app:","iss":"urn:app:","obj":[[{"path":"/f/' + - url.partition("/f/")[2].encode() + - b'"}]],"aud":["urn:service:file.download"]}' + b'{"sub":"urn:app:","iss":"urn:app:","obj":[[{"path":"/f/' + + url.partition("/f/")[2].encode() + + b'"}]],"aud":["urn:service:file.download"]}' ) deviation["_fallback"] = (content["src"],) deviation["is_original"] = True - content["src"] = ( - "{}?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.{}.".format( - url, - # base64 of 'header' is precomputed as 'eyJ0eX...' - # binascii.b2a_base64(header).rstrip(b"=\n").decode(), - binascii.b2a_base64(payload).rstrip(b"=\n").decode()) + content["src"] = "{}?token=eyJ0eXAiOiJKV1QiLCJhbGciOiJub25lIn0.{}.".format( + url, + # base64 of 'header' is precomputed as 'eyJ0eX...' + # binascii.b2a_base64(header).rstrip(b"=\n").decode(), + binascii.b2a_base64(payload).rstrip(b"=\n").decode(), ) def _extract_comments(self, target_id, target_type="deviation"): @@ -657,8 +665,7 @@ def _extract_comments(self, target_id, target_type="deviation"): comment_ids = [None] while comment_ids: - comments = self.api.comments( - target_id, target_type, comment_ids.pop()) + comments = self.api.comments(target_id, target_type, comment_ids.pop()) if results: results.extend(comments) @@ -690,8 +697,7 @@ def _fetch_premium(self, deviation): pass if not self.api.refresh_token_key: - self.log.warning( - "Unable to access premium content (no refresh-token)") + self.log.warning("Unable to access premium content (no refresh-token)") self._fetch_premium = lambda _: None return None @@ -702,28 +708,25 @@ def _fetch_premium(self, deviation): # premium_folder_data is no longer present when user has access (#5063) has_access = ("premium_folder_data" not in dev) or folder["has_access"] - if not has_access and folder["type"] == "watchers" and \ - self.config("auto-watch"): + if not has_access and folder["type"] == "watchers" and self.config("auto-watch"): if self.unwatch is not None: self.unwatch.append(username) if self.api.user_friends_watch(username): has_access = True - self.log.info( - "Watching %s for premium folder access", username) + self.log.info("Watching %s for premium folder access", username) else: self.log.warning( - "Error when trying to watch %s. " - "Try again with a new refresh-token", username) + "Error when trying to watch %s. " "Try again with a new refresh-token", + username, + ) if has_access: self.log.info("Fetching premium folder data") else: - self.log.warning("Unable to access premium content (type: %s)", - folder["type"]) + self.log.warning("Unable to access premium content (type: %s)", folder["type"]) cache = self._premium_cache - for dev in self.api.gallery( - username, folder["gallery_id"], public=False): + for dev in self.api.gallery(username, folder["gallery_id"], public=False): cache[dev["deviationid"]] = dev if has_access else None return cache[deviation["deviationid"]] @@ -734,12 +737,11 @@ def _unwatch_premium(self): self.api.user_friends_unwatch(username) def _eclipse_media(self, media, format="preview"): - url = [media["baseUri"], ] + url = [ + media["baseUri"], + ] - formats = { - fmt["t"]: fmt - for fmt in media["types"] - } + formats = {fmt["t"]: fmt for fmt in media["types"]} tokens = media["token"] if len(tokens) == 1: @@ -752,7 +754,7 @@ def _eclipse_media(self, media, format="preview"): def _eclipse_to_oauth(self, eclipse_api, deviations): for obj in deviations: - deviation = obj["deviation"] if "deviation" in obj else obj + deviation = obj.get("deviation", obj) deviation_uuid = eclipse_api.deviation_extended_fetch( deviation["deviationId"], deviation["author"]["username"], @@ -763,6 +765,7 @@ def _eclipse_to_oauth(self, eclipse_api, deviations): class DeviantartUserExtractor(DeviantartExtractor): """Extractor for an artist's user profile""" + subcategory = "user" pattern = BASE_PATTERN + r"/?$" example = "https://www.deviantart.com/USER" @@ -773,23 +776,28 @@ def initialize(self): skip = Extractor.skip def items(self): - base = "{}/{}/".format(self.root, self.user) - return self._dispatch_extractors(( - (DeviantartAvatarExtractor , base + "avatar"), - (DeviantartBackgroundExtractor, base + "banner"), - (DeviantartGalleryExtractor , base + "gallery"), - (DeviantartScrapsExtractor , base + "gallery/scraps"), - (DeviantartJournalExtractor , base + "posts"), - (DeviantartStatusExtractor , base + "posts/statuses"), - (DeviantartFavoriteExtractor , base + "favourites"), - ), ("gallery",)) + base = f"{self.root}/{self.user}/" + return self._dispatch_extractors( + ( + (DeviantartAvatarExtractor, base + "avatar"), + (DeviantartBackgroundExtractor, base + "banner"), + (DeviantartGalleryExtractor, base + "gallery"), + (DeviantartScrapsExtractor, base + "gallery/scraps"), + (DeviantartJournalExtractor, base + "posts"), + (DeviantartStatusExtractor, base + "posts/statuses"), + (DeviantartFavoriteExtractor, base + "favourites"), + ), + ("gallery",), + ) ############################################################################### # OAuth ####################################################################### + class DeviantartGalleryExtractor(DeviantartExtractor): """Extractor for all deviations from an artist's gallery""" + subcategory = "gallery" archive_fmt = "g_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/gallery(?:/all|/?\?catpath=)?/?$" @@ -804,6 +812,7 @@ def deviations(self): class DeviantartAvatarExtractor(DeviantartExtractor): """Extractor for an artist's avatar""" + subcategory = "avatar" archive_fmt = "a_{_username}_{index}" pattern = BASE_PATTERN + r"/avatar" @@ -837,27 +846,27 @@ def deviations(self): fmt, _, ext = fmt.rpartition(".") if fmt: fmt = "-" + fmt - url = "https://a.deviantart.net/avatars{}/{}/{}/{}.{}?{}".format( - fmt, name[0], name[1], name, ext, index) + url = f"https://a.deviantart.net/avatars{fmt}/{name[0]}/{name[1]}/{name}.{ext}?{index}" results.append(self._make_deviation(url, user, index, fmt)) return results def _make_deviation(self, url, user, index, fmt): return { - "author" : user, - "da_category" : "avatar", - "index" : text.parse_int(index), - "is_deleted" : False, + "author": user, + "da_category": "avatar", + "index": text.parse_int(index), + "is_deleted": False, "is_downloadable": False, - "published_time" : 0, - "title" : "avatar" + fmt, - "stats" : {"comments": 0}, - "content" : {"src": url}, + "published_time": 0, + "title": "avatar" + fmt, + "stats": {"comments": 0}, + "content": {"src": url}, } class DeviantartBackgroundExtractor(DeviantartExtractor): """Extractor for an artist's banner""" + subcategory = "background" archive_fmt = "b_{index}" pattern = BASE_PATTERN + r"/ba(?:nner|ckground)" @@ -865,14 +874,14 @@ class DeviantartBackgroundExtractor(DeviantartExtractor): def deviations(self): try: - return (self.api.user_profile(self.user.lower()) - ["cover_deviation"]["cover_deviation"],) + return (self.api.user_profile(self.user.lower())["cover_deviation"]["cover_deviation"],) except Exception: return () class DeviantartFolderExtractor(DeviantartExtractor): """Extractor for deviations inside an artist's gallery folder""" + subcategory = "folder" directory_fmt = ("{category}", "{username}", "{folder[title]}") archive_fmt = "F_{folder[uuid]}_{index}.{extension}" @@ -890,7 +899,7 @@ def deviations(self): folder = self._find_folder(folders, self.folder_name, self.folder_id) self.folder = { "title": folder["name"], - "uuid" : folder["folderid"], + "uuid": folder["folderid"], "index": self.folder_id, "owner": self.user, } @@ -903,10 +912,10 @@ def prepare(self, deviation): class DeviantartStashExtractor(DeviantartExtractor): """Extractor for sta.sh-ed deviations""" + subcategory = "stash" archive_fmt = "{index}.{extension}" - pattern = (r"(?:https?://)?(?:(?:www\.)?deviantart\.com/stash|sta\.sh)" - r"/([a-z0-9]+)") + pattern = r"(?:https?://)?(?:(?:www\.)?deviantart\.com/stash|sta\.sh)" r"/([a-z0-9]+)" example = "https://www.deviantart.com/stash/abcde" skip = Extractor.skip @@ -922,17 +931,15 @@ def deviations(self, stash_id=None): page = self._limited_request(url).text if stash_id[0] == "0": - uuid = text.extr(page, '//deviation/', '"') + uuid = text.extr(page, "//deviation/", '"') if uuid: deviation = self.api.deviation(uuid) deviation["_page"] = page - deviation["index"] = text.parse_int(text.extr( - page, '\\"deviationId\\":', ',')) + deviation["index"] = text.parse_int(text.extr(page, '\\"deviationId\\":', ",")) yield deviation return - for sid in text.extract_iter( - page, 'href="https://www.deviantart.com/stash/', '"'): + for sid in text.extract_iter(page, 'href="https://www.deviantart.com/stash/', '"'): if sid == stash_id or sid.endswith("#comments"): continue yield from self.deviations(sid) @@ -940,6 +947,7 @@ def deviations(self, stash_id=None): class DeviantartFavoriteExtractor(DeviantartExtractor): """Extractor for an artist's favorites""" + subcategory = "favorite" directory_fmt = ("{category}", "{username}", "Favourites") archive_fmt = "f_{_username}_{index}.{extension}" @@ -950,15 +958,14 @@ def deviations(self): if self.flat: return self.api.collections_all(self.user, self.offset) folders = self.api.collections_folders(self.user) - return self._folder_urls( - folders, "favourites", DeviantartCollectionExtractor) + return self._folder_urls(folders, "favourites", DeviantartCollectionExtractor) class DeviantartCollectionExtractor(DeviantartExtractor): """Extractor for a single favorite collection""" + subcategory = "collection" - directory_fmt = ("{category}", "{username}", "Favourites", - "{collection[title]}") + directory_fmt = ("{category}", "{username}", "Favourites", "{collection[title]}") archive_fmt = "C_{collection[uuid]}_{index}.{extension}" pattern = BASE_PATTERN + r"/favourites/([^/?#]+)/([^/?#]+)" example = "https://www.deviantart.com/USER/favourites/12345/TITLE" @@ -971,11 +978,10 @@ def __init__(self, match): def deviations(self): folders = self.api.collections_folders(self.user) - folder = self._find_folder( - folders, self.collection_name, self.collection_id) + folder = self._find_folder(folders, self.collection_name, self.collection_id) self.collection = { "title": folder["name"], - "uuid" : folder["folderid"], + "uuid": folder["folderid"], "index": self.collection_id, "owner": self.user, } @@ -988,6 +994,7 @@ def prepare(self, deviation): class DeviantartJournalExtractor(DeviantartExtractor): """Extractor for an artist's journals""" + subcategory = "journal" directory_fmt = ("{category}", "{username}", "Journal") archive_fmt = "j_{_username}_{index}.{extension}" @@ -1000,6 +1007,7 @@ def deviations(self): class DeviantartStatusExtractor(DeviantartExtractor): """Extractor for an artist's status updates""" + subcategory = "status" directory_fmt = ("{category}", "{username}", "Status") filename_fmt = "{category}_{index}_{title}_{date}.{extension}" @@ -1020,8 +1028,7 @@ def status(self, status): yield from self.status(item["status"].copy()) # assume is_deleted == true means necessary fields are missing if status["is_deleted"]: - self.log.warning( - "Skipping status %s (deleted)", status.get("statusid")) + self.log.warning("Skipping status %s (deleted)", status.get("statusid")) return yield status @@ -1054,13 +1061,13 @@ def prepare(self, deviation): deviation["stats"] = {"comments": comments_count} if self.comments: deviation["comments"] = ( - self._extract_comments(deviation["statusid"], "status") - if comments_count else () + self._extract_comments(deviation["statusid"], "status") if comments_count else () ) class DeviantartTagExtractor(DeviantartExtractor): """Extractor for deviations from tag searches""" + subcategory = "tag" directory_fmt = ("{category}", "Tags", "{search_tags}") archive_fmt = "T_{search_tags}_{index}.{extension}" @@ -1081,9 +1088,11 @@ def prepare(self, deviation): class DeviantartWatchExtractor(DeviantartExtractor): """Extractor for Deviations from watched users""" + subcategory = "watch" - pattern = (r"(?:https?://)?(?:www\.)?deviantart\.com" - r"/(?:watch/deviations|notifications/watch)()()") + pattern = ( + r"(?:https?://)?(?:www\.)?deviantart\.com" r"/(?:watch/deviations|notifications/watch)()()" + ) example = "https://www.deviantart.com/watch/deviations" def deviations(self): @@ -1092,6 +1101,7 @@ def deviations(self): class DeviantartWatchPostsExtractor(DeviantartExtractor): """Extractor for Posts from watched users""" + subcategory = "watch-posts" pattern = r"(?:https?://)?(?:www\.)?deviantart\.com/watch/posts()()" example = "https://www.deviantart.com/watch/posts" @@ -1103,15 +1113,19 @@ def deviations(self): ############################################################################### # Eclipse ##################################################################### + class DeviantartDeviationExtractor(DeviantartExtractor): """Extractor for single deviations""" + subcategory = "deviation" archive_fmt = "g_{_username}_{index}.{extension}" - pattern = (BASE_PATTERN + r"/(art|journal)/(?:[^/?#]+-)?(\d+)" - r"|(?:https?://)?(?:www\.)?(?:fx)?deviantart\.com/" - r"(?:view/|deviation/|view(?:-full)?\.php/*\?(?:[^#]+&)?id=)" - r"(\d+)" # bare deviation ID without slug - r"|(?:https?://)?fav\.me/d([0-9a-z]+)") # base36 + pattern = ( + BASE_PATTERN + r"/(art|journal)/(?:[^/?#]+-)?(\d+)" + r"|(?:https?://)?(?:www\.)?(?:fx)?deviantart\.com/" + r"(?:view/|deviation/|view(?:-full)?\.php/*\?(?:[^#]+&)?id=)" + r"(\d+)" # bare deviation ID without slug + r"|(?:https?://)?fav\.me/d([0-9a-z]+)" + ) # base36 example = "https://www.deviantart.com/UsER/art/TITLE-12345" skip = Extractor.skip @@ -1119,18 +1133,16 @@ class DeviantartDeviationExtractor(DeviantartExtractor): def __init__(self, match): DeviantartExtractor.__init__(self, match) self.type = match.group(3) - self.deviation_id = \ - match.group(4) or match.group(5) or id_from_base36(match.group(6)) + self.deviation_id = match.group(4) or match.group(5) or id_from_base36(match.group(6)) def deviations(self): if self.user: - url = "{}/{}/{}/{}".format( - self.root, self.user, self.type or "art", self.deviation_id) + url = "{}/{}/{}/{}".format(self.root, self.user, self.type or "art", self.deviation_id) else: - url = "{}/view/{}/".format(self.root, self.deviation_id) + url = f"{self.root}/view/{self.deviation_id}/" page = self._limited_request(url, notfound="deviation").text - uuid = text.extr(page, '"deviationUuid\\":\\"', '\\') + uuid = text.extr(page, '"deviationUuid\\":\\"', "\\") if not uuid: raise exception.NotFoundError("deviation") @@ -1141,6 +1153,7 @@ def deviations(self): class DeviantartScrapsExtractor(DeviantartExtractor): """Extractor for an artist's scraps""" + subcategory = "scraps" directory_fmt = ("{category}", "{username}", "Scraps") archive_fmt = "s_{_username}_{index}.{extension}" @@ -1152,16 +1165,18 @@ def deviations(self): eclipse_api = DeviantartEclipseAPI(self) return self._eclipse_to_oauth( - eclipse_api, eclipse_api.gallery_scraps(self.user, self.offset)) + eclipse_api, eclipse_api.gallery_scraps(self.user, self.offset) + ) class DeviantartSearchExtractor(DeviantartExtractor): """Extractor for deviantart search results""" + subcategory = "search" directory_fmt = ("{category}", "Search", "{search_tags}") archive_fmt = "Q_{search_tags}_{index}.{extension}" - pattern = (r"(?:https?://)?www\.deviantart\.com" - r"/search(?:/deviations)?/?\?([^#]+)") + cookies_domain = ".deviantart.com" + pattern = r"(?:https?://)?www\.deviantart\.com" r"/search(?:/deviations)?/?\?([^#]+)" example = "https://www.deviantart.com/search?q=QUERY" skip = Extractor.skip @@ -1175,8 +1190,7 @@ def deviations(self): logged_in = self.login() eclipse_api = DeviantartEclipseAPI(self) - search = (eclipse_api.search_deviations - if logged_in else self._search_html) + search = eclipse_api.search_deviations if logged_in else self._search_html return self._eclipse_to_oauth(eclipse_api, search(self.query)) def prepare(self, deviation): @@ -1193,15 +1207,18 @@ def _search_html(self, params): raise exception.StopExtraction("HTTP redirect to login page") page = response.text - for dev in DeviantartDeviationExtractor.pattern.findall( - page)[2::3]: + for dev in DeviantartDeviationExtractor.pattern.findall(page)[2::3]: yield { "deviationId": dev[3], "author": {"username": dev[0]}, "isJournal": dev[2] == "journal", } - cursor = text.extr(page, r'\"cursor\":\"', '\\',) + cursor = text.extr( + page, + r"\"cursor\":\"", + "\\", + ) if not cursor: return params["cursor"] = cursor @@ -1209,6 +1226,7 @@ def _search_html(self, params): class DeviantartGallerySearchExtractor(DeviantartExtractor): """Extractor for deviantart gallery searches""" + subcategory = "gallery-search" archive_fmt = "g_{_username}_{index}.{extension}" pattern = BASE_PATTERN + r"/gallery/?\?(q=[^#]+)" @@ -1226,12 +1244,14 @@ def deviations(self): self.search = query["q"] return self._eclipse_to_oauth( - eclipse_api, eclipse_api.galleries_search( + eclipse_api, + eclipse_api.galleries_search( self.user, self.search, self.offset, query.get("sort", "most-recent"), - )) + ), + ) def prepare(self, deviation): DeviantartExtractor.prepare(self, deviation) @@ -1240,6 +1260,7 @@ def prepare(self, deviation): class DeviantartFollowingExtractor(DeviantartExtractor): """Extractor for user's watched users""" + subcategory = "following" pattern = BASE_PATTERN + "/(?:about#)?watching" example = "https://www.deviantart.com/USER/about#watching" @@ -1256,11 +1277,13 @@ def items(self): ############################################################################### # API Interfaces ############################################################## -class DeviantartOAuthAPI(): + +class DeviantartOAuthAPI: """Interface for the DeviantArt OAuth API https://www.deviantart.com/developers/http/v1/20160316 """ + CLIENT_ID = "5388" CLIENT_SECRET = "76b08c69cfb27f26d6161f9ab6d061a1" @@ -1298,14 +1321,19 @@ def __init__(self, extractor): metadata = extractor.config("metadata", False) if not metadata: - metadata = True if extractor.extra else False + metadata = bool(extractor.extra) if metadata: self.metadata = True if isinstance(metadata, str): if metadata == "all": - metadata = ("submission", "camera", "stats", - "collection", "gallery") + metadata = ( + "submission", + "camera", + "stats", + "collection", + "gallery", + ) else: metadata = metadata.replace(" ", "").split(",") elif not isinstance(metadata, (list, tuple)): @@ -1318,13 +1346,16 @@ def __init__(self, extractor): self.limit = 10 for param in metadata: self._metadata_params["ext_" + param] = "1" - if "ext_collection" in self._metadata_params or \ - "ext_gallery" in self._metadata_params: + if ( + "ext_collection" in self._metadata_params + or "ext_gallery" in self._metadata_params + ): if token: self._metadata_public = False else: - self.log.error("'collection' and 'gallery' metadata " - "require a refresh token") + self.log.error( + "'collection' and 'gallery' metadata " "require a refresh token" + ) else: # base metadata self.limit = 50 @@ -1341,32 +1372,30 @@ def __init__(self, extractor): def browse_deviantsyouwatch(self, offset=0): """Yield deviations from users you watch""" endpoint = "/browse/deviantsyouwatch" - params = {"limit": 50, "offset": offset, - "mature_content": self.mature} + params = {"limit": 50, "offset": offset, "mature_content": self.mature} return self._pagination(endpoint, params, public=False) def browse_posts_deviantsyouwatch(self, offset=0): """Yield posts from users you watch""" endpoint = "/browse/posts/deviantsyouwatch" - params = {"limit": 50, "offset": offset, - "mature_content": self.mature} + params = {"limit": 50, "offset": offset, "mature_content": self.mature} return self._pagination(endpoint, params, public=False, unpack=True) def browse_tags(self, tag, offset=0): - """ Browse a tag """ + """Browse a tag""" endpoint = "/browse/tags" params = { - "tag" : tag, - "offset" : offset, - "limit" : 50, + "tag": tag, + "offset": offset, + "limit": 50, "mature_content": self.mature, } return self._pagination(endpoint, params) def browse_user_journals(self, username, offset=0): journals = filter( - lambda post: "/journal/" in post["url"], - self.user_profile_posts(username)) + lambda post: "/journal/" in post["url"], self.user_profile_posts(username) + ) if offset: journals = util.advance(journals, offset) return journals @@ -1374,34 +1403,45 @@ def browse_user_journals(self, username, offset=0): def collections(self, username, folder_id, offset=0): """Yield all Deviation-objects contained in a collection folder""" endpoint = "/collections/" + folder_id - params = {"username": username, "offset": offset, "limit": 24, - "mature_content": self.mature} + params = { + "username": username, + "offset": offset, + "limit": 24, + "mature_content": self.mature, + } return self._pagination(endpoint, params) def collections_all(self, username, offset=0): """Yield all deviations in a user's collection""" endpoint = "/collections/all" - params = {"username": username, "offset": offset, "limit": 24, - "mature_content": self.mature} + params = { + "username": username, + "offset": offset, + "limit": 24, + "mature_content": self.mature, + } return self._pagination(endpoint, params) @memcache(keyarg=1) def collections_folders(self, username, offset=0): """Yield all collection folders of a specific user""" endpoint = "/collections/folders" - params = {"username": username, "offset": offset, "limit": 50, - "mature_content": self.mature} + params = { + "username": username, + "offset": offset, + "limit": 50, + "mature_content": self.mature, + } return self._pagination_list(endpoint, params) - def comments(self, target_id, target_type="deviation", - comment_id=None, offset=0): + def comments(self, target_id, target_type="deviation", comment_id=None, offset=0): """Fetch comments posted on a target""" - endpoint = "/comments/{}/{}".format(target_type, target_id) + endpoint = f"/comments/{target_type}/{target_id}" params = { - "commentid" : comment_id, - "maxdepth" : "5", - "offset" : offset, - "limit" : 50, + "commentid": comment_id, + "maxdepth": "5", + "offset": offset, + "limit": 50, "mature_content": self.mature, } return self._pagination_list(endpoint, params=params, key="thread") @@ -1411,8 +1451,7 @@ def deviation(self, deviation_id, public=None): endpoint = "/deviation/" + deviation_id deviation = self._call(endpoint, public=public) - if deviation.get("is_mature") and public is None and \ - self.refresh_token_key: + if deviation.get("is_mature") and public is None and self.refresh_token_key: deviation = self._call(endpoint, public=False) if self.metadata: @@ -1426,8 +1465,7 @@ def deviation_content(self, deviation_id, public=None): endpoint = "/deviation/content" params = {"deviationid": deviation_id} content = self._call(endpoint, params=params, public=public) - if public and content["html"].startswith( - ' <span class=\"username-with-symbol'): + if public and content["html"].startswith(' <span class="username-with-symbol'): if self.refresh_token_key: content = self._call(endpoint, params=params, public=False) else: @@ -1440,15 +1478,14 @@ def deviation_download(self, deviation_id, public=None): params = {"mature_content": self.mature} try: - return self._call( - endpoint, params=params, public=public, log=False) + return self._call(endpoint, params=params, public=public, log=False) except Exception: if not self.refresh_token_key: raise return self._call(endpoint, params=params, public=False) def deviation_metadata(self, deviations): - """ Fetch deviation metadata for a set of deviations""" + """Fetch deviation metadata for a set of deviations""" endpoint = "/deviation/metadata?" + "&".join( "deviationids[{}]={}".format(num, deviation["deviationid"]) for num, deviation in enumerate(deviations) @@ -1462,23 +1499,36 @@ def deviation_metadata(self, deviations): def gallery(self, username, folder_id, offset=0, extend=True, public=None): """Yield all Deviation-objects contained in a gallery folder""" endpoint = "/gallery/" + folder_id - params = {"username": username, "offset": offset, "limit": 24, - "mature_content": self.mature, "mode": "newest"} + params = { + "username": username, + "offset": offset, + "limit": 24, + "mature_content": self.mature, + "mode": "newest", + } return self._pagination(endpoint, params, extend, public) def gallery_all(self, username, offset=0): """Yield all Deviation-objects of a specific user""" endpoint = "/gallery/all" - params = {"username": username, "offset": offset, "limit": 24, - "mature_content": self.mature} + params = { + "username": username, + "offset": offset, + "limit": 24, + "mature_content": self.mature, + } return self._pagination(endpoint, params) @memcache(keyarg=1) def gallery_folders(self, username, offset=0): """Yield all gallery folders of a specific user""" endpoint = "/gallery/folders" - params = {"username": username, "offset": offset, "limit": 50, - "mature_content": self.mature} + params = { + "username": username, + "offset": offset, + "limit": 50, + "mature_content": self.mature, + } return self._pagination_list(endpoint, params) def user_friends(self, username, offset=0): @@ -1491,25 +1541,32 @@ def user_friends_watch(self, username): """Watch a user""" endpoint = "/user/friends/watch/" + username data = { - "watch[friend]" : "0", - "watch[deviations]" : "0", - "watch[journals]" : "0", + "watch[friend]": "0", + "watch[deviations]": "0", + "watch[journals]": "0", "watch[forum_threads]": "0", - "watch[critiques]" : "0", - "watch[scraps]" : "0", - "watch[activity]" : "0", - "watch[collections]" : "0", - "mature_content" : self.mature, + "watch[critiques]": "0", + "watch[scraps]": "0", + "watch[activity]": "0", + "watch[collections]": "0", + "mature_content": self.mature, } return self._call( - endpoint, method="POST", data=data, public=False, fatal=False, + endpoint, + method="POST", + data=data, + public=False, + fatal=False, ).get("success") def user_friends_unwatch(self, username): """Unwatch a user""" endpoint = "/user/friends/unwatch/" + username return self._call( - endpoint, method="POST", public=False, fatal=False, + endpoint, + method="POST", + public=False, + fatal=False, ).get("success") @memcache(keyarg=1) @@ -1520,23 +1577,22 @@ def user_profile(self, username): def user_profile_posts(self, username): endpoint = "/user/profile/posts" - params = {"username": username, "limit": 50, - "mature_content": self.mature} + params = {"username": username, "limit": 50, "mature_content": self.mature} return self._pagination(endpoint, params) def user_statuses(self, username, offset=0): """Yield status updates of a specific user""" statuses = filter( lambda post: "/status-update/" in post["url"], - self.user_profile_posts(username)) + self.user_profile_posts(username), + ) if offset: statuses = util.advance(statuses, offset) return statuses def authenticate(self, refresh_token_key): """Authenticate the application by requesting an access token""" - self.headers["Authorization"] = \ - self._authenticate_impl(refresh_token_key) + self.headers["Authorization"] = self._authenticate_impl(refresh_token_key) @cache(maxage=3600, keyarg=1) def _authenticate_impl(self, refresh_token_key): @@ -1544,24 +1600,25 @@ def _authenticate_impl(self, refresh_token_key): url = "https://www.deviantart.com/oauth2/token" if refresh_token_key: self.log.info("Refreshing private access token") - data = {"grant_type": "refresh_token", - "refresh_token": _refresh_token_cache(refresh_token_key)} + data = { + "grant_type": "refresh_token", + "refresh_token": _refresh_token_cache(refresh_token_key), + } else: self.log.info("Requesting public access token") data = {"grant_type": "client_credentials"} auth = util.HTTPBasicAuth(self.client_id, self.client_secret) - response = self.extractor.request( - url, method="POST", data=data, auth=auth, fatal=False) + response = self.extractor.request(url, method="POST", data=data, auth=auth, fatal=False) data = response.json() if response.status_code != 200: self.log.debug("Server response: %s", data) - raise exception.AuthenticationError('"{}" ({})'.format( - data.get("error_description"), data.get("error"))) + raise exception.AuthenticationError( + '"{}" ({})'.format(data.get("error_description"), data.get("error")) + ) if refresh_token_key: - _refresh_token_cache.update( - refresh_token_key, data["refresh_token"]) + _refresh_token_cache.update(refresh_token_key, data["refresh_token"]) return "Bearer " + data["access_token"] def _call(self, endpoint, fatal=True, log=True, public=None, **kwargs): @@ -1601,8 +1658,7 @@ def _call(self, endpoint, fatal=True, log=True, public=None, **kwargs): raise exception.AuthorizationError() self.log.debug(response.text) - msg = "API responded with {} {}".format( - status, response.reason) + msg = f"API responded with {status} {response.reason}" if status == 429: if self.delay < 30: self.delay += 1 @@ -1615,7 +1671,8 @@ def _call(self, endpoint, fatal=True, log=True, public=None, **kwargs): "Register your own OAuth application and use its " "credentials to prevent this error: " "https://gdl-org.github.io/docs/configuration.html" - "#extractor-deviantart-client-id-client-secret") + "#extractor-deviantart-client-id-client-secret" + ) else: if log: self.log.error(msg) @@ -1632,8 +1689,7 @@ def _should_switch_tokens(self, results, params): return False - def _pagination(self, endpoint, params, - extend=True, public=None, unpack=False, key="results"): + def _pagination(self, endpoint, params, extend=True, public=None, unpack=False, key="results"): warn = True if public is None: public = self.public @@ -1650,8 +1706,7 @@ def _pagination(self, endpoint, params, return if unpack: - results = [item["journal"] for item in results - if "journal" in item] + results = [item["journal"] for item in results if "journal" in item] if extend: if public and self._should_switch_tokens(results, params): if self.refresh_token_key: @@ -1663,7 +1718,8 @@ def _pagination(self, endpoint, params, self.log.warning( "Private or mature deviations detected! " "Run 'gallery-dl oauth:deviantart' and follow the " - "instructions to be able to access them.") + "instructions to be able to access them." + ) # "statusid" cannot be used instead if results and "deviationid" in results[0]: @@ -1675,15 +1731,13 @@ def _pagination(self, endpoint, params, for dev in self._shared_content(results): if not dev["is_deleted"]: continue - patch = self._call( - "/deviation/" + dev["deviationid"], fatal=False) + patch = self._call("/deviation/" + dev["deviationid"], fatal=False) if patch: dev.update(patch) yield from results - if not data["has_more"] and ( - self.strategy != "manual" or not results or not extend): + if not data["has_more"] and (self.strategy != "manual" or not results or not extend): return if "next_cursor" in data: @@ -1719,20 +1773,20 @@ def _metadata(self, deviations): else: n = self.limit for index in range(0, len(deviations), n): - self._metadata_batch(deviations[index:index+n]) + self._metadata_batch(deviations[index : index + n]) def _metadata_batch(self, deviations): """Fetch extended metadata for a single batch of deviations""" - for deviation, metadata in zip( - deviations, self.deviation_metadata(deviations)): + for deviation, metadata in zip(deviations, self.deviation_metadata(deviations)): deviation.update(metadata) deviation["tags"] = [t["tag_name"] for t in deviation["tags"]] def _folders(self, deviations): """Add a list of all containing folders to each deviation object""" for deviation in deviations: - deviation["folders"] = self._folders_map( - deviation["author"]["username"])[deviation["deviationid"]] + deviation["folders"] = self._folders_map(deviation["author"]["username"])[ + deviation["deviationid"] + ] @memcache(keyarg=1) def _folders_map(self, username): @@ -1741,10 +1795,7 @@ def _folders_map(self, username): folders = self.gallery_folders(username) # create 'folderid'-to-'folder' mapping - fmap = { - folder["folderid"]: folder - for folder in folders - } + fmap = {folder["folderid"]: folder for folder in folders} # add parent names to folders, but ignore "Featured" as parent featured = folders[0]["folderid"] @@ -1769,13 +1820,12 @@ def _folders_map(self, username): # map deviationids to folder names dmap = collections.defaultdict(list) for folder in folders: - for deviation in self.gallery( - username, folder["folderid"], 0, False): + for deviation in self.gallery(username, folder["folderid"], 0, False): dmap[deviation["deviationid"]].append(folder["name"]) return dmap -class DeviantartEclipseAPI(): +class DeviantartEclipseAPI: """Interface to the DeviantArt Eclipse API""" def __init__(self, extractor): @@ -1787,11 +1837,11 @@ def __init__(self, extractor): def deviation_extended_fetch(self, deviation_id, user, kind=None): endpoint = "/_puppy/dadeviation/init" params = { - "deviationid" : deviation_id, - "username" : user, - "type" : kind, - "include_session" : "false", - "expand" : "deviation.related", + "deviationid": deviation_id, + "username": user, + "type": kind, + "include_session": "false", + "expand": "deviation.related", "da_minor_version": "20230710", } return self._call(endpoint, params) @@ -1799,10 +1849,10 @@ def deviation_extended_fetch(self, deviation_id, user, kind=None): def gallery_scraps(self, user, offset=0): endpoint = "/_puppy/dashared/gallection/contents" params = { - "username" : user, - "type" : "gallery", - "offset" : offset, - "limit" : 24, + "username": user, + "type": "gallery", + "offset": offset, + "limit": 24, "scraps_folder": "true", } return self._pagination(endpoint, params) @@ -1811,11 +1861,11 @@ def galleries_search(self, user, query, offset=0, order="most-recent"): endpoint = "/_puppy/dashared/gallection/search" params = { "username": user, - "type" : "gallery", - "order" : order, - "q" : query, - "offset" : offset, - "limit" : 24, + "type": "gallery", + "order": order, + "q": query, + "offset": offset, + "limit": 24, } return self._pagination(endpoint, params) @@ -1833,12 +1883,12 @@ def user_watching(self, user, offset=0): endpoint = "/_puppy/gruser/module/watching" params = { - "gruserid" : gruserid, + "gruserid": gruserid, "gruser_typeid": "4", - "username" : user, - "moduleid" : moduleid, - "offset" : offset, - "limit" : 24, + "username": user, + "moduleid": moduleid, + "offset": offset, + "limit": 24, } return self._pagination(endpoint, params) @@ -1868,7 +1918,8 @@ def _pagination(self, endpoint, params, key="results"): self.log.warning( "Private deviations detected! " "Provide login credentials or session cookies " - "to be able to access them.") + "to be able to access them." + ) yield from results if not data.get("hasMore"): @@ -1886,7 +1937,7 @@ def _pagination(self, endpoint, params, key="results"): params["offset"] = int(params["offset"]) + len(results) def _ids_watching(self, user): - url = "{}/{}/about".format(self.extractor.root, user) + url = f"{self.extractor.root}/{user}/about" page = self.request(url).text gruser_id = text.extr(page, ' data-userid="', '"') @@ -1894,8 +1945,7 @@ def _ids_watching(self, user): pos = page.find('\\"name\\":\\"watching\\"') if pos < 0: raise exception.NotFoundError("'watching' module ID") - module_id = text.rextract( - page, '\\"id\\":', ',', pos)[0].strip('" ') + module_id = text.rextract(page, '\\"id\\":', ",", pos)[0].strip('" ') self._fetch_csrf_token(page) return gruser_id, module_id @@ -1903,8 +1953,7 @@ def _ids_watching(self, user): def _fetch_csrf_token(self, page=None): if page is None: page = self.request(self.extractor.root + "/").text - self.csrf_token = token = text.extr( - page, "window.__CSRF_TOKEN__ = '", "'") + self.csrf_token = token = text.extr(page, "window.__CSRF_TOKEN__ = '", "'") return token @@ -1916,14 +1965,14 @@ def _user_details(extr, name): return None -@cache(maxage=36500*86400, keyarg=0) +@cache(maxage=36500 * 86400, keyarg=0) def _refresh_token_cache(token): if token and token[0] == "#": return None return token -@cache(maxage=28*86400, keyarg=1) +@cache(maxage=28 * 86400, keyarg=1) def _login_impl(extr, username, password): extr.log.info("Logging in as %s", username) @@ -1951,10 +2000,7 @@ def _login_impl(extr, username, password): if not response.history: raise exception.AuthenticationError() - return { - cookie.name: cookie.value - for cookie in extr.cookies - } + return {cookie.name: cookie.value for cookie in extr.cookies} def id_from_base36(base36): diff --git a/gallery_dl/extractor/directlink.py b/gallery_dl/extractor/directlink.py index 2f0230af7b..9e8dd5e0bc 100644 --- a/gallery_dl/extractor/directlink.py +++ b/gallery_dl/extractor/directlink.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,19 +6,23 @@ """Direct link handling""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class DirectlinkExtractor(Extractor): """Extractor for direct links to images and other media files""" + category = "directlink" filename_fmt = "{domain}/{path}/{filename}.{extension}" archive_fmt = filename_fmt - pattern = (r"(?i)https?://(?P<domain>[^/?#]+)/(?P<path>[^?#]+\." - r"(?:jpe?g|jpe|png|gif|bmp|svg|web[mp]|avif|heic|psd" - r"|mp4|m4v|mov|mkv|og[gmv]|wav|mp3|opus|zip|rar|7z|pdf|swf))" - r"(?:\?(?P<query>[^#]*))?(?:#(?P<fragment>.*))?$") + pattern = ( + r"(?i)https?://(?P<domain>[^/?#]+)/(?P<path>[^?#]+\." + r"(?:jpe?g|jpe|png|gif|bmp|svg|web[mp]|avif|heic|psd" + r"|mp4|m4v|mov|mkv|og[gmv]|wav|mp3|opus|zip|rar|7z|pdf|swf))" + r"(?:\?(?P<query>[^#]*))?(?:#(?P<fragment>.*))?$" + ) example = "https://en.wikipedia.org/static/images/project-logos/enwiki.png" def __init__(self, match): @@ -36,8 +38,7 @@ def items(self): data["path"], _, name = data["path"].rpartition("/") data["filename"], _, ext = name.rpartition(".") data["extension"] = ext.lower() - data["_http_headers"] = { - "Referer": self.url.encode("latin-1", "ignore")} + data["_http_headers"] = {"Referer": self.url.encode("latin-1", "ignore")} yield Message.Directory, data yield Message.Url, self.url, data diff --git a/gallery_dl/extractor/dynastyscans.py b/gallery_dl/extractor/dynastyscans.py index 583869f1ab..97bf15c8a2 100644 --- a/gallery_dl/extractor/dynastyscans.py +++ b/gallery_dl/extractor/dynastyscans.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,20 +6,26 @@ """Extractors for https://dynasty-scans.com/""" -from .common import ChapterExtractor, MangaExtractor, Extractor, Message -from .. import text, util import re +from .. import text +from .. import util +from .common import ChapterExtractor +from .common import Extractor +from .common import MangaExtractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?(?:www\.)?dynasty-scans\.com" -class DynastyscansBase(): +class DynastyscansBase: """Base class for dynastyscans extractors""" + category = "dynastyscans" root = "https://dynasty-scans.com" def _parse_image_page(self, image_id): - url = "{}/images/{}".format(self.root, image_id) + url = f"{self.root}/images/{image_id}" extr = text.extract_from(self.request(url).text) date = extr("class='create_at'>", "</span>") @@ -32,52 +36,49 @@ def _parse_image_page(self, image_id): src = text.extr(src, 'href="', '"') if "Source<" in src else "" return { - "url" : self.root + url, + "url": self.root + url, "image_id": text.parse_int(image_id), - "tags" : text.split_html(tags), - "date" : text.remove_html(date), - "source" : text.unescape(src), + "tags": text.split_html(tags), + "date": text.remove_html(date), + "source": text.unescape(src), } class DynastyscansChapterExtractor(DynastyscansBase, ChapterExtractor): """Extractor for manga-chapters from dynasty-scans.com""" + pattern = BASE_PATTERN + r"(/chapters/[^/?#]+)" example = "https://dynasty-scans.com/chapters/NAME" def metadata(self, page): extr = text.extract_from(page) match = re.match( - (r"(?:<a[^>]*>)?([^<]+)(?:</a>)?" # manga name - r"(?: ch(\d+)([^:<]*))?" # chapter info - r"(?:: (.+))?"), # title + ( + r"(?:<a[^>]*>)?([^<]+)(?:</a>)?" # manga name + r"(?: ch(\d+)([^:<]*))?" # chapter info + r"(?:: (.+))?" + ), # title extr("<h3 id='chapter-title'><b>", "</b>"), ) author = extr(" by ", "</a>") - group = extr('"icon-print"></i> ', '</span>') + group = extr('"icon-print"></i> ', "</span>") return { - "manga" : text.unescape(match.group(1)), - "chapter" : text.parse_int(match.group(2)), + "manga": text.unescape(match.group(1)), + "chapter": text.parse_int(match.group(2)), "chapter_minor": match.group(3) or "", - "title" : text.unescape(match.group(4) or ""), - "author" : text.remove_html(author), - "group" : (text.remove_html(group) or - text.extr(group, ' alt="', '"')), - "date" : text.parse_datetime(extr( - '"icon-calendar"></i> ', '<'), "%b %d, %Y"), - "tags" : text.split_html(extr( - "class='tags'>", "<div id='chapter-actions'")), - "lang" : "en", + "title": text.unescape(match.group(4) or ""), + "author": text.remove_html(author), + "group": (text.remove_html(group) or text.extr(group, ' alt="', '"')), + "date": text.parse_datetime(extr('"icon-calendar"></i> ', "<"), "%b %d, %Y"), + "tags": text.split_html(extr("class='tags'>", "<div id='chapter-actions'")), + "lang": "en", "language": "English", } def images(self, page): data = text.extr(page, "var pages = ", ";\n") - return [ - (self.root + img["image"], None) - for img in util.json_loads(data) - ] + return [(self.root + img["image"], None) for img in util.json_loads(data)] class DynastyscansMangaExtractor(DynastyscansBase, MangaExtractor): @@ -87,14 +88,12 @@ class DynastyscansMangaExtractor(DynastyscansBase, MangaExtractor): example = "https://dynasty-scans.com/series/NAME" def chapters(self, page): - return [ - (self.root + path, {}) - for path in text.extract_iter(page, '<dd>\n<a href="', '"') - ] + return [(self.root + path, {}) for path in text.extract_iter(page, '<dd>\n<a href="', '"')] class DynastyscansSearchExtractor(DynastyscansBase, Extractor): """Extrator for image search results on dynasty-scans.com""" + subcategory = "search" directory_fmt = ("{category}", "Images") filename_fmt = "{image_id}.{extension}" @@ -127,6 +126,7 @@ def images(self): class DynastyscansImageExtractor(DynastyscansSearchExtractor): """Extractor for individual images on dynasty-scans.com""" + subcategory = "image" pattern = BASE_PATTERN + r"/images/(\d+)" example = "https://dynasty-scans.com/images/12345" diff --git a/gallery_dl/extractor/e621.py b/gallery_dl/extractor/e621.py index 553ec22f94..98238d5154 100644 --- a/gallery_dl/extractor/e621.py +++ b/gallery_dl/extractor/e621.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,14 +6,16 @@ """Extractors for https://e621.net/ and other e621 instances""" -from .common import Message -from . import danbooru +from .. import text +from .. import util from ..cache import memcache -from .. import text, util +from . import danbooru +from .common import Message class E621Extractor(danbooru.DanbooruExtractor): """Base class for e621 extractors""" + basecategory = "E621" page_limit = 750 page_start = None @@ -32,8 +32,8 @@ def items(self): elif not isinstance(includes, (list, tuple)): includes = ("notes", "pools") - notes = ("notes" in includes) - pools = ("pools" in includes) + notes = "notes" in includes + pools = "pools" in includes data = self.metadata() for post in self.posts(): @@ -42,61 +42,62 @@ def items(self): if not file["url"]: md5 = file["md5"] file["url"] = "https://static1.{}/data/{}/{}/{}.{}".format( - self.root[8:], md5[0:2], md5[2:4], md5, file["ext"]) + self.root[8:], md5[0:2], md5[2:4], md5, file["ext"] + ) if notes and post.get("has_notes"): post["notes"] = self._get_notes(post["id"]) if pools and post["pools"]: - post["pools"] = self._get_pools( - ",".join(map(str, post["pools"]))) + post["pools"] = self._get_pools(",".join(map(str, post["pools"]))) post["filename"] = file["md5"] post["extension"] = file["ext"] - post["date"] = text.parse_datetime( - post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") + post["date"] = text.parse_datetime(post["created_at"], "%Y-%m-%dT%H:%M:%S.%f%z") post.update(data) yield Message.Directory, post yield Message.Url, file["url"], post def _get_notes(self, id): - return self.request( - "{}/notes.json?search[post_id]={}".format(self.root, id)).json() + return self.request(f"{self.root}/notes.json?search[post_id]={id}").json() @memcache(keyarg=1) def _get_pools(self, ids): - pools = self.request( - "{}/pools.json?search[id]={}".format(self.root, ids)).json() + pools = self.request(f"{self.root}/pools.json?search[id]={ids}").json() for pool in pools: pool["name"] = pool["name"].replace("_", " ") return pools -BASE_PATTERN = E621Extractor.update({ - "e621": { - "root": "https://e621.net", - "pattern": r"e621\.net", - }, - "e926": { - "root": "https://e926.net", - "pattern": r"e926\.net", - }, - "e6ai": { - "root": "https://e6ai.net", - "pattern": r"e6ai\.net", - }, -}) +BASE_PATTERN = E621Extractor.update( + { + "e621": { + "root": "https://e621.net", + "pattern": r"e621\.net", + }, + "e926": { + "root": "https://e926.net", + "pattern": r"e926\.net", + }, + "e6ai": { + "root": "https://e6ai.net", + "pattern": r"e6ai\.net", + }, + } +) class E621TagExtractor(E621Extractor, danbooru.DanbooruTagExtractor): """Extractor for e621 posts from tag searches""" + pattern = BASE_PATTERN + r"/posts?(?:\?.*?tags=|/index/\d+/)([^&#]+)" example = "https://e621.net/posts?tags=TAG" class E621PoolExtractor(E621Extractor, danbooru.DanbooruPoolExtractor): """Extractor for e621 pools""" + pattern = BASE_PATTERN + r"/pool(?:s|/show)/(\d+)" example = "https://e621.net/pools/12345" @@ -105,8 +106,7 @@ def posts(self): id_to_post = { post["id"]: post - for post in self._pagination( - "/posts.json", {"tags": "pool:" + self.pool_id}) + for post in self._pagination("/posts.json", {"tags": "pool:" + self.pool_id}) } posts = [] @@ -123,16 +123,18 @@ def posts(self): class E621PostExtractor(E621Extractor, danbooru.DanbooruPostExtractor): """Extractor for single e621 posts""" + pattern = BASE_PATTERN + r"/post(?:s|/show)/(\d+)" example = "https://e621.net/posts/12345" def posts(self): - url = "{}/posts/{}.json".format(self.root, self.post_id) + url = f"{self.root}/posts/{self.post_id}.json" return (self.request(url).json()["post"],) class E621PopularExtractor(E621Extractor, danbooru.DanbooruPopularExtractor): """Extractor for popular images from e621""" + pattern = BASE_PATTERN + r"/explore/posts/popular(?:\?([^#]*))?" example = "https://e621.net/explore/posts/popular" @@ -142,6 +144,7 @@ def posts(self): class E621FavoriteExtractor(E621Extractor): """Extractor for e621 favorites""" + subcategory = "favorite" directory_fmt = ("{category}", "Favorites", "{user_id}") archive_fmt = "f_{user_id}_{id}" diff --git a/gallery_dl/extractor/erome.py b/gallery_dl/extractor/erome.py index e6d136f6c7..918402e899 100644 --- a/gallery_dl/extractor/erome.py +++ b/gallery_dl/extractor/erome.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,11 +6,15 @@ """Extractors for https://www.erome.com/""" -from .common import Extractor, Message -from .. import text, util, exception -from ..cache import cache import itertools +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?(?:www\.)?erome\.com" @@ -30,40 +32,36 @@ def __init__(self, match): def items(self): for album_id in self.albums(): - url = "{}/a/{}".format(self.root, album_id) + url = f"{self.root}/a/{album_id}" try: page = self.request(url).text except exception.HttpError as exc: - self.log.warning( - "Unable to fetch album '%s' (%s)", album_id, exc) + self.log.warning("Unable to fetch album '%s' (%s)", album_id, exc) continue - title, pos = text.extract( - page, 'property="og:title" content="', '"') + title, pos = text.extract(page, 'property="og:title" content="', '"') pos = page.index('<div class="user-profile', pos) - user, pos = text.extract( - page, 'href="https://www.erome.com/', '"', pos) + user, pos = text.extract(page, 'href="https://www.erome.com/', '"', pos) urls = [] date = None groups = page.split('<div class="media-group"') for group in util.advance(groups, 1): - url = (text.extr(group, '<source src="', '"') or - text.extr(group, 'data-src="', '"')) + url = text.extr(group, '<source src="', '"') or text.extr(group, 'data-src="', '"') if url: urls.append(url) if not date: - ts = text.extr(group, '?v=', '"') + ts = text.extr(group, "?v=", '"') if len(ts) > 1: date = text.parse_timestamp(ts) data = { - "album_id" : album_id, - "title" : text.unescape(title), - "user" : text.unquote(user), - "count" : len(urls), - "date" : date, + "album_id": album_id, + "title": text.unescape(title), + "user": text.unquote(user), + "count": len(urls), + "date": date, "_http_headers": {"Referer": url}, } @@ -83,8 +81,7 @@ def request(self, url, **kwargs): response = Extractor.request(self, url, **kwargs) if response.cookies: _cookie_cache.update("", response.cookies) - if response.content.find( - b"<title>Please wait a few moments", 0, 600) < 0: + if response.content.find(b"Please wait a few moments", 0, 600) < 0: return response self.sleep(5.0, "check") @@ -101,6 +98,7 @@ def _pagination(self, url, params): class EromeAlbumExtractor(EromeExtractor): """Extractor for albums on erome.com""" + subcategory = "album" pattern = BASE_PATTERN + r"/a/(\w+)" example = "https://www.erome.com/a/ID" @@ -115,7 +113,7 @@ class EromeUserExtractor(EromeExtractor): example = "https://www.erome.com/USER" def albums(self): - url = "{}/{}".format(self.root, self.item) + url = f"{self.root}/{self.item}" return self._pagination(url, {}) diff --git a/gallery_dl/extractor/everia.py b/gallery_dl/extractor/everia.py index 94444ffb8b..f0074997a3 100644 --- a/gallery_dl/extractor/everia.py +++ b/gallery_dl/extractor/everia.py @@ -1,15 +1,15 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://everia.club""" -from .common import Extractor, Message -from .. import text import re +from .. import text +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?everia\.club" @@ -29,10 +29,7 @@ def _pagination(self, path, params=None, pnum=1): find_posts = re.compile(r'thumbnail">\s*= 300: @@ -56,12 +53,10 @@ def items(self): urls = re.findall(r'img.*?src="([^"]+)', content) data = { - "title": text.unescape( - text.extr(page, 'itemprop="headline">', "

")), + "title": text.unescape(text.extr(page, 'itemprop="headline">', "

")), "tags": list(text.extract_iter(page, 'rel="tag">', "")), "post_url": url, - "post_category": text.extr( - page, "post-in-category-", " ").capitalize(), + "post_category": text.extr(page, "post-in-category-", " ").capitalize(), "count": len(urls), } @@ -84,8 +79,7 @@ class EveriaCategoryExtractor(EveriaExtractor): class EveriaDateExtractor(EveriaExtractor): subcategory = "date" - pattern = (BASE_PATTERN + - r"(/\d{4}(?:/\d{2})?(?:/\d{2})?)(?:/page/\d+)?/?$") + pattern = BASE_PATTERN + r"(/\d{4}(?:/\d{2})?(?:/\d{2})?)(?:/page/\d+)?/?$" example = "https://everia.club/0000/00/00" diff --git a/gallery_dl/extractor/exhentai.py b/gallery_dl/extractor/exhentai.py index e7ba78e7c2..b31024d548 100644 --- a/gallery_dl/extractor/exhentai.py +++ b/gallery_dl/extractor/exhentai.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,18 +6,23 @@ """Extractors for https://e-hentai.org/ and https://exhentai.org/""" -from .common import Extractor, Message -from .. import text, util, exception -from ..cache import cache import collections import itertools import math +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?(e[x-]|g\.e-)hentai\.org" class ExhentaiExtractor(Extractor): """Base class for exhentai extractors""" + category = "exhentai" directory_fmt = ("{category}", "{gid} {title[:247]}") filename_fmt = "{gid}_{num:>04}_{image_token}_{filename}.{extension}" @@ -62,7 +65,7 @@ def login(self): raise exception.StopExtraction("Image limit reached!") if self.cookies_check(self.cookies_names): - return + return None username, password = self._get_auth_info() if username: @@ -76,7 +79,7 @@ def login(self): self.original = False self.limits = False - @cache(maxage=90*86400, keyarg=1) + @cache(maxage=90 * 86400, keyarg=1) def _login_impl(self, username, password): self.log.info("Logging in as %s", username) @@ -99,8 +102,7 @@ def _login_impl(self, username, password): content = response.content if b"You are now logged in as:" not in content: if b"The captcha was not entered correctly" in content: - raise exception.AuthenticationError( - "CAPTCHA required. Use cookies instead.") + raise exception.AuthenticationError("CAPTCHA required. Use cookies instead.") raise exception.AuthenticationError() # collect more cookies @@ -114,10 +116,9 @@ def _login_impl(self, username, password): class ExhentaiGalleryExtractor(ExhentaiExtractor): """Extractor for image galleries from exhentai.org""" + subcategory = "gallery" - pattern = (BASE_PATTERN + - r"(?:/g/(\d+)/([\da-f]{10})" - r"|/s/([\da-f]{10})/(\d+)-(\d+))") + pattern = BASE_PATTERN + r"(?:/g/(\d+)/([\da-f]{10})" r"|/s/([\da-f]{10})/(\d+)-(\d+))" example = "https://e-hentai.org/g/12345/67890abcde/" def __init__(self, match): @@ -149,22 +150,26 @@ def _init(self): def finalize(self): if self.data: - self.log.info("Use '%s/s/%s/%s-%s' as input URL " - "to continue downloading from the current position", - self.root, self.data["image_token"], - self.gallery_id, self.data["num"]) + self.log.info( + "Use '%s/s/%s/%s-%s' as input URL " + "to continue downloading from the current position", + self.root, + self.data["image_token"], + self.gallery_id, + self.data["num"], + ) def favorite(self, slot="0"): url = self.root + "/gallerypopups.php" params = { "gid": self.gallery_id, - "t" : self.gallery_token, + "t": self.gallery_token, "act": "addfav", } data = { - "favcat" : slot, - "apply" : "Apply Changes", - "update" : "1", + "favcat": slot, + "apply": "Apply Changes", + "update": "1", } self.request(url, method="POST", params=params, data=data) @@ -173,19 +178,17 @@ def items(self): if self.gallery_token: gpage = self._gallery_page() - self.image_token = text.extr(gpage, 'hentai.org/s/', '"') + self.image_token = text.extr(gpage, "hentai.org/s/", '"') if not self.image_token: self.log.debug("Page content:\n%s", gpage) - raise exception.StopExtraction( - "Failed to extract initial image token") + raise exception.StopExtraction("Failed to extract initial image token") ipage = self._image_page() else: ipage = self._image_page() - part = text.extr(ipage, 'hentai.org/g/', '"') + part = text.extr(ipage, "hentai.org/g/", '"') if not part: self.log.debug("Page content:\n%s", ipage) - raise exception.StopExtraction( - "Failed to extract gallery token") + raise exception.StopExtraction("Failed to extract gallery token") self.gallery_token = part.split("/")[1] gpage = self._gallery_page() @@ -193,8 +196,7 @@ def items(self): self.count = text.parse_int(data["filecount"]) yield Message.Directory, data - images = itertools.chain( - (self.image_from_page(ipage),), self.images_from_api()) + images = itertools.chain((self.image_from_page(ipage),), self.images_from_api()) for url, image in images: data.update(image) if self.limits: @@ -218,7 +220,8 @@ def _items_hitomi(self): data = {} from .hitomi import HitomiGalleryExtractor - url = "https://hitomi.la/galleries/{}.html".format(self.gallery_id) + + url = f"https://hitomi.la/galleries/{self.gallery_id}.html" data["_extractor"] = HitomiGalleryExtractor yield Message.Queue, url, data @@ -245,27 +248,27 @@ def metadata_from_page(self, page): self.api_url = api_url data = { - "gid" : self.gallery_id, - "token" : self.gallery_token, - "thumb" : extr("background:transparent url(", ")"), - "title" : text.unescape(extr('

', '

')), - "title_jpn" : text.unescape(extr('

', '

')), - "_" : extr('
', '<'), - "uploader" : extr('
', '
'), - "date" : text.parse_datetime(extr( - '>Posted:
'), "%Y-%m-%d %H:%M"), - "parent" : extr( - '>Parent:"), "%Y-%m-%d %H:%M" + ), + "parent": extr('>Parent:
', 'Visible:', '<'), - "language" : extr('>Language:', ' '), - "filesize" : text.parse_bytes(extr( - '>File Size:', '<').rstrip("Bbi")), - "filecount" : extr('>Length:', ' '), - "favorites" : extr('id="favcount">', ' '), - "rating" : extr(">Average: ", "<"), - "torrentcount" : extr('>Torrent Download (', ')'), + "gid": self.gallery_id, + "token": self.gallery_token, + "thumb": extr("background:transparent url(", ")"), + "title": text.unescape(extr('

', "

")), + "title_jpn": text.unescape(extr('

', "

")), + "_": extr('
", "<"), + "uploader": extr('
', "
"), + "date": text.parse_datetime( + extr('>Posted:
', "Visible:', "<") != "Yes", + "language": extr('>Language:', " "), + "filesize": text.parse_bytes( + extr('>File Size:', "<").rstrip("Bbi") + ), + "filecount": extr('>Length:', " "), + "favorites": extr('id="favcount">', " "), + "rating": extr(">Average: ", "<"), + "torrentcount": extr(">Torrent Download (", ")"), } uploader = data["uploader"] @@ -281,15 +284,15 @@ def metadata_from_page(self, page): data["lang"] = util.language_to_code(data["language"]) data["tags"] = [ text.unquote(tag.replace("+", " ")) - for tag in text.extract_iter(page, 'hentai.org/tag/', '"') + for tag in text.extract_iter(page, "hentai.org/tag/", '"') ] return data def metadata_from_api(self): data = { - "method" : "gdata", - "gidlist" : ((self.gallery_id, self.gallery_token),), + "method": "gdata", + "gidlist": ((self.gallery_id, self.gallery_token),), "namespace": 1, } @@ -307,12 +310,12 @@ def image_from_page(self, page): self.key_next = extr("'", "'") iurl = extr('= 0: origurl, pos = text.rextract(i6, '"', '"', pos) url = text.unescape(origurl) - data = self._parse_original_info(text.extract( - i6, "ownload original", "<", pos)[0]) + data = self._parse_original_info( + text.extract(i6, "ownload original", "<", pos)[0] + ) data["_fallback"] = self._fallback_original(nl, url) else: url = imgurl data = self._parse_image_info(url) - data["_fallback"] = self._fallback_1280( - nl, request["page"], imgkey) + data["_fallback"] = self._fallback_1280(nl, request["page"], imgkey) except IndexError: self.log.debug("Page content:\n%s", page) - raise exception.StopExtraction( - "Unable to parse image info for '%s'", url) + raise exception.StopExtraction("Unable to parse image info for '%s'", url) data["num"] = request["page"] data["image_token"] = imgkey @@ -385,8 +386,9 @@ def images_from_api(self): request["imgkey"] = nextkey def _validate_response(self, response): - if not response.history and response.headers.get( - "content-type", "").startswith("text/html"): + if not response.history and response.headers.get("content-type", "").startswith( + "text/html" + ): page = response.text self.log.warning("'%s'", page) @@ -394,7 +396,7 @@ def _validate_response(self, response): gp = self.config("gp") if gp == "stop": raise exception.StopExtraction("Not enough GP") - elif gp == "wait": + if gp == "wait": input("Press ENTER to continue.") return response.url @@ -423,8 +425,7 @@ def _check_509(self, url): # full 509.gif URLs # - https://exhentai.org/img/509.gif # - https://ehgt.org/g/509.gif - if url.endswith(("hentai.org/img/509.gif", - "ehgt.org/g/509.gif")): + if url.endswith(("hentai.org/img/509.gif", "ehgt.org/g/509.gif")): self.log.debug(url) self._report_limits() @@ -433,8 +434,7 @@ def _update_limits(self): cookies = { cookie.name: cookie.value for cookie in self.cookies - if cookie.domain == self.cookies_domain and - cookie.name != "igneous" + if cookie.domain == self.cookies_domain and cookie.name != "igneous" } page = self.request(url, cookies=cookies).text @@ -443,8 +443,7 @@ def _update_limits(self): self._remaining = self.limits - text.parse_int(current) def _gallery_page(self): - url = "{}/g/{}/{}/".format( - self.root, self.gallery_id, self.gallery_token) + url = f"{self.root}/g/{self.gallery_id}/{self.gallery_token}/" response = self.request(url, fatal=False) page = response.text @@ -457,8 +456,7 @@ def _gallery_page(self): return page def _image_page(self): - url = "{}/s/{}/{}-{}".format( - self.root, self.image_token, self.gallery_id, self.image_num) + url = f"{self.root}/s/{self.image_token}/{self.gallery_id}-{self.image_num}" page = self.request(url, fatal=False).text if page.startswith(("Invalid page", "Keep trying")): @@ -466,7 +464,7 @@ def _image_page(self): return page def _fallback_original(self, nl, fullimg): - url = "{}?nl={}".format(fullimg, nl) + url = f"{fullimg}?nl={nl}" for _ in util.repeat(self.fallback_retries): yield url @@ -475,8 +473,7 @@ def _fallback_1280(self, nl, num, token=None): token = self.key_start for _ in util.repeat(self.fallback_retries): - url = "{}/s/{}/{}-{}?nl={}".format( - self.root, token, self.gallery_id, num, nl) + url = f"{self.root}/s/{token}/{self.gallery_id}-{num}?nl={nl}" page = self.request(url, fatal=False).text if page.startswith(("Invalid page", "Keep trying")): @@ -498,9 +495,9 @@ def _parse_image_info(url): size = width = height = 0 return { - "cost" : 1, - "size" : text.parse_int(size), - "width" : text.parse_int(width), + "cost": 1, + "size": text.parse_int(size), + "width": text.parse_int(width), "height": text.parse_int(height), } @@ -511,15 +508,16 @@ def _parse_original_info(info): return { # 1 initial point + 1 per 0.1 MB - "cost" : 1 + math.ceil(size / 100000), - "size" : size, - "width" : text.parse_int(parts[0]), + "cost": 1 + math.ceil(size / 100000), + "size": size, + "width": text.parse_int(parts[0]), "height": text.parse_int(parts[2]), } class ExhentaiSearchExtractor(ExhentaiExtractor): """Extractor for exhentai search results""" + subcategory = "search" pattern = BASE_PATTERN + r"/(?:\?([^#]*)|tag/([^/?#]+))" example = "https://e-hentai.org/?f_search=QUERY" @@ -577,6 +575,7 @@ def items(self): class ExhentaiFavoriteExtractor(ExhentaiSearchExtractor): """Extractor for favorited exhentai galleries""" + subcategory = "favorite" pattern = BASE_PATTERN + r"/favorites\.php(?:\?([^#]*)())?" example = "https://e-hentai.org/favorites.php" diff --git a/gallery_dl/extractor/fanbox.py b/gallery_dl/extractor/fanbox.py index 9bbfb4384a..c14f659774 100644 --- a/gallery_dl/extractor/fanbox.py +++ b/gallery_dl/extractor/fanbox.py @@ -1,26 +1,25 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://www.fanbox.cc/""" -from .common import Extractor, Message +import re + from .. import text from ..cache import memcache -import re +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?fanbox\.cc" USER_PATTERN = ( - r"(?:https?://)?(?:" - r"(?!www\.)([\w-]+)\.fanbox\.cc|" - r"(?:www\.)?fanbox\.cc/@([\w-]+))" + r"(?:https?://)?(?:" r"(?!www\.)([\w-]+)\.fanbox\.cc|" r"(?:www\.)?fanbox\.cc/@([\w-]+))" ) class FanboxExtractor(Extractor): """Base class for Fanbox extractors""" + category = "fanbox" root = "https://www.fanbox.cc" directory_fmt = ("{category}", "{creatorId}") @@ -41,9 +40,9 @@ def _init(self): includes = includes.split(",") elif not isinstance(includes, (list, tuple)): includes = ("user", "plan") - self._meta_user = ("user" in includes) - self._meta_plan = ("plan" in includes) - self._meta_comments = ("comments" in includes) + self._meta_user = "user" in includes + self._meta_plan = "plan" in includes + self._meta_comments = "comments" in includes else: self._meta_user = self._meta_plan = self._meta_comments = False @@ -71,13 +70,14 @@ def _pagination(self, url): try: yield self._get_post_data(item["id"]) except Exception as exc: - self.log.warning("Skipping post %s (%s: %s)", - item["id"], exc.__class__.__name__, exc) + self.log.warning( + "Skipping post %s (%s: %s)", item["id"], exc.__class__.__name__, exc + ) url = body["nextUrl"] def _get_post_data(self, post_id): """Fetch and process post data""" - url = "https://api.fanbox.cc/post.info?postId="+post_id + url = "https://api.fanbox.cc/post.info?postId=" + post_id post = self.request(url, headers=self.headers).json()["body"] content_body = post.pop("body", None) @@ -88,7 +88,7 @@ def _get_post_data(self, post_id): post["articleBody"] = content_body.copy() if "blocks" in content_body: content = [] # text content - images = [] # image IDs in 'body' order + images = [] # image IDs in 'body' order append = content.append append_img = images.append @@ -156,16 +156,18 @@ def _get_plan_data(self, creator_id): params = {"creatorId": creator_id} data = self.request(url, params=params, headers=self.headers).json() - plans = {0: { - "id" : "", - "title" : "", - "fee" : 0, - "description" : "", - "coverImageUrl" : "", - "creatorId" : creator_id, - "hasAdultContent": None, - "paymentMethod" : None, - }} + plans = { + 0: { + "id": "", + "title": "", + "fee": 0, + "description": "", + "coverImageUrl": "", + "creatorId": creator_id, + "hasAdultContent": None, + "paymentMethod": None, + } + } for plan in data["body"]: del plan["user"] plans[plan["fee"]] = plan @@ -173,8 +175,7 @@ def _get_plan_data(self, creator_id): return plans def _get_comment_data(self, post_id): - url = ("https://api.fanbox.cc/post.listComments" - "?limit=10&postId=" + post_id) + url = "https://api.fanbox.cc/post.listComments" "?limit=10&postId=" + post_id comments = [] while url: @@ -204,12 +205,9 @@ def _get_urls_from_post(self, content_body, post): html_urls = [] for href in text.extract_iter(content_body["html"], 'href="', '"'): - if "fanbox.pixiv.net/images/entry" in href: + if "fanbox.pixiv.net/images/entry" in href or "downloads.fanbox.cc" in href: html_urls.append(href) - elif "downloads.fanbox.cc" in href: - html_urls.append(href) - for src in text.extract_iter(content_body["html"], - 'data-src-original="', '"'): + for src in text.extract_iter(content_body["html"], 'data-src-original="', '"'): html_urls.append(src) for url in html_urls: @@ -282,35 +280,38 @@ def _process_embed(self, post, embed): is_video = False if provider == "soundcloud": - url = prefix+"https://soundcloud.com/"+content_id + url = prefix + "https://soundcloud.com/" + content_id is_video = True elif provider == "youtube": - url = prefix+"https://youtube.com/watch?v="+content_id + url = prefix + "https://youtube.com/watch?v=" + content_id is_video = True elif provider == "vimeo": - url = prefix+"https://vimeo.com/"+content_id + url = prefix + "https://vimeo.com/" + content_id is_video = True elif provider == "fanbox": # this is an old URL format that redirects # to a proper Fanbox URL - url = "https://www.pixiv.net/fanbox/"+content_id + url = "https://www.pixiv.net/fanbox/" + content_id # resolve redirect try: - url = self.request(url, method="HEAD", - allow_redirects=False).headers["location"] + url = self.request(url, method="HEAD", allow_redirects=False).headers["location"] except Exception as exc: url = None - self.log.warning("Unable to extract fanbox embed %s (%s: %s)", - content_id, exc.__class__.__name__, exc) + self.log.warning( + "Unable to extract fanbox embed %s (%s: %s)", + content_id, + exc.__class__.__name__, + exc, + ) else: final_post["_extractor"] = FanboxPostExtractor elif provider == "twitter": - url = "https://twitter.com/_/status/"+content_id + url = "https://twitter.com/_/status/" + content_id elif provider == "google_forms": templ = "https://docs.google.com/forms/d/e/{}/viewform?usp=sf_link" url = templ.format(content_id) else: - self.log.warning("service not recognized: {}".format(provider)) + self.log.warning("service not recognized: %s", provider) if url: final_post["embed"] = embed @@ -324,6 +325,7 @@ def _process_embed(self, post, embed): class FanboxCreatorExtractor(FanboxExtractor): """Extractor for a Fanbox creator's works""" + subcategory = "creator" pattern = USER_PATTERN + r"(?:/posts)?/?$" example = "https://USER.fanbox.cc/" @@ -345,12 +347,14 @@ def _pagination_creator(self, url): try: yield self._get_post_data(item["id"]) except Exception as exc: - self.log.warning("Skipping post %s (%s: %s)", - item["id"], exc.__class__.__name__, exc) + self.log.warning( + "Skipping post %s (%s: %s)", item["id"], exc.__class__.__name__, exc + ) class FanboxPostExtractor(FanboxExtractor): """Extractor for media from a single Fanbox post""" + subcategory = "post" pattern = USER_PATTERN + r"/posts/(\d+)" example = "https://USER.fanbox.cc/posts/12345" @@ -365,6 +369,7 @@ def posts(self): class FanboxHomeExtractor(FanboxExtractor): """Extractor for your Fanbox home feed""" + subcategory = "home" pattern = BASE_PATTERN + r"/?$" example = "https://fanbox.cc/" @@ -376,6 +381,7 @@ def posts(self): class FanboxSupportingExtractor(FanboxExtractor): """Extractor for your supported Fanbox users feed""" + subcategory = "supporting" pattern = BASE_PATTERN + r"/home/supporting" example = "https://fanbox.cc/home/supporting" @@ -387,6 +393,7 @@ def posts(self): class FanboxRedirectExtractor(Extractor): """Extractor for pixiv redirects to fanbox.cc""" + category = "fanbox" subcategory = "redirect" pattern = r"(?:https?://)?(?:www\.)?pixiv\.net/fanbox/creator/(\d+)" @@ -399,6 +406,5 @@ def __init__(self, match): def items(self): url = "https://www.pixiv.net/fanbox/creator/" + self.user_id data = {"_extractor": FanboxCreatorExtractor} - response = self.request( - url, method="HEAD", allow_redirects=False, notfound="user") + response = self.request(url, method="HEAD", allow_redirects=False, notfound="user") yield Message.Queue, response.headers["Location"], data diff --git a/gallery_dl/extractor/fanleaks.py b/gallery_dl/extractor/fanleaks.py index 886e893483..d3fb56c5cf 100644 --- a/gallery_dl/extractor/fanleaks.py +++ b/gallery_dl/extractor/fanleaks.py @@ -1,17 +1,17 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://fanleaks.club/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class FanleaksExtractor(Extractor): """Base class for Fanleaks extractors""" + category = "fanleaks" directory_fmt = ("{category}", "{model}") filename_fmt = "{model_id}_{id}.{extension}" @@ -26,9 +26,9 @@ def extract_post(self, url): extr = text.extract_from(self.request(url, notfound="post").text) data = { "model_id": self.model_id, - "model" : text.unescape(extr('text-lg">', "")), - "id" : text.parse_int(self.id), - "type" : extr('type="', '"')[:5] or "photo", + "model": text.unescape(extr('text-lg">', "")), + "id": text.parse_int(self.id), + "type": extr('type="', '"')[:5] or "photo", } url = extr('src="', '"') yield Message.Directory, data @@ -37,6 +37,7 @@ def extract_post(self, url): class FanleaksPostExtractor(FanleaksExtractor): """Extractor for individual posts on fanleaks.club""" + subcategory = "post" pattern = r"(?:https?://)?(?:www\.)?fanleaks\.club/([^/?#]+)/(\d+)" example = "https://fanleaks.club/MODEL/12345" @@ -46,42 +47,40 @@ def __init__(self, match): self.id = match.group(2) def items(self): - url = "{}/{}/{}".format(self.root, self.model_id, self.id) + url = f"{self.root}/{self.model_id}/{self.id}" return self.extract_post(url) class FanleaksModelExtractor(FanleaksExtractor): """Extractor for all posts from a fanleaks model""" + subcategory = "model" - pattern = (r"(?:https?://)?(?:www\.)?fanleaks\.club" - r"/(?!latest/?$)([^/?#]+)/?$") + pattern = r"(?:https?://)?(?:www\.)?fanleaks\.club" r"/(?!latest/?$)([^/?#]+)/?$" example = "https://fanleaks.club/MODEL" def items(self): page_num = 1 - page = self.request( - self.root + "/" + self.model_id, notfound="model").text + page = self.request(self.root + "/" + self.model_id, notfound="model").text data = { "model_id": self.model_id, - "model" : text.unescape(text.extr(page, 'mt-4">', "")), - "type" : "photo", + "model": text.unescape(text.extr(page, 'mt-4">', "")), + "type": "photo", } page_url = text.extr(page, "url: '", "'") while True: - page = self.request("{}{}".format(page_url, page_num)).text + page = self.request(f"{page_url}{page_num}").text if not page: return for item in text.extract_iter(page, '"): self.id = id = text.extr(item, "/", '"') if "/icon-play.svg" in item: - url = "{}/{}/{}".format(self.root, self.model_id, id) + url = f"{self.root}/{self.model_id}/{id}" yield from self.extract_post(url) continue data["id"] = text.parse_int(id) - url = text.extr(item, 'src="', '"').replace( - "/thumbs/", "/", 1) + url = text.extr(item, 'src="', '"').replace("/thumbs/", "/", 1) yield Message.Directory, data yield Message.Url, url, text.nameext_from_url(url, data) page_num += 1 diff --git a/gallery_dl/extractor/fantia.py b/gallery_dl/extractor/fantia.py index 6218f19899..171c5866af 100644 --- a/gallery_dl/extractor/fantia.py +++ b/gallery_dl/extractor/fantia.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://fantia.jp/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message class FantiaExtractor(Extractor): """Base class for Fantia extractors""" + category = "fantia" root = "https://fantia.jp" directory_fmt = ("{category}", "{fanclub_id}") @@ -21,14 +22,14 @@ class FantiaExtractor(Extractor): def _init(self): self.headers = { - "Accept" : "application/json, text/plain, */*", + "Accept": "application/json, text/plain, */*", "X-Requested-With": "XMLHttpRequest", } self._empty_plan = { - "id" : 0, + "id": 0, "price": 0, "limit": 0, - "name" : "", + "name": "", "description": "", "thumb": self.root + "/images/fallback/plan/thumb_default.png", } @@ -52,15 +53,16 @@ def items(self): if content["visible_status"] != "visible": self.log.warning( - "Unable to download '%s' files from " - "%s#post-content-id-%s", content["visible_status"], - post["post_url"], content["id"]) + "Unable to download '%s' files from " "%s#post-content-id-%s", + content["visible_status"], + post["post_url"], + content["id"], + ) for file in files: post.update(file) post["num"] += 1 - text.nameext_from_url( - post["content_filename"] or file["file_url"], post) + text.nameext_from_url(post["content_filename"] or file["file_url"], post) yield Message.Url, file["file_url"], post post["content_num"] += 1 @@ -76,8 +78,7 @@ def _pagination(self, url): self._csrf_token(page) post_id = None - for post_id in text.extract_iter( - page, 'class="link-block" href="/posts/', '"'): + for post_id in text.extract_iter(page, 'class="link-block" href="/posts/', '"'): yield post_id if not post_id: @@ -87,12 +88,11 @@ def _pagination(self, url): def _csrf_token(self, page=None): if not page: page = self.request(self.root + "/").text - self.headers["X-CSRF-Token"] = text.extr( - page, 'name="csrf-token" content="', '"') + self.headers["X-CSRF-Token"] = text.extr(page, 'name="csrf-token" content="', '"') def _get_post_data(self, post_id): """Fetch and process post data""" - url = self.root+"/api/v1/posts/"+post_id + url = self.root + "/api/v1/posts/" + post_id resp = self.request(url, headers=self.headers).json()["post"] return { "post_id": resp["id"], @@ -101,13 +101,12 @@ def _get_post_data(self, post_id): "comment": resp["comment"], "rating": resp["rating"], "posted_at": resp["posted_at"], - "date": text.parse_datetime( - resp["posted_at"], "%a, %d %b %Y %H:%M:%S %z"), + "date": text.parse_datetime(resp["posted_at"], "%a, %d %b %Y %H:%M:%S %z"), "fanclub_id": resp["fanclub"]["id"], "fanclub_user_id": resp["fanclub"]["user"]["id"], "fanclub_user_name": resp["fanclub"]["user"]["name"], "fanclub_name": resp["fanclub"]["name"], - "fanclub_url": self.root+"/fanclubs/"+str(resp["fanclub"]["id"]), + "fanclub_url": self.root + "/fanclubs/" + str(resp["fanclub"]["id"]), "tags": [t["name"] for t in resp["tags"]], "_data": resp, } @@ -120,14 +119,17 @@ def _get_post_contents(self, post): except Exception: pass else: - contents.insert(0, { - "id": "thumb", - "title": "thumb", - "category": "thumb", - "download_uri": url, - "visible_status": "visible", - "plan": None, - }) + contents.insert( + 0, + { + "id": "thumb", + "title": "thumb", + "category": "thumb", + "download_uri": url, + "visible_status": "visible", + "plan": None, + }, + ) return contents @@ -144,15 +146,13 @@ def _process_content(self, post, content): if "post_content_photos" in content: for photo in content["post_content_photos"]: - files.append({"file_id" : photo["id"], - "file_url": photo["url"]["original"]}) + files.append({"file_id": photo["id"], "file_url": photo["url"]["original"]}) if "download_uri" in content: url = content["download_uri"] if url[0] == "/": url = self.root + url - files.append({"file_id" : content["id"], - "file_url": url}) + files.append({"file_id": content["id"], "file_url": url}) if content["category"] == "blog" and "comment" in content: comment_json = util.json_loads(content["comment"]) @@ -164,8 +164,9 @@ def _process_content(self, post, content): blog_text += insert elif isinstance(insert, dict) and "fantiaImage" in insert: img = insert["fantiaImage"] - files.append({"file_id" : img["id"], - "file_url": self.root + img["original_url"]}) + files.append( + {"file_id": img["id"], "file_url": self.root + img["original_url"]} + ) post["blogpost_text"] = blog_text else: post["blogpost_text"] = "" @@ -175,6 +176,7 @@ def _process_content(self, post, content): class FantiaCreatorExtractor(FantiaExtractor): """Extractor for a Fantia creator's works""" + subcategory = "creator" pattern = r"(?:https?://)?(?:www\.)?fantia\.jp/fanclubs/(\d+)" example = "https://fantia.jp/fanclubs/12345" @@ -184,12 +186,13 @@ def __init__(self, match): self.creator_id = match.group(1) def posts(self): - url = "{}/fanclubs/{}/posts".format(self.root, self.creator_id) + url = f"{self.root}/fanclubs/{self.creator_id}/posts" return self._pagination(url) class FantiaPostExtractor(FantiaExtractor): """Extractor for media from a single Fantia post""" + subcategory = "post" pattern = r"(?:https?://)?(?:www\.)?fantia\.jp/posts/(\d+)" example = "https://fantia.jp/posts/12345" diff --git a/gallery_dl/extractor/fapachi.py b/gallery_dl/extractor/fapachi.py index 80478caa39..e97af8a96f 100644 --- a/gallery_dl/extractor/fapachi.py +++ b/gallery_dl/extractor/fapachi.py @@ -1,25 +1,24 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://fapachi.com/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class FapachiPostExtractor(Extractor): """Extractor for individual posts on fapachi.com""" + category = "fapachi" subcategory = "post" root = "https://fapachi.com" directory_fmt = ("{category}", "{user}") filename_fmt = "{user}_{id}.{extension}" archive_fmt = "{user}_{id}" - pattern = (r"(?:https?://)?(?:www\.)?fapachi\.com" - r"/(?!search/)([^/?#]+)/media/(\d+)") + pattern = r"(?:https?://)?(?:www\.)?fapachi\.com" r"/(?!search/)([^/?#]+)/media/(\d+)" example = "https://fapachi.com/MODEL/media/12345" def __init__(self, match): @@ -29,10 +28,9 @@ def __init__(self, match): def items(self): data = { "user": self.user, - "id" : self.id, + "id": self.id, } - page = self.request("{}/{}/media/{}".format( - self.root, self.user, self.id)).text + page = self.request(f"{self.root}/{self.user}/media/{self.id}").text url = self.root + text.extr(page, 'd-block" src="', '"') yield Message.Directory, data yield Message.Url, url, text.nameext_from_url(url, data) @@ -40,11 +38,13 @@ def items(self): class FapachiUserExtractor(Extractor): """Extractor for all posts from a fapachi user""" + category = "fapachi" subcategory = "user" root = "https://fapachi.com" - pattern = (r"(?:https?://)?(?:www\.)?fapachi\.com" - r"/(?!search(?:/|$))([^/?#]+)(?:/page/(\d+))?$") + pattern = ( + r"(?:https?://)?(?:www\.)?fapachi\.com" r"/(?!search(?:/|$))([^/?#]+)(?:/page/(\d+))?$" + ) example = "https://fapachi.com/MODEL" def __init__(self, match): @@ -55,8 +55,7 @@ def __init__(self, match): def items(self): data = {"_extractor": FapachiPostExtractor} while True: - page = self.request("{}/{}/page/{}".format( - self.root, self.user, self.num)).text + page = self.request(f"{self.root}/{self.user}/page/{self.num}").text for post in text.extract_iter(page, 'model-media-prew">', ">"): path = text.extr(post, '", None) + self.request(url, allow_redirects=False).text, 'class="uk-align-center"', "", None + ) if page is None: raise exception.NotFoundError("post") data = { "model": self.model, - "id" : text.parse_int(self.id), - "type" : "video" if 'type="video' in page else "photo", + "id": text.parse_int(self.id), + "type": "video" if 'type="video' in page else "photo", "thumbnail": text.extr(page, 'poster="', '"'), } - url = text.extr(page, 'src="', '"').replace( - ".md", "").replace(".th", "") + url = text.extr(page, 'src="', '"').replace(".md", "").replace(".th", "") yield Message.Directory, data yield Message.Url, url, text.nameext_from_url(url, data) class FapelloModelExtractor(Extractor): """Extractor for all posts from a fapello model""" + category = "fapello" subcategory = "model" - pattern = (BASE_PATTERN + r"/(?!top-(?:likes|followers)|popular_videos" - r"|videos|trending|search/?$)" - r"([^/?#]+)/?$") + pattern = ( + BASE_PATTERN + r"/(?!top-(?:likes|followers)|popular_videos" + r"|videos|trending|search/?$)" + r"([^/?#]+)/?$" + ) example = "https://fapello.com/model/" def __init__(self, match): @@ -66,8 +68,7 @@ def items(self): num = 1 data = {"_extractor": FapelloPostExtractor} while True: - url = "{}/ajax/model/{}/page-{}/".format( - self.root, self.model, num) + url = f"{self.root}/ajax/model/{self.model}/page-{num}/" page = self.request(url).text if not page: return @@ -81,11 +82,13 @@ def items(self): class FapelloPathExtractor(Extractor): """Extractor for models and posts from fapello.com paths""" + category = "fapello" subcategory = "path" - pattern = (BASE_PATTERN + - r"/(?!search/?$)(top-(?:likes|followers)|videos|trending" - r"|popular_videos/[^/?#]+)/?$") + pattern = ( + BASE_PATTERN + r"/(?!search/?$)(top-(?:likes|followers)|videos|trending" + r"|popular_videos/[^/?#]+)/?$" + ) example = "https://fapello.com/trending/" def __init__(self, match): @@ -106,12 +109,10 @@ def items(self): data = {"_extractor": FapelloModelExtractor} while True: - page = self.request("{}/ajax/{}/page-{}/".format( - self.root, self.path, num)).text + page = self.request(f"{self.root}/ajax/{self.path}/page-{num}/").text if not page: return - for item in text.extract_iter( - page, 'uk-transition-toggle">', ""): + for item in text.extract_iter(page, 'uk-transition-toggle">', ""): yield Message.Queue, text.extr(item, ' 0 and (int(size["width"]) > self.maxsize or - int(size["height"]) > self.maxsize): + if index > 0 and ( + int(size["width"]) > self.maxsize or int(size["height"]) > self.maxsize + ): del sizes[index:] break return sizes @@ -379,17 +391,19 @@ def urls_lookupGroup(self, groupname): """Returns a group NSID, given the url to a group's page.""" params = {"url": "https://www.flickr.com/groups/" + groupname} group = self._call("urls.lookupGroup", params)["group"] - return {"nsid": group["id"], - "path_alias": groupname, - "groupname": group["groupname"]["_content"]} + return { + "nsid": group["id"], + "path_alias": groupname, + "groupname": group["groupname"]["_content"], + } def urls_lookupUser(self, username): """Returns a user NSID, given the url to a user's photos or profile.""" params = {"url": "https://www.flickr.com/photos/" + username} user = self._call("urls.lookupUser", params)["user"] return { - "nsid" : user["id"], - "username" : user["username"]["_content"], + "nsid": user["id"], + "username": user["username"]["_content"], "path_alias": username, } @@ -418,25 +432,25 @@ def _call(self, method, params): self.log.debug("Server response: %s", data) if data["code"] == 1: raise exception.NotFoundError(self.extractor.subcategory) - elif data["code"] == 2: + if data["code"] == 2: raise exception.AuthorizationError(msg) - elif data["code"] == 98: + if data["code"] == 98: raise exception.AuthenticationError(msg) - elif data["code"] == 99: + if data["code"] == 99: raise exception.AuthorizationError(msg) raise exception.StopExtraction("API request failed: %s", msg) return data def _pagination(self, method, params, key="photos"): - extras = ("description,date_upload,tags,views,media," - "path_alias,owner_name,") + extras = "description,date_upload,tags,views,media," "path_alias,owner_name," includes = self.extractor.config("metadata") if includes: if isinstance(includes, (list, tuple)): includes = ",".join(includes) elif not isinstance(includes, str): - includes = ("license,date_taken,original_format,last_update," - "geo,machine_tags,o_dims") + includes = ( + "license,date_taken,original_format,last_update," "geo,machine_tags,o_dims" + ) extras = extras + includes + "," extras += ",".join("url_" + fmt[0] for fmt in self.formats) @@ -471,8 +485,8 @@ def _extract_format(self, photo): if "owner" in photo: photo["owner"] = { - "nsid" : photo["owner"], - "username" : photo["ownername"], + "nsid": photo["owner"], + "username": photo["ownername"], "path_alias": photo["pathalias"], } else: @@ -488,17 +502,15 @@ def _extract_format(self, photo): if key in photo: photo["width"] = text.parse_int(photo["width_" + fmt]) photo["height"] = text.parse_int(photo["height_" + fmt]) - if self.maxsize and (photo["width"] > self.maxsize or - photo["height"] > self.maxsize): + if self.maxsize and ( + photo["width"] > self.maxsize or photo["height"] > self.maxsize + ): continue photo["url"] = photo[key] photo["label"] = fmtname # remove excess data - keys = [ - key for key in photo - if key.startswith(("url_", "width_", "height_")) - ] + keys = [key for key in photo if key.startswith(("url_", "width_", "height_"))] for key in keys: del photo[key] break @@ -529,7 +541,10 @@ def _extract_metadata(self, photo): except Exception as exc: self.log.warning( "Unable to retrieve 'exif' data for %s (%s: %s)", - photo["id"], exc.__class__.__name__, exc) + photo["id"], + exc.__class__.__name__, + exc, + ) if self.contexts: try: @@ -537,7 +552,10 @@ def _extract_metadata(self, photo): except Exception as exc: self.log.warning( "Unable to retrieve 'contexts' data for %s (%s: %s)", - photo["id"], exc.__class__.__name__, exc) + photo["id"], + exc.__class__.__name__, + exc, + ) @staticmethod def _clean_info(info): diff --git a/gallery_dl/extractor/foolfuuka.py b/gallery_dl/extractor/foolfuuka.py index 44c4542b53..e99c376115 100644 --- a/gallery_dl/extractor/foolfuuka.py +++ b/gallery_dl/extractor/foolfuuka.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,16 @@ """Extractors for FoolFuuka 4chan archives""" -from .common import BaseExtractor, Message -from .. import text import itertools +from .. import text +from .common import BaseExtractor +from .common import Message + class FoolfuukaExtractor(BaseExtractor): """Base extractor for FoolFuuka based boards/archives""" + basecategory = "foolfuuka" filename_fmt = "{timestamp_ms} {filename_media}.{extension}" archive_fmt = "{board[shortname]}_{num}_{timestamp}" @@ -40,11 +41,9 @@ def items(self): if url and url[0] == "/": url = self.root + url - post["filename"], _, post["extension"] = \ - media["media"].rpartition(".") + post["filename"], _, post["extension"] = media["media"].rpartition(".") post["filename_media"] = media["media_filename"].rpartition(".")[0] - post["timestamp_ms"] = text.parse_int( - media["media_orig"].rpartition(".")[0]) + post["timestamp_ms"] = text.parse_int(media["media_orig"].rpartition(".")[0]) yield Message.Url, url, post def metadata(self): @@ -57,8 +56,7 @@ def remote(self, media): """Resolve a remote media link""" page = self.request(media["remote_media_link"]).text url = text.extr(page, 'http-equiv="Refresh" content="0; url=', '"') - if url.endswith(".webm") and \ - url.startswith("https://thebarchive.com/"): + if url.endswith(".webm") and url.startswith("https://thebarchive.com/"): return url[:-1] return url @@ -67,51 +65,53 @@ def _remote_direct(media): return media["remote_media_link"] -BASE_PATTERN = FoolfuukaExtractor.update({ - "4plebs": { - "root": "https://archive.4plebs.org", - "pattern": r"(?:archive\.)?4plebs\.org", - }, - "archivedmoe": { - "root": "https://archived.moe", - "pattern": r"archived\.moe", - }, - "archiveofsins": { - "root": "https://archiveofsins.com", - "pattern": r"(?:www\.)?archiveofsins\.com", - }, - "b4k": { - "root": "https://arch.b4k.co", - "pattern": r"arch\.b4k\.co", - }, - "desuarchive": { - "root": "https://desuarchive.org", - "pattern": r"desuarchive\.org", - }, - "fireden": { - "root": "https://boards.fireden.net", - "pattern": r"boards\.fireden\.net", - }, - "palanq": { - "root": "https://archive.palanq.win", - "pattern": r"archive\.palanq\.win", - }, - "rbt": { - "root": "https://rbt.asia", - "pattern": r"(?:rbt\.asia|(?:archive\.)?rebeccablacktech\.com)", - }, - "thebarchive": { - "root": "https://thebarchive.com", - "pattern": r"thebarchive\.com", - }, -}) +BASE_PATTERN = FoolfuukaExtractor.update( + { + "4plebs": { + "root": "https://archive.4plebs.org", + "pattern": r"(?:archive\.)?4plebs\.org", + }, + "archivedmoe": { + "root": "https://archived.moe", + "pattern": r"archived\.moe", + }, + "archiveofsins": { + "root": "https://archiveofsins.com", + "pattern": r"(?:www\.)?archiveofsins\.com", + }, + "b4k": { + "root": "https://arch.b4k.co", + "pattern": r"arch\.b4k\.co", + }, + "desuarchive": { + "root": "https://desuarchive.org", + "pattern": r"desuarchive\.org", + }, + "fireden": { + "root": "https://boards.fireden.net", + "pattern": r"boards\.fireden\.net", + }, + "palanq": { + "root": "https://archive.palanq.win", + "pattern": r"archive\.palanq\.win", + }, + "rbt": { + "root": "https://rbt.asia", + "pattern": r"(?:rbt\.asia|(?:archive\.)?rebeccablacktech\.com)", + }, + "thebarchive": { + "root": "https://thebarchive.com", + "pattern": r"thebarchive\.com", + }, + } +) class FoolfuukaThreadExtractor(FoolfuukaExtractor): """Base extractor for threads on FoolFuuka based boards/archives""" + subcategory = "thread" - directory_fmt = ("{category}", "{board[shortname]}", - "{thread_num} {title|comment[:50]}") + directory_fmt = ("{category}", "{board[shortname]}", "{thread_num} {title|comment[:50]}") pattern = BASE_PATTERN + r"/([^/?#]+)/thread/(\d+)" example = "https://archived.moe/a/thread/12345/" @@ -139,6 +139,7 @@ def posts(self): class FoolfuukaBoardExtractor(FoolfuukaExtractor): """Base extractor for FoolFuuka based boards/archives""" + subcategory = "board" pattern = BASE_PATTERN + r"/([^/?#]+)(?:/(?:page/)?(\d*))?$" example = "https://archived.moe/a/" @@ -149,9 +150,8 @@ def __init__(self, match): self.page = self.groups[-1] def items(self): - index_base = "{}/_/api/chan/index/?board={}&page=".format( - self.root, self.board) - thread_base = "{}/{}/thread/".format(self.root, self.board) + index_base = f"{self.root}/_/api/chan/index/?board={self.board}&page=" + thread_base = f"{self.root}/{self.board}/thread/" page = self.page for pnum in itertools.count(text.parse_int(page, 1)): @@ -175,6 +175,7 @@ def items(self): class FoolfuukaSearchExtractor(FoolfuukaExtractor): """Base extractor for search results on FoolFuuka based boards/archives""" + subcategory = "search" directory_fmt = ("{category}", "search", "{search}") pattern = BASE_PATTERN + r"/([^/?#]+)/search((?:/[^/?#]+/[^/?#]+)+)" @@ -230,6 +231,7 @@ def posts(self): class FoolfuukaGalleryExtractor(FoolfuukaExtractor): """Base extractor for FoolFuuka galleries""" + subcategory = "gallery" directory_fmt = ("{category}", "{board}", "gallery") pattern = BASE_PATTERN + r"/([^/?#]+)/gallery(?:/(\d+))?" @@ -240,7 +242,7 @@ def __init__(self, match): board = match.group(match.lastindex) if board.isdecimal(): - self.board = match.group(match.lastindex-1) + self.board = match.group(match.lastindex - 1) self.pages = (board,) else: self.board = board @@ -250,8 +252,7 @@ def metadata(self): return {"board": self.board} def posts(self): - base = "{}/_/api/chan/gallery/?board={}&page=".format( - self.root, self.board) + base = f"{self.root}/_/api/chan/gallery/?board={self.board}&page=" for page in self.pages: with self.request(base + page) as response: diff --git a/gallery_dl/extractor/foolslide.py b/gallery_dl/extractor/foolslide.py index bb684c2d62..8c8633ce08 100644 --- a/gallery_dl/extractor/foolslide.py +++ b/gallery_dl/extractor/foolslide.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,15 @@ """Extractors for FoOlSlide based sites""" -from .common import BaseExtractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import BaseExtractor +from .common import Message class FoolslideExtractor(BaseExtractor): """Base class for FoOlSlide extractors""" + basecategory = "foolslide" def __init__(self, match): @@ -22,7 +23,8 @@ def __init__(self, match): def request(self, url): return BaseExtractor.request( - self, url, encoding="utf-8", method="POST", data={"adult": "true"}) + self, url, encoding="utf-8", method="POST", data={"adult": "true"} + ) @staticmethod def parse_chapter_url(url, data): @@ -37,16 +39,15 @@ def parse_chapter_url(url, data): return data -BASE_PATTERN = FoolslideExtractor.update({ -}) +BASE_PATTERN = FoolslideExtractor.update({}) class FoolslideChapterExtractor(FoolslideExtractor): """Base class for chapter extractors for FoOlSlide based sites""" + subcategory = "chapter" directory_fmt = ("{category}", "{manga}", "{chapter_string}") - filename_fmt = ( - "{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}") + filename_fmt = "{manga}_c{chapter:>03}{chapter_minor:?//}_{page:>03}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"(/read/[^/?#]+/[a-z-]+/\d+/\d+(?:/\d+)?)" example = "https://read.powermanga.org/read/MANGA/en/0/123/" @@ -60,8 +61,7 @@ def items(self): data["chapter_id"] = text.parse_int(imgs[0]["chapter_id"]) yield Message.Directory, data - enum = util.enumerate_reversed if self.config( - "page-reverse") else enumerate + enum = util.enumerate_reversed if self.config("page-reverse") else enumerate for data["page"], image in enum(imgs, 1): try: url = image["url"] @@ -78,11 +78,14 @@ def items(self): def metadata(self, page): extr = text.extract_from(page) - extr('

', '') - return self.parse_chapter_url(self.gallery_url, { - "manga" : text.unescape(extr('title="', '"')).strip(), - "chapter_string": text.unescape(extr('title="', '"')), - }) + extr('

', "") + return self.parse_chapter_url( + self.gallery_url, + { + "manga": text.unescape(extr('title="', '"')).strip(), + "chapter_string": text.unescape(extr('title="', '"')), + }, + ) def images(self, page): return util.json_loads(text.extr(page, "var pages = ", ";")) @@ -90,6 +93,7 @@ def images(self, page): class FoolslideMangaExtractor(FoolslideExtractor): """Base class for manga extractors for FoOlSlide based sites""" + subcategory = "manga" categorytransfer = True pattern = BASE_PATTERN + r"(/series/[^/?#]+)" @@ -108,17 +112,27 @@ def items(self): def chapters(self, page): extr = text.extract_from(page) - manga = text.unescape(extr('

', '

')).strip() - author = extr('Author: ', 'Artist: ', '', "")).strip() + author = extr("Author: ", "Artist: ", "")) + self._new_layout = "http-equiv=" not in extr("") path = extr('href="//d', '"') if not path: msg = text.remove_html( - extr('System Message', '') or - extr('System Message', '
') + extr("System Message", "") or extr("System Message", "") ).partition(" . Continue ")[0] - return self.log.warning( - "Unable to download post %s (\"%s\")", post_id, msg) + return self.log.warning('Unable to download post %s ("%s")', post_id, msg) pi = text.parse_int rh = text.remove_html - data = text.nameext_from_url(path, { - "id" : pi(post_id), - "url": "https://d" + path, - }) + data = text.nameext_from_url( + path, + { + "id": pi(post_id), + "url": "https://d" + path, + }, + ) if self._new_layout: - data["tags"] = text.split_html(extr( - 'class="tags-row">', '')) + data["tags"] = text.split_html(extr('class="tags-row">', "")) data["title"] = text.unescape(extr("

", "

")) data["artist"] = extr("", "<") data["_description"] = extr( 'class="submission-description user-submitted-links">', - ' ') - data["views"] = pi(rh(extr('class="views">', ''))) - data["favorites"] = pi(rh(extr('class="favorites">', ''))) - data["comments"] = pi(rh(extr('class="comments">', ''))) - data["rating"] = rh(extr('class="rating">', '')) - data["fa_category"] = rh(extr('>Category', '')) - data["theme"] = rh(extr('>', '<')) - data["species"] = rh(extr('>Species', '')) - data["gender"] = rh(extr('>Gender', '')) + " ", + ) + data["views"] = pi(rh(extr('class="views">', ""))) + data["favorites"] = pi(rh(extr('class="favorites">', ""))) + data["comments"] = pi(rh(extr('class="comments">', ""))) + data["rating"] = rh(extr('class="rating">', "")) + data["fa_category"] = rh(extr(">Category", "")) + data["theme"] = rh(extr(">", "<")) + data["species"] = rh(extr(">Species", "")) + data["gender"] = rh(extr(">Gender", "")) data["width"] = pi(extr("", "x")) data["height"] = pi(extr("", "p")) data["folders"] = folders = [] - for folder in extr( - "

Listed in Folders

", "").split(""): + for folder in extr("

Listed in Folders

", "").split(""): folder = rh(folder) if folder: folders.append(folder) @@ -130,12 +130,12 @@ def _parse_post(self, post_id): data["views"] = pi(extr("Views:", "<")) data["width"] = pi(extr("Resolution:", "x")) data["height"] = pi(extr("", "<")) - data["tags"] = text.split_html(extr( - 'id="keywords">', ''))[::2] - data["rating"] = extr('', ' ') + data[', ""))[::2] + data["rating"] = extr('', ', ' ') + '', + " ", + ) data["folders"] = () # folders not present in old layout data["artist_url"] = data["artist"].replace("_", "").lower() @@ -143,7 +143,8 @@ def _parse_post(self, post_id): data["date"] = text.parse_timestamp(data["filename"].partition(".")[0]) data["description"] = self._process_description(data["_description"]) data["thumbnail"] = "https://t.furaffinity.net/{}@600-{}.jpg".format( - post_id, path.rsplit("/", 2)[1]) + post_id, path.rsplit("/", 2)[1] + ) return data @@ -155,8 +156,7 @@ def _pagination(self, path): num = 1 while True: - url = "{}/{}/{}/{}/".format( - self.root, path, self.user, num) + url = f"{self.root}/{path}/{self.user}/{num}/" page = self.request(url).text post_id = None @@ -168,7 +168,7 @@ def _pagination(self, path): num += 1 def _pagination_favorites(self): - path = "/favorites/{}/".format(self.user) + path = f"/favorites/{self.user}/" while path: page = self.request(self.root + path).text @@ -189,22 +189,22 @@ def _pagination_favorites(self): def _pagination_search(self, query): url = self.root + "/search/" data = { - "page" : 1, - "order-by" : "relevancy", + "page": 1, + "order-by": "relevancy", "order-direction": "desc", - "range" : "all", - "range_from" : "", - "range_to" : "", - "rating-general" : "1", - "rating-mature" : "1", - "rating-adult" : "1", - "type-art" : "1", - "type-music" : "1", - "type-flash" : "1", - "type-story" : "1", - "type-photo" : "1", - "type-poetry" : "1", - "mode" : "extended", + "range": "all", + "range_from": "", + "range_to": "", + "rating-general": "1", + "rating-mature": "1", + "rating-adult": "1", + "type-art": "1", + "type-music": "1", + "type-flash": "1", + "type-story": "1", + "type-photo": "1", + "type-poetry": "1", + "mode": "extended", } data.update(query) @@ -229,6 +229,7 @@ def _pagination_search(self, query): class FuraffinityGalleryExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's gallery""" + subcategory = "gallery" pattern = BASE_PATTERN + r"/gallery/([^/?#]+)" example = "https://www.furaffinity.net/gallery/USER/" @@ -239,6 +240,7 @@ def posts(self): class FuraffinityScrapsExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's scraps""" + subcategory = "scraps" directory_fmt = ("{category}", "{user!l}", "Scraps") pattern = BASE_PATTERN + r"/scraps/([^/?#]+)" @@ -250,6 +252,7 @@ def posts(self): class FuraffinityFavoriteExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's favorites""" + subcategory = "favorite" directory_fmt = ("{category}", "{user!l}", "Favorites") pattern = BASE_PATTERN + r"/favorites/([^/?#]+)" @@ -267,6 +270,7 @@ def _parse_post(self, post_id): class FuraffinitySearchExtractor(FuraffinityExtractor): """Extractor for furaffinity search results""" + subcategory = "search" directory_fmt = ("{category}", "Search", "{search}") pattern = BASE_PATTERN + r"/search(?:/([^/?#]+))?/?[?&]([^#]+)" @@ -287,6 +291,7 @@ def posts(self): class FuraffinityPostExtractor(FuraffinityExtractor): """Extractor for individual posts on furaffinity""" + subcategory = "post" pattern = BASE_PATTERN + r"/(?:view|full)/(\d+)" example = "https://www.furaffinity.net/view/12345/" @@ -299,6 +304,7 @@ def posts(self): class FuraffinityUserExtractor(FuraffinityExtractor): """Extractor for furaffinity user profiles""" + subcategory = "user" cookies_domain = None pattern = BASE_PATTERN + r"/user/([^/?#]+)" @@ -310,22 +316,26 @@ def initialize(self): skip = Extractor.skip def items(self): - base = "{}/{{}}/{}/".format(self.root, self.user) - return self._dispatch_extractors(( - (FuraffinityGalleryExtractor , base.format("gallery")), - (FuraffinityScrapsExtractor , base.format("scraps")), - (FuraffinityFavoriteExtractor, base.format("favorites")), - ), ("gallery",)) + base = f"{self.root}/{{}}/{self.user}/" + return self._dispatch_extractors( + ( + (FuraffinityGalleryExtractor, base.format("gallery")), + (FuraffinityScrapsExtractor, base.format("scraps")), + (FuraffinityFavoriteExtractor, base.format("favorites")), + ), + ("gallery",), + ) class FuraffinityFollowingExtractor(FuraffinityExtractor): """Extractor for a furaffinity user's watched users""" + subcategory = "following" pattern = BASE_PATTERN + "/watchlist/by/([^/?#]+)" example = "https://www.furaffinity.net/watchlist/by/USER/" def items(self): - url = "{}/watchlist/by/{}/".format(self.root, self.user) + url = f"{self.root}/watchlist/by/{self.user}/" data = {"_extractor": FuraffinityUserExtractor} while True: @@ -342,6 +352,7 @@ def items(self): class FuraffinitySubmissionsExtractor(FuraffinityExtractor): """Extractor for new furaffinity submissions""" + subcategory = "submissions" pattern = BASE_PATTERN + r"(/msg/submissions(?:/[^/?#]+)?)" example = "https://www.furaffinity.net/msg/submissions" @@ -355,12 +366,13 @@ def _pagination_submissions(self, url): while True: page = self.request(url).text - for post_id in text.extract_iter(page, 'id="sid-', '"'): - yield post_id + yield from text.extract_iter(page, 'id="sid-', '"') - path = (text.extr(page, '", "").strip() title, _, gallery_id = title.rpartition("#") return { - "gallery_id" : text.parse_int(gallery_id), + "gallery_id": text.parse_int(gallery_id), "gallery_hash": self.gallery_hash, - "title" : text.unescape(title[:-15]), - "views" : data.get("hits"), - "score" : data.get("rating"), - "tags" : (data.get("tags") or "").split(","), + "title": text.unescape(title[:-15]), + "views": data.get("hits"), + "score": data.get("rating"), + "tags": (data.get("tags") or "").split(","), } def images(self, page): - return [ - ("https:" + image["imageUrl"], image) - for image in self.data["images"] - ] + return [("https:" + image["imageUrl"], image) for image in self.data["images"]] class FuskatorSearchExtractor(Extractor): """Extractor for search results on fuskator.com""" + category = "fuskator" subcategory = "search" root = "https://fuskator.com" @@ -80,11 +84,10 @@ def items(self): while True: page = self.request(url).text - for path in text.extract_iter( - page, 'class="pic_pad">', '>>><') + pages = text.extr(page, 'class="pages">', ">>><") if not pages: return url = self.root + text.rextract(pages, 'href="', '"')[0] diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py index 37c776e6a5..dbb457f078 100644 --- a/gallery_dl/extractor/gelbooru.py +++ b/gallery_dl/extractor/gelbooru.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2014-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,16 +6,20 @@ """Extractors for https://gelbooru.com/""" -from .common import Extractor, Message -from . import gelbooru_v02 -from .. import text, exception import binascii +from .. import exception +from .. import text +from . import gelbooru_v02 +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?(?:www\.)?gelbooru\.com/(?:index\.php)?\?" -class GelbooruBase(): +class GelbooruBase: """Base class for gelbooru extractors""" + category = "gelbooru" basecategory = "booru" root = "https://gelbooru.com" @@ -112,7 +114,7 @@ def _file_url(post): url = post["file_url"] if url.endswith((".webm", ".mp4")): md5 = post["md5"] - path = "/images/{}/{}/{}.webm".format(md5[0:2], md5[2:4], md5) + path = f"/images/{md5[0:2]}/{md5[2:4]}/{md5}.webm" post["_fallback"] = GelbooruBase._video_fallback(path) url = "https://img3.gelbooru.com" + path return url @@ -123,36 +125,38 @@ def _video_fallback(path): yield "https://img1.gelbooru.com" + path def _notes(self, post, page): - notes_data = text.extr(page, '
') + notes_data = text.extr(page, '
") if not notes_data: return post["notes"] = notes = [] extr = text.extract - for note in text.extract_iter(notes_data, ''): - notes.append({ - "width" : int(extr(note, 'data-width="', '"')[0]), - "height": int(extr(note, 'data-height="', '"')[0]), - "x" : int(extr(note, 'data-x="', '"')[0]), - "y" : int(extr(note, 'data-y="', '"')[0]), - "body" : extr(note, 'data-body="', '"')[0], - }) + for note in text.extract_iter(notes_data, ""): + notes.append( + { + "width": int(extr(note, 'data-width="', '"')[0]), + "height": int(extr(note, 'data-height="', '"')[0]), + "x": int(extr(note, 'data-x="', '"')[0]), + "y": int(extr(note, 'data-y="', '"')[0]), + "body": extr(note, 'data-body="', '"')[0], + } + ) def _skip_offset(self, num): self.offset += num return num -class GelbooruTagExtractor(GelbooruBase, - gelbooru_v02.GelbooruV02TagExtractor): +class GelbooruTagExtractor(GelbooruBase, gelbooru_v02.GelbooruV02TagExtractor): """Extractor for images from gelbooru.com based on search-tags""" + pattern = BASE_PATTERN + r"page=post&s=list&tags=([^&#]*)" example = "https://gelbooru.com/index.php?page=post&s=list&tags=TAG" -class GelbooruPoolExtractor(GelbooruBase, - gelbooru_v02.GelbooruV02PoolExtractor): +class GelbooruPoolExtractor(GelbooruBase, gelbooru_v02.GelbooruV02PoolExtractor): """Extractor for gelbooru pools""" + per_page = 45 pattern = BASE_PATTERN + r"page=pool&s=show&id=(\d+)" example = "https://gelbooru.com/index.php?page=pool&s=show&id=12345" @@ -163,8 +167,8 @@ def metadata(self): url = self.root + "/index.php" self._params = { "page": "pool", - "s" : "show", - "id" : self.pool_id, + "s": "show", + "id": self.pool_id, } page = self.request(url, params=self._params).text @@ -181,9 +185,9 @@ def posts(self): return self._pagination_html(self._params) -class GelbooruFavoriteExtractor(GelbooruBase, - gelbooru_v02.GelbooruV02FavoriteExtractor): +class GelbooruFavoriteExtractor(GelbooruBase, gelbooru_v02.GelbooruV02FavoriteExtractor): """Extractor for gelbooru favorites""" + per_page = 100 pattern = BASE_PATTERN + r"page=favorites&s=view&id=(\d+)" example = "https://gelbooru.com/index.php?page=favorites&s=view&id=12345" @@ -193,8 +197,8 @@ class GelbooruFavoriteExtractor(GelbooruBase, def posts(self): # get number of favorites params = { - "s" : "favorite", - "id" : self.favorite_id, + "s": "favorite", + "id": self.favorite_id, "limit": "2", } data = self._api_request(params, None, True) @@ -207,12 +211,11 @@ def posts(self): order = 1 if favs[0]["id"] < favs[1]["id"] else -1 except LookupError as exc: self.log.debug( - "Error when determining API favorite order (%s: %s)", - exc.__class__.__name__, exc) + "Error when determining API favorite order (%s: %s)", exc.__class__.__name__, exc + ) order = -1 else: - self.log.debug("API yields favorites in %sscending order", - "a" if order > 0 else "de") + self.log.debug("API yields favorites in %sscending order", "a" if order > 0 else "de") order_favs = self.config("order-posts") if order_favs and order_favs[0] in ("r", "a"): @@ -250,11 +253,11 @@ def _pagination(self, params, count): params["pid"] += 1 def _pagination_reverse(self, params, count): - pnum, last = divmod(count-1, self.per_page) + pnum, last = divmod(count - 1, self.per_page) if self.offset > last: # page number change self.offset -= last - diff, self.offset = divmod(self.offset-1, self.per_page) + diff, self.offset = divmod(self.offset - 1, self.per_page) pnum -= diff + 1 skip = self.offset @@ -279,20 +282,20 @@ def _pagination_reverse(self, params, count): return -class GelbooruPostExtractor(GelbooruBase, - gelbooru_v02.GelbooruV02PostExtractor): +class GelbooruPostExtractor(GelbooruBase, gelbooru_v02.GelbooruV02PostExtractor): """Extractor for single images from gelbooru.com""" - pattern = (BASE_PATTERN + - r"(?=(?:[^#]+&)?page=post(?:&|#|$))" - r"(?=(?:[^#]+&)?s=view(?:&|#|$))" - r"(?:[^#]+&)?id=(\d+)") + + pattern = ( + BASE_PATTERN + r"(?=(?:[^#]+&)?page=post(?:&|#|$))" + r"(?=(?:[^#]+&)?s=view(?:&|#|$))" + r"(?:[^#]+&)?id=(\d+)" + ) example = "https://gelbooru.com/index.php?page=post&s=view&id=12345" class GelbooruRedirectExtractor(GelbooruBase, Extractor): subcategory = "redirect" - pattern = (r"(?:https?://)?(?:www\.)?gelbooru\.com" - r"/redirect\.php\?s=([^&#]+)") + pattern = r"(?:https?://)?(?:www\.)?gelbooru\.com" r"/redirect\.php\?s=([^&#]+)" example = "https://gelbooru.com/redirect.php?s=BASE64" def __init__(self, match): @@ -300,7 +303,6 @@ def __init__(self, match): self.url_base64 = match.group(1) def items(self): - url = text.ensure_http_scheme(binascii.a2b_base64( - self.url_base64).decode()) + url = text.ensure_http_scheme(binascii.a2b_base64(self.url_base64).decode()) data = {"_extractor": GelbooruPostExtractor} yield Message.Queue, url, data diff --git a/gallery_dl/extractor/gelbooru_v01.py b/gallery_dl/extractor/gelbooru_v01.py index 0b96048b64..1d260a69a3 100644 --- a/gallery_dl/extractor/gelbooru_v01.py +++ b/gallery_dl/extractor/gelbooru_v01.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,8 +6,8 @@ """Extractors for Gelbooru Beta 0.1.11 sites""" -from . import booru from .. import text +from . import booru class GelbooruV01Extractor(booru.BooruExtractor): @@ -17,27 +15,24 @@ class GelbooruV01Extractor(booru.BooruExtractor): per_page = 20 def _parse_post(self, post_id): - url = "{}/index.php?page=post&s=view&id={}".format( - self.root, post_id) + url = f"{self.root}/index.php?page=post&s=view&id={post_id}" extr = text.extract_from(self.request(url).text) post = { - "id" : post_id, - "created_at": extr('Posted: ', ' <'), - "uploader" : extr('By: ', ' <'), - "width" : extr('Size: ', 'x'), - "height" : extr('', ' <'), - "source" : extr('Source: ', ' <'), - "rating" : (extr('Rating: ', '<') or "?")[0].lower(), - "score" : extr('Score: ', ' <'), - "file_url" : extr('img', '<')), + "id": post_id, + "created_at": extr("Posted: ", " <"), + "uploader": extr("By: ", " <"), + "width": extr("Size: ", "x"), + "height": extr("", " <"), + "source": extr("Source: ", " <"), + "rating": (extr("Rating: ", "<") or "?")[0].lower(), + "score": extr("Score: ", " <"), + "file_url": extr('img', "<")), } post["md5"] = post["file_url"].rpartition("/")[2].partition(".")[0] - post["date"] = text.parse_datetime( - post["created_at"], "%Y-%m-%d %H:%M:%S") + post["date"] = text.parse_datetime(post["created_at"], "%Y-%m-%d %H:%M:%S") return post @@ -61,28 +56,30 @@ def _pagination(self, url, begin, end): pid += self.per_page -BASE_PATTERN = GelbooruV01Extractor.update({ - "thecollection": { - "root": "https://the-collection.booru.org", - "pattern": r"the-collection\.booru\.org", - }, - "illusioncardsbooru": { - "root": "https://illusioncards.booru.org", - "pattern": r"illusioncards\.booru\.org", - }, - "allgirlbooru": { - "root": "https://allgirl.booru.org", - "pattern": r"allgirl\.booru\.org", - }, - "drawfriends": { - "root": "https://drawfriends.booru.org", - "pattern": r"drawfriends\.booru\.org", - }, - "vidyart2": { - "root": "https://vidyart2.booru.org", - "pattern": r"vidyart2\.booru\.org", - }, -}) +BASE_PATTERN = GelbooruV01Extractor.update( + { + "thecollection": { + "root": "https://the-collection.booru.org", + "pattern": r"the-collection\.booru\.org", + }, + "illusioncardsbooru": { + "root": "https://illusioncards.booru.org", + "pattern": r"illusioncards\.booru\.org", + }, + "allgirlbooru": { + "root": "https://allgirl.booru.org", + "pattern": r"allgirl\.booru\.org", + }, + "drawfriends": { + "root": "https://drawfriends.booru.org", + "pattern": r"drawfriends\.booru\.org", + }, + "vidyart2": { + "root": "https://vidyart2.booru.org", + "pattern": r"vidyart2\.booru\.org", + }, + } +) class GelbooruV01TagExtractor(GelbooruV01Extractor): @@ -100,8 +97,7 @@ def metadata(self): return {"search_tags": text.unquote(self.tags.replace("+", " "))} def posts(self): - url = "{}/index.php?page=post&s=list&tags={}&pid=".format( - self.root, self.tags) + url = f"{self.root}/index.php?page=post&s=list&tags={self.tags}&pid=" return self._pagination(url, 'class="thumb">')) + tag_container = text.extr(page, '
").replace("\r\n", "\n"), "", "" + ) + ), + "ratings": [ + text.unescape(r) + for r in text.extract_iter(extr("class='ratings_box'", ""), "title='", "'") + ], + "date": text.parse_datetime(extr("datetime='", "'")), + "views": text.parse_int(extr(">Views", "<")), + "score": text.parse_int(extr(">Vote Score", "<")), + "media": text.unescape(extr(">Media", "<").strip()), + "tags": text.split_html(extr(">Tags ", "")), } body = data["_body"] if "", "").rpartition(">")[2]), - "author" : text.unescape(extr('alt="', '"')), - "date" : text.parse_datetime(extr( - ">Updated<", "").rpartition(">")[2], "%B %d, %Y"), - "status" : extr("class='indent'>", "<"), + "user": self.user, + "title": text.unescape(extr("
", "").rpartition(">")[2]), + "author": text.unescape(extr('alt="', '"')), + "date": text.parse_datetime( + extr(">Updated<", "").rpartition(">")[2], "%B %d, %Y" + ), + "status": extr("class='indent'>", "<"), } for c in ("Chapters", "Words", "Comments", "Views", "Rating"): - data[c.lower()] = text.parse_int(extr( - ">" + c + ":", "<").replace(",", "")) + data[c.lower()] = text.parse_int(extr(">" + c + ":", "<").replace(",", "")) - data["description"] = text.unescape(extr( - "class='storyDescript'>", "", ""), "title='", "'")] + data["ratings"] = [ + text.unescape(r) + for r in text.extract_iter(extr("class='ratings_box'", "
"), "title='", "'") + ] return text.nameext_from_url(data["src"], data) @@ -140,16 +141,14 @@ def _request_check(self, url, **kwargs): # and update PHPSESSID and content filters if necessary response = self.request(url, **kwargs) content = response.content - if len(content) < 5000 and \ - b'
', '') + url = f"{self.root}/stories/user/{self.user}" + return self._pagination(url, '
', "") class HentaifoundryStoryExtractor(HentaifoundryExtractor): """Extractor for a hentaifoundry story""" + subcategory = "story" archive_fmt = "s_{index}" pattern = BASE_PATTERN + r"/stories/user/([^/?#]+)/(\d+)" @@ -354,8 +359,7 @@ def __init__(self, match): self.index = match.group(3) def items(self): - story_url = "{}/stories/user/{}/{}/x?enterAgree=1".format( - self.root, self.user, self.index) + story_url = f"{self.root}/stories/user/{self.user}/{self.index}/x?enterAgree=1" story = self._parse_story(self.request(story_url).text) yield Message.Directory, story yield Message.Url, story["src"], story diff --git a/gallery_dl/extractor/hentaifox.py b/gallery_dl/extractor/hentaifox.py index 31a302d168..e77c123d43 100644 --- a/gallery_dl/extractor/hentaifox.py +++ b/gallery_dl/extractor/hentaifox.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,18 +6,23 @@ """Extractors for https://hentaifox.com/""" -from .common import GalleryExtractor, Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import GalleryExtractor +from .common import Message -class HentaifoxBase(): +class HentaifoxBase: """Base class for hentaifox extractors""" + category = "hentaifox" root = "https://hentaifox.com" class HentaifoxGalleryExtractor(HentaifoxBase, GalleryExtractor): """Extractor for image galleries on hentaifox.com""" + pattern = r"(?:https?://)?(?:www\.)?hentaifox\.com(/gallery/(\d+))" example = "https://hentaifox.com/gallery/12345/" @@ -31,8 +34,7 @@ def __init__(self, match): def _split(txt): return [ text.remove_html(tag.partition(">")[2], "", "") - for tag in text.extract_iter( - txt, "class='tag_btn", "Tags:", "")), + "artist": split(extr(">Artists:", "")), + "group": split(extr(">Groups:", "")), + "type": text.remove_html(extr(">Category:", ""): url, _, title = info.partition('">') yield { - "url" : text.urljoin(self.root, url), - "gallery_id": text.parse_int( - url.strip("/").rpartition("/")[2]), - "title" : text.unescape(title), + "url": text.urljoin(self.root, url), + "gallery_id": text.parse_int(url.strip("/").rpartition("/")[2]), + "title": text.unescape(title), "_extractor": HentaifoxGalleryExtractor, } diff --git a/gallery_dl/extractor/hentaihand.py b/gallery_dl/extractor/hentaihand.py index f3f43c4730..c5c2dd9634 100644 --- a/gallery_dl/extractor/hentaihand.py +++ b/gallery_dl/extractor/hentaihand.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,16 @@ """Extractors for https://hentaihand.com/""" -from .common import GalleryExtractor, Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import GalleryExtractor +from .common import Message class HentaihandGalleryExtractor(GalleryExtractor): """Extractor for image galleries on hentaihand.com""" + category = "hentaihand" root = "https://hentaihand.com" pattern = r"(?:https?://)?(?:www\.)?hentaihand\.com/\w+/comic/([\w-]+)" @@ -21,25 +23,23 @@ class HentaihandGalleryExtractor(GalleryExtractor): def __init__(self, match): self.slug = match.group(1) - url = "{}/api/comics/{}".format(self.root, self.slug) + url = f"{self.root}/api/comics/{self.slug}" GalleryExtractor.__init__(self, match, url) def metadata(self, page): info = util.json_loads(page) data = { - "gallery_id" : text.parse_int(info["id"]), - "title" : info["title"], - "title_alt" : info["alternative_title"], - "slug" : self.slug, - "type" : info["category"]["name"], - "language" : info["language"]["name"], - "lang" : util.language_to_code(info["language"]["name"]), - "tags" : [t["slug"] for t in info["tags"]], - "date" : text.parse_datetime( - info["uploaded_at"], "%Y-%m-%d"), + "gallery_id": text.parse_int(info["id"]), + "title": info["title"], + "title_alt": info["alternative_title"], + "slug": self.slug, + "type": info["category"]["name"], + "language": info["language"]["name"], + "lang": util.language_to_code(info["language"]["name"]), + "tags": [t["slug"] for t in info["tags"]], + "date": text.parse_datetime(info["uploaded_at"], "%Y-%m-%d"), } - for key in ("artists", "authors", "groups", "characters", - "relationships", "parodies"): + for key in ("artists", "authors", "groups", "characters", "relationships", "parodies"): data[key] = [v["name"] for v in info[key]] return data @@ -50,12 +50,15 @@ def images(self, _): class HentaihandTagExtractor(Extractor): """Extractor for tag searches on hentaihand.com""" + category = "hentaihand" subcategory = "tag" root = "https://hentaihand.com" - pattern = (r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com" - r"/\w+/(parody|character|tag|artist|group|language" - r"|category|relationship)/([^/?#]+)") + pattern = ( + r"(?i)(?:https?://)?(?:www\.)?hentaihand\.com" + r"/\w+/(parody|character|tag|artist|group|language" + r"|category|relationship)/([^/?#]+)" + ) example = "https://hentaihand.com/en/tag/TAG" def __init__(self, match): @@ -63,22 +66,19 @@ def __init__(self, match): self.type, self.key = match.groups() def items(self): - if self.type[-1] == "y": - tpl = self.type[:-1] + "ies" - else: - tpl = self.type + "s" + tpl = self.type[:-1] + "ies" if self.type[-1] == "y" else self.type + "s" - url = "{}/api/{}/{}".format(self.root, tpl, self.key) + url = f"{self.root}/api/{tpl}/{self.key}" tid = self.request(url, notfound=self.type).json()["id"] url = self.root + "/api/comics" params = { "per_page": "18", - tpl : tid, - "page" : 1, - "q" : "", - "sort" : "uploaded_at", - "order" : "desc", + tpl: tid, + "page": 1, + "q": "", + "sort": "uploaded_at", + "order": "desc", "duration": "day", } while True: diff --git a/gallery_dl/extractor/hentaihere.py b/gallery_dl/extractor/hentaihere.py index ba9558c0dc..0f4f176805 100644 --- a/gallery_dl/extractor/hentaihere.py +++ b/gallery_dl/extractor/hentaihere.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,31 +6,36 @@ """Extractors for https://hentaihere.com/""" -from .common import ChapterExtractor, MangaExtractor -from .. import text, util import re +from .. import text +from .. import util +from .common import ChapterExtractor +from .common import MangaExtractor + -class HentaihereBase(): +class HentaihereBase: """Base class for hentaihere extractors""" + category = "hentaihere" root = "https://hentaihere.com" class HentaihereChapterExtractor(HentaihereBase, ChapterExtractor): """Extractor for a single manga chapter from hentaihere.com""" + archive_fmt = "{chapter_id}_{page}" pattern = r"(?:https?://)?(?:www\.)?hentaihere\.com/m/S(\d+)/([^/?#]+)" example = "https://hentaihere.com/m/S12345/1/1/" def __init__(self, match): self.manga_id, self.chapter = match.groups() - url = "{}/m/S{}/{}/1".format(self.root, self.manga_id, self.chapter) + url = f"{self.root}/m/S{self.manga_id}/{self.chapter}/1" ChapterExtractor.__init__(self, match, url) def metadata(self, page): title = text.extr(page, "", "") - chapter_id = text.extr(page, 'report/C', '"') + chapter_id = text.extr(page, "report/C", '"') chapter, sep, minor = self.chapter.partition(".") pattern = r"Page 1 \| (.+) \(([^)]+)\) - Chapter \d+: (.+) by (.+) at " match = re.match(pattern, title) @@ -52,14 +55,12 @@ def metadata(self, page): @staticmethod def images(page): images = text.extr(page, "var rff_imageList = ", ";") - return [ - ("https://hentaicdn.com/hentai" + part, None) - for part in util.json_loads(images) - ] + return [("https://hentaicdn.com/hentai" + part, None) for part in util.json_loads(images)] class HentaihereMangaExtractor(HentaihereBase, MangaExtractor): """Extractor for hmanga from hentaihere.com""" + chapterclass = HentaihereChapterExtractor pattern = r"(?:https?://)?(?:www\.)?hentaihere\.com(/m/S\d+)/?$" example = "https://hentaihere.com/m/S12345" @@ -68,33 +69,34 @@ def chapters(self, page): results = [] pos = page.find('itemscope itemtype="http://schema.org/Book') + 1 - manga, pos = text.extract( - page, '', '', pos) - mtype, pos = text.extract( - page, '[', ']', pos) - manga_id = text.parse_int( - self.manga_url.rstrip("/").rpartition("/")[2][1:]) + manga, pos = text.extract(page, '', "", pos) + mtype, pos = text.extract(page, '[', "]", pos) + manga_id = text.parse_int(self.manga_url.rstrip("/").rpartition("/")[2][1:]) while True: - marker, pos = text.extract( - page, '
  • ', '', pos) + marker, pos = text.extract(page, '
  • ', "", pos) if marker is None: return results url, pos = text.extract(page, '\n', '<', pos) - chapter_id, pos = text.extract(page, '/C', '"', pos) + chapter, pos = text.extract(page, 'title="Tagged: -">\n', "<", pos) + chapter_id, pos = text.extract(page, "/C", '"', pos) chapter, _, title = text.unescape(chapter).strip().partition(" - ") chapter, sep, minor = chapter.partition(".") - results.append((url, { - "manga_id": manga_id, - "manga": manga, - "chapter": text.parse_int(chapter), - "chapter_minor": sep + minor, - "chapter_id": text.parse_int(chapter_id), - "type": mtype, - "title": title, - "lang": "en", - "language": "English", - })) + results.append( + ( + url, + { + "manga_id": manga_id, + "manga": manga, + "chapter": text.parse_int(chapter), + "chapter_minor": sep + minor, + "chapter_id": text.parse_int(chapter_id), + "type": mtype, + "title": title, + "lang": "en", + "language": "English", + }, + ) + ) diff --git a/gallery_dl/extractor/hentainexus.py b/gallery_dl/extractor/hentainexus.py index 286ee381ac..aa29ee5a49 100644 --- a/gallery_dl/extractor/hentainexus.py +++ b/gallery_dl/extractor/hentainexus.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,22 +6,27 @@ """Extractors for https://hentainexus.com/""" -from .common import GalleryExtractor, Extractor, Message -from .. import text, util import binascii +from contextlib import suppress + +from .. import text +from .. import util +from .common import Extractor +from .common import GalleryExtractor +from .common import Message class HentainexusGalleryExtractor(GalleryExtractor): """Extractor for hentainexus galleries""" + category = "hentainexus" root = "https://hentainexus.com" - pattern = (r"(?i)(?:https?://)?(?:www\.)?hentainexus\.com" - r"/(?:view|read)/(\d+)") + pattern = r"(?i)(?:https?://)?(?:www\.)?hentainexus\.com" r"/(?:view|read)/(\d+)" example = "https://hentainexus.com/view/12345" def __init__(self, match): self.gallery_id = match.group(1) - url = "{}/view/{}".format(self.root, self.gallery_id) + url = f"{self.root}/view/{self.gallery_id}" GalleryExtractor.__init__(self, match, url) def metadata(self, page): @@ -31,13 +34,22 @@ def metadata(self, page): extr = text.extract_from(page) data = { "gallery_id": text.parse_int(self.gallery_id), - "cover" : extr('"og:image" content="', '"'), - "title" : extr('

    ', '

    '), + "cover": extr('"og:image" content="', '"'), + "title": extr('

    ', "

    "), } - for key in ("Artist", "Book", "Circle", "Event", "Language", - "Magazine", "Parody", "Publisher", "Description"): - value = rmve(extr('viewcolumn">' + key + '', '')) + for key in ( + "Artist", + "Book", + "Circle", + "Event", + "Language", + "Magazine", + "Parody", + "Publisher", + "Description", + ): + value = rmve(extr('viewcolumn">' + key + "", "")) value, sep, rest = value.rpartition(" (") data[key.lower()] = value if sep else rest @@ -59,10 +71,9 @@ def metadata(self, page): return data def images(self, _): - url = "{}/read/{}".format(self.root, self.gallery_id) + url = f"{self.root}/read/{self.gallery_id}" page = self.request(url).text - imgs = util.json_loads(self._decode(text.extr( - page, 'initReader("', '"'))) + imgs = util.json_loads(self._decode(text.extr(page, 'initReader("', '"'))) headers = None if not self.config("original", True): @@ -72,10 +83,8 @@ def images(self, _): results = [] for img in imgs: - try: + with suppress(KeyError): results.append((img["image"], img)) - except KeyError: - pass return results @staticmethod @@ -84,7 +93,7 @@ def _decode(data): hostname = "hentainexus.com" primes = (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53) blob = list(binascii.a2b_base64(data)) - for i in range(0, len(hostname)): + for i in range(len(hostname)): blob[i] = blob[i] ^ ord(hostname[i]) key = blob[0:64] @@ -93,10 +102,7 @@ def _decode(data): for k in key: C = C ^ k for _ in range(8): - if C & 1: - C = C >> 1 ^ 0xc - else: - C = C >> 1 + C = C >> 1 ^ 12 if C & 1 else C >> 1 k = primes[C & 0x7] x = 0 @@ -120,45 +126,45 @@ def _decode(data): @staticmethod def _join_title(data): - event = data['event'] - artist = data['artist'] - circle = data['circle'] - title = data['title'] - parody = data['parody'] - book = data['book'] - magazine = data['magazine'] + event = data["event"] + artist = data["artist"] + circle = data["circle"] + title = data["title"] + parody = data["parody"] + book = data["book"] + magazine = data["magazine"] # a few galleries have a large number of artists or parodies, # which get replaced with "Various" in the title string - if artist.count(',') >= 3: - artist = 'Various' - if parody.count(',') >= 3: - parody = 'Various' + if artist.count(",") >= 3: + artist = "Various" + if parody.count(",") >= 3: + parody = "Various" - jt = '' + jt = "" if event: - jt += '({}) '.format(event) + jt += f"({event}) " if circle: - jt += '[{} ({})] '.format(circle, artist) + jt += f"[{circle} ({artist})] " else: - jt += '[{}] '.format(artist) + jt += f"[{artist}] " jt += title - if parody.lower() != 'original work': - jt += ' ({})'.format(parody) + if parody.lower() != "original work": + jt += f" ({parody})" if book: - jt += ' ({})'.format(book) + jt += f" ({book})" if magazine: - jt += ' ({})'.format(magazine) + jt += f" ({magazine})" return jt class HentainexusSearchExtractor(Extractor): """Extractor for hentainexus search results""" + category = "hentainexus" subcategory = "search" root = "https://hentainexus.com" - pattern = (r"(?i)(?:https?://)?(?:www\.)?hentainexus\.com" - r"(?:/page/\d+)?/?(?:\?(q=[^/?#]+))?$") + pattern = r"(?i)(?:https?://)?(?:www\.)?hentainexus\.com" r"(?:/page/\d+)?/?(?:\?(q=[^/?#]+))?$" example = "https://hentainexus.com/?q=QUERY" def items(self): diff --git a/gallery_dl/extractor/hiperdex.py b/gallery_dl/extractor/hiperdex.py index c939a3c556..bbad5f1a28 100644 --- a/gallery_dl/extractor/hiperdex.py +++ b/gallery_dl/extractor/hiperdex.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,50 +6,44 @@ """Extractors for https://hipertoon.com/""" -from .common import ChapterExtractor, MangaExtractor +import re + from .. import text from ..cache import memcache -import re +from .common import ChapterExtractor +from .common import MangaExtractor -BASE_PATTERN = (r"((?:https?://)?(?:www\.)?" - r"(?:1st)?hiper(?:dex|toon)\d?\.(?:com|net|info|top))") +BASE_PATTERN = r"((?:https?://)?(?:www\.)?" r"(?:1st)?hiper(?:dex|toon)\d?\.(?:com|net|info|top))" -class HiperdexBase(): +class HiperdexBase: """Base class for hiperdex extractors""" + category = "hiperdex" root = "https://hipertoon.com" @memcache(keyarg=1) def manga_data(self, manga, page=None): if not page: - url = "{}/manga/{}/".format(self.root, manga) + url = f"{self.root}/manga/{manga}/" page = self.request(url).text extr = text.extract_from(page) return { - "url" : text.unescape(extr( - 'property="og:url" content="', '"')), - "manga" : text.unescape(extr( - ' property="name" title="', '"')), - "score" : text.parse_float(extr( - 'id="averagerate">', '<')), - "author" : text.remove_html(extr( - 'class="author-content">', '
    • ')), - "artist" : text.remove_html(extr( - 'class="artist-content">', '
    ')), - "genre" : text.split_html(extr( - 'class="genres-content">', ''))[::2], - "type" : extr( - 'class="summary-content">', '<').strip(), - "release": text.parse_int(text.remove_html(extr( - 'class="summary-content">', ''))), - "status" : extr( - 'class="summary-content">', '<').strip(), - "description": text.remove_html(text.unescape(extr( - "Summary ", ""))), + "url": text.unescape(extr('property="og:url" content="', '"')), + "manga": text.unescape(extr(' property="name" title="', '"')), + "score": text.parse_float(extr('id="averagerate">', "<")), + "author": text.remove_html(extr('class="author-content">', "")), + "artist": text.remove_html(extr('class="artist-content">', "")), + "genre": text.split_html(extr('class="genres-content">', ""))[::2], + "type": extr('class="summary-content">', "<").strip(), + "release": text.parse_int(text.remove_html(extr('class="summary-content">', ""))), + "status": extr('class="summary-content">', "<").strip(), + "description": text.remove_html( + text.unescape(extr("Summary ", "")) + ), "language": "English", - "lang" : "en", + "lang": "en", } def chapter_data(self, chapter): @@ -59,7 +51,7 @@ def chapter_data(self, chapter): chapter = chapter[8:] chapter, _, minor = chapter.partition("-") data = { - "chapter" : text.parse_int(chapter), + "chapter": text.parse_int(chapter), "chapter_minor": "." + minor if minor and minor != "end" else "", } data.update(self.manga_data(self.manga.lower())) @@ -68,6 +60,7 @@ def chapter_data(self, chapter): class HiperdexChapterExtractor(HiperdexBase, ChapterExtractor): """Extractor for hiperdex manga chapters""" + pattern = BASE_PATTERN + r"(/mangas?/([^/?#]+)/([^/?#]+))" example = "https://hipertoon.com/manga/MANGA/CHAPTER/" @@ -82,13 +75,13 @@ def metadata(self, _): def images(self, page): return [ (url.strip(), None) - for url in re.findall( - r'id="image-\d+"\s+(?:data-)?src="([^"]+)', page) + for url in re.findall(r'id="image-\d+"\s+(?:data-)?src="([^"]+)', page) ] class HiperdexMangaExtractor(HiperdexBase, MangaExtractor): """Extractor for hiperdex manga""" + chapterclass = HiperdexChapterExtractor pattern = BASE_PATTERN + r"(/mangas?/([^/?#]+))/?$" example = "https://hipertoon.com/manga/MANGA/" @@ -112,8 +105,7 @@ def chapters(self, page): html = self.request(url, method="POST", headers=headers).text results = [] - for item in text.extract_iter( - html, '
  • "): url = text.extr(item, 'href="', '"') chapter = url.rstrip("/").rpartition("/")[2] results.append((url, self.chapter_data(chapter))) @@ -122,6 +114,7 @@ def chapters(self, page): class HiperdexArtistExtractor(HiperdexBase, MangaExtractor): """Extractor for an artists's manga on hiperdex""" + subcategory = "artist" categorytransfer = False chapterclass = HiperdexMangaExtractor @@ -135,7 +128,7 @@ def __init__(self, match): def chapters(self, page): results = [] - for info in text.extract_iter(page, 'id="manga-item-', '= total: return class HitomiIndexExtractor(HitomiTagExtractor): """Extractor for galleries from index searches on hitomi.la""" + subcategory = "index" pattern = r"(?:https?://)?hitomi\.la/(\w+)-(\w+)\.html" example = "https://hitomi.la/index-LANG.html" @@ -163,8 +170,7 @@ def __init__(self, match): def items(self): data = {"_extractor": HitomiGalleryExtractor} - nozomi_url = "https://ltn.hitomi.la/{}-{}.nozomi".format( - self.tag, self.language) + nozomi_url = f"https://ltn.hitomi.la/{self.tag}-{self.language}.nozomi" headers = { "Origin": self.root, "Cache-Control": "max-age=0", @@ -173,26 +179,26 @@ def items(self): offset = 0 total = None while True: - headers["Referer"] = "{}/{}-{}.html?page={}".format( - self.root, self.tag, self.language, offset // 100 + 1) - headers["Range"] = "bytes={}-{}".format(offset, offset+99) + headers["Referer"] = ( + f"{self.root}/{self.tag}-{self.language}.html?page={offset // 100 + 1}" + ) + headers["Range"] = f"bytes={offset}-{offset + 99}" response = self.request(nozomi_url, headers=headers) for gallery_id in decode_nozomi(response.content): - gallery_url = "{}/galleries/{}.html".format( - self.root, gallery_id) + gallery_url = f"{self.root}/galleries/{gallery_id}.html" yield Message.Queue, gallery_url, data offset += 100 if total is None: - total = text.parse_int( - response.headers["content-range"].rpartition("/")[2]) + total = text.parse_int(response.headers["content-range"].rpartition("/")[2]) if offset >= total: return class HitomiSearchExtractor(Extractor): """Extractor for galleries from multiple tag searches on hitomi.la""" + category = "hitomi" subcategory = "search" root = "https://hitomi.la" @@ -211,28 +217,23 @@ def items(self): intersects = set.intersection(*results) for gallery_id in sorted(intersects, reverse=True): - gallery_url = "{}/galleries/{}.html".format( - self.root, gallery_id) + gallery_url = f"{self.root}/galleries/{gallery_id}.html" yield Message.Queue, gallery_url, data def get_nozomi_items(self, full_tag): area, tag, language = self.get_nozomi_args(full_tag) if area: - referer_base = "{}/n/{}/{}-{}.html".format( - self.root, area, tag, language) - nozomi_url = "https://ltn.hitomi.la/{}/{}-{}.nozomi".format( - area, tag, language) + referer_base = f"{self.root}/n/{area}/{tag}-{language}.html" + nozomi_url = f"https://ltn.hitomi.la/{area}/{tag}-{language}.nozomi" else: - referer_base = "{}/n/{}-{}.html".format( - self.root, tag, language) - nozomi_url = "https://ltn.hitomi.la/{}-{}.nozomi".format( - tag, language) + referer_base = f"{self.root}/n/{tag}-{language}.html" + nozomi_url = f"https://ltn.hitomi.la/{tag}-{language}.nozomi" headers = { "Origin": self.root, "Cache-Control": "max-age=0", - "Referer": "{}/search.html?{}".format(referer_base, self.query), + "Referer": f"{referer_base}/search.html?{self.query}", } response = self.request(nozomi_url, headers=headers) @@ -261,8 +262,7 @@ def _parse_gg(extr): m = {} keys = [] - for match in re.finditer( - r"case\s+(\d+):(?:\s*o\s*=\s*(\d+))?", page): + for match in re.finditer(r"case\s+(\d+):(?:\s*o\s*=\s*(\d+))?", page): key, value = match.groups() keys.append(int(key)) @@ -272,8 +272,7 @@ def _parse_gg(extr): m[key] = value keys.clear() - for match in re.finditer( - r"if\s+\(g\s*===?\s*(\d+)\)[\s{]*o\s*=\s*(\d+)", page): + for match in re.finditer(r"if\s+\(g\s*===?\s*(\d+)\)[\s{]*o\s*=\s*(\d+)", page): m[int(match.group(1))] = int(match.group(2)) d = re.search(r"(?:var\s|default:)\s*o\s*=\s*(\d+)", page) diff --git a/gallery_dl/extractor/hotleak.py b/gallery_dl/extractor/hotleak.py index ddfc54b337..14d565927b 100644 --- a/gallery_dl/extractor/hotleak.py +++ b/gallery_dl/extractor/hotleak.py @@ -1,22 +1,27 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://hotleak.vip/""" -from .common import Extractor, Message -from .. import text, exception import binascii +from .. import exception +from .. import text +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?(?:www\.)?hotleak\.vip" class HotleakExtractor(Extractor): """Base class for hotleak extractors""" + category = "hotleak" - directory_fmt = ("{category}", "{creator}",) + directory_fmt = ( + "{category}", + "{creator}", + ) filename_fmt = "{creator}_{id}.{extension}" archive_fmt = "{type}_{creator}_{id}" root = "https://hotleak.vip" @@ -25,9 +30,7 @@ def items(self): for post in self.posts(): if not post["url"].startswith("ytdl:"): post["url"] = ( - post["url"] - .replace("/storage/storage/", "/storage/") - .replace("_thumb.", ".") + post["url"].replace("/storage/storage/", "/storage/").replace("_thumb.", ".") ) post["_http_expected_status"] = (404,) yield Message.Directory, post @@ -46,8 +49,7 @@ def _pagination(self, url, params): if "" not in page: return - for item in text.extract_iter( - page, '') + page = text.extr(page, '
    ', "") data = { - "id" : text.parse_int(self.id), + "id": text.parse_int(self.id), "creator": self.creator, - "type" : self.type, + "type": self.type, } if self.type == "photo": @@ -86,8 +88,7 @@ def posts(self): text.nameext_from_url(data["url"], data) elif self.type == "video": - data["url"] = "ytdl:" + decode_video_url(text.extr( - text.unescape(page), '"src":"', '"')) + data["url"] = "ytdl:" + decode_video_url(text.extr(text.unescape(page), '"src":"', '"')) text.nameext_from_url(data["url"], data) data["extension"] = "mp4" @@ -96,9 +97,9 @@ def posts(self): class HotleakCreatorExtractor(HotleakExtractor): """Extractor for all posts from a hotleak creator""" + subcategory = "creator" - pattern = (BASE_PATTERN + r"/(?!(?:hot|creators|videos|photos)(?:$|/))" - r"([^/?#]+)/?$") + pattern = BASE_PATTERN + r"/(?!(?:hot|creators|videos|photos)(?:$|/))" r"([^/?#]+)/?$" example = "https://hotleak.vip/MODEL" def __init__(self, match): @@ -106,7 +107,7 @@ def __init__(self, match): self.creator = match.group(1) def posts(self): - url = "{}/{}".format(self.root, self.creator) + url = f"{self.root}/{self.creator}" return self._pagination(url) def _pagination(self, url): @@ -115,12 +116,10 @@ def _pagination(self, url): while True: try: - response = self.request( - url, headers=headers, params=params, notfound="creator") + response = self.request(url, headers=headers, params=params, notfound="creator") except exception.HttpError as exc: if exc.response.status_code == 429: - self.wait( - until=exc.response.headers.get("X-RateLimit-Reset")) + self.wait(until=exc.response.headers.get("X-RateLimit-Reset")) continue raise @@ -139,8 +138,7 @@ def _pagination(self, url): elif post["type"] == 1: data["type"] = "video" - data["url"] = "ytdl:" + decode_video_url( - post["stream_url_play"]) + data["url"] = "ytdl:" + decode_video_url(post["stream_url_play"]) text.nameext_from_url(data["url"], data) data["extension"] = "mp4" @@ -150,6 +148,7 @@ def _pagination(self, url): class HotleakCategoryExtractor(HotleakExtractor): """Extractor for hotleak categories""" + subcategory = "category" pattern = BASE_PATTERN + r"/(hot|creators|videos|photos)(?:/?\?([^#]+))?" example = "https://hotleak.vip/photos" @@ -159,7 +158,7 @@ def __init__(self, match): self._category, self.params = match.groups() def items(self): - url = "{}/{}".format(self.root, self._category) + url = f"{self.root}/{self._category}" if self._category in ("hot", "creators"): data = {"_extractor": HotleakCreatorExtractor} @@ -172,6 +171,7 @@ def items(self): class HotleakSearchExtractor(HotleakExtractor): """Extractor for hotleak search results""" + subcategory = "search" pattern = BASE_PATTERN + r"/search(?:/?\?([^#]+))" example = "https://hotleak.vip/search?search=QUERY" diff --git a/gallery_dl/extractor/idolcomplex.py b/gallery_dl/extractor/idolcomplex.py index dfd9a31713..fba4fc839d 100644 --- a/gallery_dl/extractor/idolcomplex.py +++ b/gallery_dl/extractor/idolcomplex.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,18 +6,22 @@ """Extractors for https://idol.sankakucomplex.com/""" -from .sankaku import SankakuExtractor -from .common import Message -from ..cache import cache -from .. import text, util, exception import collections import re +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from .common import Message +from .sankaku import SankakuExtractor + BASE_PATTERN = r"(?:https?://)?idol\.sankakucomplex\.com(?:/[a-z]{2})?" class IdolcomplexExtractor(SankakuExtractor): """Base class for idolcomplex extractors""" + category = "idolcomplex" root = "https://idol.sankakucomplex.com" cookies_domain = "idol.sankakucomplex.com" @@ -34,9 +36,7 @@ def __init__(self, match): self.start_post = 0 def _init(self): - self.find_pids = re.compile( - r" href=[\"#]/\w\w/posts/(\w+)" - ).findall + self.find_pids = re.compile(r" href=[\"#]/\w\w/posts/(\w+)").findall self.find_tags = re.compile( r'tag-type-([^"]+)">\s*]*?href="/[^?]*\?tags=([^"]+)' ).findall @@ -62,7 +62,7 @@ def post_ids(self): def login(self): if self.cookies_check(self.cookies_names): - return + return None username, password = self._get_auth_info() if username: @@ -70,7 +70,7 @@ def login(self): self.logged_in = False - @cache(maxage=90*86400, keyarg=1) + @cache(maxage=90 * 86400, keyarg=1) def _login_impl(self, username, password): self.log.info("Logging in as %s", username) @@ -80,15 +80,15 @@ def _login_impl(self, username, password): headers = { "Referer": url, } - url = self.root + (text.extr(page, '
    ") - vcnt = extr('>Votes:', "<") + vcnt = extr(">Votes:", "<") pid = extr(">Post ID:", "<") created = extr(' title="', '"') - file_url = extr('>Original:', 'id=') + file_url = extr(">Original:", "id=") if file_url: file_url = extr(' href="', '"') width = extr(">", "x") height = extr("", " ") else: - width = extr('') + width = extr("") file_url = extr('Rating:", "') + tags_html = text.extr(page, '")), - "genres" : text.split_html(extr( - "", "")), - "tags" : text.split_html(extr( - "", "")), - "uploader" : text.remove_html(extr( - "", "")), - "language" : extr(" ", "\n"), + "title": text.unescape(extr("

    ", "<").strip()), + "artists": text.split_html(extr("", "")), + "genres": text.split_html(extr("", "")), + "tags": text.split_html(extr("", "")), + "uploader": text.remove_html(extr("", "")), + "language": extr(" ", "\n"), } diff --git a/gallery_dl/extractor/toyhouse.py b/gallery_dl/extractor/toyhouse.py index 44d87ee83c..21a2704822 100644 --- a/gallery_dl/extractor/toyhouse.py +++ b/gallery_dl/extractor/toyhouse.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,14 +6,17 @@ """Extractors for https://toyhou.se/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?toyhou\.se" class ToyhouseExtractor(Extractor): """Base class for toyhouse extractors""" + category = "toyhouse" root = "https://toyhou.se" directory_fmt = ("{category}", "{user|artists!S}") @@ -51,17 +52,18 @@ def _parse_post(self, post, needle='\n
    ', '<'), - "%d %b %Y, %I:%M:%S %p"), + "date": text.parse_datetime( + extr('Credits\n

    \n
    ', "<"), "%d %b %Y, %I:%M:%S %p" + ), "artists": [ text.remove_html(artist) - for artist in extr( - '
    ', '
    \n
    ').split( - '
    ') + for artist in extr('
    ', "
    \n
    ").split( + '
    ' + ) + ], + "characters": text.split_html(extr('
    \n
    "))[ + 2: ], - "characters": text.split_html(extr( - '
    ', ''): + for post in text.extract_iter(page, '
  • ', "
    • "): cnt += 1 yield self._parse_post(post) @@ -89,24 +90,26 @@ def _pagination(self, path): def _accept_content_warning(self, page): pos = page.find(' name="_token"') + 1 token, pos = text.extract(page, ' value="', '"', pos) - user , pos = text.extract(page, ' value="', '"', pos) + user, pos = text.extract(page, ' value="', '"', pos) if not token or not user: return False data = {"_token": token, "user": user} - self.request(self.root + "/~account/warnings/accept", - method="POST", data=data, allow_redirects=False) + self.request( + self.root + "/~account/warnings/accept", method="POST", data=data, allow_redirects=False + ) return True class ToyhouseArtExtractor(ToyhouseExtractor): """Extractor for artworks of a toyhouse user""" + subcategory = "art" pattern = BASE_PATTERN + r"/([^/?#]+)/art" example = "https://www.toyhou.se/USER/art" def posts(self): - return self._pagination("/{}/art".format(self.user)) + return self._pagination(f"/{self.user}/art") def metadata(self): return {"user": self.user} @@ -114,14 +117,16 @@ def metadata(self): class ToyhouseImageExtractor(ToyhouseExtractor): """Extractor for individual toyhouse images""" + subcategory = "image" - pattern = (r"(?:https?://)?(?:" - r"(?:www\.)?toyhou\.se/~images|" - r"f\d+\.toyhou\.se/file/[^/?#]+/(?:image|watermark)s" - r")/(\d+)") + pattern = ( + r"(?:https?://)?(?:" + r"(?:www\.)?toyhou\.se/~images|" + r"f\d+\.toyhou\.se/file/[^/?#]+/(?:image|watermark)s" + r")/(\d+)" + ) example = "https://toyhou.se/~images/12345" def posts(self): - url = "{}/~images/{}".format(self.root, self.user) - return (self._parse_post( - self.request(url).text, '', '
    ')), - "date" : text.parse_datetime( - extr('id="Uploaded">', '
    ').strip(), "%Y %B %d"), - "rating" : text.parse_float(extr( - 'id="Rating">', '').partition(" ")[0]), - "type" : text.remove_html(extr('id="Category">' , '')), - "collection": text.remove_html(extr('id="Collection">', '')), - "group" : text.split_html(extr('id="Group">' , '')), - "artist" : text.split_html(extr('id="Artist">' , '')), - "parody" : text.split_html(extr('id="Parody">' , '')), - "characters": text.split_html(extr('id="Character">' , '')), - "tags" : text.split_html(extr('id="Tag">' , '')), - "language" : "English", - "lang" : "en", + "title": title_en or title_jp, + "title_en": title_en, + "title_jp": title_jp, + "thumbnail": extr('"og:image" content="', '"'), + "uploader": text.remove_html(extr('id="Uploader">', "")), + "date": text.parse_datetime(extr('id="Uploaded">', "").strip(), "%Y %B %d"), + "rating": text.parse_float(extr('id="Rating">', "").partition(" ")[0]), + "type": text.remove_html(extr('id="Category">', "")), + "collection": text.remove_html(extr('id="Collection">', "")), + "group": text.split_html(extr('id="Group">', "")), + "artist": text.split_html(extr('id="Artist">', "")), + "parody": text.split_html(extr('id="Parody">', "")), + "characters": text.split_html(extr('id="Character">', "")), + "tags": text.split_html(extr('id="Tag">', "")), + "language": "English", + "lang": "en", } def images(self, page): - url = "{}/Read/Index/{}?page=1".format(self.root, self.gallery_id) + url = f"{self.root}/Read/Index/{self.gallery_id}?page=1" headers = {"Referer": self.gallery_url} response = self.request(url, headers=headers, fatal=False) if "/Auth/" in response.url: raise exception.StopExtraction( "Failed to get gallery JSON data. Visit '%s' in a browser " - "and solve the CAPTCHA to continue.", response.url) + "and solve the CAPTCHA to continue.", + response.url, + ) page = response.text tpl, pos = text.extract(page, 'data-cdn="', '"') - cnt, pos = text.extract(page, '> of ', '<', pos) + cnt, pos = text.extract(page, "> of ", "<", pos) base, _, params = text.unescape(tpl).partition("[PAGE]") - return [ - (base + str(i) + params, None) - for i in range(1, text.parse_int(cnt)+1) - ] + return [(base + str(i) + params, None) for i in range(1, text.parse_int(cnt) + 1)] class TsuminoSearchExtractor(TsuminoBase, Extractor): """Extractor for search results on tsumino.com""" + subcategory = "search" pattern = r"(?i)(?:https?://)?(?:www\.)?tsumino\.com/(?:Books/?)?#(.+)" example = "https://www.tsumino.com/Books#QUERY" @@ -119,9 +121,9 @@ def items(self): def galleries(self): """Return all gallery results matching 'self.query'""" - url = "{}/Search/Operate?type=Book".format(self.root) + url = f"{self.root}/Search/Operate?type=Book" headers = { - "Referer": "{}/".format(self.root), + "Referer": f"{self.root}/", "X-Requested-With": "XMLHttpRequest", } data = { @@ -137,8 +139,7 @@ def galleries(self): data.update(self._parse(self.query)) while True: - info = self.request( - url, method="POST", headers=headers, data=data).json() + info = self.request(url, method="POST", headers=headers, data=data).json() for gallery in info["data"]: yield gallery["entry"] @@ -155,8 +156,7 @@ def _parse(self, query): return self._parse_simple(query) return self._parse_jsurl(query) except Exception as exc: - raise exception.StopExtraction( - "Invalid search query '%s' (%s)", query, exc) + raise exception.StopExtraction("Invalid search query '%s' (%s)", query, exc) @staticmethod def _parse_simple(query): @@ -196,8 +196,7 @@ def eat(expected): nonlocal i if data[i] != expected: - error = "bad JSURL syntax: expected '{}', got {}".format( - expected, data[i]) + error = f"bad JSURL syntax: expected '{expected}', got {data[i]}" raise ValueError(error) i += 1 @@ -217,10 +216,10 @@ def decode(): if beg < i: result += data[beg:i] if data[i + 1] == "*": - result += chr(int(data[i+2:i+6], 16)) + result += chr(int(data[i + 2 : i + 6], 16)) i += 6 else: - result += chr(int(data[i+1:i+3], 16)) + result += chr(int(data[i + 1 : i + 3], 16)) i += 3 beg = i @@ -239,7 +238,7 @@ def decode(): def parse_one(): nonlocal i - eat('~') + eat("~") result = "" ch = data[i] @@ -248,7 +247,7 @@ def parse_one(): if data[i] == "~": result = [] - if data[i+1] == ")": + if data[i + 1] == ")": i += 1 else: result.append(parse_one()) @@ -295,11 +294,11 @@ def parse_one(): def expand(key, value): if isinstance(value, list): for index, cvalue in enumerate(value): - ckey = "{}[{}]".format(key, index) + ckey = f"{key}[{index}]" yield from expand(ckey, cvalue) elif isinstance(value, dict): for ckey, cvalue in value.items(): - ckey = "{}[{}]".format(key, ckey) + ckey = f"{key}[{ckey}]" yield from expand(ckey, cvalue) else: yield key, value diff --git a/gallery_dl/extractor/tumblr.py b/gallery_dl/extractor/tumblr.py index 8d1fcde546..7e3084add7 100644 --- a/gallery_dl/extractor/tumblr.py +++ b/gallery_dl/extractor/tumblr.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,11 +6,17 @@ """Extractors for https://www.tumblr.com/""" -from .common import Extractor, Message -from .. import text, util, oauth, exception -from datetime import datetime, date, timedelta import re +from datetime import date +from datetime import datetime +from datetime import timedelta +from .. import exception +from .. import oauth +from .. import text +from .. import util +from .common import Extractor +from .common import Message BASE_PATTERN = ( r"(?:tumblr:(?:https?://)?([^/]+)|" @@ -21,12 +25,14 @@ r"([\w-]+\.tumblr\.com)))" ) -POST_TYPES = frozenset(("text", "quote", "link", "answer", "video", - "audio", "photo", "chat", "search")) +POST_TYPES = frozenset( + ("text", "quote", "link", "answer", "video", "audio", "photo", "chat", "search") +) class TumblrExtractor(Extractor): """Base class for tumblr extractors""" + category = "tumblr" directory_fmt = ("{category}", "{blog_name}") filename_fmt = "{category}_{blog_name}_{id}_{num:>02}.{extension}" @@ -67,12 +73,13 @@ def items(self): # pre-compile regular expressions self._sub_video = re.compile( - r"https?://((?:vt|vtt|ve)(?:\.media)?\.tumblr\.com" - r"/tumblr_[^_]+)_\d+\.([0-9a-z]+)").sub + r"https?://((?:vt|vtt|ve)(?:\.media)?\.tumblr\.com" r"/tumblr_[^_]+)_\d+\.([0-9a-z]+)" + ).sub if self.inline: self._sub_image = re.compile( r"https?://(\d+\.media\.tumblr\.com(?:/[0-9a-f]+)?" - r"/tumblr(?:_inline)?_[^_]+)_\d+\.([0-9a-z]+)").sub + r"/tumblr(?:_inline)?_[^_]+)_\d+\.([0-9a-z]+)" + ).sub self._subn_orig_image = re.compile(r"/s\d+x\d+/").subn _findall_image = re.compile(' best_photo["height"] or - alt_photo["width"] > best_photo["width"]): + if ( + alt_photo["height"] > best_photo["height"] + or alt_photo["width"] > best_photo["width"] + ): best_photo = alt_photo photo.update(best_photo) - if self.original and "/s2048x3072/" in photo["url"] and ( - photo["width"] == 2048 or photo["height"] == 3072): + if ( + self.original + and "/s2048x3072/" in photo["url"] + and (photo["width"] == 2048 or photo["height"] == 3072) + ): photo["url"], fb = self._original_photo(photo["url"]) if fb: post["_fallback"] = self._original_image_fallback( - photo["url"], post["id"]) + photo["url"], post["id"] + ) del photo["original_size"] del photo["alt_sizes"] - posts.append( - self._prepare_image(photo["url"], post.copy())) + posts.append(self._prepare_image(photo["url"], post.copy())) del post["photo"] post.pop("_fallback", None) @@ -142,8 +154,7 @@ def items(self): url = post.get("video_url") # type "video" if url: - posts.append(self._prepare( - self._original_video(url), post.copy())) + posts.append(self._prepare(self._original_video(url), post.copy())) if self.inline and "reblog" in post: # inline media # only "chat" posts are missing a "reblog" key in their @@ -152,8 +163,7 @@ def items(self): for url in _findall_image(body): url, fb = self._original_inline_image(url) if fb: - post["_fallback"] = self._original_image_fallback( - url, post["id"]) + post["_fallback"] = self._original_image_fallback(url, post["id"]) posts.append(self._prepare_image(url, post.copy())) post.pop("_fallback", None) for url in _findall_video(body): @@ -184,20 +194,18 @@ def _setup_posttypes(self): if types == "all": return POST_TYPES - elif not types: + if not types: return frozenset() - else: - if isinstance(types, str): - types = types.split(",") - types = frozenset(types) + if isinstance(types, str): + types = types.split(",") + types = frozenset(types) - invalid = types - POST_TYPES - if invalid: - types = types & POST_TYPES - self.log.warning("Invalid post types: '%s'", - "', '".join(sorted(invalid))) - return types + invalid = types - POST_TYPES + if invalid: + types = types & POST_TYPES + self.log.warning("Invalid post types: '%s'", "', '".join(sorted(invalid))) + return types @staticmethod def _prepare(url, post): @@ -214,9 +222,10 @@ def _prepare_image(url, post): # incorrect extensions will be corrected by 'adjust-extensions' if post["extension"] == "gif": post["_fallback"] = (url + "v",) - post["_http_headers"] = {"Accept": # copied from chrome 106 - "image/avif,image/webp,image/apng," - "image/svg+xml,image/*,*/*;q=0.8"} + post["_http_headers"] = { + "Accept": # copied from chrome 106 + "image/avif,image/webp,image/apng," "image/svg+xml,image/*,*/*;q=0.8" + } parts = post["filename"].split("_") try: @@ -270,12 +279,12 @@ def _original_image_fallback(self, url, post_id): for _ in util.repeat(self.fallback_retries): self.sleep(self.fallback_delay, "image token") yield self._update_image_token(url)[0] - self.log.warning("Unable to fetch higher-resolution " - "version of %s (%s)", url, post_id) + self.log.warning("Unable to fetch higher-resolution " "version of %s (%s)", url, post_id) class TumblrUserExtractor(TumblrExtractor): """Extractor for a Tumblr user's posts""" + subcategory = "user" pattern = BASE_PATTERN + r"(?:/page/\d+|/archive)?/?$" example = "https://www.tumblr.com/BLOG" @@ -286,6 +295,7 @@ def posts(self): class TumblrPostExtractor(TumblrExtractor): """Extractor for a single Tumblr post""" + subcategory = "post" pattern = BASE_PATTERN + r"/(?:post/|image/)?(\d+)" example = "https://www.tumblr.com/BLOG/12345" @@ -306,6 +316,7 @@ def _setup_posttypes(): class TumblrTagExtractor(TumblrExtractor): """Extractor for Tumblr user's posts by tag""" + subcategory = "tag" pattern = BASE_PATTERN + r"/tagged/([^/?#]+)" example = "https://www.tumblr.com/BLOG/tagged/TAG" @@ -320,6 +331,7 @@ def posts(self): class TumblrDayExtractor(TumblrExtractor): """Extractor for Tumblr user's posts by day""" + subcategory = "day" pattern = BASE_PATTERN + r"/day/(\d\d\d\d/\d\d/\d\d)" example = "https://www.tumblr.com/BLOG/day/1970/01/01" @@ -334,7 +346,8 @@ def _init(self): self.date_min = ( # 719163 == date(1970, 1, 1).toordinal() - (self.ordinal - 719163) * 86400) + (self.ordinal - 719163) * 86400 + ) self.api.before = self.date_min + 86400 @@ -344,6 +357,7 @@ def posts(self): class TumblrLikesExtractor(TumblrExtractor): """Extractor for a Tumblr user's liked posts""" + subcategory = "likes" directory_fmt = ("{category}", "{blog_name}", "likes") archive_fmt = "f_{blog[name]}_{id}_{num}" @@ -356,9 +370,9 @@ def posts(self): class TumblrSearchExtractor(TumblrExtractor): """Extractor for a Tumblr search""" + subcategory = "search" - pattern = (BASE_PATTERN + r"/search/([^/?#]+)" - r"(?:/([^/?#]+)(?:/([^/?#]+))?)?(?:/?\?([^#]+))?") + pattern = BASE_PATTERN + r"/search/([^/?#]+)" r"(?:/([^/?#]+)(?:/([^/?#]+))?)?(?:/?\?([^#]+))?" example = "https://www.tumblr.com/search/QUERY" def posts(self): @@ -372,6 +386,7 @@ class TumblrAPI(oauth.OAuth1API): https://github.com/tumblr/docs/blob/master/api.md """ + ROOT = "https://api.tumblr.com" API_KEY = "O3hU2tMi5e4Qs5t3vezEi6L0qRORJ5y9oUpSGsrWu8iA3UCc3B" API_SECRET = "sFdsK3PDdP2QpYMRAoq0oDnw0sFS24XigXmdfnaeNZpJpqAn03" @@ -386,7 +401,7 @@ def info(self, blog): try: return self.BLOG_CACHE[blog] except KeyError: - endpoint = "/v2/blog/{}/info".format(blog) + endpoint = f"/v2/blog/{blog}/info" params = {"api_key": self.api_key} if self.api_key else None self.BLOG_CACHE[blog] = blog = self._call(endpoint, params)["blog"] return blog @@ -394,12 +409,10 @@ def info(self, blog): def avatar(self, blog, size="512"): """Retrieve a blog avatar""" if self.api_key: - return "{}/v2/blog/{}/avatar/{}?api_key={}".format( - self.ROOT, blog, size, self.api_key) - endpoint = "/v2/blog/{}/avatar".format(blog) + return f"{self.ROOT}/v2/blog/{blog}/avatar/{size}?api_key={self.api_key}" + endpoint = f"/v2/blog/{blog}/avatar" params = {"size": size} - return self._call( - endpoint, params, allow_redirects=False)["avatar_url"] + return self._call(endpoint, params, allow_redirects=False)["avatar_url"] def posts(self, blog, params): """Retrieve published posts""" @@ -412,12 +425,12 @@ def posts(self, blog, params): if self.before and params["offset"]: self.log.warning("'offset' and 'date-max' cannot be used together") - endpoint = "/v2/blog/{}/posts".format(blog) + endpoint = f"/v2/blog/{blog}/posts" return self._pagination(endpoint, params, blog=blog, cache=True) def likes(self, blog): """Retrieve liked posts""" - endpoint = "/v2/blog/{}/likes".format(blog) + endpoint = f"/v2/blog/{blog}/likes" params = {"limit": "50", "before": self.before} if self.api_key: params["api_key"] = self.api_key @@ -465,21 +478,19 @@ def _call(self, endpoint, params, **kwargs): if status == 403: raise exception.AuthorizationError() - elif status == 404: + if status == 404: try: error = data["errors"][0]["detail"] - board = ("only viewable within the Tumblr dashboard" - in error) + board = "only viewable within the Tumblr dashboard" in error except Exception: board = False if board: - self.log.info("Run 'gallery-dl oauth:tumblr' " - "to access dashboard-only blogs") + self.log.info("Run 'gallery-dl oauth:tumblr' " "to access dashboard-only blogs") raise exception.AuthorizationError(error) raise exception.NotFoundError("user or post") - elif status == 429: + if status == 429: # daily rate limit if response.headers.get("x-ratelimit-perday-remaining") == "0": self.log.info("Daily API rate limit exceeded") @@ -491,7 +502,8 @@ def _call(self, endpoint, params, **kwargs): "Register your own OAuth application and use its " "credentials to prevent this error: " "https://gdl-org.github.io/docs/configuration.html" - "#extractor-tumblr-api-key-api-secret") + "#extractor-tumblr-api-key-api-secret" + ) if self.extractor.config("ratelimit") == "wait": self.extractor.wait(seconds=reset) @@ -500,7 +512,8 @@ def _call(self, endpoint, params, **kwargs): t = (datetime.now() + timedelta(0, float(reset))).time() raise exception.StopExtraction( "Aborting - Rate limit will reset at %s", - "{:02}:{:02}:{:02}".format(t.hour, t.minute, t.second)) + f"{t.hour:02}:{t.minute:02}:{t.second:02}", + ) # hourly rate limit reset = response.headers.get("x-ratelimit-perhour-reset") @@ -511,8 +524,7 @@ def _call(self, endpoint, params, **kwargs): raise exception.StopExtraction(data) - def _pagination(self, endpoint, params, - blog=None, key="posts", cache=False): + def _pagination(self, endpoint, params, blog=None, key="posts", cache=False): if self.api_key: params["api_key"] = self.api_key @@ -555,8 +567,7 @@ def _pagination(self, endpoint, params, params["offset"] = None else: # offset - params["offset"] = \ - text.parse_int(params["offset"]) + params["limit"] + params["offset"] = text.parse_int(params["offset"]) + params["limit"] params["before"] = None if params["offset"] >= data["total_posts"]: return diff --git a/gallery_dl/extractor/tumblrgallery.py b/gallery_dl/extractor/tumblrgallery.py index 448625ede3..49fb87cf39 100644 --- a/gallery_dl/extractor/tumblrgallery.py +++ b/gallery_dl/extractor/tumblrgallery.py @@ -1,19 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://tumblrgallery.xyz/""" -from .common import GalleryExtractor from .. import text +from .common import GalleryExtractor BASE_PATTERN = r"(?:https?://)?tumblrgallery\.xyz" class TumblrgalleryExtractor(GalleryExtractor): """Base class for tumblrgallery extractors""" + category = "tumblrgallery" filename_fmt = "{category}_{gallery_id}_{num:>03}_{id}.{extension}" directory_fmt = ("{category}", "{gallery_id} {title}") @@ -22,8 +21,7 @@ class TumblrgalleryExtractor(GalleryExtractor): @staticmethod def _urls_from_page(page): - return text.extract_iter( - page, '
    ", "")), + "title": text.unescape(text.extr(page, "

    ", "

    ")), "gallery_id": self.gallery_id, } def images(self, _): page_num = 1 while True: - url = "{}/tumblrblog/gallery/{}/{}.html".format( - self.root, self.gallery_id, page_num) + url = f"{self.root}/tumblrblog/gallery/{self.gallery_id}/{page_num}.html" response = self.request(url, allow_redirects=False, fatal=False) if response.status_code >= 300: @@ -68,6 +66,7 @@ def images(self, _): class TumblrgalleryPostExtractor(TumblrgalleryExtractor): """Extractor for Posts on tumblrgallery.xyz""" + subcategory = "post" pattern = BASE_PATTERN + r"(/post/(\d+)\.html)" example = "https://tumblrgallery.xyz/post/12345.html" @@ -78,7 +77,7 @@ def __init__(self, match): def metadata(self, page): return { - "title" : text.remove_html( + "title": text.remove_html( text.unescape(text.extr(page, "", "")) ).replace("_", "-"), "gallery_id": self.gallery_id, @@ -91,6 +90,7 @@ def images(self, page): class TumblrgallerySearchExtractor(TumblrgalleryExtractor): """Extractor for Search result on tumblrgallery.xyz""" + subcategory = "search" filename_fmt = "{category}_{num:>03}_{gallery_id}_{id}_{title}.{extension}" directory_fmt = ("{category}", "{search_term}") @@ -111,22 +111,19 @@ def images(self, _): while True: page = self.request(self.root + "/" + page_url).text - for gallery_id in text.extract_iter( - page, '
    ", "") - )).replace("_", "-") + data["title"] = text.remove_html( + text.unescape(text.extr(post_page, "", "")) + ).replace("_", "-") yield url, data - next_url = text.extr( - page, ' = 300000000000000: - date = text.parse_timestamp( - ((tweet_id >> 22) + 1288834974657) // 1000) + date = text.parse_timestamp(((tweet_id >> 22) + 1288834974657) // 1000) else: try: - date = text.parse_datetime( - legacy["created_at"], "%a %b %d %H:%M:%S %z %Y") + date = text.parse_datetime(legacy["created_at"], "%a %b %d %H:%M:%S %z %Y") except Exception: date = util.NONE source = tweet.get("source") tdata = { - "tweet_id" : tweet_id, - "retweet_id" : text.parse_int( - tget("retweeted_status_id_str")), - "quote_id" : text.parse_int( - tget("quoted_by_id_str")), - "reply_id" : text.parse_int( - tget("in_reply_to_status_id_str")), - "conversation_id": text.parse_int( - tget("conversation_id_str")), - "date" : date, - "author" : author, - "user" : self._user or author, - "lang" : legacy["lang"], - "source" : text.extr(source, ">", "<") if source else "", - "sensitive" : tget("possibly_sensitive"), + "tweet_id": tweet_id, + "retweet_id": text.parse_int(tget("retweeted_status_id_str")), + "quote_id": text.parse_int(tget("quoted_by_id_str")), + "reply_id": text.parse_int(tget("in_reply_to_status_id_str")), + "conversation_id": text.parse_int(tget("conversation_id_str")), + "date": date, + "author": author, + "user": self._user or author, + "lang": legacy["lang"], + "source": text.extr(source, ">", "<") if source else "", + "sensitive": tget("possibly_sensitive"), "favorite_count": tget("favorite_count"), - "quote_count" : tget("quote_count"), - "reply_count" : tget("reply_count"), - "retweet_count" : tget("retweet_count"), + "quote_count": tget("quote_count"), + "reply_count": tget("reply_count"), + "retweet_count": tget("retweet_count"), "bookmark_count": tget("bookmark_count"), } @@ -352,39 +342,40 @@ def _transform_tweet(self, tweet): mentions = entities.get("user_mentions") if mentions: - tdata["mentions"] = [{ - "id": text.parse_int(u["id_str"]), - "name": u["screen_name"], - "nick": u["name"], - } for u in mentions] + tdata["mentions"] = [ + { + "id": text.parse_int(u["id_str"]), + "name": u["screen_name"], + "nick": u["name"], + } + for u in mentions + ] content = text.unescape(content) urls = entities.get("urls") if urls: - for url in urls: - try: + with suppress(KeyError): + for url in urls: content = content.replace(url["url"], url["expanded_url"]) - except KeyError: - pass txt, _, tco = content.rpartition(" ") tdata["content"] = txt if tco.startswith("https://t.co/") else content if "birdwatch_pivot" in tweet: try: - tdata["birdwatch"] = \ - tweet["birdwatch_pivot"]["subtitle"]["text"] + tdata["birdwatch"] = tweet["birdwatch_pivot"]["subtitle"]["text"] except KeyError: - self.log.debug("Unable to extract 'birdwatch' note from %s", - tweet["birdwatch_pivot"]) + self.log.debug( + "Unable to extract 'birdwatch' note from %s", tweet["birdwatch_pivot"] + ) if "in_reply_to_screen_name" in legacy: tdata["reply_to"] = legacy["in_reply_to_screen_name"] if "quoted_by" in legacy: tdata["quote_by"] = legacy["quoted_by"] if tdata["retweet_id"]: - tdata["content"] = "RT @{}: {}".format( - author["name"], tdata["content"]) + tdata["content"] = "RT @{}: {}".format(author["name"], tdata["content"]) tdata["date_original"] = text.parse_timestamp( - ((tdata["retweet_id"] >> 22) + 1288834974657) // 1000) + ((tdata["retweet_id"] >> 22) + 1288834974657) // 1000 + ) return tdata @@ -409,33 +400,29 @@ def _transform_user(self, user): entities = user["entities"] self._user_cache[uid] = udata = { - "id" : text.parse_int(uid), - "name" : user["screen_name"], - "nick" : user["name"], - "location" : uget("location"), - "date" : text.parse_datetime( - uget("created_at"), "%a %b %d %H:%M:%S %z %Y"), - "verified" : uget("verified", False), - "protected" : uget("protected", False), - "profile_banner" : uget("profile_banner_url", ""), - "profile_image" : uget( - "profile_image_url_https", "").replace("_normal.", "."), + "id": text.parse_int(uid), + "name": user["screen_name"], + "nick": user["name"], + "location": uget("location"), + "date": text.parse_datetime(uget("created_at"), "%a %b %d %H:%M:%S %z %Y"), + "verified": uget("verified", False), + "protected": uget("protected", False), + "profile_banner": uget("profile_banner_url", ""), + "profile_image": uget("profile_image_url_https", "").replace("_normal.", "."), "favourites_count": uget("favourites_count"), - "followers_count" : uget("followers_count"), - "friends_count" : uget("friends_count"), - "listed_count" : uget("listed_count"), - "media_count" : uget("media_count"), - "statuses_count" : uget("statuses_count"), + "followers_count": uget("followers_count"), + "friends_count": uget("friends_count"), + "listed_count": uget("listed_count"), + "media_count": uget("media_count"), + "statuses_count": uget("statuses_count"), } descr = user["description"] urls = entities["description"].get("urls") if urls: - for url in urls: - try: + with suppress(KeyError): + for url in urls: descr = descr.replace(url["url"], url["expanded_url"]) - except KeyError: - pass udata["description"] = descr if "url" in entities: @@ -473,16 +460,14 @@ def _users_result(self, users): def _expand_tweets(self, tweets): seen = set() for tweet in tweets: - obj = tweet["legacy"] if "legacy" in tweet else tweet + obj = tweet.get("legacy", tweet) cid = obj.get("conversation_id_str") if not cid: tid = obj["id_str"] - self.log.warning( - "Unable to expand %s (no 'conversation_id')", tid) + self.log.warning("Unable to expand %s (no 'conversation_id')", tid) continue if cid in seen: - self.log.debug( - "Skipping expansion of %s (previously seen)", cid) + self.log.debug("Skipping expansion of %s (previously seen)", cid) continue seen.add(cid) try: @@ -528,12 +513,14 @@ def tweets(self): def finalize(self): if self._cursor: - self.log.info("Use '-o cursor=%s' to continue downloading " - "from the current position", self._cursor) + self.log.info( + "Use '-o cursor=%s' to continue downloading " "from the current position", + self._cursor, + ) def login(self): if self.cookies_check(self.cookies_names): - return + return None username, password = self._get_auth_info() if username: @@ -542,9 +529,12 @@ def login(self): class TwitterUserExtractor(TwitterExtractor): """Extractor for a Twitter user""" + subcategory = "user" - pattern = (BASE_PATTERN + r"/(?!search)(?:([^/?#]+)/?(?:$|[?#])" - r"|i(?:/user/|ntent/user\?user_id=)(\d+))") + pattern = ( + BASE_PATTERN + r"/(?!search)(?:([^/?#]+)/?(?:$|[?#])" + r"|i(?:/user/|ntent/user\?user_id=)(\d+))" + ) example = "https://x.com/USER" def __init__(self, match): @@ -560,21 +550,25 @@ def finalize(self): pass def items(self): - base = "{}/{}/".format(self.root, self.user) - return self._dispatch_extractors(( - (TwitterInfoExtractor , base + "info"), - (TwitterAvatarExtractor , base + "photo"), - (TwitterBackgroundExtractor, base + "header_photo"), - (TwitterTimelineExtractor , base + "timeline"), - (TwitterTweetsExtractor , base + "tweets"), - (TwitterMediaExtractor , base + "media"), - (TwitterRepliesExtractor , base + "with_replies"), - (TwitterLikesExtractor , base + "likes"), - ), ("timeline",)) + base = f"{self.root}/{self.user}/" + return self._dispatch_extractors( + ( + (TwitterInfoExtractor, base + "info"), + (TwitterAvatarExtractor, base + "photo"), + (TwitterBackgroundExtractor, base + "header_photo"), + (TwitterTimelineExtractor, base + "timeline"), + (TwitterTweetsExtractor, base + "tweets"), + (TwitterMediaExtractor, base + "media"), + (TwitterRepliesExtractor, base + "with_replies"), + (TwitterLikesExtractor, base + "likes"), + ), + ("timeline",), + ) class TwitterTimelineExtractor(TwitterExtractor): """Extractor for a Twitter user timeline""" + subcategory = "timeline" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/timeline(?!\w)" example = "https://x.com/USER/timeline" @@ -631,7 +625,7 @@ def tweets(self): query += " include:retweets include:nativeretweets" if state <= 2: - self._cursor_prefix = "2_{}/".format(tweet_id) + self._cursor_prefix = f"2_{tweet_id}/" if reset: self._cursor = self._cursor_prefix @@ -647,7 +641,7 @@ def tweets(self): if state <= 3: # yield unfiltered search results - self._cursor_prefix = "3_{}/".format(tweet_id) + self._cursor_prefix = f"3_{tweet_id}/" if reset: self._cursor = self._cursor_prefix @@ -659,8 +653,7 @@ def _select_tweet_source(self): if strategy is None or strategy == "auto": if self.retweets or self.textonly: return self.api.user_tweets - else: - return self.api.user_media + return self.api.user_media if strategy == "tweets": return self.api.user_tweets if strategy == "media": @@ -672,6 +665,7 @@ def _select_tweet_source(self): class TwitterTweetsExtractor(TwitterExtractor): """Extractor for Tweets from a user's Tweets timeline""" + subcategory = "tweets" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/tweets(?!\w)" example = "https://x.com/USER/tweets" @@ -682,6 +676,7 @@ def tweets(self): class TwitterRepliesExtractor(TwitterExtractor): """Extractor for Tweets from a user's timeline including replies""" + subcategory = "replies" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/with_replies(?!\w)" example = "https://x.com/USER/with_replies" @@ -692,6 +687,7 @@ def tweets(self): class TwitterMediaExtractor(TwitterExtractor): """Extractor for Tweets from a user's Media timeline""" + subcategory = "media" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/media(?!\w)" example = "https://x.com/USER/media" @@ -702,6 +698,7 @@ def tweets(self): class TwitterLikesExtractor(TwitterExtractor): """Extractor for liked tweets""" + subcategory = "likes" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/likes(?!\w)" example = "https://x.com/USER/likes" @@ -715,6 +712,7 @@ def tweets(self): class TwitterBookmarkExtractor(TwitterExtractor): """Extractor for bookmarked tweets""" + subcategory = "bookmark" pattern = BASE_PATTERN + r"/i/bookmarks()" example = "https://x.com/i/bookmarks" @@ -725,12 +723,14 @@ def tweets(self): def _transform_tweet(self, tweet): tdata = TwitterExtractor._transform_tweet(self, tweet) tdata["date_bookmarked"] = text.parse_timestamp( - (int(tweet["sortIndex"] or 0) >> 20) // 1000) + (int(tweet["sortIndex"] or 0) >> 20) // 1000 + ) return tdata class TwitterListExtractor(TwitterExtractor): """Extractor for Twitter lists""" + subcategory = "list" pattern = BASE_PATTERN + r"/i/lists/(\d+)/?$" example = "https://x.com/i/lists/12345" @@ -741,6 +741,7 @@ def tweets(self): class TwitterListMembersExtractor(TwitterExtractor): """Extractor for members of a Twitter list""" + subcategory = "list-members" pattern = BASE_PATTERN + r"/i/lists/(\d+)/members" example = "https://x.com/i/lists/12345/members" @@ -752,6 +753,7 @@ def items(self): class TwitterFollowingExtractor(TwitterExtractor): """Extractor for followed users""" + subcategory = "following" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/following(?!\w)" example = "https://x.com/USER/following" @@ -763,6 +765,7 @@ def items(self): class TwitterSearchExtractor(TwitterExtractor): """Extractor for Twitter search results""" + subcategory = "search" pattern = BASE_PATTERN + r"/search/?\?(?:[^&#]+&)*q=([^&#]+)" example = "https://x.com/search?q=QUERY" @@ -780,32 +783,31 @@ def tweets(self): if user: user = None break - else: - user = item[5:] + user = item[5:] if user is not None: - try: + with suppress(KeyError): self._assign_user(self.api.user_by_screen_name(user)) - except KeyError: - pass return self.api.search_timeline(query) class TwitterHashtagExtractor(TwitterExtractor): """Extractor for Twitter hashtags""" + subcategory = "hashtag" pattern = BASE_PATTERN + r"/hashtag/([^/?#]+)" example = "https://x.com/hashtag/NAME" def items(self): - url = "{}/search?q=%23{}".format(self.root, self.user) + url = f"{self.root}/search?q=%23{self.user}" data = {"_extractor": TwitterSearchExtractor} yield Message.Queue, url, data class TwitterCommunityExtractor(TwitterExtractor): """Extractor for a Twitter community""" + subcategory = "community" pattern = BASE_PATTERN + r"/i/communities/(\d+)" example = "https://x.com/i/communities/12345" @@ -818,6 +820,7 @@ def tweets(self): class TwitterCommunitiesExtractor(TwitterExtractor): """Extractor for followed Twitter communities""" + subcategory = "communities" pattern = BASE_PATTERN + r"/([^/?#]+)/communities/?$" example = "https://x.com/i/communities" @@ -828,9 +831,9 @@ def tweets(self): class TwitterEventExtractor(TwitterExtractor): """Extractor for Tweets from a Twitter Event""" + subcategory = "event" - directory_fmt = ("{category}", "Events", - "{event[id]} {event[short_title]}") + directory_fmt = ("{category}", "Events", "{event[id]} {event[short_title]}") pattern = BASE_PATTERN + r"/i/events/(\d+)" example = "https://x.com/i/events/12345" @@ -843,9 +846,9 @@ def tweets(self): class TwitterTweetExtractor(TwitterExtractor): """Extractor for individual tweets""" + subcategory = "tweet" - pattern = (BASE_PATTERN + r"/([^/?#]+|i/web)/status/(\d+)" - r"/?(?:$|\?|#|photo/|video/)") + pattern = BASE_PATTERN + r"/([^/?#]+|i/web)/status/(\d+)" r"/?(?:$|\?|#|photo/|video/)" example = "https://x.com/USER/status/12345" def __init__(self, match): @@ -855,12 +858,15 @@ def __init__(self, match): def tweets(self): conversations = self.config("conversations") if conversations: - self._accessible = (conversations == "accessible") + self._accessible = conversations == "accessible" return self._tweets_conversation(self.tweet_id) endpoint = self.config("tweet-endpoint") - if endpoint == "detail" or endpoint in (None, "auto") and \ - self.api.headers["x-twitter-auth-type"]: + if ( + endpoint == "detail" + or endpoint in (None, "auto") + and self.api.headers["x-twitter-auth-type"] + ): return self._tweets_detail(self.tweet_id) return self._tweets_single(self.tweet_id) @@ -871,8 +877,7 @@ def _tweets_single(self, tweet_id): try: self._assign_user(tweet["core"]["user_results"]["result"]) except KeyError: - raise exception.StopExtraction( - "'%s'", tweet.get("reason") or "Unavailable") + raise exception.StopExtraction("'%s'", tweet.get("reason") or "Unavailable") yield tweet @@ -891,8 +896,7 @@ def _tweets_detail(self, tweet_id): tweets = [] for tweet in self.api.tweet_detail(tweet_id): - if tweet["rest_id"] == tweet_id or \ - tweet.get("_retweet_id_str") == tweet_id: + if tweet["rest_id"] == tweet_id or tweet.get("_retweet_id_str") == tweet_id: if self._user_obj is None: self._assign_user(tweet["core"]["user_results"]["result"]) tweets.append(tweet) @@ -909,8 +913,7 @@ def _tweets_conversation(self, tweet_id): for tweet in tweets: buffer.append(tweet) - if tweet["rest_id"] == tweet_id or \ - tweet.get("_retweet_id_str") == tweet_id: + if tweet["rest_id"] == tweet_id or tweet.get("_retweet_id_str") == tweet_id: self._assign_user(tweet["core"]["user_results"]["result"]) break else: @@ -924,18 +927,20 @@ def _tweets_conversation(self, tweet_id): class TwitterQuotesExtractor(TwitterExtractor): """Extractor for quotes of a Tweet""" + subcategory = "quotes" pattern = BASE_PATTERN + r"/(?:[^/?#]+|i/web)/status/(\d+)/quotes" example = "https://x.com/USER/status/12345/quotes" def items(self): - url = "{}/search?q=quoted_tweet_id:{}".format(self.root, self.user) + url = f"{self.root}/search?q=quoted_tweet_id:{self.user}" data = {"_extractor": TwitterSearchExtractor} yield Message.Queue, url, data class TwitterInfoExtractor(TwitterExtractor): """Extractor for a user's profile data""" + subcategory = "info" pattern = BASE_PATTERN + r"/(?!search)([^/?#]+)/info" example = "https://x.com/USER/info" @@ -964,8 +969,9 @@ def tweets(self): user = self._user_obj url = user["legacy"]["profile_image_url_https"] - if url == ("https://abs.twimg.com/sticky" - "/default_profile_images/default_profile_normal.png"): + if url == ( + "https://abs.twimg.com/sticky" "/default_profile_images/default_profile_normal.png" + ): return () url = url.replace("_normal.", ".") @@ -1007,8 +1013,7 @@ def __init__(self, match): TwitterExtractor._init_sizes(self) def items(self): - base = "https://pbs.twimg.com/media/{}?format={}&name=".format( - self.id, self.fmt) + base = f"https://pbs.twimg.com/media/{self.id}?format={self.fmt}&name=" data = { "filename": self.id, @@ -1020,8 +1025,7 @@ def items(self): yield Message.Url, base + self._size_image, data -class TwitterAPI(): - +class TwitterAPI: def __init__(self, extractor): self.extractor = extractor self.log = extractor.log @@ -1057,8 +1061,8 @@ def __init__(self, extractor): "Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "same-origin", "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejR" - "COuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu" - "4FA33AGWWjCpTnA", + "COuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu" + "4FA33AGWWjCpTnA", } self.params = { "include_profile_interstitial_type": "1", @@ -1098,8 +1102,8 @@ def __init__(self, extractor): "spelling_corrections": None, "include_ext_edit_control": "true", "ext": "mediaStats,highlightedLabel,hasNftAvatar,voiceInfo," - "enrichments,superFollowMetadata,unmentionInfo,editControl," - "collab_control,vibe", + "enrichments,superFollowMetadata,unmentionInfo,editControl," + "collab_control,vibe", } self.features = { "hidden_profile_likes_enabled": True, @@ -1109,8 +1113,7 @@ def __init__(self, extractor): "highlights_tweets_tab_ui_enabled": True, "responsive_web_twitter_article_notes_tab_enabled": True, "creator_subscriptions_tweet_preview_api_enabled": True, - "responsive_web_graphql_" - "skip_user_profile_image_extensions_enabled": False, + "responsive_web_graphql_" "skip_user_profile_image_extensions_enabled": False, "responsive_web_graphql_timeline_navigation_enabled": True, } self.features_pagination = { @@ -1118,8 +1121,7 @@ def __init__(self, extractor): "verified_phone_label_enabled": False, "creator_subscriptions_tweet_preview_api_enabled": True, "responsive_web_graphql_timeline_navigation_enabled": True, - "responsive_web_graphql_skip_user_profile_" - "image_extensions_enabled": False, + "responsive_web_graphql_skip_user_profile_" "image_extensions_enabled": False, "c9s_tweet_anatomy_moderator_badge_enabled": True, "tweetypie_unmention_optimization_enabled": True, "responsive_web_edit_tweet_api_enabled": True, @@ -1130,8 +1132,7 @@ def __init__(self, extractor): "tweet_awards_web_tipping_enabled": False, "freedom_of_speech_not_reach_fetch_enabled": True, "standardized_nudges_misinfo": True, - "tweet_with_visibility_results_prefer_gql_" - "limited_actions_policy_enabled": True, + "tweet_with_visibility_results_prefer_gql_" "limited_actions_policy_enabled": True, "rweb_video_timestamps_enabled": True, "longform_notetweets_rich_text_read_enabled": True, "longform_notetweets_inline_media_enabled": True, @@ -1149,7 +1150,7 @@ def tweet_result_by_rest_id(self, tweet_id): } params = { "variables": self._json_dumps(variables), - "features" : self._json_dumps(self.features_pagination), + "features": self._json_dumps(self.features_pagination), } tweet = self._call(endpoint, params)["data"]["tweetResult"]["result"] if "tweet" in tweet: @@ -1179,7 +1180,8 @@ def tweet_detail(self, tweet_id): "withV2Timeline": True, } return self._pagination_tweets( - endpoint, variables, ("threaded_conversation_with_injections_v2",)) + endpoint, variables, ("threaded_conversation_with_injections_v2",) + ) def user_tweets(self, screen_name): endpoint = "/graphql/5ICa5d9-AitXZrIA3H-4MQ/UserTweets" @@ -1240,8 +1242,8 @@ def user_bookmarks(self): features = self.features_pagination.copy() features["graphql_timeline_v2_bookmark_timeline"] = True return self._pagination_tweets( - endpoint, variables, ("bookmark_timeline_v2", "timeline"), False, - features=features) + endpoint, variables, ("bookmark_timeline_v2", "timeline"), False, features=features + ) def list_latest_tweets_timeline(self, list_id): endpoint = "/graphql/HjsWc-nwwHKYwHenbHm-tw/ListLatestTweetsTimeline" @@ -1249,8 +1251,7 @@ def list_latest_tweets_timeline(self, list_id): "listId": list_id, "count": 100, } - return self._pagination_tweets( - endpoint, variables, ("list", "tweets_timeline", "timeline")) + return self._pagination_tweets(endpoint, variables, ("list", "tweets_timeline", "timeline")) def search_timeline(self, query): endpoint = "/graphql/fZK7JipRHWtiZsTodhsTfQ/SearchTimeline" @@ -1262,8 +1263,8 @@ def search_timeline(self, query): } return self._pagination_tweets( - endpoint, variables, - ("search_by_raw_query", "search_timeline", "timeline")) + endpoint, variables, ("search_by_raw_query", "search_timeline", "timeline") + ) def community_tweets_timeline(self, community_id): endpoint = "/graphql/7B2AdxSuC-Er8qUr3Plm_w/CommunityTweetsTimeline" @@ -1275,9 +1276,10 @@ def community_tweets_timeline(self, community_id): "withCommunity": True, } return self._pagination_tweets( - endpoint, variables, - ("communityResults", "result", "ranked_community_timeline", - "timeline")) + endpoint, + variables, + ("communityResults", "result", "ranked_community_timeline", "timeline"), + ) def community_media_timeline(self, community_id): endpoint = "/graphql/qAGUldfcIoMv5KyAyVLYog/CommunityMediaTimeline" @@ -1287,23 +1289,23 @@ def community_media_timeline(self, community_id): "withCommunity": True, } return self._pagination_tweets( - endpoint, variables, - ("communityResults", "result", "community_media_timeline", - "timeline")) + endpoint, + variables, + ("communityResults", "result", "community_media_timeline", "timeline"), + ) def communities_main_page_timeline(self, screen_name): - endpoint = ("/graphql/GtOhw2mstITBepTRppL6Uw" - "/CommunitiesMainPageTimeline") + endpoint = "/graphql/GtOhw2mstITBepTRppL6Uw" "/CommunitiesMainPageTimeline" variables = { "count": 100, "withCommunity": True, } return self._pagination_tweets( - endpoint, variables, - ("viewer", "communities_timeline", "timeline")) + endpoint, variables, ("viewer", "communities_timeline", "timeline") + ) def live_event_timeline(self, event_id): - endpoint = "/2/live_event/timeline/{}.json".format(event_id) + endpoint = f"/2/live_event/timeline/{event_id}.json" params = self.params.copy() params["timeline_id"] = "recap" params["urt"] = "true" @@ -1311,12 +1313,11 @@ def live_event_timeline(self, event_id): return self._pagination_legacy(endpoint, params) def live_event(self, event_id): - endpoint = "/1.1/live_event/1/{}/timeline.json".format(event_id) + endpoint = f"/1.1/live_event/1/{event_id}/timeline.json" params = self.params.copy() params["count"] = "0" params["urt"] = "true" - return (self._call(endpoint, params) - ["twitter_objects"]["live_events"][event_id]) + return self._call(endpoint, params)["twitter_objects"]["live_events"][event_id] def list_members(self, list_id): endpoint = "/graphql/BQp2IEYkgxuSxqbTAr1e1g/ListMembers" @@ -1325,8 +1326,7 @@ def list_members(self, list_id): "count": 100, "withSafetyModeUserFields": True, } - return self._pagination_users( - endpoint, variables, ("list", "members_timeline", "timeline")) + return self._pagination_users(endpoint, variables, ("list", "members_timeline", "timeline")) def user_following(self, screen_name): endpoint = "/graphql/PAnE9toEjRfE-4tozRcsfw/Following" @@ -1342,10 +1342,12 @@ def user_by_rest_id(self, rest_id): endpoint = "/graphql/tD8zKvQzwY3kdx5yz6YmOw/UserByRestId" features = self.features params = { - "variables": self._json_dumps({ - "userId": rest_id, - "withSafetyModeUserFields": True, - }), + "variables": self._json_dumps( + { + "userId": rest_id, + "withSafetyModeUserFields": True, + } + ), "features": self._json_dumps(features), } return self._call(endpoint, params)["data"]["user"]["result"] @@ -1354,15 +1356,15 @@ def user_by_rest_id(self, rest_id): def user_by_screen_name(self, screen_name): endpoint = "/graphql/k5XapwcSikNsEsILW5FvgA/UserByScreenName" features = self.features.copy() - features["subscriptions_verification_info_" - "is_identity_verified_enabled"] = True - features["subscriptions_verification_info_" - "verified_since_enabled"] = True + features["subscriptions_verification_info_" "is_identity_verified_enabled"] = True + features["subscriptions_verification_info_" "verified_since_enabled"] = True params = { - "variables": self._json_dumps({ - "screen_name": screen_name, - "withSafetyModeUserFields": True, - }), + "variables": self._json_dumps( + { + "screen_name": screen_name, + "withSafetyModeUserFields": True, + } + ), "features": self._json_dumps(features), } return self._call(endpoint, params)["data"]["user"]["result"] @@ -1378,26 +1380,31 @@ def _user_id_by_screen_name(self, screen_name): return user["rest_id"] except KeyError: if "unavailable_message" in user: - raise exception.NotFoundError("{} ({})".format( - user["unavailable_message"].get("text"), - user.get("reason")), False) - else: - raise exception.NotFoundError("user") + raise exception.NotFoundError( + "{} ({})".format(user["unavailable_message"].get("text"), user.get("reason")), + False, + ) + raise exception.NotFoundError("user") @cache(maxage=3600) def _guest_token(self): endpoint = "/1.1/guest/activate.json" self.log.info("Requesting guest token") - return str(self._call( - endpoint, None, "POST", False, "https://api.x.com", - )["guest_token"]) + return str( + self._call( + endpoint, + None, + "POST", + False, + "https://api.x.com", + )["guest_token"] + ) def _authenticate_guest(self): guest_token = self._guest_token() if guest_token != self.headers["x-guest-token"]: self.headers["x-guest-token"] = guest_token - self.extractor.cookies.set( - "gt", guest_token, domain=self.extractor.cookies_domain) + self.extractor.cookies.set("gt", guest_token, domain=self.extractor.cookies_domain) def _call(self, endpoint, params, method="GET", auth=True, root=None): url = (root or self.root) + endpoint @@ -1407,8 +1414,8 @@ def _call(self, endpoint, params, method="GET", auth=True, root=None): self._authenticate_guest() response = self.extractor.request( - url, method=method, params=params, - headers=self.headers, fatal=None) + url, method=method, params=params, headers=self.headers, fatal=None + ) # update 'x-csrf-token' header (#1170) csrf_token = response.cookies.get("ct0") @@ -1451,8 +1458,7 @@ def _call(self, endpoint, params, method="GET", auth=True, root=None): continue _login_impl.invalidate(username) - self.extractor.cookies_update( - _login_impl(self.extractor, username, password)) + self.extractor.cookies_update(_login_impl(self.extractor, username, password)) self.__init__(self.extractor) retry = True @@ -1463,34 +1469,31 @@ def _call(self, endpoint, params, method="GET", auth=True, root=None): if self.headers["x-twitter-auth-type"]: self.log.debug("Retrying API request") continue - else: - # fall through to "Login Required" - response.status_code = 404 + # fall through to "Login Required" + response.status_code = 404 if response.status_code < 400: return data - elif response.status_code in (403, 404) and \ - not self.headers["x-twitter-auth-type"]: + if response.status_code in (403, 404) and not self.headers["x-twitter-auth-type"]: raise exception.AuthorizationError("Login required") - elif response.status_code == 429: + if response.status_code == 429: self._handle_ratelimit(response) continue # error - try: + with suppress(Exception): errors = ", ".join(e["message"] for e in errors) - except Exception: - pass raise exception.StopExtraction( - "%s %s (%s)", response.status_code, response.reason, errors) + "%s %s (%s)", response.status_code, response.reason, errors + ) def _pagination_legacy(self, endpoint, params): extr = self.extractor cursor = extr._init_cursor() if cursor: params["cursor"] = cursor - original_retweets = (extr.retweets == "original") + original_retweets = extr.retweets == "original" bottom = ("cursor-bottom-", "sq-cursor-bottom") while True: @@ -1513,21 +1516,21 @@ def _pagination_legacy(self, endpoint, params): elif "replaceEntry" in instr: entry = instr["replaceEntry"]["entry"] if entry["entryId"].startswith(bottom): - cursor = (entry["content"]["operation"] - ["cursor"]["value"]) + cursor = entry["content"]["operation"]["cursor"]["value"] # collect tweet IDs and cursor value for entry in entries: entry_startswith = entry["entryId"].startswith if entry_startswith(("tweet-", "sq-I-t-")): - tweet_ids.append( - entry["content"]["item"]["content"]["tweet"]["id"]) + tweet_ids.append(entry["content"]["item"]["content"]["tweet"]["id"]) elif entry_startswith("homeConversation-"): tweet_ids.extend( - entry["content"]["timelineModule"]["metadata"] - ["conversationMetadata"]["allTweetIds"][::-1]) + entry["content"]["timelineModule"]["metadata"]["conversationMetadata"][ + "allTweetIds" + ][::-1] + ) elif entry_startswith(bottom): cursor = entry["content"]["operation"]["cursor"] @@ -1561,10 +1564,8 @@ def _pagination_legacy(self, endpoint, params): tweet = retweet elif retweet: tweet["author"] = users[retweet["user_id_str"]] - if "extended_entities" in retweet and \ - "extended_entities" not in tweet: - tweet["extended_entities"] = \ - retweet["extended_entities"] + if "extended_entities" in retweet and "extended_entities" not in tweet: + tweet["extended_entities"] = retweet["extended_entities"] tweet["user"] = users[tweet["user_id_str"]] yield tweet @@ -1582,10 +1583,9 @@ def _pagination_legacy(self, endpoint, params): return extr._update_cursor(None) params["cursor"] = extr._update_cursor(cursor) - def _pagination_tweets(self, endpoint, variables, - path=None, stop_tweets=True, features=None): + def _pagination_tweets(self, endpoint, variables, path=None, stop_tweets=True, features=None): extr = self.extractor - original_retweets = (extr.retweets == "original") + original_retweets = extr.retweets == "original" pinned_tweet = extr.pinned params = {"variables": None} @@ -1603,8 +1603,7 @@ def _pagination_tweets(self, endpoint, variables, try: if path is None: - instructions = (data["user"]["result"]["timeline_v2"] - ["timeline"]["instructions"]) + instructions = data["user"]["result"]["timeline_v2"]["timeline"]["instructions"] else: instructions = data for key in path: @@ -1641,23 +1640,21 @@ def _pagination_tweets(self, endpoint, variables, if user: user = user["legacy"] if user.get("blocked_by"): - if self.headers["x-twitter-auth-type"] and \ - extr.config("logout"): + if self.headers["x-twitter-auth-type"] and extr.config("logout"): extr.cookies_file = None del extr.cookies["auth_token"] self.headers["x-twitter-auth-type"] = None extr.log.info("Retrying API request as guest") continue raise exception.AuthorizationError( - "{} blocked your account".format( - user["screen_name"])) + "{} blocked your account".format(user["screen_name"]) + ) elif user.get("protected"): raise exception.AuthorizationError( - "{}'s Tweets are protected".format( - user["screen_name"])) + "{}'s Tweets are protected".format(user["screen_name"]) + ) - raise exception.StopExtraction( - "Unable to retrieve Tweets from this timeline") + raise exception.StopExtraction("Unable to retrieve Tweets from this timeline") tweets = [] tweet = None @@ -1674,20 +1671,16 @@ def _pagination_tweets(self, endpoint, variables, if esw("tweet-"): tweets.append(entry) - elif esw(("profile-grid-", - "communities-grid-")): + elif esw(("profile-grid-", "communities-grid-")): if "content" in entry: tweets.extend(entry["content"]["items"]) else: tweets.append(entry) - elif esw(("homeConversation-", - "profile-conversation-", - "conversationthread-")): + elif esw(("homeConversation-", "profile-conversation-", "conversationthread-")): tweets.extend(entry["content"]["items"]) elif esw("tombstone-"): item = entry["content"]["itemContent"] - item["tweet_results"] = \ - {"result": {"tombstone": item["tombstoneInfo"]}} + item["tweet_results"] = {"result": {"tombstone": item["tombstoneInfo"]}} tweets.append(entry) elif esw("cursor-bottom-"): cursor = entry["content"] @@ -1700,18 +1693,16 @@ def _pagination_tweets(self, endpoint, variables, for entry in tweets: try: - item = ((entry.get("content") or entry["item"]) - ["itemContent"]) + item = (entry.get("content") or entry["item"])["itemContent"] if "promotedMetadata" in item and not extr.ads: extr.log.debug( - "Skipping %s (ad)", - (entry.get("entryId") or "").rpartition("-")[2]) + "Skipping %s (ad)", (entry.get("entryId") or "").rpartition("-")[2] + ) continue tweet = item["tweet_results"]["result"] if "tombstone" in tweet: - tweet = self._process_tombstone( - entry, tweet["tombstone"]) + tweet = self._process_tombstone(entry, tweet["tombstone"]) if not tweet: continue @@ -1721,8 +1712,8 @@ def _pagination_tweets(self, endpoint, variables, tweet["sortIndex"] = entry.get("sortIndex") except KeyError: extr.log.debug( - "Skipping %s (deleted)", - (entry.get("entryId") or "").rpartition("-")[2]) + "Skipping %s (deleted)", (entry.get("entryId") or "").rpartition("-")[2] + ) continue if "retweeted_status_result" in legacy: @@ -1731,33 +1722,29 @@ def _pagination_tweets(self, endpoint, variables, retweet = retweet["tweet"] if original_retweets: try: - retweet["legacy"]["retweeted_status_id_str"] = \ - retweet["rest_id"] + retweet["legacy"]["retweeted_status_id_str"] = retweet["rest_id"] retweet["_retweet_id_str"] = tweet["rest_id"] tweet = retweet except KeyError: continue else: try: - legacy["retweeted_status_id_str"] = \ - retweet["rest_id"] - tweet["author"] = \ - retweet["core"]["user_results"]["result"] + legacy["retweeted_status_id_str"] = retweet["rest_id"] + tweet["author"] = retweet["core"]["user_results"]["result"] rtlegacy = retweet["legacy"] if "note_tweet" in retweet: tweet["note_tweet"] = retweet["note_tweet"] - if "extended_entities" in rtlegacy and \ - "extended_entities" not in legacy: - legacy["extended_entities"] = \ - rtlegacy["extended_entities"] + if ( + "extended_entities" in rtlegacy + and "extended_entities" not in legacy + ): + legacy["extended_entities"] = rtlegacy["extended_entities"] - if "withheld_scope" in rtlegacy and \ - "withheld_scope" not in legacy: - legacy["withheld_scope"] = \ - rtlegacy["withheld_scope"] + if "withheld_scope" in rtlegacy and "withheld_scope" not in legacy: + legacy["withheld_scope"] = rtlegacy["withheld_scope"] legacy["full_text"] = rtlegacy["full_text"] except KeyError: @@ -1768,17 +1755,15 @@ def _pagination_tweets(self, endpoint, variables, if "quoted_status_result" in tweet: try: quoted = tweet["quoted_status_result"]["result"] - quoted["legacy"]["quoted_by"] = ( - tweet["core"]["user_results"]["result"] - ["legacy"]["screen_name"]) + quoted["legacy"]["quoted_by"] = tweet["core"]["user_results"]["result"][ + "legacy" + ]["screen_name"] quoted["legacy"]["quoted_by_id_str"] = tweet["rest_id"] quoted["sortIndex"] = entry.get("sortIndex") yield quoted except KeyError: - extr.log.debug( - "Skipping quote of %s (deleted)", - tweet.get("rest_id")) + extr.log.debug("Skipping quote of %s (deleted)", tweet.get("rest_id")) continue if stop_tweets and not tweet: @@ -1794,7 +1779,7 @@ def _pagination_users(self, endpoint, variables, path=None): variables["cursor"] = cursor params = { "variables": None, - "features" : self._json_dumps(self.features_pagination), + "features": self._json_dumps(self.features_pagination), } while True: @@ -1804,8 +1789,7 @@ def _pagination_users(self, endpoint, variables, path=None): try: if path is None: - instructions = (data["user"]["result"]["timeline"] - ["timeline"]["instructions"]) + instructions = data["user"]["result"]["timeline"]["timeline"]["instructions"] else: for key in path: data = data[key] @@ -1818,8 +1802,7 @@ def _pagination_users(self, endpoint, variables, path=None): for entry in instr["entries"]: if entry["entryId"].startswith("user-"): try: - user = (entry["content"]["itemContent"] - ["user_results"]["result"]) + user = entry["content"]["itemContent"]["user_results"]["result"] except KeyError: pass else: @@ -1836,7 +1819,7 @@ def _handle_ratelimit(self, response): rl = self.extractor.config("ratelimit") if rl == "abort": raise exception.StopExtraction("Rate limit exceeded") - elif rl and isinstance(rl, str) and rl.startswith("wait:"): + if rl and isinstance(rl, str) and rl.startswith("wait:"): until = None seconds = text.parse_float(rl.partition(":")[2]) or 60.0 else: @@ -1848,21 +1831,19 @@ def _process_tombstone(self, entry, tombstone): text = (tombstone.get("richText") or tombstone["text"])["text"] tweet_id = entry["entryId"].rpartition("-")[2] - if text.startswith("Age-restricted"): - if self._nsfw_warning: - self._nsfw_warning = False - self.log.warning('"%s"', text) + if text.startswith("Age-restricted") and self._nsfw_warning: + self._nsfw_warning = False + self.log.warning('"%s"', text) self.log.debug("Skipping %s ('%s')", tweet_id, text) -@cache(maxage=365*86400, keyarg=1) +@cache(maxage=365 * 86400, keyarg=1) def _login_impl(extr, username, password): - def process(data, params=None): response = extr.request( - url, params=params, headers=headers, json=data, - method="POST", fatal=None) + url, params=params, headers=headers, json=data, method="POST", fatal=None + ) # update 'x-csrf-token' header (#5945) csrf_token = response.cookies.get("ct0") @@ -1876,15 +1857,14 @@ def process(data, params=None): else: if response.status_code < 400: try: - return (data["flow_token"], - data["subtasks"][0]["subtask_id"]) + return (data["flow_token"], data["subtasks"][0]["subtask_id"]) except LookupError: pass errors = [] for error in data.get("errors") or (): msg = error.get("message") - errors.append('"{}"'.format(msg) if msg else "Unknown error") + errors.append(f'"{msg}"' if msg else "Unknown error") extr.log.debug(response.text) raise exception.AuthenticationError(", ".join(errors)) @@ -1984,7 +1964,8 @@ def process(data, params=None): } elif subtask == "LoginEnterAlternateIdentifierSubtask": alt = extr.config("username-alt") or extr.input( - "Alternate Identifier (username, email, phone number): ") + "Alternate Identifier (username, email, phone number): " + ) data = { "enter_text": { "text": alt, @@ -2016,8 +1997,7 @@ def process(data, params=None): elif subtask == "DenyLoginSubtask": raise exception.AuthenticationError("Login rejected as suspicious") elif subtask == "LoginSuccessSubtask": - raise exception.AuthenticationError( - "No 'auth_token' cookie received") + raise exception.AuthenticationError("No 'auth_token' cookie received") else: raise exception.StopExtraction("Unrecognized subtask %s", subtask) @@ -2028,10 +2008,7 @@ def process(data, params=None): "subtask_inputs": [inputs], } - extr.sleep(random.uniform(1.0, 3.0), "login ({})".format(subtask)) + extr.sleep(random.uniform(1.0, 3.0), f"login ({subtask})") flow_token, subtask = process(data) - return { - cookie.name: cookie.value - for cookie in extr.cookies - } + return {cookie.name: cookie.value for cookie in extr.cookies} diff --git a/gallery_dl/extractor/unsplash.py b/gallery_dl/extractor/unsplash.py index a1b87b9c39..86a6b04dac 100644 --- a/gallery_dl/extractor/unsplash.py +++ b/gallery_dl/extractor/unsplash.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,14 +6,17 @@ """Extractors for https://unsplash.com/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?unsplash\.com" class UnsplashExtractor(Extractor): """Base class for unsplash extractors""" + category = "unsplash" directory_fmt = ("{category}", "{user[username]}") filename_fmt = "{id}.{extension}" @@ -33,8 +34,7 @@ def items(self): metadata = self.metadata() for photo in self.photos(): - util.delete_items( - photo, ("current_user_collections", "related_collections")) + util.delete_items(photo, ("current_user_collections", "related_collections")) url = photo["urls"][fmt] text.nameext_from_url(url, photo) @@ -74,41 +74,45 @@ def _pagination(self, url, params, results=False): class UnsplashImageExtractor(UnsplashExtractor): """Extractor for a single unsplash photo""" + subcategory = "image" pattern = BASE_PATTERN + r"/photos/([^/?#]+)" example = "https://unsplash.com/photos/ID" def photos(self): - url = "{}/napi/photos/{}".format(self.root, self.item) + url = f"{self.root}/napi/photos/{self.item}" return (self.request(url).json(),) class UnsplashUserExtractor(UnsplashExtractor): """Extractor for all photos of an unsplash user""" + subcategory = "user" pattern = BASE_PATTERN + r"/@(\w+)/?$" example = "https://unsplash.com/@USER" def photos(self): - url = "{}/napi/users/{}/photos".format(self.root, self.item) + url = f"{self.root}/napi/users/{self.item}/photos" params = {"order_by": "latest"} return self._pagination(url, params) class UnsplashFavoriteExtractor(UnsplashExtractor): """Extractor for all likes of an unsplash user""" + subcategory = "favorite" pattern = BASE_PATTERN + r"/@(\w+)/likes" example = "https://unsplash.com/@USER/likes" def photos(self): - url = "{}/napi/users/{}/likes".format(self.root, self.item) + url = f"{self.root}/napi/users/{self.item}/likes" params = {"order_by": "latest"} return self._pagination(url, params) class UnsplashCollectionExtractor(UnsplashExtractor): """Extractor for an unsplash collection""" + subcategory = "collection" pattern = BASE_PATTERN + r"/collections/([^/?#]+)(?:/([^/?#]+))?" example = "https://unsplash.com/collections/12345/TITLE" @@ -121,13 +125,14 @@ def metadata(self): return {"collection_id": self.item, "collection_title": self.title} def photos(self): - url = "{}/napi/collections/{}/photos".format(self.root, self.item) + url = f"{self.root}/napi/collections/{self.item}/photos" params = {"order_by": "latest"} return self._pagination(url, params) class UnsplashSearchExtractor(UnsplashExtractor): """Extractor for unsplash search results""" + subcategory = "search" pattern = BASE_PATTERN + r"/s/photos/([^/?#]+)(?:\?([^#]+))?" example = "https://unsplash.com/s/photos/QUERY" @@ -138,7 +143,7 @@ def __init__(self, match): def photos(self): url = self.root + "/napi/search/photos" - params = {"query": text.unquote(self.item.replace('-', ' '))} + params = {"query": text.unquote(self.item.replace("-", " "))} if self.query: params.update(text.parse_query(self.query)) return self._pagination(url, params, True) diff --git a/gallery_dl/extractor/uploadir.py b/gallery_dl/extractor/uploadir.py index ce34e7de82..96e49d182d 100644 --- a/gallery_dl/extractor/uploadir.py +++ b/gallery_dl/extractor/uploadir.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,14 @@ """Extractors for https://uploadir.com/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class UploadirFileExtractor(Extractor): """Extractor for uploadir files""" + category = "uploadir" subcategory = "file" root = "https://uploadir.com" @@ -27,7 +27,7 @@ def __init__(self, match): self.file_id = match.group(1) def items(self): - url = "{}/u/{}".format(self.root, self.file_id) + url = f"{self.root}/u/{self.file_id}" response = self.request(url, method="HEAD", allow_redirects=False) if 300 <= response.status_code < 400: @@ -38,18 +38,20 @@ def items(self): url = self.root + extr('class="form" action="', '"') token = extr('name="authenticity_token" value="', '"') - data = text.nameext_from_url(name, { - "_http_method": "POST", - "_http_data" : { - "authenticity_token": token, - "upload_id": self.file_id, + data = text.nameext_from_url( + name, + { + "_http_method": "POST", + "_http_data": { + "authenticity_token": token, + "upload_id": self.file_id, + }, }, - }) + ) else: hcd = response.headers.get("Content-Disposition") - name = (hcd.partition("filename*=UTF-8''")[2] or - text.extr(hcd, 'filename="', '"')) + name = hcd.partition("filename*=UTF-8''")[2] or text.extr(hcd, 'filename="', '"') data = text.nameext_from_url(name) data["id"] = self.file_id diff --git a/gallery_dl/extractor/urlgalleries.py b/gallery_dl/extractor/urlgalleries.py index bb80055841..3ee79faf2a 100644 --- a/gallery_dl/extractor/urlgalleries.py +++ b/gallery_dl/extractor/urlgalleries.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://urlgalleries.net/""" -from .common import GalleryExtractor, Message -from .. import text, exception +from .. import exception +from .. import text +from .common import GalleryExtractor +from .common import Message class UrlgalleriesGalleryExtractor(GalleryExtractor): """Base class for Urlgalleries extractors""" + category = "urlgalleries" root = "https://urlgalleries.net" request_interval = (0.5, 1.5) @@ -20,13 +21,11 @@ class UrlgalleriesGalleryExtractor(GalleryExtractor): def items(self): blog, self.gallery_id = self.groups - url = "https://{}.urlgalleries.net/porn-gallery-{}/?a=10000".format( - blog, self.gallery_id) + url = f"https://{blog}.urlgalleries.net/porn-gallery-{self.gallery_id}/?a=10000" with self.request(url, allow_redirects=False, fatal=...) as response: if 300 <= response.status_code < 500: - if response.headers.get("location", "").endswith( - "/not_found_adult.php"): + if response.headers.get("location", "").endswith("/not_found_adult.php"): raise exception.NotFoundError("gallery") raise exception.HttpError(None, response) page = response.text @@ -35,7 +34,7 @@ def items(self): data = self.metadata(page) data["count"] = len(imgs) - root = "https://{}.urlgalleries.net".format(blog) + root = f"https://{blog}.urlgalleries.net" yield Message.Directory, data for data["num"], img in enumerate(imgs, 1): page = self.request(root + img).text @@ -47,11 +46,10 @@ def metadata(self, page): return { "gallery_id": self.gallery_id, "_site": extr(' title="', '"'), # site name - "blog" : text.unescape(extr(' title="', '"')), + "blog": text.unescape(extr(' title="', '"')), "_rprt": extr(' title="', '"'), # report button "title": text.unescape(extr(' title="', '"').strip()), - "date" : text.parse_datetime( - extr(" images in gallery | ", "<"), "%B %d, %Y %H:%M"), + "date": text.parse_datetime(extr(" images in gallery | ", "<"), "%B %d, %Y %H:%M"), } def images(self, page): diff --git a/gallery_dl/extractor/urlshortener.py b/gallery_dl/extractor/urlshortener.py index 49a3debdf0..7b6f922282 100644 --- a/gallery_dl/extractor/urlshortener.py +++ b/gallery_dl/extractor/urlshortener.py @@ -1,36 +1,39 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for general-purpose URL shorteners""" -from .common import BaseExtractor, Message from .. import exception +from .common import BaseExtractor +from .common import Message class UrlshortenerExtractor(BaseExtractor): """Base class for URL shortener extractors""" + basecategory = "urlshortener" -BASE_PATTERN = UrlshortenerExtractor.update({ - "bitly": { - "root": "https://bit.ly", - "pattern": r"bit\.ly", - }, - "tco": { - # t.co sends 'http-equiv="refresh"' (200) when using browser UA - "headers": {"User-Agent": None}, - "root": "https://t.co", - "pattern": r"t\.co", - }, -}) +BASE_PATTERN = UrlshortenerExtractor.update( + { + "bitly": { + "root": "https://bit.ly", + "pattern": r"bit\.ly", + }, + "tco": { + # t.co sends 'http-equiv="refresh"' (200) when using browser UA + "headers": {"User-Agent": None}, + "root": "https://t.co", + "pattern": r"t\.co", + }, + } +) class UrlshortenerLinkExtractor(UrlshortenerExtractor): """Extractor for general-purpose URL shorteners""" + subcategory = "link" pattern = BASE_PATTERN + r"/([^/?#]+)" example = "https://bit.ly/abcde" @@ -44,8 +47,12 @@ def _init(self): def items(self): response = self.request( - "{}/{}".format(self.root, self.id), headers=self.headers, - method="HEAD", allow_redirects=False, notfound="URL") + f"{self.root}/{self.id}", + headers=self.headers, + method="HEAD", + allow_redirects=False, + notfound="URL", + ) try: yield Message.Queue, response.headers["location"], {} except KeyError: diff --git a/gallery_dl/extractor/vanillarock.py b/gallery_dl/extractor/vanillarock.py index 1ce969f7e4..40a2effb40 100644 --- a/gallery_dl/extractor/vanillarock.py +++ b/gallery_dl/extractor/vanillarock.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,14 @@ """Extractors for https://vanilla-rock.com/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class VanillarockExtractor(Extractor): """Base class for vanillarock extractors""" + category = "vanillarock" root = "https://vanilla-rock.com" @@ -24,12 +24,12 @@ def __init__(self, match): class VanillarockPostExtractor(VanillarockExtractor): """Extractor for blogposts on vanilla-rock.com""" + subcategory = "post" directory_fmt = ("{category}", "{path}") filename_fmt = "{num:>02}.{extension}" archive_fmt = "{filename}" - pattern = (r"(?:https?://)?(?:www\.)?vanilla-rock\.com" - r"(/(?!category/|tag/)[^/?#]+)/?$") + pattern = r"(?:https?://)?(?:www\.)?vanilla-rock\.com" r"(/(?!category/|tag/)[^/?#]+)/?$" example = "https://vanilla-rock.com/TITLE" def items(self): @@ -38,7 +38,7 @@ def items(self): imgs = [] while True: - img = extr('
    ', '
    ') + img = extr('
    ', "
    ") if not img: break imgs.append(text.extr(img, 'href="', '"')) @@ -46,11 +46,9 @@ def items(self): data = { "count": len(imgs), "title": text.unescape(name), - "path" : self.path.strip("/"), - "date" : text.parse_datetime(extr( - '
    ', '
    '), "%Y-%m-%d %H:%M"), - "tags" : text.split_html(extr( - '
    ', '
    '))[::2], + "path": self.path.strip("/"), + "date": text.parse_datetime(extr('
    ', "
    "), "%Y-%m-%d %H:%M"), + "tags": text.split_html(extr('
    ', "
    "))[::2], } yield Message.Directory, data @@ -60,9 +58,9 @@ def items(self): class VanillarockTagExtractor(VanillarockExtractor): """Extractor for vanillarock blog posts by tag or category""" + subcategory = "tag" - pattern = (r"(?:https?://)?(?:www\.)?vanilla-rock\.com" - r"(/(?:tag|category)/[^?#]+)") + pattern = r"(?:https?://)?(?:www\.)?vanilla-rock\.com" r"(/(?:tag|category)/[^?#]+)" example = "https://vanilla-rock.com/tag/TAG" def items(self): @@ -72,7 +70,7 @@ def items(self): while url: extr = text.extract_from(self.request(url).text) while True: - post = extr('

    ', '

    ') + post = extr('

    ', "

    ") if not post: break yield Message.Queue, text.extr(post, 'href="', '"'), data diff --git a/gallery_dl/extractor/vichan.py b/gallery_dl/extractor/vichan.py index 654c4512c1..c1e100b3f0 100644 --- a/gallery_dl/extractor/vichan.py +++ b/gallery_dl/extractor/vichan.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,29 +6,34 @@ """Extractors for vichan imageboards""" -from .common import BaseExtractor, Message from .. import text +from .common import BaseExtractor +from .common import Message class VichanExtractor(BaseExtractor): """Base class for vichan extractors""" + basecategory = "vichan" -BASE_PATTERN = VichanExtractor.update({ - "8kun": { - "root": "https://8kun.top", - "pattern": r"8kun\.top", - }, - "smugloli": { - "root": None, - "pattern": r"smuglo(?:\.li|li\.net)", - }, -}) +BASE_PATTERN = VichanExtractor.update( + { + "8kun": { + "root": "https://8kun.top", + "pattern": r"8kun\.top", + }, + "smugloli": { + "root": None, + "pattern": r"smuglo(?:\.li|li\.net)", + }, + } +) class VichanThreadExtractor(VichanExtractor): """Extractor for vichan threads""" + subcategory = "thread" directory_fmt = ("{category}", "{board}", "{thread} {title}") filename_fmt = "{time}{num:?-//} {filename}.{extension}" @@ -41,20 +44,19 @@ class VichanThreadExtractor(VichanExtractor): def __init__(self, match): VichanExtractor.__init__(self, match) index = match.lastindex - self.board = match.group(index-1) + self.board = match.group(index - 1) self.thread = match.group(index) def items(self): - url = "{}/{}/res/{}.json".format(self.root, self.board, self.thread) + url = f"{self.root}/{self.board}/res/{self.thread}.json" posts = self.request(url).json()["posts"] title = posts[0].get("sub") or text.remove_html(posts[0]["com"]) - process = (self._process_8kun if self.category == "8kun" else - self._process) + process = self._process_8kun if self.category == "8kun" else self._process data = { - "board" : self.board, + "board": self.board, "thread": self.thread, - "title" : text.unescape(title)[:50], - "num" : 0, + "title": text.unescape(title)[:50], + "num": 0, } yield Message.Directory, data @@ -62,15 +64,13 @@ def items(self): if "filename" in post: yield process(post, data) if "extra_files" in post: - for post["num"], filedata in enumerate( - post["extra_files"], 1): + for post["num"], filedata in enumerate(post["extra_files"], 1): yield process(post, filedata) def _process(self, post, data): post.update(data) post["extension"] = post["ext"][1:] - post["url"] = "{}/{}/src/{}{}".format( - self.root, post["board"], post["tim"], post["ext"]) + post["url"] = "{}/{}/src/{}{}".format(self.root, post["board"], post["tim"], post["ext"]) return Message.Url, post["url"], post @staticmethod @@ -80,17 +80,18 @@ def _process_8kun(post, data): tim = post["tim"] if len(tim) > 16: - post["url"] = "https://media.128ducks.com/file_store/{}{}".format( - tim, post["ext"]) + post["url"] = "https://media.128ducks.com/file_store/{}{}".format(tim, post["ext"]) else: post["url"] = "https://media.128ducks.com/{}/src/{}{}".format( - post["board"], tim, post["ext"]) + post["board"], tim, post["ext"] + ) return Message.Url, post["url"], post class VichanBoardExtractor(VichanExtractor): """Extractor for vichan boards""" + subcategory = "board" pattern = BASE_PATTERN + r"/([^/?#]+)(?:/index|/catalog|/\d+|/?$)" example = "https://8kun.top/a/" @@ -100,13 +101,12 @@ def __init__(self, match): self.board = match.group(match.lastindex) def items(self): - url = "{}/{}/threads.json".format(self.root, self.board) + url = f"{self.root}/{self.board}/threads.json" threads = self.request(url).json() for page in threads: for thread in page["threads"]: - url = "{}/{}/res/{}.html".format( - self.root, self.board, thread["no"]) + url = "{}/{}/res/{}.html".format(self.root, self.board, thread["no"]) thread["page"] = page["page"] thread["_extractor"] = VichanThreadExtractor yield Message.Queue, url, thread diff --git a/gallery_dl/extractor/vipergirls.py b/gallery_dl/extractor/vipergirls.py index 5cde0d6c77..149f348f21 100644 --- a/gallery_dl/extractor/vipergirls.py +++ b/gallery_dl/extractor/vipergirls.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,17 +6,21 @@ """Extractors for https://vipergirls.to/""" -from .common import Extractor, Message -from .. import text, util, exception -from ..cache import cache +from xml.etree import ElementTree as ET -from xml.etree import ElementTree +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?vipergirls\.to" class VipergirlsExtractor(Extractor): """Base class for vipergirls extractors""" + category = "vipergirls" root = "https://vipergirls.to" request_interval = 0.5 @@ -67,30 +69,29 @@ def login(self): if username: self.cookies_update(self._login_impl(username, password)) - @cache(maxage=90*86400, keyarg=1) + @cache(maxage=90 * 86400, keyarg=1) def _login_impl(self, username, password): self.log.info("Logging in as %s", username) - url = "{}/login.php?do=login".format(self.root) + url = f"{self.root}/login.php?do=login" data = { "vb_login_username": username, "vb_login_password": password, - "do" : "login", - "cookieuser" : "1", + "do": "login", + "cookieuser": "1", } response = self.request(url, method="POST", data=data) if not response.cookies.get("vg_password"): raise exception.AuthenticationError() - return {cookie.name: cookie.value - for cookie in response.cookies} + return {cookie.name: cookie.value for cookie in response.cookies} def like(self, post, user_hash): url = self.root + "/post_thanks.php" params = { - "do" : "post_thanks_add", - "p" : post.get("id"), + "do": "post_thanks_add", + "p": post.get("id"), "securitytoken": user_hash, } @@ -100,9 +101,9 @@ def like(self, post, user_hash): class VipergirlsThreadExtractor(VipergirlsExtractor): """Extractor for vipergirls threads""" + subcategory = "thread" - pattern = (BASE_PATTERN + - r"/threads/(\d+)(?:-[^/?#]+)?(/page\d+)?(?:$|#|\?(?!p=))") + pattern = BASE_PATTERN + r"/threads/(\d+)(?:-[^/?#]+)?(/page\d+)?(?:$|#|\?(?!p=))" example = "https://vipergirls.to/threads/12345-TITLE" def __init__(self, match): @@ -110,15 +111,15 @@ def __init__(self, match): self.thread_id, self.page = match.groups() def posts(self): - url = "{}/vr.php?t={}".format(self.root, self.thread_id) - return ElementTree.fromstring(self.request(url).text) + url = f"{self.root}/vr.php?t={self.thread_id}" + return ET.fromstring(self.request(url).text) class VipergirlsPostExtractor(VipergirlsExtractor): """Extractor for vipergirls posts""" + subcategory = "post" - pattern = (BASE_PATTERN + - r"/threads/(\d+)(?:-[^/?#]+)?\?p=\d+[^#]*#post(\d+)") + pattern = BASE_PATTERN + r"/threads/(\d+)(?:-[^/?#]+)?\?p=\d+[^#]*#post(\d+)" example = "https://vipergirls.to/threads/12345-TITLE?p=23456#post23456" def __init__(self, match): @@ -127,5 +128,5 @@ def __init__(self, match): self.page = 0 def posts(self): - url = "{}/vr.php?p={}".format(self.root, self.post_id) - return ElementTree.fromstring(self.request(url).text) + url = f"{self.root}/vr.php?p={self.post_id}" + return ET.fromstring(self.request(url).text) diff --git a/gallery_dl/extractor/vk.py b/gallery_dl/extractor/vk.py index ea034a79a4..bf9400e5a4 100644 --- a/gallery_dl/extractor/vk.py +++ b/gallery_dl/extractor/vk.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,15 +6,19 @@ """Extractors for https://vk.com/""" -from .common import Extractor, Message -from .. import text, exception import re +from .. import exception +from .. import text +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https://)?(?:www\.|m\.)?vk\.com" class VkExtractor(Extractor): """Base class for vk extractors""" + category = "vk" directory_fmt = ("{category}", "{user[name]|user[id]}") filename_fmt = "{id}.{extension}" @@ -39,7 +41,6 @@ def items(self): yield Message.Directory, data for photo in self.photos(): - for size in sizes: size += "_" if size in photo: @@ -74,26 +75,30 @@ def _pagination(self, photos_id): url = self.root + "/al_photos.php" headers = { "X-Requested-With": "XMLHttpRequest", - "Origin" : self.root, - "Referer" : self.root + "/" + photos_id, + "Origin": self.root, + "Referer": self.root + "/" + photos_id, } data = { - "act" : "show", - "al" : "1", + "act": "show", + "al": "1", "direction": "1", - "list" : photos_id, - "offset" : self.offset, + "list": photos_id, + "offset": self.offset, } while True: payload = self.request( - url, method="POST", headers=headers, data=data, + url, + method="POST", + headers=headers, + data=data, ).json()["payload"][1] if len(payload) < 4: self.log.debug(payload) raise exception.AuthorizationError( - text.unescape(payload[0]) if payload[0] else None) + text.unescape(payload[0]) if payload[0] else None + ) total = payload[1] photos = payload[3] @@ -114,10 +119,11 @@ def _pagination(self, photos_id): class VkPhotosExtractor(VkExtractor): """Extractor for photos from a vk user""" + subcategory = "photos" - pattern = (BASE_PATTERN + r"/(?:" - r"(?:albums|photos|id)(-?\d+)" - r"|(?!(?:album|tag)-?\d+_?)([^/?#]+))") + pattern = ( + BASE_PATTERN + r"/(?:" r"(?:albums|photos|id)(-?\d+)" r"|(?!(?:album|tag)-?\d+_?)([^/?#]+))" + ) example = "https://vk.com/id12345" def __init__(self, match): @@ -134,27 +140,28 @@ def metadata(self): url = "{}/{}{}".format(self.root, prefix, user_id.lstrip("-")) data = self._extract_profile(url) else: - url = "{}/{}".format(self.root, self.user_name) + url = f"{self.root}/{self.user_name}" data = self._extract_profile(url) self.user_id = data["user"]["id"] return data def _extract_profile(self, url): extr = text.extract_from(self.request(url).text) - return {"user": { - "name": text.unescape(extr( - 'rel="canonical" href="https://vk.com/', '"')), - "nick": text.unescape(extr( - '

    ', "<")).replace(" ", " "), - "info": text.unescape(text.remove_html(extr( - '', '', "<")).replace(" ", " "), + "info": text.unescape( + text.remove_html(extr('', ""): path = text.rextract(wp, ' src="', '"')[0] if path: image = text.nameext_from_url(path) diff --git a/gallery_dl/extractor/warosu.py b/gallery_dl/extractor/warosu.py index 61a36d5a8f..7a6e6df4cb 100644 --- a/gallery_dl/extractor/warosu.py +++ b/gallery_dl/extractor/warosu.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,14 @@ """Extractors for https://warosu.org/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class WarosuThreadExtractor(Extractor): """Extractor for threads on warosu.org""" + category = "warosu" subcategory = "thread" root = "https://warosu.org" @@ -28,14 +28,13 @@ def __init__(self, match): self.board, self.thread = match.groups() def items(self): - url = "{}/{}/thread/{}".format(self.root, self.board, self.thread) + url = f"{self.root}/{self.board}/thread/{self.thread}" page = self.request(url).text data = self.metadata(page) posts = self.posts(page) if not data["title"]: - data["title"] = text.unescape(text.remove_html( - posts[0]["com"]))[:50] + data["title"] = text.unescape(text.remove_html(posts[0]["com"]))[:50] yield Message.Directory, data for post in posts: @@ -49,10 +48,10 @@ def metadata(self, page): boardname = text.extr(page, "", "") title = text.unescape(text.extr(page, "class=filetitle>", "<")) return { - "board" : self.board, + "board": self.board, "board_name": boardname.split(" - ")[1], - "thread" : self.thread, - "title" : title, + "thread": self.thread, + "title": title, } def posts(self, page): @@ -73,12 +72,11 @@ def parse(self, post): def _extract_post(self, post): extr = text.extract_from(post) return { - "no" : extr("id=p", ">"), + "no": extr("id=p", ">"), "name": extr("class=postername>", "<").strip(), "time": extr("class=posttime title=", "000>"), - "now" : extr("", "<").strip(), - "com" : text.unescape(text.remove_html(extr( - "
    ", "
    ").strip())), + "now": extr("", "<").strip(), + "com": text.unescape(text.remove_html(extr("
    ", "
    ").strip())), } def _extract_image(self, post, data): @@ -86,8 +84,7 @@ def _extract_image(self, post, data): data["fsize"] = extr(" File: ", ", ") data["w"] = extr("", "x") data["h"] = extr("", ", ") - data["filename"] = text.unquote(extr( - "", "<").rstrip().rpartition(".")[0]) + data["filename"] = text.unquote(extr("", "<").rstrip().rpartition(".")[0]) extr("
    ", "") url = extr("    ") diff --git a/gallery_dl/extractor/weasyl.py b/gallery_dl/extractor/weasyl.py index 13b0520696..adda29a8c2 100644 --- a/gallery_dl/extractor/weasyl.py +++ b/gallery_dl/extractor/weasyl.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://www.weasyl.com/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https://)?(?:www\.)?weasyl.com/" @@ -24,22 +23,19 @@ def populate_submission(data): # Some submissions don't have content and can be skipped if "submission" in data["media"]: data["url"] = data["media"]["submission"][0]["url"] - data["date"] = text.parse_datetime( - data["posted_at"][:19], "%Y-%m-%dT%H:%M:%S") + data["date"] = text.parse_datetime(data["posted_at"][:19], "%Y-%m-%dT%H:%M:%S") text.nameext_from_url(data["url"], data) return True return False def _init(self): - self.session.headers['X-Weasyl-API-Key'] = self.config("api-key") + self.session.headers["X-Weasyl-API-Key"] = self.config("api-key") def request_submission(self, submitid): - return self.request( - "{}/api/submissions/{}/view".format(self.root, submitid)).json() + return self.request(f"{self.root}/api/submissions/{submitid}/view").json() def retrieve_journal(self, journalid): - data = self.request( - "{}/api/journals/{}/view".format(self.root, journalid)).json() + data = self.request(f"{self.root}/api/journals/{journalid}/view").json() data["extension"] = "html" data["html"] = "text:" + data["content"] data["date"] = text.parse_datetime(data["posted_at"]) @@ -47,9 +43,9 @@ def retrieve_journal(self, journalid): def submissions(self, owner_login, folderid=None): metadata = self.config("metadata") - url = "{}/api/users/{}/gallery".format(self.root, owner_login) + url = f"{self.root}/api/users/{owner_login}/gallery" params = { - "nextid" : None, + "nextid": None, "folderid": folderid, } @@ -57,8 +53,7 @@ def submissions(self, owner_login, folderid=None): data = self.request(url, params=params).json() for submission in data["submissions"]: if metadata: - submission = self.request_submission( - submission["submitid"]) + submission = self.request_submission(submission["submitid"]) if self.populate_submission(submission): submission["folderid"] = folderid # Do any submissions have more than one url? If so @@ -150,9 +145,9 @@ def __init__(self, match): def items(self): yield Message.Directory, {"owner_login": self.owner_login} - url = "{}/journals/{}".format(self.root, self.owner_login) + url = f"{self.root}/journals/{self.owner_login}" page = self.request(url).text - for journalid in text.extract_iter(page, 'href="/journal/', '/'): + for journalid in text.extract_iter(page, 'href="/journal/', "/"): data = self.retrieve_journal(journalid) yield Message.Url, data["html"], data @@ -173,7 +168,7 @@ def items(self): else: path = "/favorites" params = { - "userid" : userid, + "userid": userid, "feature": "submit", } diff --git a/gallery_dl/extractor/webmshare.py b/gallery_dl/extractor/webmshare.py index 7e2b5ea322..2c9e936737 100644 --- a/gallery_dl/extractor/webmshare.py +++ b/gallery_dl/extractor/webmshare.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,19 +6,20 @@ """Extractors for https://webmshare.com/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message class WebmshareVideoExtractor(Extractor): """Extractor for webmshare videos""" + category = "webmshare" subcategory = "video" root = "https://webmshare.com" filename_fmt = "{id}{title:? //}.{extension}" archive_fmt = "{id}" - pattern = (r"(?:https?://)?(?:s\d+\.)?webmshare\.com" - r"/(?:play/|download-webm/)?(\w{3,})") + pattern = r"(?:https?://)?(?:s\d+\.)?webmshare\.com" r"/(?:play/|download-webm/)?(\w{3,})" example = "https://webmshare.com/_ID_" def __init__(self, match): @@ -28,23 +27,19 @@ def __init__(self, match): self.video_id = match.group(1) def items(self): - url = "{}/{}".format(self.root, self.video_id) + url = f"{self.root}/{self.video_id}" extr = text.extract_from(self.request(url).text) data = { - "title": text.unescape(extr( - 'property="og:title" content="', '"').rpartition(" — ")[0]), + "title": text.unescape(extr('property="og:title" content="', '"').rpartition(" — ")[0]), "thumb": "https:" + extr('property="og:image" content="', '"'), - "url" : "https:" + extr('property="og:video" content="', '"'), - "width": text.parse_int(extr( - 'property="og:video:width" content="', '"')), - "height": text.parse_int(extr( - 'property="og:video:height" content="', '"')), - "date" : text.parse_datetime(extr( - "Added ", "<"), "%B %d, %Y"), - "views": text.parse_int(extr('glyphicon-eye-open">', '<')), - "id" : self.video_id, - "filename" : self.video_id, + "url": "https:" + extr('property="og:video" content="', '"'), + "width": text.parse_int(extr('property="og:video:width" content="', '"')), + "height": text.parse_int(extr('property="og:video:height" content="', '"')), + "date": text.parse_datetime(extr("Added ", "<"), "%B %d, %Y"), + "views": text.parse_int(extr('glyphicon-eye-open">', "<")), + "id": self.video_id, + "filename": self.video_id, "extension": "webm", } diff --git a/gallery_dl/extractor/webtoons.py b/gallery_dl/extractor/webtoons.py index 70ab259d93..2c366b541e 100644 --- a/gallery_dl/extractor/webtoons.py +++ b/gallery_dl/extractor/webtoons.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2020 Leonardo Taccari # Copyright 2021-2023 Mike Fährmann # @@ -9,74 +7,91 @@ """Extractors for https://www.webtoons.com/""" -from .common import GalleryExtractor, Extractor, Message -from .. import exception, text, util +from .. import exception +from .. import text +from .. import util +from .common import Extractor +from .common import GalleryExtractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?webtoons\.com/(([^/?#]+)" -class WebtoonsBase(): +class WebtoonsBase: category = "webtoons" root = "https://www.webtoons.com" cookies_domain = ".webtoons.com" def setup_agegate_cookies(self): - self.cookies_update({ - "atGDPR" : "AD_CONSENT", - "needCCPA" : "false", - "needCOPPA" : "false", - "needGDPR" : "false", - "pagGDPR" : "true", - "ageGatePass": "true", - }) + self.cookies_update( + { + "atGDPR": "AD_CONSENT", + "needCCPA": "false", + "needCOPPA": "false", + "needGDPR": "false", + "pagGDPR": "true", + "ageGatePass": "true", + } + ) def request(self, url, **kwargs): response = Extractor.request(self, url, **kwargs) if response.history and "/ageGate" in response.url: raise exception.StopExtraction( - "HTTP redirect to age gate check ('%s')", response.request.url) + "HTTP redirect to age gate check ('%s')", response.request.url + ) return response class WebtoonsEpisodeExtractor(WebtoonsBase, GalleryExtractor): """Extractor for an episode on webtoons.com""" + subcategory = "episode" directory_fmt = ("{category}", "{comic}") filename_fmt = "{episode_no}-{num:>02}.{extension}" archive_fmt = "{title_no}_{episode_no}_{num}" - pattern = (BASE_PATTERN + r"/([^/?#]+)/([^/?#]+)/(?:[^/?#]+))" - r"/viewer(?:\?([^#'\"]+))") - example = ("https://www.webtoons.com/en/GENRE/TITLE/NAME/viewer" - "?title_no=123&episode_no=12345") + pattern = BASE_PATTERN + r"/([^/?#]+)/([^/?#]+)/(?:[^/?#]+))" r"/viewer(?:\?([^#'\"]+))" + example = "https://www.webtoons.com/en/GENRE/TITLE/NAME/viewer" "?title_no=123&episode_no=12345" test = ( - (("https://www.webtoons.com/en/comedy/safely-endangered" - "/ep-572-earth/viewer?title_no=352&episode_no=572"), { - "url": "55bec5d7c42aba19e3d0d56db25fdf0b0b13be38", - "content": ("1748c7e82b6db910fa179f6dc7c4281b0f680fa7", - "42055e44659f6ffc410b3fb6557346dfbb993df3", - "49e1f2def04c6f7a6a3dacf245a1cd9abe77a6a9"), - "count": 5, - }), - (("https://www.webtoons.com/en/challenge/punderworld" - "/happy-earth-day-/viewer?title_no=312584&episode_no=40"), { - "exception": exception.NotFoundError, - "keyword": { - "comic": "punderworld", - "description": str, - "episode": "36", - "episode_no": "40", - "genre": "challenge", - "title": r"re:^Punderworld - .+", - "title_no": "312584", + ( + ( + "https://www.webtoons.com/en/comedy/safely-endangered" + "/ep-572-earth/viewer?title_no=352&episode_no=572" + ), + { + "url": "55bec5d7c42aba19e3d0d56db25fdf0b0b13be38", + "content": ( + "1748c7e82b6db910fa179f6dc7c4281b0f680fa7", + "42055e44659f6ffc410b3fb6557346dfbb993df3", + "49e1f2def04c6f7a6a3dacf245a1cd9abe77a6a9", + ), + "count": 5, }, - }), + ), + ( + ( + "https://www.webtoons.com/en/challenge/punderworld" + "/happy-earth-day-/viewer?title_no=312584&episode_no=40" + ), + { + "exception": exception.NotFoundError, + "keyword": { + "comic": "punderworld", + "description": str, + "episode": "36", + "episode_no": "40", + "genre": "challenge", + "title": r"re:^Punderworld - .+", + "title_no": "312584", + }, + }, + ), ) def __init__(self, match): - self.path, self.lang, self.genre, self.comic, self.query = \ - match.groups() + self.path, self.lang, self.genre, self.comic, self.query = match.groups() - url = "{}/{}/viewer?{}".format(self.root, self.path, self.query) + url = f"{self.root}/{self.path}/viewer?{self.query}" GalleryExtractor.__init__(self, match, url) def _init(self): @@ -91,60 +106,55 @@ def metadata(self, page): title = extr('', '<') + if extr('
    ", "<") episode_name = extr('

    #', '<') - else: - episode = "" + episode = extr(">#", "<") if extr('', '') + if extr('
    ", "") else: username = author_name = "" return { - "genre" : self.genre, - "comic" : self.comic, - "title_no" : self.title_no, - "episode_no" : self.episode_no, - "title" : text.unescape(title), - "episode" : episode, - "comic_name" : text.unescape(comic_name), + "genre": self.genre, + "comic": self.comic, + "title_no": self.title_no, + "episode_no": self.episode_no, + "title": text.unescape(title), + "episode": episode, + "comic_name": text.unescape(comic_name), "episode_name": text.unescape(episode_name), - "username" : username, - "author_name" : text.unescape(author_name), - "description" : text.unescape(descr), - "lang" : self.lang, - "language" : util.code_to_language(self.lang), + "username": username, + "author_name": text.unescape(author_name), + "description": text.unescape(descr), + "lang": self.lang, + "language": util.code_to_language(self.lang), } @staticmethod def images(page): return [ (url.replace("://webtoon-phinf.", "://swebtoon-phinf."), None) - for url in text.extract_iter( - page, 'class="_images" data-url="', '"') + for url in text.extract_iter(page, 'class="_images" data-url="', '"') ] class WebtoonsComicExtractor(WebtoonsBase, Extractor): """Extractor for an entire comic on webtoons.com""" + subcategory = "comic" categorytransfer = True - pattern = (BASE_PATTERN + r"/([^/?#]+)/([^/?#]+))" - r"/list(?:\?([^#]+))") + pattern = BASE_PATTERN + r"/([^/?#]+)/([^/?#]+))" r"/list(?:\?([^#]+))" example = "https://www.webtoons.com/en/GENRE/TITLE/list?title_no=123" def __init__(self, match): Extractor.__init__(self, match) - self.path, self.lang, self.genre, self.comic, self.query = \ - match.groups() + self.path, self.lang, self.genre, self.comic, self.query = match.groups() def _init(self): self.setup_agegate_cookies() @@ -157,12 +167,11 @@ def items(self): page = None data = { "_extractor": WebtoonsEpisodeExtractor, - "title_no" : text.parse_int(self.title_no), + "title_no": text.parse_int(self.title_no), } while True: - path = "/{}/list?title_no={}&page={}".format( - self.path, self.title_no, self.page_no) + path = f"/{self.path}/list?title_no={self.title_no}&page={self.page_no}" if page and path not in page: return @@ -185,8 +194,5 @@ def items(self): @staticmethod def get_episode_urls(page): """Extract and return all episode urls in 'page'""" - page = text.extr(page, 'id="_listUl"', '') - return [ - match.group(0) - for match in WebtoonsEpisodeExtractor.pattern.finditer(page) - ] + page = text.extr(page, 'id="_listUl"', "") + return [match.group(0) for match in WebtoonsEpisodeExtractor.pattern.finditer(page)] diff --git a/gallery_dl/extractor/weibo.py b/gallery_dl/extractor/weibo.py index 9885d79f71..cd7c03a325 100644 --- a/gallery_dl/extractor/weibo.py +++ b/gallery_dl/extractor/weibo.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,11 +6,15 @@ """Extractors for https://www.weibo.com/""" -from .common import Extractor, Message -from .. import text, util, exception -from ..cache import cache import random +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from .common import Extractor +from .common import Message + BASE_PATTERN = r"(?:https?://)?(?:www\.|m\.)?weibo\.c(?:om|n)" USER_PATTERN = BASE_PATTERN + r"/(?:(u|n|p(?:rofile)?)/)?([^/?#]+)(?:/home)?" @@ -34,7 +36,7 @@ def _init(self): self.retweets = self.config("retweets", False) self.videos = self.config("videos", True) self.gifs = self.config("gifs", True) - self.gifs_video = (self.gifs == "video") + self.gifs_video = self.gifs == "video" cookies = _cookie_cache() if cookies is not None: @@ -46,8 +48,8 @@ def request(self, url, **kwargs): if response.history: if "login.sina.com" in response.url: raise exception.StopExtraction( - "HTTP redirect to login page (%s)", - response.url.partition("?")[0]) + "HTTP redirect to login page (%s)", response.url.partition("?")[0] + ) if "passport.weibo.com" in response.url: self._sina_visitor_system(response) response = Extractor.request(self, url, **kwargs) @@ -55,10 +57,9 @@ def request(self, url, **kwargs): return response def items(self): - original_retweets = (self.retweets == "original") + original_retweets = self.retweets == "original" for status in self.statuses(): - if "ori_mid" in status and not self.retweets: self.log.debug("Skipping %s (快转 retweet)", status["id"]) continue @@ -80,8 +81,7 @@ def items(self): files = [] self._extract_status(status, files) - status["date"] = text.parse_datetime( - status["created_at"], "%a %b %d %H:%M:%S %z %Y") + status["date"] = text.parse_datetime(status["created_at"], "%a %b %d %H:%M:%S %z %Y") status["count"] = len(files) yield Message.Directory, status @@ -127,7 +127,6 @@ def _extract_status(self, status, files): elif pic_type == "livephoto" and self.livephoto: append(pic["largest"].copy()) append({"url": pic["video"]}) - else: append(pic["largest"].copy()) @@ -138,27 +137,23 @@ def _extract_status(self, status, files): def _extract_video(self, info): try: - media = max(info["playback_list"], - key=lambda m: m["meta"]["quality_index"]) + media = max(info["playback_list"], key=lambda m: m["meta"]["quality_index"]) except Exception: - return {"url": (info.get("stream_url_hd") or - info.get("stream_url") or "")} + return {"url": (info.get("stream_url_hd") or info.get("stream_url") or "")} else: return media["play_info"].copy() def _status_by_id(self, status_id): - url = "{}/ajax/statuses/show?id={}".format(self.root, status_id) + url = f"{self.root}/ajax/statuses/show?id={status_id}" return self.request(url).json() def _user_id(self): if len(self.user) >= 10 and self.user.isdecimal(): return self.user[-10:] - else: - url = "{}/ajax/profile/info?{}={}".format( - self.root, - "screen_name" if self._prefix == "n" else "custom", - self.user) - return self.request(url).json()["data"]["user"]["idstr"] + url = "{}/ajax/profile/info?{}={}".format( + self.root, "screen_name" if self._prefix == "n" else "custom", self.user + ) + return self.request(url).json()["data"]["user"]["idstr"] def _pagination(self, endpoint, params): url = self.root + "/ajax" + endpoint @@ -177,8 +172,7 @@ def _pagination(self, endpoint, params): if not data.get("ok"): self.log.debug(response.content) if "since_id" not in params: # first iteration - raise exception.StopExtraction( - '"%s"', data.get("msg") or "unknown error") + raise exception.StopExtraction('"%s"', data.get("msg") or "unknown error") data = data["data"] statuses = data["list"] @@ -219,22 +213,21 @@ def _sina_visitor_system(self, response): data = { "cb": "gen_callback", "fp": '{"os":"1","browser":"Gecko109,0,0,0","fonts":"undefined",' - '"screenInfo":"1920*1080*24","plugins":""}', + '"screenInfo":"1920*1080*24","plugins":""}', } - page = Extractor.request( - self, passport_url, method="POST", headers=headers, data=data).text + page = Extractor.request(self, passport_url, method="POST", headers=headers, data=data).text data = util.json_loads(text.extr(page, "(", ");"))["data"] passport_url = "https://passport.weibo.com/visitor/visitor" params = { - "a" : "incarnate", - "t" : data["tid"], - "w" : "3" if data.get("new_tid") else "2", - "c" : "{:>03}".format(data.get("confidence") or 100), - "gc" : "", - "cb" : "cross_domain", - "from" : "weibo", + "a": "incarnate", + "t": data["tid"], + "w": "3" if data.get("new_tid") else "2", + "c": "{:>03}".format(data.get("confidence") or 100), + "gc": "", + "cb": "cross_domain", + "from": "weibo", "_rand": random.random(), } response = Extractor.request(self, passport_url, params=params) @@ -243,6 +236,7 @@ def _sina_visitor_system(self, response): class WeiboUserExtractor(WeiboExtractor): """Extractor for weibo user profiles""" + subcategory = "user" pattern = USER_PATTERN + r"(?:$|#)" example = "https://weibo.com/USER" @@ -253,18 +247,22 @@ class WeiboUserExtractor(WeiboExtractor): # pass def items(self): - base = "{}/u/{}?tabtype=".format(self.root, self._user_id()) - return self._dispatch_extractors(( - (WeiboHomeExtractor , base + "home"), - (WeiboFeedExtractor , base + "feed"), - (WeiboVideosExtractor , base + "video"), - (WeiboNewvideoExtractor, base + "newVideo"), - (WeiboAlbumExtractor , base + "album"), - ), ("feed",)) + base = f"{self.root}/u/{self._user_id()}?tabtype=" + return self._dispatch_extractors( + ( + (WeiboHomeExtractor, base + "home"), + (WeiboFeedExtractor, base + "feed"), + (WeiboVideosExtractor, base + "video"), + (WeiboNewvideoExtractor, base + "newVideo"), + (WeiboAlbumExtractor, base + "album"), + ), + ("feed",), + ) class WeiboHomeExtractor(WeiboExtractor): """Extractor for weibo 'home' listings""" + subcategory = "home" pattern = USER_PATTERN + r"\?tabtype=home" example = "https://weibo.com/USER?tabtype=home" @@ -277,6 +275,7 @@ def statuses(self): class WeiboFeedExtractor(WeiboExtractor): """Extractor for weibo user feeds""" + subcategory = "feed" pattern = USER_PATTERN + r"\?tabtype=feed" example = "https://weibo.com/USER?tabtype=feed" @@ -289,6 +288,7 @@ def statuses(self): class WeiboVideosExtractor(WeiboExtractor): """Extractor for weibo 'video' listings""" + subcategory = "videos" pattern = USER_PATTERN + r"\?tabtype=video" example = "https://weibo.com/USER?tabtype=video" @@ -303,6 +303,7 @@ def statuses(self): class WeiboNewvideoExtractor(WeiboExtractor): """Extractor for weibo 'newVideo' listings""" + subcategory = "newvideo" pattern = USER_PATTERN + r"\?tabtype=newVideo" example = "https://weibo.com/USER?tabtype=newVideo" @@ -315,6 +316,7 @@ def statuses(self): class WeiboArticleExtractor(WeiboExtractor): """Extractor for weibo 'article' listings""" + subcategory = "article" pattern = USER_PATTERN + r"\?tabtype=article" example = "https://weibo.com/USER?tabtype=article" @@ -327,6 +329,7 @@ def statuses(self): class WeiboAlbumExtractor(WeiboExtractor): """Extractor for weibo 'album' listings""" + subcategory = "album" pattern = USER_PATTERN + r"\?tabtype=album" example = "https://weibo.com/USER?tabtype=album" @@ -349,6 +352,7 @@ def statuses(self): class WeiboStatusExtractor(WeiboExtractor): """Extractor for images from a status on weibo.cn""" + subcategory = "status" pattern = BASE_PATTERN + r"/(detail|status|\d+)/(\w+)" example = "https://weibo.com/detail/12345" @@ -361,6 +365,6 @@ def statuses(self): return (status,) -@cache(maxage=365*86400) +@cache(maxage=365 * 86400) def _cookie_cache(): return None diff --git a/gallery_dl/extractor/wikiart.py b/gallery_dl/extractor/wikiart.py index 938c0485a9..d0afe3c654 100644 --- a/gallery_dl/extractor/wikiart.py +++ b/gallery_dl/extractor/wikiart.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,14 +6,16 @@ """Extractors for https://www.wikiart.org/""" -from .common import Extractor, Message from .. import text +from .common import Extractor +from .common import Message BASE_PATTERN = r"(?:https?://)?(?:www\.)?wikiart\.org/([a-z]+)" class WikiartExtractor(Extractor): """Base class for wikiart extractors""" + category = "wikiart" filename_fmt = "{id}_{title}.{extension}" archive_fmt = "{id}" @@ -66,6 +66,7 @@ def _pagination(self, url, extra_params=None, key="Paintings", stop=False): class WikiartArtistExtractor(WikiartExtractor): """Extractor for an artist's paintings on wikiart.org""" + subcategory = "artist" directory_fmt = ("{category}", "{artist[artistName]}") pattern = BASE_PATTERN + r"/(?!\w+-by-)([\w-]+)/?$" @@ -77,18 +78,18 @@ def __init__(self, match): self.artist = None def metadata(self): - url = "{}/{}/{}?json=2".format(self.root, self.lang, self.artist_name) + url = f"{self.root}/{self.lang}/{self.artist_name}?json=2" self.artist = self.request(url).json() return {"artist": self.artist} def paintings(self): - url = "{}/{}/{}/mode/all-paintings".format( - self.root, self.lang, self.artist_name) + url = f"{self.root}/{self.lang}/{self.artist_name}/mode/all-paintings" return self._pagination(url) class WikiartImageExtractor(WikiartArtistExtractor): """Extractor for individual paintings on wikiart.org""" + subcategory = "image" pattern = BASE_PATTERN + r"/(?!(?:paintings|artists)-by-)([\w-]+)/([\w-]+)" example = "https://www.wikiart.org/en/ARTIST/TITLE" @@ -102,14 +103,17 @@ def paintings(self): if not sep or not year.isdecimal(): title = self.title url = "{}/{}/Search/{} {}".format( - self.root, self.lang, - self.artist.get("artistName") or self.artist_name, title, + self.root, + self.lang, + self.artist.get("artistName") or self.artist_name, + title, ) return self._pagination(url, stop=True) class WikiartArtworksExtractor(WikiartExtractor): """Extractor for artwork collections on wikiart.org""" + subcategory = "artworks" directory_fmt = ("{category}", "Artworks by {group!c}", "{type}") pattern = BASE_PATTERN + r"/paintings-by-([\w-]+)/([\w-]+)" @@ -124,15 +128,15 @@ def metadata(self): return {"group": self.group, "type": self.type} def paintings(self): - url = "{}/{}/paintings-by-{}/{}".format( - self.root, self.lang, self.group, self.type) + url = f"{self.root}/{self.lang}/paintings-by-{self.group}/{self.type}" return self._pagination(url) class WikiartArtistsExtractor(WikiartExtractor): """Extractor for artist collections on wikiart.org""" + subcategory = "artists" - pattern = (BASE_PATTERN + r"/artists-by-([\w-]+)/([\w-]+)") + pattern = BASE_PATTERN + r"/artists-by-([\w-]+)/([\w-]+)" example = "https://www.wikiart.org/en/artists-by-GROUP/TYPE" def __init__(self, match): @@ -141,8 +145,7 @@ def __init__(self, match): self.type = match.group(3) def items(self): - url = "{}/{}/App/Search/Artists-by-{}".format( - self.root, self.lang, self.group) + url = f"{self.root}/{self.lang}/App/Search/Artists-by-{self.group}" params = {"json": "3", "searchterm": self.type} for artist in self._pagination(url, params, "Artists"): diff --git a/gallery_dl/extractor/wikifeet.py b/gallery_dl/extractor/wikifeet.py index d3586c0172..ba9467e9eb 100644 --- a/gallery_dl/extractor/wikifeet.py +++ b/gallery_dl/extractor/wikifeet.py @@ -1,23 +1,22 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. """Extractors for https://www.wikifeet.com/""" +from .. import text +from .. import util from .common import GalleryExtractor -from .. import text, util class WikifeetGalleryExtractor(GalleryExtractor): """Extractor for image galleries from wikifeet.com""" + category = "wikifeet" directory_fmt = ("{category}", "{celebrity}") filename_fmt = "{category}_{celeb}_{pid}.{extension}" archive_fmt = "{type}_{celeb}_{pid}" - pattern = (r"(?:https?://)(?:(?:www\.)?wikifeetx?|" - r"men\.wikifeet)\.com/([^/?#]+)") + pattern = r"(?:https?://)(?:(?:www\.)?wikifeetx?|" r"men\.wikifeet)\.com/([^/?#]+)" example = "https://www.wikifeet.com/CELEB" def __init__(self, match): @@ -31,14 +30,15 @@ def __init__(self, match): def metadata(self, page): extr = text.extract_from(page) return { - "celeb" : self.celeb, - "type" : self.type, - "rating" : text.parse_float(extr('"ratingValue": "', '"')), - "celebrity" : text.unescape(extr("times'>", "

    ")), - "shoesize" : text.remove_html(extr("Shoe Size:", "edit")), + "celeb": self.celeb, + "type": self.type, + "rating": text.parse_float(extr('"ratingValue": "', '"')), + "celebrity": text.unescape(extr("times'>", "

    ")), + "shoesize": text.remove_html(extr("Shoe Size:", "edit")), "birthplace": text.remove_html(extr("Birthplace:", "edit")), - "birthday" : text.parse_datetime(text.remove_html( - extr("Birth Date:", "edit")), "%Y-%m-%d"), + "birthday": text.parse_datetime( + text.remove_html(extr("Birth Date:", "edit")), "%Y-%m-%d" + ), } def images(self, page): @@ -52,14 +52,14 @@ def images(self, page): } ufmt = "https://pics.wikifeet.com/" + self.celeb + "-Feet-{}.jpg" return [ - (ufmt.format(data["pid"]), { - "pid" : data["pid"], - "width" : data["pw"], - "height": data["ph"], - "tags" : [ - tagmap[tag] - for tag in data["tags"] if tag in tagmap - ], - }) + ( + ufmt.format(data["pid"]), + { + "pid": data["pid"], + "width": data["pw"], + "height": data["ph"], + "tags": [tagmap[tag] for tag in data["tags"] if tag in tagmap], + }, + ) for data in util.json_loads(text.extr(page, "['gdata'] = ", ";")) ] diff --git a/gallery_dl/extractor/wikimedia.py b/gallery_dl/extractor/wikimedia.py index 4eae53750a..8c85c34eef 100644 --- a/gallery_dl/extractor/wikimedia.py +++ b/gallery_dl/extractor/wikimedia.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022 Ailothaen # Copyright 2024 Mike Fährmann # @@ -9,13 +7,16 @@ """Extractors for Wikimedia sites""" -from .common import BaseExtractor, Message -from .. import text, exception +from .. import exception +from .. import text from ..cache import cache +from .common import BaseExtractor +from .common import Message class WikimediaExtractor(BaseExtractor): """Base class for wikimedia extractors""" + basecategory = "wikimedia" filename_fmt = "{filename} ({sha1[:8]}).{extension}" archive_fmt = "{sha1}" @@ -28,7 +29,8 @@ def __init__(self, match): self.category = self.root.split(".")[-2] elif self.category in ("fandom", "wikigg"): self.category = "{}-{}".format( - self.category, self.root.partition(".")[0].rpartition("/")[2]) + self.category, self.root.partition(".")[0].rpartition("/")[2] + ) self.per_page = self.config("limit", 50) @@ -42,7 +44,7 @@ def _init(self): else: self.api_url = None - @cache(maxage=36500*86400, keyarg=1) + @cache(maxage=36500 * 86400, keyarg=1) def _search_api_path(self, root): self.log.debug("Probing possible API endpoints") for path in ("/api.php", "/w/api.php", "/wiki/api.php"): @@ -55,18 +57,12 @@ def _search_api_path(self, root): @staticmethod def prepare(image): """Adjust the content of a image object""" - image["metadata"] = { - m["name"]: m["value"] - for m in image["metadata"] or ()} - image["commonmetadata"] = { - m["name"]: m["value"] - for m in image["commonmetadata"] or ()} + image["metadata"] = {m["name"]: m["value"] for m in image["metadata"] or ()} + image["commonmetadata"] = {m["name"]: m["value"] for m in image["commonmetadata"] or ()} filename = image["canonicaltitle"] - image["filename"], _, image["extension"] = \ - filename.partition(":")[2].rpartition(".") - image["date"] = text.parse_datetime( - image["timestamp"], "%Y-%m-%dT%H:%M:%SZ") + image["filename"], _, image["extension"] = filename.partition(":")[2].rpartition(".") + image["date"] = text.parse_datetime(image["timestamp"], "%Y-%m-%dT%H:%M:%SZ") def items(self): for info in self._pagination(self.params): @@ -126,65 +122,68 @@ def _pagination(self, params): params.update(continuation) -BASE_PATTERN = WikimediaExtractor.update({ - "wikimedia": { - "root": None, - "pattern": r"[a-z]{2,}\." - r"wik(?:i(?:pedia|quote|books|source|news|versity|data" - r"|voyage)|tionary)" - r"\.org", - "api-path": "/w/api.php", - }, - "wikispecies": { - "root": "https://species.wikimedia.org", - "pattern": r"species\.wikimedia\.org", - "api-path": "/w/api.php", - }, - "wikimediacommons": { - "root": "https://commons.wikimedia.org", - "pattern": r"commons\.wikimedia\.org", - "api-path": "/w/api.php", - }, - "mediawiki": { - "root": "https://www.mediawiki.org", - "pattern": r"(?:www\.)?mediawiki\.org", - "api-path": "/w/api.php", - }, - "fandom": { - "root": None, - "pattern": r"[\w-]+\.fandom\.com", - "api-path": "/api.php", - }, - "wikigg": { - "root": None, - "pattern": r"\w+\.wiki\.gg", - "api-path": "/api.php", - }, - "mariowiki": { - "root": "https://www.mariowiki.com", - "pattern": r"(?:www\.)?mariowiki\.com", - "api-path": "/api.php", - }, - "bulbapedia": { - "root": "https://bulbapedia.bulbagarden.net", - "pattern": r"(?:bulbapedia|archives)\.bulbagarden\.net", - "api-path": "/w/api.php", - }, - "pidgiwiki": { - "root": "https://www.pidgi.net", - "pattern": r"(?:www\.)?pidgi\.net", - "api-path": "/wiki/api.php", - }, - "azurlanewiki": { - "root": "https://azurlane.koumakan.jp", - "pattern": r"azurlane\.koumakan\.jp", - "api-path": "/w/api.php", - }, -}) +BASE_PATTERN = WikimediaExtractor.update( + { + "wikimedia": { + "root": None, + "pattern": r"[a-z]{2,}\." + r"wik(?:i(?:pedia|quote|books|source|news|versity|data" + r"|voyage)|tionary)" + r"\.org", + "api-path": "/w/api.php", + }, + "wikispecies": { + "root": "https://species.wikimedia.org", + "pattern": r"species\.wikimedia\.org", + "api-path": "/w/api.php", + }, + "wikimediacommons": { + "root": "https://commons.wikimedia.org", + "pattern": r"commons\.wikimedia\.org", + "api-path": "/w/api.php", + }, + "mediawiki": { + "root": "https://www.mediawiki.org", + "pattern": r"(?:www\.)?mediawiki\.org", + "api-path": "/w/api.php", + }, + "fandom": { + "root": None, + "pattern": r"[\w-]+\.fandom\.com", + "api-path": "/api.php", + }, + "wikigg": { + "root": None, + "pattern": r"\w+\.wiki\.gg", + "api-path": "/api.php", + }, + "mariowiki": { + "root": "https://www.mariowiki.com", + "pattern": r"(?:www\.)?mariowiki\.com", + "api-path": "/api.php", + }, + "bulbapedia": { + "root": "https://bulbapedia.bulbagarden.net", + "pattern": r"(?:bulbapedia|archives)\.bulbagarden\.net", + "api-path": "/w/api.php", + }, + "pidgiwiki": { + "root": "https://www.pidgi.net", + "pattern": r"(?:www\.)?pidgi\.net", + "api-path": "/wiki/api.php", + }, + "azurlanewiki": { + "root": "https://azurlane.koumakan.jp", + "pattern": r"azurlane\.koumakan\.jp", + "api-path": "/w/api.php", + }, + } +) class WikimediaArticleExtractor(WikimediaExtractor): """Extractor for wikimedia articles""" + subcategory = "article" directory_fmt = ("{category}", "{page}") pattern = BASE_PATTERN + r"/(?!static/)([^?#]+)" @@ -210,19 +209,19 @@ def __init__(self, match): if prefix == "category": self.params = { "generator": "categorymembers", - "gcmtitle" : path, - "gcmtype" : "file", - "gcmlimit" : self.per_page, + "gcmtitle": path, + "gcmtype": "file", + "gcmlimit": self.per_page, } elif prefix == "file": self.params = { - "titles" : path, + "titles": path, } else: self.params = { "generator": "images", - "gimlimit" : self.per_page, - "titles" : path, + "gimlimit": self.per_page, + "titles": path, } def prepare(self, image): @@ -232,6 +231,7 @@ def prepare(self, image): class WikimediaWikiExtractor(WikimediaExtractor): """Extractor for all files on a MediaWiki instance""" + subcategory = "wiki" pattern = BASE_PATTERN + r"/?$" example = "https://en.wikipedia.org/" @@ -241,7 +241,7 @@ def __init__(self, match): # ref: https://www.mediawiki.org/wiki/API:Allpages self.params = { - "generator" : "allpages", + "generator": "allpages", "gapnamespace": 6, # "File" namespace - "gaplimit" : self.per_page, + "gaplimit": self.per_page, } diff --git a/gallery_dl/extractor/xhamster.py b/gallery_dl/extractor/xhamster.py index 6dc9362a5c..1f197713da 100644 --- a/gallery_dl/extractor/xhamster.py +++ b/gallery_dl/extractor/xhamster.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,15 +6,19 @@ """Extractors for https://xhamster.com/""" -from .common import Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import Message -BASE_PATTERN = (r"(?:https?://)?((?:[\w-]+\.)?xhamster" - r"(?:\d?\.(?:com|one|desi)|\.porncache\.net))") +BASE_PATTERN = ( + r"(?:https?://)?((?:[\w-]+\.)?xhamster" r"(?:\d?\.(?:com|one|desi)|\.porncache\.net))" +) class XhamsterExtractor(Extractor): """Base class for xhamster extractors""" + category = "xhamster" def __init__(self, match): @@ -26,9 +28,9 @@ def __init__(self, match): class XhamsterGalleryExtractor(XhamsterExtractor): """Extractor for image galleries on xhamster.com""" + subcategory = "gallery" - directory_fmt = ("{category}", "{user[name]}", - "{gallery[id]} {gallery[title]}") + directory_fmt = ("{category}", "{user[name]}", "{gallery[id]} {gallery[title]}") filename_fmt = "{num:>03}_{id}.{extension}" archive_fmt = "{id}" pattern = BASE_PATTERN + r"(/photos/gallery/[^/?#]+)" @@ -54,26 +56,24 @@ def metadata(self): imgs = self.data["photosGalleryModel"] return { - "user": - { - "id" : text.parse_int(user["id"]), - "url" : user["pageURL"], - "name" : user["name"], - "retired" : user["retired"], - "verified" : user["verified"], + "user": { + "id": text.parse_int(user["id"]), + "url": user["pageURL"], + "name": user["name"], + "retired": user["retired"], + "verified": user["verified"], "subscribers": user["subscribers"], }, - "gallery": - { - "id" : text.parse_int(imgs["id"]), - "tags" : [c["name"] for c in imgs["categories"]], - "date" : text.parse_timestamp(imgs["created"]), - "views" : text.parse_int(imgs["views"]), - "likes" : text.parse_int(imgs["rating"]["likes"]), - "dislikes" : text.parse_int(imgs["rating"]["dislikes"]), - "title" : text.unescape(imgs["title"]), + "gallery": { + "id": text.parse_int(imgs["id"]), + "tags": [c["name"] for c in imgs["categories"]], + "date": text.parse_timestamp(imgs["created"]), + "views": text.parse_int(imgs["views"]), + "likes": text.parse_int(imgs["rating"]["likes"]), + "dislikes": text.parse_int(imgs["rating"]["dislikes"]), + "title": text.unescape(imgs["title"]), "description": text.unescape(imgs["description"]), - "thumbnail" : imgs["thumbURL"], + "thumbnail": imgs["thumbURL"], }, "count": text.parse_int(imgs["quantity"]), } @@ -95,12 +95,12 @@ def images(self): def _data(self, url): page = self.request(url).text - return util.json_loads(text.extr( - page, "window.initials=", "").rstrip("\n\r;")) + return util.json_loads(text.extr(page, "window.initials=", "").rstrip("\n\r;")) class XhamsterUserExtractor(XhamsterExtractor): """Extractor for all galleries of an xhamster user""" + subcategory = "user" pattern = BASE_PATTERN + r"/users/([^/?#]+)(?:/photos)?/?(?:$|[?#])" example = "https://xhamster.com/users/USER/photos" @@ -110,7 +110,7 @@ def __init__(self, match): self.user = match.group(2) def items(self): - url = "{}/users/{}/photos".format(self.root, self.user) + url = f"{self.root}/users/{self.user}/photos" data = {"_extractor": XhamsterGalleryExtractor} while url: diff --git a/gallery_dl/extractor/xvideos.py b/gallery_dl/extractor/xvideos.py index da9d6b07ab..139cd876e8 100644 --- a/gallery_dl/extractor/xvideos.py +++ b/gallery_dl/extractor/xvideos.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,24 +6,27 @@ """Extractors for https://www.xvideos.com/""" -from .common import GalleryExtractor, Extractor, Message -from .. import text, util +from .. import text +from .. import util +from .common import Extractor +from .common import GalleryExtractor +from .common import Message -BASE_PATTERN = (r"(?:https?://)?(?:www\.)?xvideos\.com" - r"/(?:profiles|(?:amateur-|model-)?channels)") +BASE_PATTERN = r"(?:https?://)?(?:www\.)?xvideos\.com" r"/(?:profiles|(?:amateur-|model-)?channels)" -class XvideosBase(): +class XvideosBase: """Base class for xvideos extractors""" + category = "xvideos" root = "https://www.xvideos.com" class XvideosGalleryExtractor(XvideosBase, GalleryExtractor): """Extractor for user profile galleries on xvideos.com""" + subcategory = "gallery" - directory_fmt = ("{category}", "{user[name]}", - "{gallery[id]} {gallery[title]}") + directory_fmt = ("{category}", "{user[name]}", "{gallery[id]} {gallery[title]}") filename_fmt = "{category}_{gallery[id]}_{num:>03}.{extension}" archive_fmt = "{gallery[id]}_{num}" pattern = BASE_PATTERN + r"/([^/?#]+)/photos/(\d+)" @@ -33,41 +34,38 @@ class XvideosGalleryExtractor(XvideosBase, GalleryExtractor): def __init__(self, match): self.user, self.gallery_id = match.groups() - url = "{}/profiles/{}/photos/{}".format( - self.root, self.user, self.gallery_id) + url = f"{self.root}/profiles/{self.user}/photos/{self.gallery_id}" GalleryExtractor.__init__(self, match, url) def metadata(self, page): extr = text.extract_from(page) user = { - "id" : text.parse_int(extr('"id_user":', ',')), + "id": text.parse_int(extr('"id_user":', ",")), "display": extr('"display":"', '"'), - "sex" : extr('"sex":"', '"'), - "name" : self.user, + "sex": extr('"sex":"', '"'), + "name": self.user, } title = extr('"title":"', '"') - user["description"] = extr( - '', '').strip() - tags = extr('Tagged:', '<').strip() + user["description"] = extr('', "").strip() + tags = extr("Tagged:", "<").strip() return { "user": user, "gallery": { - "id" : text.parse_int(self.gallery_id), + "id": text.parse_int(self.gallery_id), "title": text.unescape(title), - "tags" : text.unescape(tags).split(", ") if tags else [], + "tags": text.unescape(tags).split(", ") if tags else [], }, } def images(self, page): results = [ (url, None) - for url in text.extract_iter( - page, 'Next"))["data"] + data = util.json_loads(text.extr(page, "xv.conf=", ";"))["data"] if not isinstance(data["galleries"], dict): return @@ -107,7 +104,7 @@ def items(self): galleries = [ { - "id" : text.parse_int(gid), + "id": text.parse_int(gid), "title": text.unescape(gdata["title"]), "count": gdata["nb_pics"], "_extractor": XvideosGalleryExtractor, @@ -117,6 +114,5 @@ def items(self): galleries.sort(key=lambda x: x["id"]) for gallery in galleries: - url = "https://www.xvideos.com/profiles/{}/photos/{}".format( - self.user, gallery["id"]) + url = "https://www.xvideos.com/profiles/{}/photos/{}".format(self.user, gallery["id"]) yield Message.Queue, url, gallery diff --git a/gallery_dl/extractor/ytdl.py b/gallery_dl/extractor/ytdl.py index 168845ea7f..b7303f683c 100644 --- a/gallery_dl/extractor/ytdl.py +++ b/gallery_dl/extractor/ytdl.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,16 @@ """Extractors for sites supported by youtube-dl""" -from .common import Extractor, Message -from .. import ytdl, config, exception +from .. import config +from .. import exception +from .. import ytdl +from .common import Extractor +from .common import Message class YoutubeDLExtractor(Extractor): """Generic extractor for youtube-dl supported URLs""" + category = "ytdl" directory_fmt = ("{category}", "{subcategory}") filename_fmt = "{title}-{id}.{extension}" @@ -23,8 +25,7 @@ class YoutubeDLExtractor(Extractor): def __init__(self, match): # import main youtube_dl module - ytdl_module = ytdl.import_module(config.get( - ("extractor", "ytdl"), "module")) + ytdl_module = ytdl.import_module(config.get(("extractor", "ytdl"), "module")) self.ytdl_module_name = ytdl_module.__name__ # find suitable youtube_dl extractor @@ -49,19 +50,19 @@ def __init__(self, match): def items(self): # import subcategory module ytdl_module = ytdl.import_module( - config.get(("extractor", "ytdl", self.subcategory), "module") or - self.ytdl_module_name) + config.get(("extractor", "ytdl", self.subcategory), "module") or self.ytdl_module_name + ) self.log.debug("Using %s", ytdl_module) # construct YoutubeDL object extr_opts = { - "extract_flat" : "in_playlist", + "extract_flat": "in_playlist", "force_generic_extractor": self.force_generic_extractor, } user_opts = { - "retries" : self._retries, - "socket_timeout" : self._timeout, - "nocheckcertificate" : not self._verify, + "retries": self._retries, + "socket_timeout": self._timeout, + "nocheckcertificate": not self._verify, } if self._proxies: @@ -72,8 +73,7 @@ def items(self): user_opts["username"], user_opts["password"] = username, password del username, password - ytdl_instance = ytdl.construct_YoutubeDL( - ytdl_module, self, user_opts, extr_opts) + ytdl_instance = ytdl.construct_YoutubeDL(ytdl_module, self, user_opts, extr_opts) # transfer cookies to ytdl cookies = self.cookies @@ -85,17 +85,15 @@ def items(self): # extract youtube_dl info_dict try: info_dict = ytdl_instance._YoutubeDL__extract_info( - self.ytdl_url, - ytdl_instance.get_info_extractor(self.ytdl_ie_key), - False, {}, True) + self.ytdl_url, ytdl_instance.get_info_extractor(self.ytdl_ie_key), False, {}, True + ) except ytdl_module.utils.YoutubeDLError: raise exception.StopExtraction("Failed to extract video data") if not info_dict: return elif "entries" in info_dict: - results = self._process_entries( - ytdl_module, ytdl_instance, info_dict["entries"]) + results = self._process_entries(ytdl_module, ytdl_instance, info_dict["entries"]) else: results = (info_dict,) @@ -105,9 +103,7 @@ def items(self): info_dict["_ytdl_info_dict"] = info_dict info_dict["_ytdl_instance"] = ytdl_instance - url = "ytdl:" + (info_dict.get("url") or - info_dict.get("webpage_url") or - self.ytdl_url) + url = "ytdl:" + (info_dict.get("url") or info_dict.get("webpage_url") or self.ytdl_url) yield Message.Directory, info_dict yield Message.Url, url, info_dict @@ -120,16 +116,15 @@ def _process_entries(self, ytdl_module, ytdl_instance, entries): if entry.get("_type") in ("url", "url_transparent"): try: entry = ytdl_instance.extract_info( - entry["url"], False, - ie_key=entry.get("ie_key")) + entry["url"], False, ie_key=entry.get("ie_key") + ) except ytdl_module.utils.YoutubeDLError: continue if not entry: continue if "entries" in entry: - yield from self._process_entries( - ytdl_module, ytdl_instance, entry["entries"]) + yield from self._process_entries(ytdl_module, ytdl_instance, entry["entries"]) else: yield entry diff --git a/gallery_dl/extractor/zerochan.py b/gallery_dl/extractor/zerochan.py index f9b1a7f4c3..b6dae44f46 100644 --- a/gallery_dl/extractor/zerochan.py +++ b/gallery_dl/extractor/zerochan.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2022-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,17 +6,21 @@ """Extractors for https://www.zerochan.net/""" -from .booru import BooruExtractor -from ..cache import cache -from .. import text, util, exception import collections import re +from .. import exception +from .. import text +from .. import util +from ..cache import cache +from .booru import BooruExtractor + BASE_PATTERN = r"(?:https?://)?(?:www\.)?zerochan\.net" class ZerochanExtractor(BooruExtractor): """Base class for zerochan extractors""" + category = "zerochan" root = "https://www.zerochan.net" filename_fmt = "{id}.{extension}" @@ -32,7 +34,7 @@ class ZerochanExtractor(BooruExtractor): def login(self): self._logged_in = True if self.cookies_check(self.cookies_names): - return + return None username, password = self._get_auth_info() if username: @@ -40,20 +42,20 @@ def login(self): self._logged_in = False - @cache(maxage=90*86400, keyarg=1) + @cache(maxage=90 * 86400, keyarg=1) def _login_impl(self, username, password): self.log.info("Logging in as %s", username) url = self.root + "/login" headers = { - "Origin" : self.root, - "Referer" : url, + "Origin": self.root, + "Referer": url, } data = { - "ref" : "/", - "name" : username, + "ref": "/", + "name": username, "password": password, - "login" : "Login", + "login": "Login", } response = self.request(url, method="POST", headers=headers, data=data) @@ -63,23 +65,21 @@ def _login_impl(self, username, password): return response.cookies def _parse_entry_html(self, entry_id): - url = "{}/{}".format(self.root, entry_id) + url = f"{self.root}/{entry_id}" extr = text.extract_from(self.request(url).text) data = { - "id" : text.parse_int(entry_id), - "author" : text.parse_unicode_escapes(extr(' "name": "', '"')), + "id": text.parse_int(entry_id), + "author": text.parse_unicode_escapes(extr(' "name": "', '"')), "file_url": extr('"contentUrl": "', '"'), - "date" : text.parse_datetime(extr('"datePublished": "', '"')), - "width" : text.parse_int(extr('"width": "', ' ')), - "height" : text.parse_int(extr('"height": "', ' ')), - "size" : text.parse_bytes(extr('"contentSize": "', 'B')), - "path" : text.split_html(extr( - 'class="breadcrumbs', ''))[2:], + "date": text.parse_datetime(extr('"datePublished": "', '"')), + "width": text.parse_int(extr('"width": "', " ")), + "height": text.parse_int(extr('"height": "', " ")), + "size": text.parse_bytes(extr('"contentSize": "', "B")), + "path": text.split_html(extr('class="breadcrumbs', ""))[2:], "uploader": extr('href="/user/', '"'), - "tags" : extr('
      '), - "source" : text.unescape(text.extr( - extr('id="source-url"', ''), 'href="', '"')), + "tags": extr('
        "), + "source": text.unescape(text.extr(extr('id="source-url"', ""), 'href="', '"')), } html = data["tags"] @@ -92,7 +92,7 @@ def _parse_entry_html(self, entry_id): return data def _parse_entry_api(self, entry_id): - url = "{}/{}?json".format(self.root, entry_id) + url = f"{self.root}/{entry_id}?json" text = self.request(url).text try: item = util.json_loads(text) @@ -103,14 +103,14 @@ def _parse_entry_api(self, entry_id): item = util.json_loads(text) data = { - "id" : item["id"], + "id": item["id"], "file_url": item["full"], - "width" : item["width"], - "height" : item["height"], - "size" : item["size"], - "name" : item["primary"], - "md5" : item["hash"], - "source" : item.get("source"), + "width": item["width"], + "height": item["height"], + "size": item["size"], + "name": item["primary"], + "md5": item["hash"], + "source": item.get("source"), } if not self._logged_in: @@ -146,8 +146,7 @@ def _init(self): self.session.headers["User-Agent"] = util.USERAGENT def metadata(self): - return {"search_tags": text.unquote( - self.search_tag.replace("+", " "))} + return {"search_tags": text.unquote(self.search_tag.replace("+", " "))} def posts_html(self): url = self.root + "/" + self.search_tag @@ -157,11 +156,11 @@ def posts_html(self): while True: page = self.request(url, params=params).text - thumbs = text.extr(page, '
          ") extr = text.extract_from(thumbs) while True: - post = extr('
        • ") if not post: break @@ -172,13 +171,12 @@ def posts_html(self): yield post else: yield { - "id" : extr('href="/', '"'), - "name" : extr('alt="', '"'), - "width" : extr('title="', '✕'), - "height": extr('', ' '), - "size" : extr('', 'b'), - "file_url": "https://static." + extr( - '", ""))[:-11], + "slug": self.slug, + "title": text.unescape(text.extr(page, "", ""))[:-11], } def images(self, page): @@ -48,10 +47,15 @@ def images_v2(self, page): results = [] while True: - for path in text.extract_iter( - page, ' class="picbox"> limit: obj = obj[:limit_hint] + hint return fmt(obj) + return apply_limit def _default_format(format_spec, default): def wrap(obj): return format(obj, format_spec) + return wrap -class Literal(): +class Literal: # __getattr__, __getattribute__, and __class_getitem__ # are all slower than regular __getitem__ diff --git a/gallery_dl/job.py b/gallery_dl/job.py index be244279bd..5ccd87be6c 100644 --- a/gallery_dl/job.py +++ b/gallery_dl/job.py @@ -1,37 +1,35 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import sys +import collections import errno -import logging import functools -import collections +import logging +import sys -from . import ( - extractor, - downloader, - postprocessor, - archive, - config, - exception, - formatter, - output, - path, - text, - util, - version, -) +from . import archive +from . import config +from . import downloader +from . import exception +from . import extractor +from . import formatter +from . import output +from . import path +from . import postprocessor +from . import text +from . import util +from . import version from .extractor.message import Message + stdout_write = output.stdout_write -class Job(): +class Job: """Base class for Job types""" + ulog = None _logger_adapter = output.LoggerAdapter @@ -49,15 +47,14 @@ def __init__(self, extr, parent=None): cfgpath = [] if parent: - if extr.category == parent.extractor.category or \ - extr.category in parent.parents: + if extr.category == parent.extractor.category or extr.category in parent.parents: parents = parent.parents else: parents = parent.parents + (parent.extractor.category,) if parents: for category in parents: - cat = "{}>{}".format(category, extr.category) + cat = f"{category}>{extr.category}" cfgpath.append((cat, extr.subcategory)) cfgpath.append((extr.category, extr.subcategory)) self.parents = parents @@ -78,16 +75,18 @@ def __init__(self, extr, parent=None): actions = extr.config("actions") if actions: - from .actions import LoggerAdapter, parse + from .actions import LoggerAdapter + from .actions import parse + self._logger_adapter = LoggerAdapter self._logger_actions = parse(actions) path_proxy = output.PathfmtProxy(self) self._logger_extra = { - "job" : self, + "job": self, "extractor": extr, - "path" : path_proxy, - "keywords" : output.KwdictProxy(self), + "path": path_proxy, + "keywords": output.KwdictProxy(self), } extr.log = self._wrap_logger(extr.log) extr.log.debug("Using %s for '%s'", extr.__class__.__name__, extr.url) @@ -106,8 +105,7 @@ def __init__(self, extr, parent=None): self.metadata_http = extr.config2("metadata-http", "http-metadata") metadata_path = extr.config2("metadata-path", "path-metadata") metadata_version = extr.config2("metadata-version", "version-metadata") - metadata_extractor = extr.config2( - "metadata-extractor", "extractor-metadata") + metadata_extractor = extr.config2("metadata-extractor", "extractor-metadata") if metadata_path: self.kwdict[metadata_path] = path_proxy @@ -115,9 +113,9 @@ def __init__(self, extr, parent=None): self.kwdict[metadata_extractor] = extr if metadata_version: self.kwdict[metadata_version] = { - "version" : version.__version__, - "is_executable" : util.EXECUTABLE, - "current_git_head": util.git_head() + "version": version.__version__, + "is_executable": util.EXECUTABLE, + "current_git_head": util.git_head(), } # user-supplied metadata kwdict = extr.config("keywords") @@ -142,8 +140,7 @@ def run(self): self._init() # sleep before extractor start - sleep = util.build_duration_func( - extractor.config("sleep-extractor")) + sleep = util.build_duration_func(extractor.config("sleep-extractor")) if sleep: extractor.sleep(sleep(), "extractor") @@ -161,16 +158,20 @@ def run(self): log.debug("", exc_info=exc) self.status |= exc.code except OSError as exc: - log.error("Unable to download data: %s: %s", - exc.__class__.__name__, exc) + log.error("Unable to download data: %s: %s", exc.__class__.__name__, exc) log.debug("", exc_info=exc) self.status |= 128 except Exception as exc: - log.error(("An unexpected error occurred: %s - %s. " - "Please run gallery-dl again with the --verbose flag, " - "copy its output and report this issue on " - "https://github.com/mikf/gallery-dl/issues ."), - exc.__class__.__name__, exc) + log.error( + ( + "An unexpected error occurred: %s - %s. " + "Please run gallery-dl again with the --verbose flag, " + "copy its output and report this issue on " + "https://github.com/mikf/gallery-dl/issues ." + ), + exc.__class__.__name__, + exc, + ) log.debug("", exc_info=exc) self.status |= 1 except BaseException: @@ -256,8 +257,7 @@ def _prepare_predicates(self, target, skip=True): try: pred = util.RangePredicate(prange) except ValueError as exc: - self.extractor.log.warning( - "invalid %s range: %s", target, exc) + self.extractor.log.warning("invalid %s range: %s", target, exc) else: if skip and pred.lower > 1 and not pfilter: pred.index += self.extractor.skip(pred.lower - 1) @@ -334,7 +334,6 @@ def handle_url(self, url, kwdict): # download from URL if not self.download(url): - # use fallback URLs if available/enabled fallback = kwdict.get("_fallback", ()) if self.fallback else () for num, url in enumerate(fallback, 1): @@ -345,8 +344,7 @@ def handle_url(self, url, kwdict): else: # download failed self.status |= 4 - self.log.error("Failed to download %s", - pathfmt.filename or url) + self.log.error("Failed to download %s", pathfmt.filename or url) if "error" in hooks: for callback in hooks["error"]: callback(pathfmt) @@ -436,10 +434,12 @@ def handle_queue(self, url, kwdict): if status: self.status |= status - if (status & 95 and # not FormatError or OSError - "_fallback" in kwdict and self.fallback): - fallback = kwdict["_fallback"] = \ - iter(kwdict["_fallback"]) + if ( + status & 95 # not FormatError or OSError + and "_fallback" in kwdict + and self.fallback + ): + fallback = kwdict["_fallback"] = iter(kwdict["_fallback"]) try: url = next(fallback) except StopIteration: @@ -478,10 +478,9 @@ def handle_finalize(self): if "finalize-error" in hooks: for callback in hooks["finalize-error"]: callback(pathfmt) - else: - if "finalize-success" in hooks: - for callback in hooks["finalize-success"]: - callback(pathfmt) + elif "finalize-success" in hooks: + for callback in hooks["finalize-success"]: + callback(pathfmt) def handle_skip(self): pathfmt = self.pathfmt @@ -551,23 +550,25 @@ def initialize(self, kwdict=None): archive_path = cfg("archive") if archive_path: archive_path = util.expand_path(archive_path) - archive_format = (cfg("archive-prefix", extr.category) + - cfg("archive-format", extr.archive_fmt)) - archive_pragma = (cfg("archive-pragma")) + archive_format = cfg("archive-prefix", extr.category) + cfg( + "archive-format", extr.archive_fmt + ) + archive_pragma = cfg("archive-pragma") try: if "{" in archive_path: - archive_path = formatter.parse( - archive_path).format_map(kwdict) + archive_path = formatter.parse(archive_path).format_map(kwdict) if cfg("archive-mode") == "memory": archive_cls = archive.DownloadArchiveMemory else: archive_cls = archive.DownloadArchive - self.archive = archive_cls( - archive_path, archive_format, archive_pragma) + self.archive = archive_cls(archive_path, archive_format, archive_pragma) except Exception as exc: extr.log.warning( "Failed to open download archive at '%s' (%s: %s)", - archive_path, exc.__class__.__name__, exc) + archive_path, + exc.__class__.__name__, + exc, + ) else: extr.log.debug("Using download archive '%s'", archive_path) @@ -578,8 +579,8 @@ def initialize(self, kwdict=None): else: if isinstance(events, str): events = events.split(",") - self._archive_write_file = ("file" in events) - self._archive_write_skip = ("skip" in events) + self._archive_write_file = "file" in events + self._archive_write_skip = "skip" in events skip = cfg("skip", True) if skip: @@ -634,8 +635,7 @@ def initialize(self, kwdict=None): else: clist = pp_dict.get("blacklist") negate = True - if clist and not util.build_extractor_filter( - clist, negate)(extr): + if clist and not util.build_extractor_filter(clist, negate)(extr): continue name = pp_dict.get("name") @@ -646,8 +646,9 @@ def initialize(self, kwdict=None): try: pp_obj = pp_cls(self, pp_dict) except Exception as exc: - pp_log.error("'%s' initialization failed: %s: %s", - name, exc.__class__.__name__, exc) + pp_log.error( + "'%s' initialization failed: %s: %s", name, exc.__class__.__name__, exc + ) pp_log.debug("", exc_info=exc) else: pp_list.append(pp_obj) @@ -664,8 +665,7 @@ def register_hooks(self, hooks, options=None): if expr: condition = util.compile_filter(expr) for hook, callback in hooks.items(): - self.hooks[hook].append(functools.partial( - self._call_hook, callback, condition)) + self.hooks[hook].append(functools.partial(self._call_hook, callback, condition)) else: for hook, callback in hooks.items(): self.hooks[hook].append(callback) @@ -715,19 +715,20 @@ def __init__(self, url, parent=None): self.private = config.get(("output",), "private") def handle_url(self, url, kwdict): - stdout_write("\nKeywords for filenames and --filter:\n" - "------------------------------------\n") + stdout_write( + "\nKeywords for filenames and --filter:\n" "------------------------------------\n" + ) if self.metadata_http and url.startswith("http"): kwdict[self.metadata_http] = util.extract_headers( - self.extractor.request(url, method="HEAD")) + self.extractor.request(url, method="HEAD") + ) self.print_kwdict(kwdict) raise exception.StopExtraction() def handle_directory(self, kwdict): - stdout_write("Keywords for directory names:\n" - "-----------------------------\n") + stdout_write("Keywords for directory names:\n" "-----------------------------\n") self.print_kwdict(kwdict) def handle_queue(self, url, kwdict): @@ -738,18 +739,16 @@ def handle_queue(self, url, kwdict): if not util.filter_dict(kwdict): self.extractor.log.info( "This extractor only spawns other extractors " - "and does not provide any metadata on its own.") + "and does not provide any metadata on its own." + ) if extr: - self.extractor.log.info( - "Showing results for '%s' instead:\n", url) + self.extractor.log.info("Showing results for '%s' instead:\n", url) KeywordJob(extr, self).run() else: - self.extractor.log.info( - "Try 'gallery-dl -K \"%s\"' instead.", url) + self.extractor.log.info("Try 'gallery-dl -K \"%s\"' instead.", url) else: - stdout_write("Keywords for --chapter-filter:\n" - "------------------------------\n") + stdout_write("Keywords for --chapter-filter:\n" "------------------------------\n") self.print_kwdict(kwdict) if extr or self.extractor.categorytransfer: stdout_write("\n") @@ -765,7 +764,7 @@ def print_kwdict(self, kwdict, prefix="", markers=None): if markers is None: markers = {markerid} elif markerid in markers: - write("{}\n \n".format(prefix[:-2])) + write(f"{prefix[:-2]}\n \n") return # ignore circular reference else: markers.add(markerid) @@ -784,20 +783,21 @@ def print_kwdict(self, kwdict, prefix="", markers=None): elif isinstance(value[0], dict): self.print_kwdict(value[0], key + "[N]['", markers) else: - fmt = (" {:>%s} {}\n" % len(str(len(value)))).format + fmt = (" {:>%s} {}\n" % len(str(len(value)))).format # noqa: UP031 write(key + "[N]\n") for idx, val in enumerate(value, 0): write(fmt(idx, val)) else: # string or number - write("{}\n {}\n".format(key, value)) + write(f"{key}\n {value}\n") markers.remove(markerid) class UrlJob(Job): """Print download urls""" + maxdepth = 1 def __init__(self, url, parent=None, depth=1): @@ -819,10 +819,7 @@ def handle_url_fallback(url, kwdict): def handle_queue(self, url, kwdict): cls = kwdict.get("_extractor") - if cls: - extr = cls.from_url(url) - else: - extr = extractor.find(url) + extr = cls.from_url(url) if cls else extractor.find(url) if extr: self.status |= self.__class__(extr, self, self.depth + 1).run() @@ -839,8 +836,12 @@ def run(self): pc = self._print_config if ex.basecategory: - pm("Category / Subcategory / Basecategory", - ex.category, ex.subcategory, ex.basecategory) + pm( + "Category / Subcategory / Basecategory", + ex.category, + ex.subcategory, + ex.basecategory, + ) else: pm("Category / Subcategory", ex.category, ex.subcategory) @@ -852,28 +853,26 @@ def run(self): return 0 def _print_multi(self, title, *values): - stdout_write("{}\n {}\n\n".format( - title, " / ".join(map(util.json_dumps, values)))) + stdout_write("{}\n {}\n\n".format(title, " / ".join(map(util.json_dumps, values)))) def _print_config(self, title, optname, value): optval = self.extractor.config(optname, util.SENTINEL) + if optval is not util.SENTINEL: stdout_write( - "{} (custom):\n {}\n{} (default):\n {}\n\n".format( - title, util.json_dumps(optval), - title, util.json_dumps(value))) + f"{title} (custom):\n {util.json_dumps(optval)}\n{title} (default):\n " + f"{util.json_dumps(value)}\n\n" + ) elif value: - stdout_write( - "{} (default):\n {}\n\n".format( - title, util.json_dumps(value))) + stdout_write(f"{title} (default):\n {util.json_dumps(value)}\n\n") class DataJob(Job): """Collect extractor results and dump them""" + resolve = False - def __init__(self, url, parent=None, file=sys.stdout, ensure_ascii=True, - resolve=False): + def __init__(self, url, parent=None, file=sys.stdout, ensure_ascii=True, resolve=False): Job.__init__(self, url, parent) self.file = file self.data = [] @@ -890,8 +889,7 @@ def run(self): self._init() extractor = self.extractor - sleep = util.build_duration_func( - extractor.config("sleep-extractor")) + sleep = util.build_duration_func(extractor.config("sleep-extractor")) if sleep: extractor.sleep(sleep(), "extractor") @@ -932,14 +930,11 @@ def handle_queue(self, url, kwdict): def handle_queue_resolve(self, url, kwdict): cls = kwdict.get("_extractor") - if cls: - extr = cls.from_url(url) - else: - extr = extractor.find(url) + extr = cls.from_url(url) if cls else extractor.find(url) if not extr: return self.data.append((Message.Queue, url, self.filter(kwdict))) - job = self.__class__(extr, self, None, self.ascii, self.resolve-1) + job = self.__class__(extr, self, None, self.ascii, self.resolve - 1) job.data = self.data job.run() diff --git a/gallery_dl/oauth.py b/gallery_dl/oauth.py index 8508ee1e0d..8aab27707f 100644 --- a/gallery_dl/oauth.py +++ b/gallery_dl/oauth.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2020 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,12 @@ """OAuth helper functions and classes""" +import binascii +import hashlib import hmac -import time import random import string -import hashlib -import binascii +import time import urllib.parse import requests @@ -41,13 +39,13 @@ def concat(*args): class OAuth1Session(requests.Session): """Extension to requests.Session to support OAuth 1.0""" - def __init__(self, consumer_key, consumer_secret, - token=None, token_secret=None): - + def __init__(self, consumer_key, consumer_secret, token=None, token_secret=None): requests.Session.__init__(self) self.auth = OAuth1Client( - consumer_key, consumer_secret, - token, token_secret, + consumer_key, + consumer_secret, + token, + token_secret, ) def rebuild_auth(self, prepared_request, response): @@ -59,9 +57,7 @@ def rebuild_auth(self, prepared_request, response): class OAuth1Client(requests.auth.AuthBase): """OAuth1.0a authentication""" - def __init__(self, consumer_key, consumer_secret, - token=None, token_secret=None): - + def __init__(self, consumer_key, consumer_secret, token=None, token_secret=None): self.consumer_key = consumer_key self.consumer_secret = consumer_secret self.token = token @@ -82,7 +78,8 @@ def __call__(self, request): oauth_params.append(("oauth_signature", signature)) request.headers["Authorization"] = "OAuth " + ",".join( - key + '="' + value + '"' for key, value in oauth_params) + key + '="' + value + '"' for key, value in oauth_params + ) return request @@ -103,8 +100,9 @@ def generate_signature(self, request, params): return quote(binascii.b2a_base64(signature)[:-1].decode()) -class OAuth1API(): +class OAuth1API: """Base class for OAuth1.0 based API interfaces""" + API_KEY = None API_SECRET = None @@ -124,8 +122,7 @@ def __init__(self, extractor): if api_key and api_secret and token and token_secret: self.log.debug("Using %s OAuth1.0 authentication", key_type) - self.session = OAuth1Session( - api_key, api_secret, token, token_secret) + self.session = OAuth1Session(api_key, api_secret, token, token_secret) self.api_key = None else: self.log.debug("Using %s api_key authentication", key_type) @@ -138,6 +135,6 @@ def request(self, url, **kwargs): return self.extractor.request(url, **kwargs) -@cache(maxage=36500*86400, keyarg=0) +@cache(maxage=36500 * 86400, keyarg=0) def _token_cache(key): return None, None diff --git a/gallery_dl/option.py b/gallery_dl/option.py index a3f78e5b83..7396ed61f6 100644 --- a/gallery_dl/option.py +++ b/gallery_dl/option.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -12,17 +10,23 @@ import logging import os.path import sys -from . import job, util, version +from contextlib import suppress + +from . import job +from . import util +from . import version class ConfigAction(argparse.Action): """Set argparse results as config values""" + def __call__(self, parser, namespace, values, option_string=None): namespace.options.append(((), self.dest, values)) class ConfigConstAction(argparse.Action): """Set argparse const values as config values""" + def __call__(self, parser, namespace, values, option_string=None): namespace.options.append(((), self.dest, self.const)) @@ -38,15 +42,19 @@ def __call__(self, parser, namespace, values, option_string=None): class DeprecatedConfigConstAction(argparse.Action): """Set argparse const values as config values + deprecation warning""" + def __call__(self, parser, namespace, values, option_string=None): sys.stderr.write( "warning: {} is deprecated. Use {} instead.\n".format( - "/".join(self.option_strings), self.choices)) + "/".join(self.option_strings), self.choices + ) + ) namespace.options.append(((), self.dest, self.const)) class ConfigParseAction(argparse.Action): """Parse KEY=VALUE config options""" + def __call__(self, parser, namespace, values, option_string=None): key, value = _parse_option(values) key = key.split(".") # splitting an empty string becomes [""] @@ -55,6 +63,7 @@ def __call__(self, parser, namespace, values, option_string=None): class PPParseAction(argparse.Action): """Parse KEY=VALUE post processor options""" + def __call__(self, parser, namespace, values, option_string=None): key, value = _parse_option(values) namespace.options_pp[key] = value @@ -62,89 +71,100 @@ def __call__(self, parser, namespace, values, option_string=None): class InputfileAction(argparse.Action): """Collect input files""" + def __call__(self, parser, namespace, value, option_string=None): namespace.input_files.append((value, self.const)) class MtimeAction(argparse.Action): """Configure mtime post processors""" + def __call__(self, parser, namespace, value, option_string=None): - namespace.postprocessors.append({ - "name": "mtime", - "value": "{" + (self.const or value) + "}", - }) + namespace.postprocessors.append( + { + "name": "mtime", + "value": "{" + (self.const or value) + "}", + } + ) class RenameAction(argparse.Action): """Configure rename post processors""" + def __call__(self, parser, namespace, value, option_string=None): if self.const: - namespace.postprocessors.append({ - "name": "rename", - "to" : value, - }) + namespace.postprocessors.append( + { + "name": "rename", + "to": value, + } + ) else: - namespace.postprocessors.append({ - "name": "rename", - "from": value, - }) + namespace.postprocessors.append( + { + "name": "rename", + "from": value, + } + ) class UgoiraAction(argparse.Action): """Configure ugoira post processors""" + def __call__(self, parser, namespace, value, option_string=None): - if self.const: - value = self.const - else: - value = value.strip().lower() + value = self.const if self.const else value.strip().lower() if value in ("webm", "vp9"): pp = { - "extension" : "webm", - "ffmpeg-args" : ("-c:v", "libvpx-vp9", - "-crf", "12", - "-b:v", "0", "-an"), + "extension": "webm", + "ffmpeg-args": ("-c:v", "libvpx-vp9", "-crf", "12", "-b:v", "0", "-an"), } elif value == "vp9-lossless": pp = { - "extension" : "webm", - "ffmpeg-args" : ("-c:v", "libvpx-vp9", - "-lossless", "1", - "-pix_fmt", "yuv420p", "-an"), + "extension": "webm", + "ffmpeg-args": ( + "-c:v", + "libvpx-vp9", + "-lossless", + "1", + "-pix_fmt", + "yuv420p", + "-an", + ), } elif value == "vp8": pp = { - "extension" : "webm", - "ffmpeg-args" : ("-c:v", "libvpx", - "-crf", "4", - "-b:v", "5000k", "-an"), + "extension": "webm", + "ffmpeg-args": ("-c:v", "libvpx", "-crf", "4", "-b:v", "5000k", "-an"), } elif value == "mp4": pp = { - "extension" : "mp4", - "ffmpeg-args" : ("-c:v", "libx264", "-an", "-b:v", "5M"), + "extension": "mp4", + "ffmpeg-args": ("-c:v", "libx264", "-an", "-b:v", "5M"), "libx264-prevent-odd": True, } elif value == "gif": pp = { - "extension" : "gif", - "ffmpeg-args" : ("-filter_complex", "[0:v] split [a][b];" - "[a] palettegen [p];[b][p] paletteuse"), + "extension": "gif", + "ffmpeg-args": ( + "-filter_complex", + "[0:v] split [a][b];" "[a] palettegen [p];[b][p] paletteuse", + ), "repeat-last-frame": False, } elif value == "mkv" or value == "copy": pp = { - "extension" : "mkv", - "ffmpeg-args" : ("-c:v", "copy"), + "extension": "mkv", + "ffmpeg-args": ("-c:v", "copy"), "repeat-last-frame": False, } elif value == "zip" or value == "archive": pp = { - "mode" : "archive", + "mode": "archive", } namespace.options.append(((), "ugoira", "original")) else: - parser.error("Unsupported Ugoira format '{}'".format(value)) + parser.error(f"Unsupported Ugoira format '{value}'") pp["name"] = "ugoira" pp["whitelist"] = ("pixiv", "danbooru") @@ -170,34 +190,46 @@ def __call__(self, parser, namespace, value, option_string=None): event = ("prepare",) else: event = event.strip().lower() - if event not in {"init", "file", "after", "skip", "error", - "prepare", "prepare-after", "post", "post-after", - "finalize", "finalize-success", "finalize-error"}: + if event not in { + "init", + "file", + "after", + "skip", + "error", + "prepare", + "prepare-after", + "post", + "post-after", + "finalize", + "finalize-success", + "finalize-error", + }: format_string = value event = ("prepare",) if not format_string: return - if "{" not in format_string and \ - " " not in format_string and \ - format_string[0] != "\f": + if "{" not in format_string and " " not in format_string and format_string[0] != "\f": format_string = "{" + format_string + "}" if format_string[-1] != "\n": format_string += "\n" - namespace.postprocessors.append({ - "name" : "metadata", - "event" : event, - "filename" : filename, - "base-directory": base or ".", - "content-format": format_string, - "open" : mode, - }) + namespace.postprocessors.append( + { + "name": "metadata", + "event": event, + "filename": filename, + "base-directory": base or ".", + "content-format": format_string, + "open": mode, + } + ) class Formatter(argparse.HelpFormatter): """Custom HelpFormatter class to customize help output""" + def __init__(self, prog): argparse.HelpFormatter.__init__(self, prog, max_help_position=30) @@ -211,10 +243,9 @@ def _format_action_invocation(self, action, join=", ".join): def _parse_option(opt): key, _, value = opt.partition("=") - try: + with suppress(ValueError): value = util.json_loads(value) - except ValueError: - pass + return key, value @@ -228,585 +259,829 @@ def build_parser(): general = parser.add_argument_group("General Options") general.add_argument( - "-h", "--help", + "-h", + "--help", action="help", help="Print this help message and exit", ) general.add_argument( "--version", - action="version", version=version.__version__, + action="version", + version=version.__version__, help="Print program version and exit", ) general.add_argument( - "-f", "--filename", - dest="filename", metavar="FORMAT", - help=("Filename format string for downloaded files " - "('/O' for \"original\" filenames)"), + "-f", + "--filename", + dest="filename", + metavar="FORMAT", + help=("Filename format string for downloaded files " "('/O' for \"original\" filenames)"), ) general.add_argument( - "-d", "--destination", - dest="base-directory", metavar="PATH", action=ConfigAction, + "-d", + "--destination", + dest="base-directory", + metavar="PATH", + action=ConfigAction, help="Target location for file downloads", ) general.add_argument( - "-D", "--directory", - dest="directory", metavar="PATH", + "-D", + "--directory", + dest="directory", + metavar="PATH", help="Exact location for file downloads", ) general.add_argument( - "-X", "--extractors", - dest="extractor_sources", metavar="PATH", action="append", + "-X", + "--extractors", + dest="extractor_sources", + metavar="PATH", + action="append", help="Load external extractors from PATH", ) general.add_argument( "--user-agent", - dest="user-agent", metavar="UA", action=ConfigAction, + dest="user-agent", + metavar="UA", + action=ConfigAction, help="User-Agent request header", ) general.add_argument( "--clear-cache", - dest="clear_cache", metavar="MODULE", - help="Delete cached login sessions, cookies, etc. for MODULE " - "(ALL to delete everything)", + dest="clear_cache", + metavar="MODULE", + help="Delete cached login sessions, cookies, etc. for MODULE " "(ALL to delete everything)", ) update = parser.add_argument_group("Update Options") if util.EXECUTABLE: update.add_argument( - "-U", "--update", - dest="update", action="store_const", const="latest", + "-U", + "--update", + dest="update", + action="store_const", + const="latest", help="Update to the latest version", ) update.add_argument( "--update-to", - dest="update", metavar="CHANNEL[@TAG]", - help=("Switch to a dfferent release channel (stable or dev) " - "or upgrade/downgrade to a specific version"), + dest="update", + metavar="CHANNEL[@TAG]", + help=( + "Switch to a dfferent release channel (stable or dev) " + "or upgrade/downgrade to a specific version" + ), ) update.add_argument( "--update-check", - dest="update", action="store_const", const="check", + dest="update", + action="store_const", + const="check", help="Check if a newer version is available", ) else: update.add_argument( - "-U", "--update-check", - dest="update", action="store_const", const="check", + "-U", + "--update-check", + dest="update", + action="store_const", + const="check", help="Check if a newer version is available", ) input = parser.add_argument_group("Input Options") input.add_argument( "urls", - metavar="URL", nargs="*", + metavar="URL", + nargs="*", help=argparse.SUPPRESS, ) input.add_argument( - "-i", "--input-file", - dest="input_files", metavar="FILE", action=InputfileAction, const=None, + "-i", + "--input-file", + dest="input_files", + metavar="FILE", + action=InputfileAction, + const=None, default=[], - help=("Download URLs found in FILE ('-' for stdin). " - "More than one --input-file can be specified"), + help=( + "Download URLs found in FILE ('-' for stdin). " + "More than one --input-file can be specified" + ), ) input.add_argument( - "-I", "--input-file-comment", - dest="input_files", metavar="FILE", action=InputfileAction, const="c", - help=("Download URLs found in FILE. " - "Comment them out after they were downloaded successfully."), + "-I", + "--input-file-comment", + dest="input_files", + metavar="FILE", + action=InputfileAction, + const="c", + help=( + "Download URLs found in FILE. " + "Comment them out after they were downloaded successfully." + ), ) input.add_argument( - "-x", "--input-file-delete", - dest="input_files", metavar="FILE", action=InputfileAction, const="d", - help=("Download URLs found in FILE. " - "Delete them after they were downloaded successfully."), + "-x", + "--input-file-delete", + dest="input_files", + metavar="FILE", + action=InputfileAction, + const="d", + help=( + "Download URLs found in FILE. " "Delete them after they were downloaded successfully." + ), ) input.add_argument( "--no-input", - dest="input", nargs=0, action=ConfigConstAction, const=False, + dest="input", + nargs=0, + action=ConfigConstAction, + const=False, help=("Do not prompt for passwords/tokens"), ) output = parser.add_argument_group("Output Options") output.add_argument( - "-q", "--quiet", - dest="loglevel", default=logging.INFO, - action="store_const", const=logging.ERROR, + "-q", + "--quiet", + dest="loglevel", + default=logging.INFO, + action="store_const", + const=logging.ERROR, help="Activate quiet mode", ) output.add_argument( - "-w", "--warning", + "-w", + "--warning", dest="loglevel", - action="store_const", const=logging.WARNING, + action="store_const", + const=logging.WARNING, help="Print only warnings and errors", ) output.add_argument( - "-v", "--verbose", + "-v", + "--verbose", dest="loglevel", - action="store_const", const=logging.DEBUG, + action="store_const", + const=logging.DEBUG, help="Print various debugging information", ) output.add_argument( - "-g", "--get-urls", - dest="list_urls", action="count", + "-g", + "--get-urls", + dest="list_urls", + action="count", help="Print URLs instead of downloading", ) output.add_argument( - "-G", "--resolve-urls", - dest="list_urls", action="store_const", const=128, + "-G", + "--resolve-urls", + dest="list_urls", + action="store_const", + const=128, help="Print URLs instead of downloading; resolve intermediary URLs", ) output.add_argument( - "-j", "--dump-json", - dest="dump_json", action="count", + "-j", + "--dump-json", + dest="dump_json", + action="count", help="Print JSON information", ) output.add_argument( - "-J", "--resolve-json", - dest="dump_json", action="store_const", const=128, + "-J", + "--resolve-json", + dest="dump_json", + action="store_const", + const=128, help="Print JSON information; resolve intermediary URLs", ) output.add_argument( - "-s", "--simulate", - dest="jobtype", action="store_const", const=job.SimulationJob, + "-s", + "--simulate", + dest="jobtype", + action="store_const", + const=job.SimulationJob, help="Simulate data extraction; do not download anything", ) output.add_argument( - "-E", "--extractor-info", - dest="jobtype", action="store_const", const=job.InfoJob, + "-E", + "--extractor-info", + dest="jobtype", + action="store_const", + const=job.InfoJob, help="Print extractor defaults and settings", ) output.add_argument( - "-K", "--list-keywords", - dest="jobtype", action="store_const", const=job.KeywordJob, - help=("Print a list of available keywords and example values " - "for the given URLs"), + "-K", + "--list-keywords", + dest="jobtype", + action="store_const", + const=job.KeywordJob, + help=("Print a list of available keywords and example values " "for the given URLs"), ) output.add_argument( - "-e", "--error-file", - dest="errorfile", metavar="FILE", action=ConfigAction, + "-e", + "--error-file", + dest="errorfile", + metavar="FILE", + action=ConfigAction, help="Add input URLs which returned an error to FILE", ) output.add_argument( - "-N", "--print", - dest="postprocessors", metavar="[EVENT:]FORMAT", - action=PrintAction, const="-", default=[], - help=("Write FORMAT during EVENT (default 'prepare') to standard " - "output. Examples: 'id' or 'post:{md5[:8]}'"), + "-N", + "--print", + dest="postprocessors", + metavar="[EVENT:]FORMAT", + action=PrintAction, + const="-", + default=[], + help=( + "Write FORMAT during EVENT (default 'prepare') to standard " + "output. Examples: 'id' or 'post:{md5[:8]}'" + ), ) output.add_argument( "--print-to-file", - dest="postprocessors", metavar="[EVENT:]FORMAT FILE", - action=PrintAction, nargs=2, + dest="postprocessors", + metavar="[EVENT:]FORMAT FILE", + action=PrintAction, + nargs=2, help="Append FORMAT during EVENT to FILE", ) output.add_argument( "--list-modules", - dest="list_modules", action="store_true", + dest="list_modules", + action="store_true", help="Print a list of available extractor modules", ) output.add_argument( "--list-extractors", - dest="list_extractors", metavar="CATEGORIES", nargs="*", - help=("Print a list of extractor classes " - "with description, (sub)category and example URL"), + dest="list_extractors", + metavar="CATEGORIES", + nargs="*", + help=( + "Print a list of extractor classes " "with description, (sub)category and example URL" + ), ) output.add_argument( "--write-log", - dest="logfile", metavar="FILE", action=ConfigAction, + dest="logfile", + metavar="FILE", + action=ConfigAction, help="Write logging output to FILE", ) output.add_argument( "--write-unsupported", - dest="unsupportedfile", metavar="FILE", action=ConfigAction, - help=("Write URLs, which get emitted by other extractors but cannot " - "be handled, to FILE"), + dest="unsupportedfile", + metavar="FILE", + action=ConfigAction, + help=( + "Write URLs, which get emitted by other extractors but cannot " "be handled, to FILE" + ), ) output.add_argument( "--write-pages", - dest="write-pages", nargs=0, action=ConfigConstAction, const=True, - help=("Write downloaded intermediary pages to files " - "in the current directory to debug problems"), + dest="write-pages", + nargs=0, + action=ConfigConstAction, + const=True, + help=( + "Write downloaded intermediary pages to files " + "in the current directory to debug problems" + ), ) output.add_argument( "--print-traffic", - dest="print_traffic", action="store_true", + dest="print_traffic", + action="store_true", help=("Display sent and read HTTP traffic"), ) output.add_argument( "--no-colors", - dest="colors", action="store_false", + dest="colors", + action="store_false", help=("Do not emit ANSI color codes in output"), ) networking = parser.add_argument_group("Networking Options") networking.add_argument( - "-R", "--retries", - dest="retries", metavar="N", type=int, action=ConfigAction, - help=("Maximum number of retries for failed HTTP requests " - "or -1 for infinite retries (default: 4)"), + "-R", + "--retries", + dest="retries", + metavar="N", + type=int, + action=ConfigAction, + help=( + "Maximum number of retries for failed HTTP requests " + "or -1 for infinite retries (default: 4)" + ), ) networking.add_argument( "--http-timeout", - dest="timeout", metavar="SECONDS", type=float, action=ConfigAction, + dest="timeout", + metavar="SECONDS", + type=float, + action=ConfigAction, help="Timeout for HTTP connections (default: 30.0)", ) networking.add_argument( "--proxy", - dest="proxy", metavar="URL", action=ConfigAction, + dest="proxy", + metavar="URL", + action=ConfigAction, help="Use the specified proxy", ) networking.add_argument( "--source-address", - dest="source-address", metavar="IP", action=ConfigAction, + dest="source-address", + metavar="IP", + action=ConfigAction, help="Client-side IP address to bind to", ) networking.add_argument( - "-4", "--force-ipv4", - dest="source-address", nargs=0, action=ConfigConstAction, + "-4", + "--force-ipv4", + dest="source-address", + nargs=0, + action=ConfigConstAction, const="0.0.0.0", help="Make all connections via IPv4", ) networking.add_argument( - "-6", "--force-ipv6", - dest="source-address", nargs=0, action=ConfigConstAction, const="::", + "-6", + "--force-ipv6", + dest="source-address", + nargs=0, + action=ConfigConstAction, + const="::", help="Make all connections via IPv6", ) networking.add_argument( "--no-check-certificate", - dest="verify", nargs=0, action=ConfigConstAction, const=False, + dest="verify", + nargs=0, + action=ConfigConstAction, + const=False, help="Disable HTTPS certificate validation", ) downloader = parser.add_argument_group("Downloader Options") downloader.add_argument( - "-r", "--limit-rate", - dest="rate", metavar="RATE", action=ConfigAction, + "-r", + "--limit-rate", + dest="rate", + metavar="RATE", + action=ConfigAction, help="Maximum download rate (e.g. 500k or 2.5M)", ) downloader.add_argument( "--chunk-size", - dest="chunk-size", metavar="SIZE", action=ConfigAction, + dest="chunk-size", + metavar="SIZE", + action=ConfigAction, help="Size of in-memory data chunks (default: 32k)", ) downloader.add_argument( "--sleep", - dest="sleep", metavar="SECONDS", action=ConfigAction, - help=("Number of seconds to wait before each download. " - "This can be either a constant value or a range " - "(e.g. 2.7 or 2.0-3.5)"), + dest="sleep", + metavar="SECONDS", + action=ConfigAction, + help=( + "Number of seconds to wait before each download. " + "This can be either a constant value or a range " + "(e.g. 2.7 or 2.0-3.5)" + ), ) downloader.add_argument( "--sleep-request", - dest="sleep-request", metavar="SECONDS", action=ConfigAction, - help=("Number of seconds to wait between HTTP requests " - "during data extraction"), + dest="sleep-request", + metavar="SECONDS", + action=ConfigAction, + help=("Number of seconds to wait between HTTP requests " "during data extraction"), ) downloader.add_argument( "--sleep-extractor", - dest="sleep-extractor", metavar="SECONDS", action=ConfigAction, - help=("Number of seconds to wait before starting data extraction " - "for an input URL"), + dest="sleep-extractor", + metavar="SECONDS", + action=ConfigAction, + help=("Number of seconds to wait before starting data extraction " "for an input URL"), ) downloader.add_argument( "--no-part", - dest="part", nargs=0, action=ConfigConstAction, const=False, + dest="part", + nargs=0, + action=ConfigConstAction, + const=False, help="Do not use .part files", ) downloader.add_argument( "--no-skip", - dest="skip", nargs=0, action=ConfigConstAction, const=False, + dest="skip", + nargs=0, + action=ConfigConstAction, + const=False, help="Do not skip downloads; overwrite existing files", ) downloader.add_argument( "--no-mtime", - dest="mtime", nargs=0, action=ConfigConstAction, const=False, - help=("Do not set file modification times according to " - "Last-Modified HTTP response headers") + dest="mtime", + nargs=0, + action=ConfigConstAction, + const=False, + help=( + "Do not set file modification times according to " "Last-Modified HTTP response headers" + ), ) downloader.add_argument( "--no-download", - dest="download", nargs=0, action=ConfigConstAction, const=False, - help=("Do not download any files") + dest="download", + nargs=0, + action=ConfigConstAction, + const=False, + help=("Do not download any files"), ) configuration = parser.add_argument_group("Configuration Options") configuration.add_argument( - "-o", "--option", - dest="options", metavar="KEY=VALUE", - action=ConfigParseAction, default=[], - help=("Additional options. " - "Example: -o browser=firefox") , + "-o", + "--option", + dest="options", + metavar="KEY=VALUE", + action=ConfigParseAction, + default=[], + help=("Additional options. " "Example: -o browser=firefox"), ) configuration.add_argument( - "-c", "--config", - dest="configs_json", metavar="FILE", action="append", + "-c", + "--config", + dest="configs_json", + metavar="FILE", + action="append", help="Additional configuration files", ) configuration.add_argument( "--config-yaml", - dest="configs_yaml", metavar="FILE", action="append", + dest="configs_yaml", + metavar="FILE", + action="append", help="Additional configuration files in YAML format", ) configuration.add_argument( "--config-toml", - dest="configs_toml", metavar="FILE", action="append", + dest="configs_toml", + metavar="FILE", + action="append", help="Additional configuration files in TOML format", ) configuration.add_argument( "--config-create", - dest="config", action="store_const", const="init", + dest="config", + action="store_const", + const="init", help="Create a basic configuration file", ) configuration.add_argument( "--config-status", - dest="config", action="store_const", const="status", + dest="config", + action="store_const", + const="status", help="Show configuration file status", ) configuration.add_argument( "--config-open", - dest="config", action="store_const", const="open", + dest="config", + action="store_const", + const="open", help="Open configuration file in external application", ) configuration.add_argument( "--config-ignore", - dest="config_load", action="store_false", + dest="config_load", + action="store_false", help="Do not read default configuration files", ) configuration.add_argument( "--ignore-config", - dest="config_load", action="store_false", + dest="config_load", + action="store_false", help=argparse.SUPPRESS, ) authentication = parser.add_argument_group("Authentication Options") authentication.add_argument( - "-u", "--username", - dest="username", metavar="USER", action=ConfigAction, + "-u", + "--username", + dest="username", + metavar="USER", + action=ConfigAction, help="Username to login with", ) authentication.add_argument( - "-p", "--password", - dest="password", metavar="PASS", action=ConfigAction, + "-p", + "--password", + dest="password", + metavar="PASS", + action=ConfigAction, help="Password belonging to the given username", ) authentication.add_argument( "--netrc", - dest="netrc", nargs=0, action=ConfigConstAction, const=True, + dest="netrc", + nargs=0, + action=ConfigConstAction, + const=True, help="Enable .netrc authentication data", ) cookies = parser.add_argument_group("Cookie Options") cookies.add_argument( - "-C", "--cookies", - dest="cookies", metavar="FILE", action=ConfigAction, + "-C", + "--cookies", + dest="cookies", + metavar="FILE", + action=ConfigAction, help="File to load additional cookies from", ) cookies.add_argument( "--cookies-export", - dest="cookies-update", metavar="FILE", action=ConfigAction, + dest="cookies-update", + metavar="FILE", + action=ConfigAction, help="Export session cookies to FILE", ) cookies.add_argument( "--cookies-from-browser", dest="cookies_from_browser", metavar="BROWSER[/DOMAIN][+KEYRING][:PROFILE][::CONTAINER]", - help=("Name of the browser to load cookies from, with optional " - "domain prefixed with '/', " - "keyring name prefixed with '+', " - "profile prefixed with ':', and " - "container prefixed with '::' " - "('none' for no container (default), 'all' for all containers)"), + help=( + "Name of the browser to load cookies from, with optional " + "domain prefixed with '/', " + "keyring name prefixed with '+', " + "profile prefixed with ':', and " + "container prefixed with '::' " + "('none' for no container (default), 'all' for all containers)" + ), ) selection = parser.add_argument_group("Selection Options") selection.add_argument( - "-A", "--abort", - dest="abort", metavar="N", type=int, - help=("Stop current extractor run " - "after N consecutive file downloads were skipped"), + "-A", + "--abort", + dest="abort", + metavar="N", + type=int, + help=("Stop current extractor run " "after N consecutive file downloads were skipped"), ) selection.add_argument( - "-T", "--terminate", - dest="terminate", metavar="N", type=int, - help=("Stop current and parent extractor runs " - "after N consecutive file downloads were skipped"), + "-T", + "--terminate", + dest="terminate", + metavar="N", + type=int, + help=( + "Stop current and parent extractor runs " + "after N consecutive file downloads were skipped" + ), ) selection.add_argument( "--filesize-min", - dest="filesize-min", metavar="SIZE", action=ConfigAction, + dest="filesize-min", + metavar="SIZE", + action=ConfigAction, help="Do not download files smaller than SIZE (e.g. 500k or 2.5M)", ) selection.add_argument( "--filesize-max", - dest="filesize-max", metavar="SIZE", action=ConfigAction, + dest="filesize-max", + metavar="SIZE", + action=ConfigAction, help="Do not download files larger than SIZE (e.g. 500k or 2.5M)", ) selection.add_argument( "--download-archive", - dest="archive", metavar="FILE", action=ConfigAction, - help=("Record all downloaded or skipped files in FILE and " - "skip downloading any file already in it"), + dest="archive", + metavar="FILE", + action=ConfigAction, + help=( + "Record all downloaded or skipped files in FILE and " + "skip downloading any file already in it" + ), ) selection.add_argument( "--range", - dest="image-range", metavar="RANGE", action=ConfigAction, - help=("Index range(s) specifying which files to download. " - "These can be either a constant value, range, or slice " - "(e.g. '5', '8-20', or '1:24:3')"), + dest="image-range", + metavar="RANGE", + action=ConfigAction, + help=( + "Index range(s) specifying which files to download. " + "These can be either a constant value, range, or slice " + "(e.g. '5', '8-20', or '1:24:3')" + ), ) selection.add_argument( "--chapter-range", - dest="chapter-range", metavar="RANGE", action=ConfigAction, - help=("Like '--range', but applies to manga chapters " - "and other delegated URLs"), + dest="chapter-range", + metavar="RANGE", + action=ConfigAction, + help=("Like '--range', but applies to manga chapters " "and other delegated URLs"), ) selection.add_argument( "--filter", - dest="image-filter", metavar="EXPR", action=ConfigAction, - help=("Python expression controlling which files to download. " - "Files for which the expression evaluates to False are ignored. " - "Available keys are the filename-specific ones listed by '-K'. " - "Example: --filter \"image_width >= 1000 and " - "rating in ('s', 'q')\""), + dest="image-filter", + metavar="EXPR", + action=ConfigAction, + help=( + "Python expression controlling which files to download. " + "Files for which the expression evaluates to False are ignored. " + "Available keys are the filename-specific ones listed by '-K'. " + 'Example: --filter "image_width >= 1000 and ' + "rating in ('s', 'q')\"" + ), ) selection.add_argument( "--chapter-filter", - dest="chapter-filter", metavar="EXPR", action=ConfigAction, - help=("Like '--filter', but applies to manga chapters " - "and other delegated URLs"), + dest="chapter-filter", + metavar="EXPR", + action=ConfigAction, + help=("Like '--filter', but applies to manga chapters " "and other delegated URLs"), ) infojson = { - "name" : "metadata", - "event" : "init", + "name": "metadata", + "event": "init", "filename": "info.json", } postprocessor = parser.add_argument_group("Post-processing Options") postprocessor.add_argument( - "-P", "--postprocessor", - dest="postprocessors", metavar="NAME", action="append", + "-P", + "--postprocessor", + dest="postprocessors", + metavar="NAME", + action="append", help="Activate the specified post processor", ) postprocessor.add_argument( "--no-postprocessors", - dest="postprocess", nargs=0, action=ConfigConstAction, const=False, - help=("Do not run any post processors") + dest="postprocess", + nargs=0, + action=ConfigConstAction, + const=False, + help=("Do not run any post processors"), ) postprocessor.add_argument( - "-O", "--postprocessor-option", - dest="options_pp", metavar="KEY=VALUE", - action=PPParseAction, default={}, + "-O", + "--postprocessor-option", + dest="options_pp", + metavar="KEY=VALUE", + action=PPParseAction, + default={}, help="Additional post processor options", ) postprocessor.add_argument( "--write-metadata", dest="postprocessors", - action="append_const", const="metadata", + action="append_const", + const="metadata", help="Write metadata to separate JSON files", ) postprocessor.add_argument( "--write-info-json", dest="postprocessors", - action="append_const", const=infojson, + action="append_const", + const=infojson, help="Write gallery metadata to a info.json file", ) postprocessor.add_argument( "--write-infojson", dest="postprocessors", - action="append_const", const=infojson, + action="append_const", + const=infojson, help=argparse.SUPPRESS, ) postprocessor.add_argument( "--write-tags", dest="postprocessors", - action="append_const", const={"name": "metadata", "mode": "tags"}, + action="append_const", + const={"name": "metadata", "mode": "tags"}, help="Write image tags to separate text files", ) postprocessor.add_argument( "--zip", dest="postprocessors", - action="append_const", const="zip", + action="append_const", + const="zip", help="Store downloaded files in a ZIP archive", ) postprocessor.add_argument( "--cbz", dest="postprocessors", - action="append_const", const={ - "name" : "zip", + action="append_const", + const={ + "name": "zip", "extension": "cbz", }, help="Store downloaded files in a CBZ archive", ) postprocessor.add_argument( "--mtime", - dest="postprocessors", metavar="NAME", action=MtimeAction, - help=("Set file modification times according to metadata " - "selected by NAME. Examples: 'date' or 'status[date]'"), + dest="postprocessors", + metavar="NAME", + action=MtimeAction, + help=( + "Set file modification times according to metadata " + "selected by NAME. Examples: 'date' or 'status[date]'" + ), ) postprocessor.add_argument( "--mtime-from-date", - dest="postprocessors", nargs=0, action=MtimeAction, + dest="postprocessors", + nargs=0, + action=MtimeAction, const="date|status[date]", help=argparse.SUPPRESS, ) postprocessor.add_argument( "--rename", - dest="postprocessors", metavar="FORMAT", action=RenameAction, const=0, - help=("Rename previously downloaded files from FORMAT " - "to the current filename format"), + dest="postprocessors", + metavar="FORMAT", + action=RenameAction, + const=0, + help=("Rename previously downloaded files from FORMAT " "to the current filename format"), ) postprocessor.add_argument( "--rename-to", - dest="postprocessors", metavar="FORMAT", action=RenameAction, const=1, - help=("Rename previously downloaded files from the current filename " - "format to FORMAT"), + dest="postprocessors", + metavar="FORMAT", + action=RenameAction, + const=1, + help=("Rename previously downloaded files from the current filename " "format to FORMAT"), ) postprocessor.add_argument( "--ugoira", - dest="postprocessors", metavar="FMT", action=UgoiraAction, - help=("Convert Pixiv Ugoira to FMT using FFmpeg. " - "Supported formats are 'webm', 'mp4', 'gif', " - "'vp8', 'vp9', 'vp9-lossless', 'copy', 'zip'."), + dest="postprocessors", + metavar="FMT", + action=UgoiraAction, + help=( + "Convert Pixiv Ugoira to FMT using FFmpeg. " + "Supported formats are 'webm', 'mp4', 'gif', " + "'vp8', 'vp9', 'vp9-lossless', 'copy', 'zip'." + ), ) postprocessor.add_argument( "--ugoira-conv", - dest="postprocessors", nargs=0, action=UgoiraAction, const="vp8", + dest="postprocessors", + nargs=0, + action=UgoiraAction, + const="vp8", help=argparse.SUPPRESS, ) postprocessor.add_argument( "--ugoira-conv-lossless", - dest="postprocessors", nargs=0, action=UgoiraAction, + dest="postprocessors", + nargs=0, + action=UgoiraAction, const="vp9-lossless", help=argparse.SUPPRESS, ) postprocessor.add_argument( "--ugoira-conv-copy", - dest="postprocessors", nargs=0, action=UgoiraAction, const="copy", + dest="postprocessors", + nargs=0, + action=UgoiraAction, + const="copy", help=argparse.SUPPRESS, ) postprocessor.add_argument( "--exec", - dest="postprocessors", metavar="CMD", - action=AppendCommandAction, const={"name": "exec"}, - help=("Execute CMD for each downloaded file. " - "Supported replacement fields are " - "{} or {_path}, {_directory}, {_filename}. " - "Example: --exec \"convert {} {}.png && rm {}\""), + dest="postprocessors", + metavar="CMD", + action=AppendCommandAction, + const={"name": "exec"}, + help=( + "Execute CMD for each downloaded file. " + "Supported replacement fields are " + "{} or {_path}, {_directory}, {_filename}. " + 'Example: --exec "convert {} {}.png && rm {}"' + ), ) postprocessor.add_argument( "--exec-after", - dest="postprocessors", metavar="CMD", - action=AppendCommandAction, const={ - "name": "exec", "event": "finalize"}, - help=("Execute CMD after all files were downloaded. " - "Example: --exec-after \"cd {_directory} " - "&& convert * ../doc.pdf\""), + dest="postprocessors", + metavar="CMD", + action=AppendCommandAction, + const={"name": "exec", "event": "finalize"}, + help=( + "Execute CMD after all files were downloaded. " + 'Example: --exec-after "cd {_directory} ' + '&& convert * ../doc.pdf"' + ), ) - try: + with suppress(Exception): # restore normal behavior when adding '-4' or '-6' as arguments parser._has_negative_number_optionals.clear() - except Exception: - pass return parser diff --git a/gallery_dl/output.py b/gallery_dl/output.py index 1649487b25..e364848338 100644 --- a/gallery_dl/output.py +++ b/gallery_dl/output.py @@ -1,18 +1,18 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import logging import os -import sys import shutil -import logging +import sys import unicodedata -from . import config, util, formatter +from . import config +from . import formatter +from . import util # -------------------------------------------------------------------- # Globals @@ -72,17 +72,29 @@ class Logger(logging.Logger): """Custom Logger that includes extra info in log records""" - def makeRecord(self, name, level, fn, lno, msg, args, exc_info, - func=None, extra=None, sinfo=None, - factory=logging._logRecordFactory): + def makeRecord( + self, + name, + level, + fn, + lno, + msg, + args, + exc_info, + func=None, + extra=None, + sinfo=None, + factory=logging._logRecordFactory, + ): rv = factory(name, level, fn, lno, msg, args, exc_info, func, sinfo) if extra: rv.__dict__.update(extra) return rv -class LoggerAdapter(): +class LoggerAdapter: """Trimmed-down version of logging.LoggingAdapter""" + __slots__ = ("logger", "extra") def __init__(self, logger, job): @@ -110,7 +122,7 @@ def error(self, msg, *args, **kwargs): self.logger._log(logging.ERROR, msg, args, **kwargs) -class PathfmtProxy(): +class PathfmtProxy: __slots__ = ("job",) def __init__(self, job): @@ -127,7 +139,7 @@ def __str__(self): return "" -class KwdictProxy(): +class KwdictProxy: __slots__ = ("job",) def __init__(self, job): @@ -144,9 +156,8 @@ class Formatter(logging.Formatter): def __init__(self, fmt, datefmt): if isinstance(fmt, dict): for key in LOG_LEVELS: - value = fmt[key] if key in fmt else LOG_FORMAT - fmt[key] = (formatter.parse(value).format_map, - "{asctime" in value) + value = fmt.get(key, LOG_FORMAT) + fmt[key] = (formatter.parse(value).format_map, "{asctime" in value) else: if fmt == LOG_FORMAT: fmt = (fmt.format_map, False) @@ -225,21 +236,18 @@ def configure_logging(loglevel): lf[level] = ansifmt(c, logfmt) if c else logfmt logfmt = lf - handler.setFormatter(Formatter( - logfmt, opts.get("format-date", LOG_FORMAT_DATE))) + handler.setFormatter(Formatter(logfmt, opts.get("format-date", LOG_FORMAT_DATE))) if "level" in opts and handler.level == LOG_LEVEL: handler.setLevel(opts["level"]) - if minlevel > handler.level: - minlevel = handler.level + minlevel = min(minlevel, handler.level) # file logging handler handler = setup_logging_handler("logfile", lvl=loglevel) if handler: root.addHandler(handler) - if minlevel > handler.level: - minlevel = handler.level + minlevel = min(minlevel, handler.level) root.setLevel(minlevel) @@ -262,25 +270,26 @@ def setup_logging_handler(key, fmt=LOG_FORMAT, lvl=LOG_LEVEL, mode="w"): os.makedirs(os.path.dirname(path)) handler = logging.FileHandler(path, mode, encoding) except (OSError, ValueError) as exc: - logging.getLogger("gallery-dl").warning( - "%s: %s", key, exc) + logging.getLogger("gallery-dl").warning("%s: %s", key, exc) return None except TypeError as exc: - logging.getLogger("gallery-dl").warning( - "%s: missing or invalid path (%s)", key, exc) + logging.getLogger("gallery-dl").warning("%s: missing or invalid path (%s)", key, exc) return None handler.setLevel(opts.get("level", lvl)) - handler.setFormatter(Formatter( - opts.get("format", fmt), - opts.get("format-date", LOG_FORMAT_DATE), - )) + handler.setFormatter( + Formatter( + opts.get("format", fmt), + opts.get("format-date", LOG_FORMAT_DATE), + ) + ) return handler # -------------------------------------------------------------------- # Utility functions + def stdout_write_flush(s): sys.stdout.write(s) sys.stdout.flush() @@ -292,6 +301,7 @@ def stderr_write_flush(s): if getattr(sys.stdout, "line_buffering", None): + def stdout_write(s): sys.stdout.write(s) else: @@ -299,6 +309,7 @@ def stdout_write(s): if getattr(sys.stderr, "line_buffering", None): + def stderr_write(s): sys.stderr.write(s) else: @@ -324,28 +335,30 @@ def configure_standard_streams(): except AttributeError: # no 'reconfigure' support oget = options.get - setattr(sys, name, stream.__class__( - stream.buffer, - encoding=oget("encoding", stream.encoding), - errors=oget("errors", "replace"), - newline=oget("newline", stream.newlines), - line_buffering=oget("line_buffering", stream.line_buffering), - )) + setattr( + sys, + name, + stream.__class__( + stream.buffer, + encoding=oget("encoding", stream.encoding), + errors=oget("errors", "replace"), + newline=oget("newline", stream.newlines), + line_buffering=oget("line_buffering", stream.line_buffering), + ), + ) # -------------------------------------------------------------------- # Downloader output + def select(): """Select a suitable output class""" mode = config.get(("output",), "mode") if mode is None or mode == "auto": try: - if TTY_STDOUT: - output = ColorOutput() if ANSI else TerminalOutput() - else: - output = PipeOutput() + output = (ColorOutput() if ANSI else TerminalOutput()) if TTY_STDOUT else PipeOutput() except Exception: output = PipeOutput() elif isinstance(mode, dict): @@ -354,12 +367,12 @@ def select(): output = NullOutput() else: output = { - "default" : PipeOutput, - "pipe" : PipeOutput, - "term" : TerminalOutput, + "default": PipeOutput, + "pipe": PipeOutput, + "term": TerminalOutput, "terminal": TerminalOutput, - "color" : ColorOutput, - "null" : NullOutput, + "color": ColorOutput, + "null": NullOutput, }[mode.lower()]() if not config.get(("output",), "skip", True): @@ -367,8 +380,7 @@ def select(): return output -class NullOutput(): - +class NullOutput: def start(self, path): """Print a message indicating the start of a download""" @@ -383,7 +395,6 @@ def progress(self, bytes_total, bytes_downloaded, bytes_per_second): class PipeOutput(NullOutput): - def skip(self, path): stdout_write(CHAR_SKIP + path + "\n") @@ -391,8 +402,7 @@ def success(self, path): stdout_write(path + "\n") -class TerminalOutput(): - +class TerminalOutput: def __init__(self): shorten = config.get(("output",), "shorten", True) if shorten: @@ -416,14 +426,12 @@ def progress(self, bytes_total, bytes_downloaded, bytes_per_second): bdl = util.format_value(bytes_downloaded) bps = util.format_value(bytes_per_second) if bytes_total is None: - stderr_write("\r{:>7}B {:>7}B/s ".format(bdl, bps)) + stderr_write(f"\r{bdl:>7}B {bps:>7}B/s ") else: - stderr_write("\r{:>3}% {:>7}B {:>7}B/s ".format( - bytes_downloaded * 100 // bytes_total, bdl, bps)) + stderr_write(f"\r{bytes_downloaded * 100 // bytes_total:>3}% {bdl:>7}B {bps:>7}B/s ") class ColorOutput(TerminalOutput): - def __init__(self): TerminalOutput.__init__(self) @@ -431,10 +439,8 @@ def __init__(self): if colors is None: colors = COLORS_DEFAULT - self.color_skip = "\033[{}m".format( - colors.get("skip", "2")) - self.color_success = "\r\033[{}m".format( - colors.get("success", "1;32")) + self.color_skip = "\033[{}m".format(colors.get("skip", "2")) + self.color_success = "\r\033[{}m".format(colors.get("success", "1;32")) def start(self, path): stdout_write_flush(self.shorten(path)) @@ -446,10 +452,8 @@ def success(self, path): stdout_write(self.color_success + self.shorten(path) + "\033[0m\n") -class CustomOutput(): - +class CustomOutput: def __init__(self, options): - fmt_skip = options.get("skip") fmt_start = options.get("start") fmt_success = options.get("success") @@ -467,21 +471,18 @@ def __init__(self, options): func = shorten_string_eaw if shorten == "eaw" else shorten_string width = shutil.get_terminal_size().columns - self._fmt_skip = self._make_func( - func, fmt_skip, width - off_skip) - self._fmt_start = self._make_func( - func, fmt_start, width - off_start) - self._fmt_success = self._make_func( - func, fmt_success, width - off_success) + self._fmt_skip = self._make_func(func, fmt_skip, width - off_skip) + self._fmt_start = self._make_func(func, fmt_start, width - off_start) + self._fmt_success = self._make_func(func, fmt_success, width - off_success) else: self._fmt_skip = fmt_skip.format self._fmt_start = fmt_start.format self._fmt_success = fmt_success.format - self._fmt_progress = (options.get("progress") or - "\r{0:>7}B {1:>7}B/s ").format - self._fmt_progress_total = (options.get("progress-total") or - "\r{3:>3}% {0:>7}B {1:>7}B/s ").format + self._fmt_progress = (options.get("progress") or "\r{0:>7}B {1:>7}B/s ").format + self._fmt_progress_total = ( + options.get("progress-total") or "\r{3:>3}% {0:>7}B {1:>7}B/s " + ).format @staticmethod def _make_func(shorten, format_string, limit): @@ -503,16 +504,16 @@ def progress(self, bytes_total, bytes_downloaded, bytes_per_second): if bytes_total is None: stderr_write(self._fmt_progress(bdl, bps)) else: - stderr_write(self._fmt_progress_total( - bdl, bps, util.format_value(bytes_total), - bytes_downloaded * 100 // bytes_total)) + stderr_write( + self._fmt_progress_total( + bdl, bps, util.format_value(bytes_total), bytes_downloaded * 100 // bytes_total + ) + ) class EAWCache(dict): - def __missing__(self, key): - width = self[key] = \ - 2 if unicodedata.east_asian_width(key) in "WF" else 1 + width = self[key] = 2 if unicodedata.east_asian_width(key) in "WF" else 1 return width @@ -521,7 +522,7 @@ def shorten_string(txt, limit, sep="…"): if len(txt) <= limit: return txt limit -= len(sep) - return txt[:limit // 2] + sep + txt[-((limit+1) // 2):] + return txt[: limit // 2] + sep + txt[-((limit + 1) // 2) :] def shorten_string_eaw(txt, limit, sep="…", cache=EAWCache()): @@ -536,7 +537,7 @@ def shorten_string_eaw(txt, limit, sep="…", cache=EAWCache()): limit -= len(sep) if text_width == len(txt): # all characters have a width of 1 - return txt[:limit // 2] + sep + txt[-((limit+1) // 2):] + return txt[: limit // 2] + sep + txt[-((limit + 1) // 2) :] # wide characters left = 0 @@ -548,11 +549,11 @@ def shorten_string_eaw(txt, limit, sep="…", cache=EAWCache()): left += 1 right = -1 - rwidth = (limit+1) // 2 + (lwidth + char_widths[left]) + rwidth = (limit + 1) // 2 + (lwidth + char_widths[left]) while True: rwidth -= char_widths[right] if rwidth < 0: break right -= 1 - return txt[:left] + sep + txt[right+1:] + return txt[:left] + sep + txt[right + 1 :] diff --git a/gallery_dl/path.py b/gallery_dl/path.py index f57b02e171..26ef92f3b5 100644 --- a/gallery_dl/path.py +++ b/gallery_dl/path.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,24 +6,26 @@ """Filesystem path handling""" +import functools import os import re import shutil -import functools -from . import util, formatter, exception + +from . import exception +from . import formatter +from . import util WINDOWS = util.WINDOWS EXTENSION_MAP = { "jpeg": "jpg", - "jpe" : "jpg", + "jpe": "jpg", "jfif": "jpg", - "jif" : "jpg", - "jfi" : "jpg", + "jif": "jpg", + "jfi": "jpg", } -class PathFormat(): - +class PathFormat: def __init__(self, extractor): config = extractor.config kwdefault = config("keywords-default") @@ -38,15 +38,14 @@ def __init__(self, extractor): filename_fmt = extractor.filename_fmt elif isinstance(filename_fmt, dict): self.filename_conditions = [ - (util.compile_filter(expr), - formatter.parse(fmt, kwdefault).format_map) - for expr, fmt in filename_fmt.items() if expr + (util.compile_filter(expr), formatter.parse(fmt, kwdefault).format_map) + for expr, fmt in filename_fmt.items() + if expr ] self.build_filename = self.build_filename_conditional filename_fmt = filename_fmt.get("", extractor.filename_fmt) - self.filename_formatter = formatter.parse( - filename_fmt, kwdefault).format_map + self.filename_formatter = formatter.parse(filename_fmt, kwdefault).format_map except Exception as exc: raise exception.FilenameFormatError(exc) @@ -57,18 +56,18 @@ def __init__(self, extractor): directory_fmt = extractor.directory_fmt elif isinstance(directory_fmt, dict): self.directory_conditions = [ - (util.compile_filter(expr), [ - formatter.parse(fmt, kwdefault).format_map - for fmt in fmts - ]) - for expr, fmts in directory_fmt.items() if expr + ( + util.compile_filter(expr), + [formatter.parse(fmt, kwdefault).format_map for fmt in fmts], + ) + for expr, fmts in directory_fmt.items() + if expr ] self.build_directory = self.build_directory_conditional directory_fmt = directory_fmt.get("", extractor.directory_fmt) self.directory_formatters = [ - formatter.parse(dirfmt, kwdefault).format_map - for dirfmt in directory_fmt + formatter.parse(dirfmt, kwdefault).format_map for dirfmt in directory_fmt ] except Exception as exc: raise exception.DirectoryFormatError(exc) @@ -92,11 +91,11 @@ def __init__(self, extractor): restrict = config("path-restrict", "auto") replace = config("path-replace", "_") if restrict == "auto": - restrict = "\\\\|/<>:\"?*" if WINDOWS else "/" + restrict = '\\\\|/<>:"?*' if WINDOWS else "/" elif restrict == "unix": restrict = "/" elif restrict == "windows": - restrict = "\\\\|/<>:\"?*" + restrict = '\\\\|/<>:"?*' elif restrict == "ascii": restrict = "^0-9A-Za-z_." elif restrict == "ascii+": @@ -138,15 +137,16 @@ def __init__(self, extractor): def _build_cleanfunc(chars, repl): if not chars: return util.identity - elif isinstance(chars, dict): + if isinstance(chars, dict): + def func(x, table=str.maketrans(chars)): return x.translate(table) elif len(chars) == 1: + def func(x, c=chars, r=repl): return x.replace(c, r) else: - return functools.partial( - re.compile("[" + chars + "]").sub, repl) + return functools.partial(re.compile("[" + chars + "]").sub, repl) return func def open(self, mode="wb"): @@ -188,7 +188,8 @@ def set_directory(self, kwdict): segments = self.build_directory(kwdict) if segments: self.directory = directory = self.basedirectory + self.clean_path( - os.sep.join(segments) + os.sep) + os.sep.join(segments) + os.sep + ) else: self.directory = directory = self.basedirectory @@ -226,8 +227,7 @@ def fix_extension(self, _=None): """Fix filenames without a given filename extension""" try: if not self.extension: - self.kwdict["extension"] = \ - self.prefix + self.extension_map("", "") + self.kwdict["extension"] = self.prefix + self.extension_map("", "") self.build_path() if self.path[-1] == ".": self.path = self.path[:-1] @@ -244,8 +244,7 @@ def fix_extension(self, _=None): def build_filename(self, kwdict): """Apply 'kwdict' to filename format string""" try: - return self.clean_path(self.clean_segment( - self.filename_formatter(kwdict))) + return self.clean_path(self.clean_segment(self.filename_formatter(kwdict))) except Exception as exc: raise exception.FilenameFormatError(exc) @@ -312,8 +311,7 @@ def part_enable(self, part_directory=None): if self.extension: self.temppath += ".part" else: - self.kwdict["extension"] = self.prefix + self.extension_map( - "part", "part") + self.kwdict["extension"] = self.prefix + self.extension_map("part", "part") self.build_path() if part_directory: self.temppath = os.path.join( diff --git a/gallery_dl/postprocessor/__init__.py b/gallery_dl/postprocessor/__init__.py index 7837b063d9..0e0cf7f39e 100644 --- a/gallery_dl/postprocessor/__init__.py +++ b/gallery_dl/postprocessor/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify diff --git a/gallery_dl/postprocessor/classify.py b/gallery_dl/postprocessor/classify.py index 34af1d9408..9830b86f37 100644 --- a/gallery_dl/postprocessor/classify.py +++ b/gallery_dl/postprocessor/classify.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2021 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,31 +6,25 @@ """Categorize files by file extension""" -from .common import PostProcessor import os +from .common import PostProcessor -class ClassifyPP(PostProcessor): +class ClassifyPP(PostProcessor): DEFAULT_MAPPING = { - "Music" : ("mp3", "aac", "flac", "ogg", "wma", "m4a", "wav"), - "Video" : ("flv", "ogv", "avi", "mp4", "mpg", "mpeg", "3gp", "mkv", - "webm", "vob", "wmv"), - "Pictures" : ("jpg", "jpeg", "png", "gif", "bmp", "svg", "webp"), - "Archives" : ("zip", "rar", "7z", "tar", "gz", "bz2"), + "Music": ("mp3", "aac", "flac", "ogg", "wma", "m4a", "wav"), + "Video": ("flv", "ogv", "avi", "mp4", "mpg", "mpeg", "3gp", "mkv", "webm", "vob", "wmv"), + "Pictures": ("jpg", "jpeg", "png", "gif", "bmp", "svg", "webp"), + "Archives": ("zip", "rar", "7z", "tar", "gz", "bz2"), } def __init__(self, job, options): PostProcessor.__init__(self, job) mapping = options.get("mapping", self.DEFAULT_MAPPING) - self.mapping = { - ext: directory - for directory, exts in mapping.items() - for ext in exts - } - job.register_hooks( - {"prepare": self.prepare, "file": self.move}, options) + self.mapping = {ext: directory for directory, exts in mapping.items() for ext in exts} + job.register_hooks({"prepare": self.prepare, "file": self.move}, options) def prepare(self, pathfmt): ext = pathfmt.extension diff --git a/gallery_dl/postprocessor/common.py b/gallery_dl/postprocessor/common.py index d4e1603410..790eaad5e4 100644 --- a/gallery_dl/postprocessor/common.py +++ b/gallery_dl/postprocessor/common.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,10 +6,12 @@ """Common classes and constants used by postprocessor modules.""" -from .. import util, formatter, archive +from .. import archive +from .. import formatter +from .. import util -class PostProcessor(): +class PostProcessor: """Base class for postprocessors""" def __init__(self, job): @@ -28,24 +28,28 @@ def _init_archive(self, job, options, prefix=None): archive_path = util.expand_path(archive_path) if not prefix: prefix = "_" + self.name.upper() + "_" - archive_format = ( - options.get("archive-prefix", extr.category) + - options.get("archive-format", prefix + extr.archive_fmt)) + archive_format = options.get("archive-prefix", extr.category) + options.get( + "archive-format", prefix + extr.archive_fmt + ) try: if "{" in archive_path: - archive_path = formatter.parse(archive_path).format_map( - job.pathfmt.kwdict) + archive_path = formatter.parse(archive_path).format_map(job.pathfmt.kwdict) self.archive = archive.DownloadArchive( - archive_path, archive_format, + archive_path, + archive_format, options.get("archive-pragma"), - "_archive_" + self.name) + "_archive_" + self.name, + ) except Exception as exc: self.log.warning( "Failed to open %s archive at '%s' (%s: %s)", - self.name, archive_path, exc.__class__.__name__, exc) + self.name, + archive_path, + exc.__class__.__name__, + exc, + ) else: - self.log.debug( - "Using %s archive '%s'", self.name, archive_path) + self.log.debug("Using %s archive '%s'", self.name, archive_path) return True self.archive = None diff --git a/gallery_dl/postprocessor/compare.py b/gallery_dl/postprocessor/compare.py index 3bb63c80e1..1481e2bb02 100644 --- a/gallery_dl/postprocessor/compare.py +++ b/gallery_dl/postprocessor/compare.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2020-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,15 @@ """Compare versions of the same file and replace/enumerate them on mismatch""" -from .common import PostProcessor -from .. import text, util, exception import os +from .. import exception +from .. import text +from .. import util +from .common import PostProcessor -class ComparePP(PostProcessor): +class ComparePP(PostProcessor): def __init__(self, job, options): PostProcessor.__init__(self, job) if options.get("shallow"): @@ -32,11 +32,10 @@ def __init__(self, job, options): elif equal == "exit": self._equal_exc = SystemExit - job.register_hooks({"file": ( - self.enumerate - if options.get("action") == "enumerate" else - self.replace - )}, options) + job.register_hooks( + {"file": (self.enumerate if options.get("action") == "enumerate" else self.replace)}, + options, + ) def replace(self, pathfmt): try: diff --git a/gallery_dl/postprocessor/exec.py b/gallery_dl/postprocessor/exec.py index 7d2be2b957..11352f6407 100644 --- a/gallery_dl/postprocessor/exec.py +++ b/gallery_dl/postprocessor/exec.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,15 @@ """Execute processes""" -from .common import PostProcessor -from .. import util, formatter import os import re +from .. import formatter +from .. import util +from .common import PostProcessor if util.WINDOWS: + def quote(s): return '"' + s.replace('"', '\\"') + '"' else: @@ -22,7 +22,6 @@ def quote(s): class ExecPP(PostProcessor): - def __init__(self, job, options): PostProcessor.__init__(self, job) @@ -81,8 +80,7 @@ def _exec(self, args, shell): self.log.debug("Running '%s'", args) retcode = util.Popen(args, shell=shell).wait() if retcode: - self.log.warning("'%s' returned with non-zero exit status (%d)", - args, retcode) + self.log.warning("'%s' returned with non-zero exit status (%d)", args, retcode) def _exec_async(self, args, shell): self.log.debug("Running '%s'", args) diff --git a/gallery_dl/postprocessor/hash.py b/gallery_dl/postprocessor/hash.py index 92a7477992..c9617903b6 100644 --- a/gallery_dl/postprocessor/hash.py +++ b/gallery_dl/postprocessor/hash.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,12 @@ """Compute file hash digests""" -from .common import PostProcessor import hashlib +from .common import PostProcessor -class HashPP(PostProcessor): +class HashPP(PostProcessor): def __init__(self, job, options): PostProcessor.__init__(self, job) @@ -41,10 +39,7 @@ def __init__(self, job, options): job.register_hooks({event: self.run for event in events}, options) def run(self, pathfmt): - hashes = [ - (key, hashlib.new(name)) - for key, name in self.hashes - ] + hashes = [(key, hashlib.new(name)) for key, name in self.hashes] size = self.chunk_size with self._open(pathfmt) as fp: diff --git a/gallery_dl/postprocessor/metadata.py b/gallery_dl/postprocessor/metadata.py index 3ef9fbce13..9bf330f092 100644 --- a/gallery_dl/postprocessor/metadata.py +++ b/gallery_dl/postprocessor/metadata.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,15 +6,16 @@ """Write metadata to external files""" -from .common import PostProcessor -from .. import util, formatter import json -import sys import os +import sys +from .. import formatter +from .. import util +from .common import PostProcessor -class MetadataPP(PostProcessor): +class MetadataPP(PostProcessor): def __init__(self, job, options): PostProcessor.__init__(self, job) @@ -73,8 +72,7 @@ def __init__(self, job, options): if isinstance(directory, list): self._directory = self._directory_format self._directory_formatters = [ - formatter.parse(dirfmt, util.NONE).format_map - for dirfmt in directory + formatter.parse(dirfmt, util.NONE).format_map for dirfmt in directory ] elif directory: self._directory = self._directory_custom @@ -190,8 +188,7 @@ def _filename(self, pathfmt): return (pathfmt.filename or "metadata") + "." + self.extension def _filename_custom(self, pathfmt): - return pathfmt.clean_path(pathfmt.clean_segment( - self._filename_fmt(pathfmt.kwdict))) + return pathfmt.clean_path(pathfmt.clean_segment(self._filename_fmt(pathfmt.kwdict))) def _filename_extfmt(self, pathfmt): kwdict = pathfmt.kwdict @@ -253,10 +250,8 @@ def _make_filter(self, options): exclude = set(exclude) if private: - return lambda d: {k: v for k, v in d.items() - if k not in exclude} - return lambda d: {k: v for k, v in util.filter_dict(d).items() - if k not in exclude} + return lambda d: {k: v for k, v in d.items() if k not in exclude} + return lambda d: {k: v for k, v in util.filter_dict(d).items() if k not in exclude} if not private: return util.filter_dict diff --git a/gallery_dl/postprocessor/mtime.py b/gallery_dl/postprocessor/mtime.py index 6ded1e29dc..514893175a 100644 --- a/gallery_dl/postprocessor/mtime.py +++ b/gallery_dl/postprocessor/mtime.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2019-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,15 @@ """Use metadata as file modification time""" -from .common import PostProcessor -from .. import text, util, formatter from datetime import datetime +from .. import formatter +from .. import text +from .. import util +from .common import PostProcessor -class MtimePP(PostProcessor): +class MtimePP(PostProcessor): def __init__(self, job, options): PostProcessor.__init__(self, job) value = options.get("value") @@ -38,8 +38,8 @@ def run(self, pathfmt): pathfmt.kwdict["_mtime"] = ( util.datetime_to_timestamp(mtime) - if isinstance(mtime, datetime) else - text.parse_int(mtime) + if isinstance(mtime, datetime) + else text.parse_int(mtime) ) diff --git a/gallery_dl/postprocessor/python.py b/gallery_dl/postprocessor/python.py index db71da2502..d461d081b7 100644 --- a/gallery_dl/postprocessor/python.py +++ b/gallery_dl/postprocessor/python.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,12 +6,11 @@ """Run Python functions""" -from .common import PostProcessor from .. import util +from .common import PostProcessor class PythonPP(PostProcessor): - def __init__(self, job, options): PostProcessor.__init__(self, job) diff --git a/gallery_dl/postprocessor/rename.py b/gallery_dl/postprocessor/rename.py index f71738d2a9..081baf8c9d 100644 --- a/gallery_dl/postprocessor/rename.py +++ b/gallery_dl/postprocessor/rename.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,13 +6,13 @@ """Rename files""" -from .common import PostProcessor -from .. import formatter import os +from .. import formatter +from .common import PostProcessor -class RenamePP(PostProcessor): +class RenamePP(PostProcessor): def __init__(self, job, options): PostProcessor.__init__(self, job) @@ -24,19 +22,24 @@ def __init__(self, job, options): if old: self._old = self._apply_format(old) - self._new = (self._apply_format(new) if new else - self._apply_pathfmt) - job.register_hooks({ - "prepare": self.rename_from, - }, options) + self._new = self._apply_format(new) if new else self._apply_pathfmt + job.register_hooks( + { + "prepare": self.rename_from, + }, + options, + ) elif new: self._old = self._apply_pathfmt self._new = self._apply_format(new) - job.register_hooks({ - "skip" : self.rename_to_skip, - "prepare-after": self.rename_to_pafter, - }, options) + job.register_hooks( + { + "skip": self.rename_to_skip, + "prepare-after": self.rename_to_pafter, + }, + options, + ) else: raise ValueError("Option 'from' or 'to' is required") @@ -69,8 +72,10 @@ def rename_to_pafter(self, pathfmt): def _rename(self, path_old, name_old, path_new, name_new): if self.skip and os.path.exists(path_new): return self.log.warning( - "Not renaming '%s' to '%s' since another file with the " - "same name exists", name_old, name_new) + "Not renaming '%s' to '%s' since another file with the " "same name exists", + name_old, + name_new, + ) self.log.info("'%s' -> '%s'", name_old, name_new) os.replace(path_old, path_new) @@ -82,8 +87,7 @@ def _apply_format(self, format_string): fmt = formatter.parse(format_string).format_map def apply(pathfmt): - return pathfmt.clean_path(pathfmt.clean_segment(fmt( - pathfmt.kwdict))) + return pathfmt.clean_path(pathfmt.clean_segment(fmt(pathfmt.kwdict))) return apply diff --git a/gallery_dl/postprocessor/ugoira.py b/gallery_dl/postprocessor/ugoira.py index fec4ab0254..184976c6d5 100644 --- a/gallery_dl/postprocessor/ugoira.py +++ b/gallery_dl/postprocessor/ugoira.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,17 +6,19 @@ """Convert Pixiv Ugoira to WebM""" -from .common import PostProcessor -from .. import util +import os +import shutil import subprocess import tempfile import zipfile -import shutil -import os + +from .. import util +from .common import PostProcessor try: from math import gcd except ImportError: + def gcd(a, b): while b: a, b = b, a % b @@ -26,7 +26,6 @@ def gcd(a, b): class UgoiraPP(PostProcessor): - def __init__(self, job, options): PostProcessor.__init__(self, job) self.args = options.get("ffmpeg-args") or () @@ -48,8 +47,7 @@ def __init__(self, job, options): ext = options.get("extension") mode = options.get("mode") or options.get("ffmpeg-demuxer") if mode is None or mode == "auto": - if ext in (None, "webm", "mkv") and ( - mkvmerge or shutil.which("mkvmerge")): + if ext in (None, "webm", "mkv") and (mkvmerge or shutil.which("mkvmerge")): mode = "mkvmerge" else: mode = "concat" @@ -82,13 +80,18 @@ def __init__(self, job, options): vcodec = None for index, arg in enumerate(self.args): arg, _, stream = arg.partition(":") - if arg == "-vcodec" or arg in ("-c", "-codec") and ( - not stream or stream.partition(":")[0] in ("v", "V")): + if ( + arg == "-vcodec" + or arg in ("-c", "-codec") + and (not stream or stream.partition(":")[0] in ("v", "V")) + ): vcodec = self.args[index + 1] # use filter when using libx264/5 self.prevent_odd = ( - vcodec in ("libx264", "libx265") or - not vcodec and self.extension.lower() in ("mp4", "mkv")) + vcodec in ("libx264", "libx265") + or not vcodec + and self.extension.lower() in ("mp4", "mkv") + ) else: self.prevent_odd = False @@ -98,11 +101,14 @@ def __init__(self, job, options): if self.prevent_odd: args += ("-vf", "crop=iw-mod(iw\\,2):ih-mod(ih\\,2)") - job.register_hooks({ - "prepare": self.prepare, - "file" : self.convert_from_zip, - "after" : self.convert_from_files, - }, options) + job.register_hooks( + { + "prepare": self.prepare, + "file": self.convert_from_zip, + "after": self.convert_from_files, + }, + options, + ) def prepare(self, pathfmt): self._convert_zip = self._convert_files = False @@ -136,7 +142,7 @@ def prepare(self, pathfmt): def convert_from_zip(self, pathfmt): if not self._convert_zip: - return + return None self._zip_source = True with self._tempdir() as tempdir: @@ -146,13 +152,16 @@ def convert_from_zip(self, pathfmt): zfile.extractall(tempdir) except FileNotFoundError: pathfmt.realpath = pathfmt.temppath - return + return None except Exception as exc: pathfmt.realpath = pathfmt.temppath self.log.error( "%s: Unable to extract frames from %s (%s: %s)", - pathfmt.kwdict.get("id"), pathfmt.filename, - exc.__class__.__name__, exc) + pathfmt.kwdict.get("id"), + pathfmt.filename, + exc.__class__.__name__, + exc, + ) return self.log.debug("", exc_info=exc) if self.convert(pathfmt, tempdir): @@ -170,18 +179,17 @@ def convert_from_files(self, pathfmt): with tempfile.TemporaryDirectory() as tempdir: for frame in self._files: - # update frame filename extension - frame["file"] = name = "{}.{}".format( - frame["file"].partition(".")[0], frame["ext"]) + frame["file"] = name = "{}.{}".format(frame["file"].partition(".")[0], frame["ext"]) if tempdir: # move frame into tempdir try: self._copy_file(frame["path"], tempdir + "/" + name) except OSError as exc: - self.log.debug("Unable to copy frame %s (%s: %s)", - name, exc.__class__.__name__, exc) + self.log.debug( + "Unable to copy frame %s (%s: %s)", name, exc.__class__.__name__, exc + ) return pathfmt.kwdict["num"] = 0 @@ -227,8 +235,7 @@ def convert_to_animation(self, pathfmt, tempdir): self._finalize(pathfmt, tempdir) except OSError as exc: print() - self.log.error("Unable to invoke FFmpeg (%s: %s)", - exc.__class__.__name__, exc) + self.log.error("Unable to invoke FFmpeg (%s: %s)", exc.__class__.__name__, exc) self.log.debug("", exc_info=exc) pathfmt.realpath = pathfmt.temppath except Exception as exc: @@ -247,14 +254,10 @@ def convert_to_archive(self, pathfmt, tempdir): frames = self._frames if self.metadata: - if isinstance(self.metadata, str): - metaname = self.metadata - else: - metaname = "animation.json" - framedata = util.json_dumps([ - {"file": frame["file"], "delay": frame["delay"]} - for frame in frames - ]).encode() + metaname = self.metadata if isinstance(self.metadata, str) else "animation.json" + framedata = util.json_dumps( + [{"file": frame["file"], "delay": frame["delay"]} for frame in frames] + ).encode() if self._zip_source: self.delete = False @@ -268,17 +271,14 @@ def convert_to_archive(self, pathfmt, tempdir): else: if self.mtime: dt = pathfmt.kwdict["date_url"] or pathfmt.kwdict["date"] - mtime = (dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second) + mtime = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second) with zipfile.ZipFile(pathfmt.realpath, "w") as zfile: for frame in frames: - zinfo = zipfile.ZipInfo.from_file( - frame["path"], frame["file"]) + zinfo = zipfile.ZipInfo.from_file(frame["path"], frame["file"]) if self.mtime: zinfo.date_time = mtime - with open(frame["path"], "rb") as src, \ - zfile.open(zinfo, "w") as dst: - shutil.copyfileobj(src, dst, 1024*8) + with open(frame["path"], "rb") as src, zfile.open(zinfo, "w") as dst: + shutil.copyfileobj(src, dst, 1024 * 8) if self.metadata: zinfo = zipfile.ZipInfo(metaname) if self.mtime: @@ -297,8 +297,7 @@ def _exec(self, args): retcode = util.Popen(args, stdout=out, stderr=out).wait() if retcode: print() - self.log.error("Non-zero exit status when running %s (%s)", - args, retcode) + self.log.error("Non-zero exit status when running %s (%s)", args, retcode) raise ValueError() return retcode @@ -330,9 +329,8 @@ def _process_image2(self, pathfmt, tempdir): last_copy = last.copy() frames.append(last_copy) name, _, ext = last_copy["file"].rpartition(".") - last_copy["file"] = "{:>06}.{}".format(int(name)+1, ext) - shutil.copyfile(tempdir + last["file"], - tempdir + last_copy["file"]) + last_copy["file"] = f"{int(name) + 1:>06}.{ext}" + shutil.copyfile(tempdir + last["file"], tempdir + last_copy["file"]) # adjust frame mtime values ts = 0 @@ -342,13 +340,14 @@ def _process_image2(self, pathfmt, tempdir): return [ self.ffmpeg, - "-f", "image2", - "-ts_from_file", "2", - "-pattern_type", "sequence", - "-i", "{}%06d.{}".format( - tempdir.replace("%", "%%"), - frame["file"].rpartition(".")[2] - ), + "-f", + "image2", + "-ts_from_file", + "2", + "-pattern_type", + "sequence", + "-i", + "{}%06d.{}".format(tempdir.replace("%", "%%"), frame["file"].rpartition(".")[2]), ] def _process_mkvmerge(self, pathfmt, tempdir): @@ -357,19 +356,23 @@ def _process_mkvmerge(self, pathfmt, tempdir): return [ self.ffmpeg, - "-f", "image2", - "-pattern_type", "sequence", - "-i", "{}/%06d.{}".format( - tempdir.replace("%", "%%"), - self._frames[0]["file"].rpartition(".")[2] + "-f", + "image2", + "-pattern_type", + "sequence", + "-i", + "{}/%06d.{}".format( + tempdir.replace("%", "%%"), self._frames[0]["file"].rpartition(".")[2] ), ] def _finalize_mkvmerge(self, pathfmt, tempdir): args = [ self.mkvmerge, - "-o", pathfmt.path, # mkvmerge does not support "raw" paths - "--timecodes", "0:" + self._write_mkvmerge_timecodes(tempdir), + "-o", + pathfmt.path, # mkvmerge does not support "raw" paths + "--timecodes", + "0:" + self._write_mkvmerge_timecodes(tempdir), ] if self.extension == "webm": args.append("--webm") @@ -383,8 +386,7 @@ def _write_ffmpeg_concat(self, tempdir): append = content.append for frame in self._frames: - append("file '{}'\nduration {}".format( - frame["file"], frame["delay"] / 1000)) + append("file '{}'\nduration {}".format(frame["file"], frame["delay"] / 1000)) if self.repeat: append("file '{}'".format(frame["file"])) append("") @@ -417,7 +419,7 @@ def calculate_framerate(self, frames): if not self.uniform: gcd = self._delay_gcd(frames) if gcd >= 10: - return (None, "1000/{}".format(gcd)) + return (None, f"1000/{gcd}") return (None, None) @@ -431,10 +433,7 @@ def _delay_gcd(frames): @staticmethod def _delay_is_uniform(frames): delay = frames[0]["delay"] - for f in frames: - if f["delay"] != delay: - return False - return True + return all(f["delay"] == delay for f in frames) __postprocessor__ = UgoiraPP diff --git a/gallery_dl/postprocessor/zip.py b/gallery_dl/postprocessor/zip.py index ce36f2a7af..89b383f536 100644 --- a/gallery_dl/postprocessor/zip.py +++ b/gallery_dl/postprocessor/zip.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,19 +6,19 @@ """Store files in ZIP archives""" -from .common import PostProcessor -from .. import util -import zipfile import os +import zipfile +from .. import util +from .common import PostProcessor -class ZipPP(PostProcessor): +class ZipPP(PostProcessor): COMPRESSION_ALGORITHMS = { "store": zipfile.ZIP_STORED, - "zip" : zipfile.ZIP_DEFLATED, + "zip": zipfile.ZIP_DEFLATED, "bzip2": zipfile.ZIP_BZIP2, - "lzma" : zipfile.ZIP_LZMA, + "lzma": zipfile.ZIP_LZMA, } def __init__(self, job, options): @@ -31,19 +29,20 @@ def __init__(self, job, options): algorithm = options.get("compression", "store") if algorithm not in self.COMPRESSION_ALGORITHMS: self.log.warning( - "unknown compression algorithm '%s'; falling back to 'store'", - algorithm) + "unknown compression algorithm '%s'; falling back to 'store'", algorithm + ) algorithm = "store" self.zfile = None self.path = job.pathfmt.realdirectory[:-1] - self.args = (self.path + ext, "a", - self.COMPRESSION_ALGORITHMS[algorithm], True) - - job.register_hooks({ - "file": (self.write_safe if options.get("mode") == "safe" else - self.write_fast), - }, options) + self.args = (self.path + ext, "a", self.COMPRESSION_ALGORITHMS[algorithm], True) + + job.register_hooks( + { + "file": (self.write_safe if options.get("mode") == "safe" else self.write_fast), + }, + options, + ) job.hooks["finalize"].append(self.finalize) def open(self): @@ -80,10 +79,8 @@ def write_extra(self, pathfmt, zfile, files): try: zfile.write(path, os.path.basename(path)) except OSError as exc: - self.log.warning( - "Unable to write %s to %s", path, zfile.filename) + self.log.warning("Unable to write %s to %s", path, zfile.filename) self.log.debug("%s: %s", exc, exc.__class__.__name__) - pass else: if self.delete: util.remove_file(path) diff --git a/gallery_dl/text.py b/gallery_dl/text.py index 5fd5a40715..2db7cfb6e9 100644 --- a/gallery_dl/text.py +++ b/gallery_dl/text.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2015-2022 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,11 +6,11 @@ """Collection of functions that work on strings/text""" +import datetime +import html import re import sys -import html import time -import datetime import urllib.parse HTML_RE = re.compile("<[^>]+>") @@ -32,11 +30,7 @@ def remove_html(txt, repl=" ", sep=" "): def split_html(txt): """Split input string by HTML tags""" try: - return [ - unescape(x).strip() - for x in HTML_RE.split(txt) - if x and not x.isspace() - ] + return [unescape(x).strip() for x in HTML_RE.split(txt) if x and not x.isspace()] except TypeError: return [] @@ -62,11 +56,11 @@ def root_from_url(url, scheme="https://"): """Extract scheme and domain from a URL""" if not url.startswith(("https://", "http://")): try: - return scheme + url[:url.index("/")] + return scheme + url[: url.index("/")] except ValueError: return scheme + url try: - return url[:url.index("/", 8)] + return url[: url.index("/", 8)] except ValueError: return url @@ -123,7 +117,7 @@ def extract(txt, begin, end, pos=0): try: first = txt.index(begin, pos) + len(begin) last = txt.index(end, first) - return txt[first:last], last+len(end) + return txt[first:last], last + len(end) except Exception: return None, pos @@ -132,7 +126,7 @@ def extr(txt, begin, end, default=""): """Stripped-down version of 'extract()'""" try: first = txt.index(begin) + len(begin) - return txt[first:txt.index(end, first)] + return txt[first : txt.index(end, first)] except Exception: return default @@ -142,7 +136,7 @@ def rextract(txt, begin, end, pos=-1): lbeg = len(begin) first = txt.rindex(begin, 0, pos) last = txt.index(end, first + lbeg) - return txt[first + lbeg:last], first + return txt[first + lbeg : last], first except Exception: return None, pos @@ -175,6 +169,7 @@ def extract_iter(txt, begin, end, pos=0): def extract_from(txt, pos=0, default=""): """Returns a function object that extracts from 'txt'""" + def extr(begin, end, index=txt.index, txt=txt): nonlocal pos try: @@ -184,6 +179,7 @@ def extr(begin, end, index=txt.index, txt=txt): return txt[first:last] except Exception: return default + return extr @@ -286,7 +282,7 @@ def parse_query_list(qs): return result -if sys.hexversion < 0x30c0000: +if sys.hexversion < 0x30C0000: # Python <= 3.11 def parse_timestamp(ts, default=None): """Create a datetime object from a Unix timestamp""" diff --git a/gallery_dl/update.py b/gallery_dl/update.py index b068e37504..297c4a609e 100644 --- a/gallery_dl/update.py +++ b/gallery_dl/update.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2024 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -10,55 +8,54 @@ import re import sys -from .extractor.common import Extractor, Message +from . import exception +from . import util +from . import version +from .extractor.common import Extractor +from .extractor.common import Message from .job import DownloadJob -from . import util, version, exception REPOS = { - "stable" : "mikf/gallery-dl", - "dev" : "gdl-org/builds", + "stable": "mikf/gallery-dl", + "dev": "gdl-org/builds", "nightly": "gdl-org/builds", - "master" : "gdl-org/builds", + "master": "gdl-org/builds", } BINARIES_STABLE = { - "windows" : "gallery-dl.exe", + "windows": "gallery-dl.exe", "windows_x86": "gallery-dl.exe", "windows_x64": "gallery-dl.exe", - "linux" : "gallery-dl.bin", + "linux": "gallery-dl.bin", } BINARIES_DEV = { - "windows" : "gallery-dl_windows.exe", + "windows": "gallery-dl_windows.exe", "windows_x86": "gallery-dl_windows_x86.exe", "windows_x64": "gallery-dl_windows.exe", - "linux" : "gallery-dl_linux", - "macos" : "gallery-dl_macos", + "linux": "gallery-dl_linux", + "macos": "gallery-dl_macos", } BINARIES = { - "stable" : BINARIES_STABLE, - "dev" : BINARIES_DEV, + "stable": BINARIES_STABLE, + "dev": BINARIES_DEV, "nightly": BINARIES_DEV, - "master" : BINARIES_DEV, + "master": BINARIES_DEV, } class UpdateJob(DownloadJob): - def handle_url(self, url, kwdict): if not self._check_update(kwdict): if kwdict["_check"]: self.status |= 1 - return self.extractor.log.info( - "gallery-dl is up to date (%s)", version.__version__) + return self.extractor.log.info("gallery-dl is up to date (%s)", version.__version__) if kwdict["_check"]: return self.extractor.log.info( - "A new release is available: %s -> %s", - version.__version__, kwdict["tag_name"]) + "A new release is available: %s -> %s", version.__version__, kwdict["tag_name"] + ) - self.extractor.log.info( - "Updating from %s to %s", - version.__version__, kwdict["tag_name"]) + self.extractor.log.info("Updating from %s to %s", version.__version__, kwdict["tag_name"]) path_old = sys.executable + ".old" path_new = sys.executable + ".new" @@ -98,10 +95,13 @@ def handle_url(self, url, kwdict): import atexit import subprocess - cmd = 'ping 127.0.0.1 -n 5 -w 1000 & del /F "{}"'.format(path_old) + cmd = f'ping 127.0.0.1 -n 5 -w 1000 & del /F "{path_old}"' atexit.register( - util.Popen, cmd, shell=True, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + util.Popen, + cmd, + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, ) else: @@ -193,8 +193,7 @@ def items(self): raise exception.StopExtraction("Invalid channel '%s'", repo) path_tag = tag if tag == "latest" else "tags/" + tag - url = "{}/repos/{}/releases/{}".format( - self.root_api, path_repo, path_tag) + url = f"{self.root_api}/repos/{path_repo}/releases/{path_tag}" headers = { "Accept": "application/vnd.github+json", "User-Agent": util.USERAGENT, @@ -204,15 +203,14 @@ def items(self): data["_check"] = check data["_exact"] = exact - if binary == "linux" and \ - repo != "stable" and \ - data["tag_name"] <= "2024.05.28": + if binary == "linux" and repo != "stable" and data["tag_name"] <= "2024.05.28": binary_name = "gallery-dl_ubuntu" else: binary_name = BINARIES[repo][binary] url = "{}/{}/releases/download/{}/{}".format( - self.root, path_repo, data["tag_name"], binary_name) + self.root, path_repo, data["tag_name"], binary_name + ) yield Message.Directory, data yield Message.Url, url, data diff --git a/gallery_dl/util.py b/gallery_dl/util.py index 44ac22e23c..91f7e697aa 100644 --- a/gallery_dl/util.py +++ b/gallery_dl/util.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2017-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,24 +6,29 @@ """Utility functions and classes""" -import re -import os -import sys -import json -import time -import random -import getpass -import hashlib import binascii +import collections import datetime import functools +import getpass +import hashlib import itertools +import json +import os +import random +import re import subprocess -import collections +import sys +import time import urllib.parse +from contextlib import suppress +from email.utils import mktime_tz +from email.utils import parsedate_tz from http.cookiejar import Cookie -from email.utils import mktime_tz, parsedate_tz -from . import text, version, exception + +from . import exception +from . import text +from . import version def bencode(num, alphabet="0123456789"): @@ -49,7 +52,7 @@ def bdecode(data, alphabet="0123456789"): def advance(iterable, num): - """"Advance 'iterable' by 'num' steps""" + """ "Advance 'iterable' by 'num' steps""" iterator = iter(iterable) next(itertools.islice(iterator, num, num), None) return iterator @@ -89,16 +92,15 @@ def contains(values, elements, separator=" "): if not isinstance(elements, (tuple, list)): return elements in values - for e in elements: - if e in values: - return True - return False + return any(e in values for e in elements) def raises(cls): """Returns a function that raises 'cls' as exception""" + def wrap(*args): raise cls(*args) + return wrap @@ -151,8 +153,7 @@ def format_value(value, suffixes="kMGTPEZY"): index = value_len - 4 if index >= 0: offset = (value_len - 1) % 3 + 1 - return (value[:offset] + "." + value[offset:offset+2] + - suffixes[index // 3]) + return value[:offset] + "." + value[offset : offset + 2] + suffixes[index // 3] return value @@ -193,9 +194,9 @@ def enumerate_reversed(iterable, start=0, length=None): length = len(iterable) try: - iterable = zip(range(start-1+length, start-1, -1), reversed(iterable)) + iterable = zip(range(start - 1 + length, start - 1, -1), reversed(iterable)) except TypeError: - iterable = list(zip(range(start, start+length), iterable)) + iterable = list(zip(range(start, start + length), iterable)) iterable.reverse() return iterable @@ -231,7 +232,7 @@ def datetime_to_timestamp_string(dt): return "" -if sys.hexversion < 0x30c0000: +if sys.hexversion < 0x30C0000: # Python <= 3.11 datetime_utcfromtimestamp = datetime.datetime.utcfromtimestamp datetime_utcnow = datetime.datetime.utcnow @@ -264,7 +265,8 @@ def json_default(obj): def dump_json(obj, fp=sys.stdout, ensure_ascii=True, indent=4): """Serialize 'obj' as JSON and write it to 'fp'""" json.dump( - obj, fp, + obj, + fp, ensure_ascii=ensure_ascii, indent=indent, default=json_default, @@ -308,30 +310,31 @@ def dump_response(response, fp, headers=False, content=True, hide_auth=True): cookie = req_headers.get("Cookie") if cookie: req_headers["Cookie"] = ";".join( - c.partition("=")[0] + "=***" - for c in cookie.split(";") + c.partition("=")[0] + "=***" for c in cookie.split(";") ) set_cookie = res_headers.get("Set-Cookie") if set_cookie: res_headers["Set-Cookie"] = re.sub( - r"(^|, )([^ =]+)=[^,;]*", r"\1\2=***", set_cookie, + r"(^|, )([^ =]+)=[^,;]*", + r"\1\2=***", + set_cookie, ) fmt_nv = "{}: {}".format - fp.write(outfmt.format( - request=request, - response=response, - request_headers="\n".join( - fmt_nv(name, value) - for name, value in req_headers.items() - ), - response_headers="\n".join( - fmt_nv(name, value) - for name, value in res_headers.items() - ), - ).encode()) + fp.write( + outfmt.format( + request=request, + response=response, + request_headers="\n".join( + fmt_nv(name, value) for name, value in req_headers.items() + ), + response_headers="\n".join( + fmt_nv(name, value) for name, value in res_headers.items() + ), + ).encode() + ) if content: if headers: @@ -356,7 +359,7 @@ def extract_headers(response): return data -@functools.lru_cache(maxsize=None) +@functools.cache def git_head(): try: out, err = Popen( @@ -382,17 +385,13 @@ def expand_path(path): def remove_file(path): - try: + with suppress(OSError): os.unlink(path) - except OSError: - pass def remove_directory(path): - try: + with suppress(OSError): os.rmdir(path) - except OSError: - pass def set_mtime(path, mtime): @@ -409,7 +408,6 @@ def cookiestxt_load(fp): cookies = [] for line in fp: - line = line.lstrip(" ") # strip '#HttpOnly_' if line.startswith("#HttpOnly_"): @@ -421,24 +419,32 @@ def cookiestxt_load(fp): if line[-1] == "\n": line = line[:-1] - domain, domain_specified, path, secure, expires, name, value = \ - line.split("\t") + domain, domain_specified, path, secure, expires, name, value = line.split("\t") if not name: name = value value = None - cookies.append(Cookie( - 0, name, value, - None, False, - domain, - domain_specified == "TRUE", - domain[0] == "." if domain else False, - path, False, - secure == "TRUE", - None if expires == "0" or not expires else expires, - False, None, None, {}, - )) + cookies.append( + Cookie( + 0, + name, + value, + None, + False, + domain, + domain_specified == "TRUE", + domain[0] == "." if domain else False, + path, + False, + secure == "TRUE", + None if expires == "0" or not expires else expires, + False, + None, + None, + {}, + ) + ) return cookies @@ -460,15 +466,19 @@ def cookiestxt_store(fp, cookies): value = cookie.value domain = cookie.domain - write("\t".join(( - domain, - "TRUE" if domain and domain[0] == "." else "FALSE", - cookie.path, - "TRUE" if cookie.secure else "FALSE", - "0" if cookie.expires is None else str(cookie.expires), - name, - value + "\n", - ))) + write( + "\t".join( + ( + domain, + "TRUE" if domain and domain[0] == "." else "FALSE", + cookie.path, + "TRUE" if cookie.secure else "FALSE", + "0" if cookie.expires is None else str(cookie.expires), + name, + value + "\n", + ) + ) + ) def code_to_language(code, default=None): @@ -520,20 +530,23 @@ def language_to_code(lang, default=None): } -class HTTPBasicAuth(): +class HTTPBasicAuth: __slots__ = ("authorization",) def __init__(self, username, password): - self.authorization = b"Basic " + binascii.b2a_base64( - username.encode("latin1") + b":" + str(password).encode("latin1") - )[:-1] + self.authorization = ( + b"Basic " + + binascii.b2a_base64( + username.encode("latin1") + b":" + str(password).encode("latin1") + )[:-1] + ) def __call__(self, request): request.headers["Authorization"] = self.authorization return request -class ModuleProxy(): +class ModuleProxy: __slots__ = () def __getitem__(self, key, modules=sys.modules): @@ -551,14 +564,14 @@ def __getitem__(self, key, modules=sys.modules): __getattr__ = __getitem__ -class LazyPrompt(): +class LazyPrompt: __slots__ = () def __str__(self): return getpass.getpass() -class NullContext(): +class NullContext: __slots__ = () def __enter__(self): @@ -568,8 +581,9 @@ def __exit__(self, exc_type, exc_value, traceback): pass -class CustomNone(): +class CustomNone: """None-style type that supports more operations than regular None""" + __slots__ = () __getattribute__ = identity @@ -650,24 +664,24 @@ def __str__(): NONE = CustomNone() EPOCH = datetime.datetime(1970, 1, 1) SECOND = datetime.timedelta(0, 1) -WINDOWS = (os.name == "nt") +WINDOWS = os.name == "nt" SENTINEL = object() USERAGENT = "gallery-dl/" + version.__version__ EXECUTABLE = getattr(sys, "frozen", False) SPECIAL_EXTRACTORS = {"oauth", "recursive", "generic"} GLOBALS = { - "contains" : contains, + "contains": contains, "parse_int": text.parse_int, - "urlsplit" : urllib.parse.urlsplit, - "datetime" : datetime.datetime, + "urlsplit": urllib.parse.urlsplit, + "datetime": datetime.datetime, "timedelta": datetime.timedelta, - "abort" : raises(exception.StopExtraction), + "abort": raises(exception.StopExtraction), "terminate": raises(exception.TerminateExtraction), - "restart" : raises(exception.RestartExtraction), + "restart": raises(exception.RestartExtraction), "hash_sha1": sha1, - "hash_md5" : md5, - "std" : ModuleProxy(), - "re" : re, + "hash_md5": md5, + "std": ModuleProxy(), + "re": re, } @@ -703,7 +717,7 @@ def compile_expression_raw(expr, name="", globals=None): return functools.partial(eval, code_object, globals or GLOBALS) -def compile_expression_defaultdict(expr, name="", globals=None): +def compile_expression_defaultdict(expr, name="", globals=None): # noqa: F811 global GLOBALS_DEFAULT GLOBALS_DEFAULT = collections.defaultdict(lambda: NONE, GLOBALS) @@ -778,13 +792,11 @@ def build_duration_func(duration, min=0.0): upper = float(upper) return functools.partial( random.uniform, - lower if lower > min else min, - upper if upper > min else min, + max(min, lower), + max(min, upper), ) - else: - if lower < min: - lower = min - return lambda: lower + lower = max(lower, min) + return lambda: lower def build_extractor_filter(categories, negate=True, special=None): @@ -796,7 +808,7 @@ def build_extractor_filter(categories, negate=True, special=None): catset = set() # set of categories / basecategories subset = set() # set of subcategories - catsub = [] # list of category-subcategory pairs + catsub = [] # list of category-subcategory pairs for item in categories: category, _, subcategory = item.partition(":") @@ -817,35 +829,34 @@ def build_extractor_filter(categories, negate=True, special=None): if negate: if catset: - tests.append(lambda extr: - extr.category not in catset and - extr.basecategory not in catset) + tests.append( + lambda extr: extr.category not in catset and extr.basecategory not in catset + ) if subset: tests.append(lambda extr: extr.subcategory not in subset) else: if catset: - tests.append(lambda extr: - extr.category in catset or - extr.basecategory in catset) + tests.append(lambda extr: extr.category in catset or extr.basecategory in catset) if subset: tests.append(lambda extr: extr.subcategory in subset) if catsub: + def test(extr): for category, subcategory in catsub: if subcategory == extr.subcategory and ( - category == extr.category or - category == extr.basecategory): + category == extr.category or category == extr.basecategory + ): return not negate return negate + tests.append(test) if len(tests) == 1: return tests[0] if negate: return lambda extr: all(t(extr) for t in tests) - else: - return lambda extr: any(t(extr) for t in tests) + return lambda extr: any(t(extr) for t in tests) def build_proxy_map(proxies, log=None): @@ -871,19 +882,16 @@ def build_proxy_map(proxies, log=None): def build_predicate(predicates): if not predicates: return lambda url, kwdict: True - elif len(predicates) == 1: + if len(predicates) == 1: return predicates[0] return functools.partial(chain_predicates, predicates) def chain_predicates(predicates, url, kwdict): - for pred in predicates: - if not pred(url, kwdict): - return False - return True + return all(pred(url, kwdict) for pred in predicates) -class RangePredicate(): +class RangePredicate: """Predicate; True if the current index is in the given range(s)""" def __init__(self, rangespec): @@ -905,10 +913,7 @@ def __call__(self, _url, _kwdict): if index > self.upper: raise exception.StopExtraction() - for range in self.ranges: - if index in range: - return True - return False + return any(index in range for range in self.ranges) @staticmethod def _parse(rangespec): @@ -929,31 +934,36 @@ def _parse(rangespec): if not group: continue - elif ":" in group: + if ":" in group: start, _, stop = group.partition(":") stop, _, step = stop.partition(":") - append(range( - int(start) if start.strip() else 1, - int(stop) if stop.strip() else sys.maxsize, - int(step) if step.strip() else 1, - )) + append( + range( + int(start) if start.strip() else 1, + int(stop) if stop.strip() else sys.maxsize, + int(step) if step.strip() else 1, + ) + ) elif "-" in group: start, _, stop = group.partition("-") - append(range( - int(start) if start.strip() else 1, - int(stop) + 1 if stop.strip() else sys.maxsize, - )) + append( + range( + int(start) if start.strip() else 1, + int(stop) + 1 if stop.strip() else sys.maxsize, + ) + ) else: start = int(group) - append(range(start, start+1)) + append(range(start, start + 1)) return ranges -class UniquePredicate(): +class UniquePredicate: """Predicate; True if given URL has not been encountered before""" + def __init__(self): self.urls = set() @@ -966,11 +976,11 @@ def __call__(self, url, _): return False -class FilterPredicate(): +class FilterPredicate: """Predicate; True if evaluating the given expression returns True""" def __init__(self, expr, target="image"): - name = "<{} filter>".format(target) + name = f"<{target} filter>" self.expr = compile_filter(expr, name) def __call__(self, _, kwdict): diff --git a/gallery_dl/version.py b/gallery_dl/version.py index f85f221926..31cb5f879e 100644 --- a/gallery_dl/version.py +++ b/gallery_dl/version.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2016-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify diff --git a/gallery_dl/ytdl.py b/gallery_dl/ytdl.py index fe88c2cb4d..7bd48f6434 100644 --- a/gallery_dl/ytdl.py +++ b/gallery_dl/ytdl.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2021-2023 Mike Fährmann # # This program is free software; you can redistribute it and/or modify @@ -8,10 +6,13 @@ """Helpers for interacting with youtube-dl""" +import itertools import re import shlex -import itertools -from . import text, util, exception + +from . import exception +from . import text +from . import util def import_module(module_name): @@ -71,7 +72,7 @@ def construct_YoutubeDL(module, obj, user_opts, system_opts=None): def parse_command_line(module, argv): parser, opts, args = module.parseOpts(argv) - ytdlp = (module.__name__ == "yt_dlp") + ytdlp = module.__name__ == "yt_dlp" std_headers = module.std_headers try: @@ -126,8 +127,7 @@ def parse_command_line(module, argv): opts.remuxvideo = opts.remuxvideo.replace(" ", "") if getattr(opts, "wait_for_video", None) is not None: min_wait, _, max_wait = opts.wait_for_video.partition("-") - opts.wait_for_video = (module.parse_duration(min_wait), - module.parse_duration(max_wait)) + opts.wait_for_video = (module.parse_duration(min_wait), module.parse_duration(max_wait)) if opts.date is not None: date = module.DateRange.day(opts.date) @@ -140,21 +140,19 @@ def _unused_compat_opt(name): if name not in compat_opts: return False compat_opts.discard(name) - compat_opts.update(["*%s" % name]) + compat_opts.update([f"*{name}"]) return True - def set_default_compat( - compat_name, opt_name, default=True, remove_compat=True): + def set_default_compat(compat_name, opt_name, default=True, remove_compat=True): attr = getattr(opts, opt_name, None) if compat_name in compat_opts: if attr is None: setattr(opts, opt_name, not default) return True - else: - if remove_compat: - _unused_compat_opt(compat_name) - return False - elif attr is None: + if remove_compat: + _unused_compat_opt(compat_name) + return False + if attr is None: setattr(opts, opt_name, default) return None @@ -164,11 +162,11 @@ def set_default_compat( if "format-sort" in compat_opts: opts.format_sort.extend(module.InfoExtractor.FormatSort.ytdl_default) _video_multistreams_set = set_default_compat( - "multistreams", "allow_multiple_video_streams", - False, remove_compat=False) + "multistreams", "allow_multiple_video_streams", False, remove_compat=False + ) _audio_multistreams_set = set_default_compat( - "multistreams", "allow_multiple_audio_streams", - False, remove_compat=False) + "multistreams", "allow_multiple_audio_streams", False, remove_compat=False + ) if _video_multistreams_set is False and _audio_multistreams_set is False: _unused_compat_opt("multistreams") @@ -188,6 +186,7 @@ def set_default_compat( opts.format = "bestaudio/best" if ytdlp: + def metadataparser_actions(f): if isinstance(f, str): yield module.MetadataFromFieldPP.to_action(f) @@ -204,27 +203,29 @@ def metadataparser_actions(f): if opts.metafromtitle is not None: if "pre_process" not in parse_metadata: parse_metadata["pre_process"] = [] - parse_metadata["pre_process"].append( - "title:%s" % opts.metafromtitle) + parse_metadata["pre_process"].append(f"title:{opts.metafromtitle}") opts.parse_metadata = { - k: list(itertools.chain.from_iterable(map( - metadataparser_actions, v))) + k: list(itertools.chain.from_iterable(map(metadataparser_actions, v))) for k, v in parse_metadata.items() } else: if parse_metadata is None: parse_metadata = [] if opts.metafromtitle is not None: - parse_metadata.append("title:%s" % opts.metafromtitle) - opts.parse_metadata = list(itertools.chain.from_iterable(map( - metadataparser_actions, parse_metadata))) + parse_metadata.append(f"title:{opts.metafromtitle}") + opts.parse_metadata = list( + itertools.chain.from_iterable(map(metadataparser_actions, parse_metadata)) + ) opts.metafromtitle = None else: opts.parse_metadata = () - download_archive_fn = module.expand_path(opts.download_archive) \ - if opts.download_archive is not None else opts.download_archive + download_archive_fn = ( + module.expand_path(opts.download_archive) + if opts.download_archive is not None + else opts.download_archive + ) if getattr(opts, "getcomments", None): opts.writeinfojson = True @@ -233,30 +234,30 @@ def metadataparser_actions(f): opts.sponsorblock_mark = set() opts.sponsorblock_remove = set() else: - opts.sponsorblock_mark = \ - getattr(opts, "sponsorblock_mark", None) or set() - opts.sponsorblock_remove = \ - getattr(opts, "sponsorblock_remove", None) or set() + opts.sponsorblock_mark = getattr(opts, "sponsorblock_mark", None) or set() + opts.sponsorblock_remove = getattr(opts, "sponsorblock_remove", None) or set() opts.remove_chapters = getattr(opts, "remove_chapters", None) or () try: postprocessors = list(module.get_postprocessors(opts)) except AttributeError: - postprocessors = legacy_postprocessors( - opts, module, ytdlp, compat_opts) + postprocessors = legacy_postprocessors(opts, module, ytdlp, compat_opts) match_filter = ( - None if opts.match_filter is None - else module.match_filter_func(opts.match_filter)) + None if opts.match_filter is None else module.match_filter_func(opts.match_filter) + ) cookiesfrombrowser = getattr(opts, "cookiesfrombrowser", None) if cookiesfrombrowser: - match = re.fullmatch(r"""(?x) + match = re.fullmatch( + r"""(?x) (?P[^+:]+) (?:\s*\+\s*(?P[^:]+))? (?:\s*:\s*(?!:)(?P.+?))? (?:\s*::\s*(?P.+))? - """, cookiesfrombrowser) + """, + cookiesfrombrowser, + ) if match: browser, keyring, profile, container = match.groups() if keyring is not None: @@ -286,23 +287,17 @@ def metadataparser_actions(f): "forcefilename": opts.getfilename, "forceformat": opts.getformat, "forceprint": getattr(opts, "forceprint", None) or (), - "force_write_download_archive": getattr( - opts, "force_write_download_archive", None), + "force_write_download_archive": getattr(opts, "force_write_download_archive", None), "simulate": opts.simulate, "skip_download": opts.skip_download, "format": opts.format, - "allow_unplayable_formats": getattr( - opts, "allow_unplayable_formats", None), - "ignore_no_formats_error": getattr( - opts, "ignore_no_formats_error", None), - "format_sort": getattr( - opts, "format_sort", None), - "format_sort_force": getattr( - opts, "format_sort_force", None), + "allow_unplayable_formats": getattr(opts, "allow_unplayable_formats", None), + "ignore_no_formats_error": getattr(opts, "ignore_no_formats_error", None), + "format_sort": getattr(opts, "format_sort", None), + "format_sort_force": getattr(opts, "format_sort_force", None), "allow_multiple_video_streams": opts.allow_multiple_video_streams, "allow_multiple_audio_streams": opts.allow_multiple_audio_streams, - "check_formats": getattr( - opts, "check_formats", None), + "check_formats": getattr(opts, "check_formats", None), "outtmpl": opts.outtmpl, "outtmpl_na_placeholder": opts.outtmpl_na_placeholder, "paths": getattr(opts, "paths", None), @@ -321,8 +316,7 @@ def metadataparser_actions(f): "extractor_retries": getattr(opts, "extractor_retries", None), "skip_unavailable_fragments": opts.skip_unavailable_fragments, "keep_fragments": opts.keep_fragments, - "concurrent_fragment_downloads": getattr( - opts, "concurrent_fragment_downloads", None), + "concurrent_fragment_downloads": getattr(opts, "concurrent_fragment_downloads", None), "buffersize": opts.buffersize, "noresizebuffer": opts.noresizebuffer, "http_chunk_size": opts.http_chunk_size, @@ -344,8 +338,8 @@ def metadataparser_actions(f): "clean_infojson": opts.clean_infojson, "getcomments": getattr(opts, "getcomments", None), "writethumbnail": opts.writethumbnail is True, - "write_all_thumbnails": getattr(opts, "write_all_thumbnails", None) or - opts.writethumbnail == "all", + "write_all_thumbnails": getattr(opts, "write_all_thumbnails", None) + or opts.writethumbnail == "all", "writelink": getattr(opts, "writelink", None), "writeurllink": getattr(opts, "writeurllink", None), "writewebloclink": getattr(opts, "writewebloclink", None), @@ -377,8 +371,7 @@ def metadataparser_actions(f): "break_on_existing": getattr(opts, "break_on_existing", None), "break_on_reject": getattr(opts, "break_on_reject", None), "break_per_url": getattr(opts, "break_per_url", None), - "skip_playlist_after_errors": getattr( - opts, "skip_playlist_after_errors", None), + "skip_playlist_after_errors": getattr(opts, "skip_playlist_after_errors", None), "cookiefile": opts.cookiefile, "cookiesfrombrowser": cookiesfrombrowser, "nocheckcertificate": opts.no_check_certificate, @@ -392,10 +385,8 @@ def metadataparser_actions(f): "default_search": opts.default_search, "dynamic_mpd": getattr(opts, "dynamic_mpd", None), "extractor_args": getattr(opts, "extractor_args", None), - "youtube_include_dash_manifest": getattr( - opts, "youtube_include_dash_manifest", None), - "youtube_include_hls_manifest": getattr( - opts, "youtube_include_hls_manifest", None), + "youtube_include_dash_manifest": getattr(opts, "youtube_include_dash_manifest", None), + "youtube_include_hls_manifest": getattr(opts, "youtube_include_hls_manifest", None), "encoding": opts.encoding, "extract_flat": opts.extract_flat, "live_from_start": getattr(opts, "live_from_start", None), @@ -405,12 +396,10 @@ def metadataparser_actions(f): "postprocessors": postprocessors, "fixup": opts.fixup, "source_address": opts.source_address, - "sleep_interval_requests": getattr( - opts, "sleep_interval_requests", None), + "sleep_interval_requests": getattr(opts, "sleep_interval_requests", None), "sleep_interval": opts.sleep_interval, "max_sleep_interval": opts.max_sleep_interval, - "sleep_interval_subtitles": getattr( - opts, "sleep_interval_subtitles", None), + "sleep_interval_subtitles": getattr(opts, "sleep_interval_subtitles", None), "external_downloader": opts.external_downloader, "playlist_items": opts.playlist_items, "xattr_set_filesize": opts.xattr_set_filesize, @@ -419,18 +408,14 @@ def metadataparser_actions(f): "ffmpeg_location": opts.ffmpeg_location, "hls_prefer_native": opts.hls_prefer_native, "hls_use_mpegts": opts.hls_use_mpegts, - "hls_split_discontinuity": getattr( - opts, "hls_split_discontinuity", None), + "hls_split_discontinuity": getattr(opts, "hls_split_discontinuity", None), "external_downloader_args": opts.external_downloader_args, "postprocessor_args": opts.postprocessor_args, "cn_verification_proxy": opts.cn_verification_proxy, "geo_verification_proxy": opts.geo_verification_proxy, - "geo_bypass": getattr( - opts, "geo_bypass", "default"), - "geo_bypass_country": getattr( - opts, "geo_bypass_country", None), - "geo_bypass_ip_block": getattr( - opts, "geo_bypass_ip_block", None), + "geo_bypass": getattr(opts, "geo_bypass", "default"), + "geo_bypass_country": getattr(opts, "geo_bypass_country", None), + "geo_bypass_ip_block": getattr(opts, "geo_bypass_ip_block", None), "compat_opts": compat_opts, } @@ -446,65 +431,79 @@ def legacy_postprocessors(opts, module, ytdlp, compat_opts): sponsorblock_query = opts.sponsorblock_mark | opts.sponsorblock_remove if opts.metafromtitle: - postprocessors.append({ - "key": "MetadataFromTitle", - "titleformat": opts.metafromtitle, - }) + postprocessors.append( + { + "key": "MetadataFromTitle", + "titleformat": opts.metafromtitle, + } + ) if getattr(opts, "add_postprocessors", None): postprocessors += list(opts.add_postprocessors) if sponsorblock_query: - postprocessors.append({ - "key": "SponsorBlock", - "categories": sponsorblock_query, - "api": opts.sponsorblock_api, - "when": "pre_process", - }) + postprocessors.append( + { + "key": "SponsorBlock", + "categories": sponsorblock_query, + "api": opts.sponsorblock_api, + "when": "pre_process", + } + ) if opts.parse_metadata: - postprocessors.append({ - "key": "MetadataParser", - "actions": opts.parse_metadata, - "when": "pre_process", - }) + postprocessors.append( + { + "key": "MetadataParser", + "actions": opts.parse_metadata, + "when": "pre_process", + } + ) if opts.convertsubtitles: - pp = {"key": "FFmpegSubtitlesConvertor", - "format": opts.convertsubtitles} + pp = {"key": "FFmpegSubtitlesConvertor", "format": opts.convertsubtitles} if ytdlp: pp["when"] = "before_dl" postprocessors.append(pp) if getattr(opts, "convertthumbnails", None): - postprocessors.append({ - "key": "FFmpegThumbnailsConvertor", - "format": opts.convertthumbnails, - "when": "before_dl", - }) + postprocessors.append( + { + "key": "FFmpegThumbnailsConvertor", + "format": opts.convertthumbnails, + "when": "before_dl", + } + ) if getattr(opts, "exec_before_dl_cmd", None): - postprocessors.append({ - "key": "Exec", - "exec_cmd": opts.exec_before_dl_cmd, - "when": "before_dl", - }) + postprocessors.append( + { + "key": "Exec", + "exec_cmd": opts.exec_before_dl_cmd, + "when": "before_dl", + } + ) if opts.extractaudio: - postprocessors.append({ - "key": "FFmpegExtractAudio", - "preferredcodec": opts.audioformat, - "preferredquality": opts.audioquality, - "nopostoverwrites": opts.nopostoverwrites, - }) + postprocessors.append( + { + "key": "FFmpegExtractAudio", + "preferredcodec": opts.audioformat, + "preferredquality": opts.audioquality, + "nopostoverwrites": opts.nopostoverwrites, + } + ) if getattr(opts, "remuxvideo", None): - postprocessors.append({ - "key": "FFmpegVideoRemuxer", - "preferedformat": opts.remuxvideo, - }) + postprocessors.append( + { + "key": "FFmpegVideoRemuxer", + "preferedformat": opts.remuxvideo, + } + ) if opts.recodevideo: - postprocessors.append({ - "key": "FFmpegVideoConvertor", - "preferedformat": opts.recodevideo, - }) + postprocessors.append( + { + "key": "FFmpegVideoConvertor", + "preferedformat": opts.recodevideo, + } + ) if opts.embedsubtitles: pp = {"key": "FFmpegEmbedSubtitle"} if ytdlp: - pp["already_have_subtitle"] = ( - opts.writesubtitles and "no-keep-subs" not in compat_opts) + pp["already_have_subtitle"] = opts.writesubtitles and "no-keep-subs" not in compat_opts postprocessors.append(pp) if not opts.writeautomaticsub and "no-keep-subs" not in compat_opts: opts.writesubtitles = True @@ -519,14 +518,16 @@ def legacy_postprocessors(opts, module, ytdlp, compat_opts): continue remove_chapters_patterns.append(re.compile(regex)) if opts.remove_chapters or sponsorblock_query: - postprocessors.append({ - "key": "ModifyChapters", - "remove_chapters_patterns": remove_chapters_patterns, - "remove_sponsor_segments": opts.sponsorblock_remove, - "remove_ranges": remove_ranges, - "sponsorblock_chapter_title": opts.sponsorblock_chapter_title, - "force_keyframes": opts.force_keyframes_at_cuts, - }) + postprocessors.append( + { + "key": "ModifyChapters", + "remove_chapters_patterns": remove_chapters_patterns, + "remove_sponsor_segments": opts.sponsorblock_remove, + "remove_ranges": remove_ranges, + "sponsorblock_chapter_title": opts.sponsorblock_chapter_title, + "force_keyframes": opts.force_keyframes_at_cuts, + } + ) addchapters = getattr(opts, "addchapters", None) embed_infojson = getattr(opts, "embed_infojson", None) if opts.addmetadata or addchapters or embed_infojson: @@ -540,38 +541,45 @@ def legacy_postprocessors(opts, module, ytdlp, compat_opts): postprocessors.append(pp) if getattr(opts, "sponskrub", False) is not False: - postprocessors.append({ - "key": "SponSkrub", - "path": opts.sponskrub_path, - "args": opts.sponskrub_args, - "cut": opts.sponskrub_cut, - "force": opts.sponskrub_force, - "ignoreerror": opts.sponskrub is None, - "_from_cli": True, - }) + postprocessors.append( + { + "key": "SponSkrub", + "path": opts.sponskrub_path, + "args": opts.sponskrub_args, + "cut": opts.sponskrub_cut, + "force": opts.sponskrub_force, + "ignoreerror": opts.sponskrub is None, + "_from_cli": True, + } + ) if opts.embedthumbnail: - already_have_thumbnail = (opts.writethumbnail or - getattr(opts, "write_all_thumbnails", False)) - postprocessors.append({ - "key": "EmbedThumbnail", - "already_have_thumbnail": already_have_thumbnail, - }) + already_have_thumbnail = opts.writethumbnail or getattr(opts, "write_all_thumbnails", False) + postprocessors.append( + { + "key": "EmbedThumbnail", + "already_have_thumbnail": already_have_thumbnail, + } + ) if not already_have_thumbnail: opts.writethumbnail = True if isinstance(opts.outtmpl, dict): opts.outtmpl["pl_thumbnail"] = "" if getattr(opts, "split_chapters", None): - postprocessors.append({ - "key": "FFmpegSplitChapters", - "force_keyframes": opts.force_keyframes_at_cuts, - }) + postprocessors.append( + { + "key": "FFmpegSplitChapters", + "force_keyframes": opts.force_keyframes_at_cuts, + } + ) if opts.xattrs: postprocessors.append({"key": "XAttrMetadata"}) if opts.exec_cmd: - postprocessors.append({ - "key": "Exec", - "exec_cmd": opts.exec_cmd, - "when": "after_move", - }) + postprocessors.append( + { + "key": "Exec", + "exec_cmd": opts.exec_cmd, + "when": "after_move", + } + ) return postprocessors diff --git a/pyproject.toml b/pyproject.toml index fed528d4a7..a5a4d84b20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,128 @@ [build-system] -requires = ["setuptools"] -build-backend = "setuptools.build_meta" + build-backend = "setuptools.build_meta" + requires = ["setuptools"] + + +# ============================================================================ # +# Ruff # +# ============================================================================ # +# +# Reference: +# - [Configuration](https://docs.astral.sh/ruff/configuration/) +# +# - [Settings](https://docs.astral.sh/ruff/settings) +# - [Rules](https://docs.astral.sh/ruff/rules/) +# +[tool.ruff] + + # Set the maximum line length to 100 + # Reference: https://knox.codes/posts/line-length-limits + line-length = 100 + + # Target Python 3.9 (the latest supported version) + # Reference: https://devguide.python.org/versions/ + target-version = "py39" + +[tool.ruff.lint] + select = [ + "C4", # flake8-comprehensions + "COM", # flake8-commas + "E", # pycodestyle (errors) + "EXE", # flake8-executable + "F", # Pyflakes + "FA", # flake8-future-annotations + "FLY", # flynt + "FURB", # refurb + "G", # flake8-logging-format + "I", # isort + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PYI", # flake8-pyi + "Q", # flake8-quotes + "SIM", # flake8-simplify + "T10", # flake8-debugger + "TCH", # flake8-type-checking + "UP", # pyupgrade + + # The following checks are recommended but would require a lot of changes: + # "A", # flake8-builtins + # "ANN", # flake8-annotations + # "ARG", # flake8-unused-arguments + # "B", # flake8-bugbear + # "BLE", # flake8-blind-except + # "C", # mccabe (complexity) + # "D", # pydocstyle + # "DTZ", # flake8-datetimez + # "EM", # flake8-errmsg + # "ERA", # eradicate + # "FBT", # flake8-boolean-trap + # "INP", # flake8-no-pep420 + # "N", # pep8-naming + # "PL", # Pylint + # "PTH", # flake8-use-pathlib + # "RET", # flake8-return + # "RUF", # Ruff-specific rules + # "S", # flake8-bandit + # "SLF", # flake8-self + # "T20", # flake8-print + # "TD", # flake8-todos + # "W", # pycodestyle (warnings) + ] + + # ignore: + ignore = [ + "ANN101", # Missing type annotation for self in method (otherwise it's just t.Self) + "ANN102", # Missing type annotation for cls in classmethod + "COM812", # Trailing comma missing; ignored for compatibility with the Ruff formatter + "ISC001", # Implicitly concatenated string literals on one line; ignored for compatibility with the Ruff formatter + "SIM115", # Use a context manager for opening files; code is structured to return open file handle + ] + +[tool.ruff.lint.isort] + + # Each import should be on a separate line to minimize merge conflicts (see + # [reorder_python_imports](https://github.com/asottile/reorder_python_imports#why-this-style)). + force-single-line = true + known-first-party = ["gallery_dl"] + +# Ignore `E402` (import violations) in all `__init__.py` files, and in select subdirectories. +[tool.ruff.lint.per-file-ignores] + "gallery_dl/extractor/500px.py" = [ + "E501", # Line too long; file uses large query strings + ] + "gallery_dl/extractor/mangapark.py" = [ + "E501", # Line too long; file uses large query strings + ] + "test/**" = [ + "C4", # flake8-comprehensions + "COM", # flake8-commas + "E", # pycodestyle (errors) + "EXE", # flake8-executable + "F", # Pyflakes + "FA", # flake8-future-annotations + "FLY", # flynt + "FURB", # refurb + "G", # flake8-logging-format + "I", # isort + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PYI", # flake8-pyi + "Q", # flake8-quotes + "SIM", # flake8-simplify + "T10", # flake8-debugger + "TCH", # flake8-type-checking + "UP", # pyupgrade + ] + +[tool.ruff.lint.pydocstyle] + # Use Google-style docstrings (https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). + convention = "google" + +[tool.ruff.format] + # Enable auto-formatting of code examples in docstrings. Markdown, + # reStructuredText code/literal blocks and doctests are all supported. + docstring-code-format = true diff --git a/scripts/build_testresult_db.py b/scripts/build_testresult_db.py index 85b332e099..247ebdbc1f 100755 --- a/scripts/build_testresult_db.py +++ b/scripts/build_testresult_db.py @@ -1,26 +1,25 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """Collect results of extractor unit tests""" -import sys -import os.path import datetime +import os.path +import sys import util -from gallery_dl import extractor, job, config -from test.test_results import setup_test_config +from gallery_dl import config +from gallery_dl import extractor +from gallery_dl import job +from test.test_results import setup_test_config # filter test cases tests = [ (idx, extr, url, result) - for extr in extractor.extractors() if hasattr(extr, "test") and extr.test if len(sys.argv) <= 1 or extr.category in sys.argv - for idx, (url, result) in enumerate(extr._get_tests()) if result ] @@ -33,9 +32,8 @@ for idx, extr, url, result in tests: - # filename - name = "{}-{}-{}.json".format(extr.category, extr.subcategory, idx) + name = f"{extr.category}-{extr.subcategory}-{idx}.json" print(name) # config values @@ -46,7 +44,7 @@ key = key.split(".") config.set(key[:-1], key[-1], value) if "range" in result: - config.set((), "image-range" , result["range"]) + config.set((), "image-range", result["range"]) config.set((), "chapter-range", result["range"]) # write test data diff --git a/scripts/completion_bash.py b/scripts/completion_bash.py index b9921dc077..b921923280 100755 --- a/scripts/completion_bash.py +++ b/scripts/completion_bash.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019 Mike Fährmann # @@ -10,8 +9,8 @@ """Generate bash completion script from gallery-dl's argument parser""" import util -from gallery_dl import option +from gallery_dl import option TEMPLATE = """_gallery_dl() { @@ -36,7 +35,6 @@ diropts = [] fileopts = [] for action in option.build_parser()._actions: - if action.metavar in ("DEST",): diropts.extend(action.option_strings) @@ -49,8 +47,11 @@ PATH = util.path("data/completion/gallery-dl") with util.lazy(PATH) as fp: - fp.write(TEMPLATE % { - "opts" : " ".join(opts), - "diropts" : "|".join(diropts), - "fileopts": "|".join(fileopts), - }) + fp.write( + TEMPLATE + % { + "opts": " ".join(opts), + "diropts": "|".join(diropts), + "fileopts": "|".join(fileopts), + } + ) diff --git a/scripts/completion_fish.py b/scripts/completion_fish.py index a5dd47b957..92c7c0cb81 100755 --- a/scripts/completion_fish.py +++ b/scripts/completion_fish.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -8,8 +7,8 @@ """Generate fish completion script from gallery-dl's argument parser""" import util -from gallery_dl import option +from gallery_dl import option TEMPLATE = """complete -c gallery-dl -x %(opts)s diff --git a/scripts/completion_zsh.py b/scripts/completion_zsh.py index d96ed7334d..a617daa218 100755 --- a/scripts/completion_zsh.py +++ b/scripts/completion_zsh.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2020 Mike Fährmann # @@ -9,8 +8,10 @@ """Generate zsh completion script from gallery-dl's argument parser""" -import util import argparse + +import util + from gallery_dl import option TEMPLATE = """#compdef gallery-dl @@ -25,16 +26,17 @@ return rc """ -TR = str.maketrans({ - "'": "'\\''", - "[": "\\[", - "]": "\\]", -}) +TR = str.maketrans( + { + "'": "'\\''", + "[": "\\[", + "]": "\\]", + } +) opts = [] for action in option.build_parser()._actions: - if not action.option_strings or action.help == argparse.SUPPRESS: continue elif len(action.option_strings) == 1: diff --git a/scripts/create_test_data.py b/scripts/create_test_data.py index 34d585ff44..eac28e8092 100755 --- a/scripts/create_test_data.py +++ b/scripts/create_test_data.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2015-2019 Mike Fährmann # @@ -11,10 +10,9 @@ import argparse -import util # noqa from gallery_dl import extractor -from test.test_results import ResultJob, setup_test_config - +from test.test_results import ResultJob +from test.test_results import setup_test_config TESTDATA_FMT = """ test = ("{}", {{ @@ -41,7 +39,8 @@ def main(): if args.recreate: urls = [ test[0] - for extr in extractor.extractors() if extr.category in args.urls + for extr in extractor.extractors() + if extr.category in args.urls for test in extr.test ] else: @@ -58,12 +57,14 @@ def main(): data = (exc.__class__.__name__,) else: fmt = TESTDATA_FMT - data = (tjob.url_hash.hexdigest(), - tjob.kwdict_hash.hexdigest(), - tjob.content_hash.hexdigest()) + data = ( + tjob.url_hash.hexdigest(), + tjob.kwdict_hash.hexdigest(), + tjob.content_hash.hexdigest(), + ) print(tjob.extractor.__class__.__name__) print(fmt.format(url, *data)) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/scripts/export_tests.py b/scripts/export_tests.py index 7ea62852ce..4378218010 100755 --- a/scripts/export_tests.py +++ b/scripts/export_tests.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2023 Mike Fährmann # @@ -7,18 +6,18 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import collections +import itertools import os import re import sys -import itertools -import collections import util from pyprint import pyprint -from gallery_dl import extractor +from gallery_dl import extractor -FORMAT = '''\ +FORMAT = """\ # -*- coding: utf-8 -*- # This program is free software; you can redistribute it and/or modify @@ -31,34 +30,32 @@ __tests__ = ( {tests}\ ) -''' +""" def extract_tests_from_source(lines): tests = {} - match_url = re.compile( - r''' (?:test = | )?\(\(?"([^"]+)"(.*)''').match - match_end = re.compile( - r" (\}\)| \}\),)\n$").match + match_url = re.compile(r""" (?:test = | )?\(\(?"([^"]+)"(.*)""").match + match_end = re.compile(r" (\}\)| \}\),)\n$").match first = 0 url = "" for index, line in enumerate(lines): if first and match_end(line): - tests[url] = lines[first-1:index+1] + tests[url] = lines[first - 1 : index + 1] first = 0 - elif (m := match_url(line)): + elif m := match_url(line): offset = index while not m[2]: offset += 1 next = lines[offset] - line = line[:-2] + next[next.index('"')+1:] + line = line[:-2] + next[next.index('"') + 1 :] m = match_url(line) url = m[1] if m[2] in (",)", "),"): - tests[url] = lines[index-1:index+1] + tests[url] = lines[index - 1 : index + 1] first = 0 else: first = index @@ -88,12 +85,10 @@ def build_test(extr, data): comment = comment_from_source(source) head = { - "#url" : extr.url, - "#comment" : comment.replace('"', "'"), - "#category": (extr.basecategory, - extr.category, - extr.subcategory), - "#class" : extr.__class__, + "#url": extr.url, + "#comment": comment.replace('"', "'"), + "#category": (extr.basecategory, extr.category, extr.subcategory), + "#class": extr.__class__, } if not comment: @@ -103,19 +98,16 @@ def build_test(extr, data): if not data: data = {} - if (options := data.pop("options", None)): - instr["#options"] = { - name: value - for name, value in options - } - if (pattern := data.pop("pattern", None)): + if options := data.pop("options", None): + instr["#options"] = dict(options) + if pattern := data.pop("pattern", None): if pattern in PATTERNS: cls = PATTERNS[pattern] pattern = f"lit:{pyprint(cls)}.pattern" instr["#pattern"] = pattern - if (exception := data.pop("exception", None)): + if exception := data.pop("exception", None): instr["#exception"] = exception - if (range := data.pop("range", None)): + if range := data.pop("range", None): instr["#range"] = range if (count := data.pop("count", None)) is not None: instr["#count"] = count @@ -123,13 +115,12 @@ def build_test(extr, data): instr["#archive"] = archive if (extractor := data.pop("extractor", None)) is not None: instr["#extractor"] = extractor - if (url := data.pop("url", None)): + if url := data.pop("url", None): instr["#sha1_url"] = url - if (metadata := data.pop("keyword", None)): - if isinstance(metadata, str) and len(metadata) == 40: - instr["#sha1_metadata"] = metadata - metadata = {} - if (content := data.pop("content", None)): + if metadata := data.pop("keyword", None) and isinstance(metadata, str) and len(metadata) == 40: # noqa: F821 + instr["#sha1_metadata"] = metadata + metadata = {} + if content := data.pop("content", None): if isinstance(content, tuple): content = list(content) instr["#sha1_content"] = content @@ -144,10 +135,7 @@ def build_test(extr, data): def collect_patterns(): - return { - cls.pattern.pattern: cls - for cls in extractor._list_classes() - } + return {cls.pattern.pattern: cls for cls in extractor._list_classes()} def collect_tests(whitelist=None): @@ -155,7 +143,6 @@ def collect_tests(whitelist=None): for cls in extractor._list_classes(): for url, data in cls._get_tests(): - extr = cls.from_url(url) if whitelist and extr.category not in whitelist: continue @@ -170,7 +157,6 @@ def export_tests(data): tests = [] for head, instr, metadata in data: - for v in itertools.chain( head.values(), instr.values() if instr else (), @@ -181,9 +167,9 @@ def export_tests(data): module, _, name = v.__module__.rpartition(".") if name[0].isdecimal(): - stmt = f'''\ + stmt = f"""\ {module.partition(".")[0]} = __import__("{v.__module__}") -_{name} = getattr({module}, "{name}")''' +_{name} = getattr({module}, "{name}")""" elif module: stmt = f"from {module} import {name}" else: @@ -217,11 +203,14 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument( - "-t", "--target", + "-t", + "--target", help="target directory", ) parser.add_argument( - "-c", "--category", action="append", + "-c", + "--category", + action="append", help="extractor categories to export", ) @@ -230,7 +219,8 @@ def main(): if not args.target: args.target = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "test", "results", + "test", + "results", ) global PATTERNS diff --git a/scripts/hook-gallery_dl.py b/scripts/hook-gallery_dl.py index ae51b068a2..f89d708bbd 100644 --- a/scripts/hook-gallery_dl.py +++ b/scripts/hook-gallery_dl.py @@ -1,6 +1,6 @@ -# -*- coding: utf-8 -*- - -from gallery_dl import extractor, downloader, postprocessor +from gallery_dl import downloader +from gallery_dl import extractor +from gallery_dl import postprocessor hiddenimports = [ package.__name__ + "." + module diff --git a/scripts/man.py b/scripts/man.py index 2aff221d8f..e9ea947546 100755 --- a/scripts/man.py +++ b/scripts/man.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019-2020 Mike Fährmann # @@ -9,16 +8,16 @@ """Generate man pages""" -import re import datetime +import re import util + import gallery_dl.option import gallery_dl.version def build_gallery_dl_1(path=None): - OPTS_FMT = """.TP\n.B "{}" {}\n{}""" TEMPLATE = r""" @@ -92,24 +91,28 @@ def build_gallery_dl_1(path=None): for action in gallery_dl.option.build_parser()._actions: if action.help.startswith("=="): continue - options.append(OPTS_FMT.format( - ", ".join(action.option_strings).replace("-", r"\-"), - r"\f[I]{}\f[]".format(action.metavar) if action.metavar else "", - action.help, - )) + options.append( + OPTS_FMT.format( + ", ".join(action.option_strings).replace("-", r"\-"), + rf"\f[I]{action.metavar}\f[]" if action.metavar else "", + action.help, + ) + ) if not path: path = util.path("data/man/gallery-dl.1") with util.lazy(path) as fp: - fp.write(TEMPLATE.lstrip() % { - "options": "\n".join(options), - "version": gallery_dl.version.__version__, - "date" : datetime.datetime.now().strftime("%Y-%m-%d"), - }) + fp.write( + TEMPLATE.lstrip() + % { + "options": "\n".join(options), + "version": gallery_dl.version.__version__, + "date": datetime.datetime.now().strftime("%Y-%m-%d"), + } + ) def build_gallery_dl_conf_5(path=None): - TEMPLATE = r""" .TH "GALLERY-DL.CONF" "5" "%(date)s" "%(version)s" "gallery-dl Manual" .\" disable hyphenation @@ -210,24 +213,26 @@ def build_gallery_dl_conf_5(path=None): for field, text in option.items(): if field in ("Type", "Default"): - content.append('.IP "{}:" {}'.format(field, len(field)+2)) + content.append(f'.IP "{field}:" {len(field) + 2}') content.append(strip_rst(text)) else: - content.append('.IP "{}:" 4'.format(field)) + content.append(f'.IP "{field}:" 4') content.append(strip_rst(text, field != "Example")) if not path: path = util.path("data/man/gallery-dl.conf.5") with util.lazy(path) as fp: - fp.write(TEMPLATE.lstrip() % { - "options": "\n".join(content), - "version": gallery_dl.version.__version__, - "date" : datetime.datetime.now().strftime("%Y-%m-%d"), - }) + fp.write( + TEMPLATE.lstrip() + % { + "options": "\n".join(content), + "version": gallery_dl.version.__version__, + "date": datetime.datetime.now().strftime("%Y-%m-%d"), + } + ) def parse_docs_configuration(): - doc_path = util.path("docs", "configuration.rst") with open(doc_path, encoding="utf-8") as fp: doc_lines = fp.readlines() @@ -240,12 +245,11 @@ def parse_docs_configuration(): name = None last = None for line in doc_lines: - if line[0] == ".": continue # start of new section - elif re.match(r"^=+$", line): + if re.match(r"^=+$", line): if sec_name and options: sections[sec_name] = options sec_name = last.strip() @@ -284,7 +288,6 @@ def parse_docs_configuration(): def strip_rst(text, extended=True, *, ITALIC=r"\\f[I]\1\\f[]", REGULAR=r"\1"): - text = text.replace("\\", "\\\\") # ``foo`` diff --git a/scripts/options.py b/scripts/options.py index f9f0ace1cf..b33df31b91 100755 --- a/scripts/options.py +++ b/scripts/options.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2023 Mike Fährmann # @@ -16,9 +15,9 @@ import util import gallery_dl.util -gallery_dl.util.EXECUTABLE = True -from gallery_dl import option # noqa E402 +gallery_dl.util.EXECUTABLE = True +from gallery_dl import option # noqa: E402 TEMPLATE = """# Command-Line Options @@ -36,10 +35,11 @@ opts = opts.replace("\n ", "\n ") # indent by 4 -PATH = (sys.argv[1] if len(sys.argv) > 1 else - util.path("docs", "options.md")) +PATH = sys.argv[1] if len(sys.argv) > 1 else util.path("docs", "options.md") with util.lazy(PATH) as fp: - fp.write(TEMPLATE.format( - "/".join(os.path.normpath(__file__).split(os.sep)[-2:]), - opts, - )) + fp.write( + TEMPLATE.format( + "/".join(os.path.normpath(__file__).split(os.sep)[-2:]), + opts, + ) + ) diff --git a/scripts/pyinstaller.py b/scripts/pyinstaller.py index 58303547a3..7c6c08ad71 100755 --- a/scripts/pyinstaller.py +++ b/scripts/pyinstaller.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """Build a standalone executable using PyInstaller""" import argparse -import util import sys +import util + def main(): parser = argparse.ArgumentParser() @@ -34,21 +34,29 @@ def main(): name = "gallery-dl" if label: - name = "{}_{}".format(name, label) + name = f"{name}_{label}" if args.extension: - name = "{}.{}".format(name, args.extension.lower()) + name = f"{name}.{args.extension.lower()}" import PyInstaller.__main__ - return PyInstaller.__main__.run([ - "--onefile", - "--console", - "--name", name, - "--additional-hooks-dir", util.path("scripts"), - "--distpath", util.path("dist"), - "--workpath", util.path("build"), - "--specpath", util.path("build"), - util.path("gallery_dl", "__main__.py"), - ]) + + return PyInstaller.__main__.run( + [ + "--onefile", + "--console", + "--name", + name, + "--additional-hooks-dir", + util.path("scripts"), + "--distpath", + util.path("dist"), + "--workpath", + util.path("build"), + "--specpath", + util.path("build"), + util.path("gallery_dl", "__main__.py"), + ] + ) if __name__ == "__main__": diff --git a/scripts/pyprint.py b/scripts/pyprint.py index 99d31e41ed..369dad4146 100644 --- a/scripts/pyprint.py +++ b/scripts/pyprint.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2024 Mike Fährmann # @@ -11,15 +10,11 @@ def pyprint(obj, indent=0, lmin=9, lmax=16): - if isinstance(obj, str): if obj.startswith("lit:"): - return f'''{obj[4:]}''' + return f"""{obj[4:]}""" - if "\\" in obj or obj.startswith("re:"): - prefix = "r" - else: - prefix = "" + prefix = "r" if "\\" in obj or obj.startswith("re:") else "" if "\n" in obj: quote = '"""' @@ -29,19 +24,19 @@ def pyprint(obj, indent=0, lmin=9, lmax=16): else: quote = '"' - return f'''{prefix}{quote}{obj}{quote}''' + return f"""{prefix}{quote}{obj}{quote}""" if isinstance(obj, bytes): return f'''b"{str(obj)[2:-1]}"''' if isinstance(obj, type): if obj.__module__ == "builtins": - return f'''{obj.__name__}''' + return f"""{obj.__name__}""" name = obj.__module__.rpartition(".")[2] if name[0].isdecimal(): name = f"_{name}" - return f'''{name}.{obj.__name__}''' + return f"""{name}.{obj.__name__}""" if isinstance(obj, dict): if not obj: @@ -65,14 +60,13 @@ def pyprint(obj, indent=0, lmin=9, lmax=16): else: lines.append( f'''{ws} "{key}"''' - f'''{' '*(lkey - len(key))}: ''' - f'''{pyprint(value, indent+4)},''' + f"""{' '*(lkey - len(key))}: """ + f"""{pyprint(value, indent+4)},""" ) - lines.append(f'''{ws}}}''') + lines.append(f"""{ws}}}""") return "\n".join(lines) - else: - key, value = obj.popitem() - return f'''{{"{key}": {pyprint(value)}}}''' + key, value = obj.popitem() + return f"""{{"{key}": {pyprint(value)}}}""" if isinstance(obj, list): if not obj: @@ -83,19 +77,14 @@ def pyprint(obj, indent=0, lmin=9, lmax=16): lines = [] lines.append("[") - lines.extend( - f'''{ws} {pyprint(value, indent+4)},''' - for value in obj - ) - lines.append(f'''{ws}]''') + lines.extend(f"""{ws} {pyprint(value, indent+4)},""" for value in obj) + lines.append(f"""{ws}]""") return "\n".join(lines) - else: - return f'''[{pyprint(obj[0])}]''' + return f"""[{pyprint(obj[0])}]""" if isinstance(obj, tuple): if len(obj) == 1: - return f'''({pyprint(obj[0], indent+4)},)''' - return f'''({", ".join(pyprint(v, indent+4) for v in obj)})''' + return f"""({pyprint(obj[0], indent+4)},)""" + return f"""({", ".join(pyprint(v, indent+4) for v in obj)})""" - else: - return f'''{obj}''' + return f"""{obj}""" diff --git a/scripts/run_tests.py b/scripts/run_tests.py index d1fd1f14cf..ff4bc643d1 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- +# !/usr/bin/env python3 # Copyright 2021 Mike Fährmann # @@ -11,8 +10,7 @@ import sys import unittest -TEST_DIRECTORY = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "test") +TEST_DIRECTORY = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "test") sys.path.insert(0, TEST_DIRECTORY) @@ -23,10 +21,7 @@ if file.startswith("test_") and file != "test_results.py" ] else: - TESTS = [ - name if name.startswith("test_") else "test_" + name - for name in sys.argv[1:] - ] + TESTS = [name if name.startswith("test_") else "test_" + name for name in sys.argv[1:]] suite = unittest.TestSuite() diff --git a/scripts/supportedsites.py b/scripts/supportedsites.py index 4ba5a763f7..ba98d9bd9c 100755 --- a/scripts/supportedsites.py +++ b/scripts/supportedsites.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -7,11 +6,12 @@ """Generate a Markdown document listing all supported sites""" +import collections import os import sys -import collections import util + from gallery_dl import extractor try: @@ -21,175 +21,174 @@ CATEGORY_MAP = { - "2chan" : "Futaba Channel", - "35photo" : "35PHOTO", - "adultempire" : "Adult Empire", - "agnph" : "AGNPH", - "allgirlbooru" : "All girl", - "ao3" : "Archive of Our Own", - "archivedmoe" : "Archived.Moe", - "archiveofsins" : "Archive of Sins", - "artstation" : "ArtStation", - "aryion" : "Eka's Portal", - "atfbooru" : "ATFBooru", - "azurlanewiki" : "Azur Lane Wiki", - "b4k" : "arch.b4k.co", - "baraag" : "baraag", - "batoto" : "BATO.TO", - "bbc" : "BBC", - "cien" : "Ci-en", - "cohost" : "cohost!", - "comicvine" : "Comic Vine", - "coomerparty" : "Coomer", - "deltaporno" : "DeltaPorno", - "deviantart" : "DeviantArt", - "drawfriends" : "Draw Friends", - "dynastyscans" : "Dynasty Reader", - "e621" : "e621", - "e926" : "e926", - "e6ai" : "e6AI", - "erome" : "EroMe", - "everia" : "EVERIA.CLUB", - "e-hentai" : "E-Hentai", - "exhentai" : "ExHentai", - "fallenangels" : "Fallen Angels Scans", - "fanbox" : "pixivFANBOX", - "fashionnova" : "Fashion Nova", - "furaffinity" : "Fur Affinity", - "hatenablog" : "HatenaBlog", - "hbrowse" : "HBrowse", - "hentai2read" : "Hentai2Read", - "hentaicosplays" : "Hentai Cosplay", - "hentaifoundry" : "Hentai Foundry", - "hentaifox" : "HentaiFox", - "hentaihand" : "HentaiHand", - "hentaihere" : "HentaiHere", - "hentaiimg" : "Hentai Image", - "hentainexus" : "HentaiNexus", - "hiperdex" : "Hipertoon", - "hitomi" : "Hitomi.la", - "horne" : "horne", - "idolcomplex" : "Idol Complex", + "2chan": "Futaba Channel", + "35photo": "35PHOTO", + "adultempire": "Adult Empire", + "agnph": "AGNPH", + "allgirlbooru": "All girl", + "ao3": "Archive of Our Own", + "archivedmoe": "Archived.Moe", + "archiveofsins": "Archive of Sins", + "artstation": "ArtStation", + "aryion": "Eka's Portal", + "atfbooru": "ATFBooru", + "azurlanewiki": "Azur Lane Wiki", + "b4k": "arch.b4k.co", + "baraag": "baraag", + "batoto": "BATO.TO", + "bbc": "BBC", + "cien": "Ci-en", + "cohost": "cohost!", + "comicvine": "Comic Vine", + "coomerparty": "Coomer", + "deltaporno": "DeltaPorno", + "deviantart": "DeviantArt", + "drawfriends": "Draw Friends", + "dynastyscans": "Dynasty Reader", + "e621": "e621", + "e926": "e926", + "e6ai": "e6AI", + "erome": "EroMe", + "everia": "EVERIA.CLUB", + "e-hentai": "E-Hentai", + "exhentai": "ExHentai", + "fallenangels": "Fallen Angels Scans", + "fanbox": "pixivFANBOX", + "fashionnova": "Fashion Nova", + "furaffinity": "Fur Affinity", + "hatenablog": "HatenaBlog", + "hbrowse": "HBrowse", + "hentai2read": "Hentai2Read", + "hentaicosplays": "Hentai Cosplay", + "hentaifoundry": "Hentai Foundry", + "hentaifox": "HentaiFox", + "hentaihand": "HentaiHand", + "hentaihere": "HentaiHere", + "hentaiimg": "Hentai Image", + "hentainexus": "HentaiNexus", + "hiperdex": "Hipertoon", + "hitomi": "Hitomi.la", + "horne": "horne", + "idolcomplex": "Idol Complex", "illusioncardsbooru": "Illusion Game Cards", - "imagebam" : "ImageBam", - "imagefap" : "ImageFap", - "imgbb" : "ImgBB", - "imgbox" : "imgbox", - "imagechest" : "ImageChest", - "imgkiwi" : "IMG.Kiwi", - "imgth" : "imgth", - "imgur" : "imgur", - "joyreactor" : "JoyReactor", - "itchio" : "itch.io", - "jpgfish" : "JPG Fish", - "kabeuchi" : "かべうち", - "kemonoparty" : "Kemono", - "koharu" : "SchaleNetwork", - "livedoor" : "livedoor Blog", - "ohpolly" : "Oh Polly", + "imagebam": "ImageBam", + "imagefap": "ImageFap", + "imgbb": "ImgBB", + "imgbox": "imgbox", + "imagechest": "ImageChest", + "imgkiwi": "IMG.Kiwi", + "imgth": "imgth", + "imgur": "imgur", + "joyreactor": "JoyReactor", + "itchio": "itch.io", + "jpgfish": "JPG Fish", + "kabeuchi": "かべうち", + "kemonoparty": "Kemono", + "koharu": "SchaleNetwork", + "livedoor": "livedoor Blog", + "ohpolly": "Oh Polly", "omgmiamiswimwear": "Omg Miami Swimwear", - "mangadex" : "MangaDex", - "mangafox" : "Manga Fox", - "mangahere" : "Manga Here", - "mangakakalot" : "MangaKakalot", - "mangalife" : "MangaLife", - "manganelo" : "Manganato", - "mangapark" : "MangaPark", - "mangaread" : "MangaRead", - "mangasee" : "MangaSee", - "mariowiki" : "Super Mario Wiki", + "mangadex": "MangaDex", + "mangafox": "Manga Fox", + "mangahere": "Manga Here", + "mangakakalot": "MangaKakalot", + "mangalife": "MangaLife", + "manganelo": "Manganato", + "mangapark": "MangaPark", + "mangaread": "MangaRead", + "mangasee": "MangaSee", + "mariowiki": "Super Mario Wiki", "mastodon.social": "mastodon.social", - "mediawiki" : "MediaWiki", - "micmicidol" : "MIC MIC IDOL", + "mediawiki": "MediaWiki", + "micmicidol": "MIC MIC IDOL", "myhentaigallery": "My Hentai Gallery", - "myportfolio" : "Adobe Portfolio", - "naverwebtoon" : "NaverWebtoon", - "nhentai" : "nhentai", - "nijie" : "nijie", - "nozomi" : "Nozomi.la", - "nsfwalbum" : "NSFWalbum.com", - "paheal" : "rule #34", - "photovogue" : "PhotoVogue", - "pidgiwiki" : "PidgiWiki", - "pixeldrain" : "pixeldrain", - "pornimagesxxx" : "Porn Image", - "pornpics" : "PornPics.com", - "pornreactor" : "PornReactor", + "myportfolio": "Adobe Portfolio", + "naverwebtoon": "NaverWebtoon", + "nhentai": "nhentai", + "nijie": "nijie", + "nozomi": "Nozomi.la", + "nsfwalbum": "NSFWalbum.com", + "paheal": "rule #34", + "photovogue": "PhotoVogue", + "pidgiwiki": "PidgiWiki", + "pixeldrain": "pixeldrain", + "pornimagesxxx": "Porn Image", + "pornpics": "PornPics.com", + "pornreactor": "PornReactor", "readcomiconline": "Read Comic Online", - "rbt" : "RebeccaBlackTech", - "redgifs" : "RedGIFs", - "rozenarcana" : "Rozen Arcana", - "rule34" : "Rule 34", - "rule34hentai" : "Rule34Hentai", - "rule34us" : "Rule 34", - "rule34vault" : "R34 Vault", - "rule34xyz" : "Rule 34 XYZ", - "sankaku" : "Sankaku Channel", - "sankakucomplex" : "Sankaku Complex", - "seiga" : "Niconico Seiga", - "senmanga" : "Sen Manga", - "sensescans" : "Sense-Scans", - "sexcom" : "Sex.com", - "simplyhentai" : "Simply Hentai", - "slickpic" : "SlickPic", - "slideshare" : "SlideShare", - "smugmug" : "SmugMug", - "speakerdeck" : "Speaker Deck", - "steamgriddb" : "SteamGridDB", - "subscribestar" : "SubscribeStar", - "tbib" : "The Big ImageBoard", - "tcbscans" : "TCB Scans", - "tco" : "Twitter t.co", - "tmohentai" : "TMOHentai", - "thatpervert" : "ThatPervert", - "thebarchive" : "The /b/ Archive", - "thecollection" : "The /co/llection", - "tumblrgallery" : "TumblrGallery", - "vanillarock" : "もえぴりあ", - "vidyart2" : "/v/idyart2", - "vidyapics" : "Vidya Booru", - "vk" : "VK", - "vsco" : "VSCO", - "wallpapercave" : "Wallpaper Cave", - "webmshare" : "webmshare", - "webtoons" : "Webtoon", - "wikiart" : "WikiArt.org", - "wikigg" : "wiki.gg", + "rbt": "RebeccaBlackTech", + "redgifs": "RedGIFs", + "rozenarcana": "Rozen Arcana", + "rule34": "Rule 34", + "rule34hentai": "Rule34Hentai", + "rule34us": "Rule 34", + "rule34vault": "R34 Vault", + "rule34xyz": "Rule 34 XYZ", + "sankaku": "Sankaku Channel", + "sankakucomplex": "Sankaku Complex", + "seiga": "Niconico Seiga", + "senmanga": "Sen Manga", + "sensescans": "Sense-Scans", + "sexcom": "Sex.com", + "simplyhentai": "Simply Hentai", + "slickpic": "SlickPic", + "slideshare": "SlideShare", + "smugmug": "SmugMug", + "speakerdeck": "Speaker Deck", + "steamgriddb": "SteamGridDB", + "subscribestar": "SubscribeStar", + "tbib": "The Big ImageBoard", + "tcbscans": "TCB Scans", + "tco": "Twitter t.co", + "tmohentai": "TMOHentai", + "thatpervert": "ThatPervert", + "thebarchive": "The /b/ Archive", + "thecollection": "The /co/llection", + "tumblrgallery": "TumblrGallery", + "vanillarock": "もえぴりあ", + "vidyart2": "/v/idyart2", + "vidyapics": "Vidya Booru", + "vk": "VK", + "vsco": "VSCO", + "wallpapercave": "Wallpaper Cave", + "webmshare": "webmshare", + "webtoons": "Webtoon", + "wikiart": "WikiArt.org", + "wikigg": "wiki.gg", "wikimediacommons": "Wikimedia Commons", - "xbunkr" : "xBunkr", - "xhamster" : "xHamster", - "xvideos" : "XVideos", - "yandere" : "yande.re", + "xbunkr": "xBunkr", + "xhamster": "xHamster", + "xvideos": "XVideos", + "yandere": "yande.re", } SUBCATEGORY_MAP = { - "" : "", - "art" : "Art", - "audio" : "Audio", - "doujin" : "Doujin", - "home" : "Home Feed", - "image" : "individual Images", - "index" : "Site Index", - "info" : "User Profile Information", - "issue" : "Comic Issues", - "manga" : "Manga", - "media" : "Media Files", - "note" : "Images from Notes", + "": "", + "art": "Art", + "audio": "Audio", + "doujin": "Doujin", + "home": "Home Feed", + "image": "individual Images", + "index": "Site Index", + "info": "User Profile Information", + "issue": "Comic Issues", + "manga": "Manga", + "media": "Media Files", + "note": "Images from Notes", "popular": "Popular Images", - "recent" : "Recent Images", - "search" : "Search Results", - "status" : "Images from Statuses", - "tag" : "Tag Searches", - "tweets" : "", - "user" : "User Profiles", - "watch" : "Watches", - "following" : "Followed Users", - "related-pin" : "related Pins", + "recent": "Recent Images", + "search": "Search Results", + "status": "Images from Statuses", + "tag": "Tag Searches", + "tweets": "", + "user": "User Profiles", + "watch": "Watches", + "following": "Followed Users", + "related-pin": "related Pins", "related-board": "", - "ao3": { - "user-works" : "", - "user-series" : "", + "user-works": "", + "user-series": "", "user-bookmark": "Bookmarks", }, "artstation": { @@ -210,25 +209,25 @@ "images": "Image Listings", "user-models": "User Models", "user-images": "User Images", - "user-posts" : "User Posts", + "user-posts": "User Posts", }, "coomerparty": { - "discord" : "", + "discord": "", "discord-server": "", - "posts" : "", + "posts": "", }, "desktopography": { "site": "", }, "deviantart": { "gallery-search": "Gallery Searches", - "stash" : "Sta.sh", + "stash": "Sta.sh", "status": "Status Updates", "watch-posts": "", }, "fanbox": { "supporting": "Supported User Feed", - "redirect" : "Pixiv Redirects", + "redirect": "Pixiv Redirects", }, "fapello": { "path": ["Videos", "Trending Posts", "Popular Videos", "Top Models"], @@ -238,7 +237,7 @@ }, "hatenablog": { "archive": "Archive", - "entry" : "Individual Posts", + "entry": "Individual Posts", }, "hentaifoundry": { "story": "", @@ -255,19 +254,19 @@ "tagged": "Tagged Posts", }, "kemonoparty": { - "discord" : "Discord Servers", + "discord": "Discord Servers", "discord-server": "", - "posts" : "", + "posts": "", }, "lensdump": { "albums": "", }, "mangadex": { - "feed" : "Followed Feed", + "feed": "Followed Feed", }, "nijie": { "followed": "Followed Users", - "nuita" : "Nuita History", + "nuita": "Nuita History", }, "pinterest": { "board": "", @@ -276,7 +275,7 @@ "allpins": "All Pins", }, "pixiv": { - "me" : "pixiv.me Links", + "me": "pixiv.me Links", "novel-bookmark": "Novel Bookmarks", "novel-series": "Novel Series", "novel-user": "", @@ -293,8 +292,8 @@ }, "raddle": { "usersubmissions": "User Profiles", - "post" : "Individual Posts", - "shorturl" : "", + "post": "Individual Posts", + "shorturl": "", }, "redgifs": { "collections": "", @@ -309,7 +308,7 @@ "pins": "User Pins", }, "skeb": { - "following" : "Followed Creators", + "following": "Followed Creators", "following-users": "Followed Users", }, "smugmug": { @@ -336,13 +335,13 @@ }, "wallhaven": { "collections": "", - "uploads" : "", + "uploads": "", }, "wallpapercave": { "image": ["individual Images", "Search Results"], }, "weasyl": { - "journals" : "", + "journals": "", "submissions": "", }, "weibo": { @@ -358,88 +357,94 @@ } BASE_MAP = { - "E621" : "e621 Instances", - "foolfuuka" : "FoolFuuka 4chan Archives", - "foolslide" : "FoOlSlide Instances", + "E621": "e621 Instances", + "foolfuuka": "FoolFuuka 4chan Archives", + "foolslide": "FoOlSlide Instances", "gelbooru_v01": "Gelbooru Beta 0.1.11", "gelbooru_v02": "Gelbooru Beta 0.2", - "jschan" : "jschan Imageboards", - "lolisafe" : "lolisafe and chibisafe", - "lynxchan" : "LynxChan Imageboards", - "moebooru" : "Moebooru and MyImouto", - "szurubooru" : "szurubooru Instances", + "jschan": "jschan Imageboards", + "lolisafe": "lolisafe and chibisafe", + "lynxchan": "LynxChan Imageboards", + "moebooru": "Moebooru and MyImouto", + "szurubooru": "szurubooru Instances", "urlshortener": "URL Shorteners", - "vichan" : "vichan Imageboards", + "vichan": "vichan Imageboards", } URL_MAP = { - "blogspot" : "https://www.blogger.com/", + "blogspot": "https://www.blogger.com/", "wikimedia": "https://www.wikimedia.org/", } _OAUTH = 'OAuth' _COOKIES = 'Cookies' -_APIKEY_DB = ('API Key') -_APIKEY_WH = ('API Key') -_APIKEY_WY = ('API Key') +_APIKEY_DB = ( + 'API Key' +) +_APIKEY_WH = ( + 'API Key' +) +_APIKEY_WY = ( + 'API Key' +) AUTH_MAP = { - "aibooru" : "Supported", - "ao3" : "Supported", - "aryion" : "Supported", - "atfbooru" : "Supported", - "baraag" : _OAUTH, - "bluesky" : "Supported", - "booruvar" : "Supported", - "boosty" : _COOKIES, - "coomerparty" : "Supported", - "danbooru" : "Supported", - "derpibooru" : _APIKEY_DB, - "deviantart" : _OAUTH, - "e621" : "Supported", - "e6ai" : "Supported", - "e926" : "Supported", - "e-hentai" : "Supported", - "exhentai" : "Supported", - "fanbox" : _COOKIES, - "fantia" : _COOKIES, - "flickr" : _OAUTH, - "furaffinity" : _COOKIES, - "furbooru" : "API Key", - "horne" : "Required", - "idolcomplex" : "Supported", - "imgbb" : "Supported", - "inkbunny" : "Supported", - "instagram" : _COOKIES, - "kemonoparty" : "Supported", - "mangadex" : "Supported", - "mangoxo" : "Supported", + "aibooru": "Supported", + "ao3": "Supported", + "aryion": "Supported", + "atfbooru": "Supported", + "baraag": _OAUTH, + "bluesky": "Supported", + "booruvar": "Supported", + "boosty": _COOKIES, + "coomerparty": "Supported", + "danbooru": "Supported", + "derpibooru": _APIKEY_DB, + "deviantart": _OAUTH, + "e621": "Supported", + "e6ai": "Supported", + "e926": "Supported", + "e-hentai": "Supported", + "exhentai": "Supported", + "fanbox": _COOKIES, + "fantia": _COOKIES, + "flickr": _OAUTH, + "furaffinity": _COOKIES, + "furbooru": "API Key", + "horne": "Required", + "idolcomplex": "Supported", + "imgbb": "Supported", + "inkbunny": "Supported", + "instagram": _COOKIES, + "kemonoparty": "Supported", + "mangadex": "Supported", + "mangoxo": "Supported", "mastodon.social": _OAUTH, - "newgrounds" : "Supported", - "nijie" : "Required", - "patreon" : _COOKIES, - "pawoo" : _OAUTH, - "pillowfort" : "Supported", - "pinterest" : _COOKIES, - "pixiv" : _OAUTH, - "ponybooru" : "API Key", - "reddit" : _OAUTH, - "sankaku" : "Supported", - "scrolller" : "Supported", - "seiga" : "Supported", - "smugmug" : _OAUTH, - "subscribestar" : "Supported", - "tapas" : "Supported", - "tsumino" : "Supported", - "tumblr" : _OAUTH, - "twitter" : "Supported", - "vipergirls" : "Supported", - "wallhaven" : _APIKEY_WH, - "weasyl" : _APIKEY_WY, - "zerochan" : "Supported", + "newgrounds": "Supported", + "nijie": "Required", + "patreon": _COOKIES, + "pawoo": _OAUTH, + "pillowfort": "Supported", + "pinterest": _COOKIES, + "pixiv": _OAUTH, + "ponybooru": "API Key", + "reddit": _OAUTH, + "sankaku": "Supported", + "scrolller": "Supported", + "seiga": "Supported", + "smugmug": _OAUTH, + "subscribestar": "Supported", + "tapas": "Supported", + "tsumino": "Supported", + "tumblr": _OAUTH, + "twitter": "Supported", + "vipergirls": "Supported", + "wallhaven": _APIKEY_WH, + "weasyl": _APIKEY_WY, + "zerochan": "Supported", } IGNORE_LIST = ( @@ -466,7 +471,7 @@ def domain(cls): return cls.root + "/" url = cls.example - return url[:url.find("/", 8)+1] + return url[: url.find("/", 8) + 1] def category_text(c): @@ -577,20 +582,20 @@ def build_extractor_list(): # define table columns COLUMNS = ( - ("Site", 20, - lambda bc, c, scs, d: category_text(c)), - ("URL" , 35, - lambda bc, c, scs, d: d), - ("Capabilities", 50, - lambda bc, c, scs, d: ", ".join(subcategory_text(bc, c, sc) for sc in scs - if subcategory_text(bc, c, sc))), - ("Authentication", 16, - lambda bc, c, scs, d: AUTH_MAP.get(c, "")), + ("Site", 20, lambda bc, c, scs, d: category_text(c)), + ("URL", 35, lambda bc, c, scs, d: d), + ( + "Capabilities", + 50, + lambda bc, c, scs, d: ", ".join( + subcategory_text(bc, c, sc) for sc in scs if subcategory_text(bc, c, sc) + ), + ), + ("Authentication", 16, lambda bc, c, scs, d: AUTH_MAP.get(c, "")), ) def generate_output(columns, categories, domains): - thead = [] append = thead.append append("") @@ -602,11 +607,9 @@ def generate_output(columns, categories, domains): append = tbody.append for bcat, base in categories.items(): - if bcat and base: name = BASE_MAP.get(bcat) or (bcat.capitalize() + " Instances") - append('\n\n ' + - name + '\n') + append('\n\n ' + name + "\n") clist = base.items() else: clist = sorted(base.items(), key=category_key) @@ -641,7 +644,6 @@ def generate_output(columns, categories, domains): categories, domains = build_extractor_list() -PATH = (sys.argv[1] if len(sys.argv) > 1 else - util.path("docs", "supportedsites.md")) +PATH = sys.argv[1] if len(sys.argv) > 1 else util.path("docs", "supportedsites.md") with util.lazy(PATH) as fp: fp.write(generate_output(COLUMNS, categories, domains)) diff --git a/scripts/util.py b/scripts/util.py index 48b8d31934..6c4c9c08a4 100644 --- a/scripts/util.py +++ b/scripts/util.py @@ -1,11 +1,9 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import os import io +import os import sys ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -18,8 +16,7 @@ def path(*segments, join=os.path.join): return result -class lazy(): - +class lazy: def __init__(self, path): self.path = path self.buffer = io.StringIO() diff --git a/setup.py b/setup.py index 44acef9a6a..92ee240def 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- +import os.path import re import sys -import os.path import warnings @@ -18,8 +17,8 @@ def check_file(fname): if os.path.exists(path): return True warnings.warn( - "Not including file '{}' since it is not present. " - "Run 'make' to build all automatically generated files.".format(fname) + f"Not including file '{fname}' since it is not present. " + "Run 'make' to build all automatically generated files." ) return False @@ -34,10 +33,10 @@ def check_file(fname): (path, [f for f in files if check_file(f)]) for (path, files) in [ ("share/bash-completion/completions", ["data/completion/gallery-dl"]), - ("share/zsh/site-functions" , ["data/completion/_gallery-dl"]), - ("share/fish/vendor_completions.d" , ["data/completion/gallery-dl.fish"]), - ("share/man/man1" , ["data/man/gallery-dl.1"]), - ("share/man/man5" , ["data/man/gallery-dl.conf.5"]), + ("share/zsh/site-functions", ["data/completion/_gallery-dl"]), + ("share/fish/vendor_completions.d", ["data/completion/gallery-dl.fish"]), + ("share/man/man1", ["data/man/gallery-dl.1"]), + ("share/man/man5", ["data/man/gallery-dl.conf.5"]), ] ] @@ -48,10 +47,13 @@ def check_file(fname): "gallery_dl.postprocessor", ] -DESCRIPTION = ("Command-line program to download image galleries and " - "collections from several image hosting sites") +DESCRIPTION = ( + "Command-line program to download image galleries and " + "collections from several image hosting sites" +) LONG_DESCRIPTION = read("README.rst").replace( - "= 2", - - "board" : "70", - "board_name": "新板提案", - "com" : str, - "fsize" : r"re:\d+", - "name" : "名無し", - "no" : r"re:17\d\d\d", - "now" : r"re:2[34]/../..\(.\)..:..:..", - "post" : "無題", - "server" : "dec", - "thread" : "17222", - "tim" : r"re:^\d{13}$", - "time" : r"re:^\d{10}$", - "title" : "画像会話板", -}, - + { + "#url": "https://dec.2chan.net/70/res/17222.htm", + "#category": ("", "2chan", "thread"), + "#class": _2chan._2chanThreadExtractor, + "#pattern": r"https://dec\.2chan\.net/70/src/\d{13}\.jpg", + "#count": ">= 2", + "board": "70", + "board_name": "新板提案", + "com": str, + "fsize": r"re:\d+", + "name": "名無し", + "no": r"re:17\d\d\d", + "now": r"re:2[34]/../..\(.\)..:..:..", + "post": "無題", + "server": "dec", + "thread": "17222", + "tim": r"re:^\d{13}$", + "time": r"re:^\d{10}$", + "title": "画像会話板", + }, ) diff --git a/test/results/2chen.py b/test/results/2chen.py index 589053fa0c..cd864c8b37 100644 --- a/test/results/2chen.py +++ b/test/results/2chen.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -8,61 +6,52 @@ _2chen = getattr(gallery_dl.extractor, "2chen") import datetime - __tests__ = ( -{ - "#url" : "https://sturdychan.help/tv/268929", - "#category": ("", "2chen", "thread"), - "#class" : _2chen._2chenThreadExtractor, - "#pattern" : r"https://sturdychan\.help/assets/images/src/\w{40}\.\w+$", - "#count" : ">= 179", - - "board" : "tv", - "date" : datetime.datetime, - "hash" : r"re:[0-9a-f]{40}", - "name" : "Anonymous", - "no" : r"re:\d+", - "thread": "268929", - "time" : int, - "title" : "「/ttg/ #118: 🇧🇷 edition」", - "url" : str, -}, - -{ - "#url" : "https://2chen.club/tv/1", - "#category": ("", "2chen", "thread"), - "#class" : _2chen._2chenThreadExtractor, -}, - -{ - "#url" : "https://2chen.moe/jp/303786", - "#category": ("", "2chen", "thread"), - "#class" : _2chen._2chenThreadExtractor, -}, - -{ - "#url" : "https://sturdychan.help/co/", - "#category": ("", "2chen", "board"), - "#class" : _2chen._2chenBoardExtractor, - "#pattern" : _2chen._2chenThreadExtractor.pattern, -}, - -{ - "#url" : "https://2chen.moe/co", - "#category": ("", "2chen", "board"), - "#class" : _2chen._2chenBoardExtractor, -}, - -{ - "#url" : "https://2chen.club/tv", - "#category": ("", "2chen", "board"), - "#class" : _2chen._2chenBoardExtractor, -}, - -{ - "#url" : "https://2chen.moe/co/catalog", - "#category": ("", "2chen", "board"), - "#class" : _2chen._2chenBoardExtractor, -}, - + { + "#url": "https://sturdychan.help/tv/268929", + "#category": ("", "2chen", "thread"), + "#class": _2chen._2chenThreadExtractor, + "#pattern": r"https://sturdychan\.help/assets/images/src/\w{40}\.\w+$", + "#count": ">= 179", + "board": "tv", + "date": datetime.datetime, + "hash": r"re:[0-9a-f]{40}", + "name": "Anonymous", + "no": r"re:\d+", + "thread": "268929", + "time": int, + "title": "「/ttg/ #118: 🇧🇷 edition」", + "url": str, + }, + { + "#url": "https://2chen.club/tv/1", + "#category": ("", "2chen", "thread"), + "#class": _2chen._2chenThreadExtractor, + }, + { + "#url": "https://2chen.moe/jp/303786", + "#category": ("", "2chen", "thread"), + "#class": _2chen._2chenThreadExtractor, + }, + { + "#url": "https://sturdychan.help/co/", + "#category": ("", "2chen", "board"), + "#class": _2chen._2chenBoardExtractor, + "#pattern": _2chen._2chenThreadExtractor.pattern, + }, + { + "#url": "https://2chen.moe/co", + "#category": ("", "2chen", "board"), + "#class": _2chen._2chenBoardExtractor, + }, + { + "#url": "https://2chen.club/tv", + "#category": ("", "2chen", "board"), + "#class": _2chen._2chenBoardExtractor, + }, + { + "#url": "https://2chen.moe/co/catalog", + "#category": ("", "2chen", "board"), + "#class": _2chen._2chenBoardExtractor, + }, ) diff --git a/test/results/35photo.py b/test/results/35photo.py index c0e29d53bb..d2e5167223 100644 --- a/test/results/35photo.py +++ b/test/results/35photo.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -9,69 +7,61 @@ __tests__ = ( -{ - "#url" : "https://35photo.pro/liya", - "#category": ("", "35photo", "user"), - "#class" : _35photo._35photoUserExtractor, - "#pattern" : r"https://([a-z][0-9]\.)?35photo\.pro/photos_(main|series)/.*\.jpg", - "#count" : 9, -}, - -{ - "#url" : "https://35photo.pro/suhoveev", - "#comment" : "last photo ID (1267028) isn't given as 'photo-id=\"\" - " - "there are only 23 photos without the last one", - "#category": ("", "35photo", "user"), - "#class" : _35photo._35photoUserExtractor, - "#count" : ">= 33", -}, - -{ - "#url" : "https://en.35photo.pro/liya", - "#category": ("", "35photo", "user"), - "#class" : _35photo._35photoUserExtractor, -}, - -{ - "#url" : "https://ru.35photo.pro/liya", - "#category": ("", "35photo", "user"), - "#class" : _35photo._35photoUserExtractor, -}, - -{ - "#url" : "https://35photo.pro/tags/landscape/", - "#category": ("", "35photo", "tag"), - "#class" : _35photo._35photoTagExtractor, - "#range" : "1-25", - "#count" : 25, - "#archive" : False, -}, - -{ - "#url" : "https://35photo.pro/genre_109/", - "#category": ("", "35photo", "genre"), - "#class" : _35photo._35photoGenreExtractor, -}, - -{ - "#url" : "https://35photo.pro/photo_753340/", - "#category": ("", "35photo", "image"), - "#class" : _35photo._35photoImageExtractor, - "#count" : 1, - - "url" : r"re:https://35photo\.pro/photos_main/.*\.jpg", - "id" : 753340, - "title" : "Winter walk", - "description": str, - "tags" : list, - "views" : int, - "favorites" : int, - "score" : int, - "type" : 0, - "date" : "15 авг, 2014", - "user" : "liya", - "user_id" : 20415, - "user_name" : "Liya Mirzaeva", -}, - + { + "#url": "https://35photo.pro/liya", + "#category": ("", "35photo", "user"), + "#class": _35photo._35photoUserExtractor, + "#pattern": r"https://([a-z][0-9]\.)?35photo\.pro/photos_(main|series)/.*\.jpg", + "#count": 9, + }, + { + "#url": "https://35photo.pro/suhoveev", + "#comment": "last photo ID (1267028) isn't given as 'photo-id=\"\" - " + "there are only 23 photos without the last one", + "#category": ("", "35photo", "user"), + "#class": _35photo._35photoUserExtractor, + "#count": ">= 33", + }, + { + "#url": "https://en.35photo.pro/liya", + "#category": ("", "35photo", "user"), + "#class": _35photo._35photoUserExtractor, + }, + { + "#url": "https://ru.35photo.pro/liya", + "#category": ("", "35photo", "user"), + "#class": _35photo._35photoUserExtractor, + }, + { + "#url": "https://35photo.pro/tags/landscape/", + "#category": ("", "35photo", "tag"), + "#class": _35photo._35photoTagExtractor, + "#range": "1-25", + "#count": 25, + "#archive": False, + }, + { + "#url": "https://35photo.pro/genre_109/", + "#category": ("", "35photo", "genre"), + "#class": _35photo._35photoGenreExtractor, + }, + { + "#url": "https://35photo.pro/photo_753340/", + "#category": ("", "35photo", "image"), + "#class": _35photo._35photoImageExtractor, + "#count": 1, + "url": r"re:https://35photo\.pro/photos_main/.*\.jpg", + "id": 753340, + "title": "Winter walk", + "description": str, + "tags": list, + "views": int, + "favorites": int, + "score": int, + "type": 0, + "date": "15 авг, 2014", + "user": "liya", + "user_id": 20415, + "user_name": "Liya Mirzaeva", + }, ) diff --git a/test/results/3dbooru.py b/test/results/3dbooru.py index d3d407bd27..0eb72c83e6 100644 --- a/test/results/3dbooru.py +++ b/test/results/3dbooru.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -9,42 +7,37 @@ __tests__ = ( -{ - "#url" : "http://behoimi.org/post?tags=himekawa_azuru+dress", - "#category": ("booru", "3dbooru", "tag"), - "#class" : _3dbooru._3dbooruTagExtractor, - "#sha1_url" : "ecb30c6aaaf8a6ff8f55255737a9840832a483c1", - "#sha1_content": "11cbda40c287e026c1ce4ca430810f761f2d0b2a", -}, - -{ - "#url" : "http://behoimi.org/pool/show/27", - "#category": ("booru", "3dbooru", "pool"), - "#class" : _3dbooru._3dbooruPoolExtractor, - "#sha1_url" : "da75d2d1475449d5ef0c266cb612683b110a30f2", - "#sha1_content": "fd5b37c5c6c2de4b4d6f1facffdefa1e28176554", -}, - -{ - "#url" : "http://behoimi.org/post/show/140852", - "#category": ("booru", "3dbooru", "post"), - "#class" : _3dbooru._3dbooruPostExtractor, - "#options" : {"tags": True}, - "#sha1_url" : "ce874ea26f01d6c94795f3cc3aaaaa9bc325f2f6", - "#sha1_content": "26549d55b82aa9a6c1686b96af8bfcfa50805cd4", - - "tags_character": "furude_rika", - "tags_copyright": "higurashi_no_naku_koro_ni", - "tags_model" : "himekawa_azuru", - "tags_general" : str, -}, - -{ - "#url" : "http://behoimi.org/post/popular_by_month?month=2&year=2013", - "#category": ("booru", "3dbooru", "popular"), - "#class" : _3dbooru._3dbooruPopularExtractor, - "#pattern" : r"http://behoimi\.org/data/../../[0-9a-f]{32}\.jpg", - "#count" : 20, -}, - + { + "#url": "http://behoimi.org/post?tags=himekawa_azuru+dress", + "#category": ("booru", "3dbooru", "tag"), + "#class": _3dbooru._3dbooruTagExtractor, + "#sha1_url": "ecb30c6aaaf8a6ff8f55255737a9840832a483c1", + "#sha1_content": "11cbda40c287e026c1ce4ca430810f761f2d0b2a", + }, + { + "#url": "http://behoimi.org/pool/show/27", + "#category": ("booru", "3dbooru", "pool"), + "#class": _3dbooru._3dbooruPoolExtractor, + "#sha1_url": "da75d2d1475449d5ef0c266cb612683b110a30f2", + "#sha1_content": "fd5b37c5c6c2de4b4d6f1facffdefa1e28176554", + }, + { + "#url": "http://behoimi.org/post/show/140852", + "#category": ("booru", "3dbooru", "post"), + "#class": _3dbooru._3dbooruPostExtractor, + "#options": {"tags": True}, + "#sha1_url": "ce874ea26f01d6c94795f3cc3aaaaa9bc325f2f6", + "#sha1_content": "26549d55b82aa9a6c1686b96af8bfcfa50805cd4", + "tags_character": "furude_rika", + "tags_copyright": "higurashi_no_naku_koro_ni", + "tags_model": "himekawa_azuru", + "tags_general": str, + }, + { + "#url": "http://behoimi.org/post/popular_by_month?month=2&year=2013", + "#category": ("booru", "3dbooru", "popular"), + "#class": _3dbooru._3dbooruPopularExtractor, + "#pattern": r"http://behoimi\.org/data/../../[0-9a-f]{32}\.jpg", + "#count": 20, + }, ) diff --git a/test/results/4archive.py b/test/results/4archive.py index cebec6fc44..b9883e12b5 100644 --- a/test/results/4archive.py +++ b/test/results/4archive.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -8,53 +6,47 @@ _4archive = getattr(gallery_dl.extractor, "4archive") import datetime - __tests__ = ( -{ - "#url" : "https://4archive.org/board/u/thread/2397221", - "#category": ("", "4archive", "thread"), - "#class" : _4archive._4archiveThreadExtractor, - "#pattern" : r"https://(cdn\.4archive\.org/u/image/150\d/\d\d\d/\d+\.\w+|4archive\.org/image/image-404\.png)", - "#count" : 16, - - "board" : "u", - "com" : str, - "date" : datetime.datetime, - "name" : "Anonymous", - "no" : range(2397221, 2418158), - "thread": 2397221, - "time" : int, - "title" : "best anime", - "url" : str, - "width" : int, - "height": int, - "size" : int, -}, - -{ - "#url" : "https://4archive.org/board/jp/thread/17611798", - "#category": ("", "4archive", "thread"), - "#class" : _4archive._4archiveThreadExtractor, - "#pattern" : r"https://(cdn\.4archive\.org/jp/image/\d\d\d\d/\d\d\d/\d+\.\w+|4archive\.org/image/image-404\.png)", - "#count" : 85, -}, - -{ - "#url" : "https://4archive.org/board/u", - "#category": ("", "4archive", "board"), - "#class" : _4archive._4archiveBoardExtractor, - "#pattern" : _4archive._4archiveThreadExtractor.pattern, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://4archive.org/board/jp/10", - "#category": ("", "4archive", "board"), - "#class" : _4archive._4archiveBoardExtractor, - "#pattern" : _4archive._4archiveThreadExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -} - + { + "#url": "https://4archive.org/board/u/thread/2397221", + "#category": ("", "4archive", "thread"), + "#class": _4archive._4archiveThreadExtractor, + "#pattern": r"https://(cdn\.4archive\.org/u/image/150\d/\d\d\d/\d+\.\w+|4archive\.org/image/image-404\.png)", + "#count": 16, + "board": "u", + "com": str, + "date": datetime.datetime, + "name": "Anonymous", + "no": range(2397221, 2418158), + "thread": 2397221, + "time": int, + "title": "best anime", + "url": str, + "width": int, + "height": int, + "size": int, + }, + { + "#url": "https://4archive.org/board/jp/thread/17611798", + "#category": ("", "4archive", "thread"), + "#class": _4archive._4archiveThreadExtractor, + "#pattern": r"https://(cdn\.4archive\.org/jp/image/\d\d\d\d/\d\d\d/\d+\.\w+|4archive\.org/image/image-404\.png)", + "#count": 85, + }, + { + "#url": "https://4archive.org/board/u", + "#category": ("", "4archive", "board"), + "#class": _4archive._4archiveBoardExtractor, + "#pattern": _4archive._4archiveThreadExtractor.pattern, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://4archive.org/board/jp/10", + "#category": ("", "4archive", "board"), + "#class": _4archive._4archiveBoardExtractor, + "#pattern": _4archive._4archiveThreadExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, ) diff --git a/test/results/4chan.py b/test/results/4chan.py index 6219e1652e..343fd5a428 100644 --- a/test/results/4chan.py +++ b/test/results/4chan.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -9,29 +7,26 @@ __tests__ = ( -{ - "#url" : "https://boards.4chan.org/tg/thread/15396072/", - "#category": ("", "4chan", "thread"), - "#class" : _4chan._4chanThreadExtractor, - "#sha1_url" : "39082ad166161966d7ba8e37f2173a824eb540f0", - "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", - "#sha1_content" : "551e432d52700ff3711f14752124e9af86ecbbdf", -}, - -{ - "#url" : "https://boards.4channel.org/tg/thread/15396072/", - "#category": ("", "4chan", "thread"), - "#class" : _4chan._4chanThreadExtractor, - "#sha1_url" : "39082ad166161966d7ba8e37f2173a824eb540f0", - "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", -}, - -{ - "#url" : "https://boards.4channel.org/po/", - "#category": ("", "4chan", "board"), - "#class" : _4chan._4chanBoardExtractor, - "#pattern" : _4chan._4chanThreadExtractor.pattern, - "#count" : ">= 100", -}, - + { + "#url": "https://boards.4chan.org/tg/thread/15396072/", + "#category": ("", "4chan", "thread"), + "#class": _4chan._4chanThreadExtractor, + "#sha1_url": "39082ad166161966d7ba8e37f2173a824eb540f0", + "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", + "#sha1_content": "551e432d52700ff3711f14752124e9af86ecbbdf", + }, + { + "#url": "https://boards.4channel.org/tg/thread/15396072/", + "#category": ("", "4chan", "thread"), + "#class": _4chan._4chanThreadExtractor, + "#sha1_url": "39082ad166161966d7ba8e37f2173a824eb540f0", + "#sha1_metadata": "7ae2f4049adf0d2f835eb91b6b26b7f4ec882e0a", + }, + { + "#url": "https://boards.4channel.org/po/", + "#category": ("", "4chan", "board"), + "#class": _4chan._4chanBoardExtractor, + "#pattern": _4chan._4chanThreadExtractor.pattern, + "#count": ">= 100", + }, ) diff --git a/test/results/4chanarchives.py b/test/results/4chanarchives.py index 8aa8befd25..78dd96f404 100644 --- a/test/results/4chanarchives.py +++ b/test/results/4chanarchives.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -9,41 +7,36 @@ __tests__ = ( -{ - "#url" : "https://4chanarchives.com/board/c/thread/2707110", - "#category": ("", "4chanarchives", "thread"), - "#class" : _4chanarchives._4chanarchivesThreadExtractor, - "#pattern" : r"https://i\.imgur\.com/(0wLGseE|qbByWDc)\.jpg", - "#count" : 2, - - "board" : "c", - "com" : str, - "name" : "Anonymous", - "no" : int, - "thread": "2707110", - "time" : r"re:2016-07-1\d \d\d:\d\d:\d\d", - "title" : "Ren Kagami from 'Oyako Neburi'", -}, - -{ - "#url" : "https://4chanarchives.com/board/c/", - "#category": ("", "4chanarchives", "board"), - "#class" : _4chanarchives._4chanarchivesBoardExtractor, - "#pattern" : _4chanarchives._4chanarchivesThreadExtractor.pattern, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://4chanarchives.com/board/c", - "#category": ("", "4chanarchives", "board"), - "#class" : _4chanarchives._4chanarchivesBoardExtractor, -}, - -{ - "#url" : "https://4chanarchives.com/board/c/10", - "#category": ("", "4chanarchives", "board"), - "#class" : _4chanarchives._4chanarchivesBoardExtractor, -}, - + { + "#url": "https://4chanarchives.com/board/c/thread/2707110", + "#category": ("", "4chanarchives", "thread"), + "#class": _4chanarchives._4chanarchivesThreadExtractor, + "#pattern": r"https://i\.imgur\.com/(0wLGseE|qbByWDc)\.jpg", + "#count": 2, + "board": "c", + "com": str, + "name": "Anonymous", + "no": int, + "thread": "2707110", + "time": r"re:2016-07-1\d \d\d:\d\d:\d\d", + "title": "Ren Kagami from 'Oyako Neburi'", + }, + { + "#url": "https://4chanarchives.com/board/c/", + "#category": ("", "4chanarchives", "board"), + "#class": _4chanarchives._4chanarchivesBoardExtractor, + "#pattern": _4chanarchives._4chanarchivesThreadExtractor.pattern, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "https://4chanarchives.com/board/c", + "#category": ("", "4chanarchives", "board"), + "#class": _4chanarchives._4chanarchivesBoardExtractor, + }, + { + "#url": "https://4chanarchives.com/board/c/10", + "#category": ("", "4chanarchives", "board"), + "#class": _4chanarchives._4chanarchivesBoardExtractor, + }, ) diff --git a/test/results/4plebs.py b/test/results/4plebs.py index affe14d833..083b26d1b0 100644 --- a/test/results/4plebs.py +++ b/test/results/4plebs.py @@ -1,37 +1,30 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://archive.4plebs.org/tg/thread/54059290", - "#category": ("foolfuuka", "4plebs", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#pattern" : r"https://i\.4pcdn\.org/tg/1[34]\d{11}\.(jpg|png|gif)", - "#count" : 30, -}, - -{ - "#url" : "https://archive.4plebs.org/tg/", - "#category": ("foolfuuka", "4plebs", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://archive.4plebs.org/_/search/text/test/", - "#category": ("foolfuuka", "4plebs", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://archive.4plebs.org/tg/gallery/1", - "#category": ("foolfuuka", "4plebs", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://archive.4plebs.org/tg/thread/54059290", + "#category": ("foolfuuka", "4plebs", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#pattern": r"https://i\.4pcdn\.org/tg/1[34]\d{11}\.(jpg|png|gif)", + "#count": 30, + }, + { + "#url": "https://archive.4plebs.org/tg/", + "#category": ("foolfuuka", "4plebs", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://archive.4plebs.org/_/search/text/test/", + "#category": ("foolfuuka", "4plebs", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://archive.4plebs.org/tg/gallery/1", + "#category": ("foolfuuka", "4plebs", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/500px.py b/test/results/500px.py index 5630e78ea7..e025ee315c 100644 --- a/test/results/500px.py +++ b/test/results/500px.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -9,95 +7,86 @@ __tests__ = ( -{ - "#url" : "https://500px.com/p/fashvamp", - "#category": ("", "500px", "user"), - "#class" : _500px._500pxUserExtractor, - "#pattern" : r"https?://drscdn.500px.org/photo/\d+/m%3D4096(_k%3D1)?/v2\?sig=", - "#range" : "1-99", - "#count" : 99, -}, - -{ - "#url" : "https://500px.com/fashvamp", - "#category": ("", "500px", "user"), - "#class" : _500px._500pxUserExtractor, -}, - -{ - "#url" : "https://web.500px.com/fashvamp", - "#category": ("", "500px", "user"), - "#class" : _500px._500pxUserExtractor, -}, - -{ - "#url" : "https://500px.com/p/fashvamp/galleries/lera", - "#category": ("", "500px", "gallery"), - "#class" : _500px._500pxGalleryExtractor, - "#pattern" : r"https?://drscdn.500px.org/photo/\d+/m%3D4096_k%3D1/v2\?sig=", - "#count" : 3, - - "gallery": dict, - "user" : dict, -}, - -{ - "#url" : "https://500px.com/fashvamp/galleries/lera", - "#category": ("", "500px", "gallery"), - "#class" : _500px._500pxGalleryExtractor, -}, - -{ - "#url" : "https://500px.com/liked", - "#category": ("", "500px", "favorite"), - "#class" : _500px._500pxFavoriteExtractor, -}, - -{ - "#url" : "https://500px.com/photo/222049255/queen-of-coasts", - "#category": ("", "500px", "image"), - "#class" : _500px._500pxImageExtractor, - "#pattern" : r"https://drscdn\.500px\.org/photo/222049255/m%3D4096_k%3D1/v2\?sig=\w+", - "#count" : 1, - - "camera" : "Canon EOS 600D", - "camera_info" : dict, - "comments" : list, - "comments_count" : int, - "created_at" : "2017-08-01T08:40:05+00:00", - "description" : str, - "editored_by" : None, - "editors_choice" : False, - "extension" : "jpg", - "feature" : "popular", - "feature_date" : "2017-08-01T09:58:28+00:00", - "focal_length" : "208", - "height" : 3111, - "id" : 222049255, - "image_format" : "jpg", - "image_url" : list, - "images" : list, - "iso" : "100", - "lens" : "EF-S55-250mm f/4-5.6 IS II", - "lens_info" : dict, - "liked" : None, - "location" : None, - "location_details": dict, - "name" : "Queen Of Coasts", - "nsfw" : False, - "privacy" : False, - "profile" : True, - "rating" : float, - "status" : 1, - "tags" : list, - "taken_at" : "2017-05-04T17:36:51+00:00", - "times_viewed" : int, - "url" : "/photo/222049255/Queen-Of-Coasts-by-Alice-Nabieva", - "user" : dict, - "user_id" : 12847235, - "votes_count" : int, - "watermark" : True, - "width" : 4637, -}, - + { + "#url": "https://500px.com/p/fashvamp", + "#category": ("", "500px", "user"), + "#class": _500px._500pxUserExtractor, + "#pattern": r"https?://drscdn.500px.org/photo/\d+/m%3D4096(_k%3D1)?/v2\?sig=", + "#range": "1-99", + "#count": 99, + }, + { + "#url": "https://500px.com/fashvamp", + "#category": ("", "500px", "user"), + "#class": _500px._500pxUserExtractor, + }, + { + "#url": "https://web.500px.com/fashvamp", + "#category": ("", "500px", "user"), + "#class": _500px._500pxUserExtractor, + }, + { + "#url": "https://500px.com/p/fashvamp/galleries/lera", + "#category": ("", "500px", "gallery"), + "#class": _500px._500pxGalleryExtractor, + "#pattern": r"https?://drscdn.500px.org/photo/\d+/m%3D4096_k%3D1/v2\?sig=", + "#count": 3, + "gallery": dict, + "user": dict, + }, + { + "#url": "https://500px.com/fashvamp/galleries/lera", + "#category": ("", "500px", "gallery"), + "#class": _500px._500pxGalleryExtractor, + }, + { + "#url": "https://500px.com/liked", + "#category": ("", "500px", "favorite"), + "#class": _500px._500pxFavoriteExtractor, + }, + { + "#url": "https://500px.com/photo/222049255/queen-of-coasts", + "#category": ("", "500px", "image"), + "#class": _500px._500pxImageExtractor, + "#pattern": r"https://drscdn\.500px\.org/photo/222049255/m%3D4096_k%3D1/v2\?sig=\w+", + "#count": 1, + "camera": "Canon EOS 600D", + "camera_info": dict, + "comments": list, + "comments_count": int, + "created_at": "2017-08-01T08:40:05+00:00", + "description": str, + "editored_by": None, + "editors_choice": False, + "extension": "jpg", + "feature": "popular", + "feature_date": "2017-08-01T09:58:28+00:00", + "focal_length": "208", + "height": 3111, + "id": 222049255, + "image_format": "jpg", + "image_url": list, + "images": list, + "iso": "100", + "lens": "EF-S55-250mm f/4-5.6 IS II", + "lens_info": dict, + "liked": None, + "location": None, + "location_details": dict, + "name": "Queen Of Coasts", + "nsfw": False, + "privacy": False, + "profile": True, + "rating": float, + "status": 1, + "tags": list, + "taken_at": "2017-05-04T17:36:51+00:00", + "times_viewed": int, + "url": "/photo/222049255/Queen-Of-Coasts-by-Alice-Nabieva", + "user": dict, + "user_id": 12847235, + "votes_count": int, + "watermark": True, + "width": 4637, + }, ) diff --git a/test/results/8chan.py b/test/results/8chan.py index 1e26c71ac3..dd0673ab72 100644 --- a/test/results/8chan.py +++ b/test/results/8chan.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -9,81 +7,72 @@ __tests__ = ( -{ - "#url" : "https://8chan.moe/vhs/res/4.html", - "#class": _8chan._8chanThreadExtractor, - "#pattern": r"https://8chan\.moe/\.media/[0-9a-f]{64}\.\w+$", - "#count" : 14, - - "archived" : False, - "autoSage" : False, - "boardDescription": "Film and Cinema", - "boardMarkdown" : None, - "boardName" : "Movies", - "boardUri" : "vhs", - "creation" : r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z", - "cyclic" : False, - "email" : None, - "id" : r"re:^[0-9a-f]{6}$", - "locked" : False, - "markdown" : str, - "maxFileCount" : 5, - "maxFileSize" : "32.00 MB", - "maxMessageLength": 12000, - "message" : str, - "mime" : str, - "name" : "Anonymous", - "num" : int, - "originalName" : str, - "path" : r"re:/.media/[0-9a-f]{64}\.\w+$", - "pinned" : False, - "postId" : int, - "signedRole" : None, - "size" : int, - "threadId" : 4, - "thumb" : r"re:/.media/t_[0-9a-f]{64}$", - "uniquePosters" : 9, - "usesCustomCss" : True, - "usesCustomJs" : False, - "?wsPort" : int, - "?wssPort" : int, -}, - -{ - "#url" : "https://8chan.moe/vhs/last/4.html", - "#class": _8chan._8chanThreadExtractor, -}, - -{ - "#url" : "https://8chan.se/vhs/res/4.html", - "#class": _8chan._8chanThreadExtractor, -}, - -{ - "#url" : "https://8chan.cc/vhs/res/4.html", - "#class": _8chan._8chanThreadExtractor, -}, - -{ - "#url" : "https://8chan.moe/vhs/", - "#class": _8chan._8chanBoardExtractor, -}, - -{ - "#url" : "https://8chan.moe/vhs/2.html", - "#class": _8chan._8chanBoardExtractor, - "#pattern": _8chan._8chanThreadExtractor.pattern, - "#count" : range(24, 32), -}, - -{ - "#url" : "https://8chan.se/vhs/", - "#class": _8chan._8chanBoardExtractor, -}, - -{ - "#url" : "https://8chan.cc/vhs/", - "#class": _8chan._8chanBoardExtractor, -}, - + { + "#url": "https://8chan.moe/vhs/res/4.html", + "#class": _8chan._8chanThreadExtractor, + "#pattern": r"https://8chan\.moe/\.media/[0-9a-f]{64}\.\w+$", + "#count": 14, + "archived": False, + "autoSage": False, + "boardDescription": "Film and Cinema", + "boardMarkdown": None, + "boardName": "Movies", + "boardUri": "vhs", + "creation": r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z", + "cyclic": False, + "email": None, + "id": r"re:^[0-9a-f]{6}$", + "locked": False, + "markdown": str, + "maxFileCount": 5, + "maxFileSize": "32.00 MB", + "maxMessageLength": 12000, + "message": str, + "mime": str, + "name": "Anonymous", + "num": int, + "originalName": str, + "path": r"re:/.media/[0-9a-f]{64}\.\w+$", + "pinned": False, + "postId": int, + "signedRole": None, + "size": int, + "threadId": 4, + "thumb": r"re:/.media/t_[0-9a-f]{64}$", + "uniquePosters": 9, + "usesCustomCss": True, + "usesCustomJs": False, + "?wsPort": int, + "?wssPort": int, + }, + { + "#url": "https://8chan.moe/vhs/last/4.html", + "#class": _8chan._8chanThreadExtractor, + }, + { + "#url": "https://8chan.se/vhs/res/4.html", + "#class": _8chan._8chanThreadExtractor, + }, + { + "#url": "https://8chan.cc/vhs/res/4.html", + "#class": _8chan._8chanThreadExtractor, + }, + { + "#url": "https://8chan.moe/vhs/", + "#class": _8chan._8chanBoardExtractor, + }, + { + "#url": "https://8chan.moe/vhs/2.html", + "#class": _8chan._8chanBoardExtractor, + "#pattern": _8chan._8chanThreadExtractor.pattern, + "#count": range(24, 32), + }, + { + "#url": "https://8chan.se/vhs/", + "#class": _8chan._8chanBoardExtractor, + }, + { + "#url": "https://8chan.cc/vhs/", + "#class": _8chan._8chanBoardExtractor, + }, ) diff --git a/test/results/8kun.py b/test/results/8kun.py index 53d16e13d2..9923b6a749 100644 --- a/test/results/8kun.py +++ b/test/results/8kun.py @@ -1,39 +1,32 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import vichan - __tests__ = ( -{ - "#url" : "https://8kun.top/test/res/65248.html", - "#category": ("vichan", "8kun", "thread"), - "#class" : vichan.VichanThreadExtractor, - "#pattern" : r"https://media\.128ducks\.com/file_store/\w{64}\.\w+", - "#count" : ">= 8", -}, - -{ - "#url" : "https://8kun.top/v/index.html", - "#category": ("vichan", "8kun", "board"), - "#class" : vichan.VichanBoardExtractor, - "#pattern" : vichan.VichanThreadExtractor.pattern, - "#count" : ">= 100", -}, - -{ - "#url" : "https://8kun.top/v/2.html", - "#category": ("vichan", "8kun", "board"), - "#class" : vichan.VichanBoardExtractor, -}, - -{ - "#url" : "https://8kun.top/v/index.html?PageSpeed=noscript", - "#category": ("vichan", "8kun", "board"), - "#class" : vichan.VichanBoardExtractor, -}, - + { + "#url": "https://8kun.top/test/res/65248.html", + "#category": ("vichan", "8kun", "thread"), + "#class": vichan.VichanThreadExtractor, + "#pattern": r"https://media\.128ducks\.com/file_store/\w{64}\.\w+", + "#count": ">= 8", + }, + { + "#url": "https://8kun.top/v/index.html", + "#category": ("vichan", "8kun", "board"), + "#class": vichan.VichanBoardExtractor, + "#pattern": vichan.VichanThreadExtractor.pattern, + "#count": ">= 100", + }, + { + "#url": "https://8kun.top/v/2.html", + "#category": ("vichan", "8kun", "board"), + "#class": vichan.VichanBoardExtractor, + }, + { + "#url": "https://8kun.top/v/index.html?PageSpeed=noscript", + "#category": ("vichan", "8kun", "board"), + "#class": vichan.VichanBoardExtractor, + }, ) diff --git a/test/results/8muses.py b/test/results/8muses.py index 7dfb84608d..7c18b90a9d 100644 --- a/test/results/8muses.py +++ b/test/results/8muses.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. @@ -8,65 +6,57 @@ _8muses = getattr(gallery_dl.extractor, "8muses") from gallery_dl import exception - __tests__ = ( -{ - "#url" : "https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", - "#category": ("", "8muses", "album"), - "#class" : _8muses._8musesAlbumExtractor, - "#pattern" : r"https://comics.8muses.com/image/fl/[\w-]+", - "#sha1_url": "6286ac33087c236c5a7e51f8a9d4e4d5548212d4", - - "url" : str, - "hash" : str, - "page" : int, - "count": 6, - "album": { - "id" : 10467, - "title" : "Liar", - "path" : "Fakku Comics/mogg/Liar", - "parts" : [ - "Fakku Comics", - "mogg", - "Liar", - ], + { + "#url": "https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", + "#category": ("", "8muses", "album"), + "#class": _8muses._8musesAlbumExtractor, + "#pattern": r"https://comics.8muses.com/image/fl/[\w-]+", + "#sha1_url": "6286ac33087c236c5a7e51f8a9d4e4d5548212d4", + "url": str, + "hash": str, + "page": int, + "count": 6, + "album": { + "id": 10467, + "title": "Liar", + "path": "Fakku Comics/mogg/Liar", + "parts": [ + "Fakku Comics", + "mogg", + "Liar", + ], + "private": False, + "url": "https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", + "parent": 10464, + "views": int, + "likes": int, + "date": "dt:2018-07-10 00:00:00", + }, + }, + { + "#url": "https://www.8muses.com/comics/album/Fakku-Comics/santa", + "#category": ("", "8muses", "album"), + "#class": _8muses._8musesAlbumExtractor, + "#pattern": _8muses._8musesAlbumExtractor.pattern, + "#count": ">= 3", + "url": str, + "name": str, "private": False, - "url" : "https://comics.8muses.com/comics/album/Fakku-Comics/mogg/Liar", - "parent" : 10464, - "views" : int, - "likes" : int, - "date" : "dt:2018-07-10 00:00:00", }, -}, - -{ - "#url" : "https://www.8muses.com/comics/album/Fakku-Comics/santa", - "#category": ("", "8muses", "album"), - "#class" : _8muses._8musesAlbumExtractor, - "#pattern" : _8muses._8musesAlbumExtractor.pattern, - "#count" : ">= 3", - - "url" : str, - "name" : str, - "private": False, -}, - -{ - "#url" : "https://www.8muses.com/comics/album/Fakku-Comics/11?sort=az", - "#comment" : "custom sorting", - "#category": ("", "8muses", "album"), - "#class" : _8muses._8musesAlbumExtractor, - "#count" : ">= 70", - - "name": r"re:^[R-Zr-z]", -}, - -{ - "#url" : "https://comics.8muses.com/comics/album/Various-Authors/Chessire88/From-Trainers-to-Pokmons", - "#comment" : "non-ASCII characters", - "#category": ("", "8muses", "album"), - "#class" : _8muses._8musesAlbumExtractor, - "#exception": exception.HttpError, -}, - + { + "#url": "https://www.8muses.com/comics/album/Fakku-Comics/11?sort=az", + "#comment": "custom sorting", + "#category": ("", "8muses", "album"), + "#class": _8muses._8musesAlbumExtractor, + "#count": ">= 70", + "name": r"re:^[R-Zr-z]", + }, + { + "#url": "https://comics.8muses.com/comics/album/Various-Authors/Chessire88/From-Trainers-to-Pokmons", + "#comment": "non-ASCII characters", + "#category": ("", "8muses", "album"), + "#class": _8muses._8musesAlbumExtractor, + "#exception": exception.HttpError, + }, ) diff --git a/test/results/94chan.py b/test/results/94chan.py index 6e792e3058..b698355559 100644 --- a/test/results/94chan.py +++ b/test/results/94chan.py @@ -1,45 +1,37 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import jschan - __tests__ = ( -{ - "#url" : "https://94chan.org/art/thread/25.html", - "#category": ("jschan", "94chan", "thread"), - "#class" : jschan.JschanThreadExtractor, - "#pattern" : r"https://94chan.org/file/[0-9a-f]{64}(\.\w+)?", - "#count" : ">= 15", -}, - -{ - "#url" : "https://94chan.org/art/", - "#category": ("jschan", "94chan", "board"), - "#class" : jschan.JschanBoardExtractor, - "#pattern" : jschan.JschanThreadExtractor.pattern, - "#count" : ">= 30", -}, - -{ - "#url" : "https://94chan.org/art/2.html", - "#category": ("jschan", "94chan", "board"), - "#class" : jschan.JschanBoardExtractor, -}, - -{ - "#url" : "https://94chan.org/art/catalog.html", - "#category": ("jschan", "94chan", "board"), - "#class" : jschan.JschanBoardExtractor, -}, - -{ - "#url" : "https://94chan.org/art/index.html", - "#category": ("jschan", "94chan", "board"), - "#class" : jschan.JschanBoardExtractor, -}, - + { + "#url": "https://94chan.org/art/thread/25.html", + "#category": ("jschan", "94chan", "thread"), + "#class": jschan.JschanThreadExtractor, + "#pattern": r"https://94chan.org/file/[0-9a-f]{64}(\.\w+)?", + "#count": ">= 15", + }, + { + "#url": "https://94chan.org/art/", + "#category": ("jschan", "94chan", "board"), + "#class": jschan.JschanBoardExtractor, + "#pattern": jschan.JschanThreadExtractor.pattern, + "#count": ">= 30", + }, + { + "#url": "https://94chan.org/art/2.html", + "#category": ("jschan", "94chan", "board"), + "#class": jschan.JschanBoardExtractor, + }, + { + "#url": "https://94chan.org/art/catalog.html", + "#category": ("jschan", "94chan", "board"), + "#class": jschan.JschanBoardExtractor, + }, + { + "#url": "https://94chan.org/art/index.html", + "#category": ("jschan", "94chan", "board"), + "#class": jschan.JschanBoardExtractor, + }, ) diff --git a/test/results/__init__.py b/test/results/__init__.py index 0865693baa..8ca52b3253 100644 --- a/test/results/__init__.py +++ b/test/results/__init__.py @@ -1,16 +1,14 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import os import functools +import os __directory__ = os.path.dirname(__file__) -@functools.lru_cache(maxsize=None) +@functools.cache def tests(name): module = __import__(name, globals(), None, (), 1) return module.__tests__ diff --git a/test/results/acidimg.py b/test/results/acidimg.py index d61a7736d8..c3b4c49d83 100644 --- a/test/results/acidimg.py +++ b/test/results/acidimg.py @@ -1,20 +1,16 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://acidimg.cc/img-5acb6b9de4640.html", - "#category": ("imagehost", "acidimg", "image"), - "#class" : imagehosts.AcidimgImageExtractor, - "#sha1_url" : "f132a630006e8d84f52d59555191ed82b3b64c04", - "#sha1_metadata": "135347ab4345002fc013863c0d9419ba32d98f78", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - + { + "#url": "https://acidimg.cc/img-5acb6b9de4640.html", + "#category": ("imagehost", "acidimg", "image"), + "#class": imagehosts.AcidimgImageExtractor, + "#sha1_url": "f132a630006e8d84f52d59555191ed82b3b64c04", + "#sha1_metadata": "135347ab4345002fc013863c0d9419ba32d98f78", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, ) diff --git a/test/results/adultempire.py b/test/results/adultempire.py index 1a2953ef8c..936707d253 100644 --- a/test/results/adultempire.py +++ b/test/results/adultempire.py @@ -1,28 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import adultempire - __tests__ = ( -{ - "#url" : "https://www.adultempire.com/5998/gallery.html", - "#category": ("", "adultempire", "gallery"), - "#class" : adultempire.AdultempireGalleryExtractor, - "#range" : "1", - "#sha1_metadata": "5b3266e69801db0d78c22181da23bc102886e027", - "#sha1_content" : "5c6beb31e5e3cdc90ee5910d5c30f9aaec977b9e", -}, - -{ - "#url" : "https://www.adultdvdempire.com/5683/gallery.html", - "#category": ("", "adultempire", "gallery"), - "#class" : adultempire.AdultempireGalleryExtractor, - "#sha1_url" : "b12cd1a65cae8019d837505adb4d6a2c1ed4d70d", - "#sha1_metadata": "8d448d79c4ac5f5b10a3019d5b5129ddb43655e5", -}, - + { + "#url": "https://www.adultempire.com/5998/gallery.html", + "#category": ("", "adultempire", "gallery"), + "#class": adultempire.AdultempireGalleryExtractor, + "#range": "1", + "#sha1_metadata": "5b3266e69801db0d78c22181da23bc102886e027", + "#sha1_content": "5c6beb31e5e3cdc90ee5910d5c30f9aaec977b9e", + }, + { + "#url": "https://www.adultdvdempire.com/5683/gallery.html", + "#category": ("", "adultempire", "gallery"), + "#class": adultempire.AdultempireGalleryExtractor, + "#sha1_url": "b12cd1a65cae8019d837505adb4d6a2c1ed4d70d", + "#sha1_metadata": "8d448d79c4ac5f5b10a3019d5b5129ddb43655e5", + }, ) diff --git a/test/results/agnph.py b/test/results/agnph.py index ae5c0d24c8..ee4a0b3c5f 100644 --- a/test/results/agnph.py +++ b/test/results/agnph.py @@ -1,54 +1,47 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import agnph - __tests__ = ( -{ - "#url" : "https://agn.ph/gallery/post/?search=azuu", - "#category": ("booru", "agnph", "tag"), - "#class" : agnph.AgnphTagExtractor, - "#pattern" : r"http://agn\.ph/gallery/data/../../\w{32}\.jpg", - "#count" : ">= 50", -}, - -{ - "#url" : "https://agn.ph/gallery/post/show/501604/", - "#category": ("booru", "agnph", "post"), - "#class" : agnph.AgnphPostExtractor, - "#options" : {"tags": True}, - "#urls" : "http://agn.ph/gallery/data/7d/a5/7da50021f3e86f6cf1c215652060d772.png", - "#sha1_content": "93c8b2d3f53e891ad8fa68d5f60f8c7a70acd836", - - "artist" : "reyn_goldfur", - "created_at" : "1722041591", - "creator_id" : "-1", - "date" : "dt:2024-07-27 00:53:11", - "description" : None, - "fav_count" : "0", - "file_ext" : "png", - "file_url" : "http://agn.ph/gallery/data/7d/a5/7da50021f3e86f6cf1c215652060d772.png", - "has_children": False, - "height" : "1000", - "id" : "501604", - "md5" : "7da50021f3e86f6cf1c215652060d772", - "num_comments": "0", - "parent_id" : None, - "rating" : "e", - "source" : "https://inkbunny.net/s/2886519", - "status" : "approved", - "tags" : "anthro female hisuian_sneasel regional_form reyn_goldfur shelly_the_sneasel sneasel solo", - "tags_artist" : "reyn_goldfur", - "tags_character": "shelly_the_sneasel", - "tags_general": "anthro female solo", - "tags_species": "hisuian_sneasel regional_form sneasel", - "thumbnail_url": "http://agn.ph/gallery/data/thumb/7d/a5/7da50021f3e86f6cf1c215652060d772.png", - "width" : "953", - -}, - + { + "#url": "https://agn.ph/gallery/post/?search=azuu", + "#category": ("booru", "agnph", "tag"), + "#class": agnph.AgnphTagExtractor, + "#pattern": r"http://agn\.ph/gallery/data/../../\w{32}\.jpg", + "#count": ">= 50", + }, + { + "#url": "https://agn.ph/gallery/post/show/501604/", + "#category": ("booru", "agnph", "post"), + "#class": agnph.AgnphPostExtractor, + "#options": {"tags": True}, + "#urls": "http://agn.ph/gallery/data/7d/a5/7da50021f3e86f6cf1c215652060d772.png", + "#sha1_content": "93c8b2d3f53e891ad8fa68d5f60f8c7a70acd836", + "artist": "reyn_goldfur", + "created_at": "1722041591", + "creator_id": "-1", + "date": "dt:2024-07-27 00:53:11", + "description": None, + "fav_count": "0", + "file_ext": "png", + "file_url": "http://agn.ph/gallery/data/7d/a5/7da50021f3e86f6cf1c215652060d772.png", + "has_children": False, + "height": "1000", + "id": "501604", + "md5": "7da50021f3e86f6cf1c215652060d772", + "num_comments": "0", + "parent_id": None, + "rating": "e", + "source": "https://inkbunny.net/s/2886519", + "status": "approved", + "tags": "anthro female hisuian_sneasel regional_form reyn_goldfur shelly_the_sneasel sneasel solo", + "tags_artist": "reyn_goldfur", + "tags_character": "shelly_the_sneasel", + "tags_general": "anthro female solo", + "tags_species": "hisuian_sneasel regional_form sneasel", + "thumbnail_url": "http://agn.ph/gallery/data/thumb/7d/a5/7da50021f3e86f6cf1c215652060d772.png", + "width": "953", + }, ) diff --git a/test/results/aibooru.py b/test/results/aibooru.py index 414084239c..b27ca4609c 100644 --- a/test/results/aibooru.py +++ b/test/results/aibooru.py @@ -1,44 +1,36 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import danbooru - __tests__ = ( -{ - "#url" : "https://aibooru.online/posts?tags=center_frills&z=1", - "#category": ("Danbooru", "aibooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, - "#pattern" : r"https://cdn\.aibooru\.download/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", - "#count" : ">= 50", -}, - -{ - "#url" : "https://safe.aibooru.online/posts?tags=center_frills", - "#category": ("Danbooru", "aibooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, -}, - -{ - "#url" : "https://aibooru.online/pools/1", - "#category": ("Danbooru", "aibooru", "pool"), - "#class" : danbooru.DanbooruPoolExtractor, -}, - -{ - "#url" : "https://aibooru.online/posts/1", - "#category": ("Danbooru", "aibooru", "post"), - "#class" : danbooru.DanbooruPostExtractor, - "#sha1_content": "54d548743cd67799a62c77cbae97cfa0fec1b7e9", -}, - -{ - "#url" : "https://aibooru.online/explore/posts/popular", - "#category": ("Danbooru", "aibooru", "popular"), - "#class" : danbooru.DanbooruPopularExtractor, -}, - + { + "#url": "https://aibooru.online/posts?tags=center_frills&z=1", + "#category": ("Danbooru", "aibooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + "#pattern": r"https://cdn\.aibooru\.download/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", + "#count": ">= 50", + }, + { + "#url": "https://safe.aibooru.online/posts?tags=center_frills", + "#category": ("Danbooru", "aibooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + }, + { + "#url": "https://aibooru.online/pools/1", + "#category": ("Danbooru", "aibooru", "pool"), + "#class": danbooru.DanbooruPoolExtractor, + }, + { + "#url": "https://aibooru.online/posts/1", + "#category": ("Danbooru", "aibooru", "post"), + "#class": danbooru.DanbooruPostExtractor, + "#sha1_content": "54d548743cd67799a62c77cbae97cfa0fec1b7e9", + }, + { + "#url": "https://aibooru.online/explore/posts/popular", + "#category": ("Danbooru", "aibooru", "popular"), + "#class": danbooru.DanbooruPopularExtractor, + }, ) diff --git a/test/results/allgirlbooru.py b/test/results/allgirlbooru.py index e1bfc11fd9..83887f7c26 100644 --- a/test/results/allgirlbooru.py +++ b/test/results/allgirlbooru.py @@ -1,47 +1,40 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v01 - __tests__ = ( -{ - "#url" : "https://allgirl.booru.org/index.php?page=post&s=list&tags=dress", - "#category": ("gelbooru_v01", "allgirlbooru", "tag"), - "#class" : gelbooru_v01.GelbooruV01TagExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://allgirl.booru.org/index.php?page=favorites&s=view&id=380", - "#category": ("gelbooru_v01", "allgirlbooru", "favorite"), - "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, - "#count" : 4, -}, - -{ - "#url" : "https://allgirl.booru.org/index.php?page=post&s=view&id=107213", - "#category": ("gelbooru_v01", "allgirlbooru", "post"), - "#class" : gelbooru_v01.GelbooruV01PostExtractor, - "#sha1_url" : "b416800d2d2b072f80d3b37cfca9cb806fb25d51", - "#sha1_content": "3e3c65e0854a988696e11adf0de52f8fa90a51c7", - - "created_at": "2021-02-13 16:27:39", - "date" : "dt:2021-02-13 16:27:39", - "file_url" : "https://img.booru.org/allgirl//images/107/2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb.jpg", - "height" : "1200", - "id" : "107213", - "md5" : "2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb", - "rating" : "s", - "score" : str, - "source" : "", - "tags" : "blush dress green_eyes green_hair hatsune_miku long_hair twintails vocaloid", - "uploader" : "Honochi31", - "width" : "1600", -}, - + { + "#url": "https://allgirl.booru.org/index.php?page=post&s=list&tags=dress", + "#category": ("gelbooru_v01", "allgirlbooru", "tag"), + "#class": gelbooru_v01.GelbooruV01TagExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://allgirl.booru.org/index.php?page=favorites&s=view&id=380", + "#category": ("gelbooru_v01", "allgirlbooru", "favorite"), + "#class": gelbooru_v01.GelbooruV01FavoriteExtractor, + "#count": 4, + }, + { + "#url": "https://allgirl.booru.org/index.php?page=post&s=view&id=107213", + "#category": ("gelbooru_v01", "allgirlbooru", "post"), + "#class": gelbooru_v01.GelbooruV01PostExtractor, + "#sha1_url": "b416800d2d2b072f80d3b37cfca9cb806fb25d51", + "#sha1_content": "3e3c65e0854a988696e11adf0de52f8fa90a51c7", + "created_at": "2021-02-13 16:27:39", + "date": "dt:2021-02-13 16:27:39", + "file_url": "https://img.booru.org/allgirl//images/107/2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb.jpg", + "height": "1200", + "id": "107213", + "md5": "2aaa0438d58fc7baa75a53b4a9621bb89a9d3fdb", + "rating": "s", + "score": str, + "source": "", + "tags": "blush dress green_eyes green_hair hatsune_miku long_hair twintails vocaloid", + "uploader": "Honochi31", + "width": "1600", + }, ) diff --git a/test/results/animereactor.py b/test/results/animereactor.py index a83ac6ff82..12305d9ebc 100644 --- a/test/results/animereactor.py +++ b/test/results/animereactor.py @@ -1,29 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import reactor - __tests__ = ( -{ - "#url" : "http://anime.reactor.cc/tag/Anime+Art", - "#category": ("reactor", "anime.reactor", "tag"), - "#class" : reactor.ReactorTagExtractor, -}, - -{ - "#url" : "http://anime.reactor.cc/user/Shuster", - "#category": ("reactor", "anime.reactor", "user"), - "#class" : reactor.ReactorUserExtractor, -}, - -{ - "#url" : "http://anime.reactor.cc/post/3576250", - "#category": ("reactor", "anime.reactor", "post"), - "#class" : reactor.ReactorPostExtractor, -}, - + { + "#url": "http://anime.reactor.cc/tag/Anime+Art", + "#category": ("reactor", "anime.reactor", "tag"), + "#class": reactor.ReactorTagExtractor, + }, + { + "#url": "http://anime.reactor.cc/user/Shuster", + "#category": ("reactor", "anime.reactor", "user"), + "#class": reactor.ReactorUserExtractor, + }, + { + "#url": "http://anime.reactor.cc/post/3576250", + "#category": ("reactor", "anime.reactor", "post"), + "#class": reactor.ReactorPostExtractor, + }, ) diff --git a/test/results/ao3.py b/test/results/ao3.py index 4b36367382..d0f0f157a6 100644 --- a/test/results/ao3.py +++ b/test/results/ao3.py @@ -1,248 +1,228 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import ao3 - __tests__ = ( -{ - "#url" : "https://archiveofourown.org/works/47802076", - "#class" : ao3.Ao3WorkExtractor, - "#urls" : "https://archiveofourown.org/downloads/47802076/The_Wildcard.pdf?updated_at=1720398424", - - "author" : "Flowers_for_ghouls", - "bookmarks": range(100, 300), - "chapters": { - "120506833": "1. Showtime", - "120866506": "2. A Comedy of Errors", - "121739140": "3. Gifts", - "121941313": "4. Date Night", - "123054364": "5. Breaking the News", - "123579898": "6. Isolated Events", - "124258153": "7. The Home Stretch", - "124886536": "8. Domestic Bliss", - "125335270": "9. The Offer", - "125871166": "10. The Promise", - "126223879": "11. Gifts II", - "126692398": "12. On the Move", - "127471375": "13. The Fruit Vignettes", - "128496448": "14. Respite", - "128994919": "15. Changes", - "129492154": "16. Halloween", - "130379002": "17. GIfts III", - "131066743": "18. R.A.S.B.E.W.", - "131884072": "19. The Longest Night", - "132730264": "20. Meeting the Pack", - "133714876": "21. A Mystery", - "134663854": "22. Growing Pains", - "135499822": "23. Presentation Day", - "136500946": "24. Revelations", - "137857876": "25. The Retirement Plan", - "139463056": "26. Two Birds, One Stone", - "141697141": "27. New Management", - }, - "comments" : range(800, 2000), - "date" : "dt:2023-06-11 00:00:00", - "date_completed": "dt:2024-05-10 00:00:00", - "date_updated" : "dt:2024-07-08 00:27:04", - "extension": "pdf", - "filename" : "The_Wildcard", - "id" : 47802076, - "lang" : "en", - "language" : "English", - "likes" : range(1000, 2000), - "series" : { - "id" : "4237024", - "prev" : "", - "next" : "57205801", - "index": "1", - "name" : "The Wildcard Universe", - }, - "title" : "The Wildcard", - "views" : range(34000, 50000), - "words" : 217549, - - "categories": [ - "Gen", - "M/M", - ], - "characters": [ - "Dewdrop Ghoul | Fire Ghoul", - "Aether | Quintessence Ghoul", - "Multi Ghoul | Swiss Army Ghoul", - "Rain | Water Ghoul", - "Cirrus | Air Ghoulette", - "Cumulus | Air Ghoulette", - "Sunshine Ghoulette", - "Mountain | Earth Ghoul", - "Cardinal Copia", - "Phantom Ghoul", - "Aurora Ghoulette", - "Sister Imperator (Ghost Sweden Band)", - ], - "fandom": [ - "Ghost (Sweden Band)", - ], - "rating": [ - "Mature", - ], - "relationships": [ - "Aether | Quintessence Ghoul/Dewdrop Ghoul | Fire Ghoul", - "Multi Ghoul | Swiss Army Ghoul/Rain | Water Ghoul", - ], - "summary": [ - "Aether has been asked to stay at the ministry to manage the renovation of the new infirmary. It couldn’t have been worse timing. Barely days into the new tour, Dew realizes he’s carrying their first kit.", - ], - "tags": [ - "Domestic Fluff", - "Pack Dynamics", - "gratuitous fluff", - "How do ghouls work?", - "they don't even know", - "but it's cute", - "Pregnant Dewdrop", - "Recreational Drug Use", - "Cowbell!", - "Protective Ghouls", - "no beta we die like Nihil", - "sick dewdrop", - "TW: Vomiting", - "Aether really loves Dew", - "Nesting", - "Ghoul Piles (Ghost Sweden Band)", - "Angst", - "Hurt/Comfort", - "Original Ghoul Kit(s) (Ghost Sweden Band)", - "Kit fic", - ], - "warnings": [ - "No Archive Warnings Apply", - ], -}, - -{ - "#url" : "https://archiveofourown.org/works/47802076", - "#class" : ao3.Ao3WorkExtractor, - "#options" : {"formats": ["epub", "mobi", "azw3", "pdf", "html"]}, - "#urls" : ( - "https://archiveofourown.org/downloads/47802076/The_Wildcard.epub?updated_at=1720398424", - "https://archiveofourown.org/downloads/47802076/The_Wildcard.mobi?updated_at=1720398424", - "https://archiveofourown.org/downloads/47802076/The_Wildcard.azw3?updated_at=1720398424", - "https://archiveofourown.org/downloads/47802076/The_Wildcard.pdf?updated_at=1720398424", - "https://archiveofourown.org/downloads/47802076/The_Wildcard.html?updated_at=1720398424", - ), -}, - -{ - "#url" : "https://archiveofourown.org/works/12345", - "#comment" : "restricted work / login required", - "#class" : ao3.Ao3WorkExtractor, - "#auth" : True, - "#urls" : "https://archiveofourown.org/downloads/12345/Unquenchable.pdf?updated_at=1716029699", -}, - -{ - "#url" : "https://archiveofourown.org/series/1903930", - "#class" : ao3.Ao3SeriesExtractor, - "#urls" : ( - "https://archiveofourown.org/works/26131546", - "https://archiveofourown.org/works/26291101", - "https://archiveofourown.org/works/26325292", - ), -}, - -{ - "#url" : "https://archiveofourown.org/tags/Sunshine%20(Ghost%20Sweden%20Band)/works", - "#class" : ao3.Ao3TagExtractor, - "#pattern" : ao3.Ao3WorkExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://archiveofourown.org/works/search?work_search%5Bquery%5D=air+fire+ice+water", - "#class" : ao3.Ao3SearchExtractor, - "#pattern" : ao3.Ao3WorkExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://archiveofourown.org/users/Fyrelass", - "#class" : ao3.Ao3UserExtractor, - "#urls" : ( - "https://archiveofourown.org/users/Fyrelass/works", - "https://archiveofourown.org/users/Fyrelass/series", - ), -}, -{ - "#url" : "https://archiveofourown.com/users/Fyrelass", - "#class" : ao3.Ao3UserExtractor, -}, -{ - "#url" : "https://archiveofourown.net/users/Fyrelass", - "#class" : ao3.Ao3UserExtractor, -}, -{ - "#url" : "https://ao3.org/users/Fyrelass", - "#class" : ao3.Ao3UserExtractor, -}, - -{ - "#url" : "https://archiveofourown.org/users/Fyrelass/profile", - "#class" : ao3.Ao3UserExtractor, -}, - -{ - "#url" : "https://archiveofourown.org/users/Fyrelass/pseuds/Aileen%20Autarkeia", - "#class" : ao3.Ao3UserExtractor, -}, - -{ - "#url" : "https://archiveofourown.org/users/Fyrelass/works", - "#class" : ao3.Ao3UserWorksExtractor, - "#auth" : False, - "#urls" : ( - "https://archiveofourown.org/works/55035061", - "https://archiveofourown.org/works/58979287", - "https://archiveofourown.org/works/52704457", - "https://archiveofourown.org/works/52502743", - "https://archiveofourown.org/works/52170409", - "https://archiveofourown.org/works/52078558", - "https://archiveofourown.org/works/51699982", - "https://archiveofourown.org/works/51975193", - "https://archiveofourown.org/works/51633877", - "https://archiveofourown.org/works/51591436", - "https://archiveofourown.org/works/50860891", - ), -}, - -{ - "#url" : "https://archiveofourown.org/users/Fyrelass/series", - "#class" : ao3.Ao3UserSeriesExtractor, - "#auth" : False, - "#urls" : ( - "https://archiveofourown.org/series/3821575", - ), -}, - -{ - "#url" : "https://archiveofourown.org/users/Fyrelass/bookmarks", - "#class" : ao3.Ao3UserBookmarkExtractor, - "#pattern" : r"https://archiveofourown\.org/(work|serie)s/\d+", - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://archiveofourown.org/users/mikf/subscriptions", - "#class" : ao3.Ao3SubscriptionsExtractor, - "#auth" : True, - "#pattern" : r"https://archiveofourown\.org/(work|serie|user)s/\w+", - "#count" : range(20, 30), -}, - + { + "#url": "https://archiveofourown.org/works/47802076", + "#class": ao3.Ao3WorkExtractor, + "#urls": "https://archiveofourown.org/downloads/47802076/The_Wildcard.pdf?updated_at=1720398424", + "author": "Flowers_for_ghouls", + "bookmarks": range(100, 300), + "chapters": { + "120506833": "1. Showtime", + "120866506": "2. A Comedy of Errors", + "121739140": "3. Gifts", + "121941313": "4. Date Night", + "123054364": "5. Breaking the News", + "123579898": "6. Isolated Events", + "124258153": "7. The Home Stretch", + "124886536": "8. Domestic Bliss", + "125335270": "9. The Offer", + "125871166": "10. The Promise", + "126223879": "11. Gifts II", + "126692398": "12. On the Move", + "127471375": "13. The Fruit Vignettes", + "128496448": "14. Respite", + "128994919": "15. Changes", + "129492154": "16. Halloween", + "130379002": "17. GIfts III", + "131066743": "18. R.A.S.B.E.W.", + "131884072": "19. The Longest Night", + "132730264": "20. Meeting the Pack", + "133714876": "21. A Mystery", + "134663854": "22. Growing Pains", + "135499822": "23. Presentation Day", + "136500946": "24. Revelations", + "137857876": "25. The Retirement Plan", + "139463056": "26. Two Birds, One Stone", + "141697141": "27. New Management", + }, + "comments": range(800, 2000), + "date": "dt:2023-06-11 00:00:00", + "date_completed": "dt:2024-05-10 00:00:00", + "date_updated": "dt:2024-07-08 00:27:04", + "extension": "pdf", + "filename": "The_Wildcard", + "id": 47802076, + "lang": "en", + "language": "English", + "likes": range(1000, 2000), + "series": { + "id": "4237024", + "prev": "", + "next": "57205801", + "index": "1", + "name": "The Wildcard Universe", + }, + "title": "The Wildcard", + "views": range(34000, 50000), + "words": 217549, + "categories": [ + "Gen", + "M/M", + ], + "characters": [ + "Dewdrop Ghoul | Fire Ghoul", + "Aether | Quintessence Ghoul", + "Multi Ghoul | Swiss Army Ghoul", + "Rain | Water Ghoul", + "Cirrus | Air Ghoulette", + "Cumulus | Air Ghoulette", + "Sunshine Ghoulette", + "Mountain | Earth Ghoul", + "Cardinal Copia", + "Phantom Ghoul", + "Aurora Ghoulette", + "Sister Imperator (Ghost Sweden Band)", + ], + "fandom": [ + "Ghost (Sweden Band)", + ], + "rating": [ + "Mature", + ], + "relationships": [ + "Aether | Quintessence Ghoul/Dewdrop Ghoul | Fire Ghoul", + "Multi Ghoul | Swiss Army Ghoul/Rain | Water Ghoul", + ], + "summary": [ + "Aether has been asked to stay at the ministry to manage the renovation of the new infirmary. It couldn’t have been worse timing. Barely days into the new tour, Dew realizes he’s carrying their first kit.", + ], + "tags": [ + "Domestic Fluff", + "Pack Dynamics", + "gratuitous fluff", + "How do ghouls work?", + "they don't even know", + "but it's cute", + "Pregnant Dewdrop", + "Recreational Drug Use", + "Cowbell!", + "Protective Ghouls", + "no beta we die like Nihil", + "sick dewdrop", + "TW: Vomiting", + "Aether really loves Dew", + "Nesting", + "Ghoul Piles (Ghost Sweden Band)", + "Angst", + "Hurt/Comfort", + "Original Ghoul Kit(s) (Ghost Sweden Band)", + "Kit fic", + ], + "warnings": [ + "No Archive Warnings Apply", + ], + }, + { + "#url": "https://archiveofourown.org/works/47802076", + "#class": ao3.Ao3WorkExtractor, + "#options": {"formats": ["epub", "mobi", "azw3", "pdf", "html"]}, + "#urls": ( + "https://archiveofourown.org/downloads/47802076/The_Wildcard.epub?updated_at=1720398424", + "https://archiveofourown.org/downloads/47802076/The_Wildcard.mobi?updated_at=1720398424", + "https://archiveofourown.org/downloads/47802076/The_Wildcard.azw3?updated_at=1720398424", + "https://archiveofourown.org/downloads/47802076/The_Wildcard.pdf?updated_at=1720398424", + "https://archiveofourown.org/downloads/47802076/The_Wildcard.html?updated_at=1720398424", + ), + }, + { + "#url": "https://archiveofourown.org/works/12345", + "#comment": "restricted work / login required", + "#class": ao3.Ao3WorkExtractor, + "#auth": True, + "#urls": "https://archiveofourown.org/downloads/12345/Unquenchable.pdf?updated_at=1716029699", + }, + { + "#url": "https://archiveofourown.org/series/1903930", + "#class": ao3.Ao3SeriesExtractor, + "#urls": ( + "https://archiveofourown.org/works/26131546", + "https://archiveofourown.org/works/26291101", + "https://archiveofourown.org/works/26325292", + ), + }, + { + "#url": "https://archiveofourown.org/tags/Sunshine%20(Ghost%20Sweden%20Band)/works", + "#class": ao3.Ao3TagExtractor, + "#pattern": ao3.Ao3WorkExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://archiveofourown.org/works/search?work_search%5Bquery%5D=air+fire+ice+water", + "#class": ao3.Ao3SearchExtractor, + "#pattern": ao3.Ao3WorkExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://archiveofourown.org/users/Fyrelass", + "#class": ao3.Ao3UserExtractor, + "#urls": ( + "https://archiveofourown.org/users/Fyrelass/works", + "https://archiveofourown.org/users/Fyrelass/series", + ), + }, + { + "#url": "https://archiveofourown.com/users/Fyrelass", + "#class": ao3.Ao3UserExtractor, + }, + { + "#url": "https://archiveofourown.net/users/Fyrelass", + "#class": ao3.Ao3UserExtractor, + }, + { + "#url": "https://ao3.org/users/Fyrelass", + "#class": ao3.Ao3UserExtractor, + }, + { + "#url": "https://archiveofourown.org/users/Fyrelass/profile", + "#class": ao3.Ao3UserExtractor, + }, + { + "#url": "https://archiveofourown.org/users/Fyrelass/pseuds/Aileen%20Autarkeia", + "#class": ao3.Ao3UserExtractor, + }, + { + "#url": "https://archiveofourown.org/users/Fyrelass/works", + "#class": ao3.Ao3UserWorksExtractor, + "#auth": False, + "#urls": ( + "https://archiveofourown.org/works/55035061", + "https://archiveofourown.org/works/58979287", + "https://archiveofourown.org/works/52704457", + "https://archiveofourown.org/works/52502743", + "https://archiveofourown.org/works/52170409", + "https://archiveofourown.org/works/52078558", + "https://archiveofourown.org/works/51699982", + "https://archiveofourown.org/works/51975193", + "https://archiveofourown.org/works/51633877", + "https://archiveofourown.org/works/51591436", + "https://archiveofourown.org/works/50860891", + ), + }, + { + "#url": "https://archiveofourown.org/users/Fyrelass/series", + "#class": ao3.Ao3UserSeriesExtractor, + "#auth": False, + "#urls": ("https://archiveofourown.org/series/3821575",), + }, + { + "#url": "https://archiveofourown.org/users/Fyrelass/bookmarks", + "#class": ao3.Ao3UserBookmarkExtractor, + "#pattern": r"https://archiveofourown\.org/(work|serie)s/\d+", + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://archiveofourown.org/users/mikf/subscriptions", + "#class": ao3.Ao3SubscriptionsExtractor, + "#auth": True, + "#pattern": r"https://archiveofourown\.org/(work|serie|user)s/\w+", + "#count": range(20, 30), + }, ) diff --git a/test/results/architizer.py b/test/results/architizer.py index 3a75155693..7f2d0bdbe3 100644 --- a/test/results/architizer.py +++ b/test/results/architizer.py @@ -1,40 +1,34 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import architizer - __tests__ = ( -{ - "#url" : "https://architizer.com/projects/house-lo/", - "#category": ("", "architizer", "project"), - "#class" : architizer.ArchitizerProjectExtractor, - "#pattern" : r"https://architizer-prod\.imgix\.net/media/mediadata/uploads/.+\.jpg$", - - "count" : 27, - "description": str, - "firm" : "Atelier Lina Bellovicova", - "gid" : "225496", - "location" : "Czechia", - "num" : int, - "size" : "1000 sqft - 3000 sqft", - "slug" : "house-lo", - "status" : "Built", - "subcategory": "project", - "title" : "House LO", - "type" : "Residential › Private House", - "year" : "2020", -}, - -{ - "#url" : "https://architizer.com/firms/olson-kundig/", - "#category": ("", "architizer", "firm"), - "#class" : architizer.ArchitizerFirmExtractor, - "#pattern" : architizer.ArchitizerProjectExtractor.pattern, - "#count" : ">= 90", -}, - + { + "#url": "https://architizer.com/projects/house-lo/", + "#category": ("", "architizer", "project"), + "#class": architizer.ArchitizerProjectExtractor, + "#pattern": r"https://architizer-prod\.imgix\.net/media/mediadata/uploads/.+\.jpg$", + "count": 27, + "description": str, + "firm": "Atelier Lina Bellovicova", + "gid": "225496", + "location": "Czechia", + "num": int, + "size": "1000 sqft - 3000 sqft", + "slug": "house-lo", + "status": "Built", + "subcategory": "project", + "title": "House LO", + "type": "Residential › Private House", + "year": "2020", + }, + { + "#url": "https://architizer.com/firms/olson-kundig/", + "#category": ("", "architizer", "firm"), + "#class": architizer.ArchitizerFirmExtractor, + "#pattern": architizer.ArchitizerProjectExtractor.pattern, + "#count": ">= 90", + }, ) diff --git a/test/results/archivedmoe.py b/test/results/archivedmoe.py index 42aae1815e..afa539c6aa 100644 --- a/test/results/archivedmoe.py +++ b/test/results/archivedmoe.py @@ -1,56 +1,47 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://archived.moe/gd/thread/309639/", - "#category": ("foolfuuka", "archivedmoe", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url" : "fdd533840e2d535abd162c02d6dfadbc12e2dcd8", - "#sha1_content": "c27e2a7be3bc989b5dd859f7789cc854db3f5573", -}, - -{ - "#url" : "https://archived.moe/a/thread/159767162/", - "#category": ("foolfuuka", "archivedmoe", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "ffec05a1a1b906b5ca85992513671c9155ee9e87", -}, - -{ - "#url" : "https://archived.moe/b/thread/912594917/", - "#comment" : "broken thebarchive .webm URLs (#5116)", - "#category": ("foolfuuka", "archivedmoe", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#urls" : ( - "https://thebarchive.com/b/full_image/1705625299234839.gif", - "https://thebarchive.com/b/full_image/1705625431133806.web", - "https://thebarchive.com/b/full_image/1705626190307840.web", - ), -}, - -{ - "#url" : "https://archived.moe/gd/", - "#category": ("foolfuuka", "archivedmoe", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://archived.moe/_/search/text/test/", - "#category": ("foolfuuka", "archivedmoe", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://archived.moe/gd/gallery/2", - "#category": ("foolfuuka", "archivedmoe", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://archived.moe/gd/thread/309639/", + "#category": ("foolfuuka", "archivedmoe", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "fdd533840e2d535abd162c02d6dfadbc12e2dcd8", + "#sha1_content": "c27e2a7be3bc989b5dd859f7789cc854db3f5573", + }, + { + "#url": "https://archived.moe/a/thread/159767162/", + "#category": ("foolfuuka", "archivedmoe", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "ffec05a1a1b906b5ca85992513671c9155ee9e87", + }, + { + "#url": "https://archived.moe/b/thread/912594917/", + "#comment": "broken thebarchive .webm URLs (#5116)", + "#category": ("foolfuuka", "archivedmoe", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#urls": ( + "https://thebarchive.com/b/full_image/1705625299234839.gif", + "https://thebarchive.com/b/full_image/1705625431133806.web", + "https://thebarchive.com/b/full_image/1705626190307840.web", + ), + }, + { + "#url": "https://archived.moe/gd/", + "#category": ("foolfuuka", "archivedmoe", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://archived.moe/_/search/text/test/", + "#category": ("foolfuuka", "archivedmoe", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://archived.moe/gd/gallery/2", + "#category": ("foolfuuka", "archivedmoe", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/archiveofsins.py b/test/results/archiveofsins.py index 3f3fcb9396..77a15dde79 100644 --- a/test/results/archiveofsins.py +++ b/test/results/archiveofsins.py @@ -1,43 +1,35 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://archiveofsins.com/h/thread/4668813/", - "#category": ("foolfuuka", "archiveofsins", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url" : "f612d287087e10a228ef69517cf811539db9a102", - "#sha1_content": "0dd92d0d8a7bf6e2f7d1f5ac8954c1bcf18c22a4", -}, - -{ - "#url" : "https://archiveofsins.com/h/", - "#category": ("foolfuuka", "archiveofsins", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://archiveofsins.com/_/search/text/test/", - "#category": ("foolfuuka", "archiveofsins", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://archiveofsins.com/_/search/text/test/", - "#category": ("foolfuuka", "archiveofsins", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://archiveofsins.com/h/gallery/3", - "#category": ("foolfuuka", "archiveofsins", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://archiveofsins.com/h/thread/4668813/", + "#category": ("foolfuuka", "archiveofsins", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "f612d287087e10a228ef69517cf811539db9a102", + "#sha1_content": "0dd92d0d8a7bf6e2f7d1f5ac8954c1bcf18c22a4", + }, + { + "#url": "https://archiveofsins.com/h/", + "#category": ("foolfuuka", "archiveofsins", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://archiveofsins.com/_/search/text/test/", + "#category": ("foolfuuka", "archiveofsins", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://archiveofsins.com/_/search/text/test/", + "#category": ("foolfuuka", "archiveofsins", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://archiveofsins.com/h/gallery/3", + "#category": ("foolfuuka", "archiveofsins", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/artstation.py b/test/results/artstation.py index 4844bd74e7..7d0ebeccbd 100644 --- a/test/results/artstation.py +++ b/test/results/artstation.py @@ -1,232 +1,201 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import artstation from gallery_dl import exception - +from gallery_dl.extractor import artstation __tests__ = ( -{ - "#url" : "https://www.artstation.com/sungchoi/", - "#category": ("", "artstation", "user"), - "#class" : artstation.ArtstationUserExtractor, - "#pattern" : r"https://\w+\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", - "#range" : "1-10", - "#count" : ">= 10", -}, - -{ - "#url" : "https://www.artstation.com/sungchoi/albums/all/", - "#category": ("", "artstation", "user"), - "#class" : artstation.ArtstationUserExtractor, -}, - -{ - "#url" : "https://sungchoi.artstation.com/", - "#category": ("", "artstation", "user"), - "#class" : artstation.ArtstationUserExtractor, -}, - -{ - "#url" : "https://sungchoi.artstation.com/projects/", - "#category": ("", "artstation", "user"), - "#class" : artstation.ArtstationUserExtractor, -}, - -{ - "#url" : "https://www.artstation.com/huimeiye/albums/770899", - "#category": ("", "artstation", "album"), - "#class" : artstation.ArtstationAlbumExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://www.artstation.com/huimeiye/albums/770898", - "#category": ("", "artstation", "album"), - "#class" : artstation.ArtstationAlbumExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://huimeiye.artstation.com/albums/770899", - "#category": ("", "artstation", "album"), - "#class" : artstation.ArtstationAlbumExtractor, -}, - -{ - "#url" : "https://www.artstation.com/mikf/likes", - "#category": ("", "artstation", "likes"), - "#class" : artstation.ArtstationLikesExtractor, - "#pattern" : r"https://\w+\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", - "#count" : 6, -}, - -{ - "#url" : "https://www.artstation.com/mikf/collections/2647023", - "#category": ("", "artstation", "collection"), - "#class" : artstation.ArtstationCollectionExtractor, - "#count" : 10, - - "collection": { - "id" : 2647023, - "is_private" : False, - "name" : "テスト", - "projects_count": 3, - "user_id" : 697975, - "active_projects_count" : 3, - "micro_square_image_url": "https://cdna.artstation.com/p/assets/images/images/005/131/434/micro_square/gaeri-kim-cat-front.jpg?1488720625", - "small_square_image_url": "https://cdna.artstation.com/p/assets/images/images/005/131/434/small_square/gaeri-kim-cat-front.jpg?1488720625", - }, - "user": "mikf", -}, - -{ - "#url" : "https://www.artstation.com/mikf/collections", - "#category": ("", "artstation", "collections"), - "#class" : artstation.ArtstationCollectionsExtractor, - "#urls" : ( - "https://www.artstation.com/mikf/collections/2647023", - "https://www.artstation.com/mikf/collections/2647719", - ), - - "id" : range(2647023, 2647719), - "is_private" : False, - "name" : r"re:テスト|empty", - "projects_count": int, - "user_id" : 697975, - "active_projects_count" : int, - "micro_square_image_url": str, - "small_square_image_url": str, -}, - -{ - "#url" : "https://www.artstation.com/sungchoi/likes", - "#comment" : "no likes", - "#category": ("", "artstation", "likes"), - "#class" : artstation.ArtstationLikesExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.artstation.com/contests/thu-2017/challenges/20", - "#category": ("", "artstation", "challenge"), - "#class" : artstation.ArtstationChallengeExtractor, -}, - -{ - "#url" : "https://www.artstation.com/contests/beyond-human/challenges/23?sorting=winners", - "#category": ("", "artstation", "challenge"), - "#class" : artstation.ArtstationChallengeExtractor, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://www.artstation.com/search?query=ancient&sort_by=rank", - "#category": ("", "artstation", "search"), - "#class" : artstation.ArtstationSearchExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.artstation.com/artwork?sorting=latest", - "#category": ("", "artstation", "artwork"), - "#class" : artstation.ArtstationArtworkExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.artstation.com/artwork/LQVJr", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, - "#pattern" : r"https?://\w+\.artstation\.com/p/assets/images/images/008/760/279/4k/.+", - "#sha1_content": "3f211ce0d6ecdb502db2cdf7bbeceb11d8421170", -}, - -{ - "#url" : "https://www.artstation.com/artwork/Db3dy", - "#comment" : "multiple images per project", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, - "#count" : 4, -}, - -{ - "#url" : "https://www.artstation.com/artwork/lR8b5k", - "#comment" : "artstation video clips (#2566)", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, - "#options" : {"videos": True}, - "#range" : "2-3", - "#urls" : ( - "https://cdn.artstation.com/p/video_sources/000/819/843/infection-4.mp4", - "https://cdn.artstation.com/p/video_sources/000/819/725/infection-veinonly-2.mp4", - ), -}, - -{ - "#url" : "https://www.artstation.com/artwork/g4WPK", - "#comment" : "embedded youtube video", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, - "#options" : {"external": True}, - "#pattern" : r"ytdl:https://www\.youtube(-nocookie)?\.com/embed/JNFfJtwwrU0", - "#range" : "2", -}, - -{ - "#url" : "https://www.artstation.com/artwork/3q3mXB", - "#comment" : "404 (#3016)", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://sungchoi.artstation.com/projects/LQVJr", - "#comment" : "alternate URL patterns", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, -}, - -{ - "#url" : "https://artstn.co/p/LQVJr", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, -}, - -{ - "#url" : "https://www.artstation.com/sungchoi/following", - "#category": ("", "artstation", "following"), - "#class" : artstation.ArtstationFollowingExtractor, - "#pattern" : artstation.ArtstationUserExtractor.pattern, - "#count" : ">= 40", -}, - -{ - "#url" : "https://fede-x-rojas.artstation.com/projects/WBdaZy", - "#comment" : "dash in username", - "#category": ("", "artstation", "image"), - "#class" : artstation.ArtstationImageExtractor, -}, - -{ - "#url" : "https://fede-x-rojas.artstation.com/albums/8533110", - "#comment" : "dash in username", - "#category": ("", "artstation", "album"), - "#class" : artstation.ArtstationAlbumExtractor, -}, - -{ - "#url" : "https://fede-x-rojas.artstation.com/", - "#comment" : "dash in username", - "#category": ("", "artstation", "user"), - "#class" : artstation.ArtstationUserExtractor, -}, - + { + "#url": "https://www.artstation.com/sungchoi/", + "#category": ("", "artstation", "user"), + "#class": artstation.ArtstationUserExtractor, + "#pattern": r"https://\w+\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", + "#range": "1-10", + "#count": ">= 10", + }, + { + "#url": "https://www.artstation.com/sungchoi/albums/all/", + "#category": ("", "artstation", "user"), + "#class": artstation.ArtstationUserExtractor, + }, + { + "#url": "https://sungchoi.artstation.com/", + "#category": ("", "artstation", "user"), + "#class": artstation.ArtstationUserExtractor, + }, + { + "#url": "https://sungchoi.artstation.com/projects/", + "#category": ("", "artstation", "user"), + "#class": artstation.ArtstationUserExtractor, + }, + { + "#url": "https://www.artstation.com/huimeiye/albums/770899", + "#category": ("", "artstation", "album"), + "#class": artstation.ArtstationAlbumExtractor, + "#count": 2, + }, + { + "#url": "https://www.artstation.com/huimeiye/albums/770898", + "#category": ("", "artstation", "album"), + "#class": artstation.ArtstationAlbumExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://huimeiye.artstation.com/albums/770899", + "#category": ("", "artstation", "album"), + "#class": artstation.ArtstationAlbumExtractor, + }, + { + "#url": "https://www.artstation.com/mikf/likes", + "#category": ("", "artstation", "likes"), + "#class": artstation.ArtstationLikesExtractor, + "#pattern": r"https://\w+\.artstation\.com/p/assets/images/images/\d+/\d+/\d+/(4k|large|medium|small)/[^/]+", + "#count": 6, + }, + { + "#url": "https://www.artstation.com/mikf/collections/2647023", + "#category": ("", "artstation", "collection"), + "#class": artstation.ArtstationCollectionExtractor, + "#count": 10, + "collection": { + "id": 2647023, + "is_private": False, + "name": "テスト", + "projects_count": 3, + "user_id": 697975, + "active_projects_count": 3, + "micro_square_image_url": "https://cdna.artstation.com/p/assets/images/images/005/131/434/micro_square/gaeri-kim-cat-front.jpg?1488720625", + "small_square_image_url": "https://cdna.artstation.com/p/assets/images/images/005/131/434/small_square/gaeri-kim-cat-front.jpg?1488720625", + }, + "user": "mikf", + }, + { + "#url": "https://www.artstation.com/mikf/collections", + "#category": ("", "artstation", "collections"), + "#class": artstation.ArtstationCollectionsExtractor, + "#urls": ( + "https://www.artstation.com/mikf/collections/2647023", + "https://www.artstation.com/mikf/collections/2647719", + ), + "id": range(2647023, 2647719), + "is_private": False, + "name": r"re:テスト|empty", + "projects_count": int, + "user_id": 697975, + "active_projects_count": int, + "micro_square_image_url": str, + "small_square_image_url": str, + }, + { + "#url": "https://www.artstation.com/sungchoi/likes", + "#comment": "no likes", + "#category": ("", "artstation", "likes"), + "#class": artstation.ArtstationLikesExtractor, + "#count": 0, + }, + { + "#url": "https://www.artstation.com/contests/thu-2017/challenges/20", + "#category": ("", "artstation", "challenge"), + "#class": artstation.ArtstationChallengeExtractor, + }, + { + "#url": "https://www.artstation.com/contests/beyond-human/challenges/23?sorting=winners", + "#category": ("", "artstation", "challenge"), + "#class": artstation.ArtstationChallengeExtractor, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://www.artstation.com/search?query=ancient&sort_by=rank", + "#category": ("", "artstation", "search"), + "#class": artstation.ArtstationSearchExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.artstation.com/artwork?sorting=latest", + "#category": ("", "artstation", "artwork"), + "#class": artstation.ArtstationArtworkExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.artstation.com/artwork/LQVJr", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + "#pattern": r"https?://\w+\.artstation\.com/p/assets/images/images/008/760/279/4k/.+", + "#sha1_content": "3f211ce0d6ecdb502db2cdf7bbeceb11d8421170", + }, + { + "#url": "https://www.artstation.com/artwork/Db3dy", + "#comment": "multiple images per project", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + "#count": 4, + }, + { + "#url": "https://www.artstation.com/artwork/lR8b5k", + "#comment": "artstation video clips (#2566)", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + "#options": {"videos": True}, + "#range": "2-3", + "#urls": ( + "https://cdn.artstation.com/p/video_sources/000/819/843/infection-4.mp4", + "https://cdn.artstation.com/p/video_sources/000/819/725/infection-veinonly-2.mp4", + ), + }, + { + "#url": "https://www.artstation.com/artwork/g4WPK", + "#comment": "embedded youtube video", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + "#options": {"external": True}, + "#pattern": r"ytdl:https://www\.youtube(-nocookie)?\.com/embed/JNFfJtwwrU0", + "#range": "2", + }, + { + "#url": "https://www.artstation.com/artwork/3q3mXB", + "#comment": "404 (#3016)", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + "#count": 0, + }, + { + "#url": "https://sungchoi.artstation.com/projects/LQVJr", + "#comment": "alternate URL patterns", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + }, + { + "#url": "https://artstn.co/p/LQVJr", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + }, + { + "#url": "https://www.artstation.com/sungchoi/following", + "#category": ("", "artstation", "following"), + "#class": artstation.ArtstationFollowingExtractor, + "#pattern": artstation.ArtstationUserExtractor.pattern, + "#count": ">= 40", + }, + { + "#url": "https://fede-x-rojas.artstation.com/projects/WBdaZy", + "#comment": "dash in username", + "#category": ("", "artstation", "image"), + "#class": artstation.ArtstationImageExtractor, + }, + { + "#url": "https://fede-x-rojas.artstation.com/albums/8533110", + "#comment": "dash in username", + "#category": ("", "artstation", "album"), + "#class": artstation.ArtstationAlbumExtractor, + }, + { + "#url": "https://fede-x-rojas.artstation.com/", + "#comment": "dash in username", + "#category": ("", "artstation", "user"), + "#class": artstation.ArtstationUserExtractor, + }, ) diff --git a/test/results/aryion.py b/test/results/aryion.py index 113a4d95db..f099e150e0 100644 --- a/test/results/aryion.py +++ b/test/results/aryion.py @@ -1,97 +1,84 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import aryion - __tests__ = ( -{ - "#url" : "https://aryion.com/g4/gallery/jameshoward", - "#category": ("", "aryion", "gallery"), - "#class" : aryion.AryionGalleryExtractor, - "#options" : {"recursive": False}, - "#pattern" : r"https://aryion\.com/g4/data\.php\?id=\d+$", - "#range" : "48-52", - "#count" : 5, -}, - -{ - "#url" : "https://aryion.com/g4/user/jameshoward", - "#category": ("", "aryion", "gallery"), - "#class" : aryion.AryionGalleryExtractor, -}, - -{ - "#url" : "https://aryion.com/g4/latest.php?name=jameshoward", - "#category": ("", "aryion", "gallery"), - "#class" : aryion.AryionGalleryExtractor, -}, - -{ - "#url" : "https://aryion.com/g4/favorites/jameshoward", - "#category": ("", "aryion", "favorite"), - "#class" : aryion.AryionFavoriteExtractor, - "#range" : "1-10", - "#count" : 10, - - "user" : "jameshoward", - "artist" : "re:^((?!jameshoward).)*$", -}, - -{ - "#url" : "https://aryion.com/g4/tags.php?tag=star+wars&p=28", - "#category": ("", "aryion", "tag"), - "#class" : aryion.AryionTagExtractor, - "#count" : ">= 5", -}, - -{ - "#url" : "https://aryion.com/g4/view/510079", - "#category": ("", "aryion", "post"), - "#class" : aryion.AryionPostExtractor, - "#sha1_url": "f233286fa5558c07ae500f7f2d5cb0799881450e", - - "artist" : "jameshoward", - "user" : "jameshoward", - "filename" : "jameshoward-510079-subscribestar_150", - "extension" : "jpg", - "id" : 510079, - "width" : 1665, - "height" : 1619, - "size" : 784239, - "title" : "I'm on subscribestar now too!", - "description": r"re:Doesn't hurt to have a backup, right\?", - "tags" : [ - "Non-Vore", - "subscribestar", - ], - "date" : "dt:2019-02-16 19:30:34", - "path" : [], - "views" : int, - "favorites" : int, - "comments" : int, - "_mtime" : "Sat, 16 Feb 2019 19:30:34 GMT", -}, - -{ - "#url" : "https://aryion.com/g4/view/588928", - "#comment" : "x-folder (#694)", - "#category": ("", "aryion", "post"), - "#class" : aryion.AryionPostExtractor, - "#pattern" : aryion.AryionPostExtractor.pattern, - "#count" : ">= 8", -}, - -{ - "#url" : "https://aryion.com/g4/view/537379", - "#comment" : "x-comic-folder (#945)", - "#category": ("", "aryion", "post"), - "#class" : aryion.AryionPostExtractor, - "#pattern" : aryion.AryionPostExtractor.pattern, - "#count" : 2, -}, - + { + "#url": "https://aryion.com/g4/gallery/jameshoward", + "#category": ("", "aryion", "gallery"), + "#class": aryion.AryionGalleryExtractor, + "#options": {"recursive": False}, + "#pattern": r"https://aryion\.com/g4/data\.php\?id=\d+$", + "#range": "48-52", + "#count": 5, + }, + { + "#url": "https://aryion.com/g4/user/jameshoward", + "#category": ("", "aryion", "gallery"), + "#class": aryion.AryionGalleryExtractor, + }, + { + "#url": "https://aryion.com/g4/latest.php?name=jameshoward", + "#category": ("", "aryion", "gallery"), + "#class": aryion.AryionGalleryExtractor, + }, + { + "#url": "https://aryion.com/g4/favorites/jameshoward", + "#category": ("", "aryion", "favorite"), + "#class": aryion.AryionFavoriteExtractor, + "#range": "1-10", + "#count": 10, + "user": "jameshoward", + "artist": "re:^((?!jameshoward).)*$", + }, + { + "#url": "https://aryion.com/g4/tags.php?tag=star+wars&p=28", + "#category": ("", "aryion", "tag"), + "#class": aryion.AryionTagExtractor, + "#count": ">= 5", + }, + { + "#url": "https://aryion.com/g4/view/510079", + "#category": ("", "aryion", "post"), + "#class": aryion.AryionPostExtractor, + "#sha1_url": "f233286fa5558c07ae500f7f2d5cb0799881450e", + "artist": "jameshoward", + "user": "jameshoward", + "filename": "jameshoward-510079-subscribestar_150", + "extension": "jpg", + "id": 510079, + "width": 1665, + "height": 1619, + "size": 784239, + "title": "I'm on subscribestar now too!", + "description": r"re:Doesn't hurt to have a backup, right\?", + "tags": [ + "Non-Vore", + "subscribestar", + ], + "date": "dt:2019-02-16 19:30:34", + "path": [], + "views": int, + "favorites": int, + "comments": int, + "_mtime": "Sat, 16 Feb 2019 19:30:34 GMT", + }, + { + "#url": "https://aryion.com/g4/view/588928", + "#comment": "x-folder (#694)", + "#category": ("", "aryion", "post"), + "#class": aryion.AryionPostExtractor, + "#pattern": aryion.AryionPostExtractor.pattern, + "#count": ">= 8", + }, + { + "#url": "https://aryion.com/g4/view/537379", + "#comment": "x-comic-folder (#945)", + "#category": ("", "aryion", "post"), + "#class": aryion.AryionPostExtractor, + "#pattern": aryion.AryionPostExtractor.pattern, + "#count": 2, + }, ) diff --git a/test/results/atfbooru.py b/test/results/atfbooru.py index 4bdd1b6d46..35d5179792 100644 --- a/test/results/atfbooru.py +++ b/test/results/atfbooru.py @@ -1,39 +1,32 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import danbooru - __tests__ = ( -{ - "#url" : "https://booru.allthefallen.moe/posts?tags=yume_shokunin", - "#category": ("Danbooru", "atfbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, - "#count" : 12, -}, - -{ - "#url" : "https://booru.allthefallen.moe/pools/9", - "#category": ("Danbooru", "atfbooru", "pool"), - "#class" : danbooru.DanbooruPoolExtractor, - "#count" : 6, - "#sha1_url": "902549ffcdb00fe033c3f63e12bc3cb95c5fd8d5", -}, - -{ - "#url" : "https://booru.allthefallen.moe/posts/22", - "#category": ("Danbooru", "atfbooru", "post"), - "#class" : danbooru.DanbooruPostExtractor, - "#sha1_content": "21dda68e1d7e0a554078e62923f537d8e895cac8", -}, - -{ - "#url" : "https://booru.allthefallen.moe/explore/posts/popular", - "#category": ("Danbooru", "atfbooru", "popular"), - "#class" : danbooru.DanbooruPopularExtractor, -}, - + { + "#url": "https://booru.allthefallen.moe/posts?tags=yume_shokunin", + "#category": ("Danbooru", "atfbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + "#count": 12, + }, + { + "#url": "https://booru.allthefallen.moe/pools/9", + "#category": ("Danbooru", "atfbooru", "pool"), + "#class": danbooru.DanbooruPoolExtractor, + "#count": 6, + "#sha1_url": "902549ffcdb00fe033c3f63e12bc3cb95c5fd8d5", + }, + { + "#url": "https://booru.allthefallen.moe/posts/22", + "#category": ("Danbooru", "atfbooru", "post"), + "#class": danbooru.DanbooruPostExtractor, + "#sha1_content": "21dda68e1d7e0a554078e62923f537d8e895cac8", + }, + { + "#url": "https://booru.allthefallen.moe/explore/posts/popular", + "#category": ("Danbooru", "atfbooru", "popular"), + "#class": danbooru.DanbooruPopularExtractor, + }, ) diff --git a/test/results/azurlanewiki.py b/test/results/azurlanewiki.py index 1767342078..35ebb79519 100644 --- a/test/results/azurlanewiki.py +++ b/test/results/azurlanewiki.py @@ -1,25 +1,20 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://azurlane.koumakan.jp/wiki/Azur_Lane_Wiki", - "#category": ("wikimedia", "azurlanewiki", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://azurlane.koumakan.jp/wiki/Louisville/Gallery", - "#comment" : "entries with missing 'imageinfo' (#5384)", - "#category": ("wikimedia", "azurlanewiki", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#count" : "> 10", -}, - + { + "#url": "https://azurlane.koumakan.jp/wiki/Azur_Lane_Wiki", + "#category": ("wikimedia", "azurlanewiki", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://azurlane.koumakan.jp/wiki/Louisville/Gallery", + "#comment": "entries with missing 'imageinfo' (#5384)", + "#category": ("wikimedia", "azurlanewiki", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#count": "> 10", + }, ) diff --git a/test/results/b4k.py b/test/results/b4k.py index 5a2c13c2c4..212b5e1854 100644 --- a/test/results/b4k.py +++ b/test/results/b4k.py @@ -1,30 +1,24 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://arch.b4k.co/meta/thread/196/", - "#category": ("foolfuuka", "b4k", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "d309713d2f838797096b3e9cb44fe514a9c9d07a", -}, - -{ - "#url" : "https://arch.b4k.co/meta/", - "#category": ("foolfuuka", "b4k", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://arch.b4k.co/meta/gallery/", - "#category": ("foolfuuka", "b4k", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://arch.b4k.co/meta/thread/196/", + "#category": ("foolfuuka", "b4k", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "d309713d2f838797096b3e9cb44fe514a9c9d07a", + }, + { + "#url": "https://arch.b4k.co/meta/", + "#category": ("foolfuuka", "b4k", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://arch.b4k.co/meta/gallery/", + "#category": ("foolfuuka", "b4k", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/baraag.py b/test/results/baraag.py index 567d12c53c..30adde6c79 100644 --- a/test/results/baraag.py +++ b/test/results/baraag.py @@ -1,36 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mastodon - __tests__ = ( -{ - "#url" : "https://baraag.net/@pumpkinnsfw", - "#category": ("mastodon", "baraag", "user"), - "#class" : mastodon.MastodonUserExtractor, -}, - -{ - "#url" : "https://baraag.net/bookmarks", - "#category": ("mastodon", "baraag", "bookmark"), - "#class" : mastodon.MastodonBookmarkExtractor, -}, - -{ - "#url" : "https://baraag.net/users/pumpkinnsfw/following", - "#category": ("mastodon", "baraag", "following"), - "#class" : mastodon.MastodonFollowingExtractor, -}, - -{ - "#url" : "https://baraag.net/@pumpkinnsfw/104364170556898443", - "#category": ("mastodon", "baraag", "status"), - "#class" : mastodon.MastodonStatusExtractor, - "#sha1_content": "67748c1b828c58ad60d0fe5729b59fb29c872244", -}, - + { + "#url": "https://baraag.net/@pumpkinnsfw", + "#category": ("mastodon", "baraag", "user"), + "#class": mastodon.MastodonUserExtractor, + }, + { + "#url": "https://baraag.net/bookmarks", + "#category": ("mastodon", "baraag", "bookmark"), + "#class": mastodon.MastodonBookmarkExtractor, + }, + { + "#url": "https://baraag.net/users/pumpkinnsfw/following", + "#category": ("mastodon", "baraag", "following"), + "#class": mastodon.MastodonFollowingExtractor, + }, + { + "#url": "https://baraag.net/@pumpkinnsfw/104364170556898443", + "#category": ("mastodon", "baraag", "status"), + "#class": mastodon.MastodonStatusExtractor, + "#sha1_content": "67748c1b828c58ad60d0fe5729b59fb29c872244", + }, ) diff --git a/test/results/batoto.py b/test/results/batoto.py index 65c8401cd5..f170645cd2 100644 --- a/test/results/batoto.py +++ b/test/results/batoto.py @@ -1,290 +1,258 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import batoto from gallery_dl import exception +from gallery_dl.extractor import batoto __tests__ = ( -{ - "#url" : "https://bato.to/title/86408-i-shall-master-this-family-official/1681030-ch_8", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, - "#count" : 66, - - "chapter" : 8, - "chapter_id" : 1681030, - "chapter_minor": "", - "chapter_url" : "8", - "count" : 66, - "date" : "dt:2021-05-15 18:51:37", - "extension" : "webp", - "filename" : str, - "manga" : "I Shall Master this Family! [Official]", - "manga_id" : 86408, - "page" : range(1, 66), - "title" : "Observing", - "volume" : 0, - -}, - -{ - "#url" : "https://bato.to/title/104929-86-eighty-six-official/1943513-vol_1-ch_5", - "#comment" : "volume (vol) in url", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, - "#count" : 7, - - "manga" : "86--EIGHTY-SIX (Official)", - "title" : "The Spearhead Squadron's Power", - "volume" : 1, - "chapter": 5, -}, - -{ - "#url" : "https://mto.to/chapter/2584460", - "#comment" : "'-' in manga title (#5200)", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, - - "chapter" : 9, - "chapter_id": 2584460, - "chapter_minor": "", - "chapter_url": "9", - "count" : 18, - "date" : "dt:2023-11-26 11:01:12", - "manga" : "Isekai Teni shitara Aiken ga Saikyou ni narimashita - Silver Fenrir to Ore ga Isekai Kurashi wo Hajimetara (Official)", - "manga_id" : 126793, - "title" : "", - "volume" : 0 -}, - -{ - "#url" : "https://bato.to/title/90710-new-suitor-for-the-abandoned-wife/2089747-ch_76", - "#comment" : "duplicate info in chapter_minor / title (#5988)", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, - - "chapter" : 76, - "chapter_id" : 2089747, - "chapter_minor": "", - "chapter_url" : "76", - "title" : "Side Story 4 [END]", -}, - -{ - "#url" : "https://bato.to/title/115494-today-with-you/2631897-ch_38", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, - - "chapter" : 37, - "chapter_id" : 2631897, - "chapter_minor" : "", - "chapter_string": "S1 Episode 37 (End of season)", - "chapter_url" : "38", - "count" : 69, - "date" : "dt:2023-12-20 17:31:18", - "manga" : "Today With You", - "manga_id" : 115494, - "title" : "", - "volume" : 1, -}, - -{ - "#url" : "https://bato.to/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://bato.to/chapter/1681030", - "#comment" : "v2 URL", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://bato.to/title/113742-futsutsuka-na-akujo-de-wa-gozaimasu-ga-suuguu-chouso-torikae-den-official", - "#category": ("", "batoto", "manga"), - "#class" : batoto.BatotoMangaExtractor, - "#count" : ">= 21", - - "chapter" : int, - "chapter_minor": str, - "date" : "type:datetime", - "manga" : "Futsutsuka na Akujo de wa Gozaimasu ga - Suuguu Chouso Torikae Den", - "manga_id" : 113742, -}, - -{ - "#url" : "https://bato.to/title/104929-86-eighty-six-official", - "#comment" : "Manga with number in name", - "#category": ("", "batoto", "manga"), - "#class" : batoto.BatotoMangaExtractor, - "#count" : ">= 18", - - "manga": "86--EIGHTY-SIX (Official)", -}, - -{ - "#url" : "https://bato.to/title/140046-the-grand-duke-s-fox-princess-mgchan", - "#comment" : "Non-English translation (Indonesian)", - "#category": ("", "batoto", "manga"), - "#class" : batoto.BatotoMangaExtractor, - "#count" : ">= 29", - - "manga": "Grand Duke Dan Putri Rubah [cont by LUNABY]", -}, - -{ - "#url" : "https://bato.to/title/134270-removed", - "#comment" : "Deleted/removed manga", - "#category": ("", "batoto", "manga"), - "#class" : batoto.BatotoMangaExtractor, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://bato.to/title/86408-i-shall-master-this-family-official", - "#category": ("", "batoto", "manga"), - "#class" : batoto.BatotoMangaExtractor, -}, - -{ - "#url" : "https://bato.to/series/86408/i-shall-master-this-family-official", - "#comment" : "v2 URL", - "#category": ("", "batoto", "manga"), - "#class" : batoto.BatotoMangaExtractor, -}, - -{ - "#url" : "https://dto.to/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://hto.to/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://mto.to/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://wto.to/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://mangatoto.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://mangatoto.net/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://mangatoto.org/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://batocomic.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://batocomic.net/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://batocomic.org/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://readtoto.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://readtoto.net/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://readtoto.org/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://xbato.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://xbato.net/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://xbato.org/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://zbato.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://zbato.net/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://zbato.org/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://comiko.net/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://comiko.org/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - -{ - "#url" : "https://batotoo.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://batotwo.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, -{ - "#url" : "https://battwo.com/title/86408/1681030", - "#category": ("", "batoto", "chapter"), - "#class" : batoto.BatotoChapterExtractor, -}, - + { + "#url": "https://bato.to/title/86408-i-shall-master-this-family-official/1681030-ch_8", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + "#count": 66, + "chapter": 8, + "chapter_id": 1681030, + "chapter_minor": "", + "chapter_url": "8", + "count": 66, + "date": "dt:2021-05-15 18:51:37", + "extension": "webp", + "filename": str, + "manga": "I Shall Master this Family! [Official]", + "manga_id": 86408, + "page": range(1, 66), + "title": "Observing", + "volume": 0, + }, + { + "#url": "https://bato.to/title/104929-86-eighty-six-official/1943513-vol_1-ch_5", + "#comment": "volume (vol) in url", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + "#count": 7, + "manga": "86--EIGHTY-SIX (Official)", + "title": "The Spearhead Squadron's Power", + "volume": 1, + "chapter": 5, + }, + { + "#url": "https://mto.to/chapter/2584460", + "#comment": "'-' in manga title (#5200)", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + "chapter": 9, + "chapter_id": 2584460, + "chapter_minor": "", + "chapter_url": "9", + "count": 18, + "date": "dt:2023-11-26 11:01:12", + "manga": "Isekai Teni shitara Aiken ga Saikyou ni narimashita - Silver Fenrir to Ore ga Isekai Kurashi wo Hajimetara (Official)", + "manga_id": 126793, + "title": "", + "volume": 0, + }, + { + "#url": "https://bato.to/title/90710-new-suitor-for-the-abandoned-wife/2089747-ch_76", + "#comment": "duplicate info in chapter_minor / title (#5988)", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + "chapter": 76, + "chapter_id": 2089747, + "chapter_minor": "", + "chapter_url": "76", + "title": "Side Story 4 [END]", + }, + { + "#url": "https://bato.to/title/115494-today-with-you/2631897-ch_38", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + "chapter": 37, + "chapter_id": 2631897, + "chapter_minor": "", + "chapter_string": "S1 Episode 37 (End of season)", + "chapter_url": "38", + "count": 69, + "date": "dt:2023-12-20 17:31:18", + "manga": "Today With You", + "manga_id": 115494, + "title": "", + "volume": 1, + }, + { + "#url": "https://bato.to/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://bato.to/chapter/1681030", + "#comment": "v2 URL", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://bato.to/title/113742-futsutsuka-na-akujo-de-wa-gozaimasu-ga-suuguu-chouso-torikae-den-official", + "#category": ("", "batoto", "manga"), + "#class": batoto.BatotoMangaExtractor, + "#count": ">= 21", + "chapter": int, + "chapter_minor": str, + "date": "type:datetime", + "manga": "Futsutsuka na Akujo de wa Gozaimasu ga - Suuguu Chouso Torikae Den", + "manga_id": 113742, + }, + { + "#url": "https://bato.to/title/104929-86-eighty-six-official", + "#comment": "Manga with number in name", + "#category": ("", "batoto", "manga"), + "#class": batoto.BatotoMangaExtractor, + "#count": ">= 18", + "manga": "86--EIGHTY-SIX (Official)", + }, + { + "#url": "https://bato.to/title/140046-the-grand-duke-s-fox-princess-mgchan", + "#comment": "Non-English translation (Indonesian)", + "#category": ("", "batoto", "manga"), + "#class": batoto.BatotoMangaExtractor, + "#count": ">= 29", + "manga": "Grand Duke Dan Putri Rubah [cont by LUNABY]", + }, + { + "#url": "https://bato.to/title/134270-removed", + "#comment": "Deleted/removed manga", + "#category": ("", "batoto", "manga"), + "#class": batoto.BatotoMangaExtractor, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://bato.to/title/86408-i-shall-master-this-family-official", + "#category": ("", "batoto", "manga"), + "#class": batoto.BatotoMangaExtractor, + }, + { + "#url": "https://bato.to/series/86408/i-shall-master-this-family-official", + "#comment": "v2 URL", + "#category": ("", "batoto", "manga"), + "#class": batoto.BatotoMangaExtractor, + }, + { + "#url": "https://dto.to/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://hto.to/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://mto.to/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://wto.to/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://mangatoto.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://mangatoto.net/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://mangatoto.org/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://batocomic.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://batocomic.net/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://batocomic.org/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://readtoto.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://readtoto.net/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://readtoto.org/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://xbato.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://xbato.net/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://xbato.org/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://zbato.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://zbato.net/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://zbato.org/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://comiko.net/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://comiko.org/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://batotoo.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://batotwo.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, + { + "#url": "https://battwo.com/title/86408/1681030", + "#category": ("", "batoto", "chapter"), + "#class": batoto.BatotoChapterExtractor, + }, ) diff --git a/test/results/bbc.py b/test/results/bbc.py index 836786ae58..213d483cd0 100644 --- a/test/results/bbc.py +++ b/test/results/bbc.py @@ -1,49 +1,41 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import bbc - __tests__ = ( -{ - "#url" : "https://www.bbc.co.uk/programmes/p084qtzs/p085g9kg", - "#category": ("", "bbc", "gallery"), - "#class" : bbc.BbcGalleryExtractor, - "#pattern" : r"https://ichef\.bbci\.co\.uk/images/ic/1920xn/\w+\.jpg", - "#count" : 37, - - "programme": "p084qtzs", - "path" : [ - "BBC One", - "Doctor Who (2005–2022)", - "The Timeless Children", - ], -}, - -{ - "#url" : "https://www.bbc.co.uk/programmes/p084qtzs", - "#category": ("", "bbc", "gallery"), - "#class" : bbc.BbcGalleryExtractor, -}, - -{ - "#url" : "https://www.bbc.co.uk/programmes/b006q2x0/galleries", - "#category": ("", "bbc", "programme"), - "#class" : bbc.BbcProgrammeExtractor, - "#pattern" : bbc.BbcGalleryExtractor.pattern, - "#range" : "1-50", - "#count" : ">= 50", -}, - -{ - "#url" : "https://www.bbc.co.uk/programmes/b006q2x0/galleries?page=25", - "#category": ("", "bbc", "programme"), - "#class" : bbc.BbcProgrammeExtractor, - "#pattern" : bbc.BbcGalleryExtractor.pattern, - "#count" : ">= 100", -}, - + { + "#url": "https://www.bbc.co.uk/programmes/p084qtzs/p085g9kg", + "#category": ("", "bbc", "gallery"), + "#class": bbc.BbcGalleryExtractor, + "#pattern": r"https://ichef\.bbci\.co\.uk/images/ic/1920xn/\w+\.jpg", + "#count": 37, + "programme": "p084qtzs", + "path": [ + "BBC One", + "Doctor Who (2005–2022)", + "The Timeless Children", + ], + }, + { + "#url": "https://www.bbc.co.uk/programmes/p084qtzs", + "#category": ("", "bbc", "gallery"), + "#class": bbc.BbcGalleryExtractor, + }, + { + "#url": "https://www.bbc.co.uk/programmes/b006q2x0/galleries", + "#category": ("", "bbc", "programme"), + "#class": bbc.BbcProgrammeExtractor, + "#pattern": bbc.BbcGalleryExtractor.pattern, + "#range": "1-50", + "#count": ">= 50", + }, + { + "#url": "https://www.bbc.co.uk/programmes/b006q2x0/galleries?page=25", + "#category": ("", "bbc", "programme"), + "#class": bbc.BbcProgrammeExtractor, + "#pattern": bbc.BbcGalleryExtractor.pattern, + "#count": ">= 100", + }, ) diff --git a/test/results/bbw-chan.py b/test/results/bbw-chan.py index 1d79a431a2..1f736bbfd0 100644 --- a/test/results/bbw-chan.py +++ b/test/results/bbw-chan.py @@ -1,39 +1,32 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import lynxchan - __tests__ = ( -{ - "#url" : "https://bbw-chan.link/bbwdraw/res/499.html", - "#category": ("lynxchan", "bbw-chan", "thread"), - "#class" : lynxchan.LynxchanThreadExtractor, - "#pattern" : r"https://bbw-chan\.link/\.media/[0-9a-f]{64}(\.\w+)?$", - "#count" : ">= 352", -}, - -{ - "#url" : "https://bbw-chan.nl/bbwdraw/res/489.html", - "#category": ("lynxchan", "bbw-chan", "thread"), - "#class" : lynxchan.LynxchanThreadExtractor, -}, - -{ - "#url" : "https://bbw-chan.link/bbwdraw/", - "#category": ("lynxchan", "bbw-chan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, - "#pattern" : lynxchan.LynxchanThreadExtractor.pattern, - "#count" : ">= 148", -}, - -{ - "#url" : "https://bbw-chan.nl/bbwdraw/2.html", - "#category": ("lynxchan", "bbw-chan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, -}, - + { + "#url": "https://bbw-chan.link/bbwdraw/res/499.html", + "#category": ("lynxchan", "bbw-chan", "thread"), + "#class": lynxchan.LynxchanThreadExtractor, + "#pattern": r"https://bbw-chan\.link/\.media/[0-9a-f]{64}(\.\w+)?$", + "#count": ">= 352", + }, + { + "#url": "https://bbw-chan.nl/bbwdraw/res/489.html", + "#category": ("lynxchan", "bbw-chan", "thread"), + "#class": lynxchan.LynxchanThreadExtractor, + }, + { + "#url": "https://bbw-chan.link/bbwdraw/", + "#category": ("lynxchan", "bbw-chan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + "#pattern": lynxchan.LynxchanThreadExtractor.pattern, + "#count": ">= 148", + }, + { + "#url": "https://bbw-chan.nl/bbwdraw/2.html", + "#category": ("lynxchan", "bbw-chan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + }, ) diff --git a/test/results/bcbnsfw.py b/test/results/bcbnsfw.py index e9fcf8b215..a1f7925c77 100644 --- a/test/results/bcbnsfw.py +++ b/test/results/bcbnsfw.py @@ -1,26 +1,21 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import szurubooru - __tests__ = ( -{ - "#url" : "https://booru.bcbnsfw.space/posts/query=simple_background", - "#category": ("szurubooru", "bcbnsfw", "tag"), - "#class" : szurubooru.SzurubooruTagExtractor, -}, - -{ - "#url" : "https://booru.bcbnsfw.space/post/1599", - "#comment" : "now only available as WebP", - "#category": ("szurubooru", "bcbnsfw", "post"), - "#class" : szurubooru.SzurubooruPostExtractor, - "#pattern" : r"https://booru\.bcbnsfw\.space/data/posts/1599_53784518e92086bd\.png", - "#sha1_content": "55f8b8d187adc82f2dcaf2aa89db0ae21b08c0b0", -}, - + { + "#url": "https://booru.bcbnsfw.space/posts/query=simple_background", + "#category": ("szurubooru", "bcbnsfw", "tag"), + "#class": szurubooru.SzurubooruTagExtractor, + }, + { + "#url": "https://booru.bcbnsfw.space/post/1599", + "#comment": "now only available as WebP", + "#category": ("szurubooru", "bcbnsfw", "post"), + "#class": szurubooru.SzurubooruPostExtractor, + "#pattern": r"https://booru\.bcbnsfw\.space/data/posts/1599_53784518e92086bd\.png", + "#sha1_content": "55f8b8d187adc82f2dcaf2aa89db0ae21b08c0b0", + }, ) diff --git a/test/results/behance.py b/test/results/behance.py index 3a805f813c..bf72bc36fe 100644 --- a/test/results/behance.py +++ b/test/results/behance.py @@ -1,99 +1,86 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import behance from gallery_dl import exception - +from gallery_dl.extractor import behance __tests__ = ( -{ - "#url" : "https://www.behance.net/gallery/17386197/A-Short-Story", - "#category": ("", "behance", "gallery"), - "#class" : behance.BehanceGalleryExtractor, - "#urls" : ( - "ytdl:https://player.vimeo.com/video/97189640?title=0&byline=0&portrait=0&color=ffffff", - "https://mir-s3-cdn-cf.behance.net/project_modules/source/a5a12417386197.562bc055a107d.jpg", - ), - - "id" : 17386197, - "name" : r"re:\"Hi\". A short story about the important things ", - "owners": [ - "Place Studio", - "Julio César Velazquez", - ], - "?fields": [ - "Animation", - "Character Design", - "Directing", - ], - "tags" : list, - "module": dict, - "date" : "dt:2014-06-03 15:41:51", -}, - -{ - "#url" : "https://www.behance.net/gallery/21324767/Nevada-City", - "#category": ("", "behance", "gallery"), - "#class" : behance.BehanceGalleryExtractor, - "#count" : 6, - "#sha1_url": "0258fe194fe7d828d6f2c7f6086a9a0a4140db1d", - - "owners": ["Alex Strohl"], -}, - -{ - "#url" : "https://www.behance.net/gallery/88276087/Audi-R8-RWD", - "#comment" : "'media_collection' modules", - "#category": ("", "behance", "gallery"), - "#class" : behance.BehanceGalleryExtractor, - "#pattern" : r"https://mir-s3-cdn-cf\.behance\.net/project_modules/source/[0-9a-f]+.[0-9a-f]+\.jpg", - "#count" : 20, - "#sha1_url": "6bebff0d37f85349f9ad28bd8b76fd66627c1e2f", -}, - -{ - "#url" : "https://www.behance.net/gallery/101185577/COLCCI", - "#comment" : "'video' modules (#1282)", - "#category": ("", "behance", "gallery"), - "#class" : behance.BehanceGalleryExtractor, - "#pattern" : r"ytdl:https://cdn-prod-ccv\.adobe\.com/\w+/rend/master\.m3u8\?", - "#count" : 3, -}, - -{ - "#url" : "https://www.behance.net/gallery/89270715/Moevir", - "#comment" : "'text' modules (#4799)", - "#category": ("", "behance", "gallery"), - "#class" : behance.BehanceGalleryExtractor, - "#options" : {"modules": "text"}, - "#urls" : """text:
          Make Shift
          https://www.moevir.com/News/make-shif
          Moevir Magazine November Issue 2019
          Photography by Caesar Lima @caephoto 
          Model: Bee @phamhuongbee 
          Makeup by Monica Alvarez @monicaalvarezmakeup 
          Styling by Jessica Boal @jessicaboal 
          Hair by James Gilbert @brandnewjames
          Shot at Vila Sophia
          """, -}, - -{ - "#url" : "https://www.behance.net/gallery/177464639/Kimori", - "#comment" : "mature content (#4417)", - "#category": ("", "behance", "gallery"), - "#class" : behance.BehanceGalleryExtractor, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://www.behance.net/alexstrohl", - "#category": ("", "behance", "user"), - "#class" : behance.BehanceUserExtractor, - "#pattern" : behance.BehanceGalleryExtractor.pattern, - "#count" : ">= 11", -}, - -{ - "#url" : "https://www.behance.net/collection/71340149/inspiration", - "#category": ("", "behance", "collection"), - "#class" : behance.BehanceCollectionExtractor, - "#pattern" : behance.BehanceGalleryExtractor.pattern, - "#count" : ">= 150", -}, - + { + "#url": "https://www.behance.net/gallery/17386197/A-Short-Story", + "#category": ("", "behance", "gallery"), + "#class": behance.BehanceGalleryExtractor, + "#urls": ( + "ytdl:https://player.vimeo.com/video/97189640?title=0&byline=0&portrait=0&color=ffffff", + "https://mir-s3-cdn-cf.behance.net/project_modules/source/a5a12417386197.562bc055a107d.jpg", + ), + "id": 17386197, + "name": r"re:\"Hi\". A short story about the important things ", + "owners": [ + "Place Studio", + "Julio César Velazquez", + ], + "?fields": [ + "Animation", + "Character Design", + "Directing", + ], + "tags": list, + "module": dict, + "date": "dt:2014-06-03 15:41:51", + }, + { + "#url": "https://www.behance.net/gallery/21324767/Nevada-City", + "#category": ("", "behance", "gallery"), + "#class": behance.BehanceGalleryExtractor, + "#count": 6, + "#sha1_url": "0258fe194fe7d828d6f2c7f6086a9a0a4140db1d", + "owners": ["Alex Strohl"], + }, + { + "#url": "https://www.behance.net/gallery/88276087/Audi-R8-RWD", + "#comment": "'media_collection' modules", + "#category": ("", "behance", "gallery"), + "#class": behance.BehanceGalleryExtractor, + "#pattern": r"https://mir-s3-cdn-cf\.behance\.net/project_modules/source/[0-9a-f]+.[0-9a-f]+\.jpg", + "#count": 20, + "#sha1_url": "6bebff0d37f85349f9ad28bd8b76fd66627c1e2f", + }, + { + "#url": "https://www.behance.net/gallery/101185577/COLCCI", + "#comment": "'video' modules (#1282)", + "#category": ("", "behance", "gallery"), + "#class": behance.BehanceGalleryExtractor, + "#pattern": r"ytdl:https://cdn-prod-ccv\.adobe\.com/\w+/rend/master\.m3u8\?", + "#count": 3, + }, + { + "#url": "https://www.behance.net/gallery/89270715/Moevir", + "#comment": "'text' modules (#4799)", + "#category": ("", "behance", "gallery"), + "#class": behance.BehanceGalleryExtractor, + "#options": {"modules": "text"}, + "#urls": """text:
          Make Shift
          https://www.moevir.com/News/make-shif
          Moevir Magazine November Issue 2019
          Photography by Caesar Lima @caephoto 
          Model: Bee @phamhuongbee 
          Makeup by Monica Alvarez @monicaalvarezmakeup 
          Styling by Jessica Boal @jessicaboal 
          Hair by James Gilbert @brandnewjames
          Shot at Vila Sophia
          """, + }, + { + "#url": "https://www.behance.net/gallery/177464639/Kimori", + "#comment": "mature content (#4417)", + "#category": ("", "behance", "gallery"), + "#class": behance.BehanceGalleryExtractor, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://www.behance.net/alexstrohl", + "#category": ("", "behance", "user"), + "#class": behance.BehanceUserExtractor, + "#pattern": behance.BehanceGalleryExtractor.pattern, + "#count": ">= 11", + }, + { + "#url": "https://www.behance.net/collection/71340149/inspiration", + "#category": ("", "behance", "collection"), + "#class": behance.BehanceCollectionExtractor, + "#pattern": behance.BehanceGalleryExtractor.pattern, + "#count": ">= 150", + }, ) diff --git a/test/results/bilibili.py b/test/results/bilibili.py index c32095fddf..727ad5984a 100644 --- a/test/results/bilibili.py +++ b/test/results/bilibili.py @@ -8,38 +8,35 @@ __tests__ = ( -{ - "#url" : "https://www.bilibili.com/opus/988425412565532689", - "#class": bilibili.BilibiliArticleExtractor, - "#urls" : ( - "http://i0.hdslb.com/bfs/new_dyn/311264c4dcf45261f7d7a7fe451b05b9405279279.png", - "http://i0.hdslb.com/bfs/new_dyn/b60d8bc6996529613d617443a12c0a93405279279.png", - "http://i0.hdslb.com/bfs/new_dyn/d4494543210d9eee5310e11dc62581e4405279279.png", - "http://i0.hdslb.com/bfs/new_dyn/45268e63086b2d99811b2e6490130937405279279.png", - ), - - "count" : 4, - "detail" : dict, - "extension": "png", - "filename" : str, - "height" : 800, - "id" : "988425412565532689", - "isClient" : False, - "isPreview": False, - "num" : range(1, 4), - "size" : float, - "theme" : str, - "themeMode": "light", - "url" : str, - "username" : "平平出击", - "width" : 800, -}, - -{ - "#url" : "https://space.bilibili.com/405279279/article", - "#class" : bilibili.BilibiliUserArticlesExtractor, - "#pattern": bilibili.BilibiliArticleExtractor.pattern, - "#count" : range(50, 100), -}, - + { + "#url": "https://www.bilibili.com/opus/988425412565532689", + "#class": bilibili.BilibiliArticleExtractor, + "#urls": ( + "http://i0.hdslb.com/bfs/new_dyn/311264c4dcf45261f7d7a7fe451b05b9405279279.png", + "http://i0.hdslb.com/bfs/new_dyn/b60d8bc6996529613d617443a12c0a93405279279.png", + "http://i0.hdslb.com/bfs/new_dyn/d4494543210d9eee5310e11dc62581e4405279279.png", + "http://i0.hdslb.com/bfs/new_dyn/45268e63086b2d99811b2e6490130937405279279.png", + ), + "count": 4, + "detail": dict, + "extension": "png", + "filename": str, + "height": 800, + "id": "988425412565532689", + "isClient": False, + "isPreview": False, + "num": range(1, 4), + "size": float, + "theme": str, + "themeMode": "light", + "url": str, + "username": "平平出击", + "width": 800, + }, + { + "#url": "https://space.bilibili.com/405279279/article", + "#class": bilibili.BilibiliUserArticlesExtractor, + "#pattern": bilibili.BilibiliArticleExtractor.pattern, + "#count": range(50, 100), + }, ) diff --git a/test/results/bitly.py b/test/results/bitly.py index cbfcf0eb8c..26f3ba2ff9 100644 --- a/test/results/bitly.py +++ b/test/results/bitly.py @@ -1,19 +1,15 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import urlshortener - __tests__ = ( -{ - "#url" : "https://bit.ly/3cWIUgq", - "#category": ("urlshortener", "bitly", "link"), - "#class" : urlshortener.UrlshortenerLinkExtractor, - "#pattern" : "^https://gumroad.com/l/storm_b1", - "#count" : 1, -}, - + { + "#url": "https://bit.ly/3cWIUgq", + "#category": ("urlshortener", "bitly", "link"), + "#class": urlshortener.UrlshortenerLinkExtractor, + "#pattern": "^https://gumroad.com/l/storm_b1", + "#count": 1, + }, ) diff --git a/test/results/blogger.py b/test/results/blogger.py index eef964596f..122f653de4 100644 --- a/test/results/blogger.py +++ b/test/results/blogger.py @@ -1,37 +1,30 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import blogger - __tests__ = ( -{ - "#url" : "blogger:http://www.julianbunker.com/2010/12/moon-rise.html", - "#category": ("blogger", "www.julianbunker.com", "post"), - "#class" : blogger.BloggerPostExtractor, -}, - -{ - "#url" : "blogger:https://www.kefblog.com.ng/", - "#category": ("blogger", "www.kefblog.com.ng", "blog"), - "#class" : blogger.BloggerBlogExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "blogger:http://www.julianbunker.com/search?q=400mm", - "#category": ("blogger", "www.julianbunker.com", "search"), - "#class" : blogger.BloggerSearchExtractor, -}, - -{ - "#url" : "blogger:http://www.julianbunker.com/search/label/D%26D", - "#category": ("blogger", "www.julianbunker.com", "label"), - "#class" : blogger.BloggerLabelExtractor, -}, - + { + "#url": "blogger:http://www.julianbunker.com/2010/12/moon-rise.html", + "#category": ("blogger", "www.julianbunker.com", "post"), + "#class": blogger.BloggerPostExtractor, + }, + { + "#url": "blogger:https://www.kefblog.com.ng/", + "#category": ("blogger", "www.kefblog.com.ng", "blog"), + "#class": blogger.BloggerBlogExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "blogger:http://www.julianbunker.com/search?q=400mm", + "#category": ("blogger", "www.julianbunker.com", "search"), + "#class": blogger.BloggerSearchExtractor, + }, + { + "#url": "blogger:http://www.julianbunker.com/search/label/D%26D", + "#category": ("blogger", "www.julianbunker.com", "label"), + "#class": blogger.BloggerLabelExtractor, + }, ) diff --git a/test/results/blogspot.py b/test/results/blogspot.py index 3c7beadcfc..c542f7fd09 100644 --- a/test/results/blogspot.py +++ b/test/results/blogspot.py @@ -1,94 +1,82 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import blogger - __tests__ = ( -{ - "#url" : "https://julianbphotography.blogspot.com/2010/12/moon-rise.html", - "#category": ("blogger", "blogspot", "post"), - "#class" : blogger.BloggerPostExtractor, - "#urls" : "https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjH9WkPvLJq2moxKtyt3ieJZWSDFQwOi3PHRdlHVHEQHRwy-d86Jg6HWSMhxaa6EgvlXq-zDMmKM4kIPn27eJ9Hepk2X9e9HQhqwMfrT8RYTnFe65uexw7KSk5FdWHxRVp5crz3p_qph3Bj/s0/Icy-Moonrise---For-Web.jpg", - - "blog": { - "date" : "dt:2010-11-21 18:19:42", - "description": "", - "id" : "5623928067739466034", - "kind" : "blogger#blog", - "locale" : dict, - "name" : "Julian Bunker Photography", - "pages" : int, - "posts" : int, - "published" : "2010-11-21T10:19:42-08:00", - "updated" : str, - "url" : "http://julianbphotography.blogspot.com/", + { + "#url": "https://julianbphotography.blogspot.com/2010/12/moon-rise.html", + "#category": ("blogger", "blogspot", "post"), + "#class": blogger.BloggerPostExtractor, + "#urls": "https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjH9WkPvLJq2moxKtyt3ieJZWSDFQwOi3PHRdlHVHEQHRwy-d86Jg6HWSMhxaa6EgvlXq-zDMmKM4kIPn27eJ9Hepk2X9e9HQhqwMfrT8RYTnFe65uexw7KSk5FdWHxRVp5crz3p_qph3Bj/s0/Icy-Moonrise---For-Web.jpg", + "blog": { + "date": "dt:2010-11-21 18:19:42", + "description": "", + "id": "5623928067739466034", + "kind": "blogger#blog", + "locale": dict, + "name": "Julian Bunker Photography", + "pages": int, + "posts": int, + "published": "2010-11-21T10:19:42-08:00", + "updated": str, + "url": "http://julianbphotography.blogspot.com/", + }, + "post": { + "author": "Julian Bunker", + "content": str, + "date": "dt:2010-12-26 01:08:00", + "etag": str, + "id": "6955139236418998998", + "kind": "blogger#post", + "published": "2010-12-25T17:08:00-08:00", + "replies": "0", + "title": "Moon Rise", + "updated": "2011-12-06T05:21:24-08:00", + "url": "http://julianbphotography.blogspot.com/2010/12/moon-rise.html", + }, + "extension": "jpg", + "filename": "Icy-Moonrise---For-Web", + "num": 1, + "url": "https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjH9WkPvLJq2moxKtyt3ieJZWSDFQwOi3PHRdlHVHEQHRwy-d86Jg6HWSMhxaa6EgvlXq-zDMmKM4kIPn27eJ9Hepk2X9e9HQhqwMfrT8RYTnFe65uexw7KSk5FdWHxRVp5crz3p_qph3Bj/s0/Icy-Moonrise---For-Web.jpg", }, - "post": { - "author" : "Julian Bunker", - "content" : str, - "date" : "dt:2010-12-26 01:08:00", - "etag" : str, - "id" : "6955139236418998998", - "kind" : "blogger#post", - "published": "2010-12-25T17:08:00-08:00", - "replies" : "0", - "title" : "Moon Rise", - "updated" : "2011-12-06T05:21:24-08:00", - "url" : "http://julianbphotography.blogspot.com/2010/12/moon-rise.html", + { + "#url": "http://cfnmscenesinmovies.blogspot.com/2011/11/cfnm-scene-jenna-fischer-in-office.html", + "#comment": "video (#587)", + "#category": ("blogger", "blogspot", "post"), + "#class": blogger.BloggerPostExtractor, + "#pattern": r"https://.+\.googlevideo\.com/videoplayback", + }, + { + "#url": "https://randomthingsthroughmyletterbox.blogspot.com/2022/01/bitter-flowers-by-gunnar-staalesen-blog.html", + "#comment": "new image domain (#2204)", + "#category": ("blogger", "blogspot", "post"), + "#class": blogger.BloggerPostExtractor, + "#pattern": r"https://blogger\.googleusercontent\.com/img/.+=s0$", + "#count": 8, + }, + { + "#url": "https://julianbphotography.blogspot.com/", + "#category": ("blogger", "blogspot", "blog"), + "#class": blogger.BloggerBlogExtractor, + "#pattern": r"https://blogger\.googleusercontent\.com/img/.+/s0/", + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://julianbphotography.blogspot.com/search?q=400mm", + "#category": ("blogger", "blogspot", "search"), + "#class": blogger.BloggerSearchExtractor, + "#count": "< 10", + "query": "400mm", + }, + { + "#url": "https://dmmagazine.blogspot.com/search/label/D%26D", + "#category": ("blogger", "blogspot", "label"), + "#class": blogger.BloggerLabelExtractor, + "#range": "1-25", + "#count": 25, + "label": "D&D", }, - "extension": "jpg", - "filename" : "Icy-Moonrise---For-Web", - "num" : 1, - "url" : "https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjH9WkPvLJq2moxKtyt3ieJZWSDFQwOi3PHRdlHVHEQHRwy-d86Jg6HWSMhxaa6EgvlXq-zDMmKM4kIPn27eJ9Hepk2X9e9HQhqwMfrT8RYTnFe65uexw7KSk5FdWHxRVp5crz3p_qph3Bj/s0/Icy-Moonrise---For-Web.jpg", -}, - -{ - "#url" : "http://cfnmscenesinmovies.blogspot.com/2011/11/cfnm-scene-jenna-fischer-in-office.html", - "#comment" : "video (#587)", - "#category": ("blogger", "blogspot", "post"), - "#class" : blogger.BloggerPostExtractor, - "#pattern" : r"https://.+\.googlevideo\.com/videoplayback", -}, - -{ - "#url" : "https://randomthingsthroughmyletterbox.blogspot.com/2022/01/bitter-flowers-by-gunnar-staalesen-blog.html", - "#comment" : "new image domain (#2204)", - "#category": ("blogger", "blogspot", "post"), - "#class" : blogger.BloggerPostExtractor, - "#pattern" : r"https://blogger\.googleusercontent\.com/img/.+=s0$", - "#count" : 8, -}, - -{ - "#url" : "https://julianbphotography.blogspot.com/", - "#category": ("blogger", "blogspot", "blog"), - "#class" : blogger.BloggerBlogExtractor, - "#pattern" : r"https://blogger\.googleusercontent\.com/img/.+/s0/", - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://julianbphotography.blogspot.com/search?q=400mm", - "#category": ("blogger", "blogspot", "search"), - "#class" : blogger.BloggerSearchExtractor, - "#count" : "< 10", - - "query": "400mm", -}, - -{ - "#url" : "https://dmmagazine.blogspot.com/search/label/D%26D", - "#category": ("blogger", "blogspot", "label"), - "#class" : blogger.BloggerLabelExtractor, - "#range" : "1-25", - "#count" : 25, - - "label": "D&D", -}, - ) diff --git a/test/results/bluesky.py b/test/results/bluesky.py index e2da3c0c70..adcddc1262 100644 --- a/test/results/bluesky.py +++ b/test/results/bluesky.py @@ -6,391 +6,349 @@ from gallery_dl.extractor import bluesky - __tests__ = ( -{ - "#url" : "https://bsky.app/profile/bsky.app", - "#category": ("", "bluesky", "user"), - "#class" : bluesky.BlueskyUserExtractor, - "#urls" : ( - "https://bsky.app/profile/bsky.app/media", - ), -}, - -{ - "#url" : "https://www.bsky.app/profile/bsky.app", - "#class" : bluesky.BlueskyUserExtractor, -}, - -{ - "#url" : "https://main.bsky.dev/profile/bsky.app", - "#class" : bluesky.BlueskyUserExtractor, -}, - -{ - "#url" : "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur", - "#category": ("", "bluesky", "user"), - "#class" : bluesky.BlueskyUserExtractor, - "#options" : {"include": "all"}, - "#urls" : ( - "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/avatar", - "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/banner", - "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/posts", - "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/replies", - "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/media", - "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/likes", - ), -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/avatar", - "#category": ("", "bluesky", "avatar"), - "#class" : bluesky.BlueskyAvatarExtractor, - "#urls" : "https://puffball.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:z72i7hdynmk6r22z27h6tvur&cid=bafkreihagr2cmvl2jt4mgx3sppwe2it3fwolkrbtjrhcnwjk4jdijhsoze", -}, - -{ - "#url" : "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/banner", - "#category": ("", "bluesky", "background"), - "#class" : bluesky.BlueskyBackgroundExtractor, - "#urls" : "https://puffball.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:z72i7hdynmk6r22z27h6tvur&cid=bafkreichzyovokfzmymz36p5jibbjrhsur6n7hjnzxrpbt5jaydp2szvna", -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/posts", - "#category": ("", "bluesky", "posts"), - "#class" : bluesky.BlueskyPostsExtractor, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/replies", - "#category": ("", "bluesky", "replies"), - "#class" : bluesky.BlueskyRepliesExtractor, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/media", - "#category": ("", "bluesky", "media"), - "#class" : bluesky.BlueskyMediaExtractor, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://bsky.app/profile/did:plc:jfhpnnst6flqway4eaeqzj2a/feed/for-science", - "#category": ("", "bluesky", "feed"), - "#class" : bluesky.BlueskyFeedExtractor, - "#range" : "1-40", - "#count" : 40, - "#archive" : False, -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/follows", - "#category": ("", "bluesky", "following"), - "#class" : bluesky.BlueskyFollowingExtractor, - "#urls" : ( - "https://bsky.app/profile/did:plc:eon2iu7v3x2ukgxkqaf7e5np", - "https://bsky.app/profile/did:plc:ewvi7nxzyoun6zhxrhs64oiz", - ), -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/likes", - "#category": ("", "bluesky", "likes"), - "#class" : bluesky.BlueskyLikesExtractor, -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/lists/abcdefghijklm", - "#category": ("", "bluesky", "list"), - "#class" : bluesky.BlueskyListExtractor, -}, - -{ - "#url" : "https://bsky.app/search?q=nature", - "#category": ("", "bluesky", "search"), - "#class" : bluesky.BlueskySearchExtractor, - "#range" : "1-40", - "#count" : 40, - "#archive" : False, -}, - -{ - "#url" : "https://bsky.app/hashtag/nature", - "#class" : bluesky.BlueskyHashtagExtractor, - "#range" : "1-40", - "#count" : 40, - "#archive" : False, -}, -{ - "#url" : "https://bsky.app/hashtag/top", - "#class" : bluesky.BlueskyHashtagExtractor, -}, -{ - "#url" : "https://bsky.app/hashtag/nature/latest", - "#class" : bluesky.BlueskyHashtagExtractor, -}, - -{ - "#url" : "https://bsky.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, - "#options" : {"metadata": True}, - "#urls" : "https://puffball.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:z72i7hdynmk6r22z27h6tvur&cid=bafkreidypzoaybmfj5h7pnpiyct6ng5yae6ydp4czrm72ocg7ev6vbirri", - "#sha1_content": "ffcf25e7c511173a12de5276b85903309fcd8d14", - - "author": { - "avatar" : "https://cdn.bsky.app/img/avatar/plain/did:plc:z72i7hdynmk6r22z27h6tvur/bafkreihagr2cmvl2jt4mgx3sppwe2it3fwolkrbtjrhcnwjk4jdijhsoze@jpeg", - "did" : "did:plc:z72i7hdynmk6r22z27h6tvur", - "displayName": "Bluesky", - "handle" : "bsky.app", - "instance" : "bsky.app", - "labels" : [], - }, - "cid" : "bafyreihh7m6bfrwlcjfklwturmja7qfse5gte7lskpmgw76flivimbnoqm", - "count" : 1, - "createdAt" : "2023-12-22T18:58:32.715Z", - "date" : "dt:2023-12-22 18:58:32", - "description": "The bluesky logo with the blue butterfly", - "extension" : "jpeg", - "filename" : "bafkreidypzoaybmfj5h7pnpiyct6ng5yae6ydp4czrm72ocg7ev6vbirri", - "height" : 630, - "indexedAt" : "2023-12-22T18:58:32.715Z", - "instance" : "bsky.app", - "labels" : [], - "likeCount" : int, - "num" : 1, - "post_id" : "3kh5rarr3gn2n", - "replyCount" : int, - "repostCount": int, - "uri" : "at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.post/3kh5rarr3gn2n", - "width" : 1200, - "hashtags" : [], - "mentions" : [], - "uris" : ["https://blueskyweb.xyz/blog/12-21-2023-butterfly"], - "user" : { - "avatar" : str, - "banner" : str, - "description" : str, - "did" : "did:plc:z72i7hdynmk6r22z27h6tvur", - "displayName" : "Bluesky", - "followersCount": int, - "followsCount" : int, - "handle" : "bsky.app", - "instance" : "bsky.app", - "indexedAt" : str, - "labels" : [], - "postsCount" : int, - }, -}, - -{ - "#url" : "https://bsky.app/profile/mikf.bsky.social/post/3kkzc3xaf5m2w", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, - "#options" : {"metadata": "facets"}, - "#urls" : "https://conocybe.us-west.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:cslxjqkeexku6elp5xowxkq7&cid=bafkreib7ydpe3xxo4cq7nn32w7eqhcanfaanz6caepd2z4kzplxtx2ctgi", - "#sha1_content": "9cf5748f6d00aae83fbb3cc2c6eb3caa832b90f4", - - "author": { - "did" : "did:plc:cslxjqkeexku6elp5xowxkq7", - "displayName": "mikf", - "handle" : "mikf.bsky.social", - "instance" : "bsky.social", - "labels" : [], - }, - "cid" : "bafyreihtck7clocti2qshaiounadof74pxqhz7gnvbstxujqzhlodigqru", - "count" : 1, - "createdAt" : "2024-02-09T21:57:31.917Z", - "date" : "dt:2024-02-09 21:57:31", - "description": "reading lewd books", - "extension" : "jpeg", - "filename" : "bafkreib7ydpe3xxo4cq7nn32w7eqhcanfaanz6caepd2z4kzplxtx2ctgi", - "hashtags" : [ - "patchouli", - "patchy", - ], - "mentions" : [ - "did:plc:cslxjqkeexku6elp5xowxkq7", - ], - "uris" : [ - "https://seiga.nicovideo.jp/seiga/im5977527", - ], - "width" : 1024, - "height" : 768, - "langs" : ["en"], - "likeCount" : int, - "num" : 1, - "post_id" : "3kkzc3xaf5m2w", - "replyCount" : int, - "repostCount": int, - "text" : "testing \"facets\"\n\nsource: seiga.nicovideo.jp/seiga/im5977...\n#patchouli #patchy\n@mikf.bsky.social", - "uri" : "at://did:plc:cslxjqkeexku6elp5xowxkq7/app.bsky.feed.post/3kkzc3xaf5m2w", -}, - -{ - "#url" : "https://bsky.app/profile/go-guiltism.bsky.social/post/3klgth6lilt2l", - "#comment" : "different embed CID path", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, - "#urls" : "https://amanita.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:owc2r2dsewj3hk73rtd746zh&cid=bafkreieuhplc7fpbvi3suvacaf2dqxzvuu4hgl5o6eifqb76tf3uopldmi", -}, - -{ - "#url" : "https://bsky.app/profile/mikf.bsky.social/post/3l46q5glfex27", - "#comment" : "video (#6183)", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, - "#urls" : "https://conocybe.us-west.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:cslxjqkeexku6elp5xowxkq7&cid=bafkreihq2nsfocrnlpx4nykb4szouqszxwmy3ucnk4k46nx5t6hjnxlti4", - - "description": "kirby and reimu dance", - "text" : "video", - "width" : 1280, - "height" : 720, - "filename" : "bafkreihq2nsfocrnlpx4nykb4szouqszxwmy3ucnk4k46nx5t6hjnxlti4", - "extension" : "mp4", -}, - -{ - "#url" : "https://bsky.app/profile/mikf.bsky.social/post/3kmfodjotln2f", - "#comment" : "quote (#6183)", - "#class" : bluesky.BlueskyPostExtractor, - "#options" : {"quoted": True}, - "#urls" : "https://lionsmane.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:eyhmjdxsnthqhvvszdejaocz&cid=bafkreib6eb7tfozksquveaj3z5msyx3hkniubrulxdys3eftthvmuzrtme", - - "author": { - "associated" : dict, - "avatar" : "https://cdn.bsky.app/img/avatar/plain/did:plc:eyhmjdxsnthqhvvszdejaocz/bafkreigjrftlw7tabtpie32saydttpnoi7276v252vnycr6zt6euef7vdi@jpeg", - "createdAt" : "2024-01-11T00:27:37.404Z", - "did" : "did:plc:eyhmjdxsnthqhvvszdejaocz", - "displayName": "フナ", - "handle" : "ykfuna.bsky.social", - "labels" : list, - }, - "quote_by": { - "avatar" : "https://cdn.bsky.app/img/avatar/plain/did:plc:cslxjqkeexku6elp5xowxkq7/bafkreic5jqkn5ohqhgsm6zzi7vnapuz54trojv3io4tfkrcyaprl4b2ztm@jpeg", - "createdAt" : "2024-02-05T00:03:54.087Z", - "did" : "did:plc:cslxjqkeexku6elp5xowxkq7", - "displayName": "mikf", - "handle" : "mikf.bsky.social", - "labels" : list, - }, - "quote_id": "3kmfodjotln2f", - "post_id" : "3km4qy5y3jc2z", -}, - -{ - "#url" : "https://bsky.app/profile/mikf.bsky.social/post/3kmfp2qktil25", - "#comment" : "quote with media (#6183)", - "#class" : bluesky.BlueskyPostExtractor, - "#options" : {"quoted": True}, - "#urls" : ( - "https://conocybe.us-west.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:cslxjqkeexku6elp5xowxkq7&cid=bafkreiegcyremdrecmnpisci3a3nduc7lm3zdcl76z5o5rd4nstyolrxki", - "https://lionsmane.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:eyhmjdxsnthqhvvszdejaocz&cid=bafkreicojrnwiw5eqo3ko2q6duduyjaoyiqvdc25kuikcedlijtbgvlt5e", - - ), - - "text" : {"quote with media", ""}, -}, - -{ - "#url" : "https://bsky.app/profile/nytimes.com/post/3l7xvcjgdxg2g", - "#comment" : "instance metadata", - "#class" : bluesky.BlueskyPostExtractor, - "#options" : {"metadata": "user"}, - - "instance": "bsky.app", - "author": { - "createdAt" : "2023-06-05T18:50:31.498Z", - "did" : "did:plc:eclio37ymobqex2ncko63h4r", - "displayName": "The New York Times", - "handle" : "nytimes.com", - "instance" : "nytimes.com", - }, - "user": { - "avatar" : "https://cdn.bsky.app/img/avatar/plain/did:plc:eclio37ymobqex2ncko63h4r/bafkreidvvqj5jymmpaeklwkpq6gi532el447mjy2yultuukypzqm5ohfju@jpeg", - "banner" : "https://cdn.bsky.app/img/banner/plain/did:plc:eclio37ymobqex2ncko63h4r/bafkreiaiorkgl6t2j5w3sf6nj37drvwuvriq3e3vqwf4yn3pchpwfbekta@jpeg", - "createdAt" : "2023-06-05T18:50:31.498Z", - "description" : "In-depth, independent reporting to better understand the world, now on Bluesky. News tips? Share them here: http://nyti.ms/2FVHq9v", - "did" : "did:plc:eclio37ymobqex2ncko63h4r", - "displayName" : "The New York Times", - "followersCount": int, - "followsCount" : int, - "handle" : "nytimes.com", - "instance" : "nytimes.com", - "indexedAt" : "2024-01-20T05:04:46.757Z", - "labels" : [], - "postsCount" : int, - }, -}, - -{ - "#url" : "https://bsky.app/profile/stupidsaru.woke.cat/post/3l66wwwqw6u2w", - "#comment" : "instance metadata", - "#class" : bluesky.BlueskyPostExtractor, - - "author": { - "createdAt": "2023-08-31T23:28:42.305Z", - "did" : "did:plc:b7s3pdcjk6qvxmu3n674hlgj", - "handle" : "stupidsaru.woke.cat", - "instance" : "woke.cat", + { + "#url": "https://bsky.app/profile/bsky.app", + "#category": ("", "bluesky", "user"), + "#class": bluesky.BlueskyUserExtractor, + "#urls": ("https://bsky.app/profile/bsky.app/media",), }, -}, - -{ - "#url" : "https://bsky.app/profile/alt.bun.how/post/3l7rdfxhyds2f", - "#comment" : "non-bsky PDS (#6406)", - "#class" : bluesky.BlueskyPostExtractor, - "#urls" : "https://pds.bun.how/xrpc/com.atproto.sync.getBlob?did=did:plc:7x6rtuenkuvxq3zsvffp2ide&cid=bafkreielhgekjheckgjusx7x5hxkbrqryfdmzdwwp2zoxchovgnpzkxzae", - "#sha1_content": "1777956de0dc8cf0815c5c7eb574a24ce54a1d42", - - "author": { - "createdAt": "2024-10-17T13:55:48.833Z", - "did" : "did:plc:7x6rtuenkuvxq3zsvffp2ide", - "handle" : "alt.bun.how", - "instance" : "bun.how", + { + "#url": "https://www.bsky.app/profile/bsky.app", + "#class": bluesky.BlueskyUserExtractor, + }, + { + "#url": "https://main.bsky.dev/profile/bsky.app", + "#class": bluesky.BlueskyUserExtractor, + }, + { + "#url": "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur", + "#category": ("", "bluesky", "user"), + "#class": bluesky.BlueskyUserExtractor, + "#options": {"include": "all"}, + "#urls": ( + "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/avatar", + "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/banner", + "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/posts", + "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/replies", + "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/media", + "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/likes", + ), + }, + { + "#url": "https://bsky.app/profile/bsky.app/avatar", + "#category": ("", "bluesky", "avatar"), + "#class": bluesky.BlueskyAvatarExtractor, + "#urls": "https://puffball.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:z72i7hdynmk6r22z27h6tvur&cid=bafkreihagr2cmvl2jt4mgx3sppwe2it3fwolkrbtjrhcnwjk4jdijhsoze", + }, + { + "#url": "https://bsky.app/profile/did:plc:z72i7hdynmk6r22z27h6tvur/banner", + "#category": ("", "bluesky", "background"), + "#class": bluesky.BlueskyBackgroundExtractor, + "#urls": "https://puffball.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:z72i7hdynmk6r22z27h6tvur&cid=bafkreichzyovokfzmymz36p5jibbjrhsur6n7hjnzxrpbt5jaydp2szvna", + }, + { + "#url": "https://bsky.app/profile/bsky.app/posts", + "#category": ("", "bluesky", "posts"), + "#class": bluesky.BlueskyPostsExtractor, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "https://bsky.app/profile/bsky.app/replies", + "#category": ("", "bluesky", "replies"), + "#class": bluesky.BlueskyRepliesExtractor, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "https://bsky.app/profile/bsky.app/media", + "#category": ("", "bluesky", "media"), + "#class": bluesky.BlueskyMediaExtractor, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "https://bsky.app/profile/did:plc:jfhpnnst6flqway4eaeqzj2a/feed/for-science", + "#category": ("", "bluesky", "feed"), + "#class": bluesky.BlueskyFeedExtractor, + "#range": "1-40", + "#count": 40, + "#archive": False, + }, + { + "#url": "https://bsky.app/profile/bsky.app/follows", + "#category": ("", "bluesky", "following"), + "#class": bluesky.BlueskyFollowingExtractor, + "#urls": ( + "https://bsky.app/profile/did:plc:eon2iu7v3x2ukgxkqaf7e5np", + "https://bsky.app/profile/did:plc:ewvi7nxzyoun6zhxrhs64oiz", + ), + }, + { + "#url": "https://bsky.app/profile/bsky.app/likes", + "#category": ("", "bluesky", "likes"), + "#class": bluesky.BlueskyLikesExtractor, + }, + { + "#url": "https://bsky.app/profile/bsky.app/lists/abcdefghijklm", + "#category": ("", "bluesky", "list"), + "#class": bluesky.BlueskyListExtractor, + }, + { + "#url": "https://bsky.app/search?q=nature", + "#category": ("", "bluesky", "search"), + "#class": bluesky.BlueskySearchExtractor, + "#range": "1-40", + "#count": 40, + "#archive": False, + }, + { + "#url": "https://bsky.app/hashtag/nature", + "#class": bluesky.BlueskyHashtagExtractor, + "#range": "1-40", + "#count": 40, + "#archive": False, + }, + { + "#url": "https://bsky.app/hashtag/top", + "#class": bluesky.BlueskyHashtagExtractor, + }, + { + "#url": "https://bsky.app/hashtag/nature/latest", + "#class": bluesky.BlueskyHashtagExtractor, + }, + { + "#url": "https://bsky.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + "#options": {"metadata": True}, + "#urls": "https://puffball.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:z72i7hdynmk6r22z27h6tvur&cid=bafkreidypzoaybmfj5h7pnpiyct6ng5yae6ydp4czrm72ocg7ev6vbirri", + "#sha1_content": "ffcf25e7c511173a12de5276b85903309fcd8d14", + "author": { + "avatar": "https://cdn.bsky.app/img/avatar/plain/did:plc:z72i7hdynmk6r22z27h6tvur/bafkreihagr2cmvl2jt4mgx3sppwe2it3fwolkrbtjrhcnwjk4jdijhsoze@jpeg", + "did": "did:plc:z72i7hdynmk6r22z27h6tvur", + "displayName": "Bluesky", + "handle": "bsky.app", + "instance": "bsky.app", + "labels": [], + }, + "cid": "bafyreihh7m6bfrwlcjfklwturmja7qfse5gte7lskpmgw76flivimbnoqm", + "count": 1, + "createdAt": "2023-12-22T18:58:32.715Z", + "date": "dt:2023-12-22 18:58:32", + "description": "The bluesky logo with the blue butterfly", + "extension": "jpeg", + "filename": "bafkreidypzoaybmfj5h7pnpiyct6ng5yae6ydp4czrm72ocg7ev6vbirri", + "height": 630, + "indexedAt": "2023-12-22T18:58:32.715Z", + "instance": "bsky.app", + "labels": [], + "likeCount": int, + "num": 1, + "post_id": "3kh5rarr3gn2n", + "replyCount": int, + "repostCount": int, + "uri": "at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.post/3kh5rarr3gn2n", + "width": 1200, + "hashtags": [], + "mentions": [], + "uris": ["https://blueskyweb.xyz/blog/12-21-2023-butterfly"], + "user": { + "avatar": str, + "banner": str, + "description": str, + "did": "did:plc:z72i7hdynmk6r22z27h6tvur", + "displayName": "Bluesky", + "followersCount": int, + "followsCount": int, + "handle": "bsky.app", + "instance": "bsky.app", + "indexedAt": str, + "labels": [], + "postsCount": int, + }, + }, + { + "#url": "https://bsky.app/profile/mikf.bsky.social/post/3kkzc3xaf5m2w", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + "#options": {"metadata": "facets"}, + "#urls": "https://conocybe.us-west.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:cslxjqkeexku6elp5xowxkq7&cid=bafkreib7ydpe3xxo4cq7nn32w7eqhcanfaanz6caepd2z4kzplxtx2ctgi", + "#sha1_content": "9cf5748f6d00aae83fbb3cc2c6eb3caa832b90f4", + "author": { + "did": "did:plc:cslxjqkeexku6elp5xowxkq7", + "displayName": "mikf", + "handle": "mikf.bsky.social", + "instance": "bsky.social", + "labels": [], + }, + "cid": "bafyreihtck7clocti2qshaiounadof74pxqhz7gnvbstxujqzhlodigqru", + "count": 1, + "createdAt": "2024-02-09T21:57:31.917Z", + "date": "dt:2024-02-09 21:57:31", + "description": "reading lewd books", + "extension": "jpeg", + "filename": "bafkreib7ydpe3xxo4cq7nn32w7eqhcanfaanz6caepd2z4kzplxtx2ctgi", + "hashtags": [ + "patchouli", + "patchy", + ], + "mentions": [ + "did:plc:cslxjqkeexku6elp5xowxkq7", + ], + "uris": [ + "https://seiga.nicovideo.jp/seiga/im5977527", + ], + "width": 1024, + "height": 768, + "langs": ["en"], + "likeCount": int, + "num": 1, + "post_id": "3kkzc3xaf5m2w", + "replyCount": int, + "repostCount": int, + "text": 'testing "facets"\n\nsource: seiga.nicovideo.jp/seiga/im5977...\n#patchouli #patchy\n@mikf.bsky.social', + "uri": "at://did:plc:cslxjqkeexku6elp5xowxkq7/app.bsky.feed.post/3kkzc3xaf5m2w", + }, + { + "#url": "https://bsky.app/profile/go-guiltism.bsky.social/post/3klgth6lilt2l", + "#comment": "different embed CID path", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + "#urls": "https://amanita.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:owc2r2dsewj3hk73rtd746zh&cid=bafkreieuhplc7fpbvi3suvacaf2dqxzvuu4hgl5o6eifqb76tf3uopldmi", + }, + { + "#url": "https://bsky.app/profile/mikf.bsky.social/post/3l46q5glfex27", + "#comment": "video (#6183)", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + "#urls": "https://conocybe.us-west.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:cslxjqkeexku6elp5xowxkq7&cid=bafkreihq2nsfocrnlpx4nykb4szouqszxwmy3ucnk4k46nx5t6hjnxlti4", + "description": "kirby and reimu dance", + "text": "video", + "width": 1280, + "height": 720, + "filename": "bafkreihq2nsfocrnlpx4nykb4szouqszxwmy3ucnk4k46nx5t6hjnxlti4", + "extension": "mp4", + }, + { + "#url": "https://bsky.app/profile/mikf.bsky.social/post/3kmfodjotln2f", + "#comment": "quote (#6183)", + "#class": bluesky.BlueskyPostExtractor, + "#options": {"quoted": True}, + "#urls": "https://lionsmane.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:eyhmjdxsnthqhvvszdejaocz&cid=bafkreib6eb7tfozksquveaj3z5msyx3hkniubrulxdys3eftthvmuzrtme", + "author": { + "associated": dict, + "avatar": "https://cdn.bsky.app/img/avatar/plain/did:plc:eyhmjdxsnthqhvvszdejaocz/bafkreigjrftlw7tabtpie32saydttpnoi7276v252vnycr6zt6euef7vdi@jpeg", + "createdAt": "2024-01-11T00:27:37.404Z", + "did": "did:plc:eyhmjdxsnthqhvvszdejaocz", + "displayName": "フナ", + "handle": "ykfuna.bsky.social", + "labels": list, + }, + "quote_by": { + "avatar": "https://cdn.bsky.app/img/avatar/plain/did:plc:cslxjqkeexku6elp5xowxkq7/bafkreic5jqkn5ohqhgsm6zzi7vnapuz54trojv3io4tfkrcyaprl4b2ztm@jpeg", + "createdAt": "2024-02-05T00:03:54.087Z", + "did": "did:plc:cslxjqkeexku6elp5xowxkq7", + "displayName": "mikf", + "handle": "mikf.bsky.social", + "labels": list, + }, + "quote_id": "3kmfodjotln2f", + "post_id": "3km4qy5y3jc2z", + }, + { + "#url": "https://bsky.app/profile/mikf.bsky.social/post/3kmfp2qktil25", + "#comment": "quote with media (#6183)", + "#class": bluesky.BlueskyPostExtractor, + "#options": {"quoted": True}, + "#urls": ( + "https://conocybe.us-west.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:cslxjqkeexku6elp5xowxkq7&cid=bafkreiegcyremdrecmnpisci3a3nduc7lm3zdcl76z5o5rd4nstyolrxki", + "https://lionsmane.us-east.host.bsky.network/xrpc/com.atproto.sync.getBlob?did=did:plc:eyhmjdxsnthqhvvszdejaocz&cid=bafkreicojrnwiw5eqo3ko2q6duduyjaoyiqvdc25kuikcedlijtbgvlt5e", + ), + "text": {"quote with media", ""}, + }, + { + "#url": "https://bsky.app/profile/nytimes.com/post/3l7xvcjgdxg2g", + "#comment": "instance metadata", + "#class": bluesky.BlueskyPostExtractor, + "#options": {"metadata": "user"}, + "instance": "bsky.app", + "author": { + "createdAt": "2023-06-05T18:50:31.498Z", + "did": "did:plc:eclio37ymobqex2ncko63h4r", + "displayName": "The New York Times", + "handle": "nytimes.com", + "instance": "nytimes.com", + }, + "user": { + "avatar": "https://cdn.bsky.app/img/avatar/plain/did:plc:eclio37ymobqex2ncko63h4r/bafkreidvvqj5jymmpaeklwkpq6gi532el447mjy2yultuukypzqm5ohfju@jpeg", + "banner": "https://cdn.bsky.app/img/banner/plain/did:plc:eclio37ymobqex2ncko63h4r/bafkreiaiorkgl6t2j5w3sf6nj37drvwuvriq3e3vqwf4yn3pchpwfbekta@jpeg", + "createdAt": "2023-06-05T18:50:31.498Z", + "description": "In-depth, independent reporting to better understand the world, now on Bluesky. News tips? Share them here: http://nyti.ms/2FVHq9v", + "did": "did:plc:eclio37ymobqex2ncko63h4r", + "displayName": "The New York Times", + "followersCount": int, + "followsCount": int, + "handle": "nytimes.com", + "instance": "nytimes.com", + "indexedAt": "2024-01-20T05:04:46.757Z", + "labels": [], + "postsCount": int, + }, + }, + { + "#url": "https://bsky.app/profile/stupidsaru.woke.cat/post/3l66wwwqw6u2w", + "#comment": "instance metadata", + "#class": bluesky.BlueskyPostExtractor, + "author": { + "createdAt": "2023-08-31T23:28:42.305Z", + "did": "did:plc:b7s3pdcjk6qvxmu3n674hlgj", + "handle": "stupidsaru.woke.cat", + "instance": "woke.cat", + }, + }, + { + "#url": "https://bsky.app/profile/alt.bun.how/post/3l7rdfxhyds2f", + "#comment": "non-bsky PDS (#6406)", + "#class": bluesky.BlueskyPostExtractor, + "#urls": "https://pds.bun.how/xrpc/com.atproto.sync.getBlob?did=did:plc:7x6rtuenkuvxq3zsvffp2ide&cid=bafkreielhgekjheckgjusx7x5hxkbrqryfdmzdwwp2zoxchovgnpzkxzae", + "#sha1_content": "1777956de0dc8cf0815c5c7eb574a24ce54a1d42", + "author": { + "createdAt": "2024-10-17T13:55:48.833Z", + "did": "did:plc:7x6rtuenkuvxq3zsvffp2ide", + "handle": "alt.bun.how", + "instance": "bun.how", + }, + }, + { + "#url": "https://cbsky.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + }, + { + "#url": "https://bskye.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + }, + { + "#url": "https://bskyx.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + }, + { + "#url": "https://bsyy.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + }, + { + "#url": "https://fxbsky.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, + }, + { + "#url": "https://vxbsky.app/profile/bsky.app/post/3kh5rarr3gn2n", + "#category": ("", "bluesky", "post"), + "#class": bluesky.BlueskyPostExtractor, }, -}, - -{ - "#url" : "https://cbsky.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, -}, - -{ - "#url" : "https://bskye.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, -}, - -{ - "#url" : "https://bskyx.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, -}, - -{ - "#url" : "https://bsyy.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, -}, - -{ - "#url" : "https://fxbsky.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, -}, - -{ - "#url" : "https://vxbsky.app/profile/bsky.app/post/3kh5rarr3gn2n", - "#category": ("", "bluesky", "post"), - "#class" : bluesky.BlueskyPostExtractor, -}, - ) diff --git a/test/results/booruvar.py b/test/results/booruvar.py index 8beb45e173..dcb669e4cc 100644 --- a/test/results/booruvar.py +++ b/test/results/booruvar.py @@ -1,40 +1,33 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import danbooru - __tests__ = ( -{ - "#url" : "https://booru.borvar.art/posts?tags=chibi&z=1", - "#category": ("Danbooru", "booruvar", "tag"), - "#class" : danbooru.DanbooruTagExtractor, - "#pattern" : r"https://booru\.borvar\.art/data/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", - "#count" : ">= 3", -}, - -{ - "#url" : "https://booru.borvar.art/pools/2", - "#category": ("Danbooru", "booruvar", "pool"), - "#class" : danbooru.DanbooruPoolExtractor, - "#count" : 4, - "#sha1_url": "77fa3559a3fc919f72611f4e3dd0f919d19d3e0d", -}, - -{ - "#url" : "https://booru.borvar.art/posts/1487", - "#category": ("Danbooru", "booruvar", "post"), - "#class" : danbooru.DanbooruPostExtractor, - "#sha1_content": "91273ac1ea413a12be468841e2b5804656a50bff", -}, - -{ - "#url" : "https://booru.borvar.art/explore/posts/popular", - "#category": ("Danbooru", "booruvar", "popular"), - "#class" : danbooru.DanbooruPopularExtractor, -}, - + { + "#url": "https://booru.borvar.art/posts?tags=chibi&z=1", + "#category": ("Danbooru", "booruvar", "tag"), + "#class": danbooru.DanbooruTagExtractor, + "#pattern": r"https://booru\.borvar\.art/data/original/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w+", + "#count": ">= 3", + }, + { + "#url": "https://booru.borvar.art/pools/2", + "#category": ("Danbooru", "booruvar", "pool"), + "#class": danbooru.DanbooruPoolExtractor, + "#count": 4, + "#sha1_url": "77fa3559a3fc919f72611f4e3dd0f919d19d3e0d", + }, + { + "#url": "https://booru.borvar.art/posts/1487", + "#category": ("Danbooru", "booruvar", "post"), + "#class": danbooru.DanbooruPostExtractor, + "#sha1_content": "91273ac1ea413a12be468841e2b5804656a50bff", + }, + { + "#url": "https://booru.borvar.art/explore/posts/popular", + "#category": ("Danbooru", "booruvar", "popular"), + "#class": danbooru.DanbooruPopularExtractor, + }, ) diff --git a/test/results/boosty.py b/test/results/boosty.py index 2da0e6b3ca..00d2020469 100644 --- a/test/results/boosty.py +++ b/test/results/boosty.py @@ -6,130 +6,118 @@ from gallery_dl.extractor import boosty - __tests__ = ( -{ - "#url" : "https://boosty.to/milshoo", - "#class" : boosty.BoostyUserExtractor, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://boosty.to/milshoo?postsFrom=1706742000&postsTo=1709247599", - "#class" : boosty.BoostyUserExtractor, - "#urls" : "https://images.boosty.to/image/ff0d2006-3ee7-483d-a5fc-2a05b531742c?change_time=1707829201", -}, - -{ - "#url" : "https://boosty.to/milshoo/media/all", - "#class" : boosty.BoostyMediaExtractor, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://boosty.to/milshoo/posts/4304d8f0-3f49-4f97-a3f3-9f064bc32b2f", - "#class" : boosty.BoostyPostExtractor, - "#urls" : "https://images.boosty.to/image/75f86086-fc67-4ed2-9365-2958d3d1a8f7?change_time=1711027786", - - "count" : 1, - "num" : 1, - "extension": "", - "filename" : "75f86086-fc67-4ed2-9365-2958d3d1a8f7", - - "file": { - "height" : 2048, - "id" : "75f86086-fc67-4ed2-9365-2958d3d1a8f7", - "rendition": "", - "size" : 1094903, - "type" : "image", - "url" : "https://images.boosty.to/image/75f86086-fc67-4ed2-9365-2958d3d1a8f7?change_time=1711027786", - "width" : 2048, + { + "#url": "https://boosty.to/milshoo", + "#class": boosty.BoostyUserExtractor, + "#range": "1-40", + "#count": 40, }, - "user": { - "avatarUrl": "https://images.boosty.to/user/173542/avatar?change_time=1580888689", - "blogUrl" : "milshoo", - "flags" : { - "showPostDonations": True, - }, - "hasAvatar": True, - "id" : 173542, - "name" : "Милшу", + { + "#url": "https://boosty.to/milshoo?postsFrom=1706742000&postsTo=1709247599", + "#class": boosty.BoostyUserExtractor, + "#urls": "https://images.boosty.to/image/ff0d2006-3ee7-483d-a5fc-2a05b531742c?change_time=1707829201", + }, + { + "#url": "https://boosty.to/milshoo/media/all", + "#class": boosty.BoostyMediaExtractor, + "#range": "1-40", + "#count": 40, }, - "post": { - "comments" : dict, - "content" : [ - "Привет! Это Милшу ) Я открываю комментарии в своём телеграм канале Милшу ( ", - "https://t.me/milshoonya", - " ) и хочу, чтобы вы первые протестировали его работу :3\nСсылку на вступление в чат оставлю здесь ", - "https://t.me/+Z_5ph-XnIQU2YWMy", - "\nТакже хотела напомнить, что мы собираем деньги на два арта от Ананаси: ", - "https://boosty.to/milshoo/single-payment/donation/550562/target?share=target_link", - "\nБуду очень благодарна за помощь :D  ", - ], - "contentCounters": list, - "count" : dict, - "createdAt" : 1711027834, - "currencyPrices": { - "RUB": 0, - "USD": 0, + { + "#url": "https://boosty.to/milshoo/posts/4304d8f0-3f49-4f97-a3f3-9f064bc32b2f", + "#class": boosty.BoostyPostExtractor, + "#urls": "https://images.boosty.to/image/75f86086-fc67-4ed2-9365-2958d3d1a8f7?change_time=1711027786", + "count": 1, + "num": 1, + "extension": "", + "filename": "75f86086-fc67-4ed2-9365-2958d3d1a8f7", + "file": { + "height": 2048, + "id": "75f86086-fc67-4ed2-9365-2958d3d1a8f7", + "rendition": "", + "size": 1094903, + "type": "image", + "url": "https://images.boosty.to/image/75f86086-fc67-4ed2-9365-2958d3d1a8f7?change_time=1711027786", + "width": 2048, + }, + "user": { + "avatarUrl": "https://images.boosty.to/user/173542/avatar?change_time=1580888689", + "blogUrl": "milshoo", + "flags": { + "showPostDonations": True, + }, + "hasAvatar": True, + "id": 173542, + "name": "Милшу", + }, + "post": { + "comments": dict, + "content": [ + "Привет! Это Милшу ) Я открываю комментарии в своём телеграм канале Милшу ( ", + "https://t.me/milshoonya", + " ) и хочу, чтобы вы первые протестировали его работу :3\nСсылку на вступление в чат оставлю здесь ", + "https://t.me/+Z_5ph-XnIQU2YWMy", + "\nТакже хотела напомнить, что мы собираем деньги на два арта от Ананаси: ", + "https://boosty.to/milshoo/single-payment/donation/550562/target?share=target_link", + "\nБуду очень благодарна за помощь :D  ", + ], + "contentCounters": list, + "count": dict, + "createdAt": 1711027834, + "currencyPrices": { + "RUB": 0, + "USD": 0, + }, + "date": "dt:2024-03-21 13:30:34", + "donations": 0, + "donators": dict, + "hasAccess": True, + "id": "4304d8f0-3f49-4f97-a3f3-9f064bc32b2f", + "int_id": 5547124, + "isBlocked": False, + "isCommentsDenied": False, + "isDeleted": False, + "isLiked": False, + "isPublished": True, + "isRecord": False, + "isWaitingVideo": False, + "links": [ + "https://t.me/milshoonya", + "https://t.me/+Z_5ph-XnIQU2YWMy", + "https://boosty.to/milshoo/single-payment/donation/550562/target?share=target_link", + ], + "price": 0, + "publishTime": 1711027834, + "showViewsCounter": False, + "signedQuery": "", + "tags": [], + "teaser": [], + "title": "Открываю чат в телеге", + "updatedAt": 1711027904, }, - "date" : "dt:2024-03-21 13:30:34", - "donations" : 0, - "donators" : dict, - "hasAccess" : True, - "id" : "4304d8f0-3f49-4f97-a3f3-9f064bc32b2f", - "int_id" : 5547124, - "isBlocked" : False, - "isCommentsDenied": False, - "isDeleted" : False, - "isLiked" : False, - "isPublished": True, - "isRecord" : False, - "isWaitingVideo": False, - "links" : [ - "https://t.me/milshoonya", - "https://t.me/+Z_5ph-XnIQU2YWMy", - "https://boosty.to/milshoo/single-payment/donation/550562/target?share=target_link", - ], - "price" : 0, - "publishTime": 1711027834, - "showViewsCounter": False, - "signedQuery": "", - "tags" : [], - "teaser" : [], - "title" : "Открываю чат в телеге", - "updatedAt" : 1711027904 }, -}, - -{ - "#url" : "https://boosty.to/geekmedia/posts/31bb8fb6-83f1-404f-a597-f84bbe611d1d", - "#comment" : "video", - "#class" : boosty.BoostyPostExtractor, -}, - -{ - "#url" : "https://boosty.to/xcang/posts/5d4d6f90-5d48-4442-a7e5-2164a858681d", - "#comment" : "audio", - "#class" : boosty.BoostyPostExtractor, -}, - -{ - "#url" : "https://boosty.to/", - "#class" : boosty.BoostyFeedExtractor, - "#auth" : True, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://boosty.to/app/settings/subscriptions", - "#class" : boosty.BoostyFollowingExtractor, - "#pattern" : boosty.BoostyUserExtractor, - "#auth" : True, -}, - - + { + "#url": "https://boosty.to/geekmedia/posts/31bb8fb6-83f1-404f-a597-f84bbe611d1d", + "#comment": "video", + "#class": boosty.BoostyPostExtractor, + }, + { + "#url": "https://boosty.to/xcang/posts/5d4d6f90-5d48-4442-a7e5-2164a858681d", + "#comment": "audio", + "#class": boosty.BoostyPostExtractor, + }, + { + "#url": "https://boosty.to/", + "#class": boosty.BoostyFeedExtractor, + "#auth": True, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "https://boosty.to/app/settings/subscriptions", + "#class": boosty.BoostyFollowingExtractor, + "#pattern": boosty.BoostyUserExtractor, + "#auth": True, + }, ) diff --git a/test/results/bulbapedia.py b/test/results/bulbapedia.py index 1549a03507..5bcbb6e655 100644 --- a/test/results/bulbapedia.py +++ b/test/results/bulbapedia.py @@ -1,28 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://bulbapedia.bulbagarden.net/wiki/Jet", - "#category": ("wikimedia", "bulbapedia", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#pattern" : r"https://archives\.bulbagarden\.net/media/upload/\w+/\w+/[^/?#]+", - "#count" : range(10, 30), -}, - -{ - "#url" : "https://archives.bulbagarden.net/wiki/File:0460Abomasnow-Mega.png", - "#category": ("wikimedia", "bulbapedia", "file"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#pattern" : r"https://archives\.bulbagarden\.net/media/upload/\w+/\w+/[^/?#]+", - "#count" : range(8, 12), - "#archive" : False, -}, - + { + "#url": "https://bulbapedia.bulbagarden.net/wiki/Jet", + "#category": ("wikimedia", "bulbapedia", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#pattern": r"https://archives\.bulbagarden\.net/media/upload/\w+/\w+/[^/?#]+", + "#count": range(10, 30), + }, + { + "#url": "https://archives.bulbagarden.net/wiki/File:0460Abomasnow-Mega.png", + "#category": ("wikimedia", "bulbapedia", "file"), + "#class": wikimedia.WikimediaArticleExtractor, + "#pattern": r"https://archives\.bulbagarden\.net/media/upload/\w+/\w+/[^/?#]+", + "#count": range(8, 12), + "#archive": False, + }, ) diff --git a/test/results/bunkr.py b/test/results/bunkr.py index 4c732a9ead..1d4ea8d988 100644 --- a/test/results/bunkr.py +++ b/test/results/bunkr.py @@ -1,211 +1,176 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import bunkr - __tests__ = ( -{ - "#url" : "https://bunkr.sk/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, - "#urls" : "https://i-burger.bunkr.ru/test-%E3%83%86%E3%82%B9%E3%83%88-%22%26%3E-QjgneIQv.png?download=true", - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", - - "album_id" : "Lktg9Keq", - "album_name" : "test テスト \"&>", - "album_size" : "182 B", - "count" : 1, - "extension" : "png", - "file" : "https://i-burger.bunkr.ru/test-%E3%83%86%E3%82%B9%E3%83%88-%22%26%3E-QjgneIQv.png?download=true", - "filename" : "test-テスト-\"&>-QjgneIQv", - "id" : "QjgneIQv", - "name" : "test-テスト-\"&>", - "num" : 1, -}, - -{ - "#url" : "https://bunkr.is/a/iXTTc1o2", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, - "#urls" : ( - "https://i-milkshake.bunkr.ru/image-sZrQUeOx.jpg?download=true", - ), - "#sha1_content": "caf7c3d3439d94e83b3c24ddaf5a3a48aa057519", - - "album_id" : "iXTTc1o2", - "album_name" : "test2", - "album_size" : "534.6 KB", - "count" : 1, - "filename" : r"image-sZrQUeOx", - "id" : r"sZrQUeOx", - "name" : r"image", - "num" : 1, -}, - -{ - "#url" : "https://bunkr.cat/a/j1G29CnD", - "#comment" : "cdn12 .ru TLD (#4147)", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, - "#pattern" : r"https://(i-)?meatballs.bunkr.ru/\w+", - "#count" : 7, -}, - -{ - "#url" : "https://bunkr.ph/a/Lktg9Keq", - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.ps/a/Lktg9Keq", - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.pk/a/Lktg9Keq", - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.ax/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkrrr.org/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.ci/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.cr/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.fi/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.si/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.ac/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.media/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.site/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.ws/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkrr.ru/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkrr.su/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.la/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.su/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.ru/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.is/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.to/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "bunkr:http://example.org/a/Lktg9Keq", - "#category": ("lolisafe", "bunkr", "album"), - "#class" : bunkr.BunkrAlbumExtractor, -}, - -{ - "#url" : "https://bunkr.black/i/image-sZrQUeOx.jpg", - "#category": ("lolisafe", "bunkr", "media"), - "#class" : bunkr.BunkrMediaExtractor, - "#urls" : "https://i-milkshake.bunkr.ru/image-sZrQUeOx.jpg?download=true", - "#sha1_content": "caf7c3d3439d94e83b3c24ddaf5a3a48aa057519", - - "count" : 1, - "extension": "jpg", - "file" : "https://i-milkshake.bunkr.ru/image-sZrQUeOx.jpg?download=true", - "filename" : "image-sZrQUeOx", - "id" : "sZrQUeOx", - "name" : "image", -}, - -{ - "#url" : "https://bunkrrr.org/d/dJuETSzKLrUps", - "#category": ("lolisafe", "bunkr", "media"), - "#class" : bunkr.BunkrMediaExtractor, - "#urls" : "https://burger.bunkr.ru/file-r5fmwjdd.zip", - "#sha1_content": "102ddd7894fe39b3843098fc51f972a0af938f45", - - "count" : 1, - "extension": "zip", - "file" : "https://burger.bunkr.ru/file-r5fmwjdd.zip", - "filename" : "file-r5fmwjdd", - "id" : "r5fmwjdd", - "name" : "file", -}, - + { + "#url": "https://bunkr.sk/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + "#urls": "https://i-burger.bunkr.ru/test-%E3%83%86%E3%82%B9%E3%83%88-%22%26%3E-QjgneIQv.png?download=true", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + "album_id": "Lktg9Keq", + "album_name": 'test テスト "&>', + "album_size": "182 B", + "count": 1, + "extension": "png", + "file": "https://i-burger.bunkr.ru/test-%E3%83%86%E3%82%B9%E3%83%88-%22%26%3E-QjgneIQv.png?download=true", + "filename": 'test-テスト-"&>-QjgneIQv', + "id": "QjgneIQv", + "name": 'test-テスト-"&>', + "num": 1, + }, + { + "#url": "https://bunkr.is/a/iXTTc1o2", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + "#urls": ("https://i-milkshake.bunkr.ru/image-sZrQUeOx.jpg?download=true",), + "#sha1_content": "caf7c3d3439d94e83b3c24ddaf5a3a48aa057519", + "album_id": "iXTTc1o2", + "album_name": "test2", + "album_size": "534.6 KB", + "count": 1, + "filename": r"image-sZrQUeOx", + "id": r"sZrQUeOx", + "name": r"image", + "num": 1, + }, + { + "#url": "https://bunkr.cat/a/j1G29CnD", + "#comment": "cdn12 .ru TLD (#4147)", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + "#pattern": r"https://(i-)?meatballs.bunkr.ru/\w+", + "#count": 7, + }, + { + "#url": "https://bunkr.ph/a/Lktg9Keq", + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.ps/a/Lktg9Keq", + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.pk/a/Lktg9Keq", + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.ax/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkrrr.org/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.ci/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.cr/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.fi/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.si/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.ac/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.media/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.site/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.ws/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkrr.ru/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkrr.su/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.la/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.su/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.ru/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.is/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.to/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "bunkr:http://example.org/a/Lktg9Keq", + "#category": ("lolisafe", "bunkr", "album"), + "#class": bunkr.BunkrAlbumExtractor, + }, + { + "#url": "https://bunkr.black/i/image-sZrQUeOx.jpg", + "#category": ("lolisafe", "bunkr", "media"), + "#class": bunkr.BunkrMediaExtractor, + "#urls": "https://i-milkshake.bunkr.ru/image-sZrQUeOx.jpg?download=true", + "#sha1_content": "caf7c3d3439d94e83b3c24ddaf5a3a48aa057519", + "count": 1, + "extension": "jpg", + "file": "https://i-milkshake.bunkr.ru/image-sZrQUeOx.jpg?download=true", + "filename": "image-sZrQUeOx", + "id": "sZrQUeOx", + "name": "image", + }, + { + "#url": "https://bunkrrr.org/d/dJuETSzKLrUps", + "#category": ("lolisafe", "bunkr", "media"), + "#class": bunkr.BunkrMediaExtractor, + "#urls": "https://burger.bunkr.ru/file-r5fmwjdd.zip", + "#sha1_content": "102ddd7894fe39b3843098fc51f972a0af938f45", + "count": 1, + "extension": "zip", + "file": "https://burger.bunkr.ru/file-r5fmwjdd.zip", + "filename": "file-r5fmwjdd", + "id": "r5fmwjdd", + "name": "file", + }, ) diff --git a/test/results/catbox.py b/test/results/catbox.py index b977a52edd..fad1c2abcb 100644 --- a/test/results/catbox.py +++ b/test/results/catbox.py @@ -1,59 +1,49 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import catbox - __tests__ = ( -{ - "#url" : "https://catbox.moe/c/1igcbe", - "#category": ("", "catbox", "album"), - "#class" : catbox.CatboxAlbumExtractor, - "#pattern" : r"https://files\.catbox\.moe/\w+\.\w{3}$", - "#count" : 3, - "#sha1_url" : "35866a88c29462814f103bc22ec031eaeb380f8a", - "#sha1_content": "70ddb9de3872e2d17cc27e48e6bf395e5c8c0b32", - - "album_id" : "1igcbe", - "album_name" : "test", - "date" : "dt:2022-08-18 00:00:00", - "description": "album test &>", -}, - -{ - "#url" : "https://www.catbox.moe/c/cd90s1", - "#category": ("", "catbox", "album"), - "#class" : catbox.CatboxAlbumExtractor, -}, - -{ - "#url" : "https://catbox.moe/c/w7tm47#", - "#category": ("", "catbox", "album"), - "#class" : catbox.CatboxAlbumExtractor, -}, - -{ - "#url" : "https://files.catbox.moe/8ih3y7.png", - "#category": ("", "catbox", "file"), - "#class" : catbox.CatboxFileExtractor, - "#pattern" : r"^https://files\.catbox\.moe/8ih3y7\.png$", - "#count" : 1, - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", -}, - -{ - "#url" : "https://litter.catbox.moe/t8v3n9.png", - "#category": ("", "catbox", "file"), - "#class" : catbox.CatboxFileExtractor, -}, - -{ - "#url" : "https://de.catbox.moe/bjdmz1.jpg", - "#category": ("", "catbox", "file"), - "#class" : catbox.CatboxFileExtractor, -}, - + { + "#url": "https://catbox.moe/c/1igcbe", + "#category": ("", "catbox", "album"), + "#class": catbox.CatboxAlbumExtractor, + "#pattern": r"https://files\.catbox\.moe/\w+\.\w{3}$", + "#count": 3, + "#sha1_url": "35866a88c29462814f103bc22ec031eaeb380f8a", + "#sha1_content": "70ddb9de3872e2d17cc27e48e6bf395e5c8c0b32", + "album_id": "1igcbe", + "album_name": "test", + "date": "dt:2022-08-18 00:00:00", + "description": "album test &>", + }, + { + "#url": "https://www.catbox.moe/c/cd90s1", + "#category": ("", "catbox", "album"), + "#class": catbox.CatboxAlbumExtractor, + }, + { + "#url": "https://catbox.moe/c/w7tm47#", + "#category": ("", "catbox", "album"), + "#class": catbox.CatboxAlbumExtractor, + }, + { + "#url": "https://files.catbox.moe/8ih3y7.png", + "#category": ("", "catbox", "file"), + "#class": catbox.CatboxFileExtractor, + "#pattern": r"^https://files\.catbox\.moe/8ih3y7\.png$", + "#count": 1, + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, + { + "#url": "https://litter.catbox.moe/t8v3n9.png", + "#category": ("", "catbox", "file"), + "#class": catbox.CatboxFileExtractor, + }, + { + "#url": "https://de.catbox.moe/bjdmz1.jpg", + "#category": ("", "catbox", "file"), + "#class": catbox.CatboxFileExtractor, + }, ) diff --git a/test/results/cavemanon.py b/test/results/cavemanon.py index 3d065f43bf..c41ca4e25a 100644 --- a/test/results/cavemanon.py +++ b/test/results/cavemanon.py @@ -1,44 +1,37 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shimmie2 - __tests__ = ( -{ - "#url" : "https://booru.cavemanon.xyz/index.php?q=post/list/Amber/1", - "#category": ("shimmie2", "cavemanon", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, - "#pattern" : r"https://booru\.cavemanon\.xyz/index\.php\?q=image/\d+\.\w+", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://booru.cavemanon.xyz/post/list/Amber/1", - "#category": ("shimmie2", "cavemanon", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, -}, - -{ - "#url" : "https://booru.cavemanon.xyz/index.php?q=post/view/8335", - "#category": ("shimmie2", "cavemanon", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, - "#pattern" : r"https://booru\.cavemanon\.xyz/index\.php\?q=image/8335\.png", - "#sha1_content": "7158f7e4abbbf143bad5835eb93dbe4d68c1d4ab", - - "extension": "png", - "file_url" : "https://booru.cavemanon.xyz/index.php?q=image/8335.png", - "filename" : "8335", - "height" : 460, - "id" : 8335, - "md5" : "", - "size" : 0, - "tags" : "Color discord_emote Fang Food Pterodactyl transparent", - "width" : 459, -}, - + { + "#url": "https://booru.cavemanon.xyz/index.php?q=post/list/Amber/1", + "#category": ("shimmie2", "cavemanon", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + "#pattern": r"https://booru\.cavemanon\.xyz/index\.php\?q=image/\d+\.\w+", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://booru.cavemanon.xyz/post/list/Amber/1", + "#category": ("shimmie2", "cavemanon", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + }, + { + "#url": "https://booru.cavemanon.xyz/index.php?q=post/view/8335", + "#category": ("shimmie2", "cavemanon", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + "#pattern": r"https://booru\.cavemanon\.xyz/index\.php\?q=image/8335\.png", + "#sha1_content": "7158f7e4abbbf143bad5835eb93dbe4d68c1d4ab", + "extension": "png", + "file_url": "https://booru.cavemanon.xyz/index.php?q=image/8335.png", + "filename": "8335", + "height": 460, + "id": 8335, + "md5": "", + "size": 0, + "tags": "Color discord_emote Fang Food Pterodactyl transparent", + "width": 459, + }, ) diff --git a/test/results/chelseacrew.py b/test/results/chelseacrew.py index 7c12613fc6..5add98dda9 100644 --- a/test/results/chelseacrew.py +++ b/test/results/chelseacrew.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://chelseacrew.com/collections/flats", - "#category": ("shopify", "chelseacrew", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://chelseacrew.com/collections/flats/products/dora", - "#category": ("shopify", "chelseacrew", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://chelseacrew.com/collections/flats", + "#category": ("shopify", "chelseacrew", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://chelseacrew.com/collections/flats/products/dora", + "#category": ("shopify", "chelseacrew", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/cien.py b/test/results/cien.py index ef797216d9..662414a315 100644 --- a/test/results/cien.py +++ b/test/results/cien.py @@ -1,92 +1,81 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import cien - __tests__ = ( -{ - "#url" : "https://ci-en.net/creator/7491/article/1194568", - "#category": ("", "cien", "article"), - "#class" : cien.CienArticleExtractor, - "#pattern" : r"https://media\.ci-en\.jp/private/attachment/creator/00007491/c0c212a93027c8863bdb40668071c1525a4567f94baca13c17989045e5a3d81d/video-web\.mp4\?px-time=.+", - - "author": { - "@type" : "Person", - "image" : "https://media.ci-en.jp/public/icon/creator/00007491/9601a2a224245156335aaa839fa408d52c32c87dae5787fc03f455b7fd1d3488/image-200-c.jpg", - "name" : "やかろ", - "url" : "https://ci-en.net/creator/7491", - "sameAs": [ - "https://pokapoka0802.wixsite.com/tunousaginoie82", - "https://www.freem.ne.jp/brand/6001", - "https://store.steampowered.com/search/?developer=%E3%83%84%E3%83%8E%E3%82%A6%E3%82%B5%E3%82%AE%E3%81%AE%E5%AE%B6", - "https://plicy.net/User/87381", - "https://twitter.com/pokapoka0802", - ], + { + "#url": "https://ci-en.net/creator/7491/article/1194568", + "#category": ("", "cien", "article"), + "#class": cien.CienArticleExtractor, + "#pattern": r"https://media\.ci-en\.jp/private/attachment/creator/00007491/c0c212a93027c8863bdb40668071c1525a4567f94baca13c17989045e5a3d81d/video-web\.mp4\?px-time=.+", + "author": { + "@type": "Person", + "image": "https://media.ci-en.jp/public/icon/creator/00007491/9601a2a224245156335aaa839fa408d52c32c87dae5787fc03f455b7fd1d3488/image-200-c.jpg", + "name": "やかろ", + "url": "https://ci-en.net/creator/7491", + "sameAs": [ + "https://pokapoka0802.wixsite.com/tunousaginoie82", + "https://www.freem.ne.jp/brand/6001", + "https://store.steampowered.com/search/?developer=%E3%83%84%E3%83%8E%E3%82%A6%E3%82%B5%E3%82%AE%E3%81%AE%E5%AE%B6", + "https://plicy.net/User/87381", + "https://twitter.com/pokapoka0802", + ], + }, + "articleBody": str, + "count": 1, + "date": "dt:2024-07-21 15:36:00", + "dateModified": "2024-07-22T03:28:40+09:00", + "datePublished": "2024-07-22T00:36:00+09:00", + "description": "お知らせ 今回は雨のピリオードの解説をしたいと思うのですが、その前にいくつかお知らせがあります。 電話を使って謎を解いていくフリーゲーム 電話を通して、様々なキャラクターを会話をしていく、ノベルゲーム……", + "extension": "mp4", + "filename": "無題の動画 (1)", + "headline": "角兎図書館「雨のピリオード」No,16", + "image": "https://media.ci-en.jp/public/article_cover/creator/00007491/cb4062e8d885ab93e0d0fb3133265a7ad1056c906fd4ab81da509220620901e1/image-1280-c.jpg", + "keywords": "お知らせ,角兎図書館", + "mainEntityOfPage": "https://ci-en.net/creator/7491/article/1194568", + "name": "角兎図書館「雨のピリオード」No,16", + "num": 1, + "post_id": 1194568, + "type": "video", + "url": str, + }, + { + "#url": "https://ci-en.dlsite.com/creator/25509/article/1172460", + "#category": ("", "cien", "article"), + "#class": cien.CienArticleExtractor, + "#options": {"files": "download"}, + "#pattern": r"https://media\.ci-en\.jp/private/attachment/creator/00025509/7fd3c039d2277ba9541e82592aca6f6751f6c268404038ccbf1112bcf2f93357/upload/.+\.zip\?px-time=.+", + "filename": "VP 1.05.4 Tim-v9 ENG rec v3", + "extension": "zip", + "type": "download", + }, + { + "#url": "https://ci-en.net/creator/11962", + "#category": ("", "cien", "creator"), + "#class": cien.CienCreatorExtractor, + "#pattern": cien.CienArticleExtractor.pattern, + "#count": "> 25", + }, + { + "#url": "https://ci-en.net/mypage/recent", + "#category": ("", "cien", "recent"), + "#class": cien.CienRecentExtractor, + "#auth": True, + }, + { + "#url": "https://ci-en.net/mypage/subscription/following", + "#category": ("", "cien", "following"), + "#class": cien.CienFollowingExtractor, + "#pattern": cien.CienCreatorExtractor.pattern, + "#count": "> 3", + "#auth": True, + }, + { + "#url": "https://ci-en.net/mypage/subscription", + "#category": ("", "cien", "following"), + "#class": cien.CienFollowingExtractor, + "#auth": True, }, - "articleBody": str, - "count" : 1, - "date" : "dt:2024-07-21 15:36:00", - "dateModified" : "2024-07-22T03:28:40+09:00", - "datePublished": "2024-07-22T00:36:00+09:00", - "description": "お知らせ 今回は雨のピリオードの解説をしたいと思うのですが、その前にいくつかお知らせがあります。 電話を使って謎を解いていくフリーゲーム 電話を通して、様々なキャラクターを会話をしていく、ノベルゲーム……", - "extension" : "mp4", - "filename" : "無題の動画 (1)", - "headline" : "角兎図書館「雨のピリオード」No,16", - "image" : "https://media.ci-en.jp/public/article_cover/creator/00007491/cb4062e8d885ab93e0d0fb3133265a7ad1056c906fd4ab81da509220620901e1/image-1280-c.jpg", - "keywords" : "お知らせ,角兎図書館", - "mainEntityOfPage": "https://ci-en.net/creator/7491/article/1194568", - "name" : "角兎図書館「雨のピリオード」No,16", - "num" : 1, - "post_id" : 1194568, - "type" : "video", - "url" : str, -}, - -{ - "#url" : "https://ci-en.dlsite.com/creator/25509/article/1172460", - "#category": ("", "cien", "article"), - "#class" : cien.CienArticleExtractor, - "#options" : {"files": "download"}, - "#pattern" : r"https://media\.ci-en\.jp/private/attachment/creator/00025509/7fd3c039d2277ba9541e82592aca6f6751f6c268404038ccbf1112bcf2f93357/upload/.+\.zip\?px-time=.+", - - "filename" : "VP 1.05.4 Tim-v9 ENG rec v3", - "extension": "zip", - "type" : "download", -}, - -{ - "#url" : "https://ci-en.net/creator/11962", - "#category": ("", "cien", "creator"), - "#class" : cien.CienCreatorExtractor, - "#pattern" : cien.CienArticleExtractor.pattern, - "#count" : "> 25", -}, - -{ - "#url" : "https://ci-en.net/mypage/recent", - "#category": ("", "cien", "recent"), - "#class" : cien.CienRecentExtractor, - "#auth" : True, -}, - -{ - "#url" : "https://ci-en.net/mypage/subscription/following", - "#category": ("", "cien", "following"), - "#class" : cien.CienFollowingExtractor, - "#pattern" : cien.CienCreatorExtractor.pattern, - "#count" : "> 3", - "#auth" : True, -}, - -{ - "#url" : "https://ci-en.net/mypage/subscription", - "#category": ("", "cien", "following"), - "#class" : cien.CienFollowingExtractor, - "#auth" : True, -}, - ) diff --git a/test/results/civitai.py b/test/results/civitai.py index 2497674977..0de1a9818f 100644 --- a/test/results/civitai.py +++ b/test/results/civitai.py @@ -1,244 +1,220 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import civitai from gallery_dl import exception - +from gallery_dl.extractor import civitai __tests__ = ( -{ - "#url" : "https://civitai.com/models/703211/maid-classic", - "#class": civitai.CivitaiModelExtractor, - "#urls" : [ - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/5c4efa68-bb58-47c5-a716-98cd0f51f047/original=true/00013-4238863814.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/original=true/00014-3150861441.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/2dd1dc69-45a6-4beb-b36b-2e2bc65e3cda/original=true/00015-2885514572.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/00004-822988489.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/00005-1059918744.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/00006-3467286319.png", - ], - - "model" : { - "description": "

          The strength of Lora is recommended to be around 1.0.

          ", - "id" : 703211, - "minor" : False, - "name" : "メイド　クラシック/maid classic", - "nsfwLevel" : 1, - "type" : "LORA", + { + "#url": "https://civitai.com/models/703211/maid-classic", + "#class": civitai.CivitaiModelExtractor, + "#urls": [ + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/5c4efa68-bb58-47c5-a716-98cd0f51f047/original=true/00013-4238863814.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/original=true/00014-3150861441.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/2dd1dc69-45a6-4beb-b36b-2e2bc65e3cda/original=true/00015-2885514572.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/00004-822988489.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/00005-1059918744.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/00006-3467286319.png", + ], + "model": { + "description": "

          The strength of Lora is recommended to be around 1.0.

          ", + "id": 703211, + "minor": False, + "name": "メイド　クラシック/maid classic", + "nsfwLevel": 1, + "type": "LORA", + }, + "user": {"image": None, "username": "bolero537"}, + "file": { + "uuid": str, + }, + "version": dict, + "num": range(1, 3), }, - "user" : { - "image" : None, - "username": "bolero537" + { + "#url": "https://civitai.com/models/703211?modelVersionId=786644", + "#comment": "model version ID", + "#class": civitai.CivitaiModelExtractor, + "#urls": [ + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/00004-822988489.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/00005-1059918744.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/00006-3467286319.png", + ], + "version": { + "baseModel": "Pony", + "createdAt": "2024-08-30T15:28:47.661Z", + "date": "dt:2024-08-30 15:28:47", + "files": list, + "id": 786644, + "name": "v1.0 pony", + }, + "user": {"image": None, "username": "bolero537"}, + "file": { + "id": {26887862, 26887856, 26887852}, + "uuid": { + "52b6efa7-801c-4901-90b4-fa3964d23480", + "c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c", + "68568d22-c4f3-45cb-ac32-82f1cedf968f", + }, + }, + "model": { + "id": 703211, + }, + "num": range(1, 3), }, - "file" : { - "uuid": str, + { + "#url": "https://civitai.com/images/26962948", + "#class": civitai.CivitaiImageExtractor, + "#options": {"quality": "w", "metadata": True}, + "#urls": "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/w/00014-3150861441.png", + "#sha1_content": "a9a9d08f5fcdbc1e1eec7f203717f9df97b7a671", + "createdAt": "2024-08-31T01:11:47.021Z", + "date": "dt:2024-08-31 01:11:47", + "extension": "jpg", + "filename": "00014-3150861441", + "hash": "ULN0-w?b4nRjxGM{-;t7M_t7NGae~qRjMyt7", + "height": 1536, + "id": 26962948, + "nsfwLevel": 1, + "postId": 6030721, + "stats": dict, + "url": "69bf3279-df2c-4ec8-b795-479e9cd3db1b", + "uuid": "69bf3279-df2c-4ec8-b795-479e9cd3db1b", + "width": 1152, + "user": { + "username": "bolero537", + }, + "generation": { + "canRemix": True, + "external": None, + "generationProcess": "img2img", + "resources": list, + "techniques": [], + "tools": [], + "meta": { + "Denoising strength": "0.4", + "Model": "boleromix_XL_V1.3", + "Model hash": "afaf521da2", + "Size": "1152x1536", + "Tiled Diffusion scale factor": "1.5", + "Tiled Diffusion upscaler": "R-ESRGAN 4x+ Anime6B", + "VAE": "sdxl_vae.safetensors", + "Version": "v1.7.0", + "cfgScale": 7, + "negativePrompt": "negativeXL_D,(worst quality,extra legs,extra arms,extra ears,bad fingers,extra fingers,bad anatomy, missing fingers, lowres,username, artist name, text,pubic hair,bar censor,censored,multipul angle,split view,realistic,3D:1)", + "prompt": "masterpiece,ultra-detailed,best quality,8K,illustration,cute face,clean skin ,shiny hair,girl,ultra-detailed-eyes,simple background, maid, maid apron, maid headdress, long sleeves,tray,tea,cup,skirt lift", + "resources": list, + "sampler": "DPM++ 2M Karras", + "seed": 3150861441, + "steps": 20, + "hashes": { + "lora:add-detail-xl": "9c783c8ce46c", + "lora:classic maid_XL_V1.0": "e8f6e4297112", + "model": "afaf521da2", + "vae": "735e4c3a44", + }, + "TI hashes": { + "negativeXL_D": "fff5d51ab655", + }, + }, + }, }, - "version": dict, - "num" : range(1, 3), -}, - -{ - "#url" : "https://civitai.com/models/703211?modelVersionId=786644", - "#comment": "model version ID", - "#class" : civitai.CivitaiModelExtractor, - "#urls" : [ - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/52b6efa7-801c-4901-90b4-fa3964d23480/original=true/00004-822988489.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c/original=true/00005-1059918744.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/68568d22-c4f3-45cb-ac32-82f1cedf968f/original=true/00006-3467286319.png", - ], - - "version": { - "baseModel" : "Pony", - "createdAt" : "2024-08-30T15:28:47.661Z", - "date" : "dt:2024-08-30 15:28:47", - "files" : list, - "id" : 786644, - "name" : "v1.0 pony", + { + "#url": "https://civitai.com/posts/6877551", + "#class": civitai.CivitaiPostExtractor, + "#options": {"metadata": "generation"}, + "#urls": [ + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6220fa0f-9037-4b1d-bfbd-a740a06eeb7c/original=true/30748752.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/cd1edb7f-7b50-4da5-bf23-d38f24d8aef0/original=true/30748747.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/cfd5b231-accd-49bd-8bde-370880f63aa6/original=true/30748733.png", + ], + "post": { + "id": 6877551, + "date": "dt:2024-09-22 12:54:15", + }, + "file": { + "id": {30748752, 30748747, 30748733}, + "uuid": { + "6220fa0f-9037-4b1d-bfbd-a740a06eeb7c", + "cd1edb7f-7b50-4da5-bf23-d38f24d8aef0", + "cfd5b231-accd-49bd-8bde-370880f63aa6", + }, + "generation": { + "resources": list, + "techniques": [], + "tools": [], + "meta": { + "prompt": str, + "negativePrompt": str, + }, + }, + }, }, - "user" : { - "image" : None, - "username": "bolero537" + { + "#url": "https://civitai.com/tag/mecha", + "#class": civitai.CivitaiTagExtractor, }, - "file" : { - "id" : {26887862, 26887856, 26887852}, - "uuid": {"52b6efa7-801c-4901-90b4-fa3964d23480", - "c4d3bcd5-0e23-4f4e-9f34-d13b2f2bf14c", - "68568d22-c4f3-45cb-ac32-82f1cedf968f"}, + { + "#url": "https://civitai.com/images?tags=482", + "#class": civitai.CivitaiImagesExtractor, }, - "model" : { - "id": 703211, + { + "#url": "https://civitai.com/images?modelVersionId=786644", + "#class": civitai.CivitaiImagesExtractor, }, - "num" : range(1, 3), -}, - -{ - "#url" : "https://civitai.com/images/26962948", - "#class": civitai.CivitaiImageExtractor, - "#options" : {"quality": "w", "metadata": True}, - "#urls" : "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/w/00014-3150861441.png", - "#sha1_content": "a9a9d08f5fcdbc1e1eec7f203717f9df97b7a671", - - "createdAt": "2024-08-31T01:11:47.021Z", - "date" : "dt:2024-08-31 01:11:47", - "extension": "jpg", - "filename" : "00014-3150861441", - "hash" : "ULN0-w?b4nRjxGM{-;t7M_t7NGae~qRjMyt7", - "height" : 1536, - "id" : 26962948, - "nsfwLevel": 1, - "postId" : 6030721, - "stats" : dict, - "url" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b", - "uuid" : "69bf3279-df2c-4ec8-b795-479e9cd3db1b", - "width" : 1152, - "user" : { - "username": "bolero537", + { + "#url": "https://civitai.com/models", + "#class": civitai.CivitaiModelsExtractor, }, - "generation": { - "canRemix" : True, - "external" : None, - "generationProcess": "img2img", - "resources" : list, - "techniques": [], - "tools" : [], - "meta" : { - "Denoising strength": "0.4", - "Model" : "boleromix_XL_V1.3", - "Model hash" : "afaf521da2", - "Size" : "1152x1536", - "Tiled Diffusion scale factor": "1.5", - "Tiled Diffusion upscaler": "R-ESRGAN 4x+ Anime6B", - "VAE" : "sdxl_vae.safetensors", - "Version" : "v1.7.0", - "cfgScale" : 7, - "negativePrompt": "negativeXL_D,(worst quality,extra legs,extra arms,extra ears,bad fingers,extra fingers,bad anatomy, missing fingers, lowres,username, artist name, text,pubic hair,bar censor,censored,multipul angle,split view,realistic,3D:1)", - "prompt" : "masterpiece,ultra-detailed,best quality,8K,illustration,cute face,clean skin ,shiny hair,girl,ultra-detailed-eyes,simple background, maid, maid apron, maid headdress, long sleeves,tray,tea,cup,skirt lift", - "resources" : list, - "sampler" : "DPM++ 2M Karras", - "seed" : 3150861441, - "steps" : 20, - "hashes" : { - "lora:add-detail-xl": "9c783c8ce46c", - "lora:classic maid_XL_V1.0": "e8f6e4297112", - "model": "afaf521da2", - "vae": "735e4c3a44", - }, - "TI hashes" : { - "negativeXL_D": "fff5d51ab655", - }, - }, + { + "#url": "https://civitai.com/search/models?sortBy=models_v9&query=mecha", + "#class": civitai.CivitaiSearchExtractor, }, -}, - -{ - "#url" : "https://civitai.com/posts/6877551", - "#class" : civitai.CivitaiPostExtractor, - "#options": {"metadata": "generation"}, - "#urls" : [ - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6220fa0f-9037-4b1d-bfbd-a740a06eeb7c/original=true/30748752.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/cd1edb7f-7b50-4da5-bf23-d38f24d8aef0/original=true/30748747.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/cfd5b231-accd-49bd-8bde-370880f63aa6/original=true/30748733.png", - ], - - "post": { - "id" : 6877551, - "date": "dt:2024-09-22 12:54:15", + { + "#url": "https://civitai.com/user/waomodder", + "#class": civitai.CivitaiUserExtractor, + "#urls": [ + "https://civitai.com/user/waomodder/models", + "https://civitai.com/user/waomodder/posts", + ], }, - "file": { - "id" : {30748752, 30748747, 30748733}, - "uuid": {"6220fa0f-9037-4b1d-bfbd-a740a06eeb7c", - "cd1edb7f-7b50-4da5-bf23-d38f24d8aef0", - "cfd5b231-accd-49bd-8bde-370880f63aa6"}, - "generation": { - "resources" : list, - "techniques": [], - "tools" : [], - "meta" : { - "prompt" : str, - "negativePrompt": str, - }, - }, + { + "#url": "https://civitai.com/user/waomodder/models", + "#class": civitai.CivitaiUserModelsExtractor, + "#pattern": civitai.CivitaiModelExtractor.pattern, + "#count": ">= 8", + }, + { + "#url": "https://civitai.com/user/waomodder/posts", + "#class": civitai.CivitaiUserPostsExtractor, + "#pattern": r"https://image\.civitai\.com/xG1nkqKTMzGDvpLrqFT7WA/[0-9a-f-]+/original=true/\S+\.(jpe?g|png)", + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://civitai.com/user/waomodder/images", + "#class": civitai.CivitaiUserImagesExtractor, + "#pattern": r"https://image\.civitai\.com/xG1nkqKTMzGDvpLrqFT7WA/[0-9a-f-]+/original=true/\S+\.png", + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://civitai.com/user/USER/images?section=reactions", + "#category": ("", "civitai", "reactions"), + "#class": civitai.CivitaiUserImagesExtractor, + "#auth": True, + "#urls": ( + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/dd29c97a-1e95-4186-8df5-632736cbae79/original=true/00012-2489035818.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/5c4efa68-bb58-47c5-a716-98cd0f51f047/original=true/00013-4238863814.png", + "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/original=true/00014-3150861441.png", + ), + }, + { + "#url": "https://civitai.com/user/USER/images?section=reactions", + "#category": ("", "civitai", "reactions"), + "#class": civitai.CivitaiUserImagesExtractor, + "#auth": False, + "#exception": exception.AuthorizationError, }, -}, - -{ - "#url" : "https://civitai.com/tag/mecha", - "#class": civitai.CivitaiTagExtractor, -}, - -{ - "#url" : "https://civitai.com/images?tags=482", - "#class": civitai.CivitaiImagesExtractor, -}, - -{ - "#url" : "https://civitai.com/images?modelVersionId=786644", - "#class": civitai.CivitaiImagesExtractor, -}, - -{ - "#url" : "https://civitai.com/models", - "#class": civitai.CivitaiModelsExtractor, -}, - -{ - "#url" : "https://civitai.com/search/models?sortBy=models_v9&query=mecha", - "#class": civitai.CivitaiSearchExtractor, -}, - -{ - "#url" : "https://civitai.com/user/waomodder", - "#class": civitai.CivitaiUserExtractor, - "#urls" : [ - "https://civitai.com/user/waomodder/models", - "https://civitai.com/user/waomodder/posts", - ], -}, - -{ - "#url" : "https://civitai.com/user/waomodder/models", - "#class": civitai.CivitaiUserModelsExtractor, - "#pattern": civitai.CivitaiModelExtractor.pattern, - "#count" : ">= 8", -}, - -{ - "#url" : "https://civitai.com/user/waomodder/posts", - "#class": civitai.CivitaiUserPostsExtractor, - "#pattern": r"https://image\.civitai\.com/xG1nkqKTMzGDvpLrqFT7WA/[0-9a-f-]+/original=true/\S+\.(jpe?g|png)", - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://civitai.com/user/waomodder/images", - "#class": civitai.CivitaiUserImagesExtractor, - "#pattern": r"https://image\.civitai\.com/xG1nkqKTMzGDvpLrqFT7WA/[0-9a-f-]+/original=true/\S+\.png", - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://civitai.com/user/USER/images?section=reactions", - "#category": ("", "civitai", "reactions"), - "#class" : civitai.CivitaiUserImagesExtractor, - "#auth" : True, - "#urls" : ( - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/dd29c97a-1e95-4186-8df5-632736cbae79/original=true/00012-2489035818.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/5c4efa68-bb58-47c5-a716-98cd0f51f047/original=true/00013-4238863814.png", - "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/69bf3279-df2c-4ec8-b795-479e9cd3db1b/original=true/00014-3150861441.png", - ), -}, - -{ - "#url" : "https://civitai.com/user/USER/images?section=reactions", - "#category": ("", "civitai", "reactions"), - "#class" : civitai.CivitaiUserImagesExtractor, - "#auth" : False, - "#exception": exception.AuthorizationError, -}, - ) diff --git a/test/results/cohost.py b/test/results/cohost.py index 1184f21c0d..42c05c654f 100644 --- a/test/results/cohost.py +++ b/test/results/cohost.py @@ -1,26 +1,21 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import cohost - __tests__ = ( -{ - "#url" : "https://cohost.org/infinitebrians", - "#category": ("", "cohost", "user"), - "#class" : cohost.CohostUserExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://cohost.org/infinitebrians/post/4957017-thank-you-akira-tori", - "#category": ("", "cohost", "post"), - "#class" : cohost.CohostPostExtractor, - "#urls" : "https://staging.cohostcdn.org/attachment/58f9aa96-d2b2-4838-b81c-9aa8bac0bea0/march%204%202024.png", -}, - + { + "#url": "https://cohost.org/infinitebrians", + "#category": ("", "cohost", "user"), + "#class": cohost.CohostUserExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://cohost.org/infinitebrians/post/4957017-thank-you-akira-tori", + "#category": ("", "cohost", "post"), + "#class": cohost.CohostPostExtractor, + "#urls": "https://staging.cohostcdn.org/attachment/58f9aa96-d2b2-4838-b81c-9aa8bac0bea0/march%204%202024.png", + }, ) diff --git a/test/results/comicvine.py b/test/results/comicvine.py index b41c24e88c..4db4bd56fb 100644 --- a/test/results/comicvine.py +++ b/test/results/comicvine.py @@ -1,27 +1,22 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import comicvine - __tests__ = ( -{ - "#url" : "https://comicvine.gamespot.com/jock/4040-5653/images/", - "#category": ("", "comicvine", "tag"), - "#class" : comicvine.ComicvineTagExtractor, - "#pattern" : r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+\.(jpe?g|png)", - "#count" : ">= 140", -}, - -{ - "#url" : "https://comicvine.gamespot.com/batman/4005-1699/images/?tag=Fan%20Art%20%26%20Cosplay", - "#category": ("", "comicvine", "tag"), - "#class" : comicvine.ComicvineTagExtractor, - "#pattern" : r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+", - "#count" : ">= 400", -}, - + { + "#url": "https://comicvine.gamespot.com/jock/4040-5653/images/", + "#category": ("", "comicvine", "tag"), + "#class": comicvine.ComicvineTagExtractor, + "#pattern": r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+\.(jpe?g|png)", + "#count": ">= 140", + }, + { + "#url": "https://comicvine.gamespot.com/batman/4005-1699/images/?tag=Fan%20Art%20%26%20Cosplay", + "#category": ("", "comicvine", "tag"), + "#class": comicvine.ComicvineTagExtractor, + "#pattern": r"https://comicvine\.gamespot\.com/a/uploads/original/\d+/\d+/\d+-.+", + "#count": ">= 400", + }, ) diff --git a/test/results/coomerparty.py b/test/results/coomerparty.py index 87c932e83d..ef93be58b7 100644 --- a/test/results/coomerparty.py +++ b/test/results/coomerparty.py @@ -1,26 +1,21 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import kemonoparty - __tests__ = ( -{ - "#url" : "https://coomer.su/onlyfans/user/alinity/post/125962203", - "#comment" : "coomer (#2100)", - "#category": ("", "coomerparty", "onlyfans"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#urls" : "https://coomer.su/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8.jpg", -}, - -{ - "#url" : "https://coomer.party/onlyfans/user/alinity/post/125962203", - "#category": ("", "coomerparty", "onlyfans"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#urls" : "https://coomer.party/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8.jpg", -}, - + { + "#url": "https://coomer.su/onlyfans/user/alinity/post/125962203", + "#comment": "coomer (#2100)", + "#category": ("", "coomerparty", "onlyfans"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#urls": "https://coomer.su/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8.jpg", + }, + { + "#url": "https://coomer.party/onlyfans/user/alinity/post/125962203", + "#category": ("", "coomerparty", "onlyfans"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#urls": "https://coomer.party/data/7d/3f/7d3fd9804583dc224968c0591163ec91794552b04f00a6c2f42a15b68231d5a8.jpg", + }, ) diff --git a/test/results/cyberdrop.py b/test/results/cyberdrop.py index 7f6a8eb935..9d1dbf2a31 100644 --- a/test/results/cyberdrop.py +++ b/test/results/cyberdrop.py @@ -1,61 +1,54 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import cyberdrop - __tests__ = ( -{ - "#url" : "https://cyberdrop.me/a/8uE0wQiK", - "#category": ("lolisafe", "cyberdrop", "album"), - "#class" : cyberdrop.CyberdropAlbumExtractor, - "#pattern" : r"https://\w+\.cyberdrop\.ch/api/file/d/yyK9y8xpQK5dP\?token=ey.+", - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", - - "album_id" : "8uE0wQiK", - "album_name" : "test テスト \"&>", - "album_size" : 182, - "count" : 1, - "date" : "dt:2023-11-26 00:00:00", - "description" : "test テスト \"&>", - "extension" : "png", - "filename" : "test-テスト--22->-rwU3x9LU", - "id" : "rwU3x9LU", - "name" : "test-テスト--22->", - "num" : 1, - "size" : 182, - "slug" : "yyK9y8xpQK5dP", - "thumbnail_url": str, - "type" : "image/png", - "url" : str, -}, - -{ - "#url" : "https://cyberdrop.me/a/HriMgbuf", - "#category": ("lolisafe", "cyberdrop", "album"), - "#class" : cyberdrop.CyberdropAlbumExtractor, - "#pattern" : r"https://\w+\.cyberdrop\.ch/api/file/d/\w+\?token=ey.+", - "#count" : 3, - - "album_id" : "HriMgbuf", - "album_name" : "animations", - "album_size" : 1090519, - "count" : 3, - "date" : "dt:2023-11-26 00:00:00", - "description" : "animated stuff", - "extension" : r"re:gif|webm", - "filename" : r"re:danbooru_\d+_\w+-\w+", - "id" : str, - "name" : r"re:danbooru_\d+_\w+", - "num" : range(1, 3), - "size" : int, - "slug" : str, - "thumbnail_url": str, - "type" : r"re:image/gif|video/webm", - "url" : str, -}, - + { + "#url": "https://cyberdrop.me/a/8uE0wQiK", + "#category": ("lolisafe", "cyberdrop", "album"), + "#class": cyberdrop.CyberdropAlbumExtractor, + "#pattern": r"https://\w+\.cyberdrop\.ch/api/file/d/yyK9y8xpQK5dP\?token=ey.+", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + "album_id": "8uE0wQiK", + "album_name": 'test テスト "&>', + "album_size": 182, + "count": 1, + "date": "dt:2023-11-26 00:00:00", + "description": 'test テスト "&>', + "extension": "png", + "filename": "test-テスト--22->-rwU3x9LU", + "id": "rwU3x9LU", + "name": "test-テスト--22->", + "num": 1, + "size": 182, + "slug": "yyK9y8xpQK5dP", + "thumbnail_url": str, + "type": "image/png", + "url": str, + }, + { + "#url": "https://cyberdrop.me/a/HriMgbuf", + "#category": ("lolisafe", "cyberdrop", "album"), + "#class": cyberdrop.CyberdropAlbumExtractor, + "#pattern": r"https://\w+\.cyberdrop\.ch/api/file/d/\w+\?token=ey.+", + "#count": 3, + "album_id": "HriMgbuf", + "album_name": "animations", + "album_size": 1090519, + "count": 3, + "date": "dt:2023-11-26 00:00:00", + "description": "animated stuff", + "extension": r"re:gif|webm", + "filename": r"re:danbooru_\d+_\w+-\w+", + "id": str, + "name": r"re:danbooru_\d+_\w+", + "num": range(1, 3), + "size": int, + "slug": str, + "thumbnail_url": str, + "type": r"re:image/gif|video/webm", + "url": str, + }, ) diff --git a/test/results/danbooru.py b/test/results/danbooru.py index bad6641369..001a93898c 100644 --- a/test/results/danbooru.py +++ b/test/results/danbooru.py @@ -1,234 +1,216 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import danbooru - __tests__ = ( -{ - "#url" : "https://danbooru.donmai.us/posts?tags=bonocho", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, - "#sha1_content": "b196fb9f1668109d7774a0a82efea3ffdda07746", -}, - -{ - "#url" : "https://danbooru.donmai.us/posts?tags=mushishi", - "#comment" : "test page transitions", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, - "#count" : ">= 300", -}, - -{ - "#url" : "https://danbooru.donmai.us/posts?tags=pixiv_id%3A1476533", - "#comment" : "'external' option (#1747)", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, - "#options" : {"external": True}, - "#pattern" : r"https://i\.pximg\.net/img-original/img/2008/08/28/02/35/48/1476533_p0\.jpg", -}, - -{ - "#url" : "https://hijiribe.donmai.us/posts?tags=bonocho", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, -}, - -{ - "#url" : "https://sonohara.donmai.us/posts?tags=bonocho", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, -}, - -{ - "#url" : "https://safebooru.donmai.us/posts?tags=bonocho", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, -}, - -{ - "#url" : "https://donmai.moe/posts?tags=bonocho", - "#category": ("Danbooru", "danbooru", "tag"), - "#class" : danbooru.DanbooruTagExtractor, -}, - -{ - "#url" : "https://danbooru.donmai.us/pools/7659", - "#category": ("Danbooru", "danbooru", "pool"), - "#class" : danbooru.DanbooruPoolExtractor, - "#sha1_content": "b16bab12bea5f7ea9e0a836bf8045f280e113d99", -}, - -{ - "#url" : "https://danbooru.donmai.us/pool/show/7659", - "#category": ("Danbooru", "danbooru", "pool"), - "#class" : danbooru.DanbooruPoolExtractor, -}, - -{ - "#url" : "https://danbooru.donmai.us/posts/294929", - "#category": ("Danbooru", "danbooru", "post"), - "#class" : danbooru.DanbooruPostExtractor, - "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", - - "approver_id": None, - "bit_flags": 0, - "created_at": "2008-08-12T00:46:05.385-04:00", - "date": "dt:2008-08-12 04:46:05", - "down_score": 0, - "extension": "jpg", - "fav_count": 9, - "file_ext": "jpg", - "file_size": 358232, - "file_url": "https://cdn.donmai.us/original/ac/8e/ac8e3b92ea328ce9cf7211e69c905bf9.jpg", - "filename": "ac8e3b92ea328ce9cf7211e69c905bf9", - "has_active_children": False, - "has_children": False, - "has_large": True, - "has_visible_children": False, - "id": 294929, - "image_height": 687, - "image_width": 895, - "is_banned": False, - "is_deleted": False, - "is_flagged": False, - "is_pending": False, - "large_file_url": "https://cdn.donmai.us/sample/ac/8e/sample-ac8e3b92ea328ce9cf7211e69c905bf9.jpg", - "last_comment_bumped_at": None, - "last_commented_at": None, - "last_noted_at": None, - "md5": "ac8e3b92ea328ce9cf7211e69c905bf9", - "media_asset": dict, - "parent_id": None, - "pixiv_id": 1129835, - "preview_file_url": "https://cdn.donmai.us/180x180/ac/8e/ac8e3b92ea328ce9cf7211e69c905bf9.jpg", - "rating": "s", - "score": 1, - "source": "https://i.pximg.net/img-original/img/2008/07/09/16/10/23/1129835_p0.jpg", - "subcategory": "post", - "tag_count": 32, - "tag_count_artist": 1, - "tag_count_character": 3, - "tag_count_copyright": 3, - "tag_count_general": 23, - "tag_count_meta": 2, - "tag_string": "2boys bat_(animal) batman batman_(series) black_bodysuit bodysuit bonocho brown_eyes closed_mouth collared_shirt commentary_request copyright_name dc_comics expressionless facepaint glasgow_smile heath_ledger joker_(dc) male_focus multiple_boys outline outstretched_arm parted_lips photoshop_(medium) pink_shirt shirt sketch smile the_dark_knight upper_body white_outline wing_collar", - "tag_string_artist": "bonocho", - "tag_string_character": "batman heath_ledger joker_(dc)", - "tag_string_copyright": "batman_(series) dc_comics the_dark_knight", - "tag_string_general": "2boys bat_(animal) black_bodysuit bodysuit brown_eyes closed_mouth collared_shirt copyright_name expressionless facepaint glasgow_smile male_focus multiple_boys outline outstretched_arm parted_lips pink_shirt shirt sketch smile upper_body white_outline wing_collar", - "tag_string_meta": "commentary_request photoshop_(medium)", - "tags": [ - "2boys", - "bat_(animal)", - "batman", - "batman_(series)", - "black_bodysuit", - "bodysuit", - "bonocho", - "brown_eyes", - "closed_mouth", - "collared_shirt", - "commentary_request", - "copyright_name", - "dc_comics", - "expressionless", - "facepaint", - "glasgow_smile", - "heath_ledger", - "joker_(dc)", - "male_focus", - "multiple_boys", - "outline", - "outstretched_arm", - "parted_lips", - "photoshop_(medium)", - "pink_shirt", - "shirt", - "sketch", - "smile", - "the_dark_knight", - "upper_body", - "white_outline", - "wing_collar", - ], - "tags_artist": [ - "bonocho", - ], - "tags_character": [ - "batman", - "heath_ledger", - "joker_(dc)", - ], - "tags_copyright": [ - "batman_(series)", - "dc_comics", - "the_dark_knight", - ], - "tags_general": [ - "2boys", - "bat_(animal)", - "black_bodysuit", - "bodysuit", - "brown_eyes", - "closed_mouth", - "collared_shirt", - "copyright_name", - "expressionless", - "facepaint", - "glasgow_smile", - "male_focus", - "multiple_boys", - "outline", - "outstretched_arm", - "parted_lips", - "pink_shirt", - "shirt", - "sketch", - "smile", - "upper_body", - "white_outline", - "wing_collar", - ], - "tags_meta": [ - "commentary_request", - "photoshop_(medium)", - ], - "up_score": range(1, 5), - "updated_at": "2022-07-11T23:42:31.881-04:00", - "uploader_id": 67005, -}, - -{ - "#url" : "https://danbooru.donmai.us/posts/3613024", - "#category": ("Danbooru", "danbooru", "post"), - "#class" : danbooru.DanbooruPostExtractor, - "#options" : {"ugoira": True}, - "#pattern" : r"https?://.+\.zip$", -}, - -{ - "#url" : "https://danbooru.donmai.us/post/show/294929", - "#category": ("Danbooru", "danbooru", "post"), - "#class" : danbooru.DanbooruPostExtractor, -}, - -{ - "#url" : "https://danbooru.donmai.us/explore/posts/popular", - "#category": ("Danbooru", "danbooru", "popular"), - "#class" : danbooru.DanbooruPopularExtractor, -}, - -{ - "#url" : "https://danbooru.donmai.us/explore/posts/popular?date=2013-06-06&scale=week", - "#category": ("Danbooru", "danbooru", "popular"), - "#class" : danbooru.DanbooruPopularExtractor, - "#range" : "1-120", - "#count" : 120, -}, - + { + "#url": "https://danbooru.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + "#sha1_content": "b196fb9f1668109d7774a0a82efea3ffdda07746", + }, + { + "#url": "https://danbooru.donmai.us/posts?tags=mushishi", + "#comment": "test page transitions", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + "#count": ">= 300", + }, + { + "#url": "https://danbooru.donmai.us/posts?tags=pixiv_id%3A1476533", + "#comment": "'external' option (#1747)", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + "#options": {"external": True}, + "#pattern": r"https://i\.pximg\.net/img-original/img/2008/08/28/02/35/48/1476533_p0\.jpg", + }, + { + "#url": "https://hijiribe.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + }, + { + "#url": "https://sonohara.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + }, + { + "#url": "https://safebooru.donmai.us/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + }, + { + "#url": "https://donmai.moe/posts?tags=bonocho", + "#category": ("Danbooru", "danbooru", "tag"), + "#class": danbooru.DanbooruTagExtractor, + }, + { + "#url": "https://danbooru.donmai.us/pools/7659", + "#category": ("Danbooru", "danbooru", "pool"), + "#class": danbooru.DanbooruPoolExtractor, + "#sha1_content": "b16bab12bea5f7ea9e0a836bf8045f280e113d99", + }, + { + "#url": "https://danbooru.donmai.us/pool/show/7659", + "#category": ("Danbooru", "danbooru", "pool"), + "#class": danbooru.DanbooruPoolExtractor, + }, + { + "#url": "https://danbooru.donmai.us/posts/294929", + "#category": ("Danbooru", "danbooru", "post"), + "#class": danbooru.DanbooruPostExtractor, + "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", + "approver_id": None, + "bit_flags": 0, + "created_at": "2008-08-12T00:46:05.385-04:00", + "date": "dt:2008-08-12 04:46:05", + "down_score": 0, + "extension": "jpg", + "fav_count": 9, + "file_ext": "jpg", + "file_size": 358232, + "file_url": "https://cdn.donmai.us/original/ac/8e/ac8e3b92ea328ce9cf7211e69c905bf9.jpg", + "filename": "ac8e3b92ea328ce9cf7211e69c905bf9", + "has_active_children": False, + "has_children": False, + "has_large": True, + "has_visible_children": False, + "id": 294929, + "image_height": 687, + "image_width": 895, + "is_banned": False, + "is_deleted": False, + "is_flagged": False, + "is_pending": False, + "large_file_url": "https://cdn.donmai.us/sample/ac/8e/sample-ac8e3b92ea328ce9cf7211e69c905bf9.jpg", + "last_comment_bumped_at": None, + "last_commented_at": None, + "last_noted_at": None, + "md5": "ac8e3b92ea328ce9cf7211e69c905bf9", + "media_asset": dict, + "parent_id": None, + "pixiv_id": 1129835, + "preview_file_url": "https://cdn.donmai.us/180x180/ac/8e/ac8e3b92ea328ce9cf7211e69c905bf9.jpg", + "rating": "s", + "score": 1, + "source": "https://i.pximg.net/img-original/img/2008/07/09/16/10/23/1129835_p0.jpg", + "subcategory": "post", + "tag_count": 32, + "tag_count_artist": 1, + "tag_count_character": 3, + "tag_count_copyright": 3, + "tag_count_general": 23, + "tag_count_meta": 2, + "tag_string": "2boys bat_(animal) batman batman_(series) black_bodysuit bodysuit bonocho brown_eyes closed_mouth collared_shirt commentary_request copyright_name dc_comics expressionless facepaint glasgow_smile heath_ledger joker_(dc) male_focus multiple_boys outline outstretched_arm parted_lips photoshop_(medium) pink_shirt shirt sketch smile the_dark_knight upper_body white_outline wing_collar", + "tag_string_artist": "bonocho", + "tag_string_character": "batman heath_ledger joker_(dc)", + "tag_string_copyright": "batman_(series) dc_comics the_dark_knight", + "tag_string_general": "2boys bat_(animal) black_bodysuit bodysuit brown_eyes closed_mouth collared_shirt copyright_name expressionless facepaint glasgow_smile male_focus multiple_boys outline outstretched_arm parted_lips pink_shirt shirt sketch smile upper_body white_outline wing_collar", + "tag_string_meta": "commentary_request photoshop_(medium)", + "tags": [ + "2boys", + "bat_(animal)", + "batman", + "batman_(series)", + "black_bodysuit", + "bodysuit", + "bonocho", + "brown_eyes", + "closed_mouth", + "collared_shirt", + "commentary_request", + "copyright_name", + "dc_comics", + "expressionless", + "facepaint", + "glasgow_smile", + "heath_ledger", + "joker_(dc)", + "male_focus", + "multiple_boys", + "outline", + "outstretched_arm", + "parted_lips", + "photoshop_(medium)", + "pink_shirt", + "shirt", + "sketch", + "smile", + "the_dark_knight", + "upper_body", + "white_outline", + "wing_collar", + ], + "tags_artist": [ + "bonocho", + ], + "tags_character": [ + "batman", + "heath_ledger", + "joker_(dc)", + ], + "tags_copyright": [ + "batman_(series)", + "dc_comics", + "the_dark_knight", + ], + "tags_general": [ + "2boys", + "bat_(animal)", + "black_bodysuit", + "bodysuit", + "brown_eyes", + "closed_mouth", + "collared_shirt", + "copyright_name", + "expressionless", + "facepaint", + "glasgow_smile", + "male_focus", + "multiple_boys", + "outline", + "outstretched_arm", + "parted_lips", + "pink_shirt", + "shirt", + "sketch", + "smile", + "upper_body", + "white_outline", + "wing_collar", + ], + "tags_meta": [ + "commentary_request", + "photoshop_(medium)", + ], + "up_score": range(1, 5), + "updated_at": "2022-07-11T23:42:31.881-04:00", + "uploader_id": 67005, + }, + { + "#url": "https://danbooru.donmai.us/posts/3613024", + "#category": ("Danbooru", "danbooru", "post"), + "#class": danbooru.DanbooruPostExtractor, + "#options": {"ugoira": True}, + "#pattern": r"https?://.+\.zip$", + }, + { + "#url": "https://danbooru.donmai.us/post/show/294929", + "#category": ("Danbooru", "danbooru", "post"), + "#class": danbooru.DanbooruPostExtractor, + }, + { + "#url": "https://danbooru.donmai.us/explore/posts/popular", + "#category": ("Danbooru", "danbooru", "popular"), + "#class": danbooru.DanbooruPopularExtractor, + }, + { + "#url": "https://danbooru.donmai.us/explore/posts/popular?date=2013-06-06&scale=week", + "#category": ("Danbooru", "danbooru", "popular"), + "#class": danbooru.DanbooruPopularExtractor, + "#range": "1-120", + "#count": 120, + }, ) diff --git a/test/results/derpibooru.py b/test/results/derpibooru.py index 4fe775bb29..51acddb0f8 100644 --- a/test/results/derpibooru.py +++ b/test/results/derpibooru.py @@ -1,128 +1,113 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import philomena - __tests__ = ( -{ - "#url" : "https://derpibooru.org/images/1", - "#category": ("philomena", "derpibooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, - "#count" : 1, - "#sha1_content": "88449eeb0c4fa5d3583d0b794f6bc1d70bf7f889", - - "animated" : False, - "aspect_ratio" : 1.0, - "comment_count" : int, - "created_at" : "2012-01-02T03:12:33Z", - "date" : "dt:2012-01-02 03:12:33", - "deletion_reason" : None, - "description" : "", - "downvotes" : int, - "duplicate_of" : None, - "duration" : 0.04, - "extension" : "png", - "faves" : int, - "first_seen_at" : "2012-01-02T03:12:33Z", - "format" : "png", - "height" : 900, - "hidden_from_users": False, - "id" : 1, - "mime_type" : "image/png", - "name" : "1__safe_fluttershy_solo_cloud_happy_flying_upvotes+galore_artist-colon-speccysy_get_sunshine", - "orig_sha512_hash": None, - "processed" : True, - "representations" : dict, - "score" : int, - "sha512_hash" : "f16c98e2848c2f1bfff3985e8f1a54375cc49f78125391aeb80534ce011ead14e3e452a5c4bc98a66f56bdfcd07ef7800663b994f3f343c572da5ecc22a9660f", - "size" : 860914, - "source_url" : "https://web.archive.org/web/20110702164313/http://speccysy.deviantart.com:80/art/Afternoon-Flight-215193985", - "spoilered" : False, - "tag_count" : int, - "tag_ids" : list, - "tags" : list, - "thumbnails_generated": True, - "updated_at" : r"re:\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ", - "uploader" : "Clover the Clever", - "uploader_id" : 211188, - "upvotes" : int, - "view_url" : str, - "width" : 900, - "wilson_score" : float, -}, - -{ - "#url" : "https://derpibooru.org/images/3334658", - "#comment" : "svg (#5643)", - "#category": ("philomena", "derpibooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, - "#urls" : "https://derpicdn.net/img/view/2024/4/1/3334658__safe_alternate+version_artist-colon-jp_derpibooru+exclusive_twilight+sparkle_alicorn_pony_amending+fences_g4_season+5_-dot-svg+available_female_grin_lo.svg", - "#sha1_content": "eec5adf02e2a4fe83b9211c0444d57dc03e21f50", - - "extension": "svg", - "format" : "svg", -}, - -{ - "#url" : "https://derpibooru.org/1", - "#category": ("philomena", "derpibooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, -}, - -{ - "#url" : "https://www.derpibooru.org/1", - "#category": ("philomena", "derpibooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, -}, - -{ - "#url" : "https://www.derpibooru.org/images/1", - "#category": ("philomena", "derpibooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, -}, - -{ - "#url" : "https://derpibooru.org/search?q=cute", - "#category": ("philomena", "derpibooru", "search"), - "#class" : philomena.PhilomenaSearchExtractor, - "#range" : "40-60", - "#count" : 21, -}, - -{ - "#url" : "https://derpibooru.org/tags/cute", - "#category": ("philomena", "derpibooru", "search"), - "#class" : philomena.PhilomenaSearchExtractor, - "#range" : "40-60", - "#count" : 21, -}, - -{ - "#url" : "https://derpibooru.org/tags/artist-colon--dash-_-fwslash--fwslash-%255Bkorroki%255D_aternak", - "#category": ("philomena", "derpibooru", "search"), - "#class" : philomena.PhilomenaSearchExtractor, - "#count" : ">= 2", -}, - -{ - "#url" : "https://derpibooru.org/galleries/1", - "#category": ("philomena", "derpibooru", "gallery"), - "#class" : philomena.PhilomenaGalleryExtractor, - "#pattern" : r"https://derpicdn\.net/img/view/\d+/\d+/\d+/\d+[^/]+$", - - "gallery": { - "description" : "Indexes start at 1 :P", - "id" : 1, - "spoiler_warning": "", - "thumbnail_id" : 1, - "title" : "The Very First Gallery", - "user" : "DeliciousBlackInk", - "user_id" : 365446, + { + "#url": "https://derpibooru.org/images/1", + "#category": ("philomena", "derpibooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + "#count": 1, + "#sha1_content": "88449eeb0c4fa5d3583d0b794f6bc1d70bf7f889", + "animated": False, + "aspect_ratio": 1.0, + "comment_count": int, + "created_at": "2012-01-02T03:12:33Z", + "date": "dt:2012-01-02 03:12:33", + "deletion_reason": None, + "description": "", + "downvotes": int, + "duplicate_of": None, + "duration": 0.04, + "extension": "png", + "faves": int, + "first_seen_at": "2012-01-02T03:12:33Z", + "format": "png", + "height": 900, + "hidden_from_users": False, + "id": 1, + "mime_type": "image/png", + "name": "1__safe_fluttershy_solo_cloud_happy_flying_upvotes+galore_artist-colon-speccysy_get_sunshine", + "orig_sha512_hash": None, + "processed": True, + "representations": dict, + "score": int, + "sha512_hash": "f16c98e2848c2f1bfff3985e8f1a54375cc49f78125391aeb80534ce011ead14e3e452a5c4bc98a66f56bdfcd07ef7800663b994f3f343c572da5ecc22a9660f", + "size": 860914, + "source_url": "https://web.archive.org/web/20110702164313/http://speccysy.deviantart.com:80/art/Afternoon-Flight-215193985", + "spoilered": False, + "tag_count": int, + "tag_ids": list, + "tags": list, + "thumbnails_generated": True, + "updated_at": r"re:\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ", + "uploader": "Clover the Clever", + "uploader_id": 211188, + "upvotes": int, + "view_url": str, + "width": 900, + "wilson_score": float, + }, + { + "#url": "https://derpibooru.org/images/3334658", + "#comment": "svg (#5643)", + "#category": ("philomena", "derpibooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + "#urls": "https://derpicdn.net/img/view/2024/4/1/3334658__safe_alternate+version_artist-colon-jp_derpibooru+exclusive_twilight+sparkle_alicorn_pony_amending+fences_g4_season+5_-dot-svg+available_female_grin_lo.svg", + "#sha1_content": "eec5adf02e2a4fe83b9211c0444d57dc03e21f50", + "extension": "svg", + "format": "svg", + }, + { + "#url": "https://derpibooru.org/1", + "#category": ("philomena", "derpibooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + }, + { + "#url": "https://www.derpibooru.org/1", + "#category": ("philomena", "derpibooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + }, + { + "#url": "https://www.derpibooru.org/images/1", + "#category": ("philomena", "derpibooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + }, + { + "#url": "https://derpibooru.org/search?q=cute", + "#category": ("philomena", "derpibooru", "search"), + "#class": philomena.PhilomenaSearchExtractor, + "#range": "40-60", + "#count": 21, + }, + { + "#url": "https://derpibooru.org/tags/cute", + "#category": ("philomena", "derpibooru", "search"), + "#class": philomena.PhilomenaSearchExtractor, + "#range": "40-60", + "#count": 21, + }, + { + "#url": "https://derpibooru.org/tags/artist-colon--dash-_-fwslash--fwslash-%255Bkorroki%255D_aternak", + "#category": ("philomena", "derpibooru", "search"), + "#class": philomena.PhilomenaSearchExtractor, + "#count": ">= 2", + }, + { + "#url": "https://derpibooru.org/galleries/1", + "#category": ("philomena", "derpibooru", "gallery"), + "#class": philomena.PhilomenaGalleryExtractor, + "#pattern": r"https://derpicdn\.net/img/view/\d+/\d+/\d+/\d+[^/]+$", + "gallery": { + "description": "Indexes start at 1 :P", + "id": 1, + "spoiler_warning": "", + "thumbnail_id": 1, + "title": "The Very First Gallery", + "user": "DeliciousBlackInk", + "user_id": 365446, + }, }, -}, - ) diff --git a/test/results/desktopography.py b/test/results/desktopography.py index c6066692dc..1c4497414f 100644 --- a/test/results/desktopography.py +++ b/test/results/desktopography.py @@ -1,29 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import desktopography - __tests__ = ( -{ - "#url" : "https://desktopography.net/", - "#category": ("", "desktopography", "site"), - "#class" : desktopography.DesktopographySiteExtractor, -}, - -{ - "#url" : "https://desktopography.net/exhibition-2020/", - "#category": ("", "desktopography", "exhibition"), - "#class" : desktopography.DesktopographyExhibitionExtractor, -}, - -{ - "#url" : "https://desktopography.net/portfolios/new-era/", - "#category": ("", "desktopography", "entry"), - "#class" : desktopography.DesktopographyEntryExtractor, -}, - + { + "#url": "https://desktopography.net/", + "#category": ("", "desktopography", "site"), + "#class": desktopography.DesktopographySiteExtractor, + }, + { + "#url": "https://desktopography.net/exhibition-2020/", + "#category": ("", "desktopography", "exhibition"), + "#class": desktopography.DesktopographyExhibitionExtractor, + }, + { + "#url": "https://desktopography.net/portfolios/new-era/", + "#category": ("", "desktopography", "entry"), + "#class": desktopography.DesktopographyEntryExtractor, + }, ) diff --git a/test/results/desuarchive.py b/test/results/desuarchive.py index 43cd968951..fa56dfd9a5 100644 --- a/test/results/desuarchive.py +++ b/test/results/desuarchive.py @@ -1,56 +1,46 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://desuarchive.org/a/thread/159542679/", - "#category": ("foolfuuka", "desuarchive", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "e7d624aded15a069194e38dc731ec23217a422fb", -}, - -{ - "#url" : "https://desuarchive.org/a", - "#category": ("foolfuuka", "desuarchive", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://desuarchive.org/a/", - "#category": ("foolfuuka", "desuarchive", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://desuarchive.org/a/2", - "#category": ("foolfuuka", "desuarchive", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://desuarchive.org/a/page/2", - "#category": ("foolfuuka", "desuarchive", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, - "#pattern" : foolfuuka.FoolfuukaThreadExtractor.pattern, - "#count" : 10, -}, - -{ - "#url" : "https://desuarchive.org/_/search/text/test/", - "#category": ("foolfuuka", "desuarchive", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://desuarchive.org/a/gallery/5", - "#category": ("foolfuuka", "desuarchive", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://desuarchive.org/a/thread/159542679/", + "#category": ("foolfuuka", "desuarchive", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "e7d624aded15a069194e38dc731ec23217a422fb", + }, + { + "#url": "https://desuarchive.org/a", + "#category": ("foolfuuka", "desuarchive", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://desuarchive.org/a/", + "#category": ("foolfuuka", "desuarchive", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://desuarchive.org/a/2", + "#category": ("foolfuuka", "desuarchive", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://desuarchive.org/a/page/2", + "#category": ("foolfuuka", "desuarchive", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + "#pattern": foolfuuka.FoolfuukaThreadExtractor.pattern, + "#count": 10, + }, + { + "#url": "https://desuarchive.org/_/search/text/test/", + "#category": ("foolfuuka", "desuarchive", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://desuarchive.org/a/gallery/5", + "#category": ("foolfuuka", "desuarchive", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/deviantart.py b/test/results/deviantart.py index 0e8e371c41..285111e67c 100644 --- a/test/results/deviantart.py +++ b/test/results/deviantart.py @@ -1,1022 +1,895 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import deviantart import datetime -from gallery_dl import exception +from gallery_dl import exception +from gallery_dl.extractor import deviantart __tests__ = ( -{ - "#url" : "https://www.deviantart.com/shimoda7", - "#category": ("", "deviantart", "user"), - "#class" : deviantart.DeviantartUserExtractor, - "#urls" : "https://www.deviantart.com/shimoda7/gallery", -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7", - "#category": ("", "deviantart", "user"), - "#class" : deviantart.DeviantartUserExtractor, - "#options" : {"include": "all"}, - "#urls" : ( - "https://www.deviantart.com/shimoda7/avatar", - "https://www.deviantart.com/shimoda7/banner", - "https://www.deviantart.com/shimoda7/gallery", - "https://www.deviantart.com/shimoda7/gallery/scraps", - "https://www.deviantart.com/shimoda7/posts", - "https://www.deviantart.com/shimoda7/posts/statuses", - "https://www.deviantart.com/shimoda7/favourites", - ), -}, - -{ - "#url" : "https://shimoda7.deviantart.com/", - "#category": ("", "deviantart", "user"), - "#class" : deviantart.DeviantartUserExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, - "#pattern" : r"https://(images-)?wixmp-[^.]+\.wixmp\.com/f/.+/.+\.(jpg|png)\?token=.+", - "#count" : ">= 38", - - "allows_comments" : bool, - "author" : { - "type" : "premium", - "usericon": str, - "userid" : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", - "username": "shimoda7", + { + "#url": "https://www.deviantart.com/shimoda7", + "#category": ("", "deviantart", "user"), + "#class": deviantart.DeviantartUserExtractor, + "#urls": "https://www.deviantart.com/shimoda7/gallery", }, - "category_path" : str, - "content" : { - "filesize" : int, - "height" : int, - "src" : str, - "transparency": bool, - "width" : int, - }, - "date" : datetime.datetime, - "deviationid" : str, - "?download_filesize": int, - "extension" : str, - "index" : int, - "is_deleted" : bool, - "is_downloadable" : bool, - "is_favourited" : bool, - "is_mature" : bool, - "preview" : { - "height" : int, - "src" : str, - "transparency": bool, - "width" : int, - }, - "published_time" : int, - "stats" : { - "comments" : int, - "favourites": int, - }, - "target" : dict, - "thumbs" : list, - "title" : str, - "url" : r"re:https://www.deviantart.com/shimoda7/art/[^/]+-\d+", - "username" : "shimoda7", -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/", - "#comment" : "range/skip (#4557)", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, - "#options" : {"original": False}, - "#pattern" : r"https://images-wixmp-[0-9a-f]+\.wixmp\.com/f/0e474835-ec35-4937-b647-b6830ed58bd1/d2idul-6158ded2-37ac-413d-802e-0689f0f020ad\.jpg\?token=[\w.]+", - "#range" : "38-", - "#count" : 1, -}, - -{ - "#url" : "https://www.deviantart.com/AlloyRabbit/gallery", - "#comment" : "deactivated account", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/Shydude/gallery", - "#comment" : "deactivated account", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/zapor666/gallery", - "#comment" : "deactivated account", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/yakuzafc/gallery", - "#comment" : "group", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, - "#pattern" : r"https://www.deviantart.com/yakuzafc/gallery/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/", - "#count" : ">= 15", -}, - -{ - "#url" : "https://www.deviantart.com/yakuzafc/gallery", - "#comment" : "'group': 'skip' (#4630)", - "#category" : ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, - "#options" : {"group": "skip"}, - "#exception": exception.StopExtraction, - "#count" : 0, -}, - -{ - "#url" : "https://www.deviantart.com/justatest235723/gallery", - "#comment" : "'folders' option (#276)", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, - "#options" : { - "metadata": 1, - "folders" : 1, - "original": 0, - }, - "#count" : 3, - - "description": str, - "folders" : list, - "is_watching": bool, - "license" : str, - "tags" : list, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda8/gallery/", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/all", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/?catpath=/", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/gallery/", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/gallery/all/", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/gallery/?catpath=/", - "#category": ("", "deviantart", "gallery"), - "#class" : deviantart.DeviantartGalleryExtractor, -}, - -{ - "#url" : "https://deviantart.com/shimoda7/avatar", - "#category": ("", "deviantart", "avatar"), - "#class" : deviantart.DeviantartAvatarExtractor, - "#urls" : "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4", - "#sha1_content": "abf2cc79b842315f2e54bfdd93bf794a0f612b6f", - - "author" : { - "type" : "premium", - "usericon": "https://a.deviantart.net/avatars/s/h/shimoda7.jpg?4", - "userid" : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", + { + "#url": "https://www.deviantart.com/shimoda7", + "#category": ("", "deviantart", "user"), + "#class": deviantart.DeviantartUserExtractor, + "#options": {"include": "all"}, + "#urls": ( + "https://www.deviantart.com/shimoda7/avatar", + "https://www.deviantart.com/shimoda7/banner", + "https://www.deviantart.com/shimoda7/gallery", + "https://www.deviantart.com/shimoda7/gallery/scraps", + "https://www.deviantart.com/shimoda7/posts", + "https://www.deviantart.com/shimoda7/posts/statuses", + "https://www.deviantart.com/shimoda7/favourites", + ), + }, + { + "#url": "https://shimoda7.deviantart.com/", + "#category": ("", "deviantart", "user"), + "#class": deviantart.DeviantartUserExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + "#pattern": r"https://(images-)?wixmp-[^.]+\.wixmp\.com/f/.+/.+\.(jpg|png)\?token=.+", + "#count": ">= 38", + "allows_comments": bool, + "author": { + "type": "premium", + "usericon": str, + "userid": "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", + "username": "shimoda7", + }, + "category_path": str, + "content": { + "filesize": int, + "height": int, + "src": str, + "transparency": bool, + "width": int, + }, + "date": datetime.datetime, + "deviationid": str, + "?download_filesize": int, + "extension": str, + "index": int, + "is_deleted": bool, + "is_downloadable": bool, + "is_favourited": bool, + "is_mature": bool, + "preview": { + "height": int, + "src": str, + "transparency": bool, + "width": int, + }, + "published_time": int, + "stats": { + "comments": int, + "favourites": int, + }, + "target": dict, + "thumbs": list, + "title": str, + "url": r"re:https://www.deviantart.com/shimoda7/art/[^/]+-\d+", "username": "shimoda7", }, - "content" : { - "src": "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4" - }, - "da_category" : "avatar", - "date" : "dt:1970-01-01 00:00:00", - "extension" : "jpg", - "filename" : "avatar_by_shimoda7-d4", - "index" : 4, - "index_base36" : "4", - "is_deleted" : False, - "is_downloadable": False, - "is_original" : True, - "published_time" : 0, - "target" : { + { + "#url": "https://www.deviantart.com/shimoda7/gallery/", + "#comment": "range/skip (#4557)", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + "#options": {"original": False}, + "#pattern": r"https://images-wixmp-[0-9a-f]+\.wixmp\.com/f/0e474835-ec35-4937-b647-b6830ed58bd1/d2idul-6158ded2-37ac-413d-802e-0689f0f020ad\.jpg\?token=[\w.]+", + "#range": "38-", + "#count": 1, + }, + { + "#url": "https://www.deviantart.com/AlloyRabbit/gallery", + "#comment": "deactivated account", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://www.deviantart.com/Shydude/gallery", + "#comment": "deactivated account", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://www.deviantart.com/zapor666/gallery", + "#comment": "deactivated account", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://www.deviantart.com/yakuzafc/gallery", + "#comment": "group", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + "#pattern": r"https://www.deviantart.com/yakuzafc/gallery/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/", + "#count": ">= 15", + }, + { + "#url": "https://www.deviantart.com/yakuzafc/gallery", + "#comment": "'group': 'skip' (#4630)", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + "#options": {"group": "skip"}, + "#exception": exception.StopExtraction, + "#count": 0, + }, + { + "#url": "https://www.deviantart.com/justatest235723/gallery", + "#comment": "'folders' option (#276)", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + "#options": { + "metadata": 1, + "folders": 1, + "original": 0, + }, + "#count": 3, + "description": str, + "folders": list, + "is_watching": bool, + "license": str, + "tags": list, + }, + { + "#url": "https://www.deviantart.com/shimoda8/gallery/", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/all", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/?catpath=/", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://shimoda7.deviantart.com/gallery/", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://shimoda7.deviantart.com/gallery/all/", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://shimoda7.deviantart.com/gallery/?catpath=/", + "#category": ("", "deviantart", "gallery"), + "#class": deviantart.DeviantartGalleryExtractor, + }, + { + "#url": "https://deviantart.com/shimoda7/avatar", + "#category": ("", "deviantart", "avatar"), + "#class": deviantart.DeviantartAvatarExtractor, + "#urls": "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4", + "#sha1_content": "abf2cc79b842315f2e54bfdd93bf794a0f612b6f", + "author": { + "type": "premium", + "usericon": "https://a.deviantart.net/avatars/s/h/shimoda7.jpg?4", + "userid": "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B", + "username": "shimoda7", + }, + "content": {"src": "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4"}, + "da_category": "avatar", + "date": "dt:1970-01-01 00:00:00", "extension": "jpg", - "filename" : "avatar_by_shimoda7-d4", - "src" : "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4" + "filename": "avatar_by_shimoda7-d4", + "index": 4, + "index_base36": "4", + "is_deleted": False, + "is_downloadable": False, + "is_original": True, + "published_time": 0, + "target": { + "extension": "jpg", + "filename": "avatar_by_shimoda7-d4", + "src": "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4", + }, + "title": "avatar", + "username": "shimoda7", }, - "title" : "avatar", - "username" : "shimoda7", -}, - -{ - "#url" : "https://deviantart.com/shimoda7/avatar", - "#comment" : "'formats' option", - "#category": ("", "deviantart", "avatar"), - "#class" : deviantart.DeviantartAvatarExtractor, - "#archive" : False, - "#options" : {"formats": ["original.jpg", "big.jpg", "big.png", "big.gif"]}, - "#urls" : ( - "https://a.deviantart.net/avatars-original/s/h/shimoda7.jpg?4", - "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4", - "https://a.deviantart.net/avatars-big/s/h/shimoda7.png?4", - "https://a.deviantart.net/avatars-big/s/h/shimoda7.gif?4", - ), -}, - -{ - "#url" : "https://deviantart.com/h3813067/avatar", - "#comment" : "default avatar (#5276)", - "#category": ("", "deviantart", "avatar"), - "#class" : deviantart.DeviantartAvatarExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://deviantart.com/gdldev/banner", - "#category": ("", "deviantart", "background"), - "#class" : deviantart.DeviantartBackgroundExtractor, - "#pattern" : r"https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$", - "#sha1_content": "980eaa76ce515f1b6bef60dfadf26a5bbe9c583f", - - "allows_comments" : True, - "author" : { - "type" : "regular", - "usericon": "https://a.deviantart.net/avatars/g/d/gdldev.jpg?12", - "userid" : "1A12BA26-33C2-AA0A-7678-0B6DFBA7AC8E", - "username": "gdldev" - }, - "category_path" : "", - "content" : { - "filename" : "banner_by_gdldev_dgntyqc.png", - "filesize" : 84510, - "height" : 4000, - "src" : r"re:https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$", - "transparency": False, - "width" : 6400 - }, - "date" : "dt:2024-01-02 21:16:06", - "deviationid" : "8C8D6B28-766A-DE21-7F7D-CE055C3BD50A", - "download_filesize": 84510, - "extension" : "png", - "filename" : "banner_by_gdldev-dgntyqc", - "index" : 1007488020, - "index_base36" : "gntyqc", - "is_blocked" : False, - "is_deleted" : False, - "is_downloadable" : True, - "is_favourited" : False, - "is_mature" : False, - "is_original" : True, - "is_published" : False, - "preview" : dict, - "printid" : None, - "published_time" : 1704230166, - "stats" : { - "comments" : 0, - "favourites": 0, - }, - "target" : dict, - "thumbs" : list, - "title" : "Banner", - "url" : "https://www.deviantart.com/stash/0198jippkeys", - "username" : "gdldev", -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous", - "#comment" : "user", - "#category": ("", "deviantart", "folder"), - "#class" : deviantart.DeviantartFolderExtractor, - "#options" : {"original": False}, - "#count" : 5, -}, - -{ - "#url" : "https://www.deviantart.com/yakuzafc/gallery/37412168/Crafts", - "#comment" : "group", - "#category": ("", "deviantart", "folder"), - "#class" : deviantart.DeviantartFolderExtractor, - "#options" : {"original": False}, - "#count" : ">= 4", -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/B38E3C6A-2029-6B45-757B-3C8D3422AD1A/misc", - "#comment" : "uuid", - "#category": ("", "deviantart", "folder"), - "#class" : deviantart.DeviantartFolderExtractor, - "#options" : {"original": False}, - "#count" : 5, -}, - -{ - "#url" : "https://www.deviantart.com/justatest235723/gallery/69302698/-test-b-c-d-e-f-", - "#comment" : "name starts with '_', special characters (#1451)", - "#category": ("", "deviantart", "folder"), - "#class" : deviantart.DeviantartFolderExtractor, - "#options" : {"original": False}, - "#count" : 1, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/gallery/722019/Miscellaneous", - "#category": ("", "deviantart", "folder"), - "#class" : deviantart.DeviantartFolderExtractor, -}, - -{ - "#url" : "https://yakuzafc.deviantart.com/gallery/37412168/Crafts", - "#category": ("", "deviantart", "folder"), - "#class" : deviantart.DeviantartFolderExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/stash/022c83odnaxc", - "#category": ("", "deviantart", "stash"), - "#class" : deviantart.DeviantartStashExtractor, - "#pattern" : r"https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/dcvdmbc-e506cdcf-3208-4c20-85ab-0bfa8a7bcb16.png\?token=ey.+", - "#count" : 1, - "#sha1_content": "057eb2f2861f6c8a96876b13cca1a4b7a408c11f", - - "content": { - "filename": "01_by_justatest235723_dcvdmbc.png", - "filesize": 380, - "width" : 128, - "height" : 128, - "src" : r"re:https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/dcvdmbc-e506cdcf-3208-4c20-85ab-0bfa8a7bcb16.png\?token=ey.+", - }, - "date" : "dt:2018-12-26 14:49:27", - "deviationid" : "A4A6AD52-8857-46EE-ABFE-86D49D4FF9D0", - "download_filesize": 380, - "extension" : "png", - "filename" : "01_by_justatest235723-dcvdmbc", - "index" : 778297656, - "index_base36" : "cvdmbc", - "published_time": 1545835767, - "title" : "01", - "url" : "https://www.deviantart.com/stash/022c83odnaxc", -}, - -{ - "#url" : "https://sta.sh/022c83odnaxc", - "#category": ("", "deviantart", "stash"), - "#class" : deviantart.DeviantartStashExtractor, -}, - -{ - "#url" : "https://sta.sh/21jf51j7pzl2", - "#comment" : "multiple stash items", - "#category": ("", "deviantart", "stash"), - "#class" : deviantart.DeviantartStashExtractor, - "#options" : {"original": False}, - "#count" : 4, -}, - -{ - "#url" : "https://sta.sh/024t4coz16mi", - "#comment" : "downloadable, but no 'content' field (#307)", - "#category": ("", "deviantart", "stash"), - "#class" : deviantart.DeviantartStashExtractor, - "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.rar\?token=.+", - "#count" : 1, -}, - -{ - "#url" : "https://sta.sh/215twi387vfj", - "#comment" : "mixed folders and images (#659)", - "#category": ("", "deviantart", "stash"), - "#class" : deviantart.DeviantartStashExtractor, - "#options" : {"original": False}, - "#count" : 4, -}, - -{ - "#url" : "https://sta.sh/abcdefghijkl", - "#category": ("", "deviantart", "stash"), - "#class" : deviantart.DeviantartStashExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.deviantart.com/h3813067/favourites/", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, - "#options" : { - "metadata": True, - "flat" : False, - }, - "#count" : 1, -}, - -{ - "#url" : "https://www.deviantart.com/h3813067/favourites/", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, - "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", -}, - -{ - "#url" : "https://www.deviantart.com/h3813067/favourites/all", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/h3813067/favourites/?catpath=/", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, -}, - -{ - "#url" : "https://h3813067.deviantart.com/favourites/", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, -}, - -{ - "#url" : "https://h3813067.deviantart.com/favourites/all", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, -}, - -{ - "#url" : "https://h3813067.deviantart.com/favourites/?catpath=/", - "#category": ("", "deviantart", "favorite"), - "#class" : deviantart.DeviantartFavoriteExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/pencilshadings/favourites/70595441/3D-Favorites", - "#category": ("", "deviantart", "collection"), - "#class" : deviantart.DeviantartCollectionExtractor, - "#options" : {"original": False}, - "#count" : ">= 15", -}, - -{ - "#url" : "https://www.deviantart.com/pencilshadings/favourites/F050486B-CB62-3C66-87FB-1105A7F6379F/3D Favorites", - "#category": ("", "deviantart", "collection"), - "#class" : deviantart.DeviantartCollectionExtractor, - "#options" : {"original": False}, - "#count" : ">= 15", -}, - -{ - "#url" : "https://pencilshadings.deviantart.com/favourites/70595441/3D-Favorites", - "#category": ("", "deviantart", "collection"), - "#class" : deviantart.DeviantartCollectionExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, - "#sha1_url": "48aeed5631763d96f5391d2177ea72d9fdbee4e5", -}, - -{ - "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, - "#options" : {"journals": "text"}, - "#sha1_url": "b2a8e74d275664b1a4acee0fca0a6fd33298571e", -}, - -{ - "#url" : "https://www.deviantart.com/angrywhitewanker/posts/journals/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, - "#options" : {"journals": "none"}, - "#count" : 0, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/posts/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/journal/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/journal/?catpath=/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/journal/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/journal/?catpath=/", - "#category": ("", "deviantart", "journal"), - "#class" : deviantart.DeviantartJournalExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/t1na/posts/statuses", - "#category": ("", "deviantart", "status"), - "#class" : deviantart.DeviantartStatusExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.deviantart.com/justgalym/posts/statuses", - "#category": ("", "deviantart", "status"), - "#class" : deviantart.DeviantartStatusExtractor, - "#count" : 4, - "#sha1_url": "62ee48ff3405c7714dca70abf42e8e39731012fc", -}, - -{ - "#url" : "https://www.deviantart.com/justgalym/posts/statuses", - "#category": ("", "deviantart", "status"), - "#class" : deviantart.DeviantartStatusExtractor, - "#options" : {"journals": "none"}, - "#pattern" : r"https://images-wixmp-\w+\.wixmp\.com/intermediary/f/[^/]+/[^.]+\.jpg", - "#count" : 1, -}, - -{ - "#url" : "https://www.deviantart.com/vanillaghosties/posts/statuses", - "#comment" : "shared sta.sh item", - "#category": ("", "deviantart", "status"), - "#class" : deviantart.DeviantartStatusExtractor, - "#options" : { - "journals": "none", - "original": False, - }, - "#range" : "5-", - "#count" : 1, - - "index" : int, - "index_base36": r"re:^[0-9a-z]+$", - "url" : r"re:^https://www.deviantart.com/stash/\w+", -}, - -{ - "#url" : "https://www.deviantart.com/AndrejSKalin/posts/statuses", - "#comment" : "'deleted' deviations in 'items'", - "#category": ("", "deviantart", "status"), - "#class" : deviantart.DeviantartStatusExtractor, - "#options" : { - "journals" : "none", - "original" : 0, - "image-filter": "deviationid[:8] == '147C8B03'", - }, - "#count" : 2, - "#archive" : False, - - "deviationid": "147C8B03-7D34-AE93-9241-FA3C6DBBC655", -}, - -{ - "#url" : "https://www.deviantart.com/justgalym/posts/statuses", - "#category": ("", "deviantart", "status"), - "#class" : deviantart.DeviantartStatusExtractor, - "#options" : {"journals": "text"}, - "#sha1_url": "10a336bdee7b9692919461443a7dde44d495818c", -}, - -{ - "#url" : "https://www.deviantart.com/tag/nature", - "#category": ("", "deviantart", "tag"), - "#class" : deviantart.DeviantartTagExtractor, - "#options" : {"original": False}, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://www.deviantart.com/watch/deviations", - "#category": ("", "deviantart", "watch"), - "#class" : deviantart.DeviantartWatchExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/notifications/watch", - "#category": ("", "deviantart", "watch"), - "#class" : deviantart.DeviantartWatchExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/watch/posts", - "#category": ("", "deviantart", "watch-posts"), - "#class" : deviantart.DeviantartWatchPostsExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/art/For-the-sake-10073852", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : {"original": 0}, - "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/art/For-the-sake-10073852", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : {"metadata": "submission,camera,stats"}, - - "can_post_comment": False, - "description" : str, - "is_watching" : False, - "license" : "No License", - "stats": { - "comments" : int, - "downloads" : int, - "downloads_today": int, - "favourites" : int, - "views" : int, - "views_today" : int, - }, - "submission": { - "category" : "traditional/drawings/other", - "creation_time" : "2004-08-25T02:44:08-0700", - "file_size" : "133 KB", - "resolution" : "710x510", - "submitted_with": { - "app": "Unknown App", - "url": "" + { + "#url": "https://deviantart.com/shimoda7/avatar", + "#comment": "'formats' option", + "#category": ("", "deviantart", "avatar"), + "#class": deviantart.DeviantartAvatarExtractor, + "#archive": False, + "#options": {"formats": ["original.jpg", "big.jpg", "big.png", "big.gif"]}, + "#urls": ( + "https://a.deviantart.net/avatars-original/s/h/shimoda7.jpg?4", + "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4", + "https://a.deviantart.net/avatars-big/s/h/shimoda7.png?4", + "https://a.deviantart.net/avatars-big/s/h/shimoda7.gif?4", + ), + }, + { + "#url": "https://deviantart.com/h3813067/avatar", + "#comment": "default avatar (#5276)", + "#category": ("", "deviantart", "avatar"), + "#class": deviantart.DeviantartAvatarExtractor, + "#count": 0, + }, + { + "#url": "https://deviantart.com/gdldev/banner", + "#category": ("", "deviantart", "background"), + "#class": deviantart.DeviantartBackgroundExtractor, + "#pattern": r"https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$", + "#sha1_content": "980eaa76ce515f1b6bef60dfadf26a5bbe9c583f", + "allows_comments": True, + "author": { + "type": "regular", + "usericon": "https://a.deviantart.net/avatars/g/d/gdldev.jpg?12", + "userid": "1A12BA26-33C2-AA0A-7678-0B6DFBA7AC8E", + "username": "gdldev", + }, + "category_path": "", + "content": { + "filename": "banner_by_gdldev_dgntyqc.png", + "filesize": 84510, + "height": 4000, + "src": r"re:https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$", + "transparency": False, + "width": 6400, + }, + "date": "dt:2024-01-02 21:16:06", + "deviationid": "8C8D6B28-766A-DE21-7F7D-CE055C3BD50A", + "download_filesize": 84510, + "extension": "png", + "filename": "banner_by_gdldev-dgntyqc", + "index": 1007488020, + "index_base36": "gntyqc", + "is_blocked": False, + "is_deleted": False, + "is_downloadable": True, + "is_favourited": False, + "is_mature": False, + "is_original": True, + "is_published": False, + "preview": dict, + "printid": None, + "published_time": 1704230166, + "stats": { + "comments": 0, + "favourites": 0, }, + "target": dict, + "thumbs": list, + "title": "Banner", + "url": "https://www.deviantart.com/stash/0198jippkeys", + "username": "gdldev", }, - "tags": [], -}, - -{ - "#url" : "https://www.deviantart.com/zzz/art/zzz-1234567890", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.deviantart.com/justatest235723/art/archive-1103129101", - "#comment": "ZIP archive + preview image (#3782)", - "#class" : deviantart.DeviantartDeviationExtractor, - "#options": {"previews": True}, - "#pattern": [ - r"/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/di8rvv1-afe65948-16e1-4eca-b08d-9e6aaa9ed344\.zip", - r"/i/940f2d05-c5eb-4917-8192-7eb6a2d508c6/di8rvv1-bb9d891f-4374-4203-acd3-aea34b29a6a1\.png", - ], -}, - -{ - "#url" : "https://www.deviantart.com/myria-moon/art/Aime-Moi-261986576", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : {"comments": True}, - "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.jpg\?token=.+", - - "comments": "len:44", -}, - -{ - "#url" : "https://www.deviantart.com/justatest235723/art/Blue-811519058", - "#comment" : "nested comments (#4653)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : { - "original": False, - "comments": True, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous", + "#comment": "user", + "#category": ("", "deviantart", "folder"), + "#class": deviantart.DeviantartFolderExtractor, + "#options": {"original": False}, + "#count": 5, }, - - "comments": "len:20", -}, - -{ - "#url" : "https://www.deviantart.com/justatest235723/art/Blue-811519058", - "#comment" : "comment avatars (#4995)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : { - "original" : False, - "comments-avatars": True, - }, - "#range" : "5-", - "#pattern" : r"^https://www\.deviantart\.com/justatest235723/avatar/$", - "#count" : 16, -}, - -{ - "#url" : "https://www.deviantart.com/citizenfresh/art/Hverarond-789295466", - "#comment" : "wixmp URL rewrite /intermediary/", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#urls" : "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/4deb0f1a-cdef-444e-b194-c8d6b3f7e933/dd1xca2-7f835e62-6fd3-4b99-92c7-2bfd4e1b296f.jpg", - - "is_downloadable": False, - "is_original" : False, -}, - -{ - "#url" : "https://www.deviantart.com/skatergators/art/COM-Moni-781571783", - "#comment" : "GIF (#242)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#pattern" : r"https://wixmp-\w+\.wixmp\.com/f/03fd2413-efe9-4e5c-8734-2b72605b3fbb/dcxbsnb-1bbf0b38-42af-4070-8878-f30961955bec\.gif\?token=ey...", -}, - -{ - "#url" : "https://www.deviantart.com/yuumei/art/Flash-Comic-214724929", - "#comment" : "Flash animation with GIF preview (#1731)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#pattern" : r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.swf\?token=.+", - - "filename" : "flash_comic_tutorial_by_yuumei-d3juatd", - "extension": "swf", -}, - -{ - "#url" : "https://www.deviantart.com/justatest235723/art/video-1103119114", - "#comment" : "video", - "#class" : deviantart.DeviantartDeviationExtractor, - "#pattern" : r"/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/di8ro5m-e2a5bdf0-daee-4e18-bede-fbfc394d6c65\.mp4\?token=ey", - - "filename" : "video_63aebdd4bc0323da460796b9a2ac8522_by_justatest235723-di8ro5m", - "extension": "mp4", -}, - -{ - "#url" : "https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498", - "#comment" : "sta.sh URLs from description (#302)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : { - "extra" : 1, - "original": 0, - }, - "#pattern" : deviantart.DeviantartStashExtractor.pattern, - "#range" : "2-", - "#count" : 4, -}, - -{ - "#url" : "https://www.deviantart.com/cimar-wildehopps/art/Honorary-Vixen-859809305", - "#comment" : "sta.sh URL from deviation['text_content']['body']['features']", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : {"extra": 1}, - "#pattern" : r"""text: + { + "#url": "https://www.deviantart.com/yakuzafc/gallery/37412168/Crafts", + "#comment": "group", + "#category": ("", "deviantart", "folder"), + "#class": deviantart.DeviantartFolderExtractor, + "#options": {"original": False}, + "#count": ">= 4", + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/B38E3C6A-2029-6B45-757B-3C8D3422AD1A/misc", + "#comment": "uuid", + "#category": ("", "deviantart", "folder"), + "#class": deviantart.DeviantartFolderExtractor, + "#options": {"original": False}, + "#count": 5, + }, + { + "#url": "https://www.deviantart.com/justatest235723/gallery/69302698/-test-b-c-d-e-f-", + "#comment": "name starts with '_', special characters (#1451)", + "#category": ("", "deviantart", "folder"), + "#class": deviantart.DeviantartFolderExtractor, + "#options": {"original": False}, + "#count": 1, + }, + { + "#url": "https://shimoda7.deviantart.com/gallery/722019/Miscellaneous", + "#category": ("", "deviantart", "folder"), + "#class": deviantart.DeviantartFolderExtractor, + }, + { + "#url": "https://yakuzafc.deviantart.com/gallery/37412168/Crafts", + "#category": ("", "deviantart", "folder"), + "#class": deviantart.DeviantartFolderExtractor, + }, + { + "#url": "https://www.deviantart.com/stash/022c83odnaxc", + "#category": ("", "deviantart", "stash"), + "#class": deviantart.DeviantartStashExtractor, + "#pattern": r"https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/dcvdmbc-e506cdcf-3208-4c20-85ab-0bfa8a7bcb16.png\?token=ey.+", + "#count": 1, + "#sha1_content": "057eb2f2861f6c8a96876b13cca1a4b7a408c11f", + "content": { + "filename": "01_by_justatest235723_dcvdmbc.png", + "filesize": 380, + "width": 128, + "height": 128, + "src": r"re:https://wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/dcvdmbc-e506cdcf-3208-4c20-85ab-0bfa8a7bcb16.png\?token=ey.+", + }, + "date": "dt:2018-12-26 14:49:27", + "deviationid": "A4A6AD52-8857-46EE-ABFE-86D49D4FF9D0", + "download_filesize": 380, + "extension": "png", + "filename": "01_by_justatest235723-dcvdmbc", + "index": 778297656, + "index_base36": "cvdmbc", + "published_time": 1545835767, + "title": "01", + "url": "https://www.deviantart.com/stash/022c83odnaxc", + }, + { + "#url": "https://sta.sh/022c83odnaxc", + "#category": ("", "deviantart", "stash"), + "#class": deviantart.DeviantartStashExtractor, + }, + { + "#url": "https://sta.sh/21jf51j7pzl2", + "#comment": "multiple stash items", + "#category": ("", "deviantart", "stash"), + "#class": deviantart.DeviantartStashExtractor, + "#options": {"original": False}, + "#count": 4, + }, + { + "#url": "https://sta.sh/024t4coz16mi", + "#comment": "downloadable, but no 'content' field (#307)", + "#category": ("", "deviantart", "stash"), + "#class": deviantart.DeviantartStashExtractor, + "#pattern": r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.rar\?token=.+", + "#count": 1, + }, + { + "#url": "https://sta.sh/215twi387vfj", + "#comment": "mixed folders and images (#659)", + "#category": ("", "deviantart", "stash"), + "#class": deviantart.DeviantartStashExtractor, + "#options": {"original": False}, + "#count": 4, + }, + { + "#url": "https://sta.sh/abcdefghijkl", + "#category": ("", "deviantart", "stash"), + "#class": deviantart.DeviantartStashExtractor, + "#count": 0, + }, + { + "#url": "https://www.deviantart.com/h3813067/favourites/", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + "#options": { + "metadata": True, + "flat": False, + }, + "#count": 1, + }, + { + "#url": "https://www.deviantart.com/h3813067/favourites/", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", + }, + { + "#url": "https://www.deviantart.com/h3813067/favourites/all", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + }, + { + "#url": "https://www.deviantart.com/h3813067/favourites/?catpath=/", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + }, + { + "#url": "https://h3813067.deviantart.com/favourites/", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + }, + { + "#url": "https://h3813067.deviantart.com/favourites/all", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + }, + { + "#url": "https://h3813067.deviantart.com/favourites/?catpath=/", + "#category": ("", "deviantart", "favorite"), + "#class": deviantart.DeviantartFavoriteExtractor, + }, + { + "#url": "https://www.deviantart.com/pencilshadings/favourites/70595441/3D-Favorites", + "#category": ("", "deviantart", "collection"), + "#class": deviantart.DeviantartCollectionExtractor, + "#options": {"original": False}, + "#count": ">= 15", + }, + { + "#url": "https://www.deviantart.com/pencilshadings/favourites/F050486B-CB62-3C66-87FB-1105A7F6379F/3D Favorites", + "#category": ("", "deviantart", "collection"), + "#class": deviantart.DeviantartCollectionExtractor, + "#options": {"original": False}, + "#count": ">= 15", + }, + { + "#url": "https://pencilshadings.deviantart.com/favourites/70595441/3D-Favorites", + "#category": ("", "deviantart", "collection"), + "#class": deviantart.DeviantartCollectionExtractor, + }, + { + "#url": "https://www.deviantart.com/angrywhitewanker/posts/journals/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + "#sha1_url": "48aeed5631763d96f5391d2177ea72d9fdbee4e5", + }, + { + "#url": "https://www.deviantart.com/angrywhitewanker/posts/journals/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + "#options": {"journals": "text"}, + "#sha1_url": "b2a8e74d275664b1a4acee0fca0a6fd33298571e", + }, + { + "#url": "https://www.deviantart.com/angrywhitewanker/posts/journals/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + "#options": {"journals": "none"}, + "#count": 0, + }, + { + "#url": "https://www.deviantart.com/shimoda7/posts/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/journal/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/journal/?catpath=/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + }, + { + "#url": "https://shimoda7.deviantart.com/journal/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + }, + { + "#url": "https://shimoda7.deviantart.com/journal/?catpath=/", + "#category": ("", "deviantart", "journal"), + "#class": deviantart.DeviantartJournalExtractor, + }, + { + "#url": "https://www.deviantart.com/t1na/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class": deviantart.DeviantartStatusExtractor, + "#count": 0, + }, + { + "#url": "https://www.deviantart.com/justgalym/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class": deviantart.DeviantartStatusExtractor, + "#count": 4, + "#sha1_url": "62ee48ff3405c7714dca70abf42e8e39731012fc", + }, + { + "#url": "https://www.deviantart.com/justgalym/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class": deviantart.DeviantartStatusExtractor, + "#options": {"journals": "none"}, + "#pattern": r"https://images-wixmp-\w+\.wixmp\.com/intermediary/f/[^/]+/[^.]+\.jpg", + "#count": 1, + }, + { + "#url": "https://www.deviantart.com/vanillaghosties/posts/statuses", + "#comment": "shared sta.sh item", + "#category": ("", "deviantart", "status"), + "#class": deviantart.DeviantartStatusExtractor, + "#options": { + "journals": "none", + "original": False, + }, + "#range": "5-", + "#count": 1, + "index": int, + "index_base36": r"re:^[0-9a-z]+$", + "url": r"re:^https://www.deviantart.com/stash/\w+", + }, + { + "#url": "https://www.deviantart.com/AndrejSKalin/posts/statuses", + "#comment": "'deleted' deviations in 'items'", + "#category": ("", "deviantart", "status"), + "#class": deviantart.DeviantartStatusExtractor, + "#options": { + "journals": "none", + "original": 0, + "image-filter": "deviationid[:8] == '147C8B03'", + }, + "#count": 2, + "#archive": False, + "deviationid": "147C8B03-7D34-AE93-9241-FA3C6DBBC655", + }, + { + "#url": "https://www.deviantart.com/justgalym/posts/statuses", + "#category": ("", "deviantart", "status"), + "#class": deviantart.DeviantartStatusExtractor, + "#options": {"journals": "text"}, + "#sha1_url": "10a336bdee7b9692919461443a7dde44d495818c", + }, + { + "#url": "https://www.deviantart.com/tag/nature", + "#category": ("", "deviantart", "tag"), + "#class": deviantart.DeviantartTagExtractor, + "#options": {"original": False}, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://www.deviantart.com/watch/deviations", + "#category": ("", "deviantart", "watch"), + "#class": deviantart.DeviantartWatchExtractor, + }, + { + "#url": "https://www.deviantart.com/notifications/watch", + "#category": ("", "deviantart", "watch"), + "#class": deviantart.DeviantartWatchExtractor, + }, + { + "#url": "https://www.deviantart.com/watch/posts", + "#category": ("", "deviantart", "watch-posts"), + "#class": deviantart.DeviantartWatchPostsExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/art/For-the-sake-10073852", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": {"original": 0}, + "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", + }, + { + "#url": "https://www.deviantart.com/shimoda7/art/For-the-sake-10073852", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": {"metadata": "submission,camera,stats"}, + "can_post_comment": False, + "description": str, + "is_watching": False, + "license": "No License", + "stats": { + "comments": int, + "downloads": int, + "downloads_today": int, + "favourites": int, + "views": int, + "views_today": int, + }, + "submission": { + "category": "traditional/drawings/other", + "creation_time": "2004-08-25T02:44:08-0700", + "file_size": "133 KB", + "resolution": "710x510", + "submitted_with": {"app": "Unknown App", "url": ""}, + }, + "tags": [], + }, + { + "#url": "https://www.deviantart.com/zzz/art/zzz-1234567890", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.deviantart.com/justatest235723/art/archive-1103129101", + "#comment": "ZIP archive + preview image (#3782)", + "#class": deviantart.DeviantartDeviationExtractor, + "#options": {"previews": True}, + "#pattern": [ + r"/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/di8rvv1-afe65948-16e1-4eca-b08d-9e6aaa9ed344\.zip", + r"/i/940f2d05-c5eb-4917-8192-7eb6a2d508c6/di8rvv1-bb9d891f-4374-4203-acd3-aea34b29a6a1\.png", + ], + }, + { + "#url": "https://www.deviantart.com/myria-moon/art/Aime-Moi-261986576", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": {"comments": True}, + "#pattern": r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.jpg\?token=.+", + "comments": "len:44", + }, + { + "#url": "https://www.deviantart.com/justatest235723/art/Blue-811519058", + "#comment": "nested comments (#4653)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": { + "original": False, + "comments": True, + }, + "comments": "len:20", + }, + { + "#url": "https://www.deviantart.com/justatest235723/art/Blue-811519058", + "#comment": "comment avatars (#4995)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": { + "original": False, + "comments-avatars": True, + }, + "#range": "5-", + "#pattern": r"^https://www\.deviantart\.com/justatest235723/avatar/$", + "#count": 16, + }, + { + "#url": "https://www.deviantart.com/citizenfresh/art/Hverarond-789295466", + "#comment": "wixmp URL rewrite /intermediary/", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#urls": "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/4deb0f1a-cdef-444e-b194-c8d6b3f7e933/dd1xca2-7f835e62-6fd3-4b99-92c7-2bfd4e1b296f.jpg", + "is_downloadable": False, + "is_original": False, + }, + { + "#url": "https://www.deviantart.com/skatergators/art/COM-Moni-781571783", + "#comment": "GIF (#242)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#pattern": r"https://wixmp-\w+\.wixmp\.com/f/03fd2413-efe9-4e5c-8734-2b72605b3fbb/dcxbsnb-1bbf0b38-42af-4070-8878-f30961955bec\.gif\?token=ey...", + }, + { + "#url": "https://www.deviantart.com/yuumei/art/Flash-Comic-214724929", + "#comment": "Flash animation with GIF preview (#1731)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#pattern": r"https://wixmp-[^.]+\.wixmp\.com/f/.+/.+\.swf\?token=.+", + "filename": "flash_comic_tutorial_by_yuumei-d3juatd", + "extension": "swf", + }, + { + "#url": "https://www.deviantart.com/justatest235723/art/video-1103119114", + "#comment": "video", + "#class": deviantart.DeviantartDeviationExtractor, + "#pattern": r"/f/940f2d05-c5eb-4917-8192-7eb6a2d508c6/di8ro5m-e2a5bdf0-daee-4e18-bede-fbfc394d6c65\.mp4\?token=ey", + "filename": "video_63aebdd4bc0323da460796b9a2ac8522_by_justatest235723-di8ro5m", + "extension": "mp4", + }, + { + "#url": "https://www.deviantart.com/uotapo/art/INANAKI-Memo-590297498", + "#comment": "sta.sh URLs from description (#302)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": { + "extra": 1, + "original": 0, + }, + "#pattern": deviantart.DeviantartStashExtractor.pattern, + "#range": "2-", + "#count": 4, + }, + { + "#url": "https://www.deviantart.com/cimar-wildehopps/art/Honorary-Vixen-859809305", + "#comment": "sta.sh URL from deviation['text_content']['body']['features']", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": {"extra": 1}, + "#pattern": r"""text: |(?:https?://)?sta\.sh/([a-z0-9]+)""", - "#count" : 2, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/journal/ARTility-583755752", - "#comment" : "journal", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#pattern" : """text:\n""", - "#sha1_url": "37302947642d1e53392ef8ee9b3f473a3c578e7c", -}, - -{ - "#url" : "https://www.deviantart.com/gliitchlord/art/brashstrokes-812942668", - "#comment" : "journal-like post with isJournal == False (#419)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#pattern" : """text:\n""", - "#sha1_url": "8ca1dc8df53d3707c778d08a604f9ad9ddba7469", -}, - -{ - "#url" : "https://www.deviantart.com/stash/09z3557z648", - "#comment" : "sta.sh journal (#6207)", - "#class" : deviantart.DeviantartStashExtractor, - "#pattern" : """text:\n""", -}, - -{ - "#url" : "https://www.deviantart.com/starvinglunatic/art/Against-the-world-chapter-1-50968347", - "#comment" : "literature (#6254)", - "#class" : deviantart.DeviantartDeviationExtractor, - "#pattern" : """text:\n""", -}, - - -{ - "#url" : "https://www.deviantart.com/neotypical/art/985226590", - "#comment" : "subscription locked (#4567)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#count" : 0, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.deviantart.com/colibriworkshop/art/Crimson-Pandaren-Phoenix-World-of-Warcraft-630984457", - "#comment" : "'png' option (#4846)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#options" : {"quality": "png", "intermediary": False}, - "#sha1_content": "75fb92a820b154c061f7e1f9935260577b2365ec", - "#pattern" : r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com" - r"/f/d86d1faa-37a8-4bcb-b421-53331885d763/dafo6q1-5c4c999a-019e-4845-8c29-6fab2d05c8e8\.jpg" - r"/v1/fill/w_1024,h_1297,q_75,strp" - r"/crimson_pandaren_phoenix_world_of_warcraft_by_colibriworkshop_dafo6q1-fullview\.png" - r"\?token=ey.+", - - "extension": "png", -}, - -{ - "#url" : "https://deviantart.com/view/904858796/", - "#comment" : "/view/ URLs", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#sha1_content": "8770ec40ad1c1d60f6b602b16301d124f612948f", -}, - -{ - "#url" : "http://www.deviantart.com/view/890672057", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#sha1_content": "1497e13d925caeb13a250cd666b779a640209236", -}, - -{ - "#url" : "https://www.deviantart.com/view/706871727", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#sha1_content": "4d013515e72dec1e3977c82fd71ce4b15b8bd856", -}, - -{ - "#url" : "https://www.deviantart.com/view/1", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.deviantart.com/deviation/817215762", - "#comment" : "/deviation/ (#3558)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://fav.me/ddijrpu", - "#comment" : "fav.me (#3558)", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#count" : 1, -}, - -{ - "#url" : "https://fav.me/dddd", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/art/For-the-sake-of-a-memory-10073852", - "#comment" : "old-style URLs", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://myria-moon.deviantart.com/art/Aime-Moi-part-en-vadrouille-261986576", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://zzz.deviantart.com/art/zzz-1234567890", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/view.php?id=14864502", - "#comment" : "old /view/ URLs from the Wayback Machine", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "http://www.deviantart.com/view-full.php?id=100842", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://www.fxdeviantart.com/zzz/art/zzz-1234567890", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://www.fxdeviantart.com/view/1234567890", - "#category": ("", "deviantart", "deviation"), - "#class" : deviantart.DeviantartDeviationExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/scraps", - "#category": ("", "deviantart", "scraps"), - "#class" : deviantart.DeviantartScrapsExtractor, - "#count" : 12, -}, - -{ - "#url" : "https://www.deviantart.com/chain-man/gallery/scraps", - "#comment" : "deactivated account", - "#category": ("", "deviantart", "scraps"), - "#class" : deviantart.DeviantartScrapsExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery/?catpath=scraps", - "#category": ("", "deviantart", "scraps"), - "#class" : deviantart.DeviantartScrapsExtractor, -}, - -{ - "#url" : "https://shimoda7.deviantart.com/gallery/?catpath=scraps", - "#category": ("", "deviantart", "scraps"), - "#class" : deviantart.DeviantartScrapsExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/search?q=tree", - "#category": ("", "deviantart", "search"), - "#class" : deviantart.DeviantartSearchExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/search/deviations?order=popular-1-week", - "#category": ("", "deviantart", "search"), - "#class" : deviantart.DeviantartSearchExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery?q=memory", - "#category": ("", "deviantart", "gallery-search"), - "#class" : deviantart.DeviantartGallerySearchExtractor, - "#options" : {"original": 0}, - "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/gallery?q=memory&sort=popular", - "#category": ("", "deviantart", "gallery-search"), - "#class" : deviantart.DeviantartGallerySearchExtractor, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/about#watching", - "#category": ("", "deviantart", "following"), - "#class" : deviantart.DeviantartFollowingExtractor, - "#pattern" : deviantart.DeviantartUserExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://www.deviantart.com/shimoda7/watching", - "#category": ("", "deviantart", "following"), - "#class" : deviantart.DeviantartFollowingExtractor, -}, - + "#count": 2, + }, + { + "#url": "https://www.deviantart.com/shimoda7/journal/ARTility-583755752", + "#comment": "journal", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#pattern": """text:\n""", + "#sha1_url": "37302947642d1e53392ef8ee9b3f473a3c578e7c", + }, + { + "#url": "https://www.deviantart.com/gliitchlord/art/brashstrokes-812942668", + "#comment": "journal-like post with isJournal == False (#419)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#pattern": """text:\n""", + "#sha1_url": "8ca1dc8df53d3707c778d08a604f9ad9ddba7469", + }, + { + "#url": "https://www.deviantart.com/stash/09z3557z648", + "#comment": "sta.sh journal (#6207)", + "#class": deviantart.DeviantartStashExtractor, + "#pattern": """text:\n""", + }, + { + "#url": "https://www.deviantart.com/starvinglunatic/art/Against-the-world-chapter-1-50968347", + "#comment": "literature (#6254)", + "#class": deviantart.DeviantartDeviationExtractor, + "#pattern": """text:\n""", + }, + { + "#url": "https://www.deviantart.com/neotypical/art/985226590", + "#comment": "subscription locked (#4567)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#count": 0, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.deviantart.com/colibriworkshop/art/Crimson-Pandaren-Phoenix-World-of-Warcraft-630984457", + "#comment": "'png' option (#4846)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#options": {"quality": "png", "intermediary": False}, + "#sha1_content": "75fb92a820b154c061f7e1f9935260577b2365ec", + "#pattern": r"https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com" + r"/f/d86d1faa-37a8-4bcb-b421-53331885d763/dafo6q1-5c4c999a-019e-4845-8c29-6fab2d05c8e8\.jpg" + r"/v1/fill/w_1024,h_1297,q_75,strp" + r"/crimson_pandaren_phoenix_world_of_warcraft_by_colibriworkshop_dafo6q1-fullview\.png" + r"\?token=ey.+", + "extension": "png", + }, + { + "#url": "https://deviantart.com/view/904858796/", + "#comment": "/view/ URLs", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#sha1_content": "8770ec40ad1c1d60f6b602b16301d124f612948f", + }, + { + "#url": "http://www.deviantart.com/view/890672057", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#sha1_content": "1497e13d925caeb13a250cd666b779a640209236", + }, + { + "#url": "https://www.deviantart.com/view/706871727", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#sha1_content": "4d013515e72dec1e3977c82fd71ce4b15b8bd856", + }, + { + "#url": "https://www.deviantart.com/view/1", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.deviantart.com/deviation/817215762", + "#comment": "/deviation/ (#3558)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://fav.me/ddijrpu", + "#comment": "fav.me (#3558)", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#count": 1, + }, + { + "#url": "https://fav.me/dddd", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://shimoda7.deviantart.com/art/For-the-sake-of-a-memory-10073852", + "#comment": "old-style URLs", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://myria-moon.deviantart.com/art/Aime-Moi-part-en-vadrouille-261986576", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://zzz.deviantart.com/art/zzz-1234567890", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://www.deviantart.com/view.php?id=14864502", + "#comment": "old /view/ URLs from the Wayback Machine", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "http://www.deviantart.com/view-full.php?id=100842", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://www.fxdeviantart.com/zzz/art/zzz-1234567890", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://www.fxdeviantart.com/view/1234567890", + "#category": ("", "deviantart", "deviation"), + "#class": deviantart.DeviantartDeviationExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/scraps", + "#category": ("", "deviantart", "scraps"), + "#class": deviantart.DeviantartScrapsExtractor, + "#count": 12, + }, + { + "#url": "https://www.deviantart.com/chain-man/gallery/scraps", + "#comment": "deactivated account", + "#category": ("", "deviantart", "scraps"), + "#class": deviantart.DeviantartScrapsExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery/?catpath=scraps", + "#category": ("", "deviantart", "scraps"), + "#class": deviantart.DeviantartScrapsExtractor, + }, + { + "#url": "https://shimoda7.deviantart.com/gallery/?catpath=scraps", + "#category": ("", "deviantart", "scraps"), + "#class": deviantart.DeviantartScrapsExtractor, + }, + { + "#url": "https://www.deviantart.com/search?q=tree", + "#category": ("", "deviantart", "search"), + "#class": deviantart.DeviantartSearchExtractor, + }, + { + "#url": "https://www.deviantart.com/search/deviations?order=popular-1-week", + "#category": ("", "deviantart", "search"), + "#class": deviantart.DeviantartSearchExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery?q=memory", + "#category": ("", "deviantart", "gallery-search"), + "#class": deviantart.DeviantartGallerySearchExtractor, + "#options": {"original": 0}, + "#sha1_content": "6a7c74dc823ebbd457bdd9b3c2838a6ee728091e", + }, + { + "#url": "https://www.deviantart.com/shimoda7/gallery?q=memory&sort=popular", + "#category": ("", "deviantart", "gallery-search"), + "#class": deviantart.DeviantartGallerySearchExtractor, + }, + { + "#url": "https://www.deviantart.com/shimoda7/about#watching", + "#category": ("", "deviantart", "following"), + "#class": deviantart.DeviantartFollowingExtractor, + "#pattern": deviantart.DeviantartUserExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://www.deviantart.com/shimoda7/watching", + "#category": ("", "deviantart", "following"), + "#class": deviantart.DeviantartFollowingExtractor, + }, ) diff --git a/test/results/directlink.py b/test/results/directlink.py index 3309cef9a2..9995a3f0ad 100644 --- a/test/results/directlink.py +++ b/test/results/directlink.py @@ -1,174 +1,164 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import directlink - __tests__ = ( -{ - "#url" : "https://en.wikipedia.org/static/images/project-logos/enwiki.png", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, - "#sha1_url" : "18c5d00077332e98e53be9fed2ee4be66154b88d", - "#sha1_metadata": "105770a3f4393618ab7b811b731b22663b5d3794", - "#sha1_content" : "e6f58aaec8f31eb222f9e10fa9e9f64b79ae888c", -}, - -{ - "#url" : "https://example.org/file.webm", - "#comment" : "empty path", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, - "#sha1_url" : "2d807ed7059d1b532f1bb71dc24b510b80ff943f", - "#sha1_metadata": "29dad729c40fb09349f83edafa498dba1297464a", -}, - -{ - "#url" : "https://example.org/path/to/file.webm?que=1?&ry=2/#fragment", - "#comment" : "more complex example", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, - "#sha1_url" : "6fb1061390f8aada3db01cb24b51797c7ee42b31", - "#sha1_metadata": "3d7abc31d45ba324e59bc599c3b4862452d5f29c", -}, - -{ - "#url" : "https://example.org/%27%3C%23/%23%3E%27.jpg?key=%3C%26%3E", - "#comment" : "percent-encoded characters", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, - "#sha1_url" : "2627e8140727fdf743f86fe18f69f99a052c9718", - "#sha1_metadata": "831790fddda081bdddd14f96985ab02dc5b5341f", -}, - -{ - "#url" : "https://post-phinf.pstatic.net/MjAxOTA1MjlfMTQ4/MDAxNTU5MTI2NjcyNTkw.JUzkGb4V6dj9DXjLclrOoqR64uDxHFUO5KDriRdKpGwg.88mCtd4iT1NHlpVKSCaUpPmZPiDgT8hmQdQ5K_gYyu0g.JPEG/2.JPG", - "#comment" : "upper case file extension (#296)", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, - -{ - "#url" : "https://räksmörgås.josefsson.org/raksmorgas.jpg", - "#comment" : "internationalized domain name", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, - "#sha1_url" : "a65667f670b194afbd1e3ea5e7a78938d36747da", - "#sha1_metadata": "fd5037fe86eebd4764e176cbaf318caec0f700be", -}, - -{ - "#url" : "https://example.org/file.gif", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.bmp", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.svg", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.webp", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.avif", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.heic", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.psd", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.mp4", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.m4v", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.mov", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.mkv", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.ogg", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.ogm", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.ogv", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.wav", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.mp3", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.opus", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.zip", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.rar", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.7z", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.pdf", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, -{ - "#url" : "https://example.org/file.swf", - "#category": ("", "directlink", ""), - "#class" : directlink.DirectlinkExtractor, -}, - + { + "#url": "https://en.wikipedia.org/static/images/project-logos/enwiki.png", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + "#sha1_url": "18c5d00077332e98e53be9fed2ee4be66154b88d", + "#sha1_metadata": "105770a3f4393618ab7b811b731b22663b5d3794", + "#sha1_content": "e6f58aaec8f31eb222f9e10fa9e9f64b79ae888c", + }, + { + "#url": "https://example.org/file.webm", + "#comment": "empty path", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + "#sha1_url": "2d807ed7059d1b532f1bb71dc24b510b80ff943f", + "#sha1_metadata": "29dad729c40fb09349f83edafa498dba1297464a", + }, + { + "#url": "https://example.org/path/to/file.webm?que=1?&ry=2/#fragment", + "#comment": "more complex example", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + "#sha1_url": "6fb1061390f8aada3db01cb24b51797c7ee42b31", + "#sha1_metadata": "3d7abc31d45ba324e59bc599c3b4862452d5f29c", + }, + { + "#url": "https://example.org/%27%3C%23/%23%3E%27.jpg?key=%3C%26%3E", + "#comment": "percent-encoded characters", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + "#sha1_url": "2627e8140727fdf743f86fe18f69f99a052c9718", + "#sha1_metadata": "831790fddda081bdddd14f96985ab02dc5b5341f", + }, + { + "#url": "https://post-phinf.pstatic.net/MjAxOTA1MjlfMTQ4/MDAxNTU5MTI2NjcyNTkw.JUzkGb4V6dj9DXjLclrOoqR64uDxHFUO5KDriRdKpGwg.88mCtd4iT1NHlpVKSCaUpPmZPiDgT8hmQdQ5K_gYyu0g.JPEG/2.JPG", + "#comment": "upper case file extension (#296)", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://räksmörgås.josefsson.org/raksmorgas.jpg", + "#comment": "internationalized domain name", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + "#sha1_url": "a65667f670b194afbd1e3ea5e7a78938d36747da", + "#sha1_metadata": "fd5037fe86eebd4764e176cbaf318caec0f700be", + }, + { + "#url": "https://example.org/file.gif", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.bmp", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.svg", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.webp", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.avif", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.heic", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.psd", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.mp4", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.m4v", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.mov", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.mkv", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.ogg", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.ogm", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.ogv", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.wav", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.mp3", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.opus", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.zip", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.rar", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.7z", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.pdf", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, + { + "#url": "https://example.org/file.swf", + "#category": ("", "directlink", ""), + "#class": directlink.DirectlinkExtractor, + }, ) diff --git a/test/results/drawfriends.py b/test/results/drawfriends.py index d481fe70f9..aa91eed1b7 100644 --- a/test/results/drawfriends.py +++ b/test/results/drawfriends.py @@ -1,29 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v01 - __tests__ = ( -{ - "#url" : "https://drawfriends.booru.org/index.php?page=post&s=list&tags=all", - "#category": ("gelbooru_v01", "drawfriends", "tag"), - "#class" : gelbooru_v01.GelbooruV01TagExtractor, -}, - -{ - "#url" : "https://drawfriends.booru.org/index.php?page=favorites&s=view&id=1", - "#category": ("gelbooru_v01", "drawfriends", "favorite"), - "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, -}, - -{ - "#url" : "https://drawfriends.booru.org/index.php?page=post&s=view&id=107474", - "#category": ("gelbooru_v01", "drawfriends", "post"), - "#class" : gelbooru_v01.GelbooruV01PostExtractor, -}, - + { + "#url": "https://drawfriends.booru.org/index.php?page=post&s=list&tags=all", + "#category": ("gelbooru_v01", "drawfriends", "tag"), + "#class": gelbooru_v01.GelbooruV01TagExtractor, + }, + { + "#url": "https://drawfriends.booru.org/index.php?page=favorites&s=view&id=1", + "#category": ("gelbooru_v01", "drawfriends", "favorite"), + "#class": gelbooru_v01.GelbooruV01FavoriteExtractor, + }, + { + "#url": "https://drawfriends.booru.org/index.php?page=post&s=view&id=107474", + "#category": ("gelbooru_v01", "drawfriends", "post"), + "#class": gelbooru_v01.GelbooruV01PostExtractor, + }, ) diff --git a/test/results/dynastyscans.py b/test/results/dynastyscans.py index 0d4088005a..6179a8414d 100644 --- a/test/results/dynastyscans.py +++ b/test/results/dynastyscans.py @@ -1,59 +1,50 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import dynastyscans - __tests__ = ( -{ - "#url" : "http://dynasty-scans.com/chapters/hitoribocchi_no_oo_seikatsu_ch33", - "#category": ("", "dynastyscans", "chapter"), - "#class" : dynastyscans.DynastyscansChapterExtractor, - "#sha1_url" : "3cafa527fecec27a66f35e038c0c53e35d5e4317", - "#sha1_metadata": "7b134f2093813d45774cc68a3cd199ffce3e6fd3", -}, - -{ - "#url" : "http://dynasty-scans.com/chapters/new_game_the_spinoff_special_13", - "#category": ("", "dynastyscans", "chapter"), - "#class" : dynastyscans.DynastyscansChapterExtractor, - "#sha1_url" : "047fa6d58f90272883157a80fbf1e6f03ea5bbab", - "#sha1_metadata": "62dc42e9025c79bdd3e26e026a690f4c28548fd4", -}, - -{ - "#url" : "https://dynasty-scans.com/series/hitoribocchi_no_oo_seikatsu", - "#category": ("", "dynastyscans", "manga"), - "#class" : dynastyscans.DynastyscansMangaExtractor, - "#pattern" : dynastyscans.DynastyscansChapterExtractor.pattern, - "#count" : ">= 100", -}, - -{ - "#url" : "https://dynasty-scans.com/images?with[]=4930&with[]=5211", - "#category": ("", "dynastyscans", "search"), - "#class" : dynastyscans.DynastyscansSearchExtractor, - "#sha1_url" : "d2422163db7b1db94bf343f8cd50ba9cc08ae6b5", - "#sha1_metadata": "65f9948e7f29a1db2b3e6abb697f7476d2196708", -}, - -{ - "#url" : "https://dynasty-scans.com/images", - "#category": ("", "dynastyscans", "search"), - "#class" : dynastyscans.DynastyscansSearchExtractor, - "#range" : "1", - "#count" : 1, -}, - -{ - "#url" : "https://dynasty-scans.com/images/1245", - "#category": ("", "dynastyscans", "image"), - "#class" : dynastyscans.DynastyscansImageExtractor, - "#sha1_url" : "877054defac8ea2bbaeb632db176037668c73eea", - "#sha1_metadata": "9f6fd139c372203dcf7237e662a80963ab070cb0", -}, - + { + "#url": "http://dynasty-scans.com/chapters/hitoribocchi_no_oo_seikatsu_ch33", + "#category": ("", "dynastyscans", "chapter"), + "#class": dynastyscans.DynastyscansChapterExtractor, + "#sha1_url": "3cafa527fecec27a66f35e038c0c53e35d5e4317", + "#sha1_metadata": "7b134f2093813d45774cc68a3cd199ffce3e6fd3", + }, + { + "#url": "http://dynasty-scans.com/chapters/new_game_the_spinoff_special_13", + "#category": ("", "dynastyscans", "chapter"), + "#class": dynastyscans.DynastyscansChapterExtractor, + "#sha1_url": "047fa6d58f90272883157a80fbf1e6f03ea5bbab", + "#sha1_metadata": "62dc42e9025c79bdd3e26e026a690f4c28548fd4", + }, + { + "#url": "https://dynasty-scans.com/series/hitoribocchi_no_oo_seikatsu", + "#category": ("", "dynastyscans", "manga"), + "#class": dynastyscans.DynastyscansMangaExtractor, + "#pattern": dynastyscans.DynastyscansChapterExtractor.pattern, + "#count": ">= 100", + }, + { + "#url": "https://dynasty-scans.com/images?with[]=4930&with[]=5211", + "#category": ("", "dynastyscans", "search"), + "#class": dynastyscans.DynastyscansSearchExtractor, + "#sha1_url": "d2422163db7b1db94bf343f8cd50ba9cc08ae6b5", + "#sha1_metadata": "65f9948e7f29a1db2b3e6abb697f7476d2196708", + }, + { + "#url": "https://dynasty-scans.com/images", + "#category": ("", "dynastyscans", "search"), + "#class": dynastyscans.DynastyscansSearchExtractor, + "#range": "1", + "#count": 1, + }, + { + "#url": "https://dynasty-scans.com/images/1245", + "#category": ("", "dynastyscans", "image"), + "#class": dynastyscans.DynastyscansImageExtractor, + "#sha1_url": "877054defac8ea2bbaeb632db176037668c73eea", + "#sha1_metadata": "9f6fd139c372203dcf7237e662a80963ab070cb0", + }, ) diff --git a/test/results/e621.py b/test/results/e621.py index 5cd5c74daa..040f8d7cc0 100644 --- a/test/results/e621.py +++ b/test/results/e621.py @@ -1,134 +1,117 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import e621 - __tests__ = ( -{ - "#url" : "https://e621.net/posts?tags=anry", - "#category": ("E621", "e621", "tag"), - "#class" : e621.E621TagExtractor, - "#sha1_url" : "8021e5ea28d47c474c1ffc9bd44863c4d45700ba", - "#sha1_content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", -}, - -{ - "#url" : "https://e621.net/post/index/1/anry", - "#category": ("E621", "e621", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e621.net/post?tags=anry", - "#category": ("E621", "e621", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e621.net/pools/73", - "#category": ("E621", "e621", "pool"), - "#class" : e621.E621PoolExtractor, - "#sha1_url" : "1bd09a72715286a79eea3b7f09f51b3493eb579a", - "#sha1_content": "91abe5d5334425d9787811d7f06d34c77974cd22", -}, - -{ - "#url" : "https://e621.net/pool/show/73", - "#category": ("E621", "e621", "pool"), - "#class" : e621.E621PoolExtractor, -}, - -{ - "#url" : "https://e621.net/posts/535", - "#category": ("E621", "e621", "post"), - "#class" : e621.E621PostExtractor, - "#sha1_url" : "f7f78b44c9b88f8f09caac080adc8d6d9fdaa529", - "#sha1_content": "66f46e96a893fba8e694c4e049b23c2acc9af462", - - "date": "dt:2007-02-17 19:02:32", -}, - -{ - "#url" : "https://e621.net/posts/3181052", - "#category": ("E621", "e621", "post"), - "#class" : e621.E621PostExtractor, - "#options" : {"metadata": "notes,pools"}, - "#pattern" : r"https://static\d\.e621\.net/data/c6/8c/c68cca0643890b615f75fb2719589bff\.png", - - "notes": [ - { - "body" : "Little Legends 2", - "created_at" : "2022-05-16T13:58:38.877-04:00", - "creator_id" : 517450, - "creator_name": "EeveeCuddler69", - "height" : 475, - "id" : 321296, - "is_active" : True, - "post_id" : 3181052, - "updated_at" : "2022-05-16T13:59:02.050-04:00", - "version" : 3, - "width" : 809, - "x" : 83, - "y" : 117, - }, - ], - "pools": [ - { - "category" : "series", - "created_at" : "2022-02-17T00:29:22.669-05:00", - "creator_id" : 1077440, - "creator_name": "Yeetus90", - "description" : """\ + { + "#url": "https://e621.net/posts?tags=anry", + "#category": ("E621", "e621", "tag"), + "#class": e621.E621TagExtractor, + "#sha1_url": "8021e5ea28d47c474c1ffc9bd44863c4d45700ba", + "#sha1_content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", + }, + { + "#url": "https://e621.net/post/index/1/anry", + "#category": ("E621", "e621", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e621.net/post?tags=anry", + "#category": ("E621", "e621", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e621.net/pools/73", + "#category": ("E621", "e621", "pool"), + "#class": e621.E621PoolExtractor, + "#sha1_url": "1bd09a72715286a79eea3b7f09f51b3493eb579a", + "#sha1_content": "91abe5d5334425d9787811d7f06d34c77974cd22", + }, + { + "#url": "https://e621.net/pool/show/73", + "#category": ("E621", "e621", "pool"), + "#class": e621.E621PoolExtractor, + }, + { + "#url": "https://e621.net/posts/535", + "#category": ("E621", "e621", "post"), + "#class": e621.E621PostExtractor, + "#sha1_url": "f7f78b44c9b88f8f09caac080adc8d6d9fdaa529", + "#sha1_content": "66f46e96a893fba8e694c4e049b23c2acc9af462", + "date": "dt:2007-02-17 19:02:32", + }, + { + "#url": "https://e621.net/posts/3181052", + "#category": ("E621", "e621", "post"), + "#class": e621.E621PostExtractor, + "#options": {"metadata": "notes,pools"}, + "#pattern": r"https://static\d\.e621\.net/data/c6/8c/c68cca0643890b615f75fb2719589bff\.png", + "notes": [ + { + "body": "Little Legends 2", + "created_at": "2022-05-16T13:58:38.877-04:00", + "creator_id": 517450, + "creator_name": "EeveeCuddler69", + "height": 475, + "id": 321296, + "is_active": True, + "post_id": 3181052, + "updated_at": "2022-05-16T13:59:02.050-04:00", + "version": 3, + "width": 809, + "x": 83, + "y": 117, + }, + ], + "pools": [ + { + "category": "series", + "created_at": "2022-02-17T00:29:22.669-05:00", + "creator_id": 1077440, + "creator_name": "Yeetus90", + "description": """\ * "Little Legends":/pools/27971\r * Little Legends 2\r * "Little Legends 3":/pools/27481\ """, - "id" : 27492, - "is_active" : False, - "name" : "Little Legends 2", - "post_count" : 39, - "post_ids" : list, - "updated_at" : "2022-03-27T06:30:03.382-04:00", - }, - ], -}, - -{ - "#url" : "https://e621.net/post/show/535", - "#category": ("E621", "e621", "post"), - "#class" : e621.E621PostExtractor, -}, - -{ - "#url" : "https://e621.net/explore/posts/popular", - "#category": ("E621", "e621", "popular"), - "#class" : e621.E621PopularExtractor, -}, - -{ - "#url" : "https://e621.net/explore/posts/popular?date=2019-06-01&scale=month", - "#category": ("E621", "e621", "popular"), - "#class" : e621.E621PopularExtractor, - "#pattern" : r"https://static\d.e621.net/data/../../[0-9a-f]+", - "#count" : ">= 70", -}, - -{ - "#url" : "https://e621.net/favorites", - "#category": ("E621", "e621", "favorite"), - "#class" : e621.E621FavoriteExtractor, -}, - -{ - "#url" : "https://e621.net/favorites?page=2&user_id=53275", - "#category": ("E621", "e621", "favorite"), - "#class" : e621.E621FavoriteExtractor, - "#pattern" : r"https://static\d.e621.net/data/../../[0-9a-f]+", - "#count" : "> 260", -}, - + "id": 27492, + "is_active": False, + "name": "Little Legends 2", + "post_count": 39, + "post_ids": list, + "updated_at": "2022-03-27T06:30:03.382-04:00", + }, + ], + }, + { + "#url": "https://e621.net/post/show/535", + "#category": ("E621", "e621", "post"), + "#class": e621.E621PostExtractor, + }, + { + "#url": "https://e621.net/explore/posts/popular", + "#category": ("E621", "e621", "popular"), + "#class": e621.E621PopularExtractor, + }, + { + "#url": "https://e621.net/explore/posts/popular?date=2019-06-01&scale=month", + "#category": ("E621", "e621", "popular"), + "#class": e621.E621PopularExtractor, + "#pattern": r"https://static\d.e621.net/data/../../[0-9a-f]+", + "#count": ">= 70", + }, + { + "#url": "https://e621.net/favorites", + "#category": ("E621", "e621", "favorite"), + "#class": e621.E621FavoriteExtractor, + }, + { + "#url": "https://e621.net/favorites?page=2&user_id=53275", + "#category": ("E621", "e621", "favorite"), + "#class": e621.E621FavoriteExtractor, + "#pattern": r"https://static\d.e621.net/data/../../[0-9a-f]+", + "#count": "> 260", + }, ) diff --git a/test/results/e6ai.py b/test/results/e6ai.py index 291f3b3a25..6e1122c5cf 100644 --- a/test/results/e6ai.py +++ b/test/results/e6ai.py @@ -1,68 +1,56 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import e621 - __tests__ = ( -{ - "#url" : "https://e6ai.net/posts?tags=anry", - "#category": ("E621", "e6ai", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e6ai.net/post/index/1/anry", - "#category": ("E621", "e6ai", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e6ai.net/post?tags=anry", - "#category": ("E621", "e6ai", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e6ai.net/pools/3", - "#category": ("E621", "e6ai", "pool"), - "#class" : e621.E621PoolExtractor, - "#sha1_url": "a6d1ad67a3fa9b9f73731d34d5f6f26f7e85855f", -}, - -{ - "#url" : "https://e6ai.net/pool/show/3", - "#category": ("E621", "e6ai", "pool"), - "#class" : e621.E621PoolExtractor, -}, - -{ - "#url" : "https://e6ai.net/posts/23", - "#category": ("E621", "e6ai", "post"), - "#class" : e621.E621PostExtractor, - "#sha1_url" : "3c85a806b3d9eec861948af421fe0e8ad6b8f881", - "#sha1_content": "a05a484e4eb64637d56d751c02e659b4bc8ea5d5", -}, - -{ - "#url" : "https://e6ai.net/post/show/23", - "#category": ("E621", "e6ai", "post"), - "#class" : e621.E621PostExtractor, -}, - -{ - "#url" : "https://e6ai.net/explore/posts/popular", - "#category": ("E621", "e6ai", "popular"), - "#class" : e621.E621PopularExtractor, -}, - -{ - "#url" : "https://e6ai.net/favorites", - "#category": ("E621", "e6ai", "favorite"), - "#class" : e621.E621FavoriteExtractor, -}, - + { + "#url": "https://e6ai.net/posts?tags=anry", + "#category": ("E621", "e6ai", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e6ai.net/post/index/1/anry", + "#category": ("E621", "e6ai", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e6ai.net/post?tags=anry", + "#category": ("E621", "e6ai", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e6ai.net/pools/3", + "#category": ("E621", "e6ai", "pool"), + "#class": e621.E621PoolExtractor, + "#sha1_url": "a6d1ad67a3fa9b9f73731d34d5f6f26f7e85855f", + }, + { + "#url": "https://e6ai.net/pool/show/3", + "#category": ("E621", "e6ai", "pool"), + "#class": e621.E621PoolExtractor, + }, + { + "#url": "https://e6ai.net/posts/23", + "#category": ("E621", "e6ai", "post"), + "#class": e621.E621PostExtractor, + "#sha1_url": "3c85a806b3d9eec861948af421fe0e8ad6b8f881", + "#sha1_content": "a05a484e4eb64637d56d751c02e659b4bc8ea5d5", + }, + { + "#url": "https://e6ai.net/post/show/23", + "#category": ("E621", "e6ai", "post"), + "#class": e621.E621PostExtractor, + }, + { + "#url": "https://e6ai.net/explore/posts/popular", + "#category": ("E621", "e6ai", "popular"), + "#class": e621.E621PopularExtractor, + }, + { + "#url": "https://e6ai.net/favorites", + "#category": ("E621", "e6ai", "favorite"), + "#class": e621.E621FavoriteExtractor, + }, ) diff --git a/test/results/e926.py b/test/results/e926.py index b046b459b5..c2cd32f022 100644 --- a/test/results/e926.py +++ b/test/results/e926.py @@ -1,87 +1,73 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import e621 - __tests__ = ( -{ - "#url" : "https://e926.net/posts?tags=anry", - "#category": ("E621", "e926", "tag"), - "#class" : e621.E621TagExtractor, - "#sha1_url" : "12198b275c62ffe2de67cca676c8e64de80c425d", - "#sha1_content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", -}, - -{ - "#url" : "https://e926.net/post/index/1/anry", - "#category": ("E621", "e926", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e926.net/post?tags=anry", - "#category": ("E621", "e926", "tag"), - "#class" : e621.E621TagExtractor, -}, - -{ - "#url" : "https://e926.net/pools/73", - "#category": ("E621", "e926", "pool"), - "#class" : e621.E621PoolExtractor, - "#sha1_url" : "6936f1b6a18c5c25bee7cad700088dbc2503481b", - "#sha1_content": "91abe5d5334425d9787811d7f06d34c77974cd22", -}, - -{ - "#url" : "https://e926.net/pool/show/73", - "#category": ("E621", "e926", "pool"), - "#class" : e621.E621PoolExtractor, -}, - -{ - "#url" : "https://e926.net/posts/535", - "#category": ("E621", "e926", "post"), - "#class" : e621.E621PostExtractor, - "#sha1_url" : "17aec8ebd8fab098d321adcb62a2db59dab1f4bf", - "#sha1_content": "66f46e96a893fba8e694c4e049b23c2acc9af462", -}, - -{ - "#url" : "https://e926.net/post/show/535", - "#category": ("E621", "e926", "post"), - "#class" : e621.E621PostExtractor, -}, - -{ - "#url" : "https://e926.net/explore/posts/popular", - "#category": ("E621", "e926", "popular"), - "#class" : e621.E621PopularExtractor, -}, - -{ - "#url" : "https://e926.net/explore/posts/popular?date=2019-06-01&scale=month", - "#category": ("E621", "e926", "popular"), - "#class" : e621.E621PopularExtractor, - "#pattern" : r"https://static\d.e926.net/data/../../[0-9a-f]+", - "#count" : ">= 70", -}, - -{ - "#url" : "https://e926.net/favorites", - "#category": ("E621", "e926", "favorite"), - "#class" : e621.E621FavoriteExtractor, -}, - -{ - "#url" : "https://e926.net/favorites?page=2&user_id=53275", - "#category": ("E621", "e926", "favorite"), - "#class" : e621.E621FavoriteExtractor, - "#pattern" : r"https://static\d.e926.net/data/../../[0-9a-f]+", - "#count" : "> 260", -}, - + { + "#url": "https://e926.net/posts?tags=anry", + "#category": ("E621", "e926", "tag"), + "#class": e621.E621TagExtractor, + "#sha1_url": "12198b275c62ffe2de67cca676c8e64de80c425d", + "#sha1_content": "501d1e5d922da20ee8ff9806f5ed3ce3a684fd58", + }, + { + "#url": "https://e926.net/post/index/1/anry", + "#category": ("E621", "e926", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e926.net/post?tags=anry", + "#category": ("E621", "e926", "tag"), + "#class": e621.E621TagExtractor, + }, + { + "#url": "https://e926.net/pools/73", + "#category": ("E621", "e926", "pool"), + "#class": e621.E621PoolExtractor, + "#sha1_url": "6936f1b6a18c5c25bee7cad700088dbc2503481b", + "#sha1_content": "91abe5d5334425d9787811d7f06d34c77974cd22", + }, + { + "#url": "https://e926.net/pool/show/73", + "#category": ("E621", "e926", "pool"), + "#class": e621.E621PoolExtractor, + }, + { + "#url": "https://e926.net/posts/535", + "#category": ("E621", "e926", "post"), + "#class": e621.E621PostExtractor, + "#sha1_url": "17aec8ebd8fab098d321adcb62a2db59dab1f4bf", + "#sha1_content": "66f46e96a893fba8e694c4e049b23c2acc9af462", + }, + { + "#url": "https://e926.net/post/show/535", + "#category": ("E621", "e926", "post"), + "#class": e621.E621PostExtractor, + }, + { + "#url": "https://e926.net/explore/posts/popular", + "#category": ("E621", "e926", "popular"), + "#class": e621.E621PopularExtractor, + }, + { + "#url": "https://e926.net/explore/posts/popular?date=2019-06-01&scale=month", + "#category": ("E621", "e926", "popular"), + "#class": e621.E621PopularExtractor, + "#pattern": r"https://static\d.e926.net/data/../../[0-9a-f]+", + "#count": ">= 70", + }, + { + "#url": "https://e926.net/favorites", + "#category": ("E621", "e926", "favorite"), + "#class": e621.E621FavoriteExtractor, + }, + { + "#url": "https://e926.net/favorites?page=2&user_id=53275", + "#category": ("E621", "e926", "favorite"), + "#class": e621.E621FavoriteExtractor, + "#pattern": r"https://static\d.e926.net/data/../../[0-9a-f]+", + "#count": "> 260", + }, ) diff --git a/test/results/endchan.py b/test/results/endchan.py index 97d34c3bcd..de17d442ef 100644 --- a/test/results/endchan.py +++ b/test/results/endchan.py @@ -1,38 +1,31 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import lynxchan - __tests__ = ( -{ - "#url" : "https://endchan.org/yuri/res/33621.html", - "#category": ("lynxchan", "endchan", "thread"), - "#class" : lynxchan.LynxchanThreadExtractor, - "#urls" : "https://endchan.org/.media/358c089df4be990e9f7b636e1ce83d3e-imagejpeg.jpg", -}, - -{ - "#url" : "https://endchan.org/yuri/res/33621.html", - "#category": ("lynxchan", "endchan", "thread"), - "#class" : lynxchan.LynxchanThreadExtractor, -}, - -{ - "#url" : "https://endchan.org/yuri/", - "#category": ("lynxchan", "endchan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, - "#pattern" : lynxchan.LynxchanThreadExtractor.pattern, - "#count" : ">= 8", -}, - -{ - "#url" : "https://endchan.org/yuri/catalog.html", - "#category": ("lynxchan", "endchan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, -}, - + { + "#url": "https://endchan.org/yuri/res/33621.html", + "#category": ("lynxchan", "endchan", "thread"), + "#class": lynxchan.LynxchanThreadExtractor, + "#urls": "https://endchan.org/.media/358c089df4be990e9f7b636e1ce83d3e-imagejpeg.jpg", + }, + { + "#url": "https://endchan.org/yuri/res/33621.html", + "#category": ("lynxchan", "endchan", "thread"), + "#class": lynxchan.LynxchanThreadExtractor, + }, + { + "#url": "https://endchan.org/yuri/", + "#category": ("lynxchan", "endchan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + "#pattern": lynxchan.LynxchanThreadExtractor.pattern, + "#count": ">= 8", + }, + { + "#url": "https://endchan.org/yuri/catalog.html", + "#category": ("lynxchan", "endchan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + }, ) diff --git a/test/results/erome.py b/test/results/erome.py index 3624fe1774..9a411897d0 100644 --- a/test/results/erome.py +++ b/test/results/erome.py @@ -1,57 +1,48 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import erome - __tests__ = ( -{ - "#url" : "https://www.erome.com/a/NQgdlWvk", - "#category": ("", "erome", "album"), - "#class" : erome.EromeAlbumExtractor, - "#pattern" : r"https://v\d+\.erome\.com/\d+/NQgdlWvk/j7jlzmYB_480p\.mp4", - "#count" : 1, - - "album_id": "NQgdlWvk", - "date" : None, - "count" : 1, - "num" : 1, - "title" : "porn", - "user" : "yYgWBZw8o8qsMzM", -}, - -{ - "#url" : "https://www.erome.com/a/TdbZ4ogi", - "#category": ("", "erome", "album"), - "#class" : erome.EromeAlbumExtractor, - "#pattern" : r"https://s\d+\.erome\.com/\d+/TdbZ4ogi/\w+", - "#count" : 6, - - "album_id": "TdbZ4ogi", - "date" : "dt:2024-03-18 00:01:56", - "count" : 6, - "num" : int, - "title" : "82e78cfbb461ad87198f927fcb1fda9a1efac9ff.", - "user" : "yYgWBZw8o8qsMzM", -}, - -{ - "#url" : "https://www.erome.com/yYgWBZw8o8qsMzM", - "#category": ("", "erome", "user"), - "#class" : erome.EromeUserExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://www.erome.com/search?q=cute", - "#category": ("", "erome", "search"), - "#class" : erome.EromeSearchExtractor, - "#range" : "1-25", - "#count" : 25, -}, - + { + "#url": "https://www.erome.com/a/NQgdlWvk", + "#category": ("", "erome", "album"), + "#class": erome.EromeAlbumExtractor, + "#pattern": r"https://v\d+\.erome\.com/\d+/NQgdlWvk/j7jlzmYB_480p\.mp4", + "#count": 1, + "album_id": "NQgdlWvk", + "date": None, + "count": 1, + "num": 1, + "title": "porn", + "user": "yYgWBZw8o8qsMzM", + }, + { + "#url": "https://www.erome.com/a/TdbZ4ogi", + "#category": ("", "erome", "album"), + "#class": erome.EromeAlbumExtractor, + "#pattern": r"https://s\d+\.erome\.com/\d+/TdbZ4ogi/\w+", + "#count": 6, + "album_id": "TdbZ4ogi", + "date": "dt:2024-03-18 00:01:56", + "count": 6, + "num": int, + "title": "82e78cfbb461ad87198f927fcb1fda9a1efac9ff.", + "user": "yYgWBZw8o8qsMzM", + }, + { + "#url": "https://www.erome.com/yYgWBZw8o8qsMzM", + "#category": ("", "erome", "user"), + "#class": erome.EromeUserExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://www.erome.com/search?q=cute", + "#category": ("", "erome", "search"), + "#class": erome.EromeSearchExtractor, + "#range": "1-25", + "#count": 25, + }, ) diff --git a/test/results/everia.py b/test/results/everia.py index af22be0006..8d3a6d689d 100644 --- a/test/results/everia.py +++ b/test/results/everia.py @@ -8,44 +8,38 @@ __tests__ = ( -{ - "#url" : "https://everia.club/2024/09/23/mikacho-조미카-joapictures-someday/", - "#class": everia.EveriaPostExtractor, - "#count": 32, - - "title" : "Mikacho 조미카, JOApictures ‘Someday’", - "post_category": "Korea", - "tags" : ["[JOApictures]", "Mikacho 조미카"] -}, - -{ - "#url" : "https://everia.club/tag/miku-tanaka-%e7%94%b0%e4%b8%ad%e7%be%8e%e4%b9%85/", - "#class" : everia.EveriaTagExtractor, - "#pattern": everia.EveriaPostExtractor.pattern, - "#count" : "> 50", -}, - -{ - "#url" : "https://everia.club/category/japan/", - "#class" : everia.EveriaCategoryExtractor, - "#pattern": everia.EveriaPostExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://everia.club/2023/10/05/", - "#class" : everia.EveriaDateExtractor, - "#pattern": everia.EveriaPostExtractor.pattern, - "#count" : 34, -}, - -{ - "#url" : "https://everia.club/?s=saika", - "#class" : everia.EveriaSearchExtractor, - "#pattern": everia.EveriaPostExtractor.pattern, - "#range" : "1-15", - "#count" : 15, -}, - + { + "#url": "https://everia.club/2024/09/23/mikacho-조미카-joapictures-someday/", + "#class": everia.EveriaPostExtractor, + "#count": 32, + "title": "Mikacho 조미카, JOApictures ‘Someday’", + "post_category": "Korea", + "tags": ["[JOApictures]", "Mikacho 조미카"], + }, + { + "#url": "https://everia.club/tag/miku-tanaka-%e7%94%b0%e4%b8%ad%e7%be%8e%e4%b9%85/", + "#class": everia.EveriaTagExtractor, + "#pattern": everia.EveriaPostExtractor.pattern, + "#count": "> 50", + }, + { + "#url": "https://everia.club/category/japan/", + "#class": everia.EveriaCategoryExtractor, + "#pattern": everia.EveriaPostExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://everia.club/2023/10/05/", + "#class": everia.EveriaDateExtractor, + "#pattern": everia.EveriaPostExtractor.pattern, + "#count": 34, + }, + { + "#url": "https://everia.club/?s=saika", + "#class": everia.EveriaSearchExtractor, + "#pattern": everia.EveriaPostExtractor.pattern, + "#range": "1-15", + "#count": 15, + }, ) diff --git a/test/results/exhentai.py b/test/results/exhentai.py index df5935738a..507bf6b70c 100644 --- a/test/results/exhentai.py +++ b/test/results/exhentai.py @@ -1,137 +1,120 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import exhentai from gallery_dl import exception - +from gallery_dl.extractor import exhentai __tests__ = ( -{ - "#url" : "https://exhentai.org/g/1200119/d55c44d3d0/", - "#category": ("", "exhentai", "gallery"), - "#class" : exhentai.ExhentaiGalleryExtractor, - "#options" : {"original": False, "tags": True}, - "#sha1_content": [ - "2c68cff8a7ca540a78c36fdbf5fbae0260484f87", - "e9891a4c017ed0bb734cd1efba5cd03f594d31ff", - ], - - "cost" : int, - "date" : "dt:2018-03-18 20:14:00", - "eh_category" : "Non-H", - "expunged" : False, - "favorites" : r"re:^[12]\d$", - "filecount" : "4", - "filesize" : 1488978, - "gid" : 1200119, - "height" : int, - "image_token" : r"re:[0-9a-f]{10}", - "lang" : "ja", - "language" : "Japanese", - "parent" : "", - "rating" : r"re:\d\.\d+", - "size" : int, - "tags" : [ - "parody:komi-san wa komyushou desu.", - "character:shouko komi", - "group:seventh lowlife", - "other:sample", - ], - "tags_parody" : ["komi-san wa komyushou desu."], - "tags_character": ["shouko komi"], - "tags_group" : ["seventh lowlife"], - "tags_other" : ["sample"], - "thumb" : "https://s.exhentai.org/t/ce/0a/ce0a5bcb583229a9b07c0f83bcb1630ab1350640-624622-736-1036-jpg_250.jpg", - "title" : "C93 [Seventh_Lowlife] Komi-san ha Tokidoki Daitan desu (Komi-san wa Komyushou desu) [Sample]", - "title_jpn" : "(C93) [Comiketjack (わ！)] 古見さんは、時々大胆です。 (古見さんは、コミュ症です。) [見本]", - "token" : "d55c44d3d0", - "torrentcount": "0", - "uploader" : "klorpa", - "width" : int, -}, - -{ - "#url" : "https://exhentai.org/g/960461/4f0e369d82/", - "#category": ("", "exhentai", "gallery"), - "#class" : exhentai.ExhentaiGalleryExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "http://exhentai.org/g/962698/7f02358e00/", - "#category": ("", "exhentai", "gallery"), - "#class" : exhentai.ExhentaiGalleryExtractor, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://exhentai.org/s/f68367b4c8/1200119-3", - "#category": ("", "exhentai", "gallery"), - "#class" : exhentai.ExhentaiGalleryExtractor, - "#options" : {"original": False}, - "#count" : 2, -}, - -{ - "#url" : "https://e-hentai.org/s/f68367b4c8/1200119-3", - "#category": ("", "exhentai", "gallery"), - "#class" : exhentai.ExhentaiGalleryExtractor, - "#options" : {"original": False}, - "#count" : 2, -}, - -{ - "#url" : "https://g.e-hentai.org/g/1200119/d55c44d3d0/", - "#category": ("", "exhentai", "gallery"), - "#class" : exhentai.ExhentaiGalleryExtractor, -}, - -{ - "#url" : "https://e-hentai.org/?f_search=touhou", - "#category": ("", "exhentai", "search"), - "#class" : exhentai.ExhentaiSearchExtractor, -}, - -{ - "#url" : "https://exhentai.org/?f_cats=767&f_search=touhou", - "#category": ("", "exhentai", "search"), - "#class" : exhentai.ExhentaiSearchExtractor, -}, - -{ - "#url" : "https://exhentai.org/tag/parody:touhou+project", - "#category": ("", "exhentai", "search"), - "#class" : exhentai.ExhentaiSearchExtractor, -}, - -{ - "#url" : "https://exhentai.org/?f_doujinshi=0&f_manga=0&f_artistcg=0&f_gamecg=0&f_western=0&f_non-h=1&f_imageset=0&f_cosplay=0&f_asianporn=0&f_misc=0&f_search=touhou&f_apply=Apply+Filter", - "#category": ("", "exhentai", "search"), - "#class" : exhentai.ExhentaiSearchExtractor, - "#pattern" : exhentai.ExhentaiGalleryExtractor.pattern, - "#auth" : True, - "#range" : "1-30", - "#count" : 30, - - "gallery_id" : int, - "gallery_token": r"re:^[0-9a-f]{10}$", -}, - -{ - "#url" : "https://e-hentai.org/favorites.php", - "#category": ("", "exhentai", "favorite"), - "#class" : exhentai.ExhentaiFavoriteExtractor, - "#auth" : True, - "#urls" : "https://e-hentai.org/g/1200119/d55c44d3d0/", -}, - -{ - "#url" : "https://exhentai.org/favorites.php?favcat=1&f_search=touhou&f_apply=Search+Favorites", - "#category": ("", "exhentai", "favorite"), - "#class" : exhentai.ExhentaiFavoriteExtractor, -}, - + { + "#url": "https://exhentai.org/g/1200119/d55c44d3d0/", + "#category": ("", "exhentai", "gallery"), + "#class": exhentai.ExhentaiGalleryExtractor, + "#options": {"original": False, "tags": True}, + "#sha1_content": [ + "2c68cff8a7ca540a78c36fdbf5fbae0260484f87", + "e9891a4c017ed0bb734cd1efba5cd03f594d31ff", + ], + "cost": int, + "date": "dt:2018-03-18 20:14:00", + "eh_category": "Non-H", + "expunged": False, + "favorites": r"re:^[12]\d$", + "filecount": "4", + "filesize": 1488978, + "gid": 1200119, + "height": int, + "image_token": r"re:[0-9a-f]{10}", + "lang": "ja", + "language": "Japanese", + "parent": "", + "rating": r"re:\d\.\d+", + "size": int, + "tags": [ + "parody:komi-san wa komyushou desu.", + "character:shouko komi", + "group:seventh lowlife", + "other:sample", + ], + "tags_parody": ["komi-san wa komyushou desu."], + "tags_character": ["shouko komi"], + "tags_group": ["seventh lowlife"], + "tags_other": ["sample"], + "thumb": "https://s.exhentai.org/t/ce/0a/ce0a5bcb583229a9b07c0f83bcb1630ab1350640-624622-736-1036-jpg_250.jpg", + "title": "C93 [Seventh_Lowlife] Komi-san ha Tokidoki Daitan desu (Komi-san wa Komyushou desu) [Sample]", + "title_jpn": "(C93) [Comiketjack (わ！)] 古見さんは、時々大胆です。 (古見さんは、コミュ症です。) [見本]", + "token": "d55c44d3d0", + "torrentcount": "0", + "uploader": "klorpa", + "width": int, + }, + { + "#url": "https://exhentai.org/g/960461/4f0e369d82/", + "#category": ("", "exhentai", "gallery"), + "#class": exhentai.ExhentaiGalleryExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "http://exhentai.org/g/962698/7f02358e00/", + "#category": ("", "exhentai", "gallery"), + "#class": exhentai.ExhentaiGalleryExtractor, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://exhentai.org/s/f68367b4c8/1200119-3", + "#category": ("", "exhentai", "gallery"), + "#class": exhentai.ExhentaiGalleryExtractor, + "#options": {"original": False}, + "#count": 2, + }, + { + "#url": "https://e-hentai.org/s/f68367b4c8/1200119-3", + "#category": ("", "exhentai", "gallery"), + "#class": exhentai.ExhentaiGalleryExtractor, + "#options": {"original": False}, + "#count": 2, + }, + { + "#url": "https://g.e-hentai.org/g/1200119/d55c44d3d0/", + "#category": ("", "exhentai", "gallery"), + "#class": exhentai.ExhentaiGalleryExtractor, + }, + { + "#url": "https://e-hentai.org/?f_search=touhou", + "#category": ("", "exhentai", "search"), + "#class": exhentai.ExhentaiSearchExtractor, + }, + { + "#url": "https://exhentai.org/?f_cats=767&f_search=touhou", + "#category": ("", "exhentai", "search"), + "#class": exhentai.ExhentaiSearchExtractor, + }, + { + "#url": "https://exhentai.org/tag/parody:touhou+project", + "#category": ("", "exhentai", "search"), + "#class": exhentai.ExhentaiSearchExtractor, + }, + { + "#url": "https://exhentai.org/?f_doujinshi=0&f_manga=0&f_artistcg=0&f_gamecg=0&f_western=0&f_non-h=1&f_imageset=0&f_cosplay=0&f_asianporn=0&f_misc=0&f_search=touhou&f_apply=Apply+Filter", + "#category": ("", "exhentai", "search"), + "#class": exhentai.ExhentaiSearchExtractor, + "#pattern": exhentai.ExhentaiGalleryExtractor.pattern, + "#auth": True, + "#range": "1-30", + "#count": 30, + "gallery_id": int, + "gallery_token": r"re:^[0-9a-f]{10}$", + }, + { + "#url": "https://e-hentai.org/favorites.php", + "#category": ("", "exhentai", "favorite"), + "#class": exhentai.ExhentaiFavoriteExtractor, + "#auth": True, + "#urls": "https://e-hentai.org/g/1200119/d55c44d3d0/", + }, + { + "#url": "https://exhentai.org/favorites.php?favcat=1&f_search=touhou&f_apply=Search+Favorites", + "#category": ("", "exhentai", "favorite"), + "#class": exhentai.ExhentaiFavoriteExtractor, + }, ) diff --git a/test/results/fanbox.py b/test/results/fanbox.py index f11747f236..5cf8b551c0 100644 --- a/test/results/fanbox.py +++ b/test/results/fanbox.py @@ -1,183 +1,158 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import fanbox - __tests__ = ( -{ - "#url" : "https://xub.fanbox.cc", - "#category": ("", "fanbox", "creator"), - "#class" : fanbox.FanboxCreatorExtractor, - "#range" : "1-15", - "#count" : ">= 15", - - "creatorId": "xub", - "tags" : list, - "title" : str, -}, - -{ - "#url" : "https://xub.fanbox.cc/posts", - "#category": ("", "fanbox", "creator"), - "#class" : fanbox.FanboxCreatorExtractor, -}, - -{ - "#url" : "https://www.fanbox.cc/@xub/", - "#category": ("", "fanbox", "creator"), - "#class" : fanbox.FanboxCreatorExtractor, -}, - -{ - "#url" : "https://www.fanbox.cc/@xub/posts", - "#category": ("", "fanbox", "creator"), - "#class" : fanbox.FanboxCreatorExtractor, -}, - -{ - "#url" : "https://www.fanbox.cc/@xub/posts/1910054", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - "#count" : 3, - - "title" : "えま★おうがすと", - "tags" : list, - "hasAdultContent": True, - "isCoverImage" : False, -}, - -{ - "#url" : "https://nekoworks.fanbox.cc/posts/915", - "#comment" : "entry post type, image embedded in html of the post", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - "#count" : 2, - - "title" : "【SAYORI FAN CLUB】お届け内容", - "tags" : list, - "html" : str, - "hasAdultContent": True, -}, - -{ - "#url" : "https://steelwire.fanbox.cc/posts/285502", - "#comment" : "article post type, imageMap, 2 twitter embeds, fanbox embed", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - "#options" : {"embeds": True}, - "#count" : 8, - - "title" : "イラスト+SS｜【全体公開版】義足の探鉱夫男子が義足を見せてくれるだけ ", - "tags" : list, - "articleBody" : dict, - "hasAdultContent": True, -}, - -{ - "#url" : "https://www.fanbox.cc/@official-en/posts/4326303", - "#comment" : "'content' metadata (#3020)", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - - "content": r"re:(?s)^Greetings from FANBOX.\n \nAs of Monday, September 5th, 2022, we are happy to announce the start of the FANBOX hashtag event #MySetupTour ! \nAbout the event\nTo join this event .+ \nPlease check this page for further details regarding the Privacy & Terms.\nhttps://fanbox.pixiv.help/.+/10184952456601\n\n\nThank you for your continued support of FANBOX.$", -}, - -{ - "#url" : "https://official-en.fanbox.cc/posts/7022572", - "#comment" : "'plan' and 'user' metadata (#4921)", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - "#options" : {"metadata": True}, - - "plan": { - "coverImageUrl" : "", - "creatorId" : "official-en", - "description" : "", - "fee" : 0, - "hasAdultContent": None, - "id" : "", - "paymentMethod" : None, - "title" : "", + { + "#url": "https://xub.fanbox.cc", + "#category": ("", "fanbox", "creator"), + "#class": fanbox.FanboxCreatorExtractor, + "#range": "1-15", + "#count": ">= 15", + "creatorId": "xub", + "tags": list, + "title": str, + }, + { + "#url": "https://xub.fanbox.cc/posts", + "#category": ("", "fanbox", "creator"), + "#class": fanbox.FanboxCreatorExtractor, + }, + { + "#url": "https://www.fanbox.cc/@xub/", + "#category": ("", "fanbox", "creator"), + "#class": fanbox.FanboxCreatorExtractor, }, - "user": { - "coverImageUrl" : "https://pixiv.pximg.net/c/1620x580_90_a2_g5/fanbox/public/images/creator/74349833/cover/n9mX8q4tUXHXXj7sK1RPWyUu.jpeg", - "creatorId" : "official-en", - "description" : "This is the official English pixivFANBOX account! \n(official Japanese account: https://official.fanbox.cc/ )\n\npixivFANBOX is a subscription service for building a reliable fan community where creators can nurture creative lifestyles together with their fans.\nFollowers can be notified of the updates from their favorite creators they are following. Supporters can enjoy closer communication with creators through exclusive content and their latest information.\n", - "hasAdultContent" : False, - "hasBoothShop" : False, - "iconUrl" : "https://pixiv.pximg.net/c/160x160_90_a2_g5/fanbox/public/images/user/74349833/icon/oJH0OoGoSixLrJXlnneNvC95.jpeg", - "isAcceptingRequest": False, - "isFollowed" : False, - "isStopped" : False, - "isSupported" : False, - "name" : "pixivFANBOX English", - "profileItems" : [], - "profileLinks" : [ - "https://twitter.com/pixivfanbox", + { + "#url": "https://www.fanbox.cc/@xub/posts", + "#category": ("", "fanbox", "creator"), + "#class": fanbox.FanboxCreatorExtractor, + }, + { + "#url": "https://www.fanbox.cc/@xub/posts/1910054", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "#count": 3, + "title": "えま★おうがすと", + "tags": list, + "hasAdultContent": True, + "isCoverImage": False, + }, + { + "#url": "https://nekoworks.fanbox.cc/posts/915", + "#comment": "entry post type, image embedded in html of the post", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "#count": 2, + "title": "【SAYORI FAN CLUB】お届け内容", + "tags": list, + "html": str, + "hasAdultContent": True, + }, + { + "#url": "https://steelwire.fanbox.cc/posts/285502", + "#comment": "article post type, imageMap, 2 twitter embeds, fanbox embed", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "#options": {"embeds": True}, + "#count": 8, + "title": "イラスト+SS｜【全体公開版】義足の探鉱夫男子が義足を見せてくれるだけ ", + "tags": list, + "articleBody": dict, + "hasAdultContent": True, + }, + { + "#url": "https://www.fanbox.cc/@official-en/posts/4326303", + "#comment": "'content' metadata (#3020)", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "content": r"re:(?s)^Greetings from FANBOX.\n \nAs of Monday, September 5th, 2022, we are happy to announce the start of the FANBOX hashtag event #MySetupTour ! \nAbout the event\nTo join this event .+ \nPlease check this page for further details regarding the Privacy & Terms.\nhttps://fanbox.pixiv.help/.+/10184952456601\n\n\nThank you for your continued support of FANBOX.$", + }, + { + "#url": "https://official-en.fanbox.cc/posts/7022572", + "#comment": "'plan' and 'user' metadata (#4921)", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "#options": {"metadata": True}, + "plan": { + "coverImageUrl": "", + "creatorId": "official-en", + "description": "", + "fee": 0, + "hasAdultContent": None, + "id": "", + "paymentMethod": None, + "title": "", + }, + "user": { + "coverImageUrl": "https://pixiv.pximg.net/c/1620x580_90_a2_g5/fanbox/public/images/creator/74349833/cover/n9mX8q4tUXHXXj7sK1RPWyUu.jpeg", + "creatorId": "official-en", + "description": "This is the official English pixivFANBOX account! \n(official Japanese account: https://official.fanbox.cc/ )\n\npixivFANBOX is a subscription service for building a reliable fan community where creators can nurture creative lifestyles together with their fans.\nFollowers can be notified of the updates from their favorite creators they are following. Supporters can enjoy closer communication with creators through exclusive content and their latest information.\n", + "hasAdultContent": False, + "hasBoothShop": False, + "iconUrl": "https://pixiv.pximg.net/c/160x160_90_a2_g5/fanbox/public/images/user/74349833/icon/oJH0OoGoSixLrJXlnneNvC95.jpeg", + "isAcceptingRequest": False, + "isFollowed": False, + "isStopped": False, + "isSupported": False, + "name": "pixivFANBOX English", + "profileItems": [], + "profileLinks": [ + "https://twitter.com/pixivfanbox", + ], + "userId": "74349833", + }, + }, + { + "#url": "https://saki9184.fanbox.cc/posts/7754760", + "#comment": "missing plan for exact 'feeRequired' value (#5759)", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "#options": {"metadata": "plan"}, + "feeRequired": 300, + "plan": { + "creatorId": "saki9184", + "fee": 350, + "id": "414274", + "title": "入り浸りJKハルヒ", + }, + }, + { + "#url": "https://mochirong.fanbox.cc/posts/3746116", + "#comment": "imageMap file order (#2718); comments metadata (#6287)", + "#category": ("", "fanbox", "post"), + "#class": fanbox.FanboxPostExtractor, + "#options": {"metadata": "comments"}, + "#sha1_url": "c92ddd06f2efc4a5fe30ec67e21544f79a5c4062", + "#urls": [ + "https://pixiv.pximg.net/fanbox/public/images/post/3746116/cover/6h5w7F1MJWLeED6ODfHo6ZYQ.jpeg", + "https://downloads.fanbox.cc/images/post/3746116/ouTz7XZIeVD3FBOzoLhJ3ZTA.jpeg", + "https://downloads.fanbox.cc/images/post/3746116/hBs9bXEg6HvbqWT8QLD9g5ne.jpeg", + "https://downloads.fanbox.cc/images/post/3746116/C93E7db3C3sBqbDw6gQoZBMz.jpeg", ], - "userId" : "74349833", + "comments": "len:4", }, -}, - -{ - "#url" : "https://saki9184.fanbox.cc/posts/7754760", - "#comment" : "missing plan for exact 'feeRequired' value (#5759)", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - "#options" : {"metadata": "plan"}, - - "feeRequired": 300, - "plan" : { - "creatorId": "saki9184", - "fee" : 350, - "id" : "414274", - "title" : "入り浸りJKハルヒ", + { + "#url": "https://fanbox.cc/", + "#category": ("", "fanbox", "home"), + "#class": fanbox.FanboxHomeExtractor, + "#auth": True, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://fanbox.cc/home/supporting", + "#category": ("", "fanbox", "supporting"), + "#class": fanbox.FanboxSupportingExtractor, + "#auth": True, + "#count": 0, + }, + { + "#url": "https://www.pixiv.net/fanbox/creator/52336352", + "#category": ("", "fanbox", "redirect"), + "#class": fanbox.FanboxRedirectExtractor, + "#pattern": fanbox.FanboxCreatorExtractor.pattern, }, -}, - -{ - "#url" : "https://mochirong.fanbox.cc/posts/3746116", - "#comment" : "imageMap file order (#2718); comments metadata (#6287)", - "#category": ("", "fanbox", "post"), - "#class" : fanbox.FanboxPostExtractor, - "#options" : {"metadata": "comments"}, - "#sha1_url": "c92ddd06f2efc4a5fe30ec67e21544f79a5c4062", - "#urls" : [ - "https://pixiv.pximg.net/fanbox/public/images/post/3746116/cover/6h5w7F1MJWLeED6ODfHo6ZYQ.jpeg", - "https://downloads.fanbox.cc/images/post/3746116/ouTz7XZIeVD3FBOzoLhJ3ZTA.jpeg", - "https://downloads.fanbox.cc/images/post/3746116/hBs9bXEg6HvbqWT8QLD9g5ne.jpeg", - "https://downloads.fanbox.cc/images/post/3746116/C93E7db3C3sBqbDw6gQoZBMz.jpeg", - ], - - "comments": "len:4", -}, - -{ - "#url" : "https://fanbox.cc/", - "#category": ("", "fanbox", "home"), - "#class" : fanbox.FanboxHomeExtractor, - "#auth" : True, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://fanbox.cc/home/supporting", - "#category": ("", "fanbox", "supporting"), - "#class" : fanbox.FanboxSupportingExtractor, - "#auth" : True, - "#count" : 0, -}, - -{ - "#url" : "https://www.pixiv.net/fanbox/creator/52336352", - "#category": ("", "fanbox", "redirect"), - "#class" : fanbox.FanboxRedirectExtractor, - "#pattern" : fanbox.FanboxCreatorExtractor.pattern, -}, - ) diff --git a/test/results/fandom.py b/test/results/fandom.py index 2a314043dc..84331f9e02 100644 --- a/test/results/fandom.py +++ b/test/results/fandom.py @@ -1,117 +1,106 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.fandom.com/wiki/Title", - "#comment" : "for scripts/supportedsites.py", - "#category": ("wikimedia", "fandom-www", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://mushishi.fandom.com/wiki/Yahagi", - "#category": ("wikimedia", "fandom-mushishi", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#urls" : "https://static.wikia.nocookie.net/mushi-shi/images/f/f8/Yahagi.png/revision/latest?cb=20150128052255", - - "bitdepth" : 8, - "canonicaltitle": "File:Yahagi.png", - "comment" : "", - "commonmetadata": { - "ResolutionUnit": 3, - "XResolution" : "3779/100", - "YResolution" : "3779/100", + { + "#url": "https://www.fandom.com/wiki/Title", + "#comment": "for scripts/supportedsites.py", + "#category": ("wikimedia", "fandom-www", "article"), + "#class": wikimedia.WikimediaArticleExtractor, }, - "date" : "dt:2015-01-28 05:22:55", - "descriptionshorturl": "https://mushishi.fandom.com/index.php?curid=2595", - "descriptionurl": "https://mushishi.fandom.com/wiki/File:Yahagi.png", - "extension" : "png", - "extmetadata" : { - "DateTime": { - "hidden": "", - "source": "mediawiki-metadata", - "value": "2015-01-28T05:22:55Z", - }, - "ObjectName": { - "hidden": "", - "source": "mediawiki-metadata", - "value": "Yahagi", + { + "#url": "https://mushishi.fandom.com/wiki/Yahagi", + "#category": ("wikimedia", "fandom-mushishi", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#urls": "https://static.wikia.nocookie.net/mushi-shi/images/f/f8/Yahagi.png/revision/latest?cb=20150128052255", + "bitdepth": 8, + "canonicaltitle": "File:Yahagi.png", + "comment": "", + "commonmetadata": { + "ResolutionUnit": 3, + "XResolution": "3779/100", + "YResolution": "3779/100", }, - }, - "filename" : "Yahagi", - "height" : 410, - "metadata" : { - "bitDepth" : 8, - "colorType" : "truecolour", - "duration" : 0, - "frameCount": 0, - "loopCount" : 1, - "metadata" : [ - { - "name" : "XResolution", - "value": "3779/100", - }, - { - "name" : "YResolution", - "value": "3779/100", - }, - { - "name" : "ResolutionUnit", - "value": 3, + "date": "dt:2015-01-28 05:22:55", + "descriptionshorturl": "https://mushishi.fandom.com/index.php?curid=2595", + "descriptionurl": "https://mushishi.fandom.com/wiki/File:Yahagi.png", + "extension": "png", + "extmetadata": { + "DateTime": { + "hidden": "", + "source": "mediawiki-metadata", + "value": "2015-01-28T05:22:55Z", }, - { - "name" : "_MW_PNG_VERSION", - "value": 1, + "ObjectName": { + "hidden": "", + "source": "mediawiki-metadata", + "value": "Yahagi", }, - ], + }, + "filename": "Yahagi", + "height": 410, + "metadata": { + "bitDepth": 8, + "colorType": "truecolour", + "duration": 0, + "frameCount": 0, + "loopCount": 1, + "metadata": [ + { + "name": "XResolution", + "value": "3779/100", + }, + { + "name": "YResolution", + "value": "3779/100", + }, + { + "name": "ResolutionUnit", + "value": 3, + }, + { + "name": "_MW_PNG_VERSION", + "value": 1, + }, + ], + }, + "mime": "image/png", + "page": "Yahagi", + "sha1": "e3078a97976215323dbabb0c86b7acc55b512d16", + "size": 429912, + "timestamp": "2015-01-28T05:22:55Z", + "url": "https://static.wikia.nocookie.net/mushi-shi/images/f/f8/Yahagi.png/revision/latest?cb=20150128052255", + "user": "ITHYRIAL", + "userid": 4637089, + "width": 728, + }, + { + "#url": "https://hearthstone.fandom.com/wiki/Flame_Juggler", + "#comment": "empty 'metadata'", + "#category": ("wikimedia", "fandom-hearthstone", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "metadata": {}, + }, + { + "#url": "https://discogs.fandom.com/zh/wiki/File:CH-0430D2.jpg", + "#comment": "non-English language prefix (#6370)", + "#category": ("wikimedia", "fandom-discogs", "file"), + "#class": wikimedia.WikimediaArticleExtractor, + "#urls": "https://static.wikia.nocookie.net/discogs/images/a/ab/CH-0430D2.jpg/revision/latest?cb=20241007150151&path-prefix=zh", + }, + { + "#url": "https://projectsekai.fandom.com/wiki/Project_SEKAI_Wiki", + "#category": ("wikimedia", "fandom-projectsekai", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://youtube.fandom.com", + "#category": ("wikimedia", "fandom-youtube", "wiki"), + "#class": wikimedia.WikimediaWikiExtractor, + "#range": "1-20", + "#count": 20, }, - "mime" : "image/png", - "page" : "Yahagi", - "sha1" : "e3078a97976215323dbabb0c86b7acc55b512d16", - "size" : 429912, - "timestamp" : "2015-01-28T05:22:55Z", - "url" : "https://static.wikia.nocookie.net/mushi-shi/images/f/f8/Yahagi.png/revision/latest?cb=20150128052255", - "user" : "ITHYRIAL", - "userid" : 4637089, - "width" : 728, -}, - -{ - "#url" : "https://hearthstone.fandom.com/wiki/Flame_Juggler", - "#comment" : "empty 'metadata'", - "#category": ("wikimedia", "fandom-hearthstone", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - - "metadata" : {}, -}, - -{ - "#url" : "https://discogs.fandom.com/zh/wiki/File:CH-0430D2.jpg", - "#comment" : "non-English language prefix (#6370)", - "#category": ("wikimedia", "fandom-discogs", "file"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#urls" : "https://static.wikia.nocookie.net/discogs/images/a/ab/CH-0430D2.jpg/revision/latest?cb=20241007150151&path-prefix=zh", -}, - -{ - "#url" : "https://projectsekai.fandom.com/wiki/Project_SEKAI_Wiki", - "#category": ("wikimedia", "fandom-projectsekai", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://youtube.fandom.com", - "#category": ("wikimedia", "fandom-youtube", "wiki"), - "#class" : wikimedia.WikimediaWikiExtractor, - "#range" : "1-20", - "#count" : 20, -}, - ) diff --git a/test/results/fanleaks.py b/test/results/fanleaks.py index c420e0ca87..64fc09242f 100644 --- a/test/results/fanleaks.py +++ b/test/results/fanleaks.py @@ -1,67 +1,56 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import fanleaks from gallery_dl import exception - +from gallery_dl.extractor import fanleaks __tests__ = ( -{ - "#url" : "https://fanleaks.club/selti/880", - "#category": ("", "fanleaks", "post"), - "#class" : fanleaks.FanleaksPostExtractor, - "#pattern" : r"https://fanleaks\.club//models/selti/images/selti_0880\.jpg", - - "model_id": "selti", - "model" : "Selti", - "id" : 880, - "type" : "photo", -}, - -{ - "#url" : "https://fanleaks.club/daisy-keech/1038", - "#category": ("", "fanleaks", "post"), - "#class" : fanleaks.FanleaksPostExtractor, - "#pattern" : r"https://fanleaks\.club//models/daisy-keech/videos/daisy-keech_1038\.mp4", - - "model_id": "daisy-keech", - "model" : "Daisy Keech", - "id" : 1038, - "type" : "video", -}, - -{ - "#url" : "https://fanleaks.club/hannahowo/000", - "#category": ("", "fanleaks", "post"), - "#class" : fanleaks.FanleaksPostExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://fanleaks.club/hannahowo", - "#category": ("", "fanleaks", "model"), - "#class" : fanleaks.FanleaksModelExtractor, - "#pattern" : r"https://fanleaks\.club//models/hannahowo/(images|videos)/hannahowo_\d+\.\w+", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://fanleaks.club/belle-delphine", - "#category": ("", "fanleaks", "model"), - "#class" : fanleaks.FanleaksModelExtractor, - "#pattern" : r"https://fanleaks\.club//models/belle-delphine/(images|videos)/belle-delphine_\d+\.\w+", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://fanleaks.club/daisy-keech", - "#category": ("", "fanleaks", "model"), - "#class" : fanleaks.FanleaksModelExtractor, -}, - + { + "#url": "https://fanleaks.club/selti/880", + "#category": ("", "fanleaks", "post"), + "#class": fanleaks.FanleaksPostExtractor, + "#pattern": r"https://fanleaks\.club//models/selti/images/selti_0880\.jpg", + "model_id": "selti", + "model": "Selti", + "id": 880, + "type": "photo", + }, + { + "#url": "https://fanleaks.club/daisy-keech/1038", + "#category": ("", "fanleaks", "post"), + "#class": fanleaks.FanleaksPostExtractor, + "#pattern": r"https://fanleaks\.club//models/daisy-keech/videos/daisy-keech_1038\.mp4", + "model_id": "daisy-keech", + "model": "Daisy Keech", + "id": 1038, + "type": "video", + }, + { + "#url": "https://fanleaks.club/hannahowo/000", + "#category": ("", "fanleaks", "post"), + "#class": fanleaks.FanleaksPostExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://fanleaks.club/hannahowo", + "#category": ("", "fanleaks", "model"), + "#class": fanleaks.FanleaksModelExtractor, + "#pattern": r"https://fanleaks\.club//models/hannahowo/(images|videos)/hannahowo_\d+\.\w+", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://fanleaks.club/belle-delphine", + "#category": ("", "fanleaks", "model"), + "#class": fanleaks.FanleaksModelExtractor, + "#pattern": r"https://fanleaks\.club//models/belle-delphine/(images|videos)/belle-delphine_\d+\.\w+", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://fanleaks.club/daisy-keech", + "#category": ("", "fanleaks", "model"), + "#class": fanleaks.FanleaksModelExtractor, + }, ) diff --git a/test/results/fantia.py b/test/results/fantia.py index 5867e7861e..f3a816efc1 100644 --- a/test/results/fantia.py +++ b/test/results/fantia.py @@ -1,67 +1,58 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import fantia - __tests__ = ( -{ - "#url" : "https://fantia.jp/fanclubs/6939", - "#category": ("", "fantia", "creator"), - "#class" : fantia.FantiaCreatorExtractor, - "#range" : "1-25", - "#count" : ">= 25", - - "fanclub_user_id": 52152, - "tags" : list, - "post_title" : str, -}, - -{ - "#url" : "https://fantia.jp/posts/1166373", - "#category": ("", "fantia", "post"), - "#class" : fantia.FantiaPostExtractor, - "#pattern" : r"https://(c\.fantia\.jp/uploads/post/file/1166373/|cc\.fantia\.jp/uploads/post_content_photo/file/732549[01]|fantia\.jp/posts/1166373/album_image\?)", - - "blogpost_text" : r"re:^$|This is a test.\n\n(This is a test.)?\n\n|Link to video:\nhttps://www.youtube.com/watch\?v=5SSdvNcAagI\n\nhtml img from another site:\n\n\n\n\n\n", - "comment" : "\n\n", - "content_category": r"re:thumb|blog|photo_gallery", - "content_comment" : str, - "content_count" : 5, - "content_filename": r"re:|", - "content_num" : range(1, 5), - "content_title" : r"re:Test (Blog Content \d+|Image Gallery)|thumb", - "date" : "dt:2022-03-09 16:46:12", - "fanclub_id" : 356320, - "fanclub_name" : "Test Fantia", - "fanclub_url" : "https://fantia.jp/fanclubs/356320", - "fanclub_user_id" : 7487131, - "fanclub_user_name": "2022/03/08 15:13:52の名無し", - "file_url" : str, - "filename" : str, - "num" : int, - "plan" : dict, - "post_id" : 1166373, - "post_title" : "Test Fantia Post", - "post_url" : "https://fantia.jp/posts/1166373", - "posted_at" : "Thu, 10 Mar 2022 01:46:12 +0900", - "rating" : "general", - "tags" : [], -}, - -{ - "#url" : "https://fantia.jp/posts/508363", - "#category": ("", "fantia", "post"), - "#class" : fantia.FantiaPostExtractor, - "#count" : 6, - - "post_title": "zunda逆バニーでおしりｺｯｼｮﾘ", - "tags" : list, - "rating" : "adult", - "post_id" : 508363, -}, - + { + "#url": "https://fantia.jp/fanclubs/6939", + "#category": ("", "fantia", "creator"), + "#class": fantia.FantiaCreatorExtractor, + "#range": "1-25", + "#count": ">= 25", + "fanclub_user_id": 52152, + "tags": list, + "post_title": str, + }, + { + "#url": "https://fantia.jp/posts/1166373", + "#category": ("", "fantia", "post"), + "#class": fantia.FantiaPostExtractor, + "#pattern": r"https://(c\.fantia\.jp/uploads/post/file/1166373/|cc\.fantia\.jp/uploads/post_content_photo/file/732549[01]|fantia\.jp/posts/1166373/album_image\?)", + "blogpost_text": r"re:^$|This is a test.\n\n(This is a test.)?\n\n|Link to video:\nhttps://www.youtube.com/watch\?v=5SSdvNcAagI\n\nhtml img from another site:\n\n\n\n\n\n", + "comment": "\n\n", + "content_category": r"re:thumb|blog|photo_gallery", + "content_comment": str, + "content_count": 5, + "content_filename": r"re:|", + "content_num": range(1, 5), + "content_title": r"re:Test (Blog Content \d+|Image Gallery)|thumb", + "date": "dt:2022-03-09 16:46:12", + "fanclub_id": 356320, + "fanclub_name": "Test Fantia", + "fanclub_url": "https://fantia.jp/fanclubs/356320", + "fanclub_user_id": 7487131, + "fanclub_user_name": "2022/03/08 15:13:52の名無し", + "file_url": str, + "filename": str, + "num": int, + "plan": dict, + "post_id": 1166373, + "post_title": "Test Fantia Post", + "post_url": "https://fantia.jp/posts/1166373", + "posted_at": "Thu, 10 Mar 2022 01:46:12 +0900", + "rating": "general", + "tags": [], + }, + { + "#url": "https://fantia.jp/posts/508363", + "#category": ("", "fantia", "post"), + "#class": fantia.FantiaPostExtractor, + "#count": 6, + "post_title": "zunda逆バニーでおしりｺｯｼｮﾘ", + "tags": list, + "rating": "adult", + "post_id": 508363, + }, ) diff --git a/test/results/fapachi.py b/test/results/fapachi.py index 8907e7f8b4..6403625f1a 100644 --- a/test/results/fapachi.py +++ b/test/results/fapachi.py @@ -1,44 +1,36 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import fapachi - __tests__ = ( -{ - "#url" : "https://fapachi.com/sonson/media/0082", - "#comment" : "NSFW", - "#category": ("", "fapachi", "post"), - "#class" : fapachi.FapachiPostExtractor, - "#pattern" : r"https://fapachi\.com/models/s/o/sonson/1/full/sonson_0082\.jpeg", - - "user": "sonson", - "id" : "0082", -}, - -{ - "#url" : "https://fapachi.com/ferxiita/media/0159", - "#comment" : "NSFW", - "#category": ("", "fapachi", "post"), - "#class" : fapachi.FapachiPostExtractor, -}, - -{ - "#url" : "https://fapachi.com/sonson", - "#category": ("", "fapachi", "user"), - "#class" : fapachi.FapachiUserExtractor, - "#pattern" : fapachi.FapachiPostExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://fapachi.com/ferxiita/page/3", - "#category": ("", "fapachi", "user"), - "#class" : fapachi.FapachiUserExtractor, -}, - + { + "#url": "https://fapachi.com/sonson/media/0082", + "#comment": "NSFW", + "#category": ("", "fapachi", "post"), + "#class": fapachi.FapachiPostExtractor, + "#pattern": r"https://fapachi\.com/models/s/o/sonson/1/full/sonson_0082\.jpeg", + "user": "sonson", + "id": "0082", + }, + { + "#url": "https://fapachi.com/ferxiita/media/0159", + "#comment": "NSFW", + "#category": ("", "fapachi", "post"), + "#class": fapachi.FapachiPostExtractor, + }, + { + "#url": "https://fapachi.com/sonson", + "#category": ("", "fapachi", "user"), + "#class": fapachi.FapachiUserExtractor, + "#pattern": fapachi.FapachiPostExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://fapachi.com/ferxiita/page/3", + "#category": ("", "fapachi", "user"), + "#class": fapachi.FapachiUserExtractor, + }, ) diff --git a/test/results/fapello.py b/test/results/fapello.py index 6764b95989..68634c35d4 100644 --- a/test/results/fapello.py +++ b/test/results/fapello.py @@ -1,141 +1,119 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import fapello from gallery_dl import exception - +from gallery_dl.extractor import fapello __tests__ = ( -{ - "#url" : "https://fapello.com/carrykey/530/", - "#category": ("", "fapello", "post"), - "#class" : fapello.FapelloPostExtractor, - "#pattern" : r"https://fapello\.com/content/c/a/carrykey/1000/carrykey_0530\.jpg", - - "model" : "carrykey", - "id" : 530, - "type" : "photo", - "thumbnail": "", -}, - -{ - "#url" : "https://fapello.com/vladislava-661/693/", - "#category": ("", "fapello", "post"), - "#class" : fapello.FapelloPostExtractor, - "#pattern" : r"https://cdn\.fapello\.com/content/v/l/vladislava-661/1000/vladislava-661_0693\.mp4", - "#exception": exception.NotFoundError, - - "model" : "vladislava-661", - "id" : 693, - "type" : "video", - "thumbnail": "https://fapello.com/content/v/l/vladislava-661/1000/vladislava-661_0693.jpg", -}, - -{ - "#url" : "https://fapello.com/carrykey/000/", - "#category": ("", "fapello", "post"), - "#class" : fapello.FapelloPostExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://fapello.su/grace-charis-gracecharisxo/2038266/", - "#category": ("", "fapello", "post"), - "#class" : fapello.FapelloPostExtractor, - - "model" : "grace-charis-gracecharisxo", - "id" : 2038266, - "type" : "photo", -}, - -{ - "#url" : "https://fapello.com/hyoon/", - "#category": ("", "fapello", "model"), - "#class" : fapello.FapelloModelExtractor, - "#pattern" : fapello.FapelloPostExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://fapello.com/kobaebeefboo/", - "#category": ("", "fapello", "model"), - "#class" : fapello.FapelloModelExtractor, -}, - -{ - "#url" : "https://fapello.su/grace-charis-gracecharisxo/", - "#category": ("", "fapello", "model"), - "#class" : fapello.FapelloModelExtractor, - "#pattern" : fapello.FapelloPostExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://fapello.com/top-likes/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, - "#pattern" : fapello.FapelloModelExtractor.pattern, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://fapello.su/top-likes/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, - "#pattern" : fapello.FapelloModelExtractor.pattern, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://fapello.com/videos/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, - "#pattern" : fapello.FapelloPostExtractor.pattern, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://fapello.com/top-followers/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, -}, - -{ - "#url" : "https://fapello.su/top-followers/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, -}, - -{ - "#url" : "https://fapello.com/trending/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, -}, - -{ - "#url" : "https://fapello.su/trending/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, -}, - -{ - "#url" : "https://fapello.com/popular_videos/twelve_hours/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, -}, - -{ - "#url" : "https://fapello.com/popular_videos/week/", - "#category": ("", "fapello", "path"), - "#class" : fapello.FapelloPathExtractor, -}, - + { + "#url": "https://fapello.com/carrykey/530/", + "#category": ("", "fapello", "post"), + "#class": fapello.FapelloPostExtractor, + "#pattern": r"https://fapello\.com/content/c/a/carrykey/1000/carrykey_0530\.jpg", + "model": "carrykey", + "id": 530, + "type": "photo", + "thumbnail": "", + }, + { + "#url": "https://fapello.com/vladislava-661/693/", + "#category": ("", "fapello", "post"), + "#class": fapello.FapelloPostExtractor, + "#pattern": r"https://cdn\.fapello\.com/content/v/l/vladislava-661/1000/vladislava-661_0693\.mp4", + "#exception": exception.NotFoundError, + "model": "vladislava-661", + "id": 693, + "type": "video", + "thumbnail": "https://fapello.com/content/v/l/vladislava-661/1000/vladislava-661_0693.jpg", + }, + { + "#url": "https://fapello.com/carrykey/000/", + "#category": ("", "fapello", "post"), + "#class": fapello.FapelloPostExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://fapello.su/grace-charis-gracecharisxo/2038266/", + "#category": ("", "fapello", "post"), + "#class": fapello.FapelloPostExtractor, + "model": "grace-charis-gracecharisxo", + "id": 2038266, + "type": "photo", + }, + { + "#url": "https://fapello.com/hyoon/", + "#category": ("", "fapello", "model"), + "#class": fapello.FapelloModelExtractor, + "#pattern": fapello.FapelloPostExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://fapello.com/kobaebeefboo/", + "#category": ("", "fapello", "model"), + "#class": fapello.FapelloModelExtractor, + }, + { + "#url": "https://fapello.su/grace-charis-gracecharisxo/", + "#category": ("", "fapello", "model"), + "#class": fapello.FapelloModelExtractor, + "#pattern": fapello.FapelloPostExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://fapello.com/top-likes/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + "#pattern": fapello.FapelloModelExtractor.pattern, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://fapello.su/top-likes/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + "#pattern": fapello.FapelloModelExtractor.pattern, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://fapello.com/videos/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + "#pattern": fapello.FapelloPostExtractor.pattern, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://fapello.com/top-followers/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + }, + { + "#url": "https://fapello.su/top-followers/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + }, + { + "#url": "https://fapello.com/trending/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + }, + { + "#url": "https://fapello.su/trending/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + }, + { + "#url": "https://fapello.com/popular_videos/twelve_hours/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + }, + { + "#url": "https://fapello.com/popular_videos/week/", + "#category": ("", "fapello", "path"), + "#class": fapello.FapelloPathExtractor, + }, ) diff --git a/test/results/fappic.py b/test/results/fappic.py index b89c4e2996..dbab88b57b 100644 --- a/test/results/fappic.py +++ b/test/results/fappic.py @@ -1,26 +1,21 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://fappic.com/98wxqcklyh8k/test.png", - "#category": ("imagehost", "fappic", "image"), - "#class" : imagehosts.FappicImageExtractor, -}, - -{ - "#url" : "https://www.fappic.com/98wxqcklyh8k/test.png", - "#category": ("imagehost", "fappic", "image"), - "#class" : imagehosts.FappicImageExtractor, - "#pattern" : r"https://img\d+\.fappic\.com/img/\w+/test\.png", - "#sha1_metadata": "433b1d310b0ff12ad8a71ac7b9d8ba3f8cd1e898", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - + { + "#url": "https://fappic.com/98wxqcklyh8k/test.png", + "#category": ("imagehost", "fappic", "image"), + "#class": imagehosts.FappicImageExtractor, + }, + { + "#url": "https://www.fappic.com/98wxqcklyh8k/test.png", + "#category": ("imagehost", "fappic", "image"), + "#class": imagehosts.FappicImageExtractor, + "#pattern": r"https://img\d+\.fappic\.com/img/\w+/test\.png", + "#sha1_metadata": "433b1d310b0ff12ad8a71ac7b9d8ba3f8cd1e898", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, ) diff --git a/test/results/fashionnova.py b/test/results/fashionnova.py index 9cee0e23fe..ca59496f5d 100644 --- a/test/results/fashionnova.py +++ b/test/results/fashionnova.py @@ -1,45 +1,37 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://www.fashionnova.com/collections/mini-dresses", - "#category": ("shopify", "fashionnova", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.fashionnova.com/collections/mini-dresses/?page=1", - "#category": ("shopify", "fashionnova", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.fashionnova.com/collections/mini-dresses#1", - "#category": ("shopify", "fashionnova", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.fashionnova.com/products/all-my-life-legging-black", - "#category": ("shopify", "fashionnova", "product"), - "#class" : shopify.ShopifyProductExtractor, - "#pattern" : r"https?://cdn\d*\.shopify\.com/s/files/", - "#count" : 8, -}, - -{ - "#url" : "https://www.fashionnova.com/collections/flats/products/name", - "#category": ("shopify", "fashionnova", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://www.fashionnova.com/collections/mini-dresses", + "#category": ("shopify", "fashionnova", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.fashionnova.com/collections/mini-dresses/?page=1", + "#category": ("shopify", "fashionnova", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.fashionnova.com/collections/mini-dresses#1", + "#category": ("shopify", "fashionnova", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.fashionnova.com/products/all-my-life-legging-black", + "#category": ("shopify", "fashionnova", "product"), + "#class": shopify.ShopifyProductExtractor, + "#pattern": r"https?://cdn\d*\.shopify\.com/s/files/", + "#count": 8, + }, + { + "#url": "https://www.fashionnova.com/collections/flats/products/name", + "#category": ("shopify", "fashionnova", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/fireden.py b/test/results/fireden.py index 48549e18d5..e90334d79d 100644 --- a/test/results/fireden.py +++ b/test/results/fireden.py @@ -1,36 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://boards.fireden.net/sci/thread/11264294/", - "#category": ("foolfuuka", "fireden", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "61cab625c95584a12a30049d054931d64f8d20aa", -}, - -{ - "#url" : "https://boards.fireden.net/sci/", - "#category": ("foolfuuka", "fireden", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://boards.fireden.net/_/search/text/test/", - "#category": ("foolfuuka", "fireden", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://boards.fireden.net/sci/gallery/6", - "#category": ("foolfuuka", "fireden", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://boards.fireden.net/sci/thread/11264294/", + "#category": ("foolfuuka", "fireden", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "61cab625c95584a12a30049d054931d64f8d20aa", + }, + { + "#url": "https://boards.fireden.net/sci/", + "#category": ("foolfuuka", "fireden", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://boards.fireden.net/_/search/text/test/", + "#category": ("foolfuuka", "fireden", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://boards.fireden.net/sci/gallery/6", + "#category": ("foolfuuka", "fireden", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/flickr.py b/test/results/flickr.py index 5ae5805024..983a2f7991 100644 --- a/test/results/flickr.py +++ b/test/results/flickr.py @@ -4,168 +4,146 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import flickr from gallery_dl import exception - +from gallery_dl.extractor import flickr __tests__ = ( -{ - "#url" : "https://www.flickr.com/photos/departingyyz/16089302239", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, - "#options" : { - "contexts": True, - "exif": True, - }, - "#urls" : "https://live.staticflickr.com/7463/16089302239_de18cd8017_b_d.jpg", - "#pattern" : flickr.FlickrImageExtractor.pattern, - "#sha1_content": [ - "3133006c6d657fe54cf7d4c46b82abbcb0efaf9f", - "0821a28ee46386e85b02b67cf2720063440a228c", - ], - - "camera" : "Sony ILCE-7", - "comments" : int, - "description": str, - "exif" : list, - "extension" : "jpg", - "filename" : "16089302239_de18cd8017_b_d", - "id" : 16089302239, - "height" : 683, - "label" : "Large", - "media" : "photo", - "pool" : list, - "set" : list, - "url" : str, - "views" : int, - "width" : 1024, -}, - -{ - "#url" : "https://secure.flickr.com/photos/departingyyz/16089302239", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, -}, - -{ - "#url" : "https://m.flickr.com/photos/departingyyz/16089302239", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, -}, - -{ - "#url" : "https://flickr.com/photos/departingyyz/16089302239", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, -}, - -{ - "#url" : "https://www.flickr.com/photos/eliasroviello/52713899383/", - "#comment" : "video", - "#class" : flickr.FlickrImageExtractor, - "#pattern" : r"https://live.staticflickr\.com/video/52713899383/51dfffef79/1080p\.mp4\?s=ey.+", - - "media": "video", -}, - -{ - "#url" : "http://c2.staticflickr.com/2/1475/24531000464_9a7503ae68_b.jpg", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, -}, - -{ - "#url" : "https://farm2.static.flickr.com/1035/1188352415_cb139831d0.jpg", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, -}, - -{ - "#url" : "https://flic.kr/p/FPVo9U", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, -}, - -{ - "#url" : "https://www.flickr.com/photos/zzz/16089302238", - "#category": ("", "flickr", "image"), - "#class" : flickr.FlickrImageExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.flickr.com/photos/shona_s/albums/72157633471741607", - "#category": ("", "flickr", "album"), - "#class" : flickr.FlickrAlbumExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, - "#count" : 6, -}, - -{ - "#url" : "https://www.flickr.com/photos/shona_s/albums", - "#category": ("", "flickr", "album"), - "#class" : flickr.FlickrAlbumExtractor, - "#pattern" : flickr.FlickrAlbumExtractor.pattern, - "#count" : 2, -}, - -{ - "#url" : "https://secure.flickr.com/photos/shona_s/albums", - "#category": ("", "flickr", "album"), - "#class" : flickr.FlickrAlbumExtractor, -}, - -{ - "#url" : "https://m.flickr.com/photos/shona_s/albums", - "#category": ("", "flickr", "album"), - "#class" : flickr.FlickrAlbumExtractor, -}, - -{ - "#url" : "https://www.flickr.com/photos/flickr/galleries/72157681572514792/", - "#category": ("", "flickr", "gallery"), - "#class" : flickr.FlickrGalleryExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, - "#count" : ">= 10", -}, - -{ - "#url" : "https://www.flickr.com/groups/bird_headshots/", - "#category": ("", "flickr", "group"), - "#class" : flickr.FlickrGroupExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, - "#count" : "> 150", -}, - -{ - "#url" : "https://www.flickr.com/photos/shona_s/", - "#category": ("", "flickr", "user"), - "#class" : flickr.FlickrUserExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, - "#count" : 28, -}, - -{ - "#url" : "https://www.flickr.com/photos/shona_s/favorites", - "#category": ("", "flickr", "favorite"), - "#class" : flickr.FlickrFavoriteExtractor, - "#pattern" : flickr.FlickrImageExtractor.pattern, - "#count" : 4, -}, - -{ - "#url" : "https://flickr.com/search/?text=mountain", - "#category": ("", "flickr", "search"), - "#class" : flickr.FlickrSearchExtractor, -}, - -{ - "#url" : "https://flickr.com/search/?text=tree%20cloud%20house&color_codes=4&styles=minimalism", - "#category": ("", "flickr", "search"), - "#class" : flickr.FlickrSearchExtractor, -}, - + { + "#url": "https://www.flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + "#options": { + "contexts": True, + "exif": True, + }, + "#urls": "https://live.staticflickr.com/7463/16089302239_de18cd8017_b_d.jpg", + "#pattern": flickr.FlickrImageExtractor.pattern, + "#sha1_content": [ + "3133006c6d657fe54cf7d4c46b82abbcb0efaf9f", + "0821a28ee46386e85b02b67cf2720063440a228c", + ], + "camera": "Sony ILCE-7", + "comments": int, + "description": str, + "exif": list, + "extension": "jpg", + "filename": "16089302239_de18cd8017_b_d", + "id": 16089302239, + "height": 683, + "label": "Large", + "media": "photo", + "pool": list, + "set": list, + "url": str, + "views": int, + "width": 1024, + }, + { + "#url": "https://secure.flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + }, + { + "#url": "https://m.flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + }, + { + "#url": "https://flickr.com/photos/departingyyz/16089302239", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + }, + { + "#url": "https://www.flickr.com/photos/eliasroviello/52713899383/", + "#comment": "video", + "#class": flickr.FlickrImageExtractor, + "#pattern": r"https://live.staticflickr\.com/video/52713899383/51dfffef79/1080p\.mp4\?s=ey.+", + "media": "video", + }, + { + "#url": "http://c2.staticflickr.com/2/1475/24531000464_9a7503ae68_b.jpg", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + }, + { + "#url": "https://farm2.static.flickr.com/1035/1188352415_cb139831d0.jpg", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + }, + { + "#url": "https://flic.kr/p/FPVo9U", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + }, + { + "#url": "https://www.flickr.com/photos/zzz/16089302238", + "#category": ("", "flickr", "image"), + "#class": flickr.FlickrImageExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.flickr.com/photos/shona_s/albums/72157633471741607", + "#category": ("", "flickr", "album"), + "#class": flickr.FlickrAlbumExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + "#count": 6, + }, + { + "#url": "https://www.flickr.com/photos/shona_s/albums", + "#category": ("", "flickr", "album"), + "#class": flickr.FlickrAlbumExtractor, + "#pattern": flickr.FlickrAlbumExtractor.pattern, + "#count": 2, + }, + { + "#url": "https://secure.flickr.com/photos/shona_s/albums", + "#category": ("", "flickr", "album"), + "#class": flickr.FlickrAlbumExtractor, + }, + { + "#url": "https://m.flickr.com/photos/shona_s/albums", + "#category": ("", "flickr", "album"), + "#class": flickr.FlickrAlbumExtractor, + }, + { + "#url": "https://www.flickr.com/photos/flickr/galleries/72157681572514792/", + "#category": ("", "flickr", "gallery"), + "#class": flickr.FlickrGalleryExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + "#count": ">= 10", + }, + { + "#url": "https://www.flickr.com/groups/bird_headshots/", + "#category": ("", "flickr", "group"), + "#class": flickr.FlickrGroupExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + "#count": "> 150", + }, + { + "#url": "https://www.flickr.com/photos/shona_s/", + "#category": ("", "flickr", "user"), + "#class": flickr.FlickrUserExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + "#count": 28, + }, + { + "#url": "https://www.flickr.com/photos/shona_s/favorites", + "#category": ("", "flickr", "favorite"), + "#class": flickr.FlickrFavoriteExtractor, + "#pattern": flickr.FlickrImageExtractor.pattern, + "#count": 4, + }, + { + "#url": "https://flickr.com/search/?text=mountain", + "#category": ("", "flickr", "search"), + "#class": flickr.FlickrSearchExtractor, + }, + { + "#url": "https://flickr.com/search/?text=tree%20cloud%20house&color_codes=4&styles=minimalism", + "#category": ("", "flickr", "search"), + "#class": flickr.FlickrSearchExtractor, + }, ) diff --git a/test/results/foalcon.py b/test/results/foalcon.py index f2ee3a3af4..9d6fb48ebe 100644 --- a/test/results/foalcon.py +++ b/test/results/foalcon.py @@ -1,41 +1,34 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import szurubooru - __tests__ = ( -{ - "#url" : "https://booru.foalcon.com/posts/query=simple_background", - "#category": ("szurubooru", "foalcon", "tag"), - "#class" : szurubooru.SzurubooruTagExtractor, - "#pattern" : r"https://booru\.foalcon\.com/data/posts/\d+_[0-9a-f]{16}\.\w+", - "#range" : "1-150", - "#count" : 150, -}, - -{ - "#url" : "https://booru.foalcon.com/posts/query=", - "#category": ("szurubooru", "foalcon", "tag"), - "#class" : szurubooru.SzurubooruTagExtractor, -}, - -{ - "#url" : "https://booru.foalcon.com/posts", - "#category": ("szurubooru", "foalcon", "tag"), - "#class" : szurubooru.SzurubooruTagExtractor, -}, - -{ - "#url" : "https://booru.foalcon.com/post/30092", - "#category": ("szurubooru", "foalcon", "post"), - "#class" : szurubooru.SzurubooruPostExtractor, - "#pattern" : r"https://booru\.foalcon\.com/data/posts/30092_b7d56e941888b624\.png", - "#sha1_url" : "dad4d4c67d87cd9a4ac429b3414747c27a95d5cb", - "#sha1_content": "86d1514c0ca8197950cc4b74e7a59b2dc76ebf9c", -}, - + { + "#url": "https://booru.foalcon.com/posts/query=simple_background", + "#category": ("szurubooru", "foalcon", "tag"), + "#class": szurubooru.SzurubooruTagExtractor, + "#pattern": r"https://booru\.foalcon\.com/data/posts/\d+_[0-9a-f]{16}\.\w+", + "#range": "1-150", + "#count": 150, + }, + { + "#url": "https://booru.foalcon.com/posts/query=", + "#category": ("szurubooru", "foalcon", "tag"), + "#class": szurubooru.SzurubooruTagExtractor, + }, + { + "#url": "https://booru.foalcon.com/posts", + "#category": ("szurubooru", "foalcon", "tag"), + "#class": szurubooru.SzurubooruTagExtractor, + }, + { + "#url": "https://booru.foalcon.com/post/30092", + "#category": ("szurubooru", "foalcon", "post"), + "#class": szurubooru.SzurubooruPostExtractor, + "#pattern": r"https://booru\.foalcon\.com/data/posts/30092_b7d56e941888b624\.png", + "#sha1_url": "dad4d4c67d87cd9a4ac429b3414747c27a95d5cb", + "#sha1_content": "86d1514c0ca8197950cc4b74e7a59b2dc76ebf9c", + }, ) diff --git a/test/results/furaffinity.py b/test/results/furaffinity.py index 75535591dd..53db402ffb 100644 --- a/test/results/furaffinity.py +++ b/test/results/furaffinity.py @@ -1,248 +1,220 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import furaffinity - __tests__ = ( -{ - "#url" : "https://www.furaffinity.net/gallery/mirlinthloth/", - "#category": ("", "furaffinity", "gallery"), - "#class" : furaffinity.FuraffinityGalleryExtractor, - "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/mirlinthloth/\d+/\d+.\w+\.\w+", - "#range" : "45-50", - "#count" : 6, -}, - -{ - "#url" : "https://www.furaffinity.net/scraps/mirlinthloth/", - "#category": ("", "furaffinity", "scraps"), - "#class" : furaffinity.FuraffinityScrapsExtractor, - "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+(/stories)?/\d+/\d+.\w+.", - "#count" : ">= 3", -}, - -{ - "#url" : "https://www.furaffinity.net/favorites/mirlinthloth/", - "#category": ("", "furaffinity", "favorite"), - "#class" : furaffinity.FuraffinityFavoriteExtractor, - "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+/\d+/\d+.\w+\.\w+", - "#range" : "45-50", - "#count" : 6, - - "favorite_id": int, -}, - -{ - "#url" : "https://www.furaffinity.net/search/?q=cute", - "#category": ("", "furaffinity", "search"), - "#class" : furaffinity.FuraffinitySearchExtractor, - "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+/\d+/\d+.\w+\.\w+", - "#range" : "45-50", - "#count" : 6, -}, - -{ - "#url" : "https://www.furaffinity.net/search/?q=leaf&range=1day", - "#comment" : "first page of search results (#2402)", - "#category": ("", "furaffinity", "search"), - "#class" : furaffinity.FuraffinitySearchExtractor, - "#range" : "1-3", - "#count" : 3, -}, - -{ - "#url" : "https://www.furaffinity.net/view/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, - "#pattern" : r"https://d\d*\.f(uraffinity|acdn)\.net/(download/)?art/mirlinthloth/music/1488278723/1480267446.mirlinthloth_dj_fennmink_-_bude_s_4_ever\.mp3", - - "artist" : "mirlinthloth", - "artist_url" : "mirlinthloth", - "date" : "dt:2016-11-27 17:24:06", - "description": "A Song made playing the game Cosmic DJ.", - "extension" : "mp3", - "filename" : r"re:\d+\.\w+_dj_fennmink_-_bude_s_4_ever", - "id" : 21835115, - "tags" : list, - "title" : "Bude's 4 Ever", - "url" : r"re:https://d\d?\.f(uraffinity|acdn)\.net/art", - "user" : "mirlinthloth", - "views" : int, - "favorites" : int, - "comments" : int, - "rating" : "General", - "fa_category": "Music", - "theme" : "All", - "species" : "Unspecified / Any", - "gender" : "Any", - "width" : 120, - "height" : 120, -}, - -{ - "#url" : "https://www.furaffinity.net/view/42166511/", - "#comment" : "'external' option (#1492)", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, - "#options" : {"external": True}, - "#pattern" : r"https://d\d*\.f(uraffinity|acdn)\.net/|http://www\.postybirb\.com", - "#count" : 2, -}, - -{ - "#url" : "https://www.furaffinity.net/view/45331225/", - "#comment" : "no tags (#2277)", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, - - "artist" : "Kota_Remminders", - "artist_url" : "kotaremminders", - "date" : "dt:2022-01-03 17:49:33", - "fa_category": "Adoptables", - "filename" : "1641232173.kotaremminders_chidopts1", - "gender" : "Any", - "height" : 905, - "id" : 45331225, - "rating" : "General", - "species" : "Unspecified / Any", - "tags" : [], - "theme" : "All", - "title" : "REMINDER", - "width" : 1280, -}, - -{ - "#url" : "https://www.furaffinity.net/view/22964019/", - "#comment" : "get thumbnails for posts (#1284)", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, - - "artist" : "Dwale", - "artist_url" : "dwale", - "date" : "dt:2017-03-21 14:21:29", - "fa_category" : "Poetry", - "filename" : "1490106089.dwale_poem_for_children", - "folders" : [], - "height" : 50, - "id" : 22964019, - "rating" : "General", - "title" : "Poem for Children Wishing to Summon Evil Spirits", - "thumbnail" : "https://t.furaffinity.net/22964019@600-1490106089.jpg", - "width" : 50, -}, - -{ - "#url" : "https://www.furaffinity.net/view/34260156/", - "#comment" : "list gallery folders for image", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, - - "artist" : "dbd", - "artist_url" : "dbd", - "date" : "dt:2019-12-17 22:52:01", - "fa_category" : "All", - "filename" : "1576623121.dbd_patreoncustom-wdg13-web", - "folders" : ["By Year - 2019", - "Custom Character Folder - All Custom Characters", - "Custom Character Folder - Other Ungulates", - "Custom Character Folder - Female", - "Custom Character Folder - Patreon Supported Custom Characters"], - "height" : 900, - "id" : 34260156, - "rating" : "General", - "title" : "Patreon Custom Deer", - "thumbnail" : "https://t.furaffinity.net/34260156@600-1576623121.jpg", - "width" : 488, -}, - -{ - "#url" : "https://www.furaffinity.net/view/57587562", - "#comment" : "login required", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://furaffinity.net/view/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, -}, - -{ - "#url" : "https://fxfuraffinity.net/view/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, -}, - -{ - "#url" : "https://xfuraffinity.net/view/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, -}, - -{ - "#url" : "https://fxraffinity.net/view/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, -}, - -{ - "#url" : "https://sfw.furaffinity.net/view/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, -}, - -{ - "#url" : "https://www.furaffinity.net/full/21835115/", - "#category": ("", "furaffinity", "post"), - "#class" : furaffinity.FuraffinityPostExtractor, -}, - -{ - "#url" : "https://www.furaffinity.net/user/mirlinthloth/", - "#category": ("", "furaffinity", "user"), - "#class" : furaffinity.FuraffinityUserExtractor, - "#pattern" : "/gallery/mirlinthloth/$", -}, - -{ - "#url" : "https://www.furaffinity.net/user/mirlinthloth/", - "#category": ("", "furaffinity", "user"), - "#class" : furaffinity.FuraffinityUserExtractor, - "#options" : {"include": "all"}, - "#pattern" : "/(gallery|scraps|favorites)/mirlinthloth/$", - "#count" : 3, -}, - -{ - "#url" : "https://www.furaffinity.net/watchlist/by/mirlinthloth/", - "#category": ("", "furaffinity", "following"), - "#class" : furaffinity.FuraffinityFollowingExtractor, - "#pattern" : furaffinity.FuraffinityUserExtractor.pattern, - "#range" : "176-225", - "#count" : 50, -}, - -{ - "#url" : "https://www.furaffinity.net/msg/submissions", - "#category": ("", "furaffinity", "submissions"), - "#class" : furaffinity.FuraffinitySubmissionsExtractor, - "#auth" : True, - "#pattern" : r"https://d\d?\.f(uraffinity|acdn)\.net/art/mirlinthloth/\d+/\d+.\w+\.\w+", - "#range" : "45-50", - "#count" : 6, -}, - -{ - "#url" : "https://www.furaffinity.net/msg/submissions/new~56789000@48/", - "#category": ("", "furaffinity", "submissions"), - "#class" : furaffinity.FuraffinitySubmissionsExtractor, - "#auth" : True, -}, - + { + "#url": "https://www.furaffinity.net/gallery/mirlinthloth/", + "#category": ("", "furaffinity", "gallery"), + "#class": furaffinity.FuraffinityGalleryExtractor, + "#pattern": r"https://d\d?\.f(uraffinity|acdn)\.net/art/mirlinthloth/\d+/\d+.\w+\.\w+", + "#range": "45-50", + "#count": 6, + }, + { + "#url": "https://www.furaffinity.net/scraps/mirlinthloth/", + "#category": ("", "furaffinity", "scraps"), + "#class": furaffinity.FuraffinityScrapsExtractor, + "#pattern": r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+(/stories)?/\d+/\d+.\w+.", + "#count": ">= 3", + }, + { + "#url": "https://www.furaffinity.net/favorites/mirlinthloth/", + "#category": ("", "furaffinity", "favorite"), + "#class": furaffinity.FuraffinityFavoriteExtractor, + "#pattern": r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+/\d+/\d+.\w+\.\w+", + "#range": "45-50", + "#count": 6, + "favorite_id": int, + }, + { + "#url": "https://www.furaffinity.net/search/?q=cute", + "#category": ("", "furaffinity", "search"), + "#class": furaffinity.FuraffinitySearchExtractor, + "#pattern": r"https://d\d?\.f(uraffinity|acdn)\.net/art/[^/]+/\d+/\d+.\w+\.\w+", + "#range": "45-50", + "#count": 6, + }, + { + "#url": "https://www.furaffinity.net/search/?q=leaf&range=1day", + "#comment": "first page of search results (#2402)", + "#category": ("", "furaffinity", "search"), + "#class": furaffinity.FuraffinitySearchExtractor, + "#range": "1-3", + "#count": 3, + }, + { + "#url": "https://www.furaffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + "#pattern": r"https://d\d*\.f(uraffinity|acdn)\.net/(download/)?art/mirlinthloth/music/1488278723/1480267446.mirlinthloth_dj_fennmink_-_bude_s_4_ever\.mp3", + "artist": "mirlinthloth", + "artist_url": "mirlinthloth", + "date": "dt:2016-11-27 17:24:06", + "description": "A Song made playing the game Cosmic DJ.", + "extension": "mp3", + "filename": r"re:\d+\.\w+_dj_fennmink_-_bude_s_4_ever", + "id": 21835115, + "tags": list, + "title": "Bude's 4 Ever", + "url": r"re:https://d\d?\.f(uraffinity|acdn)\.net/art", + "user": "mirlinthloth", + "views": int, + "favorites": int, + "comments": int, + "rating": "General", + "fa_category": "Music", + "theme": "All", + "species": "Unspecified / Any", + "gender": "Any", + "width": 120, + "height": 120, + }, + { + "#url": "https://www.furaffinity.net/view/42166511/", + "#comment": "'external' option (#1492)", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + "#options": {"external": True}, + "#pattern": r"https://d\d*\.f(uraffinity|acdn)\.net/|http://www\.postybirb\.com", + "#count": 2, + }, + { + "#url": "https://www.furaffinity.net/view/45331225/", + "#comment": "no tags (#2277)", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + "artist": "Kota_Remminders", + "artist_url": "kotaremminders", + "date": "dt:2022-01-03 17:49:33", + "fa_category": "Adoptables", + "filename": "1641232173.kotaremminders_chidopts1", + "gender": "Any", + "height": 905, + "id": 45331225, + "rating": "General", + "species": "Unspecified / Any", + "tags": [], + "theme": "All", + "title": "REMINDER", + "width": 1280, + }, + { + "#url": "https://www.furaffinity.net/view/22964019/", + "#comment": "get thumbnails for posts (#1284)", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + "artist": "Dwale", + "artist_url": "dwale", + "date": "dt:2017-03-21 14:21:29", + "fa_category": "Poetry", + "filename": "1490106089.dwale_poem_for_children", + "folders": [], + "height": 50, + "id": 22964019, + "rating": "General", + "title": "Poem for Children Wishing to Summon Evil Spirits", + "thumbnail": "https://t.furaffinity.net/22964019@600-1490106089.jpg", + "width": 50, + }, + { + "#url": "https://www.furaffinity.net/view/34260156/", + "#comment": "list gallery folders for image", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + "artist": "dbd", + "artist_url": "dbd", + "date": "dt:2019-12-17 22:52:01", + "fa_category": "All", + "filename": "1576623121.dbd_patreoncustom-wdg13-web", + "folders": [ + "By Year - 2019", + "Custom Character Folder - All Custom Characters", + "Custom Character Folder - Other Ungulates", + "Custom Character Folder - Female", + "Custom Character Folder - Patreon Supported Custom Characters", + ], + "height": 900, + "id": 34260156, + "rating": "General", + "title": "Patreon Custom Deer", + "thumbnail": "https://t.furaffinity.net/34260156@600-1576623121.jpg", + "width": 488, + }, + { + "#url": "https://www.furaffinity.net/view/57587562", + "#comment": "login required", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + "#count": 0, + }, + { + "#url": "https://furaffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + }, + { + "#url": "https://fxfuraffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + }, + { + "#url": "https://xfuraffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + }, + { + "#url": "https://fxraffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + }, + { + "#url": "https://sfw.furaffinity.net/view/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + }, + { + "#url": "https://www.furaffinity.net/full/21835115/", + "#category": ("", "furaffinity", "post"), + "#class": furaffinity.FuraffinityPostExtractor, + }, + { + "#url": "https://www.furaffinity.net/user/mirlinthloth/", + "#category": ("", "furaffinity", "user"), + "#class": furaffinity.FuraffinityUserExtractor, + "#pattern": "/gallery/mirlinthloth/$", + }, + { + "#url": "https://www.furaffinity.net/user/mirlinthloth/", + "#category": ("", "furaffinity", "user"), + "#class": furaffinity.FuraffinityUserExtractor, + "#options": {"include": "all"}, + "#pattern": "/(gallery|scraps|favorites)/mirlinthloth/$", + "#count": 3, + }, + { + "#url": "https://www.furaffinity.net/watchlist/by/mirlinthloth/", + "#category": ("", "furaffinity", "following"), + "#class": furaffinity.FuraffinityFollowingExtractor, + "#pattern": furaffinity.FuraffinityUserExtractor.pattern, + "#range": "176-225", + "#count": 50, + }, + { + "#url": "https://www.furaffinity.net/msg/submissions", + "#category": ("", "furaffinity", "submissions"), + "#class": furaffinity.FuraffinitySubmissionsExtractor, + "#auth": True, + "#pattern": r"https://d\d?\.f(uraffinity|acdn)\.net/art/mirlinthloth/\d+/\d+.\w+\.\w+", + "#range": "45-50", + "#count": 6, + }, + { + "#url": "https://www.furaffinity.net/msg/submissions/new~56789000@48/", + "#category": ("", "furaffinity", "submissions"), + "#class": furaffinity.FuraffinitySubmissionsExtractor, + "#auth": True, + }, ) diff --git a/test/results/furbooru.py b/test/results/furbooru.py index 682aa0054e..449eea2f56 100644 --- a/test/results/furbooru.py +++ b/test/results/furbooru.py @@ -1,33 +1,27 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import philomena - __tests__ = ( -{ - "#url" : "https://furbooru.org/images/1", - "#category": ("philomena", "furbooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, - "#sha1_content": "9eaa1e1b32fa0f16520912257dbefaff238d5fd2", -}, - -{ - "#url" : "https://furbooru.org/search?q=cute", - "#category": ("philomena", "furbooru", "search"), - "#class" : philomena.PhilomenaSearchExtractor, - "#range" : "40-60", - "#count" : 21, -}, - -{ - "#url" : "https://furbooru.org/galleries/27", - "#category": ("philomena", "furbooru", "gallery"), - "#class" : philomena.PhilomenaGalleryExtractor, - "#count" : ">= 13", -}, - + { + "#url": "https://furbooru.org/images/1", + "#category": ("philomena", "furbooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + "#sha1_content": "9eaa1e1b32fa0f16520912257dbefaff238d5fd2", + }, + { + "#url": "https://furbooru.org/search?q=cute", + "#category": ("philomena", "furbooru", "search"), + "#class": philomena.PhilomenaSearchExtractor, + "#range": "40-60", + "#count": 21, + }, + { + "#url": "https://furbooru.org/galleries/27", + "#category": ("philomena", "furbooru", "gallery"), + "#class": philomena.PhilomenaGalleryExtractor, + "#count": ">= 13", + }, ) diff --git a/test/results/fuskator.py b/test/results/fuskator.py index cd68c9c2f3..c51d40f227 100644 --- a/test/results/fuskator.py +++ b/test/results/fuskator.py @@ -1,47 +1,39 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import fuskator - __tests__ = ( -{ - "#url" : "https://fuskator.com/thumbs/d0GnIzXrSKU/", - "#category": ("", "fuskator", "gallery"), - "#class" : fuskator.FuskatorGalleryExtractor, - "#pattern" : r"https://i\d+.fuskator.com/large/d0GnIzXrSKU/.+\.jpg", - "#count" : 22, - - "gallery_id" : 473023, - "gallery_hash": "d0GnIzXrSKU", - "title" : r"re:Shaved Brunette Babe Maria Ryabushkina with ", - "views" : int, - "score" : float, - "count" : 22, - "tags" : list, -}, - -{ - "#url" : "https://fuskator.com/expanded/gXpKzjgIidA/index.html", - "#category": ("", "fuskator", "gallery"), - "#class" : fuskator.FuskatorGalleryExtractor, -}, - -{ - "#url" : "https://fuskator.com/search/red_swimsuit/", - "#category": ("", "fuskator", "search"), - "#class" : fuskator.FuskatorSearchExtractor, - "#pattern" : fuskator.FuskatorGalleryExtractor.pattern, - "#count" : ">= 40", -}, - -{ - "#url" : "https://fuskator.com/page/3/swimsuit/quality/", - "#category": ("", "fuskator", "search"), - "#class" : fuskator.FuskatorSearchExtractor, -}, - + { + "#url": "https://fuskator.com/thumbs/d0GnIzXrSKU/", + "#category": ("", "fuskator", "gallery"), + "#class": fuskator.FuskatorGalleryExtractor, + "#pattern": r"https://i\d+.fuskator.com/large/d0GnIzXrSKU/.+\.jpg", + "#count": 22, + "gallery_id": 473023, + "gallery_hash": "d0GnIzXrSKU", + "title": r"re:Shaved Brunette Babe Maria Ryabushkina with ", + "views": int, + "score": float, + "count": 22, + "tags": list, + }, + { + "#url": "https://fuskator.com/expanded/gXpKzjgIidA/index.html", + "#category": ("", "fuskator", "gallery"), + "#class": fuskator.FuskatorGalleryExtractor, + }, + { + "#url": "https://fuskator.com/search/red_swimsuit/", + "#category": ("", "fuskator", "search"), + "#class": fuskator.FuskatorSearchExtractor, + "#pattern": fuskator.FuskatorGalleryExtractor.pattern, + "#count": ">= 40", + }, + { + "#url": "https://fuskator.com/page/3/swimsuit/quality/", + "#category": ("", "fuskator", "search"), + "#class": fuskator.FuskatorSearchExtractor, + }, ) diff --git a/test/results/gelbooru.py b/test/results/gelbooru.py index 3f09ea6902..b1808029c9 100644 --- a/test/results/gelbooru.py +++ b/test/results/gelbooru.py @@ -1,192 +1,167 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru - __tests__ = ( -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", - "#category": ("booru", "gelbooru", "tag"), - "#class" : gelbooru.GelbooruTagExtractor, - "#count" : 5, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=all", - "#category": ("booru", "gelbooru", "tag"), - "#class" : gelbooru.GelbooruTagExtractor, - "#range" : "1-3", - "#count" : 3, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=", - "#category": ("booru", "gelbooru", "tag"), - "#class" : gelbooru.GelbooruTagExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=meiya_neon", - "#category": ("booru", "gelbooru", "tag"), - "#class" : gelbooru.GelbooruTagExtractor, - "#pattern" : r"https://img\d\.gelbooru\.com/images/../../[0-9a-f]{32}\.jpg", - "#range" : "196-204", - "#count" : 9, - "#sha1_url": "845a61aa1f90fb4ced841e8b7e62098be2e967bf", -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=id:>=67800+id:<=68000", - "#comment" : "meta tags (#5478)", - "#category": ("booru", "gelbooru", "tag"), - "#class" : gelbooru.GelbooruTagExtractor, - "#count" : 187, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=list&tags=id:>=67800+id:<=68000+sort:id:asc", - "#comment" : "meta + sort tags (#5478)", - "#category": ("booru", "gelbooru", "tag"), - "#class" : gelbooru.GelbooruTagExtractor, - "#count" : 187, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=pool&s=show&id=761", - "#category": ("booru", "gelbooru", "pool"), - "#class" : gelbooru.GelbooruPoolExtractor, - "#count" : 6, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=favorites&s=view&id=1435674", - "#category": ("booru", "gelbooru", "favorite"), - "#class" : gelbooru.GelbooruFavoriteExtractor, - "#urls" : ( - "https://img3.gelbooru.com/images/5d/30/5d30fc056ed8668616b3c440df9bac89.jpg", - "https://img3.gelbooru.com/images/4c/2d/4c2da867ed643acdadd8105177dcdaf0.png", - "https://img3.gelbooru.com/images/c8/26/c826f3cb90d9aaca8d0632a96bf4abe8.jpg", - "https://img3.gelbooru.com/images/c1/fe/c1fe59c0bc8ce955dd353544b1015d0c.jpg", - "https://img3.gelbooru.com/images/e6/6d/e66d8883c184f5d3b2591dfcdf0d007c.jpg", - ), -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=favorites&s=view&id=1435674", - "#category": ("booru", "gelbooru", "favorite"), - "#class" : gelbooru.GelbooruFavoriteExtractor, - "#options" : {"order-posts": "reverse"}, - "#urls" : ( - "https://img3.gelbooru.com/images/e6/6d/e66d8883c184f5d3b2591dfcdf0d007c.jpg", - "https://img3.gelbooru.com/images/c1/fe/c1fe59c0bc8ce955dd353544b1015d0c.jpg", - "https://img3.gelbooru.com/images/c8/26/c826f3cb90d9aaca8d0632a96bf4abe8.jpg", - "https://img3.gelbooru.com/images/4c/2d/4c2da867ed643acdadd8105177dcdaf0.png", - "https://img3.gelbooru.com/images/5d/30/5d30fc056ed8668616b3c440df9bac89.jpg", - ), -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=313638", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, - "#count" : 1, - "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=313638", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?s=view&page=post&id=313638", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&id=313638&s=view", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?s=view&id=313638&page=post", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?id=313638&page=post&s=view", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?id=313638&s=view&page=post", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=6018318", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, - "#options" : {"tags": True}, - "#sha1_content": "977caf22f27c72a5d07ea4d4d9719acdab810991", - - "tags_artist" : "kirisaki_shuusei", - "tags_character": str, - "tags_copyright": "vocaloid", - "tags_general" : str, - "tags_metadata" : str, -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=5938076", - "#comment" : "video", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, - "#pattern" : r"https://img\d\.gelbooru\.com/images/22/61/226111273615049235b001b381707bd0\.webm", - "#sha1_content": "6360452fa8c2f0c1137749e81471238564df832a", -}, - -{ - "#url" : "https://gelbooru.com/index.php?page=post&s=view&id=5997331", - "#comment" : "notes", - "#category": ("booru", "gelbooru", "post"), - "#class" : gelbooru.GelbooruPostExtractor, - "#options" : {"notes": True}, - - "notes": [ - { - "body" : "Look over this way when you talk~", - "height": 553, - "width" : 246, - "x" : 35, - "y" : 72, - }, - { - "body" : """Hey~ + { + "#url": "https://gelbooru.com/index.php?page=post&s=list&tags=bonocho", + "#category": ("booru", "gelbooru", "tag"), + "#class": gelbooru.GelbooruTagExtractor, + "#count": 5, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=list&tags=all", + "#category": ("booru", "gelbooru", "tag"), + "#class": gelbooru.GelbooruTagExtractor, + "#range": "1-3", + "#count": 3, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=list&tags=", + "#category": ("booru", "gelbooru", "tag"), + "#class": gelbooru.GelbooruTagExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=list&tags=meiya_neon", + "#category": ("booru", "gelbooru", "tag"), + "#class": gelbooru.GelbooruTagExtractor, + "#pattern": r"https://img\d\.gelbooru\.com/images/../../[0-9a-f]{32}\.jpg", + "#range": "196-204", + "#count": 9, + "#sha1_url": "845a61aa1f90fb4ced841e8b7e62098be2e967bf", + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=list&tags=id:>=67800+id:<=68000", + "#comment": "meta tags (#5478)", + "#category": ("booru", "gelbooru", "tag"), + "#class": gelbooru.GelbooruTagExtractor, + "#count": 187, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=list&tags=id:>=67800+id:<=68000+sort:id:asc", + "#comment": "meta + sort tags (#5478)", + "#category": ("booru", "gelbooru", "tag"), + "#class": gelbooru.GelbooruTagExtractor, + "#count": 187, + }, + { + "#url": "https://gelbooru.com/index.php?page=pool&s=show&id=761", + "#category": ("booru", "gelbooru", "pool"), + "#class": gelbooru.GelbooruPoolExtractor, + "#count": 6, + }, + { + "#url": "https://gelbooru.com/index.php?page=favorites&s=view&id=1435674", + "#category": ("booru", "gelbooru", "favorite"), + "#class": gelbooru.GelbooruFavoriteExtractor, + "#urls": ( + "https://img3.gelbooru.com/images/5d/30/5d30fc056ed8668616b3c440df9bac89.jpg", + "https://img3.gelbooru.com/images/4c/2d/4c2da867ed643acdadd8105177dcdaf0.png", + "https://img3.gelbooru.com/images/c8/26/c826f3cb90d9aaca8d0632a96bf4abe8.jpg", + "https://img3.gelbooru.com/images/c1/fe/c1fe59c0bc8ce955dd353544b1015d0c.jpg", + "https://img3.gelbooru.com/images/e6/6d/e66d8883c184f5d3b2591dfcdf0d007c.jpg", + ), + }, + { + "#url": "https://gelbooru.com/index.php?page=favorites&s=view&id=1435674", + "#category": ("booru", "gelbooru", "favorite"), + "#class": gelbooru.GelbooruFavoriteExtractor, + "#options": {"order-posts": "reverse"}, + "#urls": ( + "https://img3.gelbooru.com/images/e6/6d/e66d8883c184f5d3b2591dfcdf0d007c.jpg", + "https://img3.gelbooru.com/images/c1/fe/c1fe59c0bc8ce955dd353544b1015d0c.jpg", + "https://img3.gelbooru.com/images/c8/26/c826f3cb90d9aaca8d0632a96bf4abe8.jpg", + "https://img3.gelbooru.com/images/4c/2d/4c2da867ed643acdadd8105177dcdaf0.png", + "https://img3.gelbooru.com/images/5d/30/5d30fc056ed8668616b3c440df9bac89.jpg", + ), + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=view&id=313638", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + "#count": 1, + "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=view&id=313638", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?s=view&page=post&id=313638", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&id=313638&s=view", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?s=view&id=313638&page=post", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?id=313638&page=post&s=view", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?id=313638&s=view&page=post", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=view&id=6018318", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + "#options": {"tags": True}, + "#sha1_content": "977caf22f27c72a5d07ea4d4d9719acdab810991", + "tags_artist": "kirisaki_shuusei", + "tags_character": str, + "tags_copyright": "vocaloid", + "tags_general": str, + "tags_metadata": str, + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=view&id=5938076", + "#comment": "video", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + "#pattern": r"https://img\d\.gelbooru\.com/images/22/61/226111273615049235b001b381707bd0\.webm", + "#sha1_content": "6360452fa8c2f0c1137749e81471238564df832a", + }, + { + "#url": "https://gelbooru.com/index.php?page=post&s=view&id=5997331", + "#comment": "notes", + "#category": ("booru", "gelbooru", "post"), + "#class": gelbooru.GelbooruPostExtractor, + "#options": {"notes": True}, + "notes": [ + { + "body": "Look over this way when you talk~", + "height": 553, + "width": 246, + "x": 35, + "y": 72, + }, + { + "body": """Hey~ Are you listening~?""", - "height": 557, - "width" : 246, - "x" : 1233, - "y" : 109, - }, - ], -}, - -{ - "#url" : "https://gelbooru.com/redirect.php?s=Ly9nZWxib29ydS5jb20vaW5kZXgucGhwP3BhZ2U9cG9zdCZzPXZpZXcmaWQ9MTgzMDA0Ng==", - "#category": ("booru", "gelbooru", "redirect"), - "#class" : gelbooru.GelbooruRedirectExtractor, - "#pattern" : r"https://gelbooru.com/index.php\?page=post&s=view&id=1830046", -}, - + "height": 557, + "width": 246, + "x": 1233, + "y": 109, + }, + ], + }, + { + "#url": "https://gelbooru.com/redirect.php?s=Ly9nZWxib29ydS5jb20vaW5kZXgucGhwP3BhZ2U9cG9zdCZzPXZpZXcmaWQ9MTgzMDA0Ng==", + "#category": ("booru", "gelbooru", "redirect"), + "#class": gelbooru.GelbooruRedirectExtractor, + "#pattern": r"https://gelbooru.com/index.php\?page=post&s=view&id=1830046", + }, ) diff --git a/test/results/generic.py b/test/results/generic.py index 4d940afee0..e440ef8fe1 100644 --- a/test/results/generic.py +++ b/test/results/generic.py @@ -1,68 +1,56 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import generic - __tests__ = ( -{ - "#url" : "generic:https://www.nongnu.org/lzip/", - "#category": ("", "generic", "www.nongnu.org"), - "#class" : generic.GenericExtractor, - "#count" : 1, - "#sha1_content": "40be5c77773d3e91db6e1c5df720ee30afb62368", - - "description": "Lossless data compressor", - "imageurl" : "https://www.nongnu.org/lzip/lzip.png", - "keywords" : "lzip, clzip, plzip, lzlib, LZMA, bzip2, gzip, data compression, GNU, free software", - "pageurl" : "https://www.nongnu.org/lzip/", -}, - -{ - "#url" : "generic:https://räksmörgås.josefsson.org/", - "#category": ("", "generic", "räksmörgås.josefsson.org"), - "#class" : generic.GenericExtractor, - "#pattern" : "^https://räksmörgås.josefsson.org/", - "#count" : 2, -}, - -{ - "#url" : "generic:https://en.wikipedia.org/Main_Page", - "#category": ("", "generic", "en.wikipedia.org"), - "#class" : generic.GenericExtractor, -}, - -{ - "#url" : "generic:https://example.org/path/to/file?que=1?&ry=2/#fragment", - "#category": ("", "generic", "example.org"), - "#class" : generic.GenericExtractor, -}, - -{ - "#url" : "generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", - "#category": ("", "generic", "example.org"), - "#class" : generic.GenericExtractor, -}, - -{ - "#url" : "generic:https://en.wikipedia.org/Main_Page", - "#category": ("", "generic", "en.wikipedia.org"), - "#class" : generic.GenericExtractor, -}, - -{ - "#url" : "generic:https://example.org/path/to/file?que=1?&ry=2/#fragment", - "#category": ("", "generic", "example.org"), - "#class" : generic.GenericExtractor, -}, - -{ - "#url" : "generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", - "#category": ("", "generic", "example.org"), - "#class" : generic.GenericExtractor, -}, - + { + "#url": "generic:https://www.nongnu.org/lzip/", + "#category": ("", "generic", "www.nongnu.org"), + "#class": generic.GenericExtractor, + "#count": 1, + "#sha1_content": "40be5c77773d3e91db6e1c5df720ee30afb62368", + "description": "Lossless data compressor", + "imageurl": "https://www.nongnu.org/lzip/lzip.png", + "keywords": "lzip, clzip, plzip, lzlib, LZMA, bzip2, gzip, data compression, GNU, free software", + "pageurl": "https://www.nongnu.org/lzip/", + }, + { + "#url": "generic:https://räksmörgås.josefsson.org/", + "#category": ("", "generic", "räksmörgås.josefsson.org"), + "#class": generic.GenericExtractor, + "#pattern": "^https://räksmörgås.josefsson.org/", + "#count": 2, + }, + { + "#url": "generic:https://en.wikipedia.org/Main_Page", + "#category": ("", "generic", "en.wikipedia.org"), + "#class": generic.GenericExtractor, + }, + { + "#url": "generic:https://example.org/path/to/file?que=1?&ry=2/#fragment", + "#category": ("", "generic", "example.org"), + "#class": generic.GenericExtractor, + }, + { + "#url": "generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", + "#category": ("", "generic", "example.org"), + "#class": generic.GenericExtractor, + }, + { + "#url": "generic:https://en.wikipedia.org/Main_Page", + "#category": ("", "generic", "en.wikipedia.org"), + "#class": generic.GenericExtractor, + }, + { + "#url": "generic:https://example.org/path/to/file?que=1?&ry=2/#fragment", + "#category": ("", "generic", "example.org"), + "#class": generic.GenericExtractor, + }, + { + "#url": "generic:https://example.org/%27%3C%23/%23%3E%27.htm?key=%3C%26%3E", + "#category": ("", "generic", "example.org"), + "#class": generic.GenericExtractor, + }, ) diff --git a/test/results/giantessbooru.py b/test/results/giantessbooru.py index e35b9d1074..7683ddcc02 100644 --- a/test/results/giantessbooru.py +++ b/test/results/giantessbooru.py @@ -1,62 +1,52 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shimmie2 - __tests__ = ( -{ - "#url" : "https://sizechangebooru.com/index.php?q=/post/list/drawing/1", - "#category": ("shimmie2", "giantessbooru", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, - "#pattern" : r"https://sizechangebooru\.com/index\.php\?q=/image/\d+\.jpg", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://giantessbooru.com/index.php?q=/post/list/drawing/1", - "#category": ("shimmie2", "giantessbooru", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, -}, - -{ - "#url" : "https://giantessbooru.com/post/list/drawing/1", - "#category": ("shimmie2", "giantessbooru", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, -}, - -{ - "#url" : "https://sizechangebooru.com/index.php?q=/post/view/41", - "#category": ("shimmie2", "giantessbooru", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, - "#urls" : "https://sizechangebooru.com/index.php?q=/image/41.jpg", - "#sha1_content": "79115ed309d1f4e82e7bead6948760e889139c91", - - "extension": "jpg", - "file_url" : "https://sizechangebooru.com/index.php?q=/image/41.jpg", - "filename" : "41", - "height" : 0, - "id" : 41, - "md5" : "", - "size" : 0, - "tags" : "anime bare_midriff color drawing gentle giantess karbo looking_at_tinies negeyari outdoors smiling snake_girl white_hair", - "width" : 1387, -}, - -{ - "#url" : "https://giantessbooru.com/index.php?q=/post/view/41", - "#category": ("shimmie2", "giantessbooru", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, -}, - -{ - "#url" : "https://giantessbooru.com/post/view/41", - "#category": ("shimmie2", "giantessbooru", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, -}, - + { + "#url": "https://sizechangebooru.com/index.php?q=/post/list/drawing/1", + "#category": ("shimmie2", "giantessbooru", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + "#pattern": r"https://sizechangebooru\.com/index\.php\?q=/image/\d+\.jpg", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://giantessbooru.com/index.php?q=/post/list/drawing/1", + "#category": ("shimmie2", "giantessbooru", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + }, + { + "#url": "https://giantessbooru.com/post/list/drawing/1", + "#category": ("shimmie2", "giantessbooru", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + }, + { + "#url": "https://sizechangebooru.com/index.php?q=/post/view/41", + "#category": ("shimmie2", "giantessbooru", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + "#urls": "https://sizechangebooru.com/index.php?q=/image/41.jpg", + "#sha1_content": "79115ed309d1f4e82e7bead6948760e889139c91", + "extension": "jpg", + "file_url": "https://sizechangebooru.com/index.php?q=/image/41.jpg", + "filename": "41", + "height": 0, + "id": 41, + "md5": "", + "size": 0, + "tags": "anime bare_midriff color drawing gentle giantess karbo looking_at_tinies negeyari outdoors smiling snake_girl white_hair", + "width": 1387, + }, + { + "#url": "https://giantessbooru.com/index.php?q=/post/view/41", + "#category": ("shimmie2", "giantessbooru", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + }, + { + "#url": "https://giantessbooru.com/post/view/41", + "#category": ("shimmie2", "giantessbooru", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + }, ) diff --git a/test/results/gofile.py b/test/results/gofile.py index b7d2670271..94d6acb95d 100644 --- a/test/results/gofile.py +++ b/test/results/gofile.py @@ -1,57 +1,51 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gofile - __tests__ = ( -{ - "#url" : "https://gofile.io/d/k6BomI", - "#category": ("", "gofile", "folder"), - "#class" : gofile.GofileFolderExtractor, - "#pattern" : r"https://store\d+\.gofile\.io/download/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/test-%E3%83%86%E3%82%B9%E3%83%88-%2522%26!\.png", - - "createTime" : int, - "directLink" : r"re:https://store5.gofile.io/download/direct/.+", - "downloadCount": int, - "extension" : "png", - "filename" : "test-テスト-%22&!", - "folder" : { - "childs" : [ - "b0367d79-b8ba-407f-8342-aaf8eb815443", - "7fd4a36a-c1dd-49ff-9223-d93f7d24093f", - ], - "code" : "k6BomI", - "createTime" : 1654076165, - "id" : "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", - "name" : "root", - "public" : True, - "totalDownloadCount": int, - "totalSize" : 182, - "type" : "folder", + { + "#url": "https://gofile.io/d/k6BomI", + "#category": ("", "gofile", "folder"), + "#class": gofile.GofileFolderExtractor, + "#pattern": r"https://store\d+\.gofile\.io/download/\w{8}-\w{4}-\w{4}-\w{4}-\w{12}/test-%E3%83%86%E3%82%B9%E3%83%88-%2522%26!\.png", + "createTime": int, + "directLink": r"re:https://store5.gofile.io/download/direct/.+", + "downloadCount": int, + "extension": "png", + "filename": "test-テスト-%22&!", + "folder": { + "childs": [ + "b0367d79-b8ba-407f-8342-aaf8eb815443", + "7fd4a36a-c1dd-49ff-9223-d93f7d24093f", + ], + "code": "k6BomI", + "createTime": 1654076165, + "id": "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", + "name": "root", + "public": True, + "totalDownloadCount": int, + "totalSize": 182, + "type": "folder", + }, + "id": r"re:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}", + "link": r"re:https://store5.gofile.io/download/.+\.png", + "md5": r"re:[0-9a-f]{32}", + "mimetype": "image/png", + "name": "test-テスト-%22&!.png", + "num": int, + "parentFolder": "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", + "serverChoosen": "store5", + "size": 182, + "thumbnail": r"re:https://store5.gofile.io/download/.+\.png", + "type": "file", + }, + { + "#url": "https://gofile.io/d/7fd4a36a-c1dd-49ff-9223-d93f7d24093f", + "#category": ("", "gofile", "folder"), + "#class": gofile.GofileFolderExtractor, + "#options": {"website-token": None}, + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", }, - "id" : r"re:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}", - "link" : r"re:https://store5.gofile.io/download/.+\.png", - "md5" : r"re:[0-9a-f]{32}", - "mimetype" : "image/png", - "name" : "test-テスト-%22&!.png", - "num" : int, - "parentFolder" : "fafb59f9-a7c7-4fea-a098-b29b8d97b03c", - "serverChoosen": "store5", - "size" : 182, - "thumbnail" : r"re:https://store5.gofile.io/download/.+\.png", - "type" : "file", -}, - -{ - "#url" : "https://gofile.io/d/7fd4a36a-c1dd-49ff-9223-d93f7d24093f", - "#category": ("", "gofile", "folder"), - "#class" : gofile.GofileFolderExtractor, - "#options" : {"website-token": None}, - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", -}, - ) diff --git a/test/results/hatenablog.py b/test/results/hatenablog.py index 4a306f9a18..f6e24f0224 100644 --- a/test/results/hatenablog.py +++ b/test/results/hatenablog.py @@ -1,144 +1,123 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hatenablog - __tests__ = ( -{ - "#url" : "https://cosmiclatte.hatenablog.com/entry/2020/05/28/003227", - "#category": ("", "hatenablog", "entry"), - "#class" : hatenablog.HatenablogEntryExtractor, - "#count" : 20, -}, - -{ - "#url" : "https://moko0908.hatenablog.jp/entry/2023/12/31/083846", - "#category": ("", "hatenablog", "entry"), - "#class" : hatenablog.HatenablogEntryExtractor, -}, - -{ - "#url" : "https://p-shirokuma.hatenadiary.com/entry/20231227/1703685600", - "#category": ("", "hatenablog", "entry"), - "#class" : hatenablog.HatenablogEntryExtractor, -}, - -{ - "#url" : "https://urakatahero.hateblo.jp/entry/2ndlife", - "#category": ("", "hatenablog", "entry"), - "#class" : hatenablog.HatenablogEntryExtractor, -}, - -{ - "#url" : "hatenablog:https://blog.hyouhon.com/entry/2023/12/22/133549", - "#category": ("", "hatenablog", "entry"), - "#class" : hatenablog.HatenablogEntryExtractor, -}, - -{ - "#url" : "https://cetriolo.hatenablog.com", - "#category": ("", "hatenablog", "home"), - "#class" : hatenablog.HatenablogHomeExtractor, - "#range" : "1-7", - "#count" : 7, -}, - -{ - "#url" : "https://moko0908.hatenablog.jp/", - "#category": ("", "hatenablog", "home"), - "#class" : hatenablog.HatenablogHomeExtractor, -}, - -{ - "#url" : "https://p-shirokuma.hatenadiary.com/", - "#category": ("", "hatenablog", "home"), - "#class" : hatenablog.HatenablogHomeExtractor, -}, - -{ - "#url" : "https://urakatahero.hateblo.jp/", - "#category": ("", "hatenablog", "home"), - "#class" : hatenablog.HatenablogHomeExtractor, -}, - -{ - "#url" : "hatenablog:https://blog.hyouhon.com/", - "#category": ("", "hatenablog", "home"), - "#class" : hatenablog.HatenablogHomeExtractor, -}, - -{ - "#url" : ("https://8saki.hatenablog.com/archive/category/%E3%82%BB%E3" - "%83%AB%E3%83%95%E3%82%B8%E3%82%A7%E3%83%AB%E3%83%8D%E3%82" - "%A4%E3%83%AB"), - "#category": ("", "hatenablog", "archive"), - "#class" : hatenablog.HatenablogArchiveExtractor, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://moko0908.hatenablog.jp/archive/2023", - "#category": ("", "hatenablog", "archive"), - "#class" : hatenablog.HatenablogArchiveExtractor, - "#count" : 13, -}, - -{ - "#url" : "https://p-shirokuma.hatenadiary.com/archive/2023/01", - "#category": ("", "hatenablog", "archive"), - "#class" : hatenablog.HatenablogArchiveExtractor, - "#count" : 5, -}, - -{ - "#url" : "https://urakatahero.hateblo.jp/archive", - "#category": ("", "hatenablog", "archive"), - "#class" : hatenablog.HatenablogArchiveExtractor, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "hatenablog:https://blog.hyouhon.com/archive/2024/01/01", - "#category": ("", "hatenablog", "archive"), - "#class" : hatenablog.HatenablogArchiveExtractor, -}, - -{ - "#url" : "hatenablog:https://blog.hyouhon.com/search?q=a", - "#category": ("", "hatenablog", "search"), - "#class" : hatenablog.HatenablogSearchExtractor, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://cosmiclatte.hatenablog.com/search?q=a", - "#category": ("", "hatenablog", "search"), - "#class" : hatenablog.HatenablogSearchExtractor, -}, - -{ - "#url" : "https://moko0908.hatenablog.jp/search?q=a", - "#category": ("", "hatenablog", "search"), - "#class" : hatenablog.HatenablogSearchExtractor, -}, - -{ - "#url" : "https://p-shirokuma.hatenadiary.com/search?q=a", - "#category": ("", "hatenablog", "search"), - "#class" : hatenablog.HatenablogSearchExtractor, -}, - -{ - "#url" : "https://urakatahero.hateblo.jp/search?q=a", - "#category": ("", "hatenablog", "search"), - "#class" : hatenablog.HatenablogSearchExtractor, -}, - + { + "#url": "https://cosmiclatte.hatenablog.com/entry/2020/05/28/003227", + "#category": ("", "hatenablog", "entry"), + "#class": hatenablog.HatenablogEntryExtractor, + "#count": 20, + }, + { + "#url": "https://moko0908.hatenablog.jp/entry/2023/12/31/083846", + "#category": ("", "hatenablog", "entry"), + "#class": hatenablog.HatenablogEntryExtractor, + }, + { + "#url": "https://p-shirokuma.hatenadiary.com/entry/20231227/1703685600", + "#category": ("", "hatenablog", "entry"), + "#class": hatenablog.HatenablogEntryExtractor, + }, + { + "#url": "https://urakatahero.hateblo.jp/entry/2ndlife", + "#category": ("", "hatenablog", "entry"), + "#class": hatenablog.HatenablogEntryExtractor, + }, + { + "#url": "hatenablog:https://blog.hyouhon.com/entry/2023/12/22/133549", + "#category": ("", "hatenablog", "entry"), + "#class": hatenablog.HatenablogEntryExtractor, + }, + { + "#url": "https://cetriolo.hatenablog.com", + "#category": ("", "hatenablog", "home"), + "#class": hatenablog.HatenablogHomeExtractor, + "#range": "1-7", + "#count": 7, + }, + { + "#url": "https://moko0908.hatenablog.jp/", + "#category": ("", "hatenablog", "home"), + "#class": hatenablog.HatenablogHomeExtractor, + }, + { + "#url": "https://p-shirokuma.hatenadiary.com/", + "#category": ("", "hatenablog", "home"), + "#class": hatenablog.HatenablogHomeExtractor, + }, + { + "#url": "https://urakatahero.hateblo.jp/", + "#category": ("", "hatenablog", "home"), + "#class": hatenablog.HatenablogHomeExtractor, + }, + { + "#url": "hatenablog:https://blog.hyouhon.com/", + "#category": ("", "hatenablog", "home"), + "#class": hatenablog.HatenablogHomeExtractor, + }, + { + "#url": ( + "https://8saki.hatenablog.com/archive/category/%E3%82%BB%E3" + "%83%AB%E3%83%95%E3%82%B8%E3%82%A7%E3%83%AB%E3%83%8D%E3%82" + "%A4%E3%83%AB" + ), + "#category": ("", "hatenablog", "archive"), + "#class": hatenablog.HatenablogArchiveExtractor, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://moko0908.hatenablog.jp/archive/2023", + "#category": ("", "hatenablog", "archive"), + "#class": hatenablog.HatenablogArchiveExtractor, + "#count": 13, + }, + { + "#url": "https://p-shirokuma.hatenadiary.com/archive/2023/01", + "#category": ("", "hatenablog", "archive"), + "#class": hatenablog.HatenablogArchiveExtractor, + "#count": 5, + }, + { + "#url": "https://urakatahero.hateblo.jp/archive", + "#category": ("", "hatenablog", "archive"), + "#class": hatenablog.HatenablogArchiveExtractor, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "hatenablog:https://blog.hyouhon.com/archive/2024/01/01", + "#category": ("", "hatenablog", "archive"), + "#class": hatenablog.HatenablogArchiveExtractor, + }, + { + "#url": "hatenablog:https://blog.hyouhon.com/search?q=a", + "#category": ("", "hatenablog", "search"), + "#class": hatenablog.HatenablogSearchExtractor, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://cosmiclatte.hatenablog.com/search?q=a", + "#category": ("", "hatenablog", "search"), + "#class": hatenablog.HatenablogSearchExtractor, + }, + { + "#url": "https://moko0908.hatenablog.jp/search?q=a", + "#category": ("", "hatenablog", "search"), + "#class": hatenablog.HatenablogSearchExtractor, + }, + { + "#url": "https://p-shirokuma.hatenadiary.com/search?q=a", + "#category": ("", "hatenablog", "search"), + "#class": hatenablog.HatenablogSearchExtractor, + }, + { + "#url": "https://urakatahero.hateblo.jp/search?q=a", + "#category": ("", "hatenablog", "search"), + "#class": hatenablog.HatenablogSearchExtractor, + }, ) diff --git a/test/results/hentai2read.py b/test/results/hentai2read.py index 01349c2ecc..eb8f544daa 100644 --- a/test/results/hentai2read.py +++ b/test/results/hentai2read.py @@ -1,74 +1,64 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hentai2read - __tests__ = ( -{ - "#url" : "https://hentai2read.com/amazon_elixir/1/", - "#category": ("", "hentai2read", "chapter"), - "#class" : hentai2read.Hentai2readChapterExtractor, - "#sha1_url" : "964b942cf492b3a129d2fe2608abfc475bc99e71", - "#sha1_metadata": "85645b02d34aa11b3deb6dadd7536863476e1bad", -}, - -{ - "#url" : "https://hentai2read.com/popuni_kei_joshi_panic/2.5/", - "#category": ("", "hentai2read", "chapter"), - "#class" : hentai2read.Hentai2readChapterExtractor, - "#pattern" : r"https://hentaicdn\.com/hentai/13088/2\.5y/ccdn00\d+\.jpg", - "#count" : 36, - - "author" : "Kurisu", - "chapter" : 2, - "chapter_id" : 75152, - "chapter_minor": ".5", - "count" : 36, - "lang" : "en", - "language" : "English", - "manga" : "Popuni Kei Joshi Panic!", - "manga_id" : 13088, - "page" : int, - "title" : "Popuni Kei Joshi Panic! 2.5", - "type" : "Original", -}, - -{ - "#url" : "https://hentai2read.com/amazon_elixir/", - "#category": ("", "hentai2read", "manga"), - "#class" : hentai2read.Hentai2readMangaExtractor, - "#sha1_url" : "273073752d418ec887d7f7211e42b832e8c403ba", - "#sha1_metadata": "5c1b712258e78e120907121d3987c71f834d13e1", -}, - -{ - "#url" : "https://hentai2read.com/oshikage_riot/", - "#category": ("", "hentai2read", "manga"), - "#class" : hentai2read.Hentai2readMangaExtractor, - "#sha1_url" : "6595f920a3088a15c2819c502862d45f8eb6bea6", - "#sha1_metadata": "a2e9724acb221040d4b29bf9aa8cb75b2240d8af", -}, - -{ - "#url" : "https://hentai2read.com/popuni_kei_joshi_panic/", - "#category": ("", "hentai2read", "manga"), - "#class" : hentai2read.Hentai2readMangaExtractor, - "#pattern" : hentai2read.Hentai2readChapterExtractor.pattern, - "#range" : "2-3", - - "chapter" : int, - "chapter_id" : int, - "chapter_minor": ".5", - "lang" : "en", - "language" : "English", - "manga" : "Popuni Kei Joshi Panic!", - "manga_id" : 13088, - "title" : str, - "type" : "Original", -}, - + { + "#url": "https://hentai2read.com/amazon_elixir/1/", + "#category": ("", "hentai2read", "chapter"), + "#class": hentai2read.Hentai2readChapterExtractor, + "#sha1_url": "964b942cf492b3a129d2fe2608abfc475bc99e71", + "#sha1_metadata": "85645b02d34aa11b3deb6dadd7536863476e1bad", + }, + { + "#url": "https://hentai2read.com/popuni_kei_joshi_panic/2.5/", + "#category": ("", "hentai2read", "chapter"), + "#class": hentai2read.Hentai2readChapterExtractor, + "#pattern": r"https://hentaicdn\.com/hentai/13088/2\.5y/ccdn00\d+\.jpg", + "#count": 36, + "author": "Kurisu", + "chapter": 2, + "chapter_id": 75152, + "chapter_minor": ".5", + "count": 36, + "lang": "en", + "language": "English", + "manga": "Popuni Kei Joshi Panic!", + "manga_id": 13088, + "page": int, + "title": "Popuni Kei Joshi Panic! 2.5", + "type": "Original", + }, + { + "#url": "https://hentai2read.com/amazon_elixir/", + "#category": ("", "hentai2read", "manga"), + "#class": hentai2read.Hentai2readMangaExtractor, + "#sha1_url": "273073752d418ec887d7f7211e42b832e8c403ba", + "#sha1_metadata": "5c1b712258e78e120907121d3987c71f834d13e1", + }, + { + "#url": "https://hentai2read.com/oshikage_riot/", + "#category": ("", "hentai2read", "manga"), + "#class": hentai2read.Hentai2readMangaExtractor, + "#sha1_url": "6595f920a3088a15c2819c502862d45f8eb6bea6", + "#sha1_metadata": "a2e9724acb221040d4b29bf9aa8cb75b2240d8af", + }, + { + "#url": "https://hentai2read.com/popuni_kei_joshi_panic/", + "#category": ("", "hentai2read", "manga"), + "#class": hentai2read.Hentai2readMangaExtractor, + "#pattern": hentai2read.Hentai2readChapterExtractor.pattern, + "#range": "2-3", + "chapter": int, + "chapter_id": int, + "chapter_minor": ".5", + "lang": "en", + "language": "English", + "manga": "Popuni Kei Joshi Panic!", + "manga_id": 13088, + "title": str, + "type": "Original", + }, ) diff --git a/test/results/hentaicosplays.py b/test/results/hentaicosplays.py index 5964b445b0..3afc0fe379 100644 --- a/test/results/hentaicosplays.py +++ b/test/results/hentaicosplays.py @@ -1,57 +1,46 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hentaicosplays - __tests__ = ( -{ - "#url" : "https://hentai-cosplay-xxx.com/image/---devilism--tide-kurihara-/", - "#category": ("", "hentaicosplays", "gallery"), - "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, - "#pattern" : r"https://static\d?\.hentai-cosplay-xxx\.com/upload/\d+/\d+/\d+/\d+\.jpg$", - - "count": 18, - "site" : "hentai-cosplay-xxx", - "slug" : "---devilism--tide-kurihara-", - "title": "艦 こ れ-devilism の tide Kurihara 憂", -}, - -{ - "#url" : "https://hentai-cosplays.com/image/---devilism--tide-kurihara-/", - "#category": ("", "hentaicosplays", "gallery"), - "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, - "#pattern" : r"https://static\d?\.hentai-cosplay-xxx\.com/upload/\d+/\d+/\d+/\d+\.jpg$", - - "count": 18, - "site" : "hentai-cosplays", - "slug" : "---devilism--tide-kurihara-", - "title": "艦 こ れ-devilism の tide Kurihara 憂", -}, - -{ - "#url" : "https://fr.porn-images-xxx.com/image/enako-enako-24/", - "#category": ("", "hentaicosplays", "gallery"), - "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, - "#pattern" : r"https://static\d?.porn-images-xxx.com/upload/\d+/\d+/\d+/\d+.jpg$", - - "count": 11, - "site" : "porn-images-xxx", - "title": str, -}, - -{ - "#url" : "https://ja.hentai-img.com/image/hollow-cora-502/", - "#category": ("", "hentaicosplays", "gallery"), - "#class" : hentaicosplays.HentaicosplaysGalleryExtractor, - "#pattern" : r"https://static\d?.hentai-img.com/upload/\d+/\d+/\d+/\d+.jpg$", - - "count": 2, - "site" : "hentai-img", - "title": str, -}, - + { + "#url": "https://hentai-cosplay-xxx.com/image/---devilism--tide-kurihara-/", + "#category": ("", "hentaicosplays", "gallery"), + "#class": hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern": r"https://static\d?\.hentai-cosplay-xxx\.com/upload/\d+/\d+/\d+/\d+\.jpg$", + "count": 18, + "site": "hentai-cosplay-xxx", + "slug": "---devilism--tide-kurihara-", + "title": "艦 こ れ-devilism の tide Kurihara 憂", + }, + { + "#url": "https://hentai-cosplays.com/image/---devilism--tide-kurihara-/", + "#category": ("", "hentaicosplays", "gallery"), + "#class": hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern": r"https://static\d?\.hentai-cosplay-xxx\.com/upload/\d+/\d+/\d+/\d+\.jpg$", + "count": 18, + "site": "hentai-cosplays", + "slug": "---devilism--tide-kurihara-", + "title": "艦 こ れ-devilism の tide Kurihara 憂", + }, + { + "#url": "https://fr.porn-images-xxx.com/image/enako-enako-24/", + "#category": ("", "hentaicosplays", "gallery"), + "#class": hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern": r"https://static\d?.porn-images-xxx.com/upload/\d+/\d+/\d+/\d+.jpg$", + "count": 11, + "site": "porn-images-xxx", + "title": str, + }, + { + "#url": "https://ja.hentai-img.com/image/hollow-cora-502/", + "#category": ("", "hentaicosplays", "gallery"), + "#class": hentaicosplays.HentaicosplaysGalleryExtractor, + "#pattern": r"https://static\d?.hentai-img.com/upload/\d+/\d+/\d+/\d+.jpg$", + "count": 2, + "site": "hentai-img", + "title": str, + }, ) diff --git a/test/results/hentaifoundry.py b/test/results/hentaifoundry.py index 0335bfee66..a3c0506aa4 100644 --- a/test/results/hentaifoundry.py +++ b/test/results/hentaifoundry.py @@ -1,196 +1,171 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import hentaifoundry import datetime +from gallery_dl.extractor import hentaifoundry __tests__ = ( -{ - "#url" : "https://www.hentai-foundry.com/user/Tenpura/profile", - "#category": ("", "hentaifoundry", "user"), - "#class" : hentaifoundry.HentaifoundryUserExtractor, -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura", - "#category": ("", "hentaifoundry", "pictures"), - "#class" : hentaifoundry.HentaifoundryPicturesExtractor, - "#sha1_url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28", -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura/page/3", - "#category": ("", "hentaifoundry", "pictures"), - "#class" : hentaifoundry.HentaifoundryPicturesExtractor, -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Ethevian/scraps", - "#category": ("", "hentaifoundry", "scraps"), - "#class" : hentaifoundry.HentaifoundryScrapsExtractor, - "#pattern" : r"https://pictures\.hentai-foundry\.com/e/Ethevian/.+", - "#count" : ">= 10", -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps/page/3", - "#category": ("", "hentaifoundry", "scraps"), - "#class" : hentaifoundry.HentaifoundryScrapsExtractor, -}, - -{ - "#url" : "https://www.hentai-foundry.com/user/Tenpura/faves/pictures", - "#category": ("", "hentaifoundry", "favorite"), - "#class" : hentaifoundry.HentaifoundryFavoriteExtractor, - "#sha1_url": "56f9ae2e89fe855e9fe1da9b81e5ec6212b0320b", -}, - -{ - "#url" : "https://www.hentai-foundry.com/user/Tenpura/faves/pictures/page/3", - "#category": ("", "hentaifoundry", "favorite"), - "#class" : hentaifoundry.HentaifoundryFavoriteExtractor, -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/tagged/kancolle", - "#category": ("", "hentaifoundry", "tag"), - "#class" : hentaifoundry.HentaifoundryTagExtractor, - "#pattern" : r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", - "#range" : "20-30", - - "search_tags": "kancolle", -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/recent/2018-09-20", - "#category": ("", "hentaifoundry", "recent"), - "#class" : hentaifoundry.HentaifoundryRecentExtractor, - "#pattern" : r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", - "#range" : "20-30", - - "date": "2018-09-20", -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/popular", - "#category": ("", "hentaifoundry", "popular"), - "#class" : hentaifoundry.HentaifoundryPopularExtractor, - "#pattern" : r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", - "#range" : "20-30", -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura/407501/shimakaze", - "#category": ("", "hentaifoundry", "image"), - "#class" : hentaifoundry.HentaifoundryImageExtractor, - "#sha1_url" : "fbf2fd74906738094e2575d2728e8dc3de18a8a3", - "#sha1_content": "91bf01497c39254b6dfb234a18e8f01629c77fd1", - - "artist" : "Tenpura", - "date" : "dt:2016-02-22 14:41:19", - "description": "Thank you!", - "height" : 700, - "index" : 407501, - "media" : "Other digital art", - "ratings" : [ - "Sexual content", - "Contains female nudity", - ], - "score" : int, - "tags" : [ - "collection", - "kancolle", - "kantai", - "shimakaze", - ], - "title" : "shimakaze", - "user" : "Tenpura", - "views" : int, - "width" : 495, -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Soloid/186714/Osaloop", - "#comment" : "SWF / rumble embed (#4641)", - "#category": ("", "hentaifoundry", "image"), - "#class" : hentaifoundry.HentaifoundryImageExtractor, - "#urls" : "https://pictures.hentai-foundry.com/s/Soloid/186714/Soloid-186714-Osaloop.swf", - - "artist" : "Soloid", - "date" : "dt:2013-02-07 17:25:54", - "description": "It took me ages.\nI hope you'll like it.\nSorry for the bad quality, I made it on after effect because Flash works like shit when you have 44 layers to animate, and the final ae SWF file is 55mo big.", - "extension" : "swf", - "filename" : "Soloid-186714-Osaloop", - "height" : 768, - "index" : 186714, - "media" : "Digital drawing or painting", - "ratings" : [ - "Nudity", - "Sexual content", - "Contains female nudity", - ], - "score" : range(80, 120), - "src" : "https://pictures.hentai-foundry.com/s/Soloid/186714/Soloid-186714-Osaloop.swf", - "tags" : [ - "soloid", - ], - "title" : "Osaloop", - "user" : "Soloid", - "views" : range(45000, 60000), - "width" : 613, -}, - -{ - "#url" : "http://www.hentai-foundry.com/pictures/user/Tenpura/407501/", - "#category": ("", "hentaifoundry", "image"), - "#class" : hentaifoundry.HentaifoundryImageExtractor, - "#pattern" : "http://pictures.hentai-foundry.com/t/Tenpura/407501/", -}, - -{ - "#url" : "https://www.hentai-foundry.com/pictures/user/Tenpura/407501/", - "#category": ("", "hentaifoundry", "image"), - "#class" : hentaifoundry.HentaifoundryImageExtractor, -}, - -{ - "#url" : "https://pictures.hentai-foundry.com/t/Tenpura/407501/Tenpura-407501-shimakaze.png", - "#category": ("", "hentaifoundry", "image"), - "#class" : hentaifoundry.HentaifoundryImageExtractor, -}, - -{ - "#url" : "https://www.hentai-foundry.com/stories/user/SnowWolf35", - "#category": ("", "hentaifoundry", "stories"), - "#class" : hentaifoundry.HentaifoundryStoriesExtractor, - "#count" : ">= 35", - - "author" : "SnowWolf35", - "chapters" : int, - "comments" : int, - "date" : datetime.datetime, - "description": str, - "index" : int, - "rating" : int, - "ratings" : list, - "status" : r"re:(Inc|C)omplete", - "title" : str, - "user" : "SnowWolf35", - "views" : int, - "words" : int, -}, - -{ - "#url" : "https://www.hentai-foundry.com/stories/user/SnowWolf35/26416/Overwatch-High-Chapter-Voting-Location", - "#category": ("", "hentaifoundry", "story"), - "#class" : hentaifoundry.HentaifoundryStoryExtractor, - "#sha1_url": "5a67cfa8c3bf7634c8af8485dd07c1ea74ee0ae8", - - "title": "Overwatch High Chapter Voting Location", -}, - + { + "#url": "https://www.hentai-foundry.com/user/Tenpura/profile", + "#category": ("", "hentaifoundry", "user"), + "#class": hentaifoundry.HentaifoundryUserExtractor, + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Tenpura", + "#category": ("", "hentaifoundry", "pictures"), + "#class": hentaifoundry.HentaifoundryPicturesExtractor, + "#sha1_url": "ebbc981a85073745e3ca64a0f2ab31fab967fc28", + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Tenpura/page/3", + "#category": ("", "hentaifoundry", "pictures"), + "#class": hentaifoundry.HentaifoundryPicturesExtractor, + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Ethevian/scraps", + "#category": ("", "hentaifoundry", "scraps"), + "#class": hentaifoundry.HentaifoundryScrapsExtractor, + "#pattern": r"https://pictures\.hentai-foundry\.com/e/Ethevian/.+", + "#count": ">= 10", + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Evulchibi/scraps/page/3", + "#category": ("", "hentaifoundry", "scraps"), + "#class": hentaifoundry.HentaifoundryScrapsExtractor, + }, + { + "#url": "https://www.hentai-foundry.com/user/Tenpura/faves/pictures", + "#category": ("", "hentaifoundry", "favorite"), + "#class": hentaifoundry.HentaifoundryFavoriteExtractor, + "#sha1_url": "56f9ae2e89fe855e9fe1da9b81e5ec6212b0320b", + }, + { + "#url": "https://www.hentai-foundry.com/user/Tenpura/faves/pictures/page/3", + "#category": ("", "hentaifoundry", "favorite"), + "#class": hentaifoundry.HentaifoundryFavoriteExtractor, + }, + { + "#url": "https://www.hentai-foundry.com/pictures/tagged/kancolle", + "#category": ("", "hentaifoundry", "tag"), + "#class": hentaifoundry.HentaifoundryTagExtractor, + "#pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", + "#range": "20-30", + "search_tags": "kancolle", + }, + { + "#url": "https://www.hentai-foundry.com/pictures/recent/2018-09-20", + "#category": ("", "hentaifoundry", "recent"), + "#class": hentaifoundry.HentaifoundryRecentExtractor, + "#pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", + "#range": "20-30", + "date": "2018-09-20", + }, + { + "#url": "https://www.hentai-foundry.com/pictures/popular", + "#category": ("", "hentaifoundry", "popular"), + "#class": hentaifoundry.HentaifoundryPopularExtractor, + "#pattern": r"https://pictures.hentai-foundry.com/[^/]/[^/?#]+/\d+/", + "#range": "20-30", + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Tenpura/407501/shimakaze", + "#category": ("", "hentaifoundry", "image"), + "#class": hentaifoundry.HentaifoundryImageExtractor, + "#sha1_url": "fbf2fd74906738094e2575d2728e8dc3de18a8a3", + "#sha1_content": "91bf01497c39254b6dfb234a18e8f01629c77fd1", + "artist": "Tenpura", + "date": "dt:2016-02-22 14:41:19", + "description": "Thank you!", + "height": 700, + "index": 407501, + "media": "Other digital art", + "ratings": [ + "Sexual content", + "Contains female nudity", + ], + "score": int, + "tags": [ + "collection", + "kancolle", + "kantai", + "shimakaze", + ], + "title": "shimakaze", + "user": "Tenpura", + "views": int, + "width": 495, + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Soloid/186714/Osaloop", + "#comment": "SWF / rumble embed (#4641)", + "#category": ("", "hentaifoundry", "image"), + "#class": hentaifoundry.HentaifoundryImageExtractor, + "#urls": "https://pictures.hentai-foundry.com/s/Soloid/186714/Soloid-186714-Osaloop.swf", + "artist": "Soloid", + "date": "dt:2013-02-07 17:25:54", + "description": "It took me ages.\nI hope you'll like it.\nSorry for the bad quality, I made it on after effect because Flash works like shit when you have 44 layers to animate, and the final ae SWF file is 55mo big.", + "extension": "swf", + "filename": "Soloid-186714-Osaloop", + "height": 768, + "index": 186714, + "media": "Digital drawing or painting", + "ratings": [ + "Nudity", + "Sexual content", + "Contains female nudity", + ], + "score": range(80, 120), + "src": "https://pictures.hentai-foundry.com/s/Soloid/186714/Soloid-186714-Osaloop.swf", + "tags": [ + "soloid", + ], + "title": "Osaloop", + "user": "Soloid", + "views": range(45000, 60000), + "width": 613, + }, + { + "#url": "http://www.hentai-foundry.com/pictures/user/Tenpura/407501/", + "#category": ("", "hentaifoundry", "image"), + "#class": hentaifoundry.HentaifoundryImageExtractor, + "#pattern": "http://pictures.hentai-foundry.com/t/Tenpura/407501/", + }, + { + "#url": "https://www.hentai-foundry.com/pictures/user/Tenpura/407501/", + "#category": ("", "hentaifoundry", "image"), + "#class": hentaifoundry.HentaifoundryImageExtractor, + }, + { + "#url": "https://pictures.hentai-foundry.com/t/Tenpura/407501/Tenpura-407501-shimakaze.png", + "#category": ("", "hentaifoundry", "image"), + "#class": hentaifoundry.HentaifoundryImageExtractor, + }, + { + "#url": "https://www.hentai-foundry.com/stories/user/SnowWolf35", + "#category": ("", "hentaifoundry", "stories"), + "#class": hentaifoundry.HentaifoundryStoriesExtractor, + "#count": ">= 35", + "author": "SnowWolf35", + "chapters": int, + "comments": int, + "date": datetime.datetime, + "description": str, + "index": int, + "rating": int, + "ratings": list, + "status": r"re:(Inc|C)omplete", + "title": str, + "user": "SnowWolf35", + "views": int, + "words": int, + }, + { + "#url": "https://www.hentai-foundry.com/stories/user/SnowWolf35/26416/Overwatch-High-Chapter-Voting-Location", + "#category": ("", "hentaifoundry", "story"), + "#class": hentaifoundry.HentaifoundryStoryExtractor, + "#sha1_url": "5a67cfa8c3bf7634c8af8485dd07c1ea74ee0ae8", + "title": "Overwatch High Chapter Voting Location", + }, ) diff --git a/test/results/hentaifox.py b/test/results/hentaifox.py index 123bd27748..385df8a2ce 100644 --- a/test/results/hentaifox.py +++ b/test/results/hentaifox.py @@ -1,108 +1,93 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hentaifox - __tests__ = ( -{ - "#url" : "https://hentaifox.com/gallery/56622/", - "#category": ("", "hentaifox", "gallery"), - "#class" : hentaifox.HentaifoxGalleryExtractor, - "#pattern" : r"https://i\d*\.hentaifox\.com/\d+/\d+/\d+\.jpg", - "#count" : 24, - "#sha1_metadata": "bcd6b67284f378e5cc30b89b761140e3e60fcd92", -}, - -{ - "#url" : "https://hentaifox.com/gallery/630/", - "#comment" : "'split_tag' element (#1378)", - "#category": ("", "hentaifox", "gallery"), - "#class" : hentaifox.HentaifoxGalleryExtractor, - - "artist" : [ - "beti", - "betty", - "magi", - "mimikaki", - ], - "characters": [ - "aerith gainsborough", - "tifa lockhart", - "yuffie kisaragi", - ], - "count" : 32, - "gallery_id": 630, - "group" : ["cu-little2"], - "parody" : [ - "darkstalkers | vampire", - "final fantasy vii", - ], - "tags" : [ - "femdom", - "fingering", - "masturbation", - "yuri", - ], - "title" : "Cu-Little Bakanya～", - "type" : "doujinshi", -}, - -{ - "#url" : "https://hentaifox.com/gallery/35261/", - "#comment" : "email-protected title (#4201)", - "#category": ("", "hentaifox", "gallery"), - "#class" : hentaifox.HentaifoxGalleryExtractor, - - "gallery_id": 35261, - "title" : "ManageM@ster!", - "artist" : ["haritama hiroki"], - "group" : ["studio n.ball"], -}, - -{ - "#url" : "https://hentaifox.com/parody/touhou-project/", - "#category": ("", "hentaifox", "search"), - "#class" : hentaifox.HentaifoxSearchExtractor, -}, - -{ - "#url" : "https://hentaifox.com/character/reimu-hakurei/", - "#category": ("", "hentaifox", "search"), - "#class" : hentaifox.HentaifoxSearchExtractor, -}, - -{ - "#url" : "https://hentaifox.com/artist/distance/", - "#category": ("", "hentaifox", "search"), - "#class" : hentaifox.HentaifoxSearchExtractor, -}, - -{ - "#url" : "https://hentaifox.com/search/touhou/", - "#category": ("", "hentaifox", "search"), - "#class" : hentaifox.HentaifoxSearchExtractor, -}, - -{ - "#url" : "https://hentaifox.com/group/v-slash/", - "#category": ("", "hentaifox", "search"), - "#class" : hentaifox.HentaifoxSearchExtractor, -}, - -{ - "#url" : "https://hentaifox.com/tag/heterochromia/", - "#category": ("", "hentaifox", "search"), - "#class" : hentaifox.HentaifoxSearchExtractor, - "#pattern" : hentaifox.HentaifoxGalleryExtractor.pattern, - "#count" : ">= 60", - - "url" : str, - "gallery_id": int, - "title" : str, -}, - + { + "#url": "https://hentaifox.com/gallery/56622/", + "#category": ("", "hentaifox", "gallery"), + "#class": hentaifox.HentaifoxGalleryExtractor, + "#pattern": r"https://i\d*\.hentaifox\.com/\d+/\d+/\d+\.jpg", + "#count": 24, + "#sha1_metadata": "bcd6b67284f378e5cc30b89b761140e3e60fcd92", + }, + { + "#url": "https://hentaifox.com/gallery/630/", + "#comment": "'split_tag' element (#1378)", + "#category": ("", "hentaifox", "gallery"), + "#class": hentaifox.HentaifoxGalleryExtractor, + "artist": [ + "beti", + "betty", + "magi", + "mimikaki", + ], + "characters": [ + "aerith gainsborough", + "tifa lockhart", + "yuffie kisaragi", + ], + "count": 32, + "gallery_id": 630, + "group": ["cu-little2"], + "parody": [ + "darkstalkers | vampire", + "final fantasy vii", + ], + "tags": [ + "femdom", + "fingering", + "masturbation", + "yuri", + ], + "title": "Cu-Little Bakanya～", + "type": "doujinshi", + }, + { + "#url": "https://hentaifox.com/gallery/35261/", + "#comment": "email-protected title (#4201)", + "#category": ("", "hentaifox", "gallery"), + "#class": hentaifox.HentaifoxGalleryExtractor, + "gallery_id": 35261, + "title": "ManageM@ster!", + "artist": ["haritama hiroki"], + "group": ["studio n.ball"], + }, + { + "#url": "https://hentaifox.com/parody/touhou-project/", + "#category": ("", "hentaifox", "search"), + "#class": hentaifox.HentaifoxSearchExtractor, + }, + { + "#url": "https://hentaifox.com/character/reimu-hakurei/", + "#category": ("", "hentaifox", "search"), + "#class": hentaifox.HentaifoxSearchExtractor, + }, + { + "#url": "https://hentaifox.com/artist/distance/", + "#category": ("", "hentaifox", "search"), + "#class": hentaifox.HentaifoxSearchExtractor, + }, + { + "#url": "https://hentaifox.com/search/touhou/", + "#category": ("", "hentaifox", "search"), + "#class": hentaifox.HentaifoxSearchExtractor, + }, + { + "#url": "https://hentaifox.com/group/v-slash/", + "#category": ("", "hentaifox", "search"), + "#class": hentaifox.HentaifoxSearchExtractor, + }, + { + "#url": "https://hentaifox.com/tag/heterochromia/", + "#category": ("", "hentaifox", "search"), + "#class": hentaifox.HentaifoxSearchExtractor, + "#pattern": hentaifox.HentaifoxGalleryExtractor.pattern, + "#count": ">= 60", + "url": str, + "gallery_id": int, + "title": str, + }, ) diff --git a/test/results/hentaihand.py b/test/results/hentaihand.py index 3d9a34b6d1..610596454a 100644 --- a/test/results/hentaihand.py +++ b/test/results/hentaihand.py @@ -1,57 +1,48 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hentaihand - __tests__ = ( -{ - "#url" : "https://hentaihand.com/en/comic/c75-takumi-na-muchi-choudenji-hou-no-aishi-kata-how-to-love-a-super-electromagnetic-gun-toaru-kagaku-no-railgun-english", - "#category": ("", "hentaihand", "gallery"), - "#class" : hentaihand.HentaihandGalleryExtractor, - "#pattern" : r"https://cdn.hentaihand.com/.*/images/37387/\d+.jpg$", - "#count" : 50, - - "artists" : ["Takumi Na Muchi"], - "date" : "dt:2014-06-28 00:00:00", - "gallery_id" : 37387, - "lang" : "en", - "language" : "English", - "parodies" : ["Toaru Kagaku No Railgun"], - "relationships": list, - "tags" : list, - "title" : r"re:\(C75\) \[Takumi na Muchi\] Choudenji Hou ", - "title_alt" : r"re:\(C75\) \[たくみなむち\] 超電磁砲のあいしかた", - "type" : "Doujinshi", -}, - -{ - "#url" : "https://hentaihand.com/en/artist/takumi-na-muchi", - "#category": ("", "hentaihand", "tag"), - "#class" : hentaihand.HentaihandTagExtractor, - "#pattern" : hentaihand.HentaihandGalleryExtractor.pattern, - "#count" : ">= 6", -}, - -{ - "#url" : "https://hentaihand.com/en/tag/full-color", - "#category": ("", "hentaihand", "tag"), - "#class" : hentaihand.HentaihandTagExtractor, -}, - -{ - "#url" : "https://hentaihand.com/fr/language/japanese", - "#category": ("", "hentaihand", "tag"), - "#class" : hentaihand.HentaihandTagExtractor, -}, - -{ - "#url" : "https://hentaihand.com/zh/category/manga", - "#category": ("", "hentaihand", "tag"), - "#class" : hentaihand.HentaihandTagExtractor, -}, - + { + "#url": "https://hentaihand.com/en/comic/c75-takumi-na-muchi-choudenji-hou-no-aishi-kata-how-to-love-a-super-electromagnetic-gun-toaru-kagaku-no-railgun-english", + "#category": ("", "hentaihand", "gallery"), + "#class": hentaihand.HentaihandGalleryExtractor, + "#pattern": r"https://cdn.hentaihand.com/.*/images/37387/\d+.jpg$", + "#count": 50, + "artists": ["Takumi Na Muchi"], + "date": "dt:2014-06-28 00:00:00", + "gallery_id": 37387, + "lang": "en", + "language": "English", + "parodies": ["Toaru Kagaku No Railgun"], + "relationships": list, + "tags": list, + "title": r"re:\(C75\) \[Takumi na Muchi\] Choudenji Hou ", + "title_alt": r"re:\(C75\) \[たくみなむち\] 超電磁砲のあいしかた", + "type": "Doujinshi", + }, + { + "#url": "https://hentaihand.com/en/artist/takumi-na-muchi", + "#category": ("", "hentaihand", "tag"), + "#class": hentaihand.HentaihandTagExtractor, + "#pattern": hentaihand.HentaihandGalleryExtractor.pattern, + "#count": ">= 6", + }, + { + "#url": "https://hentaihand.com/en/tag/full-color", + "#category": ("", "hentaihand", "tag"), + "#class": hentaihand.HentaihandTagExtractor, + }, + { + "#url": "https://hentaihand.com/fr/language/japanese", + "#category": ("", "hentaihand", "tag"), + "#class": hentaihand.HentaihandTagExtractor, + }, + { + "#url": "https://hentaihand.com/zh/category/manga", + "#category": ("", "hentaihand", "tag"), + "#class": hentaihand.HentaihandTagExtractor, + }, ) diff --git a/test/results/hentaihere.py b/test/results/hentaihere.py index abe52fe91f..659388efb9 100644 --- a/test/results/hentaihere.py +++ b/test/results/hentaihere.py @@ -1,65 +1,56 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hentaihere - __tests__ = ( -{ - "#url" : "https://hentaihere.com/m/S13812/1/1/", - "#category": ("", "hentaihere", "chapter"), - "#class" : hentaihere.HentaihereChapterExtractor, - "#sha1_url" : "964b942cf492b3a129d2fe2608abfc475bc99e71", - "#sha1_metadata": "0207d20eea3a15d2a8d1496755bdfa49de7cfa9d", -}, - -{ - "#url" : "https://hentaihere.com/m/S23048/1.5/1/", - "#category": ("", "hentaihere", "chapter"), - "#class" : hentaihere.HentaihereChapterExtractor, - "#pattern" : r"https://hentaicdn\.com/hentai/23048/1\.5/ccdn00\d+\.jpg", - "#count" : 32, - - "author" : "Shinozuka Yuuji", - "chapter" : 1, - "chapter_id" : 80186, - "chapter_minor": ".5", - "count" : 32, - "lang" : "en", - "language" : "English", - "manga" : "High School Slut's Love Consultation", - "manga_id" : 23048, - "page" : int, - "title" : "High School Slut's Love Consultation + Girlfriend [Full Color]", - "type" : "Original", -}, - -{ - "#url" : "https://hentaihere.com/m/S13812", - "#category": ("", "hentaihere", "manga"), - "#class" : hentaihere.HentaihereMangaExtractor, - "#sha1_url" : "d1ba6e28bb2162e844f8559c2b2725ba0a093559", - "#sha1_metadata": "5c1b712258e78e120907121d3987c71f834d13e1", -}, - -{ - "#url" : "https://hentaihere.com/m/S7608", - "#category": ("", "hentaihere", "manga"), - "#class" : hentaihere.HentaihereMangaExtractor, - "#sha1_url": "6c5239758dc93f6b1b4175922836c10391b174f7", - - "chapter" : int, - "chapter_id" : int, - "chapter_minor": "", - "lang" : "en", - "language" : "English", - "manga" : "Oshikake Riot", - "manga_id" : 7608, - "title" : r"re:Oshikake Riot( \d+)?", - "type" : "Original", -}, - + { + "#url": "https://hentaihere.com/m/S13812/1/1/", + "#category": ("", "hentaihere", "chapter"), + "#class": hentaihere.HentaihereChapterExtractor, + "#sha1_url": "964b942cf492b3a129d2fe2608abfc475bc99e71", + "#sha1_metadata": "0207d20eea3a15d2a8d1496755bdfa49de7cfa9d", + }, + { + "#url": "https://hentaihere.com/m/S23048/1.5/1/", + "#category": ("", "hentaihere", "chapter"), + "#class": hentaihere.HentaihereChapterExtractor, + "#pattern": r"https://hentaicdn\.com/hentai/23048/1\.5/ccdn00\d+\.jpg", + "#count": 32, + "author": "Shinozuka Yuuji", + "chapter": 1, + "chapter_id": 80186, + "chapter_minor": ".5", + "count": 32, + "lang": "en", + "language": "English", + "manga": "High School Slut's Love Consultation", + "manga_id": 23048, + "page": int, + "title": "High School Slut's Love Consultation + Girlfriend [Full Color]", + "type": "Original", + }, + { + "#url": "https://hentaihere.com/m/S13812", + "#category": ("", "hentaihere", "manga"), + "#class": hentaihere.HentaihereMangaExtractor, + "#sha1_url": "d1ba6e28bb2162e844f8559c2b2725ba0a093559", + "#sha1_metadata": "5c1b712258e78e120907121d3987c71f834d13e1", + }, + { + "#url": "https://hentaihere.com/m/S7608", + "#category": ("", "hentaihere", "manga"), + "#class": hentaihere.HentaihereMangaExtractor, + "#sha1_url": "6c5239758dc93f6b1b4175922836c10391b174f7", + "chapter": int, + "chapter_id": int, + "chapter_minor": "", + "lang": "en", + "language": "English", + "manga": "Oshikake Riot", + "manga_id": 7608, + "title": r"re:Oshikake Riot( \d+)?", + "type": "Original", + }, ) diff --git a/test/results/hentainexus.py b/test/results/hentainexus.py index 01091350d1..3b3e583578 100644 --- a/test/results/hentainexus.py +++ b/test/results/hentainexus.py @@ -1,81 +1,72 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hentainexus - __tests__ = ( -{ - "#url" : "https://hentainexus.com/view/5688", - "#category": ("", "hentainexus", "gallery"), - "#class" : hentainexus.HentainexusGalleryExtractor, - - "artist" : "Tsukiriran", - "book" : "", - "circle" : "", - "count" : 4, - "cover" : str, - "description": "The cherry blossom blooms for one final graduation memory. ❤", - "event" : "", - "extension" : "png", - "filename" : str, - "gallery_id" : 5688, - "image" : str, - "label" : str, - "lang" : "en", - "language" : "English", - "magazine" : "Comic Bavel 2018-08", - "num" : range(1, 4), - "parody" : "Original Work", - "publisher" : "FAKKU", - "tags" : [ - "busty", - "color", - "creampie", - "exhibitionism", - "hentai", - "kimono", - "pubic hair", - "uncensored", - "unlimited", - "vanilla", - ], - "title" : "Graduation!", - "title_conventional": "[Tsukiriran] Graduation! (Comic Bavel 2018-08)", - "type" : "image", - "url_label" : str, -}, - -{ - "#url" : "https://hentainexus.com/read/5688", - "#category": ("", "hentainexus", "gallery"), - "#class" : hentainexus.HentainexusGalleryExtractor, -}, - -{ - "#url" : "https://hentainexus.com/view/715", - "#comment" : "combined left-right pages (#5827)", - "#category": ("", "hentainexus", "gallery"), - "#class" : hentainexus.HentainexusGalleryExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://hentainexus.com/?q=tag:%22heart+pupils%22%20tag:group", - "#category": ("", "hentainexus", "search"), - "#class" : hentainexus.HentainexusSearchExtractor, - "#pattern" : hentainexus.HentainexusGalleryExtractor.pattern, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://hentainexus.com/page/3?q=tag:%22heart+pupils%22", - "#category": ("", "hentainexus", "search"), - "#class" : hentainexus.HentainexusSearchExtractor, -}, - + { + "#url": "https://hentainexus.com/view/5688", + "#category": ("", "hentainexus", "gallery"), + "#class": hentainexus.HentainexusGalleryExtractor, + "artist": "Tsukiriran", + "book": "", + "circle": "", + "count": 4, + "cover": str, + "description": "The cherry blossom blooms for one final graduation memory. ❤", + "event": "", + "extension": "png", + "filename": str, + "gallery_id": 5688, + "image": str, + "label": str, + "lang": "en", + "language": "English", + "magazine": "Comic Bavel 2018-08", + "num": range(1, 4), + "parody": "Original Work", + "publisher": "FAKKU", + "tags": [ + "busty", + "color", + "creampie", + "exhibitionism", + "hentai", + "kimono", + "pubic hair", + "uncensored", + "unlimited", + "vanilla", + ], + "title": "Graduation!", + "title_conventional": "[Tsukiriran] Graduation! (Comic Bavel 2018-08)", + "type": "image", + "url_label": str, + }, + { + "#url": "https://hentainexus.com/read/5688", + "#category": ("", "hentainexus", "gallery"), + "#class": hentainexus.HentainexusGalleryExtractor, + }, + { + "#url": "https://hentainexus.com/view/715", + "#comment": "combined left-right pages (#5827)", + "#category": ("", "hentainexus", "gallery"), + "#class": hentainexus.HentainexusGalleryExtractor, + "#count": 2, + }, + { + "#url": "https://hentainexus.com/?q=tag:%22heart+pupils%22%20tag:group", + "#category": ("", "hentainexus", "search"), + "#class": hentainexus.HentainexusSearchExtractor, + "#pattern": hentainexus.HentainexusGalleryExtractor.pattern, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://hentainexus.com/page/3?q=tag:%22heart+pupils%22", + "#category": ("", "hentainexus", "search"), + "#class": hentainexus.HentainexusSearchExtractor, + }, ) diff --git a/test/results/hiperdex.py b/test/results/hiperdex.py index a65c0928fb..736b0c9627 100644 --- a/test/results/hiperdex.py +++ b/test/results/hiperdex.py @@ -1,148 +1,125 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hiperdex - __tests__ = ( -{ - "#url" : "https://hipertoon.com/manga/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, - "#pattern" : r"https://(1st)?hiper(dex|toon)\d?.(com|net|info|top)/wp-content/uploads/WP-manga/data/manga_\w+/[0-9a-f]{32}/\d+\.webp", - "#count" : 9, - - "artist" : "Sasuga Kei", - "author" : "Sasuga Kei", - "chapter" : 154, - "chapter_minor": ".5", - "description" : "Natsuo Fujii is in love with his teacher, Hina. Attempting to forget his feelings towards her, Natsuo goes to a mixer with his classmates where he meets an odd girl named Rui Tachibana. In a strange turn of events, Rui asks Natsuo to sneak out with her and do her a favor. To his surprise, their destination is Rui’s house—and her request is for him to have sex with her. There’s no love behind the act; she just wants to learn from the experience. Thinking that it might help him forget about Hina, Natsuo hesitantly agrees. After this unusual encounter Natsuo now faces a new problem. With his father remarrying, he ends up with a new pair of stepsisters; unfortunately, he knows these two girls all too well. He soon finds out his new siblings are none other than Hina and Rui! Now living with both the teacher he loves and the girl with whom he had his “first time,” Natsuo finds himself in an unexpected love triangle as he climbs ever closer towards adulthood.", - "genre" : list, - "manga" : "Domestic na Kanojo", - "release" : 2014, - "score" : float, - "type" : "Manga", -}, - -{ - "#url" : "https://hiperdex.com/mangas/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, -}, - -{ - "#url" : "https://1sthiperdex.com/manga/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, -}, - -{ - "#url" : "https://hiperdex2.com/manga/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, -}, - -{ - "#url" : "https://hiperdex.net/manga/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, -}, - -{ - "#url" : "https://hiperdex.info/manga/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, -}, - -{ - "#url" : "https://hiperdex.top/manga/domestic-na-kanojo/154-5/", - "#category": ("", "hiperdex", "chapter"), - "#class" : hiperdex.HiperdexChapterExtractor, -}, - -{ - "#url" : "https://hiperdex.com/manga/1603231576-youre-not-that-special/", - "#category": ("", "hiperdex", "manga"), - "#class" : hiperdex.HiperdexMangaExtractor, - "#pattern" : hiperdex.HiperdexChapterExtractor.pattern, - "#count" : 51, - - "artist" : "Bolp", - "author" : "Abyo4", - "chapter" : int, - "chapter_minor": "", - "description" : r"re:I didn’t think much of the creepy girl in ", - "genre" : list, - "manga" : "You’re Not That Special!", - "release" : 2019, - "score" : float, - "status" : "Completed", - "type" : "Manhwa", -}, - -{ - "#url" : "https://hiperdex.com/manga/youre-not-that-special/", - "#category": ("", "hiperdex", "manga"), - "#class" : hiperdex.HiperdexMangaExtractor, -}, - -{ - "#url" : "https://1sthiperdex.com/manga/youre-not-that-special/", - "#category": ("", "hiperdex", "manga"), - "#class" : hiperdex.HiperdexMangaExtractor, -}, - -{ - "#url" : "https://hiperdex2.com/manga/youre-not-that-special/", - "#category": ("", "hiperdex", "manga"), - "#class" : hiperdex.HiperdexMangaExtractor, -}, - -{ - "#url" : "https://hiperdex.net/manga/youre-not-that-special/", - "#category": ("", "hiperdex", "manga"), - "#class" : hiperdex.HiperdexMangaExtractor, -}, - -{ - "#url" : "https://hiperdex.info/manga/youre-not-that-special/", - "#category": ("", "hiperdex", "manga"), - "#class" : hiperdex.HiperdexMangaExtractor, -}, - -{ - "#url" : "https://1sthiperdex.com/manga-artist/beck-ho-an/", - "#category": ("", "hiperdex", "artist"), - "#class" : hiperdex.HiperdexArtistExtractor, -}, - -{ - "#url" : "https://hiperdex.net/manga-artist/beck-ho-an/", - "#category": ("", "hiperdex", "artist"), - "#class" : hiperdex.HiperdexArtistExtractor, -}, - -{ - "#url" : "https://hiperdex2.com/manga-artist/beck-ho-an/", - "#category": ("", "hiperdex", "artist"), - "#class" : hiperdex.HiperdexArtistExtractor, -}, - -{ - "#url" : "https://hiperdex.info/manga-artist/beck-ho-an/", - "#category": ("", "hiperdex", "artist"), - "#class" : hiperdex.HiperdexArtistExtractor, -}, - -{ - "#url" : "https://hiperdex.com/manga-author/viagra/", - "#category": ("", "hiperdex", "artist"), - "#class" : hiperdex.HiperdexArtistExtractor, - "#pattern" : hiperdex.HiperdexMangaExtractor.pattern, - "#count" : ">= 6", -}, - + { + "#url": "https://hipertoon.com/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + "#pattern": r"https://(1st)?hiper(dex|toon)\d?.(com|net|info|top)/wp-content/uploads/WP-manga/data/manga_\w+/[0-9a-f]{32}/\d+\.webp", + "#count": 9, + "artist": "Sasuga Kei", + "author": "Sasuga Kei", + "chapter": 154, + "chapter_minor": ".5", + "description": "Natsuo Fujii is in love with his teacher, Hina. Attempting to forget his feelings towards her, Natsuo goes to a mixer with his classmates where he meets an odd girl named Rui Tachibana. In a strange turn of events, Rui asks Natsuo to sneak out with her and do her a favor. To his surprise, their destination is Rui’s house—and her request is for him to have sex with her. There’s no love behind the act; she just wants to learn from the experience. Thinking that it might help him forget about Hina, Natsuo hesitantly agrees. After this unusual encounter Natsuo now faces a new problem. With his father remarrying, he ends up with a new pair of stepsisters; unfortunately, he knows these two girls all too well. He soon finds out his new siblings are none other than Hina and Rui! Now living with both the teacher he loves and the girl with whom he had his “first time,” Natsuo finds himself in an unexpected love triangle as he climbs ever closer towards adulthood.", + "genre": list, + "manga": "Domestic na Kanojo", + "release": 2014, + "score": float, + "type": "Manga", + }, + { + "#url": "https://hiperdex.com/mangas/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + }, + { + "#url": "https://1sthiperdex.com/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + }, + { + "#url": "https://hiperdex2.com/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + }, + { + "#url": "https://hiperdex.net/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + }, + { + "#url": "https://hiperdex.info/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + }, + { + "#url": "https://hiperdex.top/manga/domestic-na-kanojo/154-5/", + "#category": ("", "hiperdex", "chapter"), + "#class": hiperdex.HiperdexChapterExtractor, + }, + { + "#url": "https://hiperdex.com/manga/1603231576-youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class": hiperdex.HiperdexMangaExtractor, + "#pattern": hiperdex.HiperdexChapterExtractor.pattern, + "#count": 51, + "artist": "Bolp", + "author": "Abyo4", + "chapter": int, + "chapter_minor": "", + "description": r"re:I didn’t think much of the creepy girl in ", + "genre": list, + "manga": "You’re Not That Special!", + "release": 2019, + "score": float, + "status": "Completed", + "type": "Manhwa", + }, + { + "#url": "https://hiperdex.com/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class": hiperdex.HiperdexMangaExtractor, + }, + { + "#url": "https://1sthiperdex.com/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class": hiperdex.HiperdexMangaExtractor, + }, + { + "#url": "https://hiperdex2.com/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class": hiperdex.HiperdexMangaExtractor, + }, + { + "#url": "https://hiperdex.net/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class": hiperdex.HiperdexMangaExtractor, + }, + { + "#url": "https://hiperdex.info/manga/youre-not-that-special/", + "#category": ("", "hiperdex", "manga"), + "#class": hiperdex.HiperdexMangaExtractor, + }, + { + "#url": "https://1sthiperdex.com/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class": hiperdex.HiperdexArtistExtractor, + }, + { + "#url": "https://hiperdex.net/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class": hiperdex.HiperdexArtistExtractor, + }, + { + "#url": "https://hiperdex2.com/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class": hiperdex.HiperdexArtistExtractor, + }, + { + "#url": "https://hiperdex.info/manga-artist/beck-ho-an/", + "#category": ("", "hiperdex", "artist"), + "#class": hiperdex.HiperdexArtistExtractor, + }, + { + "#url": "https://hiperdex.com/manga-author/viagra/", + "#category": ("", "hiperdex", "artist"), + "#class": hiperdex.HiperdexArtistExtractor, + "#pattern": hiperdex.HiperdexMangaExtractor.pattern, + "#count": ">= 6", + }, ) diff --git a/test/results/hitomi.py b/test/results/hitomi.py index 1b0ffcba85..c5b6cead29 100644 --- a/test/results/hitomi.py +++ b/test/results/hitomi.py @@ -1,234 +1,207 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import hitomi - __tests__ = ( -{ - "#url" : "https://hitomi.la/galleries/867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, - "#pattern" : r"https://[a-c]a\.hitomi\.la/webp/\d+/\d+/[0-9a-f]{64}\.webp", - "#count" : 16, - - "artist" : ["morris"], - "characters": [], - "count" : 16, - "date" : "dt:2015-10-27 19:20:00", - "extension" : "webp", - "extension_original" : "jpg", - "filename" : str, - "gallery_id": 867789, - "group" : [], - "lang" : "en", - "language" : "English", - "num" : range(1, 16), - "parody" : [], - "tags" : [ - "Cheating ♀", - "Drugs ♀", - "Drugs ♂", - "Incest", - "Milf ♀", - "Mother ♀", - "Sole Female ♀", - "Sole Male ♂", - "Uncensored" - ], - "title" : "Amazon no Hiyaku | Amazon Elixir (decensored)", - "title_jpn" : "", - "type" : "Manga", -}, - -{ - "#url" : "https://hitomi.la/galleries/1401410.html", - "#comment" : "download test", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, - "#range" : "1", - "#sha1_content": "d75d5a3d1302a48469016b20e53c26b714d17745", -}, - -{ - "#url" : "https://hitomi.la/galleries/733697.html", - "#comment" : "Game CG with scenes (#321)", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, - "#count" : 210, -}, - -{ - "#url" : "https://hitomi.la/galleries/1045954.html", - "#comment" : "fallback for galleries only available through /reader/ URLs", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, - "#count" : 1413, -}, - -{ - "#url" : "https://hitomi.la/cg/scathacha-sama-okuchi-ecchi-1291900.html", - "#comment" : "gallery with 'broken' redirect", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/cg/1615823.html", - "#comment" : "no tags", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, - "#options" : {"format": "avif"}, - "#pattern" : r"https://[a-c]a\.hitomi\.la/avif/\d+/\d+/[0-9a-f]{64}\.avif", - "#count" : 22, - - "artist" : ["sorairo len"], - "characters": [], - "count" : 22, - "date" : "dt:2020-04-19 06:33:00", - "extension" : "avif", - "filename" : str, - "gallery_id": 1615823, - "group" : [], - "lang" : "ja", - "language" : "Japanese", - "num" : range(1, 22), - "parody" : [], - "tags" : [ - "Blowjob ♀", - "Focus Blowjob ♀", - "Fox Girl ♀", - "Kemonomimi ♀", - "Loli ♀", - "Miko ♀", - "No Penetration", - "Unusual Pupils ♀", - "Variant Set" - ], - "title" : "Kouko-sama ga Okuchi de Reiryoku Hokyuu", - "title_jpn" : "コウコ様がお口で霊力補給♡", - "type" : "Artistcg", -}, - -{ - "#url" : "https://hitomi.la/manga/amazon-no-hiyaku-867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/manga/867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/doujinshi/867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/cg/867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/gamecg/867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/imageset/867789.html", - "#comment" : "/imageset/ gallery (#4756)", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/reader/867789.html", - "#category": ("", "hitomi", "gallery"), - "#class" : hitomi.HitomiGalleryExtractor, -}, - -{ - "#url" : "https://hitomi.la/tag/screenshots-japanese.html", - "#category": ("", "hitomi", "tag"), - "#class" : hitomi.HitomiTagExtractor, - "#pattern" : hitomi.HitomiGalleryExtractor.pattern, - "#count" : ">= 35", -}, - -{ - "#url" : "https://hitomi.la/artist/a1-all-1.html", - "#category": ("", "hitomi", "tag"), - "#class" : hitomi.HitomiTagExtractor, -}, - -{ - "#url" : "https://hitomi.la/group/initial%2Dg-all-1.html", - "#category": ("", "hitomi", "tag"), - "#class" : hitomi.HitomiTagExtractor, -}, - -{ - "#url" : "https://hitomi.la/series/amnesia-all-1.html", - "#category": ("", "hitomi", "tag"), - "#class" : hitomi.HitomiTagExtractor, -}, - -{ - "#url" : "https://hitomi.la/type/doujinshi-all-1.html", - "#category": ("", "hitomi", "tag"), - "#class" : hitomi.HitomiTagExtractor, -}, - -{ - "#url" : "https://hitomi.la/character/a2-all-1.html", - "#category": ("", "hitomi", "tag"), - "#class" : hitomi.HitomiTagExtractor, -}, - -{ - "#url" : "https://hitomi.la/index-japanese.html", - "#class" : hitomi.HitomiIndexExtractor, - "#pattern" : hitomi.HitomiGalleryExtractor.pattern, - "#range" : "1-150", - "#count" : 150, -}, - -{ - "#url" : "https://hitomi.la/search.html?tag%3Ascreenshots%20language%3Ajapanese", - "#class" : hitomi.HitomiSearchExtractor, - "#pattern" : hitomi.HitomiGalleryExtractor.pattern, - "#range" : "1-150", - "#count" : 150, -}, - -{ - "#url" : "https://hitomi.la/search.html?language%3Ajapanese%20artist%3Asumiya", - "#class" : hitomi.HitomiSearchExtractor, -}, -{ - "#url" : "https://hitomi.la/search.html?group:initial_g", - "#class" : hitomi.HitomiSearchExtractor, -}, -{ - "#url" : "https://hitomi.la/search.html?series:amnesia", - "#class" : hitomi.HitomiSearchExtractor, -}, -{ - "#url" : "https://hitomi.la/search.html?type%3Adoujinshi", - "#class" : hitomi.HitomiSearchExtractor, -}, -{ - "#url" : "https://hitomi.la/search.html?character%3Aa2", - "#class" : hitomi.HitomiSearchExtractor, -}, - + { + "#url": "https://hitomi.la/galleries/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + "#pattern": r"https://[a-c]a\.hitomi\.la/webp/\d+/\d+/[0-9a-f]{64}\.webp", + "#count": 16, + "artist": ["morris"], + "characters": [], + "count": 16, + "date": "dt:2015-10-27 19:20:00", + "extension": "webp", + "extension_original": "jpg", + "filename": str, + "gallery_id": 867789, + "group": [], + "lang": "en", + "language": "English", + "num": range(1, 16), + "parody": [], + "tags": [ + "Cheating ♀", + "Drugs ♀", + "Drugs ♂", + "Incest", + "Milf ♀", + "Mother ♀", + "Sole Female ♀", + "Sole Male ♂", + "Uncensored", + ], + "title": "Amazon no Hiyaku | Amazon Elixir (decensored)", + "title_jpn": "", + "type": "Manga", + }, + { + "#url": "https://hitomi.la/galleries/1401410.html", + "#comment": "download test", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + "#range": "1", + "#sha1_content": "d75d5a3d1302a48469016b20e53c26b714d17745", + }, + { + "#url": "https://hitomi.la/galleries/733697.html", + "#comment": "Game CG with scenes (#321)", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + "#count": 210, + }, + { + "#url": "https://hitomi.la/galleries/1045954.html", + "#comment": "fallback for galleries only available through /reader/ URLs", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + "#count": 1413, + }, + { + "#url": "https://hitomi.la/cg/scathacha-sama-okuchi-ecchi-1291900.html", + "#comment": "gallery with 'broken' redirect", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/cg/1615823.html", + "#comment": "no tags", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + "#options": {"format": "avif"}, + "#pattern": r"https://[a-c]a\.hitomi\.la/avif/\d+/\d+/[0-9a-f]{64}\.avif", + "#count": 22, + "artist": ["sorairo len"], + "characters": [], + "count": 22, + "date": "dt:2020-04-19 06:33:00", + "extension": "avif", + "filename": str, + "gallery_id": 1615823, + "group": [], + "lang": "ja", + "language": "Japanese", + "num": range(1, 22), + "parody": [], + "tags": [ + "Blowjob ♀", + "Focus Blowjob ♀", + "Fox Girl ♀", + "Kemonomimi ♀", + "Loli ♀", + "Miko ♀", + "No Penetration", + "Unusual Pupils ♀", + "Variant Set", + ], + "title": "Kouko-sama ga Okuchi de Reiryoku Hokyuu", + "title_jpn": "コウコ様がお口で霊力補給♡", + "type": "Artistcg", + }, + { + "#url": "https://hitomi.la/manga/amazon-no-hiyaku-867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/manga/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/doujinshi/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/cg/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/gamecg/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/imageset/867789.html", + "#comment": "/imageset/ gallery (#4756)", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/reader/867789.html", + "#category": ("", "hitomi", "gallery"), + "#class": hitomi.HitomiGalleryExtractor, + }, + { + "#url": "https://hitomi.la/tag/screenshots-japanese.html", + "#category": ("", "hitomi", "tag"), + "#class": hitomi.HitomiTagExtractor, + "#pattern": hitomi.HitomiGalleryExtractor.pattern, + "#count": ">= 35", + }, + { + "#url": "https://hitomi.la/artist/a1-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class": hitomi.HitomiTagExtractor, + }, + { + "#url": "https://hitomi.la/group/initial%2Dg-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class": hitomi.HitomiTagExtractor, + }, + { + "#url": "https://hitomi.la/series/amnesia-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class": hitomi.HitomiTagExtractor, + }, + { + "#url": "https://hitomi.la/type/doujinshi-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class": hitomi.HitomiTagExtractor, + }, + { + "#url": "https://hitomi.la/character/a2-all-1.html", + "#category": ("", "hitomi", "tag"), + "#class": hitomi.HitomiTagExtractor, + }, + { + "#url": "https://hitomi.la/index-japanese.html", + "#class": hitomi.HitomiIndexExtractor, + "#pattern": hitomi.HitomiGalleryExtractor.pattern, + "#range": "1-150", + "#count": 150, + }, + { + "#url": "https://hitomi.la/search.html?tag%3Ascreenshots%20language%3Ajapanese", + "#class": hitomi.HitomiSearchExtractor, + "#pattern": hitomi.HitomiGalleryExtractor.pattern, + "#range": "1-150", + "#count": 150, + }, + { + "#url": "https://hitomi.la/search.html?language%3Ajapanese%20artist%3Asumiya", + "#class": hitomi.HitomiSearchExtractor, + }, + { + "#url": "https://hitomi.la/search.html?group:initial_g", + "#class": hitomi.HitomiSearchExtractor, + }, + { + "#url": "https://hitomi.la/search.html?series:amnesia", + "#class": hitomi.HitomiSearchExtractor, + }, + { + "#url": "https://hitomi.la/search.html?type%3Adoujinshi", + "#class": hitomi.HitomiSearchExtractor, + }, + { + "#url": "https://hitomi.la/search.html?character%3Aa2", + "#class": hitomi.HitomiSearchExtractor, + }, ) diff --git a/test/results/horne.py b/test/results/horne.py index f6bddba866..f68c71cc95 100644 --- a/test/results/horne.py +++ b/test/results/horne.py @@ -1,133 +1,118 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import nijie import datetime +from gallery_dl.extractor import nijie __tests__ = ( -{ - "#url" : "https://horne.red/members.php?id=58000", - "#category": ("Nijie", "horne", "user"), - "#class" : nijie.NijieUserExtractor, - "#urls" : ( - "https://horne.red/members_illust.php?id=58000", - "https://horne.red/members_dojin.php?id=58000", - ), -}, - -{ - "#url" : "https://horne.red/members_illust.php?id=58000", - "#category": ("Nijie", "horne", "illustration"), - "#class" : nijie.NijieIllustrationExtractor, - "#pattern" : r"https://pic\.nijie\.net/\d+/horne/\w+/\d+/\d+/illust/\d+_\d+_[0-9a-f]+_[0-9a-f]+\.png", - "#range" : "1-20", - "#count" : 20, - - "artist_id" : 58000, - "artist_name": "のえるわ", - "date" : datetime.datetime, - "description": str, - "image_id" : int, - "num" : int, - "tags" : list, - "title" : str, - "url" : str, - "user_id" : 58000, - "user_name" : "のえるわ", -}, - -{ - "#url" : "https://horne.red/members_dojin.php?id=58000", - "#category": ("Nijie", "horne", "doujin"), - "#class" : nijie.NijieDoujinExtractor, -}, - -{ - "#url" : "https://horne.red/user_like_illust_view.php?id=58000", - "#category": ("Nijie", "horne", "favorite"), - "#class" : nijie.NijieFavoriteExtractor, - "#range" : "1-5", - "#count" : 5, - - "user_id" : 58000, - "user_name": "のえるわ", -}, - -{ - "#url" : "https://horne.red/history_nuita.php?id=58000", - "#category": ("Nijie", "horne", "nuita"), - "#class" : nijie.NijieNuitaExtractor, -}, - -{ - "#url" : "https://horne.red/like_user_view.php", - "#category": ("Nijie", "horne", "feed"), - "#class" : nijie.NijieFeedExtractor, -}, - -{ - "#url" : "https://horne.red/like_my.php", - "#category": ("Nijie", "horne", "followed"), - "#class" : nijie.NijieFollowedExtractor, -}, - -{ - "#url" : "https://horne.red/view.php?id=8708", - "#category": ("Nijie", "horne", "image"), - "#class" : nijie.NijieImageExtractor, - "#urls" : "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_c8f715a8f3d53943_db6231.png", - - "artist_id" : 58000, - "artist_name": "のえるわ", - "count" : 1, - "date" : "dt:2018-01-29 14:25:39", - "description": "前回とシチュがまるかぶり　\r\n竿野郎は塗るのだるかった", - "extension" : "png", - "filename" : "0_0_c8f715a8f3d53943_db6231", - "image_id" : 8708, - "num" : 0, - "tags" : [ - "男の娘", - "オリキャラ", - "うちのこ", - ], - "title" : "うちのこえっち", - "url" : "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_c8f715a8f3d53943_db6231.png", - "user_id" : 58000, - "user_name" : "のえるわ", -}, - -{ - "#url" : "https://horne.red/view.php?id=8716", - "#category": ("Nijie", "horne", "image"), - "#class" : nijie.NijieImageExtractor, - "#urls" : ( - "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_b4ffb4b6f7ec6d51_1a32c0.png", - "https://pic.nijie.net/07/horne/18/00/58000/illust/8716_0_2690972a4f6270bb_85ed8f.png", - "https://pic.nijie.net/07/horne/18/00/58000/illust/8716_1_09348508d8b76f36_f6cf47.png", - "https://pic.nijie.net/08/horne/18/00/58000/illust/8716_2_5151d956d3789277_d76e75.png", - ), - - "artist_id" : 58000, - "artist_name": "のえるわ", - "count" : 4, - "date" : "dt:2018-02-04 14:47:24", - "description": "ノエル「そんなことしなくても、言ってくれたら咥えるのに・・・♡」", - "image_id" : 8716, - "num" : range(0, 3), - "tags" : [ - "男の娘", - "フェラ", - "オリキャラ", - "うちのこ", - ], - "title" : "ノエル「いまどきそんな、恵方巻ネタなんてやらなくても・・・」", - "user_id" : 58000, - "user_name" : "のえるわ", -}, - + { + "#url": "https://horne.red/members.php?id=58000", + "#category": ("Nijie", "horne", "user"), + "#class": nijie.NijieUserExtractor, + "#urls": ( + "https://horne.red/members_illust.php?id=58000", + "https://horne.red/members_dojin.php?id=58000", + ), + }, + { + "#url": "https://horne.red/members_illust.php?id=58000", + "#category": ("Nijie", "horne", "illustration"), + "#class": nijie.NijieIllustrationExtractor, + "#pattern": r"https://pic\.nijie\.net/\d+/horne/\w+/\d+/\d+/illust/\d+_\d+_[0-9a-f]+_[0-9a-f]+\.png", + "#range": "1-20", + "#count": 20, + "artist_id": 58000, + "artist_name": "のえるわ", + "date": datetime.datetime, + "description": str, + "image_id": int, + "num": int, + "tags": list, + "title": str, + "url": str, + "user_id": 58000, + "user_name": "のえるわ", + }, + { + "#url": "https://horne.red/members_dojin.php?id=58000", + "#category": ("Nijie", "horne", "doujin"), + "#class": nijie.NijieDoujinExtractor, + }, + { + "#url": "https://horne.red/user_like_illust_view.php?id=58000", + "#category": ("Nijie", "horne", "favorite"), + "#class": nijie.NijieFavoriteExtractor, + "#range": "1-5", + "#count": 5, + "user_id": 58000, + "user_name": "のえるわ", + }, + { + "#url": "https://horne.red/history_nuita.php?id=58000", + "#category": ("Nijie", "horne", "nuita"), + "#class": nijie.NijieNuitaExtractor, + }, + { + "#url": "https://horne.red/like_user_view.php", + "#category": ("Nijie", "horne", "feed"), + "#class": nijie.NijieFeedExtractor, + }, + { + "#url": "https://horne.red/like_my.php", + "#category": ("Nijie", "horne", "followed"), + "#class": nijie.NijieFollowedExtractor, + }, + { + "#url": "https://horne.red/view.php?id=8708", + "#category": ("Nijie", "horne", "image"), + "#class": nijie.NijieImageExtractor, + "#urls": "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_c8f715a8f3d53943_db6231.png", + "artist_id": 58000, + "artist_name": "のえるわ", + "count": 1, + "date": "dt:2018-01-29 14:25:39", + "description": "前回とシチュがまるかぶり　\r\n竿野郎は塗るのだるかった", + "extension": "png", + "filename": "0_0_c8f715a8f3d53943_db6231", + "image_id": 8708, + "num": 0, + "tags": [ + "男の娘", + "オリキャラ", + "うちのこ", + ], + "title": "うちのこえっち", + "url": "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_c8f715a8f3d53943_db6231.png", + "user_id": 58000, + "user_name": "のえるわ", + }, + { + "#url": "https://horne.red/view.php?id=8716", + "#category": ("Nijie", "horne", "image"), + "#class": nijie.NijieImageExtractor, + "#urls": ( + "https://pic.nijie.net/07/horne/18/00/58000/illust/0_0_b4ffb4b6f7ec6d51_1a32c0.png", + "https://pic.nijie.net/07/horne/18/00/58000/illust/8716_0_2690972a4f6270bb_85ed8f.png", + "https://pic.nijie.net/07/horne/18/00/58000/illust/8716_1_09348508d8b76f36_f6cf47.png", + "https://pic.nijie.net/08/horne/18/00/58000/illust/8716_2_5151d956d3789277_d76e75.png", + ), + "artist_id": 58000, + "artist_name": "のえるわ", + "count": 4, + "date": "dt:2018-02-04 14:47:24", + "description": "ノエル「そんなことしなくても、言ってくれたら咥えるのに・・・♡」", + "image_id": 8716, + "num": range(3), + "tags": [ + "男の娘", + "フェラ", + "オリキャラ", + "うちのこ", + ], + "title": "ノエル「いまどきそんな、恵方巻ネタなんてやらなくても・・・」", + "user_id": 58000, + "user_name": "のえるわ", + }, ) diff --git a/test/results/hotleak.py b/test/results/hotleak.py index 7305180c19..e9ed093ae9 100644 --- a/test/results/hotleak.py +++ b/test/results/hotleak.py @@ -1,104 +1,88 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import hotleak from gallery_dl import exception - +from gallery_dl.extractor import hotleak __tests__ = ( -{ - "#url" : "https://hotleak.vip/kaiyakawaii/photo/1617145", - "#category": ("", "hotleak", "post"), - "#class" : hotleak.HotleakPostExtractor, - "#urls" : "https://image-cdn.hotleak.vip/storage/images/e98/18ad68/18ad68.webp", - - "id" : 1617145, - "creator" : "kaiyakawaii", - "type" : "photo", - "filename" : "18ad68", - "extension": "webp", -}, - -{ - "#url" : "https://hotleak.vip/lilmochidoll/video/1625538", - "#category": ("", "hotleak", "post"), - "#class" : hotleak.HotleakPostExtractor, - "#pattern" : r"ytdl:https://cdn\d+-leak\.camhdxx\.com/.+,\d+/1661/1625538/index\.m3u8", - - "id" : 1625538, - "creator" : "lilmochidoll", - "type" : "video", - "filename" : "index", - "extension": "mp4", -}, - -{ - "#url" : "https://hotleak.vip/kaiyakawaii", - "#category": ("", "hotleak", "creator"), - "#class" : hotleak.HotleakCreatorExtractor, - "#range" : "1-200", - "#count" : 200, -}, - -{ - "#url" : "https://hotleak.vip/stellaviolet", - "#category": ("", "hotleak", "creator"), - "#class" : hotleak.HotleakCreatorExtractor, - "#count" : "> 600", -}, - -{ - "#url" : "https://hotleak.vip/doesnotexist", - "#category": ("", "hotleak", "creator"), - "#class" : hotleak.HotleakCreatorExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://hotleak.vip/photos", - "#category": ("", "hotleak", "category"), - "#class" : hotleak.HotleakCategoryExtractor, - "#pattern" : hotleak.HotleakPostExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://hotleak.vip/videos", - "#category": ("", "hotleak", "category"), - "#class" : hotleak.HotleakCategoryExtractor, -}, - -{ - "#url" : "https://hotleak.vip/creators", - "#category": ("", "hotleak", "category"), - "#class" : hotleak.HotleakCategoryExtractor, - "#pattern" : hotleak.HotleakCreatorExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://hotleak.vip/hot", - "#category": ("", "hotleak", "category"), - "#class" : hotleak.HotleakCategoryExtractor, -}, - -{ - "#url" : "https://hotleak.vip/search?search=gallery-dl", - "#category": ("", "hotleak", "search"), - "#class" : hotleak.HotleakSearchExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://hotleak.vip/search?search=hannah", - "#category": ("", "hotleak", "search"), - "#class" : hotleak.HotleakSearchExtractor, - "#count" : "> 30", -}, - + { + "#url": "https://hotleak.vip/kaiyakawaii/photo/1617145", + "#category": ("", "hotleak", "post"), + "#class": hotleak.HotleakPostExtractor, + "#urls": "https://image-cdn.hotleak.vip/storage/images/e98/18ad68/18ad68.webp", + "id": 1617145, + "creator": "kaiyakawaii", + "type": "photo", + "filename": "18ad68", + "extension": "webp", + }, + { + "#url": "https://hotleak.vip/lilmochidoll/video/1625538", + "#category": ("", "hotleak", "post"), + "#class": hotleak.HotleakPostExtractor, + "#pattern": r"ytdl:https://cdn\d+-leak\.camhdxx\.com/.+,\d+/1661/1625538/index\.m3u8", + "id": 1625538, + "creator": "lilmochidoll", + "type": "video", + "filename": "index", + "extension": "mp4", + }, + { + "#url": "https://hotleak.vip/kaiyakawaii", + "#category": ("", "hotleak", "creator"), + "#class": hotleak.HotleakCreatorExtractor, + "#range": "1-200", + "#count": 200, + }, + { + "#url": "https://hotleak.vip/stellaviolet", + "#category": ("", "hotleak", "creator"), + "#class": hotleak.HotleakCreatorExtractor, + "#count": "> 600", + }, + { + "#url": "https://hotleak.vip/doesnotexist", + "#category": ("", "hotleak", "creator"), + "#class": hotleak.HotleakCreatorExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://hotleak.vip/photos", + "#category": ("", "hotleak", "category"), + "#class": hotleak.HotleakCategoryExtractor, + "#pattern": hotleak.HotleakPostExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://hotleak.vip/videos", + "#category": ("", "hotleak", "category"), + "#class": hotleak.HotleakCategoryExtractor, + }, + { + "#url": "https://hotleak.vip/creators", + "#category": ("", "hotleak", "category"), + "#class": hotleak.HotleakCategoryExtractor, + "#pattern": hotleak.HotleakCreatorExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://hotleak.vip/hot", + "#category": ("", "hotleak", "category"), + "#class": hotleak.HotleakCategoryExtractor, + }, + { + "#url": "https://hotleak.vip/search?search=gallery-dl", + "#category": ("", "hotleak", "search"), + "#class": hotleak.HotleakSearchExtractor, + "#count": 0, + }, + { + "#url": "https://hotleak.vip/search?search=hannah", + "#category": ("", "hotleak", "search"), + "#class": hotleak.HotleakSearchExtractor, + "#count": "> 30", + }, ) diff --git a/test/results/hypnohub.py b/test/results/hypnohub.py index d979b620e2..bc888fad3f 100644 --- a/test/results/hypnohub.py +++ b/test/results/hypnohub.py @@ -1,77 +1,69 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v02 - __tests__ = ( -{ - "#url" : "https://hypnohub.net/index.php?page=post&s=list&tags=gonoike_biwa", - "#category": ("gelbooru_v02", "hypnohub", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#sha1_url": "fe662b86d38c331fcac9c62af100167d404937dc", -}, - -{ - "#url" : "https://hypnohub.net/index.php?page=pool&s=show&id=61", - "#category": ("gelbooru_v02", "hypnohub", "pool"), - "#class" : gelbooru_v02.GelbooruV02PoolExtractor, - "#count" : 3, - "#sha1_url": "d314826280073441a2da609f70ee814d1f4b9407", -}, - -{ - "#url" : "https://hypnohub.net/index.php?page=favorites&s=view&id=43546", - "#category": ("gelbooru_v02", "hypnohub", "favorite"), - "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, - "#count" : 3, -}, - -{ - "#url" : "https://hypnohub.net/index.php?page=post&s=view&id=1439", - "#category": ("gelbooru_v02", "hypnohub", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#options" : { - "tags" : True, - "notes": True, + { + "#url": "https://hypnohub.net/index.php?page=post&s=list&tags=gonoike_biwa", + "#category": ("gelbooru_v02", "hypnohub", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#sha1_url": "fe662b86d38c331fcac9c62af100167d404937dc", }, - "#pattern" : r"https://hypnohub\.net/images/90/24/90245c3c5250c2a8173255d3923a010b\.jpg", - "#sha1_content": "5987c5d2354f22e5fa9b7ee7ce4a6f7beb8b2b71", - - "tags_artist" : "brokenteapot", - "tags_character": "hsien-ko", - "tags_copyright": "capcom darkstalkers", - "tags_general" : str, - "tags_metadata" : "dialogue text translated", - "notes" : [ - { - "body" : "Master Master Master Master Master Master", - "height": 83, - "id" : 10577, - "width" : 129, - "x" : 259, - "y" : 20, - }, - { - "body" : "Response Response Response Response Response Response", - "height": 86, - "id" : 10578, - "width" : 125, - "x" : 126, - "y" : 20, - }, - { - "body" : "Obedience Obedience Obedience Obedience Obedience Obedience", - "height": 80, - "id" : 10579, - "width" : 98, - "x" : 20, - "y" : 20, + { + "#url": "https://hypnohub.net/index.php?page=pool&s=show&id=61", + "#category": ("gelbooru_v02", "hypnohub", "pool"), + "#class": gelbooru_v02.GelbooruV02PoolExtractor, + "#count": 3, + "#sha1_url": "d314826280073441a2da609f70ee814d1f4b9407", + }, + { + "#url": "https://hypnohub.net/index.php?page=favorites&s=view&id=43546", + "#category": ("gelbooru_v02", "hypnohub", "favorite"), + "#class": gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count": 3, + }, + { + "#url": "https://hypnohub.net/index.php?page=post&s=view&id=1439", + "#category": ("gelbooru_v02", "hypnohub", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#options": { + "tags": True, + "notes": True, }, - ], -}, - + "#pattern": r"https://hypnohub\.net/images/90/24/90245c3c5250c2a8173255d3923a010b\.jpg", + "#sha1_content": "5987c5d2354f22e5fa9b7ee7ce4a6f7beb8b2b71", + "tags_artist": "brokenteapot", + "tags_character": "hsien-ko", + "tags_copyright": "capcom darkstalkers", + "tags_general": str, + "tags_metadata": "dialogue text translated", + "notes": [ + { + "body": "Master Master Master Master Master Master", + "height": 83, + "id": 10577, + "width": 129, + "x": 259, + "y": 20, + }, + { + "body": "Response Response Response Response Response Response", + "height": 86, + "id": 10578, + "width": 125, + "x": 126, + "y": 20, + }, + { + "body": "Obedience Obedience Obedience Obedience Obedience Obedience", + "height": 80, + "id": 10579, + "width": 98, + "x": 20, + "y": 20, + }, + ], + }, ) diff --git a/test/results/idolcomplex.py b/test/results/idolcomplex.py index dfcff19947..a4832682b5 100644 --- a/test/results/idolcomplex.py +++ b/test/results/idolcomplex.py @@ -1,122 +1,104 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import idolcomplex - __tests__ = ( -{ - "#url" : "https://idol.sankakucomplex.com/en/posts?tags=lyumos", - "#category": ("booru", "idolcomplex", "tag"), - "#class" : idolcomplex.IdolcomplexTagExtractor, - "#pattern" : r"https://i[sv]\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[^/]{32}\.\w+\?e=\d+&m=[^&#]+", - "#range" : "18-22", - "#count" : 5, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/posts/?tags=lyumos", - "#category": ("booru", "idolcomplex", "tag"), - "#class" : idolcomplex.IdolcomplexTagExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/en/?tags=lyumos", - "#category": ("booru", "idolcomplex", "tag"), - "#class" : idolcomplex.IdolcomplexTagExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/?tags=lyumos", - "#category": ("booru", "idolcomplex", "tag"), - "#class" : idolcomplex.IdolcomplexTagExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/?tags=lyumos+wreath&page=3&next=694215", - "#category": ("booru", "idolcomplex", "tag"), - "#class" : idolcomplex.IdolcomplexTagExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/en/pools/e9PMwnwRBK3", - "#category": ("booru", "idolcomplex", "pool"), - "#class" : idolcomplex.IdolcomplexPoolExtractor, - "#count" : 3, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/en/pools/show/145", - "#category": ("booru", "idolcomplex", "pool"), - "#class" : idolcomplex.IdolcomplexPoolExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/pool/show/145", - "#category": ("booru", "idolcomplex", "pool"), - "#class" : idolcomplex.IdolcomplexPoolExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/en/posts/vkr36qdOaZ4", - "#category": ("booru", "idolcomplex", "post"), - "#class" : idolcomplex.IdolcomplexPostExtractor, - "#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd", - - "created_at" : "2017-11-24 17:01:27.696", - "date" : "dt:2017-11-24 17:01:27", - "extension" : "jpg", - "file_url" : r"re:https://i[sv]\.sankakucomplex\.com/data/50/9e/509eccbba54a43cea6b275a65b93c51d\.jpg\?", - "filename" : "509eccbba54a43cea6b275a65b93c51d", - "height" : 683, - "id" : "vkr36qdOaZ4", # legacy ID: 694215 - "md5" : "509eccbba54a43cea6b275a65b93c51d", - "rating" : "g", - "tags" : "lyumos the_witcher shani_(the_witcher) 1girl green_eyes non-asian redhead waistcoat wreath cosplay 3:2_aspect_ratio", - "tags_character": "shani_(the_witcher)", - "tags_copyright": "the_witcher", - "tags_general" : "1girl green_eyes non-asian redhead waistcoat wreath", - "tags_genre" : "cosplay", - "tags_idol" : "lyumos", - "tags_medium" : "3:2_aspect_ratio", - "vote_average" : range(4, 5), - "vote_count" : range(25, 40), - "width" : 1024, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/en/posts/509eccbba54a43cea6b275a65b93c51d", - "#category": ("booru", "idolcomplex", "post"), - "#class" : idolcomplex.IdolcomplexPostExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/en/posts/show/509eccbba54a43cea6b275a65b93c51d", - "#category": ("booru", "idolcomplex", "post"), - "#class" : idolcomplex.IdolcomplexPostExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/posts/509eccbba54a43cea6b275a65b93c51d", - "#category": ("booru", "idolcomplex", "post"), - "#class" : idolcomplex.IdolcomplexPostExtractor, -}, - -{ - "#url" : "https://idol.sankakucomplex.com/post/show/694215", - "#category": ("booru", "idolcomplex", "post"), - "#class" : idolcomplex.IdolcomplexPostExtractor, - "#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd", - - "id" : "vkr36qdOaZ4", # legacy ID: 694215 - "tags_character": "shani_(the_witcher)", - "tags_copyright": "the_witcher", - "tags_idol" : str, - "tags_medium" : str, - "tags_general" : str, -}, - + { + "#url": "https://idol.sankakucomplex.com/en/posts?tags=lyumos", + "#category": ("booru", "idolcomplex", "tag"), + "#class": idolcomplex.IdolcomplexTagExtractor, + "#pattern": r"https://i[sv]\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[^/]{32}\.\w+\?e=\d+&m=[^&#]+", + "#range": "18-22", + "#count": 5, + }, + { + "#url": "https://idol.sankakucomplex.com/posts/?tags=lyumos", + "#category": ("booru", "idolcomplex", "tag"), + "#class": idolcomplex.IdolcomplexTagExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/en/?tags=lyumos", + "#category": ("booru", "idolcomplex", "tag"), + "#class": idolcomplex.IdolcomplexTagExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/?tags=lyumos", + "#category": ("booru", "idolcomplex", "tag"), + "#class": idolcomplex.IdolcomplexTagExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/?tags=lyumos+wreath&page=3&next=694215", + "#category": ("booru", "idolcomplex", "tag"), + "#class": idolcomplex.IdolcomplexTagExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/en/pools/e9PMwnwRBK3", + "#category": ("booru", "idolcomplex", "pool"), + "#class": idolcomplex.IdolcomplexPoolExtractor, + "#count": 3, + }, + { + "#url": "https://idol.sankakucomplex.com/en/pools/show/145", + "#category": ("booru", "idolcomplex", "pool"), + "#class": idolcomplex.IdolcomplexPoolExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/pool/show/145", + "#category": ("booru", "idolcomplex", "pool"), + "#class": idolcomplex.IdolcomplexPoolExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/en/posts/vkr36qdOaZ4", + "#category": ("booru", "idolcomplex", "post"), + "#class": idolcomplex.IdolcomplexPostExtractor, + "#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd", + "created_at": "2017-11-24 17:01:27.696", + "date": "dt:2017-11-24 17:01:27", + "extension": "jpg", + "file_url": r"re:https://i[sv]\.sankakucomplex\.com/data/50/9e/509eccbba54a43cea6b275a65b93c51d\.jpg\?", + "filename": "509eccbba54a43cea6b275a65b93c51d", + "height": 683, + "id": "vkr36qdOaZ4", # legacy ID: 694215 + "md5": "509eccbba54a43cea6b275a65b93c51d", + "rating": "g", + "tags": "lyumos the_witcher shani_(the_witcher) 1girl green_eyes non-asian redhead waistcoat wreath cosplay 3:2_aspect_ratio", + "tags_character": "shani_(the_witcher)", + "tags_copyright": "the_witcher", + "tags_general": "1girl green_eyes non-asian redhead waistcoat wreath", + "tags_genre": "cosplay", + "tags_idol": "lyumos", + "tags_medium": "3:2_aspect_ratio", + "vote_average": range(4, 5), + "vote_count": range(25, 40), + "width": 1024, + }, + { + "#url": "https://idol.sankakucomplex.com/en/posts/509eccbba54a43cea6b275a65b93c51d", + "#category": ("booru", "idolcomplex", "post"), + "#class": idolcomplex.IdolcomplexPostExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/en/posts/show/509eccbba54a43cea6b275a65b93c51d", + "#category": ("booru", "idolcomplex", "post"), + "#class": idolcomplex.IdolcomplexPostExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/posts/509eccbba54a43cea6b275a65b93c51d", + "#category": ("booru", "idolcomplex", "post"), + "#class": idolcomplex.IdolcomplexPostExtractor, + }, + { + "#url": "https://idol.sankakucomplex.com/post/show/694215", + "#category": ("booru", "idolcomplex", "post"), + "#class": idolcomplex.IdolcomplexPostExtractor, + "#sha1_content": "694ec2491240787d75bf5d0c75d0082b53a85afd", + "id": "vkr36qdOaZ4", # legacy ID: 694215 + "tags_character": "shani_(the_witcher)", + "tags_copyright": "the_witcher", + "tags_idol": str, + "tags_medium": str, + "tags_general": str, + }, ) diff --git a/test/results/illusioncardsbooru.py b/test/results/illusioncardsbooru.py index ba0793641c..c13e76c022 100644 --- a/test/results/illusioncardsbooru.py +++ b/test/results/illusioncardsbooru.py @@ -1,34 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v01 - __tests__ = ( -{ - "#url" : "https://illusioncards.booru.org/index.php?page=post&s=list&tags=koikatsu", - "#category": ("gelbooru_v01", "illusioncardsbooru", "tag"), - "#class" : gelbooru_v01.GelbooruV01TagExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://illusioncards.booru.org/index.php?page=favorites&s=view&id=84887", - "#category": ("gelbooru_v01", "illusioncardsbooru", "favorite"), - "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://illusioncards.booru.org/index.php?page=post&s=view&id=82746", - "#category": ("gelbooru_v01", "illusioncardsbooru", "post"), - "#class" : gelbooru_v01.GelbooruV01PostExtractor, - "#sha1_url" : "3f9cd2fadf78869b90bc5422f27b48f1af0e0909", - "#sha1_content": "159e60b92d05597bd1bb63510c2c3e4a4bada1dc", -}, - + { + "#url": "https://illusioncards.booru.org/index.php?page=post&s=list&tags=koikatsu", + "#category": ("gelbooru_v01", "illusioncardsbooru", "tag"), + "#class": gelbooru_v01.GelbooruV01TagExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://illusioncards.booru.org/index.php?page=favorites&s=view&id=84887", + "#category": ("gelbooru_v01", "illusioncardsbooru", "favorite"), + "#class": gelbooru_v01.GelbooruV01FavoriteExtractor, + "#count": 2, + }, + { + "#url": "https://illusioncards.booru.org/index.php?page=post&s=view&id=82746", + "#category": ("gelbooru_v01", "illusioncardsbooru", "post"), + "#class": gelbooru_v01.GelbooruV01PostExtractor, + "#sha1_url": "3f9cd2fadf78869b90bc5422f27b48f1af0e0909", + "#sha1_content": "159e60b92d05597bd1bb63510c2c3e4a4bada1dc", + }, ) diff --git a/test/results/imagebam.py b/test/results/imagebam.py index 759e2dc488..2de9cf8132 100644 --- a/test/results/imagebam.py +++ b/test/results/imagebam.py @@ -1,78 +1,67 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imagebam from gallery_dl import exception - +from gallery_dl.extractor import imagebam __tests__ = ( -{ - "#url" : "https://www.imagebam.com/gallery/adz2y0f9574bjpmonaismyrhtjgvey4o", - "#category": ("", "imagebam", "gallery"), - "#class" : imagebam.ImagebamGalleryExtractor, - "#sha1_url" : "76d976788ae2757ac81694736b07b72356f5c4c8", - "#sha1_metadata": "b048478b1bbba3072a7fa9fcc40630b3efad1f6c", - "#sha1_content" : "596e6bfa157f2c7169805d50075c2986549973a8", -}, - -{ - "#url" : "http://www.imagebam.com/gallery/op9dwcklwdrrguibnkoe7jxgvig30o5p", - "#category": ("", "imagebam", "gallery"), - "#class" : imagebam.ImagebamGalleryExtractor, - "#count" : 107, - "#sha1_url": "32ae6fe5dc3e4ca73ff6252e522d16473595d1d1", -}, - -{ - "#url" : "http://www.imagebam.com/gallery/gsl8teckymt4vbvx1stjkyk37j70va2c", - "#category": ("", "imagebam", "gallery"), - "#class" : imagebam.ImagebamGalleryExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://www.imagebam.com/view/GA3MT1", - "#comment" : "/view/ path (#2378)", - "#category": ("", "imagebam", "gallery"), - "#class" : imagebam.ImagebamGalleryExtractor, - "#sha1_url" : "35018ce1e00a2d2825a33d3cd37857edaf804919", - "#sha1_metadata": "3a9f98178f73694c527890c0d7ca9a92b46987ba", -}, - -{ - "#url" : "https://www.imagebam.com/image/94d56c502511890", - "#category": ("", "imagebam", "image"), - "#class" : imagebam.ImagebamImageExtractor, - "#sha1_url" : "5e9ba3b1451f8ded0ae3a1b84402888893915d4a", - "#sha1_metadata": "2a4380d4b57554ff793898c2d6ec60987c86d1a1", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - -{ - "#url" : "http://images3.imagebam.com/1d/8c/44/94d56c502511890.png", - "#category": ("", "imagebam", "image"), - "#class" : imagebam.ImagebamImageExtractor, -}, - -{ - "#url" : "https://www.imagebam.com/image/0850951366904951", - "#comment" : "NSFW (#1534)", - "#category": ("", "imagebam", "image"), - "#class" : imagebam.ImagebamImageExtractor, - "#sha1_url": "d37297b17ed1615b4311c8ed511e50ce46e4c748", -}, - -{ - "#url" : "https://www.imagebam.com/view/ME8JOQP", - "#comment" : "/view/ path (#2378)", - "#category": ("", "imagebam", "image"), - "#class" : imagebam.ImagebamImageExtractor, - "#sha1_url" : "4dca72bbe61a0360185cf4ab2bed8265b49565b8", - "#sha1_metadata": "15a494c02fd30846b41b42a26117aedde30e4ceb", - "#sha1_content" : "f81008666b17a42d8834c4749b910e1dc10a6e83", -}, - + { + "#url": "https://www.imagebam.com/gallery/adz2y0f9574bjpmonaismyrhtjgvey4o", + "#category": ("", "imagebam", "gallery"), + "#class": imagebam.ImagebamGalleryExtractor, + "#sha1_url": "76d976788ae2757ac81694736b07b72356f5c4c8", + "#sha1_metadata": "b048478b1bbba3072a7fa9fcc40630b3efad1f6c", + "#sha1_content": "596e6bfa157f2c7169805d50075c2986549973a8", + }, + { + "#url": "http://www.imagebam.com/gallery/op9dwcklwdrrguibnkoe7jxgvig30o5p", + "#category": ("", "imagebam", "gallery"), + "#class": imagebam.ImagebamGalleryExtractor, + "#count": 107, + "#sha1_url": "32ae6fe5dc3e4ca73ff6252e522d16473595d1d1", + }, + { + "#url": "http://www.imagebam.com/gallery/gsl8teckymt4vbvx1stjkyk37j70va2c", + "#category": ("", "imagebam", "gallery"), + "#class": imagebam.ImagebamGalleryExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://www.imagebam.com/view/GA3MT1", + "#comment": "/view/ path (#2378)", + "#category": ("", "imagebam", "gallery"), + "#class": imagebam.ImagebamGalleryExtractor, + "#sha1_url": "35018ce1e00a2d2825a33d3cd37857edaf804919", + "#sha1_metadata": "3a9f98178f73694c527890c0d7ca9a92b46987ba", + }, + { + "#url": "https://www.imagebam.com/image/94d56c502511890", + "#category": ("", "imagebam", "image"), + "#class": imagebam.ImagebamImageExtractor, + "#sha1_url": "5e9ba3b1451f8ded0ae3a1b84402888893915d4a", + "#sha1_metadata": "2a4380d4b57554ff793898c2d6ec60987c86d1a1", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, + { + "#url": "http://images3.imagebam.com/1d/8c/44/94d56c502511890.png", + "#category": ("", "imagebam", "image"), + "#class": imagebam.ImagebamImageExtractor, + }, + { + "#url": "https://www.imagebam.com/image/0850951366904951", + "#comment": "NSFW (#1534)", + "#category": ("", "imagebam", "image"), + "#class": imagebam.ImagebamImageExtractor, + "#sha1_url": "d37297b17ed1615b4311c8ed511e50ce46e4c748", + }, + { + "#url": "https://www.imagebam.com/view/ME8JOQP", + "#comment": "/view/ path (#2378)", + "#category": ("", "imagebam", "image"), + "#class": imagebam.ImagebamImageExtractor, + "#sha1_url": "4dca72bbe61a0360185cf4ab2bed8265b49565b8", + "#sha1_metadata": "15a494c02fd30846b41b42a26117aedde30e4ceb", + "#sha1_content": "f81008666b17a42d8834c4749b910e1dc10a6e83", + }, ) diff --git a/test/results/imagechest.py b/test/results/imagechest.py index 41254a3907..60cd7887af 100644 --- a/test/results/imagechest.py +++ b/test/results/imagechest.py @@ -1,52 +1,44 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imagechest from gallery_dl import exception - +from gallery_dl.extractor import imagechest __tests__ = ( -{ - "#url" : "https://imgchest.com/p/3na7kr3by8d", - "#category": ("", "imagechest", "gallery"), - "#class" : imagechest.ImagechestGalleryExtractor, - "#pattern" : r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", - "#count" : 3, - "#sha1_url" : "7328ca4ec2459378d725e3be19f661d2b045feda", - "#sha1_content": "076959e65be30249a2c651fbe6090dc30ba85193", - - "count" : 3, - "gallery_id": "3na7kr3by8d", - "num" : int, - "title" : "Wizardry - Video Game From The Mid 80's", -}, - -{ - "#url" : "https://imgchest.com/p/9p4n3q2z7nq", - "#comment" : "'Load More Files' button (#4028)", - "#category": ("", "imagechest", "gallery"), - "#class" : imagechest.ImagechestGalleryExtractor, - "#pattern" : r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", - "#count" : 52, - "#sha1_url": "f5674e8ba79d336193c9f698708d9dcc10e78cc7", -}, - -{ - "#url" : "https://imgchest.com/p/xxxxxxxxxxx", - "#category": ("", "imagechest", "gallery"), - "#class" : imagechest.ImagechestGalleryExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://imgchest.com/u/LunarLandr", - "#category": ("", "imagechest", "user"), - "#class" : imagechest.ImagechestUserExtractor, - "#pattern" : imagechest.ImagechestGalleryExtractor.pattern, - "#count" : 279, -}, - + { + "#url": "https://imgchest.com/p/3na7kr3by8d", + "#category": ("", "imagechest", "gallery"), + "#class": imagechest.ImagechestGalleryExtractor, + "#pattern": r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", + "#count": 3, + "#sha1_url": "7328ca4ec2459378d725e3be19f661d2b045feda", + "#sha1_content": "076959e65be30249a2c651fbe6090dc30ba85193", + "count": 3, + "gallery_id": "3na7kr3by8d", + "num": int, + "title": "Wizardry - Video Game From The Mid 80's", + }, + { + "#url": "https://imgchest.com/p/9p4n3q2z7nq", + "#comment": "'Load More Files' button (#4028)", + "#category": ("", "imagechest", "gallery"), + "#class": imagechest.ImagechestGalleryExtractor, + "#pattern": r"https://cdn\.imgchest\.com/files/\w+\.(jpg|png)", + "#count": 52, + "#sha1_url": "f5674e8ba79d336193c9f698708d9dcc10e78cc7", + }, + { + "#url": "https://imgchest.com/p/xxxxxxxxxxx", + "#category": ("", "imagechest", "gallery"), + "#class": imagechest.ImagechestGalleryExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://imgchest.com/u/LunarLandr", + "#category": ("", "imagechest", "user"), + "#class": imagechest.ImagechestUserExtractor, + "#pattern": imagechest.ImagechestGalleryExtractor.pattern, + "#count": 279, + }, ) diff --git a/test/results/imagefap.py b/test/results/imagefap.py index 8cdaf9b1c4..2b4768a339 100644 --- a/test/results/imagefap.py +++ b/test/results/imagefap.py @@ -1,211 +1,182 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imagefap from gallery_dl import exception - +from gallery_dl.extractor import imagefap __tests__ = ( -{ - "#url" : "https://www.imagefap.com/gallery/7102714", - "#category": ("", "imagefap", "gallery"), - "#class" : imagefap.ImagefapGalleryExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://www.imagefap.com/gallery/7876223", - "#category": ("", "imagefap", "gallery"), - "#class" : imagefap.ImagefapGalleryExtractor, - "#pattern" : r"https://cdn[ch]?\.imagefap\.com/images/full/\d+/\d+/\d+\.jpg", - "#count" : 44, - - "categories" : [ - "Asses", - "Softcore", - "Pornstars", - ], - "count" : 44, - "description": "", - "gallery_id" : 7876223, - "image_id" : int, - "num" : int, - "tags" : [ - "big ass", - "panties", - "horny", - "pussy", - "exposed", - "outdoor", - ], - "title" : "Kelsi Monroe in lingerie", - "uploader" : "BdRachel", -}, - -{ - "#url" : "https://www.imagefap.com/gallery/6706356", - "#comment" : "description (#3905)", - "#category": ("", "imagefap", "gallery"), - "#class" : imagefap.ImagefapGalleryExtractor, - "#range" : "1", - - "categories" : [ - "Lesbian", - "Fetish", - "Animated GIFS", - ], - "count" : 75, - "description": "A mixed collection of pics and gifs depicting lesbian femdom.\n\nAll images originally found on various Tumblr blogs and through the internet.\n\nObviously I don't own any of the images so if you do and you would like them removed please just let me know and I shall remove them straight away.", - "gallery_id" : 6706356, - "tags" : [ - "lesbian", - "femdom", - "lesbian femdom", - "lezdom", - "dominant women", - "submissive women", - ], - "title" : "Lezdom, Lesbian Femdom, Lesbian Domination - 3", - "uploader" : "pussysimon", -}, - -{ - "#url" : "https://www.imagefap.com/pictures/7102714", - "#category": ("", "imagefap", "gallery"), - "#class" : imagefap.ImagefapGalleryExtractor, -}, - -{ - "#url" : "https://www.imagefap.com/gallery.php?gid=7102714", - "#category": ("", "imagefap", "gallery"), - "#class" : imagefap.ImagefapGalleryExtractor, -}, - -{ - "#url" : "https://beta.imagefap.com/gallery.php?gid=7102714", - "#category": ("", "imagefap", "gallery"), - "#class" : imagefap.ImagefapGalleryExtractor, -}, - -{ - "#url" : "https://www.imagefap.com/photo/1962981893", - "#category": ("", "imagefap", "image"), - "#class" : imagefap.ImagefapImageExtractor, - "#pattern" : r"https://cdn[ch]?\.imagefap\.com/images/full/65/196/1962981893\.jpg", - - "date" : "21/08/2014", - "gallery_id": 7876223, - "height" : 1600, - "image_id" : 1962981893, - "title" : "Kelsi Monroe in lingerie", - "uploader" : "BdRachel", - "width" : 1066, -}, - -{ - "#url" : "https://beta.imagefap.com/photo/1962981893", - "#category": ("", "imagefap", "image"), - "#class" : imagefap.ImagefapImageExtractor, -}, - -{ - "#url" : "https://www.imagefap.com/organizer/409758", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#pattern" : r"https://www\.imagefap\.com/gallery/7876223", - "#count" : 1, - "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", -}, - -{ - "#url" : "https://www.imagefap.com/organizer/613950/Grace-Stout", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, - "#count" : 31, - - "title": r"re:Grace Stout .+", -}, - -{ - "#url" : "https://www.imagefap.com/usergallery.php?userid=1981976&folderid=409758", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#urls" : "https://www.imagefap.com/gallery/7876223", - - "folder" : "Softcore", - "gallery_id": "7876223", - "title" : "Kelsi Monroe in lingerie", -}, - -{ - "#url" : "https://www.imagefap.com/usergallery.php?user=BdRachel&folderid=409758", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", -}, - -{ - "#url" : "https://www.imagefap.com/profile/BdRachel/galleries?folderid=-1", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, - "#range" : "1-40", - - "folder": "Uncategorized", -}, - -{ - "#url" : "https://www.imagefap.com/usergallery.php?userid=1981976&folderid=-1", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, - "#range" : "1-40", -}, - -{ - "#url" : "https://www.imagefap.com/usergallery.php?user=BdRachel&folderid=-1", - "#category": ("", "imagefap", "folder"), - "#class" : imagefap.ImagefapFolderExtractor, - "#pattern" : imagefap.ImagefapGalleryExtractor.pattern, - "#range" : "1-40", -}, - -{ - "#url" : "https://www.imagefap.com/profile/BdRachel", - "#category": ("", "imagefap", "user"), - "#class" : imagefap.ImagefapUserExtractor, - "#pattern" : imagefap.ImagefapFolderExtractor.pattern, - "#count" : ">= 18", -}, - -{ - "#url" : "https://www.imagefap.com/usergallery.php?userid=1862791", - "#category": ("", "imagefap", "user"), - "#class" : imagefap.ImagefapUserExtractor, - "#pattern" : r"https://www\.imagefap\.com/profile/LucyRae/galleries\?folderid=-1", - "#count" : 1, -}, - -{ - "#url" : "https://www.imagefap.com/profile/BdRachel/galleries", - "#category": ("", "imagefap", "user"), - "#class" : imagefap.ImagefapUserExtractor, -}, - -{ - "#url" : "https://www.imagefap.com/profile.php?user=BdRachel", - "#category": ("", "imagefap", "user"), - "#class" : imagefap.ImagefapUserExtractor, -}, - -{ - "#url" : "https://beta.imagefap.com/profile.php?user=BdRachel", - "#category": ("", "imagefap", "user"), - "#class" : imagefap.ImagefapUserExtractor, -}, - + { + "#url": "https://www.imagefap.com/gallery/7102714", + "#category": ("", "imagefap", "gallery"), + "#class": imagefap.ImagefapGalleryExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://www.imagefap.com/gallery/7876223", + "#category": ("", "imagefap", "gallery"), + "#class": imagefap.ImagefapGalleryExtractor, + "#pattern": r"https://cdn[ch]?\.imagefap\.com/images/full/\d+/\d+/\d+\.jpg", + "#count": 44, + "categories": [ + "Asses", + "Softcore", + "Pornstars", + ], + "count": 44, + "description": "", + "gallery_id": 7876223, + "image_id": int, + "num": int, + "tags": [ + "big ass", + "panties", + "horny", + "pussy", + "exposed", + "outdoor", + ], + "title": "Kelsi Monroe in lingerie", + "uploader": "BdRachel", + }, + { + "#url": "https://www.imagefap.com/gallery/6706356", + "#comment": "description (#3905)", + "#category": ("", "imagefap", "gallery"), + "#class": imagefap.ImagefapGalleryExtractor, + "#range": "1", + "categories": [ + "Lesbian", + "Fetish", + "Animated GIFS", + ], + "count": 75, + "description": "A mixed collection of pics and gifs depicting lesbian femdom.\n\nAll images originally found on various Tumblr blogs and through the internet.\n\nObviously I don't own any of the images so if you do and you would like them removed please just let me know and I shall remove them straight away.", + "gallery_id": 6706356, + "tags": [ + "lesbian", + "femdom", + "lesbian femdom", + "lezdom", + "dominant women", + "submissive women", + ], + "title": "Lezdom, Lesbian Femdom, Lesbian Domination - 3", + "uploader": "pussysimon", + }, + { + "#url": "https://www.imagefap.com/pictures/7102714", + "#category": ("", "imagefap", "gallery"), + "#class": imagefap.ImagefapGalleryExtractor, + }, + { + "#url": "https://www.imagefap.com/gallery.php?gid=7102714", + "#category": ("", "imagefap", "gallery"), + "#class": imagefap.ImagefapGalleryExtractor, + }, + { + "#url": "https://beta.imagefap.com/gallery.php?gid=7102714", + "#category": ("", "imagefap", "gallery"), + "#class": imagefap.ImagefapGalleryExtractor, + }, + { + "#url": "https://www.imagefap.com/photo/1962981893", + "#category": ("", "imagefap", "image"), + "#class": imagefap.ImagefapImageExtractor, + "#pattern": r"https://cdn[ch]?\.imagefap\.com/images/full/65/196/1962981893\.jpg", + "date": "21/08/2014", + "gallery_id": 7876223, + "height": 1600, + "image_id": 1962981893, + "title": "Kelsi Monroe in lingerie", + "uploader": "BdRachel", + "width": 1066, + }, + { + "#url": "https://beta.imagefap.com/photo/1962981893", + "#category": ("", "imagefap", "image"), + "#class": imagefap.ImagefapImageExtractor, + }, + { + "#url": "https://www.imagefap.com/organizer/409758", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#pattern": r"https://www\.imagefap\.com/gallery/7876223", + "#count": 1, + "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", + }, + { + "#url": "https://www.imagefap.com/organizer/613950/Grace-Stout", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#pattern": imagefap.ImagefapGalleryExtractor.pattern, + "#count": 31, + "title": r"re:Grace Stout .+", + }, + { + "#url": "https://www.imagefap.com/usergallery.php?userid=1981976&folderid=409758", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#urls": "https://www.imagefap.com/gallery/7876223", + "folder": "Softcore", + "gallery_id": "7876223", + "title": "Kelsi Monroe in lingerie", + }, + { + "#url": "https://www.imagefap.com/usergallery.php?user=BdRachel&folderid=409758", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#sha1_url": "37822523e6e4a56feb9dea35653760c86b44ff89", + }, + { + "#url": "https://www.imagefap.com/profile/BdRachel/galleries?folderid=-1", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#pattern": imagefap.ImagefapGalleryExtractor.pattern, + "#range": "1-40", + "folder": "Uncategorized", + }, + { + "#url": "https://www.imagefap.com/usergallery.php?userid=1981976&folderid=-1", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#pattern": imagefap.ImagefapGalleryExtractor.pattern, + "#range": "1-40", + }, + { + "#url": "https://www.imagefap.com/usergallery.php?user=BdRachel&folderid=-1", + "#category": ("", "imagefap", "folder"), + "#class": imagefap.ImagefapFolderExtractor, + "#pattern": imagefap.ImagefapGalleryExtractor.pattern, + "#range": "1-40", + }, + { + "#url": "https://www.imagefap.com/profile/BdRachel", + "#category": ("", "imagefap", "user"), + "#class": imagefap.ImagefapUserExtractor, + "#pattern": imagefap.ImagefapFolderExtractor.pattern, + "#count": ">= 18", + }, + { + "#url": "https://www.imagefap.com/usergallery.php?userid=1862791", + "#category": ("", "imagefap", "user"), + "#class": imagefap.ImagefapUserExtractor, + "#pattern": r"https://www\.imagefap\.com/profile/LucyRae/galleries\?folderid=-1", + "#count": 1, + }, + { + "#url": "https://www.imagefap.com/profile/BdRachel/galleries", + "#category": ("", "imagefap", "user"), + "#class": imagefap.ImagefapUserExtractor, + }, + { + "#url": "https://www.imagefap.com/profile.php?user=BdRachel", + "#category": ("", "imagefap", "user"), + "#class": imagefap.ImagefapUserExtractor, + }, + { + "#url": "https://beta.imagefap.com/profile.php?user=BdRachel", + "#category": ("", "imagefap", "user"), + "#class": imagefap.ImagefapUserExtractor, + }, ) diff --git a/test/results/imagetwist.py b/test/results/imagetwist.py index d00ad2729a..a230d1ca70 100644 --- a/test/results/imagetwist.py +++ b/test/results/imagetwist.py @@ -1,56 +1,47 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://imagetwist.com/f1i2s4vhvbrq/test.png", - "#category": ("imagehost", "imagetwist", "image"), - "#class" : imagehosts.ImagetwistImageExtractor, - "#sha1_url" : "8d5e168c0bee30211f821c6f3b2116e419d42671", - "#sha1_metadata": "d1060a4c2e3b73b83044e20681712c0ffdd6cfef", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - -{ - "#url" : "https://www.imagetwist.com/f1i2s4vhvbrq/test.png", - "#category": ("imagehost", "imagetwist", "image"), - "#class" : imagehosts.ImagetwistImageExtractor, -}, - -{ - "#url" : "https://phun.imagetwist.com/f1i2s4vhvbrq/test.png", - "#category": ("imagehost", "imagetwist", "image"), - "#class" : imagehosts.ImagetwistImageExtractor, -}, - -{ - "#url" : "https://imagehaha.com/f1i2s4vhvbrq/test.png", - "#category": ("imagehost", "imagetwist", "image"), - "#class" : imagehosts.ImagetwistImageExtractor, -}, - -{ - "#url" : "https://www.imagehaha.com/f1i2s4vhvbrq/test.png", - "#category": ("imagehost", "imagetwist", "image"), - "#class" : imagehosts.ImagetwistImageExtractor, -}, - -{ - "#url" : "https://imagetwist.com/p/gdldev/747223/digits", - "#category": ("imagehost", "imagetwist", "gallery"), - "#class" : imagehosts.ImagetwistGalleryExtractor, - "#urls" : ( - "https://imagetwist.com/j6eu91sbl9bs", - "https://imagetwist.com/vx4oh119izyr", - "https://imagetwist.com/n3td3a6vzzed", - "https://imagetwist.com/8uz6lmg31nmc", - ), -}, - + { + "#url": "https://imagetwist.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class": imagehosts.ImagetwistImageExtractor, + "#sha1_url": "8d5e168c0bee30211f821c6f3b2116e419d42671", + "#sha1_metadata": "d1060a4c2e3b73b83044e20681712c0ffdd6cfef", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, + { + "#url": "https://www.imagetwist.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class": imagehosts.ImagetwistImageExtractor, + }, + { + "#url": "https://phun.imagetwist.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class": imagehosts.ImagetwistImageExtractor, + }, + { + "#url": "https://imagehaha.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class": imagehosts.ImagetwistImageExtractor, + }, + { + "#url": "https://www.imagehaha.com/f1i2s4vhvbrq/test.png", + "#category": ("imagehost", "imagetwist", "image"), + "#class": imagehosts.ImagetwistImageExtractor, + }, + { + "#url": "https://imagetwist.com/p/gdldev/747223/digits", + "#category": ("imagehost", "imagetwist", "gallery"), + "#class": imagehosts.ImagetwistGalleryExtractor, + "#urls": ( + "https://imagetwist.com/j6eu91sbl9bs", + "https://imagetwist.com/vx4oh119izyr", + "https://imagetwist.com/n3td3a6vzzed", + "https://imagetwist.com/8uz6lmg31nmc", + ), + }, ) diff --git a/test/results/imagevenue.py b/test/results/imagevenue.py index f57b1c9474..e18b2c4fdd 100644 --- a/test/results/imagevenue.py +++ b/test/results/imagevenue.py @@ -1,34 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://www.imagevenue.com/ME13LS07", - "#category": ("imagehost", "imagevenue", "image"), - "#class" : imagehosts.ImagevenueImageExtractor, - "#pattern" : r"https://cdn-images\.imagevenue\.com/10/ac/05/ME13LS07_o\.png", - "#sha1_metadata": "ae15d6e3b2095f019eee84cd896700cd34b09c36", - "#sha1_content" : "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", -}, - -{ - "#url" : "https://www.imagevenue.com/view/o?i=92518_13732377annakarina424200712535AM_122_486lo.jpg&h=img150&l=loc486", - "#category": ("imagehost", "imagevenue", "image"), - "#class" : imagehosts.ImagevenueImageExtractor, - "#sha1_url": "8bf0254e29250d8f5026c0105bbdda3ee3d84980", -}, - -{ - "#url" : "http://img28116.imagevenue.com/img.php?image=th_52709_test_122_64lo.jpg", - "#category": ("imagehost", "imagevenue", "image"), - "#class" : imagehosts.ImagevenueImageExtractor, - "#sha1_url": "f98e3091df7f48a05fb60fbd86f789fc5ec56331", -}, - + { + "#url": "https://www.imagevenue.com/ME13LS07", + "#category": ("imagehost", "imagevenue", "image"), + "#class": imagehosts.ImagevenueImageExtractor, + "#pattern": r"https://cdn-images\.imagevenue\.com/10/ac/05/ME13LS07_o\.png", + "#sha1_metadata": "ae15d6e3b2095f019eee84cd896700cd34b09c36", + "#sha1_content": "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", + }, + { + "#url": "https://www.imagevenue.com/view/o?i=92518_13732377annakarina424200712535AM_122_486lo.jpg&h=img150&l=loc486", + "#category": ("imagehost", "imagevenue", "image"), + "#class": imagehosts.ImagevenueImageExtractor, + "#sha1_url": "8bf0254e29250d8f5026c0105bbdda3ee3d84980", + }, + { + "#url": "http://img28116.imagevenue.com/img.php?image=th_52709_test_122_64lo.jpg", + "#category": ("imagehost", "imagevenue", "image"), + "#class": imagehosts.ImagevenueImageExtractor, + "#sha1_url": "f98e3091df7f48a05fb60fbd86f789fc5ec56331", + }, ) diff --git a/test/results/imgbb.py b/test/results/imgbb.py index e2d1bc33d0..21a3e5d947 100644 --- a/test/results/imgbb.py +++ b/test/results/imgbb.py @@ -1,92 +1,80 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imgbb from gallery_dl import exception - +from gallery_dl.extractor import imgbb __tests__ = ( -{ - "#url" : "https://ibb.co/album/i5PggF", - "#category": ("", "imgbb", "album"), - "#class" : imgbb.ImgbbAlbumExtractor, - "#patten" : r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", - "#count" : 91, - "#sha1_url" : "efe7e5a76531436e3b82c87e4ebd34c4dfeb484c", - "#sha1_metadata": "f1ab5492adb6333409f3367566a6dd7110537e21", - - "album_id" : "i5PggF", - "album_name" : "British Scrap Book", - "extension" : "jpg", - "id" : r"re:^\w{7}$", - "title" : str, - "url" : r"re:https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", - "user" : "folkie", - "user_id" : "GvFMGK", - "displayname": "Folkie", - "width" : range(501, 1034), - "height" : range(335, 768), - "size" : range(74758, 439037), -}, - -{ - "#url" : "https://ibb.co/album/i5PggF?sort=title_asc", - "#comment" : "'sort' query argument", - "#category": ("", "imgbb", "album"), - "#class" : imgbb.ImgbbAlbumExtractor, - "#patten" : r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", - "#count" : 91, - "#sha1_url" : "cde36552cc132a27178f22a1b9aceaa4df7e1575", - "#sha1_metadata": "b98bbb7671e31ebf9c7585fb9fc691b71bcdb546", -}, - -{ - "#url" : "https://ibb.co/album/kYKpwF", - "#comment" : "no user data (#471)", - "#category": ("", "imgbb", "album"), - "#class" : imgbb.ImgbbAlbumExtractor, - "#sha1_url": "ac0abcfcb89f4df6adc2f7e4ff872f3b03ef1bc7", - - "displayname": "", - "user" : "", - "user_id" : "", -}, - -{ - "#url" : "https://ibb.co/album/hqgWrF", - "#comment" : "private", - "#category": ("", "imgbb", "album"), - "#class" : imgbb.ImgbbAlbumExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://folkie.imgbb.com", - "#category": ("", "imgbb", "user"), - "#class" : imgbb.ImgbbUserExtractor, - "#patten" : r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", - "#range" : "1-80", -}, - -{ - "#url" : "https://ibb.co/fUqh5b", - "#category": ("", "imgbb", "image"), - "#class" : imgbb.ImgbbImageExtractor, - "#pattern" : r"https://i\.ibb\.co/g3kvx80/Arundel-Ireeman-5\.jpg", - "#sha1_content": "c5a0965178a8b357acd8aa39660092918c63795e", - - "id" : "fUqh5b", - "title" : "Arundel Ireeman 5", - "url" : "https://i.ibb.co/g3kvx80/Arundel-Ireeman-5.jpg", - "width" : 960, - "height" : 719, - "user" : "folkie", - "user_id" : "GvFMGK", - "displayname": "Folkie", - "extension" : "jpg", -}, - + { + "#url": "https://ibb.co/album/i5PggF", + "#category": ("", "imgbb", "album"), + "#class": imgbb.ImgbbAlbumExtractor, + "#patten": r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", + "#count": 91, + "#sha1_url": "efe7e5a76531436e3b82c87e4ebd34c4dfeb484c", + "#sha1_metadata": "f1ab5492adb6333409f3367566a6dd7110537e21", + "album_id": "i5PggF", + "album_name": "British Scrap Book", + "extension": "jpg", + "id": r"re:^\w{7}$", + "title": str, + "url": r"re:https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", + "user": "folkie", + "user_id": "GvFMGK", + "displayname": "Folkie", + "width": range(501, 1034), + "height": range(335, 768), + "size": range(74758, 439037), + }, + { + "#url": "https://ibb.co/album/i5PggF?sort=title_asc", + "#comment": "'sort' query argument", + "#category": ("", "imgbb", "album"), + "#class": imgbb.ImgbbAlbumExtractor, + "#patten": r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", + "#count": 91, + "#sha1_url": "cde36552cc132a27178f22a1b9aceaa4df7e1575", + "#sha1_metadata": "b98bbb7671e31ebf9c7585fb9fc691b71bcdb546", + }, + { + "#url": "https://ibb.co/album/kYKpwF", + "#comment": "no user data (#471)", + "#category": ("", "imgbb", "album"), + "#class": imgbb.ImgbbAlbumExtractor, + "#sha1_url": "ac0abcfcb89f4df6adc2f7e4ff872f3b03ef1bc7", + "displayname": "", + "user": "", + "user_id": "", + }, + { + "#url": "https://ibb.co/album/hqgWrF", + "#comment": "private", + "#category": ("", "imgbb", "album"), + "#class": imgbb.ImgbbAlbumExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://folkie.imgbb.com", + "#category": ("", "imgbb", "user"), + "#class": imgbb.ImgbbUserExtractor, + "#patten": r"https://i\.ibb\.co/\w{7}/[\w-]+\.jpg", + "#range": "1-80", + }, + { + "#url": "https://ibb.co/fUqh5b", + "#category": ("", "imgbb", "image"), + "#class": imgbb.ImgbbImageExtractor, + "#pattern": r"https://i\.ibb\.co/g3kvx80/Arundel-Ireeman-5\.jpg", + "#sha1_content": "c5a0965178a8b357acd8aa39660092918c63795e", + "id": "fUqh5b", + "title": "Arundel Ireeman 5", + "url": "https://i.ibb.co/g3kvx80/Arundel-Ireeman-5.jpg", + "width": 960, + "height": 719, + "user": "folkie", + "user_id": "GvFMGK", + "displayname": "Folkie", + "extension": "jpg", + }, ) diff --git a/test/results/imgbox.py b/test/results/imgbox.py index ed0be59385..8beb6a750d 100644 --- a/test/results/imgbox.py +++ b/test/results/imgbox.py @@ -1,52 +1,44 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imgbox from gallery_dl import exception - +from gallery_dl.extractor import imgbox __tests__ = ( -{ - "#url" : "https://imgbox.com/g/JaX5V5HX7g", - "#category": ("", "imgbox", "gallery"), - "#class" : imgbox.ImgboxGalleryExtractor, - "#sha1_url" : "da4f15b161461119ee78841d4b8e8d054d95f906", - "#sha1_metadata": "4b1e62820ac2c6205b7ad0b6322cc8e00dbe1b0c", - "#sha1_content" : "d20307dc8511ac24d688859c55abf2e2cc2dd3cc", -}, - -{ - "#url" : "https://imgbox.com/g/cUGEkRbdZZ", - "#category": ("", "imgbox", "gallery"), - "#class" : imgbox.ImgboxGalleryExtractor, - "#sha1_url" : "76506a3aab175c456910851f66227e90484ca9f7", - "#sha1_metadata": "fb0427b87983197849fb2887905e758f3e50cb6e", -}, - -{ - "#url" : "https://imgbox.com/g/JaX5V5HX7h", - "#category": ("", "imgbox", "gallery"), - "#class" : imgbox.ImgboxGalleryExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://imgbox.com/qHhw7lpG", - "#category": ("", "imgbox", "image"), - "#class" : imgbox.ImgboxImageExtractor, - "#sha1_url" : "ee9cdea6c48ad0161c1b5f81f6b0c9110997038c", - "#sha1_metadata": "dfc72310026b45f3feb4f9cada20c79b2575e1af", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - -{ - "#url" : "https://imgbox.com/qHhw7lpH", - "#category": ("", "imgbox", "image"), - "#class" : imgbox.ImgboxImageExtractor, - "#exception": exception.NotFoundError, -}, - + { + "#url": "https://imgbox.com/g/JaX5V5HX7g", + "#category": ("", "imgbox", "gallery"), + "#class": imgbox.ImgboxGalleryExtractor, + "#sha1_url": "da4f15b161461119ee78841d4b8e8d054d95f906", + "#sha1_metadata": "4b1e62820ac2c6205b7ad0b6322cc8e00dbe1b0c", + "#sha1_content": "d20307dc8511ac24d688859c55abf2e2cc2dd3cc", + }, + { + "#url": "https://imgbox.com/g/cUGEkRbdZZ", + "#category": ("", "imgbox", "gallery"), + "#class": imgbox.ImgboxGalleryExtractor, + "#sha1_url": "76506a3aab175c456910851f66227e90484ca9f7", + "#sha1_metadata": "fb0427b87983197849fb2887905e758f3e50cb6e", + }, + { + "#url": "https://imgbox.com/g/JaX5V5HX7h", + "#category": ("", "imgbox", "gallery"), + "#class": imgbox.ImgboxGalleryExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://imgbox.com/qHhw7lpG", + "#category": ("", "imgbox", "image"), + "#class": imgbox.ImgboxImageExtractor, + "#sha1_url": "ee9cdea6c48ad0161c1b5f81f6b0c9110997038c", + "#sha1_metadata": "dfc72310026b45f3feb4f9cada20c79b2575e1af", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, + { + "#url": "https://imgbox.com/qHhw7lpH", + "#category": ("", "imgbox", "image"), + "#class": imgbox.ImgboxImageExtractor, + "#exception": exception.NotFoundError, + }, ) diff --git a/test/results/imgclick.py b/test/results/imgclick.py index 983f21f475..5191a1cdc8 100644 --- a/test/results/imgclick.py +++ b/test/results/imgclick.py @@ -1,20 +1,16 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "http://imgclick.net/4tbrre1oxew9/test-_-_.png.html", - "#category": ("imagehost", "imgclick", "image"), - "#class" : imagehosts.ImgclickImageExtractor, - "#sha1_url" : "140dcb250a325f2d26b2d918c18b8ac6a2a0f6ab", - "#sha1_metadata": "6895256143eab955622fc149aa367777a8815ba3", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - + { + "#url": "http://imgclick.net/4tbrre1oxew9/test-_-_.png.html", + "#category": ("imagehost", "imgclick", "image"), + "#class": imagehosts.ImgclickImageExtractor, + "#sha1_url": "140dcb250a325f2d26b2d918c18b8ac6a2a0f6ab", + "#sha1_metadata": "6895256143eab955622fc149aa367777a8815ba3", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, ) diff --git a/test/results/imgkiwi.py b/test/results/imgkiwi.py index 16a4aa9623..9d8a716a3e 100644 --- a/test/results/imgkiwi.py +++ b/test/results/imgkiwi.py @@ -1,51 +1,43 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import chevereto - __tests__ = ( -{ - "#url" : "https://img.kiwi/image/79de2c41-70f9-4a87-bd6d-00fe9997c0c4.JR2wZz", - "#category": ("chevereto", "imgkiwi", "image"), - "#class" : chevereto.CheveretoImageExtractor, - "#urls" : "https://img.kiwi/images/2023/02/28/11ac1ebf28a2eae8265026b28e9c4413.jpg", - "#sha1_content": "9ea704a77e2038b9008350682cfad53a614a60bd", - - "album" : "Kins3y Wolansk1", - "extension": "jpg", - "filename" : "11ac1ebf28a2eae8265026b28e9c4413", - "id" : "JR2wZz", - "url" : "https://img.kiwi/images/2023/02/28/11ac1ebf28a2eae8265026b28e9c4413.jpg", - "user" : "johnirl", -}, - -{ - "#url" : "https://img.kiwi/album/kins3y-wolansk1.8Jxc", - "#category": ("chevereto", "imgkiwi", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, - "#pattern" : chevereto.CheveretoImageExtractor.pattern, - "#count" : 19, -}, - -{ - "#url" : "https://img.kiwi/johnirl", - "#category": ("chevereto", "imgkiwi", "user"), - "#class" : chevereto.CheveretoUserExtractor, - "#pattern" : chevereto.CheveretoImageExtractor.pattern, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://img.kiwi/johnirl/albums", - "#category": ("chevereto", "imgkiwi", "user"), - "#class" : chevereto.CheveretoUserExtractor, - "#pattern" : chevereto.CheveretoAlbumExtractor.pattern, - "#count" : 50, -}, - + { + "#url": "https://img.kiwi/image/79de2c41-70f9-4a87-bd6d-00fe9997c0c4.JR2wZz", + "#category": ("chevereto", "imgkiwi", "image"), + "#class": chevereto.CheveretoImageExtractor, + "#urls": "https://img.kiwi/images/2023/02/28/11ac1ebf28a2eae8265026b28e9c4413.jpg", + "#sha1_content": "9ea704a77e2038b9008350682cfad53a614a60bd", + "album": "Kins3y Wolansk1", + "extension": "jpg", + "filename": "11ac1ebf28a2eae8265026b28e9c4413", + "id": "JR2wZz", + "url": "https://img.kiwi/images/2023/02/28/11ac1ebf28a2eae8265026b28e9c4413.jpg", + "user": "johnirl", + }, + { + "#url": "https://img.kiwi/album/kins3y-wolansk1.8Jxc", + "#category": ("chevereto", "imgkiwi", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + "#pattern": chevereto.CheveretoImageExtractor.pattern, + "#count": 19, + }, + { + "#url": "https://img.kiwi/johnirl", + "#category": ("chevereto", "imgkiwi", "user"), + "#class": chevereto.CheveretoUserExtractor, + "#pattern": chevereto.CheveretoImageExtractor.pattern, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://img.kiwi/johnirl/albums", + "#category": ("chevereto", "imgkiwi", "user"), + "#class": chevereto.CheveretoUserExtractor, + "#pattern": chevereto.CheveretoAlbumExtractor.pattern, + "#count": 50, + }, ) diff --git a/test/results/imgspice.py b/test/results/imgspice.py index c90393c7b4..adb75af22c 100644 --- a/test/results/imgspice.py +++ b/test/results/imgspice.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://imgspice.com/q1taxkhxprrn/58410038_cal022jsp_308191001.jpg.html", - "#category": ("imagehost", "imgspice", "image"), - "#class" : imagehosts.ImgspiceImageExtractor, - "#urls" : "https://img30.imgspice.com/i/03792/q1taxkhxprrn.jpg", - "#sha1_content" : "f1de8e58a7c2ef747a206a38f96c5623b8a83edc", - - "extension": "jpg", - "filename" : "58410038_cal022jsp_308191001", - "token" : "q1taxkhxprrn", -}, - + { + "#url": "https://imgspice.com/q1taxkhxprrn/58410038_cal022jsp_308191001.jpg.html", + "#category": ("imagehost", "imgspice", "image"), + "#class": imagehosts.ImgspiceImageExtractor, + "#urls": "https://img30.imgspice.com/i/03792/q1taxkhxprrn.jpg", + "#sha1_content": "f1de8e58a7c2ef747a206a38f96c5623b8a83edc", + "extension": "jpg", + "filename": "58410038_cal022jsp_308191001", + "token": "q1taxkhxprrn", + }, ) diff --git a/test/results/imgth.py b/test/results/imgth.py index 333e135224..c79b671629 100644 --- a/test/results/imgth.py +++ b/test/results/imgth.py @@ -1,34 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imgth - __tests__ = ( -{ - "#url" : "https://imgth.com/gallery/37/wallpaper-anime", - "#category": ("", "imgth", "gallery"), - "#class" : imgth.ImgthGalleryExtractor, - "#pattern" : r"https://imgth\.com/images/2009/11/25/wallpaper-anime_\w+\.jpg", - "#sha1_url": "4ae1d281ca2b48952cf5cca57e9914402ad72748", - - "count" : 12, - "date" : "dt:2009-11-25 18:21:00", - "extension" : "jpg", - "filename" : r"re:wallpaper-anime_\w+", - "gallery_id": 37, - "num" : int, - "title" : "Wallpaper anime", - "user" : "celebrities", -}, - -{ - "#url" : "https://www.imgth.com/gallery/37/wallpaper-anime", - "#category": ("", "imgth", "gallery"), - "#class" : imgth.ImgthGalleryExtractor, -}, - + { + "#url": "https://imgth.com/gallery/37/wallpaper-anime", + "#category": ("", "imgth", "gallery"), + "#class": imgth.ImgthGalleryExtractor, + "#pattern": r"https://imgth\.com/images/2009/11/25/wallpaper-anime_\w+\.jpg", + "#sha1_url": "4ae1d281ca2b48952cf5cca57e9914402ad72748", + "count": 12, + "date": "dt:2009-11-25 18:21:00", + "extension": "jpg", + "filename": r"re:wallpaper-anime_\w+", + "gallery_id": 37, + "num": int, + "title": "Wallpaper anime", + "user": "celebrities", + }, + { + "#url": "https://www.imgth.com/gallery/37/wallpaper-anime", + "#category": ("", "imgth", "gallery"), + "#class": imgth.ImgthGalleryExtractor, + }, ) diff --git a/test/results/imgur.py b/test/results/imgur.py index 991cf54695..8ac6c91f69 100644 --- a/test/results/imgur.py +++ b/test/results/imgur.py @@ -1,410 +1,365 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imgur -from gallery_dl import exception import datetime +from gallery_dl import exception +from gallery_dl.extractor import imgur __tests__ = ( -{ - "#url" : "https://imgur.com/21yMxCS", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, - "#sha1_url" : "6f2dcfb86815bdd72808c313e5f715610bc7b9b2", - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", - "#urls" : "https://i.imgur.com/21yMxCS.png", - - "account_id" : 0, - "comment_count" : int, - "cover_id" : "21yMxCS", - "date" : "dt:2016-11-10 14:24:35", - "description" : "", - "downvote_count": int, - "duration" : 0, - "ext" : "png", - "favorite" : False, - "favorite_count": 0, - "has_sound" : False, - "height" : 32, - "id" : "21yMxCS", - "image_count" : 1, - "in_most_viral" : False, - "is_ad" : False, - "is_album" : False, - "is_animated" : False, - "is_looping" : False, - "is_mature" : False, - "is_pending" : False, - "mime_type" : "image/png", - "name" : "test-テスト", - "point_count" : int, - "privacy" : "", - "score" : int, - "size" : 182, - "title" : "Test", - "upvote_count" : int, - "url" : "https://i.imgur.com/21yMxCS.png", - "view_count" : int, - "width" : 64, -}, - -{ - "#url" : "http://imgur.com/0gybAXR", - "#comment" : "gifv/mp4 video", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, - "#sha1_url" : "a2220eb265a55b0c95e0d3d721ec7665460e3fd7", - "#sha1_content": "a3c080e43f58f55243ab830569ba02309d59abfc", -}, - -{ - "#url" : "https://imgur.com/XFfsmuC", - "#comment" : "missing title in API response (#467)", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, - - "title": "Tears are a natural response to irritants", -}, - -{ - "#url" : "https://imgur.com/1Nily2P", - "#comment" : "animated png", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, - "#pattern" : "https://i.imgur.com/1Nily2P.png", -}, - -{ - "#url" : "https://imgur.com/zzzzzzz", - "#comment" : "not found", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://imgur.com/test-21yMxCS", - "#comment" : "slug", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://m.imgur.com/r/Celebs/iHJ7tsM", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://www.imgur.com/21yMxCS", - "#comment" : "www", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://m.imgur.com/21yMxCS", - "#comment" : "mobile", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://imgur.com/zxaY6", - "#comment" : "5 character key", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://imgur.io/zxaY6", - "#comment" : ".io", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://i.imgur.com/21yMxCS.png", - "#comment" : "direct link", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://i.imgur.io/21yMxCS.png", - "#comment" : "direct link .io", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://i.imgur.com/21yMxCSh.png", - "#comment" : "direct link thumbnail", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://i.imgur.com/zxaY6.gif", - "#comment" : "direct link (short)", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://i.imgur.com/zxaY6s.gif", - "#comment" : "direct link (short; thumb)", - "#category": ("", "imgur", "image"), - "#class" : imgur.ImgurImageExtractor, -}, - -{ - "#url" : "https://imgur.com/a/TcBmP", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, - "#sha1_url": "ce3552f550a5b5316bd9c7ae02e21e39f30c0563", - "#urls" : ( - "https://i.imgur.com/693j2Kr.jpg", - "https://i.imgur.com/ZNalkAC.jpg", - "https://i.imgur.com/lMox9Ek.jpg", - "https://i.imgur.com/6PryGOv.jpg", - "https://i.imgur.com/ecasnH2.jpg", - "https://i.imgur.com/NlJDmFG.jpg", - "https://i.imgur.com/aCwKs8S.jpg", - "https://i.imgur.com/Oz4rpxo.jpg", - "https://i.imgur.com/hE93Xsn.jpg", - "https://i.imgur.com/A5uBLSx.jpg", - "https://i.imgur.com/zZghWiD.jpg", - "https://i.imgur.com/ALV4fYV.jpg", - "https://i.imgur.com/FDd90t9.jpg", - "https://i.imgur.com/Txw37NO.jpg", - "https://i.imgur.com/DcLw7Cl.jpg", - "https://i.imgur.com/a4VChy8.jpg", - "https://i.imgur.com/auCwCig.jpg", - "https://i.imgur.com/Z8VihIb.jpg", - "https://i.imgur.com/6WDRFne.jpg", - ), - - "album" : { - "account_id" : 0, - "comment_count" : int, - "cover_id" : "693j2Kr", - "date" : "dt:2015-10-09 10:37:50", - "description" : "", - "downvote_count": 0, - "favorite" : False, + { + "#url": "https://imgur.com/21yMxCS", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + "#sha1_url": "6f2dcfb86815bdd72808c313e5f715610bc7b9b2", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + "#urls": "https://i.imgur.com/21yMxCS.png", + "account_id": 0, + "comment_count": int, + "cover_id": "21yMxCS", + "date": "dt:2016-11-10 14:24:35", + "description": "", + "downvote_count": int, + "duration": 0, + "ext": "png", + "favorite": False, "favorite_count": 0, - "id" : "TcBmP", - "image_count" : 19, - "in_most_viral" : False, - "is_ad" : False, - "is_album" : True, - "is_mature" : False, - "is_pending" : False, - "privacy" : "private", - "score" : int, - "title" : "138", - "upvote_count" : int, - "url" : "https://imgur.com/a/TcBmP", - "view_count" : int, - "virality" : int, - }, - "account_id" : 0, - "count" : 19, - "date" : datetime.datetime, - "description": "", - "ext" : "jpg", - "has_sound" : False, - "height" : int, - "id" : str, - "is_animated": False, - "is_looping" : False, - "mime_type" : "image/jpeg", - "name" : str, - "num" : int, - "size" : int, - "title" : str, - "type" : "image", - "updated_at" : None, - "url" : str, - "width" : int, -}, - -{ - "#url" : "https://imgur.com/a/eD9CT", - "#comment" : "large album", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://imgur.com/a/RhJXhVT/all", - "#comment" : "7 character album hash", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, - "#sha1_url": "695ef0c950023362a0163ee5041796300db76674", -}, - -{ - "#url" : "https://imgur.com/a/TcBmQ", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://imgur.com/a/pjOnJA0", - "#comment" : "empty, no 'media' (#2557)", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://imgur.com/a/138-TcBmP", - "#comment" : "slug", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, -}, - -{ - "#url" : "https://www.imgur.com/a/TcBmP", - "#comment" : "www", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, -}, - -{ - "#url" : "https://imgur.io/a/TcBmP", - "#comment" : ".io", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, -}, - -{ - "#url" : "https://m.imgur.com/a/TcBmP", - "#comment" : "mobile", - "#category": ("", "imgur", "album"), - "#class" : imgur.ImgurAlbumExtractor, -}, - -{ - "#url" : "https://imgur.com/gallery/zf2fIms", - "#comment" : "non-album gallery (#380)", - "#category": ("", "imgur", "gallery"), - "#class" : imgur.ImgurGalleryExtractor, - "#pattern" : "https://imgur.com/zf2fIms", -}, - -{ - "#url" : "https://imgur.com/gallery/eD9CT", - "#category": ("", "imgur", "gallery"), - "#class" : imgur.ImgurGalleryExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://imgur.com/gallery/guy-gets-out-of-car-during-long-traffic-jam-to-pet-dog-zf2fIms", - "#comment" : "slug", - "#category": ("", "imgur", "gallery"), - "#class" : imgur.ImgurGalleryExtractor, -}, - -{ - "#url" : "https://imgur.com/t/unmuted/26sEhNr", - "#category": ("", "imgur", "gallery"), - "#class" : imgur.ImgurGalleryExtractor, -}, - -{ - "#url" : "https://imgur.com/t/cat/qSB8NbN", - "#category": ("", "imgur", "gallery"), - "#class" : imgur.ImgurGalleryExtractor, -}, - -{ - "#url" : "https://imgur.io/t/cat/qSB8NbN", - "#comment" : ".io", - "#category": ("", "imgur", "gallery"), - "#class" : imgur.ImgurGalleryExtractor, -}, - -{ - "#url" : "https://imgur.com/user/Miguenzo", - "#category": ("", "imgur", "user"), - "#class" : imgur.ImgurUserExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://imgur.com/user/Miguenzo/posts", - "#category": ("", "imgur", "user"), - "#class" : imgur.ImgurUserExtractor, -}, - -{ - "#url" : "https://imgur.com/user/Miguenzo/submitted", - "#category": ("", "imgur", "user"), - "#class" : imgur.ImgurUserExtractor, -}, - -{ - "#url" : "https://imgur.com/user/Miguenzo/favorites", - "#category": ("", "imgur", "favorite"), - "#class" : imgur.ImgurFavoriteExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://imgur.com/user/mikf1/favorites/folder/11896757/public", - "#category": ("", "imgur", "favorite-folder"), - "#class" : imgur.ImgurFavoriteFolderExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#count" : 3, -}, - -{ - "#url" : "https://imgur.com/user/mikf1/favorites/folder/11896741/private", - "#category": ("", "imgur", "favorite-folder"), - "#class" : imgur.ImgurFavoriteFolderExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#count" : 5, -}, - -{ - "#url" : "https://imgur.com/r/pics", - "#category": ("", "imgur", "subreddit"), - "#class" : imgur.ImgurSubredditExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://imgur.com/t/animals", - "#category": ("", "imgur", "tag"), - "#class" : imgur.ImgurTagExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://imgur.com/search?q=cute+cat", - "#category": ("", "imgur", "search"), - "#class" : imgur.ImgurSearchExtractor, - "#pattern" : r"https://imgur\.com(/a)?/\w+$", - "#range" : "1-100", - "#count" : 100, -}, - + "has_sound": False, + "height": 32, + "id": "21yMxCS", + "image_count": 1, + "in_most_viral": False, + "is_ad": False, + "is_album": False, + "is_animated": False, + "is_looping": False, + "is_mature": False, + "is_pending": False, + "mime_type": "image/png", + "name": "test-テスト", + "point_count": int, + "privacy": "", + "score": int, + "size": 182, + "title": "Test", + "upvote_count": int, + "url": "https://i.imgur.com/21yMxCS.png", + "view_count": int, + "width": 64, + }, + { + "#url": "http://imgur.com/0gybAXR", + "#comment": "gifv/mp4 video", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + "#sha1_url": "a2220eb265a55b0c95e0d3d721ec7665460e3fd7", + "#sha1_content": "a3c080e43f58f55243ab830569ba02309d59abfc", + }, + { + "#url": "https://imgur.com/XFfsmuC", + "#comment": "missing title in API response (#467)", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + "title": "Tears are a natural response to irritants", + }, + { + "#url": "https://imgur.com/1Nily2P", + "#comment": "animated png", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + "#pattern": "https://i.imgur.com/1Nily2P.png", + }, + { + "#url": "https://imgur.com/zzzzzzz", + "#comment": "not found", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://imgur.com/test-21yMxCS", + "#comment": "slug", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://m.imgur.com/r/Celebs/iHJ7tsM", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://www.imgur.com/21yMxCS", + "#comment": "www", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://m.imgur.com/21yMxCS", + "#comment": "mobile", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://imgur.com/zxaY6", + "#comment": "5 character key", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://imgur.io/zxaY6", + "#comment": ".io", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://i.imgur.com/21yMxCS.png", + "#comment": "direct link", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://i.imgur.io/21yMxCS.png", + "#comment": "direct link .io", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://i.imgur.com/21yMxCSh.png", + "#comment": "direct link thumbnail", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://i.imgur.com/zxaY6.gif", + "#comment": "direct link (short)", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://i.imgur.com/zxaY6s.gif", + "#comment": "direct link (short; thumb)", + "#category": ("", "imgur", "image"), + "#class": imgur.ImgurImageExtractor, + }, + { + "#url": "https://imgur.com/a/TcBmP", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + "#sha1_url": "ce3552f550a5b5316bd9c7ae02e21e39f30c0563", + "#urls": ( + "https://i.imgur.com/693j2Kr.jpg", + "https://i.imgur.com/ZNalkAC.jpg", + "https://i.imgur.com/lMox9Ek.jpg", + "https://i.imgur.com/6PryGOv.jpg", + "https://i.imgur.com/ecasnH2.jpg", + "https://i.imgur.com/NlJDmFG.jpg", + "https://i.imgur.com/aCwKs8S.jpg", + "https://i.imgur.com/Oz4rpxo.jpg", + "https://i.imgur.com/hE93Xsn.jpg", + "https://i.imgur.com/A5uBLSx.jpg", + "https://i.imgur.com/zZghWiD.jpg", + "https://i.imgur.com/ALV4fYV.jpg", + "https://i.imgur.com/FDd90t9.jpg", + "https://i.imgur.com/Txw37NO.jpg", + "https://i.imgur.com/DcLw7Cl.jpg", + "https://i.imgur.com/a4VChy8.jpg", + "https://i.imgur.com/auCwCig.jpg", + "https://i.imgur.com/Z8VihIb.jpg", + "https://i.imgur.com/6WDRFne.jpg", + ), + "album": { + "account_id": 0, + "comment_count": int, + "cover_id": "693j2Kr", + "date": "dt:2015-10-09 10:37:50", + "description": "", + "downvote_count": 0, + "favorite": False, + "favorite_count": 0, + "id": "TcBmP", + "image_count": 19, + "in_most_viral": False, + "is_ad": False, + "is_album": True, + "is_mature": False, + "is_pending": False, + "privacy": "private", + "score": int, + "title": "138", + "upvote_count": int, + "url": "https://imgur.com/a/TcBmP", + "view_count": int, + "virality": int, + }, + "account_id": 0, + "count": 19, + "date": datetime.datetime, + "description": "", + "ext": "jpg", + "has_sound": False, + "height": int, + "id": str, + "is_animated": False, + "is_looping": False, + "mime_type": "image/jpeg", + "name": str, + "num": int, + "size": int, + "title": str, + "type": "image", + "updated_at": None, + "url": str, + "width": int, + }, + { + "#url": "https://imgur.com/a/eD9CT", + "#comment": "large album", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://imgur.com/a/RhJXhVT/all", + "#comment": "7 character album hash", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + "#sha1_url": "695ef0c950023362a0163ee5041796300db76674", + }, + { + "#url": "https://imgur.com/a/TcBmQ", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://imgur.com/a/pjOnJA0", + "#comment": "empty, no 'media' (#2557)", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://imgur.com/a/138-TcBmP", + "#comment": "slug", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + }, + { + "#url": "https://www.imgur.com/a/TcBmP", + "#comment": "www", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + }, + { + "#url": "https://imgur.io/a/TcBmP", + "#comment": ".io", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + }, + { + "#url": "https://m.imgur.com/a/TcBmP", + "#comment": "mobile", + "#category": ("", "imgur", "album"), + "#class": imgur.ImgurAlbumExtractor, + }, + { + "#url": "https://imgur.com/gallery/zf2fIms", + "#comment": "non-album gallery (#380)", + "#category": ("", "imgur", "gallery"), + "#class": imgur.ImgurGalleryExtractor, + "#pattern": "https://imgur.com/zf2fIms", + }, + { + "#url": "https://imgur.com/gallery/eD9CT", + "#category": ("", "imgur", "gallery"), + "#class": imgur.ImgurGalleryExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://imgur.com/gallery/guy-gets-out-of-car-during-long-traffic-jam-to-pet-dog-zf2fIms", + "#comment": "slug", + "#category": ("", "imgur", "gallery"), + "#class": imgur.ImgurGalleryExtractor, + }, + { + "#url": "https://imgur.com/t/unmuted/26sEhNr", + "#category": ("", "imgur", "gallery"), + "#class": imgur.ImgurGalleryExtractor, + }, + { + "#url": "https://imgur.com/t/cat/qSB8NbN", + "#category": ("", "imgur", "gallery"), + "#class": imgur.ImgurGalleryExtractor, + }, + { + "#url": "https://imgur.io/t/cat/qSB8NbN", + "#comment": ".io", + "#category": ("", "imgur", "gallery"), + "#class": imgur.ImgurGalleryExtractor, + }, + { + "#url": "https://imgur.com/user/Miguenzo", + "#category": ("", "imgur", "user"), + "#class": imgur.ImgurUserExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://imgur.com/user/Miguenzo/posts", + "#category": ("", "imgur", "user"), + "#class": imgur.ImgurUserExtractor, + }, + { + "#url": "https://imgur.com/user/Miguenzo/submitted", + "#category": ("", "imgur", "user"), + "#class": imgur.ImgurUserExtractor, + }, + { + "#url": "https://imgur.com/user/Miguenzo/favorites", + "#category": ("", "imgur", "favorite"), + "#class": imgur.ImgurFavoriteExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://imgur.com/user/mikf1/favorites/folder/11896757/public", + "#category": ("", "imgur", "favorite-folder"), + "#class": imgur.ImgurFavoriteFolderExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#count": 3, + }, + { + "#url": "https://imgur.com/user/mikf1/favorites/folder/11896741/private", + "#category": ("", "imgur", "favorite-folder"), + "#class": imgur.ImgurFavoriteFolderExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#count": 5, + }, + { + "#url": "https://imgur.com/r/pics", + "#category": ("", "imgur", "subreddit"), + "#class": imgur.ImgurSubredditExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://imgur.com/t/animals", + "#category": ("", "imgur", "tag"), + "#class": imgur.ImgurTagExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://imgur.com/search?q=cute+cat", + "#category": ("", "imgur", "search"), + "#class": imgur.ImgurSearchExtractor, + "#pattern": r"https://imgur\.com(/a)?/\w+$", + "#range": "1-100", + "#count": 100, + }, ) diff --git a/test/results/imxto.py b/test/results/imxto.py index 430b388b83..663757ec40 100644 --- a/test/results/imxto.py +++ b/test/results/imxto.py @@ -1,65 +1,54 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import imagehosts from gallery_dl import exception - +from gallery_dl.extractor import imagehosts __tests__ = ( -{ - "#url" : "https://imx.to/i/1qdeva", - "#comment" : "new-style URL", - "#category": ("imagehost", "imxto", "image"), - "#class" : imagehosts.ImxtoImageExtractor, - "#sha1_url" : "ab2173088a6cdef631d7a47dec4a5da1c6a00130", - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", - - "size" : 18, - "width" : 64, - "height": 32, - "hash" : "94d56c599223c59f3feb71ea603484d1", -}, - -{ - "#url" : "https://imx.to/img-57a2050547b97.html", - "#comment" : "old-style URL", - "#category": ("imagehost", "imxto", "image"), - "#class" : imagehosts.ImxtoImageExtractor, - "#sha1_url" : "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", - "#sha1_content": "54592f2635674c25677c6872db3709d343cdf92f", - - "size" : 5284, - "width" : 320, - "height": 160, - "hash" : "40da6aaa7b8c42b18ef74309bbc713fc", -}, - -{ - "#url" : "https://img.yt/img-57a2050547b97.html", - "#comment" : "img.yt domain", - "#category": ("imagehost", "imxto", "image"), - "#class" : imagehosts.ImxtoImageExtractor, - "#sha1_url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", -}, - -{ - "#url" : "https://imx.to/img-57a2050547b98.html", - "#category": ("imagehost", "imxto", "image"), - "#class" : imagehosts.ImxtoImageExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://imx.to/g/ozdy", - "#category": ("imagehost", "imxto", "gallery"), - "#class" : imagehosts.ImxtoGalleryExtractor, - "#pattern" : imagehosts.ImxtoImageExtractor.pattern, - "#count" : 40, - - "title": "untitled gallery", -}, - + { + "#url": "https://imx.to/i/1qdeva", + "#comment": "new-style URL", + "#category": ("imagehost", "imxto", "image"), + "#class": imagehosts.ImxtoImageExtractor, + "#sha1_url": "ab2173088a6cdef631d7a47dec4a5da1c6a00130", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + "size": 18, + "width": 64, + "height": 32, + "hash": "94d56c599223c59f3feb71ea603484d1", + }, + { + "#url": "https://imx.to/img-57a2050547b97.html", + "#comment": "old-style URL", + "#category": ("imagehost", "imxto", "image"), + "#class": imagehosts.ImxtoImageExtractor, + "#sha1_url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", + "#sha1_content": "54592f2635674c25677c6872db3709d343cdf92f", + "size": 5284, + "width": 320, + "height": 160, + "hash": "40da6aaa7b8c42b18ef74309bbc713fc", + }, + { + "#url": "https://img.yt/img-57a2050547b97.html", + "#comment": "img.yt domain", + "#category": ("imagehost", "imxto", "image"), + "#class": imagehosts.ImxtoImageExtractor, + "#sha1_url": "a83fe6ef1909a318c4d49fcf2caf62f36c3f9204", + }, + { + "#url": "https://imx.to/img-57a2050547b98.html", + "#category": ("imagehost", "imxto", "image"), + "#class": imagehosts.ImxtoImageExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://imx.to/g/ozdy", + "#category": ("imagehost", "imxto", "gallery"), + "#class": imagehosts.ImxtoGalleryExtractor, + "#pattern": imagehosts.ImxtoImageExtractor.pattern, + "#count": 40, + "title": "untitled gallery", + }, ) diff --git a/test/results/inkbunny.py b/test/results/inkbunny.py index 4e05da7cae..06c5d6b5d4 100644 --- a/test/results/inkbunny.py +++ b/test/results/inkbunny.py @@ -1,183 +1,157 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import inkbunny import datetime +from gallery_dl.extractor import inkbunny __tests__ = ( -{ - "#url" : "https://inkbunny.net/soina", - "#category": ("", "inkbunny", "user"), - "#class" : inkbunny.InkbunnyUserExtractor, - "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_soina_.+", - "#range" : "20-50", - - "date" : datetime.datetime, - "deleted" : bool, - "file_id" : r"re:[0-9]+", - "filename" : r"re:[0-9]+_soina_\w+", - "full_file_md5" : r"re:[0-9a-f]{32}", - "mimetype" : str, - "submission_id" : r"re:[0-9]+", - "user_id" : "20969", - "comments_count" : r"re:[0-9]+", - "favorite" : bool, - "favorites_count": r"re:[0-9]+", - "friends_only" : bool, - "guest_block" : bool, - "hidden" : bool, - "pagecount" : r"re:[0-9]+", - "pools" : list, - "pools_count" : int, - "public" : bool, - "rating_id" : r"re:[0-9]+", - "rating_name" : str, - "ratings" : list, - "scraps" : bool, - "tags" : list, - "title" : str, - "type_name" : str, - "username" : "soina", - "views" : str, -}, - -{ - "#url" : "https://inkbunny.net/gallery/soina", - "#category": ("", "inkbunny", "gallery"), - "#class" : inkbunny.InkbunnyUserExtractor, - "#range" : "1-25", - - "scraps": False, -}, - -{ - "#url" : "https://inkbunny.net/scraps/soina", - "#category": ("", "inkbunny", "scraps"), - "#class" : inkbunny.InkbunnyUserExtractor, - "#range" : "1-25", - - "scraps": True, -}, - -{ - "#url" : "https://inkbunny.net/poolview_process.php?pool_id=28985", - "#category": ("", "inkbunny", "pool"), - "#class" : inkbunny.InkbunnyPoolExtractor, - "#count" : 9, - - "pool_id": "28985", -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=pool&pool_id=28985&page=1&orderby=pool_order&random=no", - "#category": ("", "inkbunny", "pool"), - "#class" : inkbunny.InkbunnyPoolExtractor, -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?mode=pool&pool_id=28985", - "#category": ("", "inkbunny", "pool"), - "#class" : inkbunny.InkbunnyPoolExtractor, -}, - -{ - "#url" : "https://inkbunny.net/userfavorites_process.php?favs_user_id=20969", - "#category": ("", "inkbunny", "favorite"), - "#class" : inkbunny.InkbunnyFavoriteExtractor, - "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_\w+_.+", - "#range" : "20-50", - - "favs_user_id" : "20969", - "favs_username": "soina", -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=userfavs&random=no&orderby=fav_datetime&page=1&user_id=20969", - "#category": ("", "inkbunny", "favorite"), - "#class" : inkbunny.InkbunnyFavoriteExtractor, -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?mode=userfavs&user_id=20969", - "#category": ("", "inkbunny", "favorite"), - "#class" : inkbunny.InkbunnyFavoriteExtractor, -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=unreadsubs&page=1&orderby=unread_datetime", - "#category": ("", "inkbunny", "unread"), - "#class" : inkbunny.InkbunnyUnreadExtractor, -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?mode=unreadsubs", - "#category": ("", "inkbunny", "unread"), - "#class" : inkbunny.InkbunnyUnreadExtractor, -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=search&page=1&orderby=create_datetime&text=cute&stringtype=and&keywords=yes&title=yes&description=no&artist=&favsby=&type=&days=&keyword_id=&user_id=&random=&md5=", - "#category": ("", "inkbunny", "search"), - "#class" : inkbunny.InkbunnySearchExtractor, - "#range" : "1-10", - "#count" : 10, - - "search": { - "rid" : "ffffffffff", - "mode" : "search", - "page" : "1", - "orderby" : "create_datetime", - "text" : "cute", - "stringtype" : "and", - "keywords" : "yes", - "title" : "yes", - "description": "no", - }, -}, - -{ - "#url" : "https://inkbunny.net/submissionsviewall.php?mode=search", - "#category": ("", "inkbunny", "search"), - "#class" : inkbunny.InkbunnySearchExtractor, -}, - -{ - "#url" : "https://inkbunny.net/watchlist_process.php?mode=watching&user_id=20969", - "#category": ("", "inkbunny", "following"), - "#class" : inkbunny.InkbunnyFollowingExtractor, - "#pattern" : inkbunny.InkbunnyUserExtractor.pattern, - "#count" : ">= 90", -}, - -{ - "#url" : "https://inkbunny.net/usersviewall.php?rid=ffffffffff&mode=watching&page=1&user_id=20969&orderby=added&namesonly=", - "#category": ("", "inkbunny", "following"), - "#class" : inkbunny.InkbunnyFollowingExtractor, -}, - -{ - "#url" : "https://inkbunny.net/usersviewall.php?mode=watching&user_id=20969", - "#category": ("", "inkbunny", "following"), - "#class" : inkbunny.InkbunnyFollowingExtractor, -}, - -{ - "#url" : "https://inkbunny.net/s/1829715", - "#category": ("", "inkbunny", "post"), - "#class" : inkbunny.InkbunnyPostExtractor, - "#pattern" : r"https://[\w.]+\.metapix\.net/files/full/2626/2626843_soina_dscn2296\.jpg", - "#sha1_content": "cf69d8dddf0822a12b4eef1f4b2258bd600b36c8", -}, - -{ - "#url" : "https://inkbunny.net/s/2044094", - "#category": ("", "inkbunny", "post"), - "#class" : inkbunny.InkbunnyPostExtractor, - "#count" : 4, -}, - + { + "#url": "https://inkbunny.net/soina", + "#category": ("", "inkbunny", "user"), + "#class": inkbunny.InkbunnyUserExtractor, + "#pattern": r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_soina_.+", + "#range": "20-50", + "date": datetime.datetime, + "deleted": bool, + "file_id": r"re:[0-9]+", + "filename": r"re:[0-9]+_soina_\w+", + "full_file_md5": r"re:[0-9a-f]{32}", + "mimetype": str, + "submission_id": r"re:[0-9]+", + "user_id": "20969", + "comments_count": r"re:[0-9]+", + "favorite": bool, + "favorites_count": r"re:[0-9]+", + "friends_only": bool, + "guest_block": bool, + "hidden": bool, + "pagecount": r"re:[0-9]+", + "pools": list, + "pools_count": int, + "public": bool, + "rating_id": r"re:[0-9]+", + "rating_name": str, + "ratings": list, + "scraps": bool, + "tags": list, + "title": str, + "type_name": str, + "username": "soina", + "views": str, + }, + { + "#url": "https://inkbunny.net/gallery/soina", + "#category": ("", "inkbunny", "gallery"), + "#class": inkbunny.InkbunnyUserExtractor, + "#range": "1-25", + "scraps": False, + }, + { + "#url": "https://inkbunny.net/scraps/soina", + "#category": ("", "inkbunny", "scraps"), + "#class": inkbunny.InkbunnyUserExtractor, + "#range": "1-25", + "scraps": True, + }, + { + "#url": "https://inkbunny.net/poolview_process.php?pool_id=28985", + "#category": ("", "inkbunny", "pool"), + "#class": inkbunny.InkbunnyPoolExtractor, + "#count": 9, + "pool_id": "28985", + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=pool&pool_id=28985&page=1&orderby=pool_order&random=no", + "#category": ("", "inkbunny", "pool"), + "#class": inkbunny.InkbunnyPoolExtractor, + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?mode=pool&pool_id=28985", + "#category": ("", "inkbunny", "pool"), + "#class": inkbunny.InkbunnyPoolExtractor, + }, + { + "#url": "https://inkbunny.net/userfavorites_process.php?favs_user_id=20969", + "#category": ("", "inkbunny", "favorite"), + "#class": inkbunny.InkbunnyFavoriteExtractor, + "#pattern": r"https://[\w.]+\.metapix\.net/files/full/\d+/\d+_\w+_.+", + "#range": "20-50", + "favs_user_id": "20969", + "favs_username": "soina", + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=userfavs&random=no&orderby=fav_datetime&page=1&user_id=20969", + "#category": ("", "inkbunny", "favorite"), + "#class": inkbunny.InkbunnyFavoriteExtractor, + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?mode=userfavs&user_id=20969", + "#category": ("", "inkbunny", "favorite"), + "#class": inkbunny.InkbunnyFavoriteExtractor, + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=unreadsubs&page=1&orderby=unread_datetime", + "#category": ("", "inkbunny", "unread"), + "#class": inkbunny.InkbunnyUnreadExtractor, + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?mode=unreadsubs", + "#category": ("", "inkbunny", "unread"), + "#class": inkbunny.InkbunnyUnreadExtractor, + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?rid=ffffffffff&mode=search&page=1&orderby=create_datetime&text=cute&stringtype=and&keywords=yes&title=yes&description=no&artist=&favsby=&type=&days=&keyword_id=&user_id=&random=&md5=", + "#category": ("", "inkbunny", "search"), + "#class": inkbunny.InkbunnySearchExtractor, + "#range": "1-10", + "#count": 10, + "search": { + "rid": "ffffffffff", + "mode": "search", + "page": "1", + "orderby": "create_datetime", + "text": "cute", + "stringtype": "and", + "keywords": "yes", + "title": "yes", + "description": "no", + }, + }, + { + "#url": "https://inkbunny.net/submissionsviewall.php?mode=search", + "#category": ("", "inkbunny", "search"), + "#class": inkbunny.InkbunnySearchExtractor, + }, + { + "#url": "https://inkbunny.net/watchlist_process.php?mode=watching&user_id=20969", + "#category": ("", "inkbunny", "following"), + "#class": inkbunny.InkbunnyFollowingExtractor, + "#pattern": inkbunny.InkbunnyUserExtractor.pattern, + "#count": ">= 90", + }, + { + "#url": "https://inkbunny.net/usersviewall.php?rid=ffffffffff&mode=watching&page=1&user_id=20969&orderby=added&namesonly=", + "#category": ("", "inkbunny", "following"), + "#class": inkbunny.InkbunnyFollowingExtractor, + }, + { + "#url": "https://inkbunny.net/usersviewall.php?mode=watching&user_id=20969", + "#category": ("", "inkbunny", "following"), + "#class": inkbunny.InkbunnyFollowingExtractor, + }, + { + "#url": "https://inkbunny.net/s/1829715", + "#category": ("", "inkbunny", "post"), + "#class": inkbunny.InkbunnyPostExtractor, + "#pattern": r"https://[\w.]+\.metapix\.net/files/full/2626/2626843_soina_dscn2296\.jpg", + "#sha1_content": "cf69d8dddf0822a12b4eef1f4b2258bd600b36c8", + }, + { + "#url": "https://inkbunny.net/s/2044094", + "#category": ("", "inkbunny", "post"), + "#class": inkbunny.InkbunnyPostExtractor, + "#count": 4, + }, ) diff --git a/test/results/instagram.py b/test/results/instagram.py index 3cdb7cd86a..1d057ea2e2 100644 --- a/test/results/instagram.py +++ b/test/results/instagram.py @@ -1,280 +1,244 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import instagram - __tests__ = ( -{ - "#url" : "https://www.instagram.com/instagram/", - "#category": ("", "instagram", "user"), - "#class" : instagram.InstagramUserExtractor, - "#auth" : False, - "#options" : {"include": "all"}, - "#urls": [ - "https://www.instagram.com/instagram/info/", - "https://www.instagram.com/instagram/avatar/", - "https://www.instagram.com/stories/instagram/", - "https://www.instagram.com/instagram/highlights/", - "https://www.instagram.com/instagram/posts/", - "https://www.instagram.com/instagram/reels/", - "https://www.instagram.com/instagram/tagged/", - ], -}, - -{ - "#url" : "https://www.instagram.com/instagram/?hl=en", - "#category": ("", "instagram", "user"), - "#class" : instagram.InstagramUserExtractor, -}, - -{ - "#url" : "https://www.instagram.com/id:25025320/", - "#category": ("", "instagram", "user"), - "#class" : instagram.InstagramUserExtractor, -}, - -{ - "#url" : "https://www.instagram.com/instagram/posts/", - "#category": ("", "instagram", "posts"), - "#class" : instagram.InstagramPostsExtractor, - "#range" : "1-16", - "#count" : ">= 16", -}, - -{ - "#url" : "https://www.instagram.com/instagram/reels/", - "#category": ("", "instagram", "reels"), - "#class" : instagram.InstagramReelsExtractor, - "#range" : "40-60", - "#count" : ">= 20", -}, - -{ - "#url" : "https://www.instagram.com/instagram/tagged/", - "#category": ("", "instagram", "tagged"), - "#class" : instagram.InstagramTaggedExtractor, - "#range" : "1-16", - "#count" : ">= 16", - - "tagged_owner_id" : "25025320", - "tagged_username" : "instagram", - "tagged_full_name": "Instagram", -}, - -{ - "#url" : "https://www.instagram.com/kadakaofficial/guide/knit-i-need-collection/18131821684305217/", - "#category": ("", "instagram", "guide"), - "#class" : instagram.InstagramGuideExtractor, - "#range" : "1-16", - "#count" : ">= 16", -}, - -{ - "#url" : "https://www.instagram.com/instagram/saved/", - "#category": ("", "instagram", "saved"), - "#class" : instagram.InstagramSavedExtractor, -}, - -{ - "#url" : "https://www.instagram.com/instagram/saved/all-posts/", - "#category": ("", "instagram", "saved"), - "#class" : instagram.InstagramSavedExtractor, -}, - -{ - "#url" : "https://www.instagram.com/instagram/saved/collection_name/123456789/", - "#category": ("", "instagram", "collection"), - "#class" : instagram.InstagramCollectionExtractor, -}, - -{ - "#url" : "https://www.instagram.com/stories/instagram/", - "#category": ("", "instagram", "stories"), - "#class" : instagram.InstagramStoriesExtractor, -}, - -{ - "#url" : "https://www.instagram.com/stories/highlights/18042509488170095/", - "#category": ("", "instagram", "highlights"), - "#class" : instagram.InstagramStoriesExtractor, -}, - -{ - "#url" : "https://instagram.com/stories/geekmig/2724343156064789461", - "#category": ("", "instagram", "stories"), - "#class" : instagram.InstagramStoriesExtractor, -}, - -{ - "#url" : "https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1", - "#category": ("", "instagram", "highlights"), - "#class" : instagram.InstagramStoriesExtractor, -}, - -{ - "#url" : "https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1?story_media_id=2724343156064789461", - "#category": ("", "instagram", "highlights"), - "#class" : instagram.InstagramStoriesExtractor, -}, - -{ - "#url" : "https://www.instagram.com/instagram/highlights", - "#category": ("", "instagram", "highlights"), - "#class" : instagram.InstagramHighlightsExtractor, -}, - -{ - "#url" : "https://www.instagram.com/instagram/following", - "#category": ("", "instagram", "following"), - "#class" : instagram.InstagramFollowingExtractor, - "#range" : "1-16", - "#count" : ">= 16", -}, - -{ - "#url" : "https://www.instagram.com/explore/tags/instagram/", - "#category": ("", "instagram", "tag"), - "#class" : instagram.InstagramTagExtractor, - "#range" : "1-16", - "#count" : ">= 16", -}, - -{ - "#url" : "https://www.instagram.com/instagram/info", - "#category": ("", "instagram", "info"), - "#class" : instagram.InstagramInfoExtractor, - "#auth" : False, -}, - -{ - "#url" : "https://www.instagram.com/instagram/avatar", - "#category": ("", "instagram", "avatar"), - "#class" : instagram.InstagramAvatarExtractor, - "#pattern" : r"https://instagram\.[\w.-]+\.fbcdn\.net/v/t51\.2885-19/281440578_1088265838702675_6233856337905829714_n\.jpg", -}, - -{ - "#url" : "https://www.instagram.com/p/BqvsDleB3lV/", - "#comment" : "GraphImage", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, - "#pattern" : r"https://[^/]+\.(cdninstagram\.com|fbcdn\.net)/v(p/[0-9a-f]+/[0-9A-F]+)?/t51.2885-15/e35/44877605_725955034447492_3123079845831750529_n.jpg", - - "date" : "dt:2018-11-29 01:04:04", - "description" : str, - "height" : int, - "likes" : int, - "location_id" : "214424288", - "location_slug" : "hong-kong", - "location_url" : r"re:/explore/locations/214424288/hong-kong/", - "media_id" : "1922949326347663701", - "shortcode" : "BqvsDleB3lV", - "post_id" : "1922949326347663701", - "post_shortcode": "BqvsDleB3lV", - "post_url" : "https://www.instagram.com/p/BqvsDleB3lV/", - "tags" : ["#WHPsquares"], - "typename" : "GraphImage", - "username" : "instagram", - "width" : int, -}, - -{ - "#url" : "https://www.instagram.com/p/BoHk1haB5tM/", - "#comment" : "GraphSidecar", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, - "#count" : 5, - - "sidecar_media_id": "1875629777499953996", - "sidecar_shortcode": "BoHk1haB5tM", - "post_id" : "1875629777499953996", - "post_shortcode" : "BoHk1haB5tM", - "post_url" : "https://www.instagram.com/p/BoHk1haB5tM/", - "num" : int, - "likes" : int, - "username" : "instagram", -}, - -{ - "#url" : "https://www.instagram.com/p/Bqxp0VSBgJg/", - "#comment" : "GraphVideo", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, - "#pattern" : r"/46840863_726311431074534_7805566102611403091_n\.mp4", - - "date" : "dt:2018-11-29 19:23:58", - "description": str, - "height" : int, - "likes" : int, - "media_id" : "1923502432034620000", - "post_url" : "https://www.instagram.com/p/Bqxp0VSBgJg/", - "shortcode" : "Bqxp0VSBgJg", - "tags" : ["#ASMR"], - "typename" : "GraphVideo", - "username" : "instagram", - "width" : int, -}, - -{ - "#url" : "https://www.instagram.com/tv/BkQjCfsBIzi/", - "#comment" : "GraphVideo (IGTV)", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, - "#pattern" : r"/10000000_597132547321814_702169244961988209_n\.mp4", - - "date" : "dt:2018-06-20 19:51:32", - "description": str, - "height" : int, - "likes" : int, - "media_id" : "1806097553666903266", - "post_url" : "https://www.instagram.com/p/BkQjCfsBIzi/", - "shortcode" : "BkQjCfsBIzi", - "typename" : "GraphVideo", - "username" : "instagram", - "width" : int, -}, - -{ - "#url" : "https://www.instagram.com/p/BtOvDOfhvRr/", - "#comment" : "GraphSidecar with 2 embedded GraphVideo objects", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, - "#count" : 2, - - "post_url" : "https://www.instagram.com/p/BtOvDOfhvRr/", - "sidecar_media_id": "1967717017113261163", - "sidecar_shortcode": "BtOvDOfhvRr", - "video_url" : str, -}, - -{ - "#url" : "https://www.instagram.com/p/B_2lf3qAd3y/", - "#comment" : "GraphImage with tagged user", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, - - "tagged_users": [{ - "id" : "1246468638", - "username" : "kaaymbl", - "full_name": "Call Me Kay", -}], -}, - -{ - "#url" : "https://www.instagram.com/dm/p/CW042g7B9CY/", - "#comment" : "URL with username (#2085)", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, -}, - -{ - "#url" : "https://www.instagram.com/reel/CDg_6Y1pxWu/", - "#category": ("", "instagram", "post"), - "#class" : instagram.InstagramPostExtractor, -}, - + { + "#url": "https://www.instagram.com/instagram/", + "#category": ("", "instagram", "user"), + "#class": instagram.InstagramUserExtractor, + "#auth": False, + "#options": {"include": "all"}, + "#urls": [ + "https://www.instagram.com/instagram/info/", + "https://www.instagram.com/instagram/avatar/", + "https://www.instagram.com/stories/instagram/", + "https://www.instagram.com/instagram/highlights/", + "https://www.instagram.com/instagram/posts/", + "https://www.instagram.com/instagram/reels/", + "https://www.instagram.com/instagram/tagged/", + ], + }, + { + "#url": "https://www.instagram.com/instagram/?hl=en", + "#category": ("", "instagram", "user"), + "#class": instagram.InstagramUserExtractor, + }, + { + "#url": "https://www.instagram.com/id:25025320/", + "#category": ("", "instagram", "user"), + "#class": instagram.InstagramUserExtractor, + }, + { + "#url": "https://www.instagram.com/instagram/posts/", + "#category": ("", "instagram", "posts"), + "#class": instagram.InstagramPostsExtractor, + "#range": "1-16", + "#count": ">= 16", + }, + { + "#url": "https://www.instagram.com/instagram/reels/", + "#category": ("", "instagram", "reels"), + "#class": instagram.InstagramReelsExtractor, + "#range": "40-60", + "#count": ">= 20", + }, + { + "#url": "https://www.instagram.com/instagram/tagged/", + "#category": ("", "instagram", "tagged"), + "#class": instagram.InstagramTaggedExtractor, + "#range": "1-16", + "#count": ">= 16", + "tagged_owner_id": "25025320", + "tagged_username": "instagram", + "tagged_full_name": "Instagram", + }, + { + "#url": "https://www.instagram.com/kadakaofficial/guide/knit-i-need-collection/18131821684305217/", + "#category": ("", "instagram", "guide"), + "#class": instagram.InstagramGuideExtractor, + "#range": "1-16", + "#count": ">= 16", + }, + { + "#url": "https://www.instagram.com/instagram/saved/", + "#category": ("", "instagram", "saved"), + "#class": instagram.InstagramSavedExtractor, + }, + { + "#url": "https://www.instagram.com/instagram/saved/all-posts/", + "#category": ("", "instagram", "saved"), + "#class": instagram.InstagramSavedExtractor, + }, + { + "#url": "https://www.instagram.com/instagram/saved/collection_name/123456789/", + "#category": ("", "instagram", "collection"), + "#class": instagram.InstagramCollectionExtractor, + }, + { + "#url": "https://www.instagram.com/stories/instagram/", + "#category": ("", "instagram", "stories"), + "#class": instagram.InstagramStoriesExtractor, + }, + { + "#url": "https://www.instagram.com/stories/highlights/18042509488170095/", + "#category": ("", "instagram", "highlights"), + "#class": instagram.InstagramStoriesExtractor, + }, + { + "#url": "https://instagram.com/stories/geekmig/2724343156064789461", + "#category": ("", "instagram", "stories"), + "#class": instagram.InstagramStoriesExtractor, + }, + { + "#url": "https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1", + "#category": ("", "instagram", "highlights"), + "#class": instagram.InstagramStoriesExtractor, + }, + { + "#url": "https://www.instagram.com/s/aGlnaGxpZ2h0OjE4MDQyNTA5NDg4MTcwMDk1?story_media_id=2724343156064789461", + "#category": ("", "instagram", "highlights"), + "#class": instagram.InstagramStoriesExtractor, + }, + { + "#url": "https://www.instagram.com/instagram/highlights", + "#category": ("", "instagram", "highlights"), + "#class": instagram.InstagramHighlightsExtractor, + }, + { + "#url": "https://www.instagram.com/instagram/following", + "#category": ("", "instagram", "following"), + "#class": instagram.InstagramFollowingExtractor, + "#range": "1-16", + "#count": ">= 16", + }, + { + "#url": "https://www.instagram.com/explore/tags/instagram/", + "#category": ("", "instagram", "tag"), + "#class": instagram.InstagramTagExtractor, + "#range": "1-16", + "#count": ">= 16", + }, + { + "#url": "https://www.instagram.com/instagram/info", + "#category": ("", "instagram", "info"), + "#class": instagram.InstagramInfoExtractor, + "#auth": False, + }, + { + "#url": "https://www.instagram.com/instagram/avatar", + "#category": ("", "instagram", "avatar"), + "#class": instagram.InstagramAvatarExtractor, + "#pattern": r"https://instagram\.[\w.-]+\.fbcdn\.net/v/t51\.2885-19/281440578_1088265838702675_6233856337905829714_n\.jpg", + }, + { + "#url": "https://www.instagram.com/p/BqvsDleB3lV/", + "#comment": "GraphImage", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + "#pattern": r"https://[^/]+\.(cdninstagram\.com|fbcdn\.net)/v(p/[0-9a-f]+/[0-9A-F]+)?/t51.2885-15/e35/44877605_725955034447492_3123079845831750529_n.jpg", + "date": "dt:2018-11-29 01:04:04", + "description": str, + "height": int, + "likes": int, + "location_id": "214424288", + "location_slug": "hong-kong", + "location_url": r"re:/explore/locations/214424288/hong-kong/", + "media_id": "1922949326347663701", + "shortcode": "BqvsDleB3lV", + "post_id": "1922949326347663701", + "post_shortcode": "BqvsDleB3lV", + "post_url": "https://www.instagram.com/p/BqvsDleB3lV/", + "tags": ["#WHPsquares"], + "typename": "GraphImage", + "username": "instagram", + "width": int, + }, + { + "#url": "https://www.instagram.com/p/BoHk1haB5tM/", + "#comment": "GraphSidecar", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + "#count": 5, + "sidecar_media_id": "1875629777499953996", + "sidecar_shortcode": "BoHk1haB5tM", + "post_id": "1875629777499953996", + "post_shortcode": "BoHk1haB5tM", + "post_url": "https://www.instagram.com/p/BoHk1haB5tM/", + "num": int, + "likes": int, + "username": "instagram", + }, + { + "#url": "https://www.instagram.com/p/Bqxp0VSBgJg/", + "#comment": "GraphVideo", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + "#pattern": r"/46840863_726311431074534_7805566102611403091_n\.mp4", + "date": "dt:2018-11-29 19:23:58", + "description": str, + "height": int, + "likes": int, + "media_id": "1923502432034620000", + "post_url": "https://www.instagram.com/p/Bqxp0VSBgJg/", + "shortcode": "Bqxp0VSBgJg", + "tags": ["#ASMR"], + "typename": "GraphVideo", + "username": "instagram", + "width": int, + }, + { + "#url": "https://www.instagram.com/tv/BkQjCfsBIzi/", + "#comment": "GraphVideo (IGTV)", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + "#pattern": r"/10000000_597132547321814_702169244961988209_n\.mp4", + "date": "dt:2018-06-20 19:51:32", + "description": str, + "height": int, + "likes": int, + "media_id": "1806097553666903266", + "post_url": "https://www.instagram.com/p/BkQjCfsBIzi/", + "shortcode": "BkQjCfsBIzi", + "typename": "GraphVideo", + "username": "instagram", + "width": int, + }, + { + "#url": "https://www.instagram.com/p/BtOvDOfhvRr/", + "#comment": "GraphSidecar with 2 embedded GraphVideo objects", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + "#count": 2, + "post_url": "https://www.instagram.com/p/BtOvDOfhvRr/", + "sidecar_media_id": "1967717017113261163", + "sidecar_shortcode": "BtOvDOfhvRr", + "video_url": str, + }, + { + "#url": "https://www.instagram.com/p/B_2lf3qAd3y/", + "#comment": "GraphImage with tagged user", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + "tagged_users": [ + { + "id": "1246468638", + "username": "kaaymbl", + "full_name": "Call Me Kay", + } + ], + }, + { + "#url": "https://www.instagram.com/dm/p/CW042g7B9CY/", + "#comment": "URL with username (#2085)", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + }, + { + "#url": "https://www.instagram.com/reel/CDg_6Y1pxWu/", + "#category": ("", "instagram", "post"), + "#class": instagram.InstagramPostExtractor, + }, ) diff --git a/test/results/issuu.py b/test/results/issuu.py index 4a90be13b3..b43e324d3e 100644 --- a/test/results/issuu.py +++ b/test/results/issuu.py @@ -1,46 +1,40 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import issuu - __tests__ = ( -{ - "#url" : "https://issuu.com/issuu/docs/motions-1-2019/", - "#category": ("", "issuu", "publication"), - "#class" : issuu.IssuuPublicationExtractor, - "#pattern" : r"https://image.isu.pub/190916155301-\w+/jpg/page_\d+.jpg", - "#count" : 36, - - "document" : { - "access" : "PUBLIC", - "contentRating": { - "isAdsafe" : True, - "isExplicit": False, - "isReviewed": True, + { + "#url": "https://issuu.com/issuu/docs/motions-1-2019/", + "#category": ("", "issuu", "publication"), + "#class": issuu.IssuuPublicationExtractor, + "#pattern": r"https://image.isu.pub/190916155301-\w+/jpg/page_\d+.jpg", + "#count": 36, + "document": { + "access": "PUBLIC", + "contentRating": { + "isAdsafe": True, + "isExplicit": False, + "isReviewed": True, + }, + "date": "dt:2019-09-16 00:00:00", + "description": r"re:Motions, the brand new publication by I", + "documentName": "motions-1-2019", + "pageCount": 36, + "publicationId": "d99ec95935f15091b040cb8060f05510", + "title": "Motions by Issuu - Issue 1", + "username": "issuu", }, - "date" : "dt:2019-09-16 00:00:00", - "description" : r"re:Motions, the brand new publication by I", - "documentName" : "motions-1-2019", - "pageCount" : 36, - "publicationId": "d99ec95935f15091b040cb8060f05510", - "title" : "Motions by Issuu - Issue 1", - "username" : "issuu", + "extension": "jpg", + "filename": r"re:page_\d+", + "num": int, + }, + { + "#url": "https://issuu.com/issuu", + "#category": ("", "issuu", "user"), + "#class": issuu.IssuuUserExtractor, + "#pattern": issuu.IssuuPublicationExtractor.pattern, + "#count": "> 25", }, - "extension": "jpg", - "filename" : r"re:page_\d+", - "num" : int, -}, - -{ - "#url" : "https://issuu.com/issuu", - "#category": ("", "issuu", "user"), - "#class" : issuu.IssuuUserExtractor, - "#pattern" : issuu.IssuuPublicationExtractor.pattern, - "#count" : "> 25", -}, - ) diff --git a/test/results/itaku.py b/test/results/itaku.py index 27b59414b1..7cc6ef40d6 100644 --- a/test/results/itaku.py +++ b/test/results/itaku.py @@ -1,81 +1,74 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import itaku - __tests__ = ( -{ - "#url" : "https://itaku.ee/profile/piku/gallery", - "#category": ("", "itaku", "gallery"), - "#class" : itaku.ItakuGalleryExtractor, - "#pattern" : r"https://itaku\.ee/api/media/gallery_imgs/[^/?#]+\.(jpg|png|gif)", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://itaku.ee/images/100471", - "#category": ("", "itaku", "image"), - "#class" : itaku.ItakuImageExtractor, - "#urls" : "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT.png", - - "already_pinned" : None, - "blacklisted" : { - "blacklisted_tags": [], - "is_blacklisted" : False, + { + "#url": "https://itaku.ee/profile/piku/gallery", + "#category": ("", "itaku", "gallery"), + "#class": itaku.ItakuGalleryExtractor, + "#pattern": r"https://itaku\.ee/api/media/gallery_imgs/[^/?#]+\.(jpg|png|gif)", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://itaku.ee/images/100471", + "#category": ("", "itaku", "image"), + "#class": itaku.ItakuImageExtractor, + "#urls": "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT.png", + "already_pinned": None, + "blacklisted": { + "blacklisted_tags": [], + "is_blacklisted": False, + }, + "can_reshare": True, + "date": "dt:2022-05-05 19:21:17", + "date_added": "2022-05-05T19:21:17.674148Z", + "date_edited": "2022-05-25T14:37:46.220612Z", + "description": "sketch from drawpile", + "extension": "png", + "filename": "220504_oUNIAFT", + "hotness_score": float, + "id": 100471, + "image": "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT.png", + "image_xl": "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT/lg.jpg", + "liked_by_you": False, + "maturity_rating": "SFW", + "num_comments": int, + "num_likes": int, + "num_reshares": int, + "obj_tags": 136446, + "owner": 16775, + "owner_avatar": "https://itaku.ee/api/media/profile_pics/av2022r_vKYVywc/md.jpg", + "owner_displayname": "Piku", + "owner_username": "piku", + "reshared_by_you": False, + "sections": ["Fanart/Miku"], + "tags": list, + "tags_character": ["hatsune_miku"], + "tags_copyright": ["vocaloid"], + "tags_general": [ + "twintails", + "green_hair", + "flag", + "gloves", + "green_eyes", + "female", + "racing_miku", + ], + "title": "Racing Miku 2022 Ver.", + "too_mature": False, + "uncompressed_filesize": "0.62", + "video": None, + "visibility": "PUBLIC", + }, + { + "#url": "https://itaku.ee/images/19465", + "#comment": "video", + "#category": ("", "itaku", "image"), + "#class": itaku.ItakuImageExtractor, + "#urls": "https://itaku.ee/api/media/gallery_vids/sleepy_af_OY5GHWw.mp4", }, - "can_reshare" : True, - "date" : "dt:2022-05-05 19:21:17", - "date_added" : "2022-05-05T19:21:17.674148Z", - "date_edited" : "2022-05-25T14:37:46.220612Z", - "description" : "sketch from drawpile", - "extension" : "png", - "filename" : "220504_oUNIAFT", - "hotness_score" : float, - "id" : 100471, - "image" : "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT.png", - "image_xl" : "https://itaku.ee/api/media/gallery_imgs/220504_oUNIAFT/lg.jpg", - "liked_by_you" : False, - "maturity_rating" : "SFW", - "num_comments" : int, - "num_likes" : int, - "num_reshares" : int, - "obj_tags" : 136446, - "owner" : 16775, - "owner_avatar" : "https://itaku.ee/api/media/profile_pics/av2022r_vKYVywc/md.jpg", - "owner_displayname": "Piku", - "owner_username" : "piku", - "reshared_by_you" : False, - "sections" : ["Fanart/Miku"], - "tags" : list, - "tags_character" : ["hatsune_miku"], - "tags_copyright" : ["vocaloid"], - "tags_general": [ - "twintails", - "green_hair", - "flag", - "gloves", - "green_eyes", - "female", - "racing_miku", - ], - "title" : "Racing Miku 2022 Ver.", - "too_mature" : False, - "uncompressed_filesize": "0.62", - "video" : None, - "visibility" : "PUBLIC", -}, - -{ - "#url" : "https://itaku.ee/images/19465", - "#comment" : "video", - "#category": ("", "itaku", "image"), - "#class" : itaku.ItakuImageExtractor, - "#urls" : "https://itaku.ee/api/media/gallery_vids/sleepy_af_OY5GHWw.mp4", -}, - ) diff --git a/test/results/itchio.py b/test/results/itchio.py index f49bd69f31..445754d54e 100644 --- a/test/results/itchio.py +++ b/test/results/itchio.py @@ -1,33 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import itchio - __tests__ = ( -{ - "#url" : "https://sirtartarus.itch.io/a-craft-of-mine", - "#category": ("", "itchio", "game"), - "#class" : itchio.ItchioGameExtractor, - "#pattern" : r"https://(dl.itch.zone|itchio-mirror.\w+.r2.cloudflarestorage.com)/upload2/game/1983311/\d+\?", - "#count" : 3, - - "extension": "", - "filename" : r"re:\d+", - "game" : { - "id" : 1983311, - "noun" : "game", - "title": "A Craft Of Mine", - "url" : "https://sirtartarus.itch.io/a-craft-of-mine", + { + "#url": "https://sirtartarus.itch.io/a-craft-of-mine", + "#category": ("", "itchio", "game"), + "#class": itchio.ItchioGameExtractor, + "#pattern": r"https://(dl.itch.zone|itchio-mirror.\w+.r2.cloudflarestorage.com)/upload2/game/1983311/\d+\?", + "#count": 3, + "extension": "", + "filename": r"re:\d+", + "game": { + "id": 1983311, + "noun": "game", + "title": "A Craft Of Mine", + "url": "https://sirtartarus.itch.io/a-craft-of-mine", + }, + "user": { + "id": 4060052, + "name": "SirTartarus", + "url": "https://sirtartarus.itch.io", + }, }, - "user" : { - "id" : 4060052, - "name": "SirTartarus", - "url" : "https://sirtartarus.itch.io", - }, -}, - ) diff --git a/test/results/joyreactor.py b/test/results/joyreactor.py index 4c29c2130c..320e8e150c 100644 --- a/test/results/joyreactor.py +++ b/test/results/joyreactor.py @@ -1,108 +1,93 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import reactor - __tests__ = ( -{ - "#url" : "http://joyreactor.cc/tag/Advent+Cirno", - "#category": ("reactor", "joyreactor", "tag"), - "#class" : reactor.ReactorTagExtractor, - "#count" : ">= 15", -}, - -{ - "#url" : "http://joyreactor.com/tag/Cirno", - "#category": ("reactor", "joyreactor", "tag"), - "#class" : reactor.ReactorTagExtractor, - "#sha1_url": "aa59090590b26f4654881301fe8fe748a51625a8", -}, - -{ - "#url" : "http://joyreactor.com/tag/Dark+Souls+2/best", - "#comment" : "'best' rating (#3073)", - "#category": ("reactor", "joyreactor", "tag"), - "#class" : reactor.ReactorTagExtractor, - "#count" : 4, -}, - -{ - "#url" : "http://joyreactor.cc/search/Nature", - "#category": ("reactor", "joyreactor", "search"), - "#class" : reactor.ReactorSearchExtractor, - "#range" : "1-25", - "#count" : ">= 20", -}, - -{ - "#url" : "http://joyreactor.com/search?q=Nature", - "#category": ("reactor", "joyreactor", "search"), - "#class" : reactor.ReactorSearchExtractor, - "#range" : "1-25", - "#count" : ">= 20", -}, - -{ - "#url" : "http://joyreactor.cc/user/hemantic", - "#category": ("reactor", "joyreactor", "user"), - "#class" : reactor.ReactorUserExtractor, -}, - -{ - "#url" : "http://joyreactor.com/user/Tacoman123", - "#category": ("reactor", "joyreactor", "user"), - "#class" : reactor.ReactorUserExtractor, - "#sha1_url": "60ce9a3e3db791a0899f7fb7643b5b87d09ae3b5", -}, - -{ - "#url" : "http://joyreactor.com/post/3721876", - "#comment" : "single image", - "#category": ("reactor", "joyreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#pattern" : r"http://img\d\.joyreactor\.com/pics/post/full/cartoon-painting-monster-lake-4841316.jpeg", - "#count" : 1, - "#sha1_metadata": "2207a7dfed55def2042b6c2554894c8d7fda386e", -}, - -{ - "#url" : "http://joyreactor.com/post/3713804", - "#comment" : "4 images", - "#category": ("reactor", "joyreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#pattern" : r"http://img\d\.joyreactor\.com/pics/post/full/movie-tv-godzilla-monsters-\d+\.jpeg", - "#count" : 4, - "#sha1_metadata": "d7da9ba7809004c809eedcf6f1c06ad0fbb3df21", -}, - -{ - "#url" : "http://joyreactor.com/post/3726210", - "#comment" : "gif / video", - "#category": ("reactor", "joyreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#sha1_url" : "60f3b9a0a3918b269bea9b4f8f1a5ab3c2c550f8", - "#sha1_metadata": "8949d9d5fc469dab264752432efbaa499561664a", -}, - -{ - "#url" : "http://joyreactor.com/post/3668724", - "#comment" : "youtube embed", - "#category": ("reactor", "joyreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#sha1_url" : "bf1666eddcff10c9b58f6be63fa94e4e13074214", - "#sha1_metadata": "e18b1ffbd79d76f9a0e90b6d474cc2499e343f0b", -}, - -{ - "#url" : "http://joyreactor.cc/post/1299", - "#comment" : "'malformed' JSON", - "#category": ("reactor", "joyreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#sha1_url": "ab02c6eb7b4035ad961b29ee0770ee41be2fcc39", -}, - + { + "#url": "http://joyreactor.cc/tag/Advent+Cirno", + "#category": ("reactor", "joyreactor", "tag"), + "#class": reactor.ReactorTagExtractor, + "#count": ">= 15", + }, + { + "#url": "http://joyreactor.com/tag/Cirno", + "#category": ("reactor", "joyreactor", "tag"), + "#class": reactor.ReactorTagExtractor, + "#sha1_url": "aa59090590b26f4654881301fe8fe748a51625a8", + }, + { + "#url": "http://joyreactor.com/tag/Dark+Souls+2/best", + "#comment": "'best' rating (#3073)", + "#category": ("reactor", "joyreactor", "tag"), + "#class": reactor.ReactorTagExtractor, + "#count": 4, + }, + { + "#url": "http://joyreactor.cc/search/Nature", + "#category": ("reactor", "joyreactor", "search"), + "#class": reactor.ReactorSearchExtractor, + "#range": "1-25", + "#count": ">= 20", + }, + { + "#url": "http://joyreactor.com/search?q=Nature", + "#category": ("reactor", "joyreactor", "search"), + "#class": reactor.ReactorSearchExtractor, + "#range": "1-25", + "#count": ">= 20", + }, + { + "#url": "http://joyreactor.cc/user/hemantic", + "#category": ("reactor", "joyreactor", "user"), + "#class": reactor.ReactorUserExtractor, + }, + { + "#url": "http://joyreactor.com/user/Tacoman123", + "#category": ("reactor", "joyreactor", "user"), + "#class": reactor.ReactorUserExtractor, + "#sha1_url": "60ce9a3e3db791a0899f7fb7643b5b87d09ae3b5", + }, + { + "#url": "http://joyreactor.com/post/3721876", + "#comment": "single image", + "#category": ("reactor", "joyreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#pattern": r"http://img\d\.joyreactor\.com/pics/post/full/cartoon-painting-monster-lake-4841316.jpeg", + "#count": 1, + "#sha1_metadata": "2207a7dfed55def2042b6c2554894c8d7fda386e", + }, + { + "#url": "http://joyreactor.com/post/3713804", + "#comment": "4 images", + "#category": ("reactor", "joyreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#pattern": r"http://img\d\.joyreactor\.com/pics/post/full/movie-tv-godzilla-monsters-\d+\.jpeg", + "#count": 4, + "#sha1_metadata": "d7da9ba7809004c809eedcf6f1c06ad0fbb3df21", + }, + { + "#url": "http://joyreactor.com/post/3726210", + "#comment": "gif / video", + "#category": ("reactor", "joyreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#sha1_url": "60f3b9a0a3918b269bea9b4f8f1a5ab3c2c550f8", + "#sha1_metadata": "8949d9d5fc469dab264752432efbaa499561664a", + }, + { + "#url": "http://joyreactor.com/post/3668724", + "#comment": "youtube embed", + "#category": ("reactor", "joyreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#sha1_url": "bf1666eddcff10c9b58f6be63fa94e4e13074214", + "#sha1_metadata": "e18b1ffbd79d76f9a0e90b6d474cc2499e343f0b", + }, + { + "#url": "http://joyreactor.cc/post/1299", + "#comment": "'malformed' JSON", + "#category": ("reactor", "joyreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#sha1_url": "ab02c6eb7b4035ad961b29ee0770ee41be2fcc39", + }, ) diff --git a/test/results/jpgfish.py b/test/results/jpgfish.py index bef1d6e052..81d4944cf5 100644 --- a/test/results/jpgfish.py +++ b/test/results/jpgfish.py @@ -1,155 +1,129 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import chevereto - __tests__ = ( -{ - "#url" : "https://jpg4.su/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, - "#urls" : "https://simp3.host.church/images/funnymeme.jpg", - "#sha1_content": "098e5e9b17ad634358426e0ffd1c93871474d13c", - - "album" : "", - "extension": "jpg", - "filename" : "funnymeme", - "id" : "LecXGS", - "url" : "https://simp3.host.church/images/funnymeme.jpg", - "user" : "exearco", -}, - -{ - "#url" : "https://jpg.church/img/auCruA", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, - "#pattern" : r"https://simp2\.host\.church/hannahowo_00457\.jpg", - - "album": "401-500", -}, - -{ - "#url" : "https://jpg1.su/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, -}, - -{ - "#url" : "https://jpeg.pet/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, -}, - -{ - "#url" : "https://jpg.pet/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, -}, - -{ - "#url" : "https://jpg.fishing/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, -}, - -{ - "#url" : "https://jpg.fish/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, -}, - -{ - "#url" : "https://jpg.church/img/funnymeme.LecXGS", - "#category": ("chevereto", "jpgfish", "image"), - "#class" : chevereto.CheveretoImageExtractor, -}, - -{ - "#url" : "https://jpg1.su/album/CDilP/?sort=date_desc&page=1", - "#category": ("chevereto", "jpgfish", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://jpg.fishing/a/gunggingnsk.N9OOI", - "#category": ("chevereto", "jpgfish", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, - "#count" : 114, -}, - -{ - "#url" : "https://jpg.fish/a/101-200.aNJ6A/", - "#category": ("chevereto", "jpgfish", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, - "#count" : 100, -}, - -{ - "#url" : "https://jpg.church/a/hannahowo.aNTdH/sub", - "#category": ("chevereto", "jpgfish", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, - "#count" : 606, -}, - -{ - "#url" : "https://jpeg.pet/album/CDilP/?sort=date_desc&page=1", - "#category": ("chevereto", "jpgfish", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, -}, - -{ - "#url" : "https://jpg.pet/album/CDilP/?sort=date_desc&page=1", - "#category": ("chevereto", "jpgfish", "album"), - "#class" : chevereto.CheveretoAlbumExtractor, -}, - -{ - "#url" : "https://jpg1.su/exearco", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, - "#count" : 3, -}, - -{ - "#url" : "https://jpg.church/exearco/albums", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, - "#count" : 1, -}, - -{ - "#url" : "https://jpeg.pet/exearco", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, -}, - -{ - "#url" : "https://jpg.pet/exearco", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, -}, - -{ - "#url" : "https://jpg.fishing/exearco", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, -}, - -{ - "#url" : "https://jpg.fish/exearco", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, -}, - -{ - "#url" : "https://jpg.church/exearco", - "#category": ("chevereto", "jpgfish", "user"), - "#class" : chevereto.CheveretoUserExtractor, -}, - + { + "#url": "https://jpg4.su/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + "#urls": "https://simp3.host.church/images/funnymeme.jpg", + "#sha1_content": "098e5e9b17ad634358426e0ffd1c93871474d13c", + "album": "", + "extension": "jpg", + "filename": "funnymeme", + "id": "LecXGS", + "url": "https://simp3.host.church/images/funnymeme.jpg", + "user": "exearco", + }, + { + "#url": "https://jpg.church/img/auCruA", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + "#pattern": r"https://simp2\.host\.church/hannahowo_00457\.jpg", + "album": "401-500", + }, + { + "#url": "https://jpg1.su/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + }, + { + "#url": "https://jpeg.pet/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + }, + { + "#url": "https://jpg.pet/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + }, + { + "#url": "https://jpg.fishing/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + }, + { + "#url": "https://jpg.fish/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + }, + { + "#url": "https://jpg.church/img/funnymeme.LecXGS", + "#category": ("chevereto", "jpgfish", "image"), + "#class": chevereto.CheveretoImageExtractor, + }, + { + "#url": "https://jpg1.su/album/CDilP/?sort=date_desc&page=1", + "#category": ("chevereto", "jpgfish", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + "#count": 2, + }, + { + "#url": "https://jpg.fishing/a/gunggingnsk.N9OOI", + "#category": ("chevereto", "jpgfish", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + "#count": 114, + }, + { + "#url": "https://jpg.fish/a/101-200.aNJ6A/", + "#category": ("chevereto", "jpgfish", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + "#count": 100, + }, + { + "#url": "https://jpg.church/a/hannahowo.aNTdH/sub", + "#category": ("chevereto", "jpgfish", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + "#count": 606, + }, + { + "#url": "https://jpeg.pet/album/CDilP/?sort=date_desc&page=1", + "#category": ("chevereto", "jpgfish", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + }, + { + "#url": "https://jpg.pet/album/CDilP/?sort=date_desc&page=1", + "#category": ("chevereto", "jpgfish", "album"), + "#class": chevereto.CheveretoAlbumExtractor, + }, + { + "#url": "https://jpg1.su/exearco", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + "#count": 3, + }, + { + "#url": "https://jpg.church/exearco/albums", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + "#count": 1, + }, + { + "#url": "https://jpeg.pet/exearco", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + }, + { + "#url": "https://jpg.pet/exearco", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + }, + { + "#url": "https://jpg.fishing/exearco", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + }, + { + "#url": "https://jpg.fish/exearco", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + }, + { + "#url": "https://jpg.church/exearco", + "#category": ("chevereto", "jpgfish", "user"), + "#class": chevereto.CheveretoUserExtractor, + }, ) diff --git a/test/results/kabeuchi.py b/test/results/kabeuchi.py index 37b6a3d8f0..dd5b00bb47 100644 --- a/test/results/kabeuchi.py +++ b/test/results/kabeuchi.py @@ -1,27 +1,22 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import kabeuchi from gallery_dl import exception - +from gallery_dl.extractor import kabeuchi __tests__ = ( -{ - "#url" : "https://kabe-uchiroom.com/mypage/?id=919865303848255493", - "#category": ("", "kabeuchi", "user"), - "#class" : kabeuchi.KabeuchiUserExtractor, - "#pattern" : r"https://kabe-uchiroom\.com/accounts/upfile/3/919865303848255493/\w+\.jpe?g", - "#count" : ">= 24", -}, - -{ - "#url" : "https://kabe-uchiroom.com/mypage/?id=123456789", - "#category": ("", "kabeuchi", "user"), - "#class" : kabeuchi.KabeuchiUserExtractor, - "#exception": exception.NotFoundError, -}, - + { + "#url": "https://kabe-uchiroom.com/mypage/?id=919865303848255493", + "#category": ("", "kabeuchi", "user"), + "#class": kabeuchi.KabeuchiUserExtractor, + "#pattern": r"https://kabe-uchiroom\.com/accounts/upfile/3/919865303848255493/\w+\.jpe?g", + "#count": ">= 24", + }, + { + "#url": "https://kabe-uchiroom.com/mypage/?id=123456789", + "#category": ("", "kabeuchi", "user"), + "#class": kabeuchi.KabeuchiUserExtractor, + "#exception": exception.NotFoundError, + }, ) diff --git a/test/results/keenspot.py b/test/results/keenspot.py index 9f756ad8b7..241ca39c5f 100644 --- a/test/results/keenspot.py +++ b/test/results/keenspot.py @@ -1,55 +1,47 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import keenspot - __tests__ = ( -{ - "#url" : "http://marksmen.keenspot.com/", - "#comment" : "link", - "#category": ("", "keenspot", "comic"), - "#class" : keenspot.KeenspotComicExtractor, - "#range" : "1-3", - "#sha1_url": "83bcf029103bf8bc865a1988afa4aaeb23709ba6", -}, - -{ - "#url" : "http://barkercomic.keenspot.com/", - "#comment" : "id", - "#category": ("", "keenspot", "comic"), - "#class" : keenspot.KeenspotComicExtractor, - "#range" : "1-3", - "#sha1_url": "c4080926db18d00bac641fdd708393b7d61379e6", -}, - -{ - "#url" : "http://crowscare.keenspot.com/", - "#comment" : "id v2", - "#category": ("", "keenspot", "comic"), - "#class" : keenspot.KeenspotComicExtractor, - "#range" : "1-3", - "#sha1_url": "a00e66a133dd39005777317da90cef921466fcaa", -}, - -{ - "#url" : "http://supernovas.keenspot.com/", - "#comment" : "ks", - "#category": ("", "keenspot", "comic"), - "#class" : keenspot.KeenspotComicExtractor, - "#range" : "1-3", - "#sha1_url": "de21b12887ef31ff82edccbc09d112e3885c3aab", -}, - -{ - "#url" : "http://twokinds.keenspot.com/comic/1066/", - "#category": ("", "keenspot", "comic"), - "#class" : keenspot.KeenspotComicExtractor, - "#range" : "1-3", - "#sha1_url": "6a784e11370abfb343dcad9adbb7718f9b7be350", -}, - + { + "#url": "http://marksmen.keenspot.com/", + "#comment": "link", + "#category": ("", "keenspot", "comic"), + "#class": keenspot.KeenspotComicExtractor, + "#range": "1-3", + "#sha1_url": "83bcf029103bf8bc865a1988afa4aaeb23709ba6", + }, + { + "#url": "http://barkercomic.keenspot.com/", + "#comment": "id", + "#category": ("", "keenspot", "comic"), + "#class": keenspot.KeenspotComicExtractor, + "#range": "1-3", + "#sha1_url": "c4080926db18d00bac641fdd708393b7d61379e6", + }, + { + "#url": "http://crowscare.keenspot.com/", + "#comment": "id v2", + "#category": ("", "keenspot", "comic"), + "#class": keenspot.KeenspotComicExtractor, + "#range": "1-3", + "#sha1_url": "a00e66a133dd39005777317da90cef921466fcaa", + }, + { + "#url": "http://supernovas.keenspot.com/", + "#comment": "ks", + "#category": ("", "keenspot", "comic"), + "#class": keenspot.KeenspotComicExtractor, + "#range": "1-3", + "#sha1_url": "de21b12887ef31ff82edccbc09d112e3885c3aab", + }, + { + "#url": "http://twokinds.keenspot.com/comic/1066/", + "#category": ("", "keenspot", "comic"), + "#class": keenspot.KeenspotComicExtractor, + "#range": "1-3", + "#sha1_url": "6a784e11370abfb343dcad9adbb7718f9b7be350", + }, ) diff --git a/test/results/kemonoparty.py b/test/results/kemonoparty.py index 4c37008979..34ff16e3bc 100644 --- a/test/results/kemonoparty.py +++ b/test/results/kemonoparty.py @@ -1,414 +1,360 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import kemonoparty from gallery_dl import exception - +from gallery_dl.extractor import kemonoparty __tests__ = ( -{ - "#url" : "https://kemono.su/fanbox/user/6993449", - "#category": ("", "kemonoparty", "fanbox"), - "#class" : kemonoparty.KemonopartyUserExtractor, - "#range" : "1-500", - "#count" : 500, -}, - -{ - "#url" : "https://kemono.su/patreon/user/881792?o=150", - "#comment" : "'max-posts' option, 'o' query parameter (#1674)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyUserExtractor, - "#options" : {"max-posts": 100}, - "#count" : range(200, 300), -}, - -{ - "#url" : "https://kemono.su/fanbox/user/6993449?q=お蔵入りになった", - "#comment" : "search / 'q' query parameter (#3385, #4057)", - "#category": ("", "kemonoparty", "fanbox"), - "#class" : kemonoparty.KemonopartyUserExtractor, - "#urls" : ( - "https://kemono.su/data/ef/7b/ef7b4398a2f4ada597421fd3c116cff86e85695911f7cd2a459b0e566b864e46.png", - "https://kemono.su/data/73/e6/73e615f6645b9d1af6329448601673c9275f07fd11eb37670c97e307e29a9ee9.png", - ), - - "id": "8779", -}, - -{ - "#url" : "https://kemono.su/subscribestar/user/alcorart", - "#category": ("", "kemonoparty", "subscribestar"), - "#class" : kemonoparty.KemonopartyUserExtractor, -}, - -{ - "#url" : "https://kemono.su/subscribestar/user/alcorart", - "#category": ("", "kemonoparty", "subscribestar"), - "#class" : kemonoparty.KemonopartyUserExtractor, -}, - -{ - "#url" : "https://kemono.su/fanbox/user/6993449/post/506575", - "#category": ("", "kemonoparty", "fanbox"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#pattern" : r"https://kemono.su/data/21/0f/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg", - "#sha1_content": "900949cefc97ab8dc1979cc3664785aac5ba70dd", - - "added" : "2020-05-06T20:28:02.302000", - "content" : str, - "count" : 1, - "date" : "dt:2019-08-10 17:09:04", - "edited" : None, - "embed" : dict, - "extension" : "jpeg", - "filename" : "P058kDFYus7DbqAkGlfWTlOr", - "hash" : "210f35388e28bbcf756db18dd516e2d82ce758e0d32881eeee76d43e1716d382", - "id" : "506575", - "num" : 1, - "published" : "2019-08-10T17:09:04", - "service" : "fanbox", - "shared_file": False, - "subcategory": "fanbox", - "title" : "c96取り置き", - "type" : "file", - "user" : "6993449", -}, - -{ - "#url" : "https://kemono.su/fanbox/user/7356311/post/802343", - "#comment" : "inline image (#1286)", - "#category": ("", "kemonoparty", "fanbox"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#pattern" : r"https://kemono\.su/data/47/b5/47b5c014ecdcfabdf2c85eec53f1133a76336997ae8596f332e97d956a460ad2\.jpg", - - "hash": "47b5c014ecdcfabdf2c85eec53f1133a76336997ae8596f332e97d956a460ad2", -}, - -{ - "#url" : "https://kemono.su/gumroad/user/3101696181060/post/tOWyf", - "#category": ("", "kemonoparty", "gumroad"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#urls" : "https://kemono.su/data/6f/13/6f1394b19516396ea520254350662c254bbea30c1e111fd4b0f042c61c426d07.zip", -}, - -{ - "#url" : "https://kemono.party/gumroad/user/3252870377455/post/aJnAH", - "#comment" : "username (#1548, #1652)", - "#category": ("", "kemonoparty", "gumroad"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#options" : {"metadata": True}, - - "username": "Kudalyn's Creations", -}, - -{ - "#url" : "https://kemono.su/patreon/user/4158582/post/32099982", - "#comment" : "allow duplicates (#2440)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://kemono.su/patreon/user/4158582/post/32099982", - "#comment" : "allow duplicates (#2440)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#options" : {"duplicates": True}, - "#count" : 3, -}, - -{ - "#url" : "https://kemono.su/patreon/user/34134344/post/38129255", - "#comment" : "DMs (#2008)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#options" : {"dms": True}, - - "dms": [{ - "body": r"re:Hi! Thank you very much for supporting the work I did in May. Here's your reward pack! I hope you find something you enjoy in it. :\)\n\nhttps://www.mediafire.com/file/\w+/Set13_tier_2.zip/file", - "date": "2021-06", - }], -}, - -{ - "#url" : "https://kemono.su/patreon/user/3161935/post/68231671", - "#comment" : "announcements", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#options" : {"announcements": True}, - - "announcements": [{ - "body": "
          Thank you so much for the support!
          This Patreon is more of a tip jar for supporting what I make. I have to clarify that there are no exclusive Patreon animations because all are released for the public. You will get earlier access to WIPs. Direct downloads to my works are also available for $5 and $10 Tiers.
          ", - "date": "2023-02", - }], -}, - -{ - "#url" : "https://kemono.su/patreon/user/19623797/post/29035449", - "#comment" : "invalid file (#3510)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#pattern" : r"907ba78b4545338d3539683e63ecb51cf51c10adc9dabd86e92bd52339f298b9\.txt", - "#sha1_content": "da39a3ee5e6b4b0d3255bfef95601890afd80709", -}, - -{ - "#url" : "https://kemono.su/subscribestar/user/alcorart/post/184330", - "#category": ("", "kemonoparty", "subscribestar"), - "#class" : kemonoparty.KemonopartyPostExtractor, -}, - -{ - "#url" : "https://kemono.su/subscribestar/user/alcorart/post/184330", - "#category": ("", "kemonoparty", "subscribestar"), - "#class" : kemonoparty.KemonopartyPostExtractor, -}, - -{ - "#url" : "https://www.kemono.su/subscribestar/user/alcorart/post/184330", - "#category": ("", "kemonoparty", "subscribestar"), - "#class" : kemonoparty.KemonopartyPostExtractor, -}, - -{ - "#url" : "https://beta.kemono.su/subscribestar/user/alcorart/post/184330", - "#category": ("", "kemonoparty", "subscribestar"), - "#class" : kemonoparty.KemonopartyPostExtractor, -}, - -{ - "#url" : "https://kemono.su/patreon/user/3161935/post/68231671/revision/142470", - "#comment" : "revisions (#4498)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#urls" : "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", - - "file" : { - "hash": "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", - "name": "wip update.jpg", - "path": "/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", + { + "#url": "https://kemono.su/fanbox/user/6993449", + "#category": ("", "kemonoparty", "fanbox"), + "#class": kemonoparty.KemonopartyUserExtractor, + "#range": "1-500", + "#count": 500, + }, + { + "#url": "https://kemono.su/patreon/user/881792?o=150", + "#comment": "'max-posts' option, 'o' query parameter (#1674)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyUserExtractor, + "#options": {"max-posts": 100}, + "#count": range(200, 300), + }, + { + "#url": "https://kemono.su/fanbox/user/6993449?q=お蔵入りになった", + "#comment": "search / 'q' query parameter (#3385, #4057)", + "#category": ("", "kemonoparty", "fanbox"), + "#class": kemonoparty.KemonopartyUserExtractor, + "#urls": ( + "https://kemono.su/data/ef/7b/ef7b4398a2f4ada597421fd3c116cff86e85695911f7cd2a459b0e566b864e46.png", + "https://kemono.su/data/73/e6/73e615f6645b9d1af6329448601673c9275f07fd11eb37670c97e307e29a9ee9.png", + ), + "id": "8779", + }, + { + "#url": "https://kemono.su/subscribestar/user/alcorart", + "#category": ("", "kemonoparty", "subscribestar"), + "#class": kemonoparty.KemonopartyUserExtractor, + }, + { + "#url": "https://kemono.su/subscribestar/user/alcorart", + "#category": ("", "kemonoparty", "subscribestar"), + "#class": kemonoparty.KemonopartyUserExtractor, + }, + { + "#url": "https://kemono.su/fanbox/user/6993449/post/506575", + "#category": ("", "kemonoparty", "fanbox"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#pattern": r"https://kemono.su/data/21/0f/210f35388e28bbcf756db18dd516e2d82ce75[0-9a-f]+\.jpg", + "#sha1_content": "900949cefc97ab8dc1979cc3664785aac5ba70dd", + "added": "2020-05-06T20:28:02.302000", + "content": str, + "count": 1, + "date": "dt:2019-08-10 17:09:04", + "edited": None, + "embed": dict, + "extension": "jpeg", + "filename": "P058kDFYus7DbqAkGlfWTlOr", + "hash": "210f35388e28bbcf756db18dd516e2d82ce758e0d32881eeee76d43e1716d382", + "id": "506575", + "num": 1, + "published": "2019-08-10T17:09:04", + "service": "fanbox", + "shared_file": False, + "subcategory": "fanbox", + "title": "c96取り置き", "type": "file", + "user": "6993449", + }, + { + "#url": "https://kemono.su/fanbox/user/7356311/post/802343", + "#comment": "inline image (#1286)", + "#category": ("", "kemonoparty", "fanbox"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#pattern": r"https://kemono\.su/data/47/b5/47b5c014ecdcfabdf2c85eec53f1133a76336997ae8596f332e97d956a460ad2\.jpg", + "hash": "47b5c014ecdcfabdf2c85eec53f1133a76336997ae8596f332e97d956a460ad2", + }, + { + "#url": "https://kemono.su/gumroad/user/3101696181060/post/tOWyf", + "#category": ("", "kemonoparty", "gumroad"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#urls": "https://kemono.su/data/6f/13/6f1394b19516396ea520254350662c254bbea30c1e111fd4b0f042c61c426d07.zip", }, - "attachments": [ - { + { + "#url": "https://kemono.party/gumroad/user/3252870377455/post/aJnAH", + "#comment": "username (#1548, #1652)", + "#category": ("", "kemonoparty", "gumroad"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#options": {"metadata": True}, + "username": "Kudalyn's Creations", + }, + { + "#url": "https://kemono.su/patreon/user/4158582/post/32099982", + "#comment": "allow duplicates (#2440)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#count": 2, + }, + { + "#url": "https://kemono.su/patreon/user/4158582/post/32099982", + "#comment": "allow duplicates (#2440)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#options": {"duplicates": True}, + "#count": 3, + }, + { + "#url": "https://kemono.su/patreon/user/34134344/post/38129255", + "#comment": "DMs (#2008)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#options": {"dms": True}, + "dms": [ + { + "body": r"re:Hi! Thank you very much for supporting the work I did in May. Here's your reward pack! I hope you find something you enjoy in it. :\)\n\nhttps://www.mediafire.com/file/\w+/Set13_tier_2.zip/file", + "date": "2021-06", + } + ], + }, + { + "#url": "https://kemono.su/patreon/user/3161935/post/68231671", + "#comment": "announcements", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#options": {"announcements": True}, + "announcements": [ + { + "body": "
          Thank you so much for the support!
          This Patreon is more of a tip jar for supporting what I make. I have to clarify that there are no exclusive Patreon animations because all are released for the public. You will get earlier access to WIPs. Direct downloads to my works are also available for $5 and $10 Tiers.
          ", + "date": "2023-02", + } + ], + }, + { + "#url": "https://kemono.su/patreon/user/19623797/post/29035449", + "#comment": "invalid file (#3510)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#pattern": r"907ba78b4545338d3539683e63ecb51cf51c10adc9dabd86e92bd52339f298b9\.txt", + "#sha1_content": "da39a3ee5e6b4b0d3255bfef95601890afd80709", + }, + { + "#url": "https://kemono.su/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class": kemonoparty.KemonopartyPostExtractor, + }, + { + "#url": "https://kemono.su/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class": kemonoparty.KemonopartyPostExtractor, + }, + { + "#url": "https://www.kemono.su/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class": kemonoparty.KemonopartyPostExtractor, + }, + { + "#url": "https://beta.kemono.su/subscribestar/user/alcorart/post/184330", + "#category": ("", "kemonoparty", "subscribestar"), + "#class": kemonoparty.KemonopartyPostExtractor, + }, + { + "#url": "https://kemono.su/patreon/user/3161935/post/68231671/revision/142470", + "#comment": "revisions (#4498)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#urls": "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", + "file": { "hash": "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", "name": "wip update.jpg", "path": "/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", - "type": "attachment", + "type": "file", }, - ], - "filename" : "wip update", - "extension" : "jpg", - "hash" : "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", - "revision_id" : 142470, - "revision_index": 2, - "revision_count": 9, - "revision_hash" : "e0e93281495e151b11636c156e52bfe9234c2a40", -}, - -{ - "#url" : "https://kemono.su/patreon/user/3161935/post/68231671", - "#comment" : "unique revisions (#5013)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#options" : {"revisions": "unique"}, - "#urls" : "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", - - "filename" : "wip update", - "hash" : "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", - "revision_id" : 0, - "revision_index": 1, - "revision_count": 1, - "revision_hash" : "e0e93281495e151b11636c156e52bfe9234c2a40", -}, - -{ - "#url" : "https://kemono.su/patreon/user/3161935/post/68231671/revisions", - "#comment" : "revisions (#4498)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#pattern" : r"https://kemono\.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86\.jpg", - "#count" : 9, - "#archive" : False, - - "revision_id": range(134996, 3052965), - "revision_index": range(1, 9), - "revision_count": 9, - "revision_hash": "e0e93281495e151b11636c156e52bfe9234c2a40", -}, - - -{ - "#url" : "https://kemono.su/patreon/user/3161935/post/68231671/revision/12345", - "#comment" : "revisions (#4498)", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://kemono.su/patreon/user/6298789/post/69764693", - "#comment" : "'published' metadata with extra microsecond data", - "#category": ("", "kemonoparty", "patreon"), - "#class" : kemonoparty.KemonopartyPostExtractor, - - "date" : "dt:2022-07-29 21:12:11", - "published": "2022-07-29T21:12:11.483000", -}, - -{ - "#url" : "https://kemono.su/gumroad/user/3267960360326/post/jwwag", - "#comment" : "empty 'file' with no 'path' (#5368)", - "#category": ("", "kemonoparty", "gumroad"), - "#class" : kemonoparty.KemonopartyPostExtractor, - "#count" : 8, - - "type" : "attachment", -}, - -{ - "#url" : "https://kemono.su/discord/server/488668827274444803#608504710906904576", - "#category": ("", "kemonoparty", "discord"), - "#class" : kemonoparty.KemonopartyDiscordExtractor, - "#count" : 4, - - "channel" : "608504710906904576", - "channel_name": "finish-work", -}, - -{ - "#url" : "https://kemono.su/discord/server/488668827274444803#finish-work", - "#category": ("", "kemonoparty", "discord"), - "#class" : kemonoparty.KemonopartyDiscordExtractor, - "#count" : 4, - - "channel" : "608504710906904576", - "channel_name": "finish-work", -}, - -{ - "#url" : "https://kemono.su/discord/server/488668827274444803/channel/608504710906904576#finish-work", - "#category": ("", "kemonoparty", "discord"), - "#class" : kemonoparty.KemonopartyDiscordExtractor, - "#count" : 4, - - "channel" : "608504710906904576", - "channel_name": "finish-work", - "date" : "type:datetime", -}, - -{ - "#url" : "https://kemono.su/discord/server/818188637329031199#818343747275456522", - "#comment" : "pagination", - "#category": ("", "kemonoparty", "discord"), - "#class" : kemonoparty.KemonopartyDiscordExtractor, - "#range" : "1-250", - "#count" : 250, - - "channel" : "818343747275456522", - "channel_name": "wraith-sfw-gallery", -}, - -{ - "#url" : "https://kemono.su/discord/server/256559665620451329/channel/462437519519383555#", - "#category": ("", "kemonoparty", "discord"), - "#class" : kemonoparty.KemonopartyDiscordExtractor, - "#pattern" : r"https://kemono\.su/data/(e3/77/e377e3525164559484ace2e64425b0cec1db08.*\.png|51/45/51453640a5e0a4d23fbf57fb85390f9c5ec154.*\.gif)", - "#count" : ">= 2", - - "hash": r"re:e377e3525164559484ace2e64425b0cec1db08|51453640a5e0a4d23fbf57fb85390f9c5ec154", -}, - -{ - "#url" : "https://kemono.su/discord/server/315262215055736843/channel/315262215055736843#general", - "#comment" : "'inline' files", - "#category": ("", "kemonoparty", "discord"), - "#class" : kemonoparty.KemonopartyDiscordExtractor, - "#options" : {"image-filter": "type == 'inline'"}, - "#pattern" : r"https://cdn\.discordapp\.com/attachments/\d+/\d+/.+$", - "#range" : "1-5", - - "hash": "", -}, - -{ - "#url" : "https://kemono.su/discord/server/488668827274444803", - "#category": ("", "kemonoparty", "discord-server"), - "#class" : kemonoparty.KemonopartyDiscordServerExtractor, - "#pattern" : kemonoparty.KemonopartyDiscordExtractor.pattern, - "#count" : 13, -}, - -{ - "#url" : "https://kemono.su/discord/server/488668827274444803", - "#category": ("", "kemonoparty", "discord-server"), - "#class" : kemonoparty.KemonopartyDiscordServerExtractor, - "#pattern" : kemonoparty.KemonopartyDiscordExtractor.pattern, - "#count" : 13, -}, - -{ - "#url" : "https://kemono.su/posts?q=foobar", - "#category": ("", "kemonoparty", "posts"), - "#class" : kemonoparty.KemonopartyPostsExtractor, - "#count" : range(60, 100), -}, - -{ - "#url" : "https://kemono.su/favorites", - "#category": ("", "kemonoparty", "favorite"), - "#class" : kemonoparty.KemonopartyFavoriteExtractor, - "#pattern" : kemonoparty.KemonopartyUserExtractor.pattern, - "#auth" : True, - "#urls" : ( - "https://kemono.su/patreon/user/881792", - "https://kemono.su/fanbox/user/6993449", - "https://kemono.su/subscribestar/user/alcorart", - ), -}, - -{ - "#url" : "https://kemono.su/favorites?type=artist&sort=faved_seq&order=asc", - "#category": ("", "kemonoparty", "favorite"), - "#class" : kemonoparty.KemonopartyFavoriteExtractor, - "#pattern" : kemonoparty.KemonopartyUserExtractor.pattern, - "#auth" : True, - "#urls" : ( - "https://kemono.su/fanbox/user/6993449", - "https://kemono.su/patreon/user/881792", - "https://kemono.su/subscribestar/user/alcorart", - ), -}, - -{ - "#url" : "https://kemono.su/favorites?type=post", - "#category": ("", "kemonoparty", "favorite"), - "#class" : kemonoparty.KemonopartyFavoriteExtractor, - "#pattern" : kemonoparty.KemonopartyPostExtractor.pattern, - "#auth" : True, - "#urls" : ( - "https://kemono.su/subscribestar/user/alcorart/post/184329", - "https://kemono.su/fanbox/user/6993449/post/23913", - "https://kemono.su/patreon/user/881792/post/4769638", - ), -}, - -{ - "#url" : "https://kemono.su/favorites?type=post&sort=published&order=asc", - "#category": ("", "kemonoparty", "favorite"), - "#class" : kemonoparty.KemonopartyFavoriteExtractor, - "#pattern" : kemonoparty.KemonopartyPostExtractor.pattern, - "#auth" : True, - "#urls" : ( - "https://kemono.su/patreon/user/881792/post/4769638", - "https://kemono.su/fanbox/user/6993449/post/23913", - "https://kemono.su/subscribestar/user/alcorart/post/184329", - ), -}, - + "attachments": [ + { + "hash": "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", + "name": "wip update.jpg", + "path": "/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", + "type": "attachment", + }, + ], + "filename": "wip update", + "extension": "jpg", + "hash": "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", + "revision_id": 142470, + "revision_index": 2, + "revision_count": 9, + "revision_hash": "e0e93281495e151b11636c156e52bfe9234c2a40", + }, + { + "#url": "https://kemono.su/patreon/user/3161935/post/68231671", + "#comment": "unique revisions (#5013)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#options": {"revisions": "unique"}, + "#urls": "https://kemono.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86.jpg", + "filename": "wip update", + "hash": "88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86", + "revision_id": 0, + "revision_index": 1, + "revision_count": 1, + "revision_hash": "e0e93281495e151b11636c156e52bfe9234c2a40", + }, + { + "#url": "https://kemono.su/patreon/user/3161935/post/68231671/revisions", + "#comment": "revisions (#4498)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#pattern": r"https://kemono\.su/data/88/52/88521f71822dfa2f42df3beba319ea4fceda2a2d6dc59da0276a75238f743f86\.jpg", + "#count": 9, + "#archive": False, + "revision_id": range(134996, 3052965), + "revision_index": range(1, 9), + "revision_count": 9, + "revision_hash": "e0e93281495e151b11636c156e52bfe9234c2a40", + }, + { + "#url": "https://kemono.su/patreon/user/3161935/post/68231671/revision/12345", + "#comment": "revisions (#4498)", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://kemono.su/patreon/user/6298789/post/69764693", + "#comment": "'published' metadata with extra microsecond data", + "#category": ("", "kemonoparty", "patreon"), + "#class": kemonoparty.KemonopartyPostExtractor, + "date": "dt:2022-07-29 21:12:11", + "published": "2022-07-29T21:12:11.483000", + }, + { + "#url": "https://kemono.su/gumroad/user/3267960360326/post/jwwag", + "#comment": "empty 'file' with no 'path' (#5368)", + "#category": ("", "kemonoparty", "gumroad"), + "#class": kemonoparty.KemonopartyPostExtractor, + "#count": 8, + "type": "attachment", + }, + { + "#url": "https://kemono.su/discord/server/488668827274444803#608504710906904576", + "#category": ("", "kemonoparty", "discord"), + "#class": kemonoparty.KemonopartyDiscordExtractor, + "#count": 4, + "channel": "608504710906904576", + "channel_name": "finish-work", + }, + { + "#url": "https://kemono.su/discord/server/488668827274444803#finish-work", + "#category": ("", "kemonoparty", "discord"), + "#class": kemonoparty.KemonopartyDiscordExtractor, + "#count": 4, + "channel": "608504710906904576", + "channel_name": "finish-work", + }, + { + "#url": "https://kemono.su/discord/server/488668827274444803/channel/608504710906904576#finish-work", + "#category": ("", "kemonoparty", "discord"), + "#class": kemonoparty.KemonopartyDiscordExtractor, + "#count": 4, + "channel": "608504710906904576", + "channel_name": "finish-work", + "date": "type:datetime", + }, + { + "#url": "https://kemono.su/discord/server/818188637329031199#818343747275456522", + "#comment": "pagination", + "#category": ("", "kemonoparty", "discord"), + "#class": kemonoparty.KemonopartyDiscordExtractor, + "#range": "1-250", + "#count": 250, + "channel": "818343747275456522", + "channel_name": "wraith-sfw-gallery", + }, + { + "#url": "https://kemono.su/discord/server/256559665620451329/channel/462437519519383555#", + "#category": ("", "kemonoparty", "discord"), + "#class": kemonoparty.KemonopartyDiscordExtractor, + "#pattern": r"https://kemono\.su/data/(e3/77/e377e3525164559484ace2e64425b0cec1db08.*\.png|51/45/51453640a5e0a4d23fbf57fb85390f9c5ec154.*\.gif)", + "#count": ">= 2", + "hash": r"re:e377e3525164559484ace2e64425b0cec1db08|51453640a5e0a4d23fbf57fb85390f9c5ec154", + }, + { + "#url": "https://kemono.su/discord/server/315262215055736843/channel/315262215055736843#general", + "#comment": "'inline' files", + "#category": ("", "kemonoparty", "discord"), + "#class": kemonoparty.KemonopartyDiscordExtractor, + "#options": {"image-filter": "type == 'inline'"}, + "#pattern": r"https://cdn\.discordapp\.com/attachments/\d+/\d+/.+$", + "#range": "1-5", + "hash": "", + }, + { + "#url": "https://kemono.su/discord/server/488668827274444803", + "#category": ("", "kemonoparty", "discord-server"), + "#class": kemonoparty.KemonopartyDiscordServerExtractor, + "#pattern": kemonoparty.KemonopartyDiscordExtractor.pattern, + "#count": 13, + }, + { + "#url": "https://kemono.su/discord/server/488668827274444803", + "#category": ("", "kemonoparty", "discord-server"), + "#class": kemonoparty.KemonopartyDiscordServerExtractor, + "#pattern": kemonoparty.KemonopartyDiscordExtractor.pattern, + "#count": 13, + }, + { + "#url": "https://kemono.su/posts?q=foobar", + "#category": ("", "kemonoparty", "posts"), + "#class": kemonoparty.KemonopartyPostsExtractor, + "#count": range(60, 100), + }, + { + "#url": "https://kemono.su/favorites", + "#category": ("", "kemonoparty", "favorite"), + "#class": kemonoparty.KemonopartyFavoriteExtractor, + "#pattern": kemonoparty.KemonopartyUserExtractor.pattern, + "#auth": True, + "#urls": ( + "https://kemono.su/patreon/user/881792", + "https://kemono.su/fanbox/user/6993449", + "https://kemono.su/subscribestar/user/alcorart", + ), + }, + { + "#url": "https://kemono.su/favorites?type=artist&sort=faved_seq&order=asc", + "#category": ("", "kemonoparty", "favorite"), + "#class": kemonoparty.KemonopartyFavoriteExtractor, + "#pattern": kemonoparty.KemonopartyUserExtractor.pattern, + "#auth": True, + "#urls": ( + "https://kemono.su/fanbox/user/6993449", + "https://kemono.su/patreon/user/881792", + "https://kemono.su/subscribestar/user/alcorart", + ), + }, + { + "#url": "https://kemono.su/favorites?type=post", + "#category": ("", "kemonoparty", "favorite"), + "#class": kemonoparty.KemonopartyFavoriteExtractor, + "#pattern": kemonoparty.KemonopartyPostExtractor.pattern, + "#auth": True, + "#urls": ( + "https://kemono.su/subscribestar/user/alcorart/post/184329", + "https://kemono.su/fanbox/user/6993449/post/23913", + "https://kemono.su/patreon/user/881792/post/4769638", + ), + }, + { + "#url": "https://kemono.su/favorites?type=post&sort=published&order=asc", + "#category": ("", "kemonoparty", "favorite"), + "#class": kemonoparty.KemonopartyFavoriteExtractor, + "#pattern": kemonoparty.KemonopartyPostExtractor.pattern, + "#auth": True, + "#urls": ( + "https://kemono.su/patreon/user/881792/post/4769638", + "https://kemono.su/fanbox/user/6993449/post/23913", + "https://kemono.su/subscribestar/user/alcorart/post/184329", + ), + }, ) diff --git a/test/results/khinsider.py b/test/results/khinsider.py index 7013069f2d..f1057966a6 100644 --- a/test/results/khinsider.py +++ b/test/results/khinsider.py @@ -1,30 +1,25 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import khinsider - __tests__ = ( -{ - "#url" : "https://downloads.khinsider.com/game-soundtracks/album/horizon-riders-wii", - "#category": ("", "khinsider", "soundtrack"), - "#class" : khinsider.KhinsiderSoundtrackExtractor, - "#pattern" : r"https?://(dl\.)?vgm(site|downloads)\.com/soundtracks/horizon-riders-wii/[^/]+/Horizon%20Riders%20Wii%20-%20Full%20Soundtrack\.mp3", - "#count" : 1, - - "album" : { - "count" : 1, - "date" : "Sep 18th, 2016", - "name" : "Horizon Riders", - "platform": "Wii", - "size" : 26214400, - "type" : "Gamerip", + { + "#url": "https://downloads.khinsider.com/game-soundtracks/album/horizon-riders-wii", + "#category": ("", "khinsider", "soundtrack"), + "#class": khinsider.KhinsiderSoundtrackExtractor, + "#pattern": r"https?://(dl\.)?vgm(site|downloads)\.com/soundtracks/horizon-riders-wii/[^/]+/Horizon%20Riders%20Wii%20-%20Full%20Soundtrack\.mp3", + "#count": 1, + "album": { + "count": 1, + "date": "Sep 18th, 2016", + "name": "Horizon Riders", + "platform": "Wii", + "size": 26214400, + "type": "Gamerip", + }, + "extension": "mp3", + "filename": "Horizon Riders Wii - Full Soundtrack", }, - "extension": "mp3", - "filename" : "Horizon Riders Wii - Full Soundtrack", -}, - ) diff --git a/test/results/koharu.py b/test/results/koharu.py index 0aed25f63f..b5841d0c19 100644 --- a/test/results/koharu.py +++ b/test/results/koharu.py @@ -1,156 +1,144 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import koharu - __tests__ = ( -{ - "#url" : "https://niyaniya.moe/g/14216/6c67076fdd45", - "#category": ("", "koharu", "gallery"), - "#class" : koharu.KoharuGalleryExtractor, - "#options" : {"tags": True}, - "#pattern" : r"https://kisakisexo.xyz/download/59896/a4fbd1828229/f47639c6abaf1903dd69c36a3d961da84741a1831aa07a2906ce9c74156a5d75\?v=1721626410802&w=0", - "#count" : 1, - - "count" : 22, - "created_at": 1721626410802, - "date" : "dt:2024-07-22 05:33:30", - "extension" : "cbz", - "filename" : "f47639c6abaf1903dd69c36a3d961da84741a1831aa07a2906ce9c74156a5d75", - "id" : 14216, - "num" : 1, - "public_key": "6c67076fdd45", - "tags": [ - "general:beach", - "general:booty", - "general:dark skin", - "general:fingering", - "general:handjob", - "general:light hair", - "general:nakadashi", - "general:outdoors", - "general:ponytail", - "general:swimsuit", - "general:x-ray", - "artist:ouchi kaeru", - "magazine:comic kairakuten 2024-08", - "female:busty", - "language:english", - "language:translated", - "other:uncensored", - "other:vanilla", - ], - "tags_artist": [ - "ouchi kaeru", - ], - "tags_female": [ - "busty", - ], - "tags_general": [ - "beach", - "booty", - "dark skin", - "fingering", - "handjob", - "light hair", - "nakadashi", - "outdoors", - "ponytail", - "swimsuit", - "x-ray", - ], - "tags_language": [ - "english", - "translated", - ], - "tags_magazine": [ - "comic kairakuten 2024-08", - ], - "tags_other": [ - "uncensored", - "vanilla", - ], - "title" : "[Ouchi Kaeru] Summer Business (Comic Kairakuten 2024-08)", - "updated_at": 1721626410802, -}, - -{ - "#url" : "https://niyaniya.moe/g/14216/6c67076fdd45", - "#category": ("", "koharu", "gallery"), - "#class" : koharu.KoharuGalleryExtractor, - "#options" : {"cbz": False, "format": "780"}, - "#pattern" : r"https://koharusexo.xyz/data/59905/2df9110af7f1/a7cbeca3fb9c83aa87582a8a74cc8f8ce1b9e9b434dc1af293628871642f42df/[0-9a-f]+/.+", - "#count" : 22, -}, - -{ - "#url" : "https://niyaniya.moe/g/14216/6c67076fdd45", - "#category": ("", "koharu", "gallery"), - "#class" : koharu.KoharuGalleryExtractor, - "#options" : {"cbz": False, "format": "780"}, - "#range" : "1", - "#sha1_content": "08954e0ae18a900ee7ca144d1661c664468c2525", -}, - -{ - "#url" : "https://koharu.to/g/14216/6c67076fdd45", - "#class": koharu.KoharuGalleryExtractor, -}, -{ - "#url" : "https://anchira.to/g/14216/6c67076fdd45", - "#class": koharu.KoharuGalleryExtractor, -}, -{ - "#url" : "https://seia.to/g/14216/6c67076fdd45", - "#class": koharu.KoharuGalleryExtractor, -}, -{ - "#url" : "https://shupogaki.moe/g/14216/6c67076fdd45", - "#class": koharu.KoharuGalleryExtractor, -}, -{ - "#url" : "https://hoshino.one/g/14216/6c67076fdd45", - "#class": koharu.KoharuGalleryExtractor, -}, - -{ - "#url" : "https://niyaniya.moe/reader/14216/6c67076fdd45", - "#category": ("", "koharu", "gallery"), - "#class" : koharu.KoharuGalleryExtractor, -}, - -{ - "#url" : "https://niyaniya.moe/?s=tag:^beach$", - "#category": ("", "koharu", "search"), - "#class" : koharu.KoharuSearchExtractor, - "#pattern" : koharu.KoharuGalleryExtractor.pattern, - "#count" : ">= 50", -}, - -{ - "#url" : "https://niyaniya.moe/favorites", - "#category": ("", "koharu", "favorite"), - "#class" : koharu.KoharuFavoriteExtractor, - "#pattern" : koharu.KoharuGalleryExtractor.pattern, - "#auth" : True, - "#urls" : [ - "https://niyaniya.moe/g/14216/6c67076fdd45", - ], -}, - -{ - "#url" : "https://niyaniya.moe/favorites?cat=6&sort=4", - "#category": ("", "koharu", "favorite"), - "#class" : koharu.KoharuFavoriteExtractor, - "#pattern" : koharu.KoharuGalleryExtractor.pattern, - "#auth" : True, - "#urls" : [ - "https://niyaniya.moe/g/14216/6c67076fdd45", - ], -}, - + { + "#url": "https://niyaniya.moe/g/14216/6c67076fdd45", + "#category": ("", "koharu", "gallery"), + "#class": koharu.KoharuGalleryExtractor, + "#options": {"tags": True}, + "#pattern": r"https://kisakisexo.xyz/download/59896/a4fbd1828229/f47639c6abaf1903dd69c36a3d961da84741a1831aa07a2906ce9c74156a5d75\?v=1721626410802&w=0", + "#count": 1, + "count": 22, + "created_at": 1721626410802, + "date": "dt:2024-07-22 05:33:30", + "extension": "cbz", + "filename": "f47639c6abaf1903dd69c36a3d961da84741a1831aa07a2906ce9c74156a5d75", + "id": 14216, + "num": 1, + "public_key": "6c67076fdd45", + "tags": [ + "general:beach", + "general:booty", + "general:dark skin", + "general:fingering", + "general:handjob", + "general:light hair", + "general:nakadashi", + "general:outdoors", + "general:ponytail", + "general:swimsuit", + "general:x-ray", + "artist:ouchi kaeru", + "magazine:comic kairakuten 2024-08", + "female:busty", + "language:english", + "language:translated", + "other:uncensored", + "other:vanilla", + ], + "tags_artist": [ + "ouchi kaeru", + ], + "tags_female": [ + "busty", + ], + "tags_general": [ + "beach", + "booty", + "dark skin", + "fingering", + "handjob", + "light hair", + "nakadashi", + "outdoors", + "ponytail", + "swimsuit", + "x-ray", + ], + "tags_language": [ + "english", + "translated", + ], + "tags_magazine": [ + "comic kairakuten 2024-08", + ], + "tags_other": [ + "uncensored", + "vanilla", + ], + "title": "[Ouchi Kaeru] Summer Business (Comic Kairakuten 2024-08)", + "updated_at": 1721626410802, + }, + { + "#url": "https://niyaniya.moe/g/14216/6c67076fdd45", + "#category": ("", "koharu", "gallery"), + "#class": koharu.KoharuGalleryExtractor, + "#options": {"cbz": False, "format": "780"}, + "#pattern": r"https://koharusexo.xyz/data/59905/2df9110af7f1/a7cbeca3fb9c83aa87582a8a74cc8f8ce1b9e9b434dc1af293628871642f42df/[0-9a-f]+/.+", + "#count": 22, + }, + { + "#url": "https://niyaniya.moe/g/14216/6c67076fdd45", + "#category": ("", "koharu", "gallery"), + "#class": koharu.KoharuGalleryExtractor, + "#options": {"cbz": False, "format": "780"}, + "#range": "1", + "#sha1_content": "08954e0ae18a900ee7ca144d1661c664468c2525", + }, + { + "#url": "https://koharu.to/g/14216/6c67076fdd45", + "#class": koharu.KoharuGalleryExtractor, + }, + { + "#url": "https://anchira.to/g/14216/6c67076fdd45", + "#class": koharu.KoharuGalleryExtractor, + }, + { + "#url": "https://seia.to/g/14216/6c67076fdd45", + "#class": koharu.KoharuGalleryExtractor, + }, + { + "#url": "https://shupogaki.moe/g/14216/6c67076fdd45", + "#class": koharu.KoharuGalleryExtractor, + }, + { + "#url": "https://hoshino.one/g/14216/6c67076fdd45", + "#class": koharu.KoharuGalleryExtractor, + }, + { + "#url": "https://niyaniya.moe/reader/14216/6c67076fdd45", + "#category": ("", "koharu", "gallery"), + "#class": koharu.KoharuGalleryExtractor, + }, + { + "#url": "https://niyaniya.moe/?s=tag:^beach$", + "#category": ("", "koharu", "search"), + "#class": koharu.KoharuSearchExtractor, + "#pattern": koharu.KoharuGalleryExtractor.pattern, + "#count": ">= 50", + }, + { + "#url": "https://niyaniya.moe/favorites", + "#category": ("", "koharu", "favorite"), + "#class": koharu.KoharuFavoriteExtractor, + "#pattern": koharu.KoharuGalleryExtractor.pattern, + "#auth": True, + "#urls": [ + "https://niyaniya.moe/g/14216/6c67076fdd45", + ], + }, + { + "#url": "https://niyaniya.moe/favorites?cat=6&sort=4", + "#category": ("", "koharu", "favorite"), + "#class": koharu.KoharuFavoriteExtractor, + "#pattern": koharu.KoharuGalleryExtractor.pattern, + "#auth": True, + "#urls": [ + "https://niyaniya.moe/g/14216/6c67076fdd45", + ], + }, ) diff --git a/test/results/kohlchan.py b/test/results/kohlchan.py index 479ed7154f..4314994a49 100644 --- a/test/results/kohlchan.py +++ b/test/results/kohlchan.py @@ -1,39 +1,32 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import lynxchan - __tests__ = ( -{ - "#url" : "https://kohlchan.net/a/res/4594.html", - "#category": ("lynxchan", "kohlchan", "thread"), - "#class" : lynxchan.LynxchanThreadExtractor, - "#pattern" : r"https://kohlchan\.net/\.media/[0-9a-f]{64}(\.\w+)?$", - "#count" : ">= 80", -}, - -{ - "#url" : "https://kohlchan.net/a/", - "#category": ("lynxchan", "kohlchan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, - "#pattern" : lynxchan.LynxchanThreadExtractor.pattern, - "#count" : ">= 100", -}, - -{ - "#url" : "https://kohlchan.net/a/2.html", - "#category": ("lynxchan", "kohlchan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, -}, - -{ - "#url" : "https://kohlchan.net/a/catalog.html", - "#category": ("lynxchan", "kohlchan", "board"), - "#class" : lynxchan.LynxchanBoardExtractor, -}, - + { + "#url": "https://kohlchan.net/a/res/4594.html", + "#category": ("lynxchan", "kohlchan", "thread"), + "#class": lynxchan.LynxchanThreadExtractor, + "#pattern": r"https://kohlchan\.net/\.media/[0-9a-f]{64}(\.\w+)?$", + "#count": ">= 80", + }, + { + "#url": "https://kohlchan.net/a/", + "#category": ("lynxchan", "kohlchan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + "#pattern": lynxchan.LynxchanThreadExtractor.pattern, + "#count": ">= 100", + }, + { + "#url": "https://kohlchan.net/a/2.html", + "#category": ("lynxchan", "kohlchan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + }, + { + "#url": "https://kohlchan.net/a/catalog.html", + "#category": ("lynxchan", "kohlchan", "board"), + "#class": lynxchan.LynxchanBoardExtractor, + }, ) diff --git a/test/results/komikcast.py b/test/results/komikcast.py index 0bd76121ff..fdc97f6aa2 100644 --- a/test/results/komikcast.py +++ b/test/results/komikcast.py @@ -1,85 +1,66 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import komikcast - __tests__ = ( -{ - "#url" : "https://komikcast.lol/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", - "#category": ("", "komikcast", "chapter"), - "#class" : komikcast.KomikcastChapterExtractor, - "#pattern" : r"https://svr\d+\.imgkc\d+\.my\.id/wp-content/img/A/Apotheosis/002-2/\d{3}\.jpg", - "#count" : 18, - - "chapter" : 2, - "chapter_minor": ".2", - "count" : 18, - "extension": "jpg", - "filename" : r"re:0\d{2}", - "lang" : "id", - "language" : "Indonesian", - "manga" : "Apotheosis", - "page" : range(1, 18), - "title" : "", -}, - -{ - "#url" : "https://komikcast.site/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", - "#category": ("", "komikcast", "chapter"), - "#class" : komikcast.KomikcastChapterExtractor, -}, - -{ - "#url" : "https://komikcast.me/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", - "#category": ("", "komikcast", "chapter"), - "#class" : komikcast.KomikcastChapterExtractor, -}, - -{ - "#url" : "https://komikcast.com/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", - "#category": ("", "komikcast", "chapter"), - "#class" : komikcast.KomikcastChapterExtractor, -}, - -{ - "#url" : "https://komikcast.me/chapter/soul-land-ii-chapter-300-1-bahasa-indonesia/", - "#category": ("", "komikcast", "chapter"), - "#class" : komikcast.KomikcastChapterExtractor, - "#pattern" : r"https://svr\d\.imgkc\d*\.my\.id/wp-content/img/S/Soul_Land_II/300\.1/\d\d\.jpg", - "#count" : 9, - "#sha1_metadata": "cb646cfed3d45105bd645ab38b2e9f7d8c436436", -}, - -{ - "#url" : "https://komikcast.site/komik/090-eko-to-issho/", - "#category": ("", "komikcast", "manga"), - "#class" : komikcast.KomikcastMangaExtractor, - "#pattern" : komikcast.KomikcastChapterExtractor.pattern, - "#count" : 12, - - "author" : "Asakura Maru", - "chapter": range(1, 12), - "chapter_minor": "", - "genres" : [ - "Comedy", - "Drama", - "Romance", - "School Life", - "Sci-Fi", - "Shounen" - ], - "manga" : "090 Eko to Issho", - "type" : "Manga", -}, - -{ - "#url" : "https://komikcast.me/tonari-no-kashiwagi-san/", - "#category": ("", "komikcast", "manga"), - "#class" : komikcast.KomikcastMangaExtractor, -}, - + { + "#url": "https://komikcast.lol/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class": komikcast.KomikcastChapterExtractor, + "#pattern": r"https://svr\d+\.imgkc\d+\.my\.id/wp-content/img/A/Apotheosis/002-2/\d{3}\.jpg", + "#count": 18, + "chapter": 2, + "chapter_minor": ".2", + "count": 18, + "extension": "jpg", + "filename": r"re:0\d{2}", + "lang": "id", + "language": "Indonesian", + "manga": "Apotheosis", + "page": range(1, 18), + "title": "", + }, + { + "#url": "https://komikcast.site/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class": komikcast.KomikcastChapterExtractor, + }, + { + "#url": "https://komikcast.me/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class": komikcast.KomikcastChapterExtractor, + }, + { + "#url": "https://komikcast.com/chapter/apotheosis-chapter-02-2-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class": komikcast.KomikcastChapterExtractor, + }, + { + "#url": "https://komikcast.me/chapter/soul-land-ii-chapter-300-1-bahasa-indonesia/", + "#category": ("", "komikcast", "chapter"), + "#class": komikcast.KomikcastChapterExtractor, + "#pattern": r"https://svr\d\.imgkc\d*\.my\.id/wp-content/img/S/Soul_Land_II/300\.1/\d\d\.jpg", + "#count": 9, + "#sha1_metadata": "cb646cfed3d45105bd645ab38b2e9f7d8c436436", + }, + { + "#url": "https://komikcast.site/komik/090-eko-to-issho/", + "#category": ("", "komikcast", "manga"), + "#class": komikcast.KomikcastMangaExtractor, + "#pattern": komikcast.KomikcastChapterExtractor.pattern, + "#count": 12, + "author": "Asakura Maru", + "chapter": range(1, 12), + "chapter_minor": "", + "genres": ["Comedy", "Drama", "Romance", "School Life", "Sci-Fi", "Shounen"], + "manga": "090 Eko to Issho", + "type": "Manga", + }, + { + "#url": "https://komikcast.me/tonari-no-kashiwagi-san/", + "#category": ("", "komikcast", "manga"), + "#class": komikcast.KomikcastMangaExtractor, + }, ) diff --git a/test/results/konachan.py b/test/results/konachan.py index ed4d2d78a9..2d51e6d907 100644 --- a/test/results/konachan.py +++ b/test/results/konachan.py @@ -1,92 +1,77 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import moebooru from gallery_dl import exception - +from gallery_dl.extractor import moebooru __tests__ = ( -{ - "#url" : "https://konachan.com/post/show/205189", - "#category": ("moebooru", "konachan", "post"), - "#class" : moebooru.MoebooruPostExtractor, - "#options" : {"tags": True}, - "#sha1_content": "674e75a753df82f5ad80803f575818b8e46e4b65", - - "tags_artist" : "patata", - "tags_character": "clownpiece", - "tags_copyright": "touhou", - "tags_general" : str, -}, - -{ - "#url" : "https://konachan.net/post/show/205189", - "#category": ("moebooru", "konachan", "post"), - "#class" : moebooru.MoebooruPostExtractor, -}, - -{ - "#url" : "https://konachan.com/post?tags=patata", - "#category": ("moebooru", "konachan", "tag"), - "#class" : moebooru.MoebooruTagExtractor, - "#sha1_content": "838cfb815e31f48160855435655ddf7bfc4ecb8d", -}, - -{ - "#url" : "https://konachan.com/post?tags=", - "#comment" : "empty 'tags' (#4354)", - "#category": ("moebooru", "konachan", "tag"), - "#class" : moebooru.MoebooruTagExtractor, -}, - -{ - "#url" : "https://konachan.net/post?tags=patata", - "#category": ("moebooru", "konachan", "tag"), - "#class" : moebooru.MoebooruTagExtractor, -}, - -{ - "#url" : "https://konachan.com/pool/show/95", - "#category": ("moebooru", "konachan", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, - "#sha1_content": "cf0546e38a93c2c510a478f8744e60687b7a8426", -}, - -{ - "#url" : "https://konachan.com/pool/show/95", - "#comment" : "'metadata' option (#4646)", - "#category": ("moebooru", "konachan", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, - "#options" : {"metadata": True}, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://konachan.net/pool/show/95", - "#category": ("moebooru", "konachan", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, -}, - -{ - "#url" : "https://konachan.com/post/popular_by_month?month=11&year=2010", - "#category": ("moebooru", "konachan", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, - "#count" : 20, -}, - -{ - "#url" : "https://konachan.com/post/popular_recent", - "#category": ("moebooru", "konachan", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, -}, - -{ - "#url" : "https://konachan.net/post/popular_recent", - "#category": ("moebooru", "konachan", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, -}, - + { + "#url": "https://konachan.com/post/show/205189", + "#category": ("moebooru", "konachan", "post"), + "#class": moebooru.MoebooruPostExtractor, + "#options": {"tags": True}, + "#sha1_content": "674e75a753df82f5ad80803f575818b8e46e4b65", + "tags_artist": "patata", + "tags_character": "clownpiece", + "tags_copyright": "touhou", + "tags_general": str, + }, + { + "#url": "https://konachan.net/post/show/205189", + "#category": ("moebooru", "konachan", "post"), + "#class": moebooru.MoebooruPostExtractor, + }, + { + "#url": "https://konachan.com/post?tags=patata", + "#category": ("moebooru", "konachan", "tag"), + "#class": moebooru.MoebooruTagExtractor, + "#sha1_content": "838cfb815e31f48160855435655ddf7bfc4ecb8d", + }, + { + "#url": "https://konachan.com/post?tags=", + "#comment": "empty 'tags' (#4354)", + "#category": ("moebooru", "konachan", "tag"), + "#class": moebooru.MoebooruTagExtractor, + }, + { + "#url": "https://konachan.net/post?tags=patata", + "#category": ("moebooru", "konachan", "tag"), + "#class": moebooru.MoebooruTagExtractor, + }, + { + "#url": "https://konachan.com/pool/show/95", + "#category": ("moebooru", "konachan", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + "#sha1_content": "cf0546e38a93c2c510a478f8744e60687b7a8426", + }, + { + "#url": "https://konachan.com/pool/show/95", + "#comment": "'metadata' option (#4646)", + "#category": ("moebooru", "konachan", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + "#options": {"metadata": True}, + "#exception": exception.HttpError, + }, + { + "#url": "https://konachan.net/pool/show/95", + "#category": ("moebooru", "konachan", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + }, + { + "#url": "https://konachan.com/post/popular_by_month?month=11&year=2010", + "#category": ("moebooru", "konachan", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + "#count": 20, + }, + { + "#url": "https://konachan.com/post/popular_recent", + "#category": ("moebooru", "konachan", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + }, + { + "#url": "https://konachan.net/post/popular_recent", + "#category": ("moebooru", "konachan", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + }, ) diff --git a/test/results/lensdump.py b/test/results/lensdump.py index 91f633c037..6f528392c0 100644 --- a/test/results/lensdump.py +++ b/test/results/lensdump.py @@ -1,101 +1,86 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import lensdump - __tests__ = ( -{ - "#url" : "https://lensdump.com/a/1IhJr", - "#class" : lensdump.LensdumpAlbumExtractor, - "#pattern" : r"https://[abcd]\.l3n\.co/i/tq\w{4}\.png", - - "extension": "png", - "name" : str, - "num" : int, - "title" : str, - "url" : str, - "width" : int, -}, - -{ - "#url" : "https://lensdump.com/a/tA4lA", - "#comment" : "2 pages", - "#class" : lensdump.LensdumpAlbumExtractor, - "#pattern" : r"https://[abcd]\.l3n\.co/i/\w{6}\.(jpe?g|png)", - "#count" : 64, -}, - -{ - "#url" : "https://lensdump.com/vstar925", - "#class" : lensdump.LensdumpAlbumsExtractor, - "#urls" : ( - "https://lensdump.com/a/tX1uA", - "https://lensdump.com/a/R0gfK", - "https://lensdump.com/a/RSOMv", - "https://lensdump.com/a/9TbdT", - ), -}, - -{ - "#url" : "https://lensdump.com/vstar925/?sort=likes_desc&page=1", - "#comment" : "custom sort order", - "#class" : lensdump.LensdumpAlbumsExtractor, - "#urls" : ( - "https://lensdump.com/a/9TbdT", - "https://lensdump.com/a/RSOMv", - "https://lensdump.com/a/R0gfK", - "https://lensdump.com/a/tX1uA", - ), -}, - -{ - "#url" : "https://lensdump.com/vstar925/albums", - "#class" : lensdump.LensdumpAlbumsExtractor, -}, - -{ - "#url" : "https://lensdump.com/i/tyoAyM", - "#class" : lensdump.LensdumpImageExtractor, - "#urls" : "https://c.l3n.co/i/tyoAyM.webp", - "#sha1_content": "1aa749ed2c0cf679ec8e1df60068edaf3875de46", - - "date" : "dt:2022-08-01 08:24:28", - "extension": "webp", - "filename" : "tyoAyM", - "height" : 400, - "id" : "tyoAyM", - "title" : "MYOBI clovis bookcaseset", - "url" : "https://c.l3n.co/i/tyoAyM.webp", - "width" : 620, -}, - -{ - "#url" : "https://c.l3n.co/i/tyoAyM.webp", - "#class" : lensdump.LensdumpImageExtractor, - "#urls" : "https://c.l3n.co/i/tyoAyM.webp", - - "date" : "dt:2022-08-01 08:24:28", - "extension": "webp", - "filename" : "tyoAyM", - "height" : 400, - "id" : "tyoAyM", - "title" : "MYOBI clovis bookcaseset", - "url" : "https://c.l3n.co/i/tyoAyM.webp", - "width" : 620, -}, - -{ - "#url" : "https://i.lensdump.com/i/tyoAyM", - "#class" : lensdump.LensdumpImageExtractor, -}, - -{ - "#url" : "https://i3.lensdump.com/i/tyoAyM", - "#class" : lensdump.LensdumpImageExtractor, -}, - + { + "#url": "https://lensdump.com/a/1IhJr", + "#class": lensdump.LensdumpAlbumExtractor, + "#pattern": r"https://[abcd]\.l3n\.co/i/tq\w{4}\.png", + "extension": "png", + "name": str, + "num": int, + "title": str, + "url": str, + "width": int, + }, + { + "#url": "https://lensdump.com/a/tA4lA", + "#comment": "2 pages", + "#class": lensdump.LensdumpAlbumExtractor, + "#pattern": r"https://[abcd]\.l3n\.co/i/\w{6}\.(jpe?g|png)", + "#count": 64, + }, + { + "#url": "https://lensdump.com/vstar925", + "#class": lensdump.LensdumpAlbumsExtractor, + "#urls": ( + "https://lensdump.com/a/tX1uA", + "https://lensdump.com/a/R0gfK", + "https://lensdump.com/a/RSOMv", + "https://lensdump.com/a/9TbdT", + ), + }, + { + "#url": "https://lensdump.com/vstar925/?sort=likes_desc&page=1", + "#comment": "custom sort order", + "#class": lensdump.LensdumpAlbumsExtractor, + "#urls": ( + "https://lensdump.com/a/9TbdT", + "https://lensdump.com/a/RSOMv", + "https://lensdump.com/a/R0gfK", + "https://lensdump.com/a/tX1uA", + ), + }, + { + "#url": "https://lensdump.com/vstar925/albums", + "#class": lensdump.LensdumpAlbumsExtractor, + }, + { + "#url": "https://lensdump.com/i/tyoAyM", + "#class": lensdump.LensdumpImageExtractor, + "#urls": "https://c.l3n.co/i/tyoAyM.webp", + "#sha1_content": "1aa749ed2c0cf679ec8e1df60068edaf3875de46", + "date": "dt:2022-08-01 08:24:28", + "extension": "webp", + "filename": "tyoAyM", + "height": 400, + "id": "tyoAyM", + "title": "MYOBI clovis bookcaseset", + "url": "https://c.l3n.co/i/tyoAyM.webp", + "width": 620, + }, + { + "#url": "https://c.l3n.co/i/tyoAyM.webp", + "#class": lensdump.LensdumpImageExtractor, + "#urls": "https://c.l3n.co/i/tyoAyM.webp", + "date": "dt:2022-08-01 08:24:28", + "extension": "webp", + "filename": "tyoAyM", + "height": 400, + "id": "tyoAyM", + "title": "MYOBI clovis bookcaseset", + "url": "https://c.l3n.co/i/tyoAyM.webp", + "width": 620, + }, + { + "#url": "https://i.lensdump.com/i/tyoAyM", + "#class": lensdump.LensdumpImageExtractor, + }, + { + "#url": "https://i3.lensdump.com/i/tyoAyM", + "#class": lensdump.LensdumpImageExtractor, + }, ) diff --git a/test/results/lesbianenergy.py b/test/results/lesbianenergy.py index 650671f96b..feef42c961 100644 --- a/test/results/lesbianenergy.py +++ b/test/results/lesbianenergy.py @@ -1,45 +1,37 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import misskey - __tests__ = ( -{ - "#url" : "https://lesbian.energy/@rerorero", - "#category": ("misskey", "lesbian.energy", "user"), - "#class" : misskey.MisskeyUserExtractor, - "#pattern" : r"https://(lesbian.energy/files/\w+|.+/media_attachments/files/.+)", - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://lesbian.energy/@nano@mk.yopo.work", - "#category": ("misskey", "lesbian.energy", "user"), - "#class" : misskey.MisskeyUserExtractor, -}, - -{ - "#url" : "https://lesbian.energy/notes/995ig09wqy", - "#category": ("misskey", "lesbian.energy", "note"), - "#class" : misskey.MisskeyNoteExtractor, - "#count" : 1, -}, - -{ - "#url" : "https://lesbian.energy/notes/96ynd9w5kc", - "#category": ("misskey", "lesbian.energy", "note"), - "#class" : misskey.MisskeyNoteExtractor, -}, - -{ - "#url" : "https://lesbian.energy/my/favorites", - "#category": ("misskey", "lesbian.energy", "favorite"), - "#class" : misskey.MisskeyFavoriteExtractor, -}, - + { + "#url": "https://lesbian.energy/@rerorero", + "#category": ("misskey", "lesbian.energy", "user"), + "#class": misskey.MisskeyUserExtractor, + "#pattern": r"https://(lesbian.energy/files/\w+|.+/media_attachments/files/.+)", + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://lesbian.energy/@nano@mk.yopo.work", + "#category": ("misskey", "lesbian.energy", "user"), + "#class": misskey.MisskeyUserExtractor, + }, + { + "#url": "https://lesbian.energy/notes/995ig09wqy", + "#category": ("misskey", "lesbian.energy", "note"), + "#class": misskey.MisskeyNoteExtractor, + "#count": 1, + }, + { + "#url": "https://lesbian.energy/notes/96ynd9w5kc", + "#category": ("misskey", "lesbian.energy", "note"), + "#class": misskey.MisskeyNoteExtractor, + }, + { + "#url": "https://lesbian.energy/my/favorites", + "#category": ("misskey", "lesbian.energy", "favorite"), + "#class": misskey.MisskeyFavoriteExtractor, + }, ) diff --git a/test/results/lexica.py b/test/results/lexica.py index 07183f6759..4f26cdeb90 100644 --- a/test/results/lexica.py +++ b/test/results/lexica.py @@ -1,42 +1,37 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import lexica - __tests__ = ( -{ - "#url" : "https://lexica.art/?q=tree", - "#category": ("", "lexica", "search"), - "#class" : lexica.LexicaSearchExtractor, - "#pattern" : r"https://lexica-serve-encoded-images2\.sharif\.workers.dev/full_jpg/[0-9a-f-]{36}$", - "#range" : "1-80", - "#count" : 80, - - "height" : int, - "id" : str, - "upscaled_height": int, - "upscaled_width" : int, - "userid" : str, - "width" : int, - "prompt" : { - "c" : int, - "grid" : bool, - "height" : int, - "id" : str, - "images" : list, - "initImage" : None, - "initImageStrength": None, - "model" : "lexica-aperture-v2", - "negativePrompt" : str, - "prompt" : str, - "seed" : str, - "timestamp" : r"re:\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\dZ", - "width" : int, + { + "#url": "https://lexica.art/?q=tree", + "#category": ("", "lexica", "search"), + "#class": lexica.LexicaSearchExtractor, + "#pattern": r"https://lexica-serve-encoded-images2\.sharif\.workers.dev/full_jpg/[0-9a-f-]{36}$", + "#range": "1-80", + "#count": 80, + "height": int, + "id": str, + "upscaled_height": int, + "upscaled_width": int, + "userid": str, + "width": int, + "prompt": { + "c": int, + "grid": bool, + "height": int, + "id": str, + "images": list, + "initImage": None, + "initImageStrength": None, + "model": "lexica-aperture-v2", + "negativePrompt": str, + "prompt": str, + "seed": str, + "timestamp": r"re:\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d.\d\d\dZ", + "width": int, + }, }, -}, - ) diff --git a/test/results/lightroom.py b/test/results/lightroom.py index 4c1a053266..97e1da03db 100644 --- a/test/results/lightroom.py +++ b/test/results/lightroom.py @@ -1,31 +1,24 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import lightroom - __tests__ = ( -{ - "#url" : "https://lightroom.adobe.com/shares/0c9cce2033f24d24975423fe616368bf", - "#category": ("", "lightroom", "gallery"), - "#class" : lightroom.LightroomGalleryExtractor, - "#count" : ">= 55", - - "title": "Sterne und Nachtphotos", - "user" : "Christian Schrang", -}, - -{ - "#url" : "https://lightroom.adobe.com/shares/7ba68ad5a97e48608d2e6c57e6082813", - "#category": ("", "lightroom", "gallery"), - "#class" : lightroom.LightroomGalleryExtractor, - "#count" : ">= 180", - - "title": "HEBFC Snr/Res v Brighton", - "user" : "", -}, - + { + "#url": "https://lightroom.adobe.com/shares/0c9cce2033f24d24975423fe616368bf", + "#category": ("", "lightroom", "gallery"), + "#class": lightroom.LightroomGalleryExtractor, + "#count": ">= 55", + "title": "Sterne und Nachtphotos", + "user": "Christian Schrang", + }, + { + "#url": "https://lightroom.adobe.com/shares/7ba68ad5a97e48608d2e6c57e6082813", + "#category": ("", "lightroom", "gallery"), + "#class": lightroom.LightroomGalleryExtractor, + "#count": ">= 180", + "title": "HEBFC Snr/Res v Brighton", + "user": "", + }, ) diff --git a/test/results/livedoor.py b/test/results/livedoor.py index 7ab366f9ca..c5c4335bf7 100644 --- a/test/results/livedoor.py +++ b/test/results/livedoor.py @@ -1,66 +1,57 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import livedoor - __tests__ = ( -{ - "#url" : "http://blog.livedoor.jp/zatsu_ke/", - "#category": ("", "livedoor", "blog"), - "#class" : livedoor.LivedoorBlogExtractor, - "#pattern" : r"https?://livedoor.blogimg.jp/\w+/imgs/\w/\w/\w+\.\w+", - "#range" : "1-50", - "#count" : 50, - "#archive" : False, - - "post" : { - "categories" : tuple, - "date" : "type:datetime", - "description": str, - "id" : int, - "tags" : list, - "title" : str, - "user" : "zatsu_ke", + { + "#url": "http://blog.livedoor.jp/zatsu_ke/", + "#category": ("", "livedoor", "blog"), + "#class": livedoor.LivedoorBlogExtractor, + "#pattern": r"https?://livedoor.blogimg.jp/\w+/imgs/\w/\w/\w+\.\w+", + "#range": "1-50", + "#count": 50, + "#archive": False, + "post": { + "categories": tuple, + "date": "type:datetime", + "description": str, + "id": int, + "tags": list, + "title": str, + "user": "zatsu_ke", + }, + "filename": str, + "hash": r"re:\w{4,}", + "num": int, + }, + { + "#url": "http://blog.livedoor.jp/uotapo/", + "#category": ("", "livedoor", "blog"), + "#class": livedoor.LivedoorBlogExtractor, + "#range": "1-5", + "#count": 5, + }, + { + "#url": "http://blog.livedoor.jp/zatsu_ke/archives/51493859.html", + "#category": ("", "livedoor", "post"), + "#class": livedoor.LivedoorPostExtractor, + "#sha1_url": "9ca3bbba62722c8155be79ad7fc47be409e4a7a2", + "#sha1_metadata": "1f5b558492e0734f638b760f70bfc0b65c5a97b9", + }, + { + "#url": "http://blog.livedoor.jp/amaumauma/archives/7835811.html", + "#category": ("", "livedoor", "post"), + "#class": livedoor.LivedoorPostExtractor, + "#sha1_url": "204bbd6a9db4969c50e0923855aeede04f2e4a62", + "#sha1_metadata": "05821c7141360e6057ef2d382b046f28326a799d", + }, + { + "#url": "http://blog.livedoor.jp/uotapo/archives/1050616939.html", + "#category": ("", "livedoor", "post"), + "#class": livedoor.LivedoorPostExtractor, + "#sha1_url": "4b5ab144b7309eb870d9c08f8853d1abee9946d2", + "#sha1_metadata": "84fbf6e4eef16675013d6333039a7cfcb22c2d50", }, - "filename": str, - "hash" : r"re:\w{4,}", - "num" : int, -}, - -{ - "#url" : "http://blog.livedoor.jp/uotapo/", - "#category": ("", "livedoor", "blog"), - "#class" : livedoor.LivedoorBlogExtractor, - "#range" : "1-5", - "#count" : 5, -}, - -{ - "#url" : "http://blog.livedoor.jp/zatsu_ke/archives/51493859.html", - "#category": ("", "livedoor", "post"), - "#class" : livedoor.LivedoorPostExtractor, - "#sha1_url" : "9ca3bbba62722c8155be79ad7fc47be409e4a7a2", - "#sha1_metadata": "1f5b558492e0734f638b760f70bfc0b65c5a97b9", -}, - -{ - "#url" : "http://blog.livedoor.jp/amaumauma/archives/7835811.html", - "#category": ("", "livedoor", "post"), - "#class" : livedoor.LivedoorPostExtractor, - "#sha1_url" : "204bbd6a9db4969c50e0923855aeede04f2e4a62", - "#sha1_metadata": "05821c7141360e6057ef2d382b046f28326a799d", -}, - -{ - "#url" : "http://blog.livedoor.jp/uotapo/archives/1050616939.html", - "#category": ("", "livedoor", "post"), - "#class" : livedoor.LivedoorPostExtractor, - "#sha1_url" : "4b5ab144b7309eb870d9c08f8853d1abee9946d2", - "#sha1_metadata": "84fbf6e4eef16675013d6333039a7cfcb22c2d50", -}, - ) diff --git a/test/results/lolibooru.py b/test/results/lolibooru.py index f8750269b2..f03ccf1c36 100644 --- a/test/results/lolibooru.py +++ b/test/results/lolibooru.py @@ -1,45 +1,36 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import moebooru - __tests__ = ( -{ - "#url" : "https://lolibooru.moe/post/show/281305/", - "#category": ("moebooru", "lolibooru", "post"), - "#class" : moebooru.MoebooruPostExtractor, - "#options" : {"notes": True}, - "#sha1_content": "a331430223ffc5b23c31649102e7d49f52489b57", - - "notes": list, -}, - -{ - "#url" : "https://lolibooru.moe/post/show/287835", - "#category": ("moebooru", "lolibooru", "post"), - "#class" : moebooru.MoebooruPostExtractor, -}, - -{ - "#url" : "https://lolibooru.moe/post?tags=ruu_%28tksymkw%29", - "#category": ("moebooru", "lolibooru", "tag"), - "#class" : moebooru.MoebooruTagExtractor, -}, - -{ - "#url" : "https://lolibooru.moe/pool/show/239", - "#category": ("moebooru", "lolibooru", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, -}, - -{ - "#url" : "https://lolibooru.moe/post/popular_recent", - "#category": ("moebooru", "lolibooru", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, -}, - + { + "#url": "https://lolibooru.moe/post/show/281305/", + "#category": ("moebooru", "lolibooru", "post"), + "#class": moebooru.MoebooruPostExtractor, + "#options": {"notes": True}, + "#sha1_content": "a331430223ffc5b23c31649102e7d49f52489b57", + "notes": list, + }, + { + "#url": "https://lolibooru.moe/post/show/287835", + "#category": ("moebooru", "lolibooru", "post"), + "#class": moebooru.MoebooruPostExtractor, + }, + { + "#url": "https://lolibooru.moe/post?tags=ruu_%28tksymkw%29", + "#category": ("moebooru", "lolibooru", "tag"), + "#class": moebooru.MoebooruTagExtractor, + }, + { + "#url": "https://lolibooru.moe/pool/show/239", + "#category": ("moebooru", "lolibooru", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + }, + { + "#url": "https://lolibooru.moe/post/popular_recent", + "#category": ("moebooru", "lolibooru", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + }, ) diff --git a/test/results/loungeunderwear.py b/test/results/loungeunderwear.py index 521de25cc9..e556972376 100644 --- a/test/results/loungeunderwear.py +++ b/test/results/loungeunderwear.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://loungeunderwear.com/collections/apparel", - "#category": ("shopify", "loungeunderwear", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://de.loungeunderwear.com/products/ribbed-crop-top-black", - "#category": ("shopify", "loungeunderwear", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://loungeunderwear.com/collections/apparel", + "#category": ("shopify", "loungeunderwear", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://de.loungeunderwear.com/products/ribbed-crop-top-black", + "#category": ("shopify", "loungeunderwear", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/luscious.py b/test/results/luscious.py index 5e7a1460d5..df82b0f8aa 100644 --- a/test/results/luscious.py +++ b/test/results/luscious.py @@ -1,115 +1,104 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import luscious import datetime -from gallery_dl import exception +from gallery_dl import exception +from gallery_dl.extractor import luscious __tests__ = ( -{ - "#url" : "https://luscious.net/albums/okinami-no-koigokoro_277031/", - "#category": ("", "luscious", "album"), - "#class" : luscious.LusciousAlbumExtractor, - "#pattern" : r"https://storage\.bhs\.cloud\.ovh\.net/v1/AUTH_\w+/images/NTRshouldbeillegal/277031/luscious_net_\d+_\d+\.jpg$", - - "album" : { - "__typename" : "Album", - "audiences" : list, - "content" : "Hentai", - "cover" : r"re:https://storage\.bhs\.cloud\.ovh\.net/v1/.+/277031/", - "created" : 1479625853, - "created_by" : "Hive Mind", - "date" : "dt:2016-11-20 07:10:53", - "description" : "Enjoy.", - "download_url" : "/download/r/25/277031/", - "genres" : list, - "id" : 277031, - "is_manga" : True, - "labels" : list, - "language" : "English", - "like_status" : "none", - "modified" : int, - "permissions" : list, - "rating" : None, - "slug" : "okinami-no-koigokoro", - "status" : None, - "tags" : list, - "title" : "Okinami no Koigokoro", - "url" : "/albums/okinami-no-koigokoro_277031/", - "marked_for_deletion" : False, - "marked_for_processing" : False, - "number_of_animated_pictures": 0, - "number_of_favorites" : int, - "number_of_pictures" : 18, + { + "#url": "https://luscious.net/albums/okinami-no-koigokoro_277031/", + "#category": ("", "luscious", "album"), + "#class": luscious.LusciousAlbumExtractor, + "#pattern": r"https://storage\.bhs\.cloud\.ovh\.net/v1/AUTH_\w+/images/NTRshouldbeillegal/277031/luscious_net_\d+_\d+\.jpg$", + "album": { + "__typename": "Album", + "audiences": list, + "content": "Hentai", + "cover": r"re:https://storage\.bhs\.cloud\.ovh\.net/v1/.+/277031/", + "created": 1479625853, + "created_by": "Hive Mind", + "date": "dt:2016-11-20 07:10:53", + "description": "Enjoy.", + "download_url": "/download/r/25/277031/", + "genres": list, + "id": 277031, + "is_manga": True, + "labels": list, + "language": "English", + "like_status": "none", + "modified": int, + "permissions": list, + "rating": None, + "slug": "okinami-no-koigokoro", + "status": None, + "tags": list, + "title": "Okinami no Koigokoro", + "url": "/albums/okinami-no-koigokoro_277031/", + "marked_for_deletion": False, + "marked_for_processing": False, + "number_of_animated_pictures": 0, + "number_of_favorites": int, + "number_of_pictures": 18, + }, + "aspect_ratio": r"re:\d+:\d+", + "category": "luscious", + "created": int, + "date": datetime.datetime, + "height": int, + "id": int, + "is_animated": False, + "like_status": "none", + "position": int, + "resolution": r"re:\d+x\d+", + "status": None, + "tags": list, + "thumbnail": str, + "title": str, + "width": int, + "number_of_comments": int, + "number_of_favorites": int, + }, + { + "#url": "https://luscious.net/albums/not-found_277035/", + "#category": ("", "luscious", "album"), + "#class": luscious.LusciousAlbumExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://members.luscious.net/albums/login-required_323871/", + "#category": ("", "luscious", "album"), + "#class": luscious.LusciousAlbumExtractor, + "#count": 64, + }, + { + "#url": "https://www.luscious.net/albums/okinami_277031/", + "#category": ("", "luscious", "album"), + "#class": luscious.LusciousAlbumExtractor, + }, + { + "#url": "https://members.luscious.net/albums/okinami_277031/", + "#category": ("", "luscious", "album"), + "#class": luscious.LusciousAlbumExtractor, + }, + { + "#url": "https://luscious.net/pictures/c/video_game_manga/album/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1", + "#category": ("", "luscious", "album"), + "#class": luscious.LusciousAlbumExtractor, + }, + { + "#url": "https://members.luscious.net/albums/list/", + "#category": ("", "luscious", "search"), + "#class": luscious.LusciousSearchExtractor, + }, + { + "#url": "https://members.luscious.net/albums/list/?display=date_newest&language_ids=%2B1&tagged=+full_color&page=1", + "#category": ("", "luscious", "search"), + "#class": luscious.LusciousSearchExtractor, + "#pattern": luscious.LusciousAlbumExtractor.pattern, + "#range": "41-60", + "#count": 20, }, - "aspect_ratio" : r"re:\d+:\d+", - "category" : "luscious", - "created" : int, - "date" : datetime.datetime, - "height" : int, - "id" : int, - "is_animated" : False, - "like_status" : "none", - "position" : int, - "resolution" : r"re:\d+x\d+", - "status" : None, - "tags" : list, - "thumbnail" : str, - "title" : str, - "width" : int, - "number_of_comments": int, - "number_of_favorites": int, -}, - -{ - "#url" : "https://luscious.net/albums/not-found_277035/", - "#category": ("", "luscious", "album"), - "#class" : luscious.LusciousAlbumExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://members.luscious.net/albums/login-required_323871/", - "#category": ("", "luscious", "album"), - "#class" : luscious.LusciousAlbumExtractor, - "#count" : 64, -}, - -{ - "#url" : "https://www.luscious.net/albums/okinami_277031/", - "#category": ("", "luscious", "album"), - "#class" : luscious.LusciousAlbumExtractor, -}, - -{ - "#url" : "https://members.luscious.net/albums/okinami_277031/", - "#category": ("", "luscious", "album"), - "#class" : luscious.LusciousAlbumExtractor, -}, - -{ - "#url" : "https://luscious.net/pictures/c/video_game_manga/album/okinami-no-koigokoro_277031/sorted/position/id/16528978/@_1", - "#category": ("", "luscious", "album"), - "#class" : luscious.LusciousAlbumExtractor, -}, - -{ - "#url" : "https://members.luscious.net/albums/list/", - "#category": ("", "luscious", "search"), - "#class" : luscious.LusciousSearchExtractor, -}, - -{ - "#url" : "https://members.luscious.net/albums/list/?display=date_newest&language_ids=%2B1&tagged=+full_color&page=1", - "#category": ("", "luscious", "search"), - "#class" : luscious.LusciousSearchExtractor, - "#pattern" : luscious.LusciousAlbumExtractor.pattern, - "#range" : "41-60", - "#count" : 20, -}, - ) diff --git a/test/results/mangadex.py b/test/results/mangadex.py index 817b094138..2121fecc43 100644 --- a/test/results/mangadex.py +++ b/test/results/mangadex.py @@ -1,164 +1,145 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangadex -from gallery_dl import exception import datetime +from gallery_dl import exception +from gallery_dl.extractor import mangadex __tests__ = ( -{ - "#url" : "https://mangadex.org/chapter/f946ac53-0b71-4b5d-aeb2-7931b13c4aaa", - "#category": ("", "mangadex", "chapter"), - "#class" : mangadex.MangadexChapterExtractor, - "#sha1_metadata": "e86128a79ebe7201b648f1caa828496a2878dc8f", -}, - -{ - "#url" : "https://mangadex.org/chapter/61a88817-9c29-4281-bdf1-77b3c1be9831", - "#comment" : "oneshot", - "#category": ("", "mangadex", "chapter"), - "#class" : mangadex.MangadexChapterExtractor, - "#count" : 64, - "#sha1_metadata": "d11ed057a919854696853362be35fc0ba7dded4c", -}, - -{ - "#url" : "https://mangadex.org/chapter/74149a55-e7c4-44ea-8a37-98e879c1096f", - "#comment" : "MANGA Plus (#1154)", - "#category": ("", "mangadex", "chapter"), - "#class" : mangadex.MangadexChapterExtractor, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://mangadex.org/chapter/364728a4-6909-4164-9eea-6b56354f7c78", - "#comment" : "'externalUrl', but still downloadable / 404 (#2503)", - "#category": ("", "mangadex", "chapter"), - "#class" : mangadex.MangadexChapterExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", - "#comment" : "mutliple values for 'lang' (#4093)", - "#category": ("", "mangadex", "manga"), - "#class" : mangadex.MangadexMangaExtractor, - "#count" : ">= 5", - - "manga" : "Souten no Koumori", - "manga_id" : "f90c4398-8aad-4f51-8a1f-024ca09fdcbc", - "title" : r"re:One[Ss]hot", - "volume" : 0, - "chapter" : 0, - "chapter_minor": "", - "chapter_id" : str, - "date" : datetime.datetime, - "lang" : str, - "language" : str, - "artist" : ["Arakawa Hiromu"], - "author" : ["Arakawa Hiromu"], - "status" : "completed", - "tags" : [ - "Oneshot", - "Historical", - "Action", - "Martial Arts", - "Drama", - "Tragedy", - ], -}, - -{ - "#url" : "https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", - "#comment" : "mutliple values for 'lang' (#4093)", - "#category": ("", "mangadex", "manga"), - "#class" : mangadex.MangadexMangaExtractor, - "#options" : {"lang": "fr,it"}, - "#count" : 2, - - "manga" : "Souten no Koumori", - "lang" : {"fr", "it"}, - "language": {"French", "Italian"}, -}, - -{ - "#url" : "https://mangadex.cc/manga/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a/", - "#category": ("", "mangadex", "manga"), - "#class" : mangadex.MangadexMangaExtractor, - "#options" : {"lang": "en"}, - "#count" : ">= 100", -}, - -{ - "#url" : "https://mangadex.org/title/7c1e2742-a086-4fd3-a3be-701fd6cf0be9", - "#category": ("", "mangadex", "manga"), - "#class" : mangadex.MangadexMangaExtractor, - "#count" : ">= 25", -}, - -{ - "#url" : "https://mangadex.org/title/584ef094-b2ab-40ce-962c-bce341fb9d10", - "#category": ("", "mangadex", "manga"), - "#class" : mangadex.MangadexMangaExtractor, - "#count" : ">= 20", -}, - -{ - "#url" : "https://mangadex.org/title/feed", - "#category": ("", "mangadex", "feed"), - "#class" : mangadex.MangadexFeedExtractor, -}, - -{ - "#url" : "https://mangadex.org/list/3a0982c5-65aa-4de2-8a4a-2175be7383ab/test", - "#category": ("", "mangadex", "list"), - "#class" : mangadex.MangadexListExtractor, - "#urls" : ( - "https://mangadex.org/title/cba4e5d6-67a0-47a0-b37a-c06e9bf25d93", - "https://mangadex.org/title/cad76ec6-ca22-42f6-96f8-eca164da6545", - ), -}, - -{ - "#url" : "https://mangadex.org/list/3a0982c5-65aa-4de2-8a4a-2175be7383ab/test?tab=titles", - "#category": ("", "mangadex", "list"), - "#class" : mangadex.MangadexListExtractor, -}, - -{ - "#url" : "https://mangadex.org/list/3a0982c5-65aa-4de2-8a4a-2175be7383ab/test?tab=feed", - "#category": ("", "mangadex", "list-feed"), - "#class" : mangadex.MangadexListExtractor, - "#urls" : ( - "https://mangadex.org/chapter/fa8a695d-260f-4dcc-95a3-1f30e66d6571", - "https://mangadex.org/chapter/c765d6d5-5712-4360-be0b-0c8e0914fc94", - "https://mangadex.org/chapter/788766b9-41c6-422e-97ba-552f03ba9655", - ), -}, - -{ - "#url" : "https://mangadex.org/author/7222d0d5-836c-4bf3-9174-72bceade8c87/kotoyama", - "#class" : mangadex.MangadexAuthorExtractor, - "#urls" : ( - "https://mangadex.org/title/259dfd8a-f06a-4825-8fa6-a2dcd7274230", - "https://mangadex.org/title/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a", - "https://mangadex.org/title/f48bbb5f-8a23-4dea-8177-eb2dbbcbf4fa", - "https://mangadex.org/title/00b68132-4e69-4ff9-ad4b-29138b377dc8", - "https://mangadex.org/title/f1b70bba-3873-4c22-afa3-1d1c78299cd9", - "https://mangadex.org/title/41cd6fa7-3e53-4900-88e6-4a06cd7df9ad", - ), -}, - -{ - "#url" : "https://mangadex.org/author/254efca2-0ac0-432c-a3a3-55b7e207e87d/flipflops", - "#class" : mangadex.MangadexAuthorExtractor, - "#pattern" : mangadex.MangadexMangaExtractor.pattern, - "#options" : {"lang": "en"}, - "#count" : ">= 15", -}, - + { + "#url": "https://mangadex.org/chapter/f946ac53-0b71-4b5d-aeb2-7931b13c4aaa", + "#category": ("", "mangadex", "chapter"), + "#class": mangadex.MangadexChapterExtractor, + "#sha1_metadata": "e86128a79ebe7201b648f1caa828496a2878dc8f", + }, + { + "#url": "https://mangadex.org/chapter/61a88817-9c29-4281-bdf1-77b3c1be9831", + "#comment": "oneshot", + "#category": ("", "mangadex", "chapter"), + "#class": mangadex.MangadexChapterExtractor, + "#count": 64, + "#sha1_metadata": "d11ed057a919854696853362be35fc0ba7dded4c", + }, + { + "#url": "https://mangadex.org/chapter/74149a55-e7c4-44ea-8a37-98e879c1096f", + "#comment": "MANGA Plus (#1154)", + "#category": ("", "mangadex", "chapter"), + "#class": mangadex.MangadexChapterExtractor, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://mangadex.org/chapter/364728a4-6909-4164-9eea-6b56354f7c78", + "#comment": "'externalUrl', but still downloadable / 404 (#2503)", + "#category": ("", "mangadex", "chapter"), + "#class": mangadex.MangadexChapterExtractor, + "#count": 0, + }, + { + "#url": "https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", + "#comment": "mutliple values for 'lang' (#4093)", + "#category": ("", "mangadex", "manga"), + "#class": mangadex.MangadexMangaExtractor, + "#count": ">= 5", + "manga": "Souten no Koumori", + "manga_id": "f90c4398-8aad-4f51-8a1f-024ca09fdcbc", + "title": r"re:One[Ss]hot", + "volume": 0, + "chapter": 0, + "chapter_minor": "", + "chapter_id": str, + "date": datetime.datetime, + "lang": str, + "language": str, + "artist": ["Arakawa Hiromu"], + "author": ["Arakawa Hiromu"], + "status": "completed", + "tags": [ + "Oneshot", + "Historical", + "Action", + "Martial Arts", + "Drama", + "Tragedy", + ], + }, + { + "#url": "https://mangadex.org/title/f90c4398-8aad-4f51-8a1f-024ca09fdcbc", + "#comment": "mutliple values for 'lang' (#4093)", + "#category": ("", "mangadex", "manga"), + "#class": mangadex.MangadexMangaExtractor, + "#options": {"lang": "fr,it"}, + "#count": 2, + "manga": "Souten no Koumori", + "lang": {"fr", "it"}, + "language": {"French", "Italian"}, + }, + { + "#url": "https://mangadex.cc/manga/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a/", + "#category": ("", "mangadex", "manga"), + "#class": mangadex.MangadexMangaExtractor, + "#options": {"lang": "en"}, + "#count": ">= 100", + }, + { + "#url": "https://mangadex.org/title/7c1e2742-a086-4fd3-a3be-701fd6cf0be9", + "#category": ("", "mangadex", "manga"), + "#class": mangadex.MangadexMangaExtractor, + "#count": ">= 25", + }, + { + "#url": "https://mangadex.org/title/584ef094-b2ab-40ce-962c-bce341fb9d10", + "#category": ("", "mangadex", "manga"), + "#class": mangadex.MangadexMangaExtractor, + "#count": ">= 20", + }, + { + "#url": "https://mangadex.org/title/feed", + "#category": ("", "mangadex", "feed"), + "#class": mangadex.MangadexFeedExtractor, + }, + { + "#url": "https://mangadex.org/list/3a0982c5-65aa-4de2-8a4a-2175be7383ab/test", + "#category": ("", "mangadex", "list"), + "#class": mangadex.MangadexListExtractor, + "#urls": ( + "https://mangadex.org/title/cba4e5d6-67a0-47a0-b37a-c06e9bf25d93", + "https://mangadex.org/title/cad76ec6-ca22-42f6-96f8-eca164da6545", + ), + }, + { + "#url": "https://mangadex.org/list/3a0982c5-65aa-4de2-8a4a-2175be7383ab/test?tab=titles", + "#category": ("", "mangadex", "list"), + "#class": mangadex.MangadexListExtractor, + }, + { + "#url": "https://mangadex.org/list/3a0982c5-65aa-4de2-8a4a-2175be7383ab/test?tab=feed", + "#category": ("", "mangadex", "list-feed"), + "#class": mangadex.MangadexListExtractor, + "#urls": ( + "https://mangadex.org/chapter/fa8a695d-260f-4dcc-95a3-1f30e66d6571", + "https://mangadex.org/chapter/c765d6d5-5712-4360-be0b-0c8e0914fc94", + "https://mangadex.org/chapter/788766b9-41c6-422e-97ba-552f03ba9655", + ), + }, + { + "#url": "https://mangadex.org/author/7222d0d5-836c-4bf3-9174-72bceade8c87/kotoyama", + "#class": mangadex.MangadexAuthorExtractor, + "#urls": ( + "https://mangadex.org/title/259dfd8a-f06a-4825-8fa6-a2dcd7274230", + "https://mangadex.org/title/d0c88e3b-ea64-4e07-9841-c1d2ac982f4a", + "https://mangadex.org/title/f48bbb5f-8a23-4dea-8177-eb2dbbcbf4fa", + "https://mangadex.org/title/00b68132-4e69-4ff9-ad4b-29138b377dc8", + "https://mangadex.org/title/f1b70bba-3873-4c22-afa3-1d1c78299cd9", + "https://mangadex.org/title/41cd6fa7-3e53-4900-88e6-4a06cd7df9ad", + ), + }, + { + "#url": "https://mangadex.org/author/254efca2-0ac0-432c-a3a3-55b7e207e87d/flipflops", + "#class": mangadex.MangadexAuthorExtractor, + "#pattern": mangadex.MangadexMangaExtractor.pattern, + "#options": {"lang": "en"}, + "#count": ">= 15", + }, ) diff --git a/test/results/mangafox.py b/test/results/mangafox.py index dc0cc9a04b..1e1a826049 100644 --- a/test/results/mangafox.py +++ b/test/results/mangafox.py @@ -1,71 +1,62 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangafox import datetime +from gallery_dl.extractor import mangafox __tests__ = ( -{ - "#url" : "http://fanfox.net/manga/kidou_keisatsu_patlabor/v05/c006.2/1.html", - "#category": ("", "mangafox", "chapter"), - "#class" : mangafox.MangafoxChapterExtractor, - "#sha1_metadata": "5661dab258d42d09d98f194f7172fb9851a49766", - "#sha1_content" : "5c50c252dcf12ffecf68801f4db8a2167265f66c", -}, - -{ - "#url" : "http://mangafox.me/manga/kidou_keisatsu_patlabor/v05/c006.2/", - "#category": ("", "mangafox", "chapter"), - "#class" : mangafox.MangafoxChapterExtractor, -}, - -{ - "#url" : "http://fanfox.net/manga/black_clover/vTBD/c295/1.html", - "#category": ("", "mangafox", "chapter"), - "#class" : mangafox.MangafoxChapterExtractor, -}, - -{ - "#url" : "https://fanfox.net/manga/kanojo_mo_kanojo", - "#category": ("", "mangafox", "manga"), - "#class" : mangafox.MangafoxMangaExtractor, - "#pattern" : mangafox.MangafoxChapterExtractor.pattern, - "#count" : ">=60", - - "author" : "HIROYUKI", - "chapter" : int, - "chapter_minor" : r"re:^(\.\d+)?$", - "chapter_string": r"re:(v\d+/)?c\d+", - "date" : datetime.datetime, - "description" : "High school boy Naoya gets a confession from Momi, a cute and friendly girl. However, Naoya already has a girlfriend, Seki... but Momi is too good a catch to let go. Momi and Nagoya's goal becomes clear: convince Seki to accept being an item with the two of them. Will she budge?", - "lang" : "en", - "language" : "English", - "manga" : "Kanojo mo Kanojo", - "tags" : [ - "Comedy", - "Romance", - "School Life", - "Shounen", - ], - "volume" : int, -}, - -{ - "#url" : "https://mangafox.me/manga/shangri_la_frontier", - "#category": ("", "mangafox", "manga"), - "#class" : mangafox.MangafoxMangaExtractor, - "#pattern" : mangafox.MangafoxChapterExtractor.pattern, - "#count" : ">=45", -}, - -{ - "#url" : "https://m.fanfox.net/manga/sentai_daishikkaku", - "#category": ("", "mangafox", "manga"), - "#class" : mangafox.MangafoxMangaExtractor, -}, - + { + "#url": "http://fanfox.net/manga/kidou_keisatsu_patlabor/v05/c006.2/1.html", + "#category": ("", "mangafox", "chapter"), + "#class": mangafox.MangafoxChapterExtractor, + "#sha1_metadata": "5661dab258d42d09d98f194f7172fb9851a49766", + "#sha1_content": "5c50c252dcf12ffecf68801f4db8a2167265f66c", + }, + { + "#url": "http://mangafox.me/manga/kidou_keisatsu_patlabor/v05/c006.2/", + "#category": ("", "mangafox", "chapter"), + "#class": mangafox.MangafoxChapterExtractor, + }, + { + "#url": "http://fanfox.net/manga/black_clover/vTBD/c295/1.html", + "#category": ("", "mangafox", "chapter"), + "#class": mangafox.MangafoxChapterExtractor, + }, + { + "#url": "https://fanfox.net/manga/kanojo_mo_kanojo", + "#category": ("", "mangafox", "manga"), + "#class": mangafox.MangafoxMangaExtractor, + "#pattern": mangafox.MangafoxChapterExtractor.pattern, + "#count": ">=60", + "author": "HIROYUKI", + "chapter": int, + "chapter_minor": r"re:^(\.\d+)?$", + "chapter_string": r"re:(v\d+/)?c\d+", + "date": datetime.datetime, + "description": "High school boy Naoya gets a confession from Momi, a cute and friendly girl. However, Naoya already has a girlfriend, Seki... but Momi is too good a catch to let go. Momi and Nagoya's goal becomes clear: convince Seki to accept being an item with the two of them. Will she budge?", + "lang": "en", + "language": "English", + "manga": "Kanojo mo Kanojo", + "tags": [ + "Comedy", + "Romance", + "School Life", + "Shounen", + ], + "volume": int, + }, + { + "#url": "https://mangafox.me/manga/shangri_la_frontier", + "#category": ("", "mangafox", "manga"), + "#class": mangafox.MangafoxMangaExtractor, + "#pattern": mangafox.MangafoxChapterExtractor.pattern, + "#count": ">=45", + }, + { + "#url": "https://m.fanfox.net/manga/sentai_daishikkaku", + "#category": ("", "mangafox", "manga"), + "#class": mangafox.MangafoxMangaExtractor, + }, ) diff --git a/test/results/mangahere.py b/test/results/mangahere.py index db17a6b082..cd78ad9a01 100644 --- a/test/results/mangahere.py +++ b/test/results/mangahere.py @@ -1,77 +1,65 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mangahere - __tests__ = ( -{ - "#url" : "https://www.mangahere.cc/manga/dongguo_xiaojie/c004.2/", - "#category": ("", "mangahere", "chapter"), - "#class" : mangahere.MangahereChapterExtractor, - "#sha1_metadata": "7c98d7b50a47e6757b089aa875a53aa970cac66f", - "#sha1_content" : "708d475f06893b88549cbd30df1e3f9428f2c884", -}, - -{ - "#url" : "https://www.mangahere.cc/manga/beastars/c196/1.html", - "#comment" : "URLs without HTTP scheme (#1070)", - "#category": ("", "mangahere", "chapter"), - "#class" : mangahere.MangahereChapterExtractor, - "#pattern" : "https://zjcdn.mangahere.org/.*", -}, - -{ - "#url" : "http://www.mangahere.co/manga/dongguo_xiaojie/c003.2/", - "#category": ("", "mangahere", "chapter"), - "#class" : mangahere.MangahereChapterExtractor, -}, - -{ - "#url" : "http://m.mangahere.co/manga/dongguo_xiaojie/c003.2/", - "#category": ("", "mangahere", "chapter"), - "#class" : mangahere.MangahereChapterExtractor, -}, - -{ - "#url" : "https://www.mangahere.cc/manga/aria/", - "#category": ("", "mangahere", "manga"), - "#class" : mangahere.MangahereMangaExtractor, - "#count" : 71, - "#sha1_url" : "9c2e54ec42e9a87ad53096c328b33c90750af3e4", - "#sha1_metadata": "71503c682c5d0c277a50409a8c5fd78e871e3d69", -}, - -{ - "#url" : "https://www.mangahere.cc/manga/hiyokoi/#50", - "#category": ("", "mangahere", "manga"), - "#class" : mangahere.MangahereMangaExtractor, - "#sha1_url" : "654850570aa03825cd57e2ae2904af489602c523", - "#sha1_metadata": "c8084d89a9ea6cf40353093669f9601a39bf5ca2", -}, - -{ - "#url" : "http://www.mangahere.cc/manga/gunnm_mars_chronicle/", - "#comment" : "adult filter (#556)", - "#category": ("", "mangahere", "manga"), - "#class" : mangahere.MangahereMangaExtractor, - "#pattern" : mangahere.MangahereChapterExtractor.pattern, - "#count" : ">= 50", -}, - -{ - "#url" : "https://www.mangahere.co/manga/aria/", - "#category": ("", "mangahere", "manga"), - "#class" : mangahere.MangahereMangaExtractor, -}, - -{ - "#url" : "https://m.mangahere.co/manga/aria/", - "#category": ("", "mangahere", "manga"), - "#class" : mangahere.MangahereMangaExtractor, -}, - + { + "#url": "https://www.mangahere.cc/manga/dongguo_xiaojie/c004.2/", + "#category": ("", "mangahere", "chapter"), + "#class": mangahere.MangahereChapterExtractor, + "#sha1_metadata": "7c98d7b50a47e6757b089aa875a53aa970cac66f", + "#sha1_content": "708d475f06893b88549cbd30df1e3f9428f2c884", + }, + { + "#url": "https://www.mangahere.cc/manga/beastars/c196/1.html", + "#comment": "URLs without HTTP scheme (#1070)", + "#category": ("", "mangahere", "chapter"), + "#class": mangahere.MangahereChapterExtractor, + "#pattern": "https://zjcdn.mangahere.org/.*", + }, + { + "#url": "http://www.mangahere.co/manga/dongguo_xiaojie/c003.2/", + "#category": ("", "mangahere", "chapter"), + "#class": mangahere.MangahereChapterExtractor, + }, + { + "#url": "http://m.mangahere.co/manga/dongguo_xiaojie/c003.2/", + "#category": ("", "mangahere", "chapter"), + "#class": mangahere.MangahereChapterExtractor, + }, + { + "#url": "https://www.mangahere.cc/manga/aria/", + "#category": ("", "mangahere", "manga"), + "#class": mangahere.MangahereMangaExtractor, + "#count": 71, + "#sha1_url": "9c2e54ec42e9a87ad53096c328b33c90750af3e4", + "#sha1_metadata": "71503c682c5d0c277a50409a8c5fd78e871e3d69", + }, + { + "#url": "https://www.mangahere.cc/manga/hiyokoi/#50", + "#category": ("", "mangahere", "manga"), + "#class": mangahere.MangahereMangaExtractor, + "#sha1_url": "654850570aa03825cd57e2ae2904af489602c523", + "#sha1_metadata": "c8084d89a9ea6cf40353093669f9601a39bf5ca2", + }, + { + "#url": "http://www.mangahere.cc/manga/gunnm_mars_chronicle/", + "#comment": "adult filter (#556)", + "#category": ("", "mangahere", "manga"), + "#class": mangahere.MangahereMangaExtractor, + "#pattern": mangahere.MangahereChapterExtractor.pattern, + "#count": ">= 50", + }, + { + "#url": "https://www.mangahere.co/manga/aria/", + "#category": ("", "mangahere", "manga"), + "#class": mangahere.MangahereMangaExtractor, + }, + { + "#url": "https://m.mangahere.co/manga/aria/", + "#category": ("", "mangahere", "manga"), + "#class": mangahere.MangahereMangaExtractor, + }, ) diff --git a/test/results/mangakakalot.py b/test/results/mangakakalot.py index b0b8badabc..91b52d1f24 100644 --- a/test/results/mangakakalot.py +++ b/test/results/mangakakalot.py @@ -1,40 +1,33 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mangakakalot - __tests__ = ( -{ - "#url" : "https://ww3.mangakakalot.tv/chapter/manga-jk986845/chapter-34.2", - "#category": ("", "mangakakalot", "chapter"), - "#class" : mangakakalot.MangakakalotChapterExtractor, - "#pattern" : r"https://cm\.blazefast\.co/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.jpg", - "#count" : 9, - "#sha1_metadata": "0f1586ff52f0f9cbbb25306ae64ab718f8a6a633", -}, - -{ - "#url" : "https://mangakakalot.tv/chapter/hatarakanai_futari_the_jobless_siblings/chapter_20.1", - "#category": ("", "mangakakalot", "chapter"), - "#class" : mangakakalot.MangakakalotChapterExtractor, -}, - -{ - "#url" : "https://ww3.mangakakalot.tv/manga/manga-jk986845", - "#category": ("", "mangakakalot", "manga"), - "#class" : mangakakalot.MangakakalotMangaExtractor, - "#pattern" : mangakakalot.MangakakalotChapterExtractor.pattern, - "#count" : ">= 30", -}, - -{ - "#url" : "https://mangakakalot.tv/manga/lk921810", - "#category": ("", "mangakakalot", "manga"), - "#class" : mangakakalot.MangakakalotMangaExtractor, -}, - + { + "#url": "https://ww3.mangakakalot.tv/chapter/manga-jk986845/chapter-34.2", + "#category": ("", "mangakakalot", "chapter"), + "#class": mangakakalot.MangakakalotChapterExtractor, + "#pattern": r"https://cm\.blazefast\.co/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.jpg", + "#count": 9, + "#sha1_metadata": "0f1586ff52f0f9cbbb25306ae64ab718f8a6a633", + }, + { + "#url": "https://mangakakalot.tv/chapter/hatarakanai_futari_the_jobless_siblings/chapter_20.1", + "#category": ("", "mangakakalot", "chapter"), + "#class": mangakakalot.MangakakalotChapterExtractor, + }, + { + "#url": "https://ww3.mangakakalot.tv/manga/manga-jk986845", + "#category": ("", "mangakakalot", "manga"), + "#class": mangakakalot.MangakakalotMangaExtractor, + "#pattern": mangakakalot.MangakakalotChapterExtractor.pattern, + "#count": ">= 30", + }, + { + "#url": "https://mangakakalot.tv/manga/lk921810", + "#category": ("", "mangakakalot", "manga"), + "#class": mangakakalot.MangakakalotMangaExtractor, + }, ) diff --git a/test/results/mangalife.py b/test/results/mangalife.py index 80226afef3..64bd40b8a1 100644 --- a/test/results/mangalife.py +++ b/test/results/mangalife.py @@ -1,69 +1,63 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangasee import datetime +from gallery_dl.extractor import mangasee __tests__ = ( -{ - "#url" : "https://manga4life.com/read-online/One-Piece-chapter-1063-page-1.html", - "#category": ("", "mangalife", "chapter"), - "#class" : mangasee.MangaseeChapterExtractor, - "#pattern" : r"https://[^/]+/manga/One-Piece/1063-0\d\d\.png", - "#count" : 13, - - "author" : ["ODA Eiichiro"], - "chapter" : 1063, - "chapter_minor" : "", - "chapter_string": "110630", - "count" : 13, - "date" : "dt:2022-10-16 17:32:54", - "extension" : "png", - "filename" : r"re:1063-0\d\d", - "genre" : [ - "Action", - "Adventure", - "Comedy", - "Drama", - "Fantasy", - "Shounen", - ], - "index" : "1", - "lang" : "en", - "language" : "English", - "manga" : "One Piece", - "page" : int, - "title" : "", -}, - -{ - "#url" : "https://manga4life.com/manga/Ano-Musume-Ni-Kiss-To-Shirayuri-O", - "#category": ("", "mangalife", "manga"), - "#class" : mangasee.MangaseeMangaExtractor, - "#pattern" : mangasee.MangaseeChapterExtractor.pattern, - "#count" : ">= 50", - - "author" : ["Canno"], - "chapter" : int, - "chapter_minor" : r"re:^|\.5$", - "chapter_string": r"re:100\d\d\d", - "date" : datetime.datetime, - "genre" : [ - "Comedy", - "Romance", - "School Life", - "Seinen", - "Shoujo Ai", - ], - "index" : "1", - "lang" : "en", - "language" : "English", - "manga" : "Ano-Musume-Ni-Kiss-To-Shirayuri-O", - "title" : "", -}, - + { + "#url": "https://manga4life.com/read-online/One-Piece-chapter-1063-page-1.html", + "#category": ("", "mangalife", "chapter"), + "#class": mangasee.MangaseeChapterExtractor, + "#pattern": r"https://[^/]+/manga/One-Piece/1063-0\d\d\.png", + "#count": 13, + "author": ["ODA Eiichiro"], + "chapter": 1063, + "chapter_minor": "", + "chapter_string": "110630", + "count": 13, + "date": "dt:2022-10-16 17:32:54", + "extension": "png", + "filename": r"re:1063-0\d\d", + "genre": [ + "Action", + "Adventure", + "Comedy", + "Drama", + "Fantasy", + "Shounen", + ], + "index": "1", + "lang": "en", + "language": "English", + "manga": "One Piece", + "page": int, + "title": "", + }, + { + "#url": "https://manga4life.com/manga/Ano-Musume-Ni-Kiss-To-Shirayuri-O", + "#category": ("", "mangalife", "manga"), + "#class": mangasee.MangaseeMangaExtractor, + "#pattern": mangasee.MangaseeChapterExtractor.pattern, + "#count": ">= 50", + "author": ["Canno"], + "chapter": int, + "chapter_minor": r"re:^|\.5$", + "chapter_string": r"re:100\d\d\d", + "date": datetime.datetime, + "genre": [ + "Comedy", + "Romance", + "School Life", + "Seinen", + "Shoujo Ai", + ], + "index": "1", + "lang": "en", + "language": "English", + "manga": "Ano-Musume-Ni-Kiss-To-Shirayuri-O", + "title": "", + }, ) diff --git a/test/results/manganelo.py b/test/results/manganelo.py index b5772656cd..a052c2e732 100644 --- a/test/results/manganelo.py +++ b/test/results/manganelo.py @@ -1,91 +1,76 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import manganelo - __tests__ = ( -{ - "#url" : "https://chapmanganato.com/manga-gn983696/chapter-23", - "#category": ("", "manganelo", "chapter"), - "#class" : manganelo.ManganeloChapterExtractor, - "#pattern" : r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/03/23/39/gn983696/vol_3_chapter_23_24_yen/\d+-[no]\.jpg", - "#count" : 25, - "#sha1_metadata": "17faaea7f0fb8c2675a327bf3aa0bcd7a6311d68", -}, - -{ - "#url" : "https://chapmanganelo.com/manga-ti107776/chapter-4", - "#category": ("", "manganelo", "chapter"), - "#class" : manganelo.ManganeloChapterExtractor, - "#pattern" : r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/01/92/08/ti970565/chapter_4_caster/\d+-o\.jpg", - "#count" : 45, - "#sha1_metadata": "06e01fa9b3fc9b5b954c0d4a98f0153b40922ded", -}, - -{ - "#url" : "https://chapmanganato.com/manga-no991297/chapter-8", - "#category": ("", "manganelo", "chapter"), - "#class" : manganelo.ManganeloChapterExtractor, - "#count" : 20, - - "chapter" : 8, - "chapter_minor": "-1", -}, - -{ - "#url" : "https://readmanganato.com/manga-gn983696/chapter-23", - "#category": ("", "manganelo", "chapter"), - "#class" : manganelo.ManganeloChapterExtractor, -}, - -{ - "#url" : "https://manganelo.com/chapter/gamers/chapter_15", - "#category": ("", "manganelo", "chapter"), - "#class" : manganelo.ManganeloChapterExtractor, -}, - -{ - "#url" : "https://manganelo.com/chapter/gq921227/chapter_23", - "#category": ("", "manganelo", "chapter"), - "#class" : manganelo.ManganeloChapterExtractor, -}, - -{ - "#url" : "https://chapmanganato.com/manga-gn983696", - "#category": ("", "manganelo", "manga"), - "#class" : manganelo.ManganeloMangaExtractor, - "#pattern" : manganelo.ManganeloChapterExtractor.pattern, - "#count" : ">= 25", -}, - -{ - "#url" : "https://m.manganelo.com/manga-ti107776", - "#category": ("", "manganelo", "manga"), - "#class" : manganelo.ManganeloMangaExtractor, - "#pattern" : manganelo.ManganeloChapterExtractor.pattern, - "#count" : ">= 12", -}, - -{ - "#url" : "https://readmanganato.com/manga-gn983696", - "#category": ("", "manganelo", "manga"), - "#class" : manganelo.ManganeloMangaExtractor, -}, - -{ - "#url" : "https://manganelo.com/manga/read_otome_no_teikoku", - "#category": ("", "manganelo", "manga"), - "#class" : manganelo.ManganeloMangaExtractor, -}, - -{ - "#url" : "https://manganelo.com/manga/ol921234/", - "#category": ("", "manganelo", "manga"), - "#class" : manganelo.ManganeloMangaExtractor, -}, - + { + "#url": "https://chapmanganato.com/manga-gn983696/chapter-23", + "#category": ("", "manganelo", "chapter"), + "#class": manganelo.ManganeloChapterExtractor, + "#pattern": r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/03/23/39/gn983696/vol_3_chapter_23_24_yen/\d+-[no]\.jpg", + "#count": 25, + "#sha1_metadata": "17faaea7f0fb8c2675a327bf3aa0bcd7a6311d68", + }, + { + "#url": "https://chapmanganelo.com/manga-ti107776/chapter-4", + "#category": ("", "manganelo", "chapter"), + "#class": manganelo.ManganeloChapterExtractor, + "#pattern": r"https://v\d+\.mkklcdnv6tempv5\.com/img/tab_17/01/92/08/ti970565/chapter_4_caster/\d+-o\.jpg", + "#count": 45, + "#sha1_metadata": "06e01fa9b3fc9b5b954c0d4a98f0153b40922ded", + }, + { + "#url": "https://chapmanganato.com/manga-no991297/chapter-8", + "#category": ("", "manganelo", "chapter"), + "#class": manganelo.ManganeloChapterExtractor, + "#count": 20, + "chapter": 8, + "chapter_minor": "-1", + }, + { + "#url": "https://readmanganato.com/manga-gn983696/chapter-23", + "#category": ("", "manganelo", "chapter"), + "#class": manganelo.ManganeloChapterExtractor, + }, + { + "#url": "https://manganelo.com/chapter/gamers/chapter_15", + "#category": ("", "manganelo", "chapter"), + "#class": manganelo.ManganeloChapterExtractor, + }, + { + "#url": "https://manganelo.com/chapter/gq921227/chapter_23", + "#category": ("", "manganelo", "chapter"), + "#class": manganelo.ManganeloChapterExtractor, + }, + { + "#url": "https://chapmanganato.com/manga-gn983696", + "#category": ("", "manganelo", "manga"), + "#class": manganelo.ManganeloMangaExtractor, + "#pattern": manganelo.ManganeloChapterExtractor.pattern, + "#count": ">= 25", + }, + { + "#url": "https://m.manganelo.com/manga-ti107776", + "#category": ("", "manganelo", "manga"), + "#class": manganelo.ManganeloMangaExtractor, + "#pattern": manganelo.ManganeloChapterExtractor.pattern, + "#count": ">= 12", + }, + { + "#url": "https://readmanganato.com/manga-gn983696", + "#category": ("", "manganelo", "manga"), + "#class": manganelo.ManganeloMangaExtractor, + }, + { + "#url": "https://manganelo.com/manga/read_otome_no_teikoku", + "#category": ("", "manganelo", "manga"), + "#class": manganelo.ManganeloMangaExtractor, + }, + { + "#url": "https://manganelo.com/manga/ol921234/", + "#category": ("", "manganelo", "manga"), + "#class": manganelo.ManganeloMangaExtractor, + }, ) diff --git a/test/results/mangapark.py b/test/results/mangapark.py index 432f535fae..3685c1ac9b 100644 --- a/test/results/mangapark.py +++ b/test/results/mangapark.py @@ -1,140 +1,122 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangapark import datetime +from gallery_dl.extractor import mangapark __tests__ = ( -{ - "#url" : "https://mangapark.net/title/114972-aria/6710214-en-ch.60.2", - "#category": ("", "mangapark", "chapter"), - "#class" : mangapark.MangaparkChapterExtractor, - "#pattern" : r"https://[\w-]+\.mpcdn\.org/comic/2002/e67/61e29278a583b9227964076e/\d+_\d+_\d+_\d+\.jpeg\?acc=[^&#]+&exp=\d+", - "#count" : 70, - - "artist" : [], - "author" : ["Amano Kozue"], - "chapter" : 60, - "chapter_id" : 6710214, - "chapter_minor": ".2", - "count" : 70, - "date" : "dt:2022-01-15 09:25:03", - "extension" : "jpeg", - "filename" : str, - "genre" : [ - "adventure", - "comedy", - "drama", - "sci_fi", - "shounen", - "slice_of_life", - ], - "lang" : "en", - "language" : "English", - "manga" : "Aria", - "manga_id" : 114972, - "page" : int, - "source" : "Koala", - "title" : "Special Navigation - Aquaria Ii", - "volume" : 12, -}, - -{ - "#url" : "https://mangapark.com/title/114972-aria/6710214-en-ch.60.2", - "#category": ("", "mangapark", "chapter"), - "#class" : mangapark.MangaparkChapterExtractor, -}, - -{ - "#url" : "https://mangapark.org/title/114972-aria/6710214-en-ch.60.2", - "#category": ("", "mangapark", "chapter"), - "#class" : mangapark.MangaparkChapterExtractor, -}, - -{ - "#url" : "https://mangapark.io/title/114972-aria/6710214-en-ch.60.2", - "#category": ("", "mangapark", "chapter"), - "#class" : mangapark.MangaparkChapterExtractor, -}, - -{ - "#url" : "https://mangapark.me/title/114972-aria/6710214-en-ch.60.2", - "#category": ("", "mangapark", "chapter"), - "#class" : mangapark.MangaparkChapterExtractor, -}, - -{ - "#url" : "https://mangapark.net/title/114972-aria", - "#comment" : "'source' option", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, - "#pattern" : mangapark.MangaparkChapterExtractor.pattern, - "#count" : 141, - - "chapter" : int, - "chapter_id" : int, - "chapter_minor": str, - "date" : datetime.datetime, - "lang" : "en", - "language" : "English", - "manga_id" : 114972, - "source" : r"re:Horse|Koala", - "source_id" : int, - "title" : str, - "volume" : int, -}, - -{ - "#url" : "https://mangapark.net/title/114972-aria", - "#comment" : "'source' option", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, - "#options" : {"source": "koala"}, - "#pattern" : mangapark.MangaparkChapterExtractor.pattern, - "#count" : 70, - - "source" : "Koala", - "source_id": 15150116, -}, - -{ - "#url" : "https://mangapark.com/title/114972-", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, -}, - -{ - "#url" : "https://mangapark.com/title/114972", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, -}, - -{ - "#url" : "https://mangapark.com/title/114972-aria", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, -}, - -{ - "#url" : "https://mangapark.org/title/114972-aria", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, -}, - -{ - "#url" : "https://mangapark.io/title/114972-aria", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, -}, - -{ - "#url" : "https://mangapark.me/title/114972-aria", - "#category": ("", "mangapark", "manga"), - "#class" : mangapark.MangaparkMangaExtractor, -}, - + { + "#url": "https://mangapark.net/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class": mangapark.MangaparkChapterExtractor, + "#pattern": r"https://[\w-]+\.mpcdn\.org/comic/2002/e67/61e29278a583b9227964076e/\d+_\d+_\d+_\d+\.jpeg\?acc=[^&#]+&exp=\d+", + "#count": 70, + "artist": [], + "author": ["Amano Kozue"], + "chapter": 60, + "chapter_id": 6710214, + "chapter_minor": ".2", + "count": 70, + "date": "dt:2022-01-15 09:25:03", + "extension": "jpeg", + "filename": str, + "genre": [ + "adventure", + "comedy", + "drama", + "sci_fi", + "shounen", + "slice_of_life", + ], + "lang": "en", + "language": "English", + "manga": "Aria", + "manga_id": 114972, + "page": int, + "source": "Koala", + "title": "Special Navigation - Aquaria Ii", + "volume": 12, + }, + { + "#url": "https://mangapark.com/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class": mangapark.MangaparkChapterExtractor, + }, + { + "#url": "https://mangapark.org/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class": mangapark.MangaparkChapterExtractor, + }, + { + "#url": "https://mangapark.io/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class": mangapark.MangaparkChapterExtractor, + }, + { + "#url": "https://mangapark.me/title/114972-aria/6710214-en-ch.60.2", + "#category": ("", "mangapark", "chapter"), + "#class": mangapark.MangaparkChapterExtractor, + }, + { + "#url": "https://mangapark.net/title/114972-aria", + "#comment": "'source' option", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + "#pattern": mangapark.MangaparkChapterExtractor.pattern, + "#count": 141, + "chapter": int, + "chapter_id": int, + "chapter_minor": str, + "date": datetime.datetime, + "lang": "en", + "language": "English", + "manga_id": 114972, + "source": r"re:Horse|Koala", + "source_id": int, + "title": str, + "volume": int, + }, + { + "#url": "https://mangapark.net/title/114972-aria", + "#comment": "'source' option", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + "#options": {"source": "koala"}, + "#pattern": mangapark.MangaparkChapterExtractor.pattern, + "#count": 70, + "source": "Koala", + "source_id": 15150116, + }, + { + "#url": "https://mangapark.com/title/114972-", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + }, + { + "#url": "https://mangapark.com/title/114972", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + }, + { + "#url": "https://mangapark.com/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + }, + { + "#url": "https://mangapark.org/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + }, + { + "#url": "https://mangapark.io/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + }, + { + "#url": "https://mangapark.me/title/114972-aria", + "#category": ("", "mangapark", "manga"), + "#class": mangapark.MangaparkMangaExtractor, + }, ) diff --git a/test/results/mangaread.py b/test/results/mangaread.py index 4330a13d36..f458049403 100644 --- a/test/results/mangaread.py +++ b/test/results/mangaread.py @@ -1,122 +1,107 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangaread from gallery_dl import exception - +from gallery_dl.extractor import mangaread __tests__ = ( -{ - "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1053-3/", - "#category": ("", "mangaread", "chapter"), - "#class" : mangaread.MangareadChapterExtractor, - "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", - "#count" : 11, - - "manga" : "One Piece", - "title" : "", - "chapter" : 1053, - "chapter_minor": ".3", - "tags" : ["Oda Eiichiro"], - "lang" : "en", - "language" : "English", -}, - -{ - "#url" : "https://www.mangaread.org/manga/one-piece/chapter-1000000/", - "#category": ("", "mangaread", "chapter"), - "#class" : mangaread.MangareadChapterExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi/chapter-10/", - "#category": ("", "mangaread", "chapter"), - "#class" : mangaread.MangareadChapterExtractor, - "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", - "#count" : 9, - - "manga" : "Kanan-sama wa Akumade Choroi", - "title" : "", - "chapter" : 10, - "chapter_minor": "", - "tags" : list, - "lang" : "en", - "language" : "English", -}, - -{ - "#url" : "https://www.mangaread.org/manga/above-all-gods/chapter146-5/", - "#comment" : "^^ no whitespace", - "#category": ("", "mangaread", "chapter"), - "#class" : mangaread.MangareadChapterExtractor, - "#pattern" : r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", - "#count" : 6, - - "manga" : "Above All Gods", - "title" : "", - "chapter" : 146, - "chapter_minor": ".5", - "tags" : list, - "lang" : "en", - "language" : "English", -}, - -{ - "#url" : "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi", - "#category": ("", "mangaread", "manga"), - "#class" : mangaread.MangareadMangaExtractor, - "#pattern" : r"https://www\.mangaread\.org/manga/kanan-sama-wa-akumade-choroi/chapter-\d+([_-].+)?/", - "#count" : ">= 13", - - "manga" : "Kanan-sama wa Akumade Choroi", - "author" : ["nonco"], - "artist" : ["nonco"], - "type" : "Manga", - "genres" : [ - "Comedy", - "Romance", - "Shounen", - "Supernatural", - ], - "rating" : float, - "release" : 2022, - "status" : "OnGoing", - "lang" : "en", - "language" : "English", - "manga_alt" : list, - "description": str, -}, - -{ - "#url" : "https://www.mangaread.org/manga/one-piece", - "#category": ("", "mangaread", "manga"), - "#class" : mangaread.MangareadMangaExtractor, - "#pattern" : r"https://www\.mangaread\.org/manga/one-piece/chapter-\d+(-.+)?/", - "#count" : ">= 1066", - - "manga" : "One Piece", - "author" : ["Oda Eiichiro"], - "artist" : ["Oda Eiichiro"], - "type" : "Manga", - "genres" : list, - "rating" : float, - "release" : 1997, - "status" : "OnGoing", - "lang" : "en", - "language" : "English", - "manga_alt" : ["One Piece"], - "description": str, -}, - -{ - "#url" : "https://www.mangaread.org/manga/doesnotexist", - "#category": ("", "mangaread", "manga"), - "#class" : mangaread.MangareadMangaExtractor, - "#exception": exception.HttpError, -}, - + { + "#url": "https://www.mangaread.org/manga/one-piece/chapter-1053-3/", + "#category": ("", "mangaread", "chapter"), + "#class": mangaread.MangareadChapterExtractor, + "#pattern": r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count": 11, + "manga": "One Piece", + "title": "", + "chapter": 1053, + "chapter_minor": ".3", + "tags": ["Oda Eiichiro"], + "lang": "en", + "language": "English", + }, + { + "#url": "https://www.mangaread.org/manga/one-piece/chapter-1000000/", + "#category": ("", "mangaread", "chapter"), + "#class": mangaread.MangareadChapterExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi/chapter-10/", + "#category": ("", "mangaread", "chapter"), + "#class": mangaread.MangareadChapterExtractor, + "#pattern": r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count": 9, + "manga": "Kanan-sama wa Akumade Choroi", + "title": "", + "chapter": 10, + "chapter_minor": "", + "tags": list, + "lang": "en", + "language": "English", + }, + { + "#url": "https://www.mangaread.org/manga/above-all-gods/chapter146-5/", + "#comment": "^^ no whitespace", + "#category": ("", "mangaread", "chapter"), + "#class": mangaread.MangareadChapterExtractor, + "#pattern": r"https://www\.mangaread\.org/wp-content/uploads/WP-manga/data/manga_[^/]+/[^/]+/[^.]+\.\w+", + "#count": 6, + "manga": "Above All Gods", + "title": "", + "chapter": 146, + "chapter_minor": ".5", + "tags": list, + "lang": "en", + "language": "English", + }, + { + "#url": "https://www.mangaread.org/manga/kanan-sama-wa-akumade-choroi", + "#category": ("", "mangaread", "manga"), + "#class": mangaread.MangareadMangaExtractor, + "#pattern": r"https://www\.mangaread\.org/manga/kanan-sama-wa-akumade-choroi/chapter-\d+([_-].+)?/", + "#count": ">= 13", + "manga": "Kanan-sama wa Akumade Choroi", + "author": ["nonco"], + "artist": ["nonco"], + "type": "Manga", + "genres": [ + "Comedy", + "Romance", + "Shounen", + "Supernatural", + ], + "rating": float, + "release": 2022, + "status": "OnGoing", + "lang": "en", + "language": "English", + "manga_alt": list, + "description": str, + }, + { + "#url": "https://www.mangaread.org/manga/one-piece", + "#category": ("", "mangaread", "manga"), + "#class": mangaread.MangareadMangaExtractor, + "#pattern": r"https://www\.mangaread\.org/manga/one-piece/chapter-\d+(-.+)?/", + "#count": ">= 1066", + "manga": "One Piece", + "author": ["Oda Eiichiro"], + "artist": ["Oda Eiichiro"], + "type": "Manga", + "genres": list, + "rating": float, + "release": 1997, + "status": "OnGoing", + "lang": "en", + "language": "English", + "manga_alt": ["One Piece"], + "description": str, + }, + { + "#url": "https://www.mangaread.org/manga/doesnotexist", + "#category": ("", "mangaread", "manga"), + "#class": mangaread.MangareadMangaExtractor, + "#exception": exception.HttpError, + }, ) diff --git a/test/results/mangasee.py b/test/results/mangasee.py index 8cf51d4e4a..0f226b1057 100644 --- a/test/results/mangasee.py +++ b/test/results/mangasee.py @@ -1,69 +1,63 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import mangasee import datetime +from gallery_dl.extractor import mangasee __tests__ = ( -{ - "#url" : "https://mangasee123.com/read-online/Tokyo-Innocent-chapter-4.5-page-1.html", - "#category": ("", "mangasee", "chapter"), - "#class" : mangasee.MangaseeChapterExtractor, - "#pattern" : r"https://[^/]+/manga/Tokyo-Innocent/0004\.5-00\d\.png", - "#count" : 8, - - "author" : ["NARUMI Naru"], - "chapter" : 4, - "chapter_minor" : ".5", - "chapter_string": "100045", - "count" : 8, - "date" : "dt:2020-01-20 21:52:53", - "extension" : "png", - "filename" : r"re:0004\.5-00\d", - "genre" : [ - "Comedy", - "Fantasy", - "Harem", - "Romance", - "Shounen", - "Supernatural", - ], - "index" : "1", - "lang" : "en", - "language" : "English", - "manga" : "Tokyo Innocent", - "page" : int, - "title" : "", -}, - -{ - "#url" : "https://mangasee123.com/manga/Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", - "#category": ("", "mangasee", "manga"), - "#class" : mangasee.MangaseeMangaExtractor, - "#pattern" : mangasee.MangaseeChapterExtractor.pattern, - "#count" : ">= 17", - - "author" : ["TAKASE Masaya"], - "chapter" : int, - "chapter_minor" : r"re:^|\.5$", - "chapter_string": r"re:100\d\d\d", - "date" : datetime.datetime, - "genre" : [ - "Comedy", - "Romance", - "School Life", - "Shounen", - "Slice of Life", - ], - "index" : "1", - "lang" : "en", - "language" : "English", - "manga" : "Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", - "title" : "", -}, - + { + "#url": "https://mangasee123.com/read-online/Tokyo-Innocent-chapter-4.5-page-1.html", + "#category": ("", "mangasee", "chapter"), + "#class": mangasee.MangaseeChapterExtractor, + "#pattern": r"https://[^/]+/manga/Tokyo-Innocent/0004\.5-00\d\.png", + "#count": 8, + "author": ["NARUMI Naru"], + "chapter": 4, + "chapter_minor": ".5", + "chapter_string": "100045", + "count": 8, + "date": "dt:2020-01-20 21:52:53", + "extension": "png", + "filename": r"re:0004\.5-00\d", + "genre": [ + "Comedy", + "Fantasy", + "Harem", + "Romance", + "Shounen", + "Supernatural", + ], + "index": "1", + "lang": "en", + "language": "English", + "manga": "Tokyo Innocent", + "page": int, + "title": "", + }, + { + "#url": "https://mangasee123.com/manga/Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", + "#category": ("", "mangasee", "manga"), + "#class": mangasee.MangaseeMangaExtractor, + "#pattern": mangasee.MangaseeChapterExtractor.pattern, + "#count": ">= 17", + "author": ["TAKASE Masaya"], + "chapter": int, + "chapter_minor": r"re:^|\.5$", + "chapter_string": r"re:100\d\d\d", + "date": datetime.datetime, + "genre": [ + "Comedy", + "Romance", + "School Life", + "Shounen", + "Slice of Life", + ], + "index": "1", + "lang": "en", + "language": "English", + "manga": "Nakamura-Koedo-To-Daizu-Keisuke-Wa-Umaku-Ikanai", + "title": "", + }, ) diff --git a/test/results/mangoxo.py b/test/results/mangoxo.py index b7c392a4ab..a88fb147c9 100644 --- a/test/results/mangoxo.py +++ b/test/results/mangoxo.py @@ -1,42 +1,36 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mangoxo - __tests__ = ( -{ - "#url" : "https://www.mangoxo.com/album/lzVOv1Q9", - "#category": ("", "mangoxo", "album"), - "#class" : mangoxo.MangoxoAlbumExtractor, - "#sha1_url": "ad921fe62663b06e7d73997f7d00646cab7bdd0d", - - "channel": { - "id" : "gaxO16d8", - "name" : "Phoenix", - "cover": str, + { + "#url": "https://www.mangoxo.com/album/lzVOv1Q9", + "#category": ("", "mangoxo", "album"), + "#class": mangoxo.MangoxoAlbumExtractor, + "#sha1_url": "ad921fe62663b06e7d73997f7d00646cab7bdd0d", + "channel": { + "id": "gaxO16d8", + "name": "Phoenix", + "cover": str, + }, + "album": { + "id": "lzVOv1Q9", + "name": r"re:池永康晟 Ikenaga Yasunari 透出古朴", + "date": "dt:2019-03-22 14:42:00", + "description": str, + }, + "id": int, + "num": int, + "count": 65, }, - "album" : { - "id" : "lzVOv1Q9", - "name" : r"re:池永康晟 Ikenaga Yasunari 透出古朴", - "date" : "dt:2019-03-22 14:42:00", - "description": str, + { + "#url": "https://www.mangoxo.com/phoenix/album", + "#category": ("", "mangoxo", "channel"), + "#class": mangoxo.MangoxoChannelExtractor, + "#pattern": mangoxo.MangoxoAlbumExtractor.pattern, + "#range": "1-30", + "#count": "> 20", }, - "id" : int, - "num" : int, - "count" : 65, -}, - -{ - "#url" : "https://www.mangoxo.com/phoenix/album", - "#category": ("", "mangoxo", "channel"), - "#class" : mangoxo.MangoxoChannelExtractor, - "#pattern" : mangoxo.MangoxoAlbumExtractor.pattern, - "#range" : "1-30", - "#count" : "> 20", -}, - ) diff --git a/test/results/mariowiki.py b/test/results/mariowiki.py index 72c4cd523c..15bc33b90f 100644 --- a/test/results/mariowiki.py +++ b/test/results/mariowiki.py @@ -1,19 +1,15 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.mariowiki.com/Rabbit", - "#category": ("wikimedia", "mariowiki", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#pattern" : r"https://mario\.wiki\.gallery/images/.+", - "#count" : range(20, 50), -}, - + { + "#url": "https://www.mariowiki.com/Rabbit", + "#category": ("wikimedia", "mariowiki", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#pattern": r"https://mario\.wiki\.gallery/images/.+", + "#count": range(20, 50), + }, ) diff --git a/test/results/mastodon.py b/test/results/mastodon.py index b6cb34641e..ad160c6063 100644 --- a/test/results/mastodon.py +++ b/test/results/mastodon.py @@ -1,32 +1,25 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mastodon - __tests__ = ( -{ - "#url" : "mastodon:https://donotsta.re/@elly/AcoUaA7EH1igiYKmFU", - "#category": ("mastodon", "donotsta.re", "status"), - "#class" : mastodon.MastodonStatusExtractor, - "#urls" : "https://asdf.donotsta.re/media/917e7722dd30d510686ce9f3717a1f722dac96fd974b5af5ec2ccbc8cbd740c6.png", - - "instance": "donotsta.re", - "instance_remote": None, -}, - -{ - "#url" : "mastodon:https://wanderingwires.net/@quarc/9qppkxzyd1ee3i9p", - "#comment" : "null moved account", - "#category": ("mastodon", "wanderingwires.net", "status"), - "#class" : mastodon.MastodonStatusExtractor, - "#urls" : "https://s3.wanderingwires.net/null/4377e826-72ab-4659-885c-fa12945eb207.png", - - "instance": "wanderingwires.net", - "instance_remote": None, -}, - + { + "#url": "mastodon:https://donotsta.re/@elly/AcoUaA7EH1igiYKmFU", + "#category": ("mastodon", "donotsta.re", "status"), + "#class": mastodon.MastodonStatusExtractor, + "#urls": "https://asdf.donotsta.re/media/917e7722dd30d510686ce9f3717a1f722dac96fd974b5af5ec2ccbc8cbd740c6.png", + "instance": "donotsta.re", + "instance_remote": None, + }, + { + "#url": "mastodon:https://wanderingwires.net/@quarc/9qppkxzyd1ee3i9p", + "#comment": "null moved account", + "#category": ("mastodon", "wanderingwires.net", "status"), + "#class": mastodon.MastodonStatusExtractor, + "#urls": "https://s3.wanderingwires.net/null/4377e826-72ab-4659-885c-fa12945eb207.png", + "instance": "wanderingwires.net", + "instance_remote": None, + }, ) diff --git a/test/results/mastodonsocial.py b/test/results/mastodonsocial.py index 1eed8a2469..da12a75105 100644 --- a/test/results/mastodonsocial.py +++ b/test/results/mastodonsocial.py @@ -1,199 +1,174 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mastodon - __tests__ = ( -{ - "#url" : "https://mastodon.social/@jk", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, - "#pattern" : r"https://files.mastodon.social/media_attachments/files/(\d+/){3,}original/\w+", - "#range" : "1-60", - "#count" : 60, -}, - -{ - "#url" : "https://mastodon.social/@ponapalt@ukadon.shillest.net", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, - "#pattern" : r"https://files\.mastodon\.social/cache/media_attachments/files/.+/original/\w{16}\.\w+$", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://mastodon.social/@gallerydl", - "#comment" : "reblogged/'boosted' posts (#4580)", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, - "#options" : {"reblogs": True}, - "#archive" : False, - "#urls": ( - "https://files.mastodon.social/media_attachments/files/111/330/852/486/713/967/original/2c25ade55a9d1af2.jpg", - "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", - "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", - ), -}, - -{ - "#url" : "https://mastodon.social/@id:10843", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, -}, - -{ - "#url" : "https://mastodon.social/users/id:10843", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, -}, - -{ - "#url" : "https://mastodon.social/users/jk", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, -}, - -{ - "#url" : "https://mastodon.social/users/yoru_nine@pawoo.net", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, -}, - -{ - "#url" : "https://mastodon.social/web/@jk", - "#category": ("mastodon", "mastodon.social", "user"), - "#class" : mastodon.MastodonUserExtractor, -}, - -{ - "#url" : "https://mastodon.social/bookmarks", - "#category": ("mastodon", "mastodon.social", "bookmark"), - "#class" : mastodon.MastodonBookmarkExtractor, - "#auth" : True, - "#urls" : "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", -}, - -{ - "#url" : "https://mastodon.social/favourites", - "#category": ("mastodon", "mastodon.social", "favorite"), - "#class" : mastodon.MastodonFavoriteExtractor, - "#auth" : True, - "#urls" : "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", -}, - -{ - "#url" : "https://mastodon.social/lists/92653", - "#category": ("mastodon", "mastodon.social", "list"), - "#class" : mastodon.MastodonListExtractor, - "#auth" : True, - "#pattern" : r"https://files\.mastodon\.social/media_attachments/files/(\d+/){3,}original/\w+", - "#range" : "1-10", -}, - -{ - "#url" : "https://mastodon.social/tags/mastodon", - "#category": ("mastodon", "mastodon.social", "hashtag"), - "#class" : mastodon.MastodonHashtagExtractor, - "#pattern" : r"https://files\.mastodon\.social/media_attachments/files/(\d+/){3,}original/\w+", - "#range" : "1-10", -}, - -{ - "#url" : "https://mastodon.social/@gallerydl/following", - "#category": ("mastodon", "mastodon.social", "following"), - "#class" : mastodon.MastodonFollowingExtractor, - "#extractor": False, - "#urls" : ( - "https://mastodon.ie/@RustyBertrand", - "https://ravenation.club/@soundwarrior20", - "https://mastodon.social/@0x4f", - "https://mastodon.social/@christianselig", - "https://saturation.social/@clive", - "https://mastodon.social/@sjvn", - ), - - "acct" : str, - "avatar" : r"re:https://files.mastodon.social/.+\.\w+$", - "avatar_static" : r"re:https://files.mastodon.social/.+\.\w+$", - "bot" : False, - "created_at" : r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z", - "discoverable" : True, - "display_name" : str, - "emojis" : [], - "fields" : list, - "followers_count": int, - "following_count": int, - "group" : False, - "header" : str, - "header_static" : str, - "id" : r"re:\d+", - "last_status_at": r"re:\d{4}-\d{2}-\d{2}", - "locked" : bool, - "note" : str, - "statuses_count": int, - "uri" : str, - "url" : str, - "username" : str, - -}, - -{ - "#url" : "https://mastodon.social/@0x4f/following", - "#category": ("mastodon", "mastodon.social", "following"), - "#class" : mastodon.MastodonFollowingExtractor, -}, - -{ - "#url" : "https://mastodon.social/users/id:10843/following", - "#category": ("mastodon", "mastodon.social", "following"), - "#class" : mastodon.MastodonFollowingExtractor, -}, - -{ - "#url" : "https://mastodon.social/@jk/103794036899778366", - "#category": ("mastodon", "mastodon.social", "status"), - "#class" : mastodon.MastodonStatusExtractor, - "#count" : 4, - - "count": 4, - "num" : int, -}, - -{ - "#url" : "https://mastodon.social/@technewsbot@assortedflotsam.com/112360601113258881", - "#comment" : "card image", - "#category": ("mastodon", "mastodon.social", "status"), - "#class" : mastodon.MastodonStatusExtractor, - "#options" : {"cards": True}, - "#urls" : "https://files.mastodon.social/cache/preview_cards/images/095/900/335/original/83f0b4a793c84123.jpg", - - "media": { - "author_name" : "Tom Warren", - "author_url" : "https://www.theverge.com/authors/tom-warren", - "blurhash" : "UHBDWMCjVGM0k,XjnPM#0h+vkpb^RkjYSh$*", - "description" : "Microsoft’s big Xbox games showcase will take place on June 9th. It will include more games than last year and a special Call of Duty Direct will follow.", - "embed_url" : "", - "height" : 628, - "html" : "", - "id" : "card95900335", - "image" : "https://files.mastodon.social/cache/preview_cards/images/095/900/335/original/83f0b4a793c84123.jpg", - "image_description": "The Xbox showcase illustration", - "language" : "en", - "provider_name": "The Verge", - "provider_url": "", - "published_at": "2024-04-30T14:15:30.341Z", - "title" : "The Xbox games showcase airs June 9th, followed by a Call of Duty Direct", - "type" : "link", - "url" : "https://files.mastodon.social/cache/preview_cards/images/095/900/335/original/83f0b4a793c84123.jpg", - "weburl" : "https://www.theverge.com/2024/4/30/24145262/xbox-games-showcase-summer-2024-call-of-duty-direct", - "width" : 1200, + { + "#url": "https://mastodon.social/@jk", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + "#pattern": r"https://files.mastodon.social/media_attachments/files/(\d+/){3,}original/\w+", + "#range": "1-60", + "#count": 60, + }, + { + "#url": "https://mastodon.social/@ponapalt@ukadon.shillest.net", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + "#pattern": r"https://files\.mastodon\.social/cache/media_attachments/files/.+/original/\w{16}\.\w+$", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://mastodon.social/@gallerydl", + "#comment": "reblogged/'boosted' posts (#4580)", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + "#options": {"reblogs": True}, + "#archive": False, + "#urls": ( + "https://files.mastodon.social/media_attachments/files/111/330/852/486/713/967/original/2c25ade55a9d1af2.jpg", + "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", + "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", + ), + }, + { + "#url": "https://mastodon.social/@id:10843", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + }, + { + "#url": "https://mastodon.social/users/id:10843", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + }, + { + "#url": "https://mastodon.social/users/jk", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + }, + { + "#url": "https://mastodon.social/users/yoru_nine@pawoo.net", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + }, + { + "#url": "https://mastodon.social/web/@jk", + "#category": ("mastodon", "mastodon.social", "user"), + "#class": mastodon.MastodonUserExtractor, + }, + { + "#url": "https://mastodon.social/bookmarks", + "#category": ("mastodon", "mastodon.social", "bookmark"), + "#class": mastodon.MastodonBookmarkExtractor, + "#auth": True, + "#urls": "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", + }, + { + "#url": "https://mastodon.social/favourites", + "#category": ("mastodon", "mastodon.social", "favorite"), + "#class": mastodon.MastodonFavoriteExtractor, + "#auth": True, + "#urls": "https://files.mastodon.social/media_attachments/files/111/331/603/082/304/823/original/e12cde371c88c1b0.png", + }, + { + "#url": "https://mastodon.social/lists/92653", + "#category": ("mastodon", "mastodon.social", "list"), + "#class": mastodon.MastodonListExtractor, + "#auth": True, + "#pattern": r"https://files\.mastodon\.social/media_attachments/files/(\d+/){3,}original/\w+", + "#range": "1-10", + }, + { + "#url": "https://mastodon.social/tags/mastodon", + "#category": ("mastodon", "mastodon.social", "hashtag"), + "#class": mastodon.MastodonHashtagExtractor, + "#pattern": r"https://files\.mastodon\.social/media_attachments/files/(\d+/){3,}original/\w+", + "#range": "1-10", + }, + { + "#url": "https://mastodon.social/@gallerydl/following", + "#category": ("mastodon", "mastodon.social", "following"), + "#class": mastodon.MastodonFollowingExtractor, + "#extractor": False, + "#urls": ( + "https://mastodon.ie/@RustyBertrand", + "https://ravenation.club/@soundwarrior20", + "https://mastodon.social/@0x4f", + "https://mastodon.social/@christianselig", + "https://saturation.social/@clive", + "https://mastodon.social/@sjvn", + ), + "acct": str, + "avatar": r"re:https://files.mastodon.social/.+\.\w+$", + "avatar_static": r"re:https://files.mastodon.social/.+\.\w+$", + "bot": False, + "created_at": r"re:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z", + "discoverable": True, + "display_name": str, + "emojis": [], + "fields": list, + "followers_count": int, + "following_count": int, + "group": False, + "header": str, + "header_static": str, + "id": r"re:\d+", + "last_status_at": r"re:\d{4}-\d{2}-\d{2}", + "locked": bool, + "note": str, + "statuses_count": int, + "uri": str, + "url": str, + "username": str, + }, + { + "#url": "https://mastodon.social/@0x4f/following", + "#category": ("mastodon", "mastodon.social", "following"), + "#class": mastodon.MastodonFollowingExtractor, + }, + { + "#url": "https://mastodon.social/users/id:10843/following", + "#category": ("mastodon", "mastodon.social", "following"), + "#class": mastodon.MastodonFollowingExtractor, + }, + { + "#url": "https://mastodon.social/@jk/103794036899778366", + "#category": ("mastodon", "mastodon.social", "status"), + "#class": mastodon.MastodonStatusExtractor, + "#count": 4, + "count": 4, + "num": int, + }, + { + "#url": "https://mastodon.social/@technewsbot@assortedflotsam.com/112360601113258881", + "#comment": "card image", + "#category": ("mastodon", "mastodon.social", "status"), + "#class": mastodon.MastodonStatusExtractor, + "#options": {"cards": True}, + "#urls": "https://files.mastodon.social/cache/preview_cards/images/095/900/335/original/83f0b4a793c84123.jpg", + "media": { + "author_name": "Tom Warren", + "author_url": "https://www.theverge.com/authors/tom-warren", + "blurhash": "UHBDWMCjVGM0k,XjnPM#0h+vkpb^RkjYSh$*", + "description": "Microsoft’s big Xbox games showcase will take place on June 9th. It will include more games than last year and a special Call of Duty Direct will follow.", + "embed_url": "", + "height": 628, + "html": "", + "id": "card95900335", + "image": "https://files.mastodon.social/cache/preview_cards/images/095/900/335/original/83f0b4a793c84123.jpg", + "image_description": "The Xbox showcase illustration", + "language": "en", + "provider_name": "The Verge", + "provider_url": "", + "published_at": "2024-04-30T14:15:30.341Z", + "title": "The Xbox games showcase airs June 9th, followed by a Call of Duty Direct", + "type": "link", + "url": "https://files.mastodon.social/cache/preview_cards/images/095/900/335/original/83f0b4a793c84123.jpg", + "weburl": "https://www.theverge.com/2024/4/30/24145262/xbox-games-showcase-summer-2024-call-of-duty-direct", + "width": 1200, + }, }, - -}, - ) diff --git a/test/results/mediawiki.py b/test/results/mediawiki.py index 683d0d36b2..fd01685ec5 100644 --- a/test/results/mediawiki.py +++ b/test/results/mediawiki.py @@ -1,24 +1,20 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.mediawiki.org/wiki/Help:Navigation", - "#category": ("wikimedia", "mediawiki", "help"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#urls" : ( - "https://upload.wikimedia.org/wikipedia/commons/e/ec/OOjs_UI_icon_information-progressive.svg", - "https://upload.wikimedia.org/wikipedia/commons/6/62/PD-icon.svg", - "https://upload.wikimedia.org/wikipedia/commons/0/0e/Vector_Sidebar.png", - "https://upload.wikimedia.org/wikipedia/commons/7/77/Vector_page_tabs.png", - "https://upload.wikimedia.org/wikipedia/commons/6/6e/Vector_user_links.png", - ), -}, - + { + "#url": "https://www.mediawiki.org/wiki/Help:Navigation", + "#category": ("wikimedia", "mediawiki", "help"), + "#class": wikimedia.WikimediaArticleExtractor, + "#urls": ( + "https://upload.wikimedia.org/wikipedia/commons/e/ec/OOjs_UI_icon_information-progressive.svg", + "https://upload.wikimedia.org/wikipedia/commons/6/62/PD-icon.svg", + "https://upload.wikimedia.org/wikipedia/commons/0/0e/Vector_Sidebar.png", + "https://upload.wikimedia.org/wikipedia/commons/7/77/Vector_page_tabs.png", + "https://upload.wikimedia.org/wikipedia/commons/6/6e/Vector_user_links.png", + ), + }, ) diff --git a/test/results/michaelscameras.py b/test/results/michaelscameras.py index 42c76bbc60..9c9fecd703 100644 --- a/test/results/michaelscameras.py +++ b/test/results/michaelscameras.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://michaels.com.au/collections/microphones", - "#category": ("shopify", "michaelscameras", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://michaels.com.au/collections/audio/products/boya-by-wm4-pro-k5-2-4ghz-mic-android-1-1-101281", - "#category": ("shopify", "michaelscameras", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://michaels.com.au/collections/microphones", + "#category": ("shopify", "michaelscameras", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://michaels.com.au/collections/audio/products/boya-by-wm4-pro-k5-2-4ghz-mic-android-1-1-101281", + "#category": ("shopify", "michaelscameras", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/misskeydesign.py b/test/results/misskeydesign.py index 3560597bd4..9b54984edd 100644 --- a/test/results/misskeydesign.py +++ b/test/results/misskeydesign.py @@ -1,53 +1,44 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import misskey - __tests__ = ( -{ - "#url" : "https://misskey.design/@machina_3D", - "#category": ("misskey", "misskey.design", "user"), - "#class" : misskey.MisskeyUserExtractor, - "#pattern" : r"https://file\.misskey\.design/post/[\w-]{36}\.\w+", - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://misskey.design/@blooddj@pawoo.net", - "#category": ("misskey", "misskey.design", "user"), - "#class" : misskey.MisskeyUserExtractor, - "#count" : "> 30", -}, - -{ - "#url" : "https://misskey.design/@kujyo_t/following", - "#category": ("misskey", "misskey.design", "following"), - "#class" : misskey.MisskeyFollowingExtractor, - "#count" : ">= 250", -}, - -{ - "#url" : "https://misskey.design/notes/9jva1danjc", - "#category": ("misskey", "misskey.design", "note"), - "#class" : misskey.MisskeyNoteExtractor, - "#urls" : "https://file.misskey.design/post/a8d27901-24e1-42ab-b8a6-1e09c98c6f55.webp", -}, - -{ - "#url" : "https://misskey.design/my/favorites", - "#category": ("misskey", "misskey.design", "favorite"), - "#class" : misskey.MisskeyFavoriteExtractor, -}, - -{ - "#url" : "https://misskey.design/api/i/favorites", - "#category": ("misskey", "misskey.design", "favorite"), - "#class" : misskey.MisskeyFavoriteExtractor, -}, - + { + "#url": "https://misskey.design/@machina_3D", + "#category": ("misskey", "misskey.design", "user"), + "#class": misskey.MisskeyUserExtractor, + "#pattern": r"https://file\.misskey\.design/post/[\w-]{36}\.\w+", + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://misskey.design/@blooddj@pawoo.net", + "#category": ("misskey", "misskey.design", "user"), + "#class": misskey.MisskeyUserExtractor, + "#count": "> 30", + }, + { + "#url": "https://misskey.design/@kujyo_t/following", + "#category": ("misskey", "misskey.design", "following"), + "#class": misskey.MisskeyFollowingExtractor, + "#count": ">= 250", + }, + { + "#url": "https://misskey.design/notes/9jva1danjc", + "#category": ("misskey", "misskey.design", "note"), + "#class": misskey.MisskeyNoteExtractor, + "#urls": "https://file.misskey.design/post/a8d27901-24e1-42ab-b8a6-1e09c98c6f55.webp", + }, + { + "#url": "https://misskey.design/my/favorites", + "#category": ("misskey", "misskey.design", "favorite"), + "#class": misskey.MisskeyFavoriteExtractor, + }, + { + "#url": "https://misskey.design/api/i/favorites", + "#category": ("misskey", "misskey.design", "favorite"), + "#class": misskey.MisskeyFavoriteExtractor, + }, ) diff --git a/test/results/misskeyio.py b/test/results/misskeyio.py index 4de64aa433..006aae18e7 100644 --- a/test/results/misskeyio.py +++ b/test/results/misskeyio.py @@ -1,62 +1,52 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import misskey - __tests__ = ( -{ - "#url" : "https://misskey.io/@lithla", - "#category": ("misskey", "misskey.io", "user"), - "#class" : misskey.MisskeyUserExtractor, - "#pattern" : r"https://(media.misskeyusercontent.com|s\d+\.arkjp\.net)/(misskey|io)/[\w-]+\.\w+", - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://misskey.io/@blooddj@pawoo.net", - "#category": ("misskey", "misskey.io", "user"), - "#class" : misskey.MisskeyUserExtractor, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://misskey.io/@blooddj@pawoo.net/following", - "#category": ("misskey", "misskey.io", "following"), - "#class" : misskey.MisskeyFollowingExtractor, - "#count" : ">= 6", - "#extractor": False, -}, - -{ - "#url" : "https://misskey.io/notes/9bhqfo835v", - "#category": ("misskey", "misskey.io", "note"), - "#class" : misskey.MisskeyNoteExtractor, - "#pattern" : r"https://(media\.misskeyusercontent\.com|s\d+\.arkjp\.net)/misskey/[\w-]+\.\w+", - "#count" : 4, -}, - -{ - "#url" : "https://misskey.io/notes/9brq7z1re6", - "#category": ("misskey", "misskey.io", "note"), - "#class" : misskey.MisskeyNoteExtractor, -}, - -{ - "#url" : "https://misskey.io/my/favorites", - "#category": ("misskey", "misskey.io", "favorite"), - "#class" : misskey.MisskeyFavoriteExtractor, -}, - -{ - "#url" : "https://misskey.io/api/i/favorites", - "#category": ("misskey", "misskey.io", "favorite"), - "#class" : misskey.MisskeyFavoriteExtractor, -}, - + { + "#url": "https://misskey.io/@lithla", + "#category": ("misskey", "misskey.io", "user"), + "#class": misskey.MisskeyUserExtractor, + "#pattern": r"https://(media.misskeyusercontent.com|s\d+\.arkjp\.net)/(misskey|io)/[\w-]+\.\w+", + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://misskey.io/@blooddj@pawoo.net", + "#category": ("misskey", "misskey.io", "user"), + "#class": misskey.MisskeyUserExtractor, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://misskey.io/@blooddj@pawoo.net/following", + "#category": ("misskey", "misskey.io", "following"), + "#class": misskey.MisskeyFollowingExtractor, + "#count": ">= 6", + "#extractor": False, + }, + { + "#url": "https://misskey.io/notes/9bhqfo835v", + "#category": ("misskey", "misskey.io", "note"), + "#class": misskey.MisskeyNoteExtractor, + "#pattern": r"https://(media\.misskeyusercontent\.com|s\d+\.arkjp\.net)/misskey/[\w-]+\.\w+", + "#count": 4, + }, + { + "#url": "https://misskey.io/notes/9brq7z1re6", + "#category": ("misskey", "misskey.io", "note"), + "#class": misskey.MisskeyNoteExtractor, + }, + { + "#url": "https://misskey.io/my/favorites", + "#category": ("misskey", "misskey.io", "favorite"), + "#class": misskey.MisskeyFavoriteExtractor, + }, + { + "#url": "https://misskey.io/api/i/favorites", + "#category": ("misskey", "misskey.io", "favorite"), + "#class": misskey.MisskeyFavoriteExtractor, + }, ) diff --git a/test/results/modcloth.py b/test/results/modcloth.py index 67ad082eff..fcb21b9091 100644 --- a/test/results/modcloth.py +++ b/test/results/modcloth.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://modcloth.com/collections/shoes", - "#category": ("shopify", "modcloth", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://modcloth.com/collections/shoes/products/heidii-brn", - "#category": ("shopify", "modcloth", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://modcloth.com/collections/shoes", + "#category": ("shopify", "modcloth", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://modcloth.com/collections/shoes/products/heidii-brn", + "#category": ("shopify", "modcloth", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/myhentaigallery.py b/test/results/myhentaigallery.py index a90e067a6f..e0e6aa4945 100644 --- a/test/results/myhentaigallery.py +++ b/test/results/myhentaigallery.py @@ -1,38 +1,31 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import myhentaigallery - __tests__ = ( -{ - "#url" : "https://myhentaigallery.com/g/16247", - "#category": ("", "myhentaigallery", "gallery"), - "#class" : myhentaigallery.MyhentaigalleryGalleryExtractor, - "#pattern" : r"https://(cdn|images)\.myhentaicomics\.com/m\w\w/images/[^/]+/original/\d+\.jpg", - - "artist" : list, - "count" : 11, - "gallery_id": 16247, - "group" : list, - "parodies" : list, - "tags" : ["Giantess"], - "title" : "Attack Of The 50ft Woman 1", -}, - -{ - "#url" : "https://myhentaigallery.com/gallery/thumbnails/16247", - "#category": ("", "myhentaigallery", "gallery"), - "#class" : myhentaigallery.MyhentaigalleryGalleryExtractor, -}, - -{ - "#url" : "https://myhentaigallery.com/gallery/show/16247/1", - "#category": ("", "myhentaigallery", "gallery"), - "#class" : myhentaigallery.MyhentaigalleryGalleryExtractor, -}, - + { + "#url": "https://myhentaigallery.com/g/16247", + "#category": ("", "myhentaigallery", "gallery"), + "#class": myhentaigallery.MyhentaigalleryGalleryExtractor, + "#pattern": r"https://(cdn|images)\.myhentaicomics\.com/m\w\w/images/[^/]+/original/\d+\.jpg", + "artist": list, + "count": 11, + "gallery_id": 16247, + "group": list, + "parodies": list, + "tags": ["Giantess"], + "title": "Attack Of The 50ft Woman 1", + }, + { + "#url": "https://myhentaigallery.com/gallery/thumbnails/16247", + "#category": ("", "myhentaigallery", "gallery"), + "#class": myhentaigallery.MyhentaigalleryGalleryExtractor, + }, + { + "#url": "https://myhentaigallery.com/gallery/show/16247/1", + "#category": ("", "myhentaigallery", "gallery"), + "#class": myhentaigallery.MyhentaigalleryGalleryExtractor, + }, ) diff --git a/test/results/myportfolio.py b/test/results/myportfolio.py index 7185129379..fc76e7a695 100644 --- a/test/results/myportfolio.py +++ b/test/results/myportfolio.py @@ -1,51 +1,43 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import myportfolio from gallery_dl import exception - +from gallery_dl.extractor import myportfolio __tests__ = ( -{ - "#url" : "https://andrewling.myportfolio.com/volvo-xc-90-hybrid", - "#category": ("", "myportfolio", "gallery"), - "#class" : myportfolio.MyportfolioGalleryExtractor, - "#sha1_url" : "acea0690c76db0e5cf267648cefd86e921bc3499", - "#sha1_metadata": "6ac6befe2ee0af921d24cf1dd4a4ed71be06db6d", -}, - -{ - "#url" : "https://andrewling.myportfolio.com/", - "#category": ("", "myportfolio", "gallery"), - "#class" : myportfolio.MyportfolioGalleryExtractor, - "#pattern" : r"https://andrewling\.myportfolio\.com/[^/?#+]+$", - "#count" : ">= 6", -}, - -{ - "#url" : "https://stevenilousphotography.myportfolio.com/society", - "#category": ("", "myportfolio", "gallery"), - "#class" : myportfolio.MyportfolioGalleryExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "myportfolio:https://tooco.com.ar/6-of-diamonds-paradise-bird", - "#comment" : "custom domain", - "#category": ("", "myportfolio", "gallery"), - "#class" : myportfolio.MyportfolioGalleryExtractor, - "#count" : 3, -}, - -{ - "#url" : "myportfolio:https://tooco.com.ar/", - "#category": ("", "myportfolio", "gallery"), - "#class" : myportfolio.MyportfolioGalleryExtractor, - "#pattern" : myportfolio.MyportfolioGalleryExtractor.pattern, - "#count" : ">= 40", -}, - + { + "#url": "https://andrewling.myportfolio.com/volvo-xc-90-hybrid", + "#category": ("", "myportfolio", "gallery"), + "#class": myportfolio.MyportfolioGalleryExtractor, + "#sha1_url": "acea0690c76db0e5cf267648cefd86e921bc3499", + "#sha1_metadata": "6ac6befe2ee0af921d24cf1dd4a4ed71be06db6d", + }, + { + "#url": "https://andrewling.myportfolio.com/", + "#category": ("", "myportfolio", "gallery"), + "#class": myportfolio.MyportfolioGalleryExtractor, + "#pattern": r"https://andrewling\.myportfolio\.com/[^/?#+]+$", + "#count": ">= 6", + }, + { + "#url": "https://stevenilousphotography.myportfolio.com/society", + "#category": ("", "myportfolio", "gallery"), + "#class": myportfolio.MyportfolioGalleryExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "myportfolio:https://tooco.com.ar/6-of-diamonds-paradise-bird", + "#comment": "custom domain", + "#category": ("", "myportfolio", "gallery"), + "#class": myportfolio.MyportfolioGalleryExtractor, + "#count": 3, + }, + { + "#url": "myportfolio:https://tooco.com.ar/", + "#category": ("", "myportfolio", "gallery"), + "#class": myportfolio.MyportfolioGalleryExtractor, + "#pattern": myportfolio.MyportfolioGalleryExtractor.pattern, + "#count": ">= 40", + }, ) diff --git a/test/results/naver.py b/test/results/naver.py index a763a5b52b..4d517939c1 100644 --- a/test/results/naver.py +++ b/test/results/naver.py @@ -1,84 +1,73 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import naver - __tests__ = ( -{ - "#url" : "https://blog.naver.com/rlfqjxm0/221430673006", - "#category": ("", "naver", "post"), - "#class" : naver.NaverPostExtractor, - "#sha1_url" : "6c694f3aced075ed5e9511f1e796d14cb26619cc", - "#sha1_metadata": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", -}, - -{ - "#url" : "https://blog.naver.com/PostView.nhn?blogId=rlfqjxm0&logNo=221430673006", - "#category": ("", "naver", "post"), - "#class" : naver.NaverPostExtractor, - "#sha1_url" : "6c694f3aced075ed5e9511f1e796d14cb26619cc", - "#sha1_metadata": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", -}, - -{ - "#url" : "https://blog.naver.com/PostView.nhn?blogId=rlfqjxm0&logNo=70161391809", - "#comment" : "filenames in EUC-KR encoding (#5126)", - "#category": ("", "naver", "post"), - "#class" : naver.NaverPostExtractor, - "#urls": ( - "https://blogfiles.pstatic.net/20130305_23/ping9303_1362411028002Dpz9z_PNG/1_사본.png", - "https://blogfiles.pstatic.net/20130305_46/rlfqjxm0_1362473322580x33zi_PNG/오마갓합작.png", - ), - - "blog": { - "id" : "rlfqjxm0", - "num" : 43030507, - "user": "에나", + { + "#url": "https://blog.naver.com/rlfqjxm0/221430673006", + "#category": ("", "naver", "post"), + "#class": naver.NaverPostExtractor, + "#sha1_url": "6c694f3aced075ed5e9511f1e796d14cb26619cc", + "#sha1_metadata": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", }, - "post": { - "date" : "dt:2013-03-05 17:48:00", - "description": " ◈ PROMOTER ：핑수 ˚ 아담 EDITOR：핑수 넵：이크：핑수...", - "num" : 70161391809, - "title" : "[공유] { 합작}  OH, MY GOD! ~ 아 또 무슨 종말을 한다 그래~", + { + "#url": "https://blog.naver.com/PostView.nhn?blogId=rlfqjxm0&logNo=221430673006", + "#category": ("", "naver", "post"), + "#class": naver.NaverPostExtractor, + "#sha1_url": "6c694f3aced075ed5e9511f1e796d14cb26619cc", + "#sha1_metadata": "a6e23d19afbee86b37d6e7ad934650c379d2cb1e", + }, + { + "#url": "https://blog.naver.com/PostView.nhn?blogId=rlfqjxm0&logNo=70161391809", + "#comment": "filenames in EUC-KR encoding (#5126)", + "#category": ("", "naver", "post"), + "#class": naver.NaverPostExtractor, + "#urls": ( + "https://blogfiles.pstatic.net/20130305_23/ping9303_1362411028002Dpz9z_PNG/1_사본.png", + "https://blogfiles.pstatic.net/20130305_46/rlfqjxm0_1362473322580x33zi_PNG/오마갓합작.png", + ), + "blog": { + "id": "rlfqjxm0", + "num": 43030507, + "user": "에나", + }, + "post": { + "date": "dt:2013-03-05 17:48:00", + "description": " ◈ PROMOTER ：핑수 ˚ 아담 EDITOR：핑수 넵：이크：핑수...", + "num": 70161391809, + "title": "[공유] { 합작}  OH, MY GOD! ~ 아 또 무슨 종말을 한다 그래~", + }, + "count": 2, + "num": range(1, 2), + "filename": r"re:1_사본|오마갓합작", + "extension": "png", + }, + { + "#url": "https://blog.naver.com/PostView.naver?blogId=rlfqjxm0&logNo=221430673006", + "#category": ("", "naver", "post"), + "#class": naver.NaverPostExtractor, + }, + { + "#url": "https://blog.naver.com/gukjung", + "#category": ("", "naver", "blog"), + "#class": naver.NaverBlogExtractor, + "#pattern": naver.NaverPostExtractor.pattern, + "#range": "1-12", + "#count": 12, + }, + { + "#url": "https://blog.naver.com/PostList.nhn?blogId=gukjung", + "#category": ("", "naver", "blog"), + "#class": naver.NaverBlogExtractor, + "#pattern": naver.NaverPostExtractor.pattern, + "#range": "1-12", + "#count": 12, + }, + { + "#url": "https://blog.naver.com/PostList.naver?blogId=gukjung", + "#category": ("", "naver", "blog"), + "#class": naver.NaverBlogExtractor, }, - "count" : 2, - "num" : range(1, 2), - "filename" : r"re:1_사본|오마갓합작", - "extension": "png", -}, - -{ - "#url" : "https://blog.naver.com/PostView.naver?blogId=rlfqjxm0&logNo=221430673006", - "#category": ("", "naver", "post"), - "#class" : naver.NaverPostExtractor, -}, - -{ - "#url" : "https://blog.naver.com/gukjung", - "#category": ("", "naver", "blog"), - "#class" : naver.NaverBlogExtractor, - "#pattern" : naver.NaverPostExtractor.pattern, - "#range" : "1-12", - "#count" : 12, -}, - -{ - "#url" : "https://blog.naver.com/PostList.nhn?blogId=gukjung", - "#category": ("", "naver", "blog"), - "#class" : naver.NaverBlogExtractor, - "#pattern" : naver.NaverPostExtractor.pattern, - "#range" : "1-12", - "#count" : 12, -}, - -{ - "#url" : "https://blog.naver.com/PostList.naver?blogId=gukjung", - "#category": ("", "naver", "blog"), - "#class" : naver.NaverBlogExtractor, -}, - ) diff --git a/test/results/naverwebtoon.py b/test/results/naverwebtoon.py index 36b6708630..fdff61560f 100644 --- a/test/results/naverwebtoon.py +++ b/test/results/naverwebtoon.py @@ -1,123 +1,109 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import naverwebtoon - __tests__ = ( -{ - "#url" : "https://comic.naver.com/webtoon/detail?titleId=26458&no=1&weekday=tue", - "#category": ("", "naverwebtoon", "episode"), - "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, - "#count" : 14, - "#sha1_url" : "47a956ba8c7a837213d5985f50c569fcff986f75", - "#sha1_content": "3806b6e8befbb1920048de9888dfce6220f69a60", - - "author" : ["김규삼"], - "artist" : ["김규삼"], - "comic" : "N의등대-눈의등대", - "count" : 14, - "episode" : "1", - "extension": "jpg", - "num" : int, - "tags" : [ - "스릴러", - "완결무료", - "완결스릴러", - ], - "title" : "n의 등대 - 눈의 등대 1화", - "title_id" : "26458", -}, - -{ - "#url" : "https://comic.naver.com/challenge/detail?titleId=765124&no=1", - "#category": ("", "naverwebtoon", "episode"), - "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, - "#pattern" : r"https://image-comic\.pstatic\.net/user_contents_data/challenge_comic/2021/01/19/342586/upload_7149856273586337846\.jpeg", - "#count" : 1, - - "author" : ["kemi****"], - "artist" : [], - "comic" : "우니 모두의 이야기", - "count" : 1, - "episode" : "1", - "extension": "jpeg", - "filename" : "upload_7149856273586337846", - "num" : 1, - "tags" : [ - "일상툰", - "우니모두의이야기", - "퇴사", - "입사", - "신입사원", - "사회초년생", - "회사원", - "20대", - ], - "title" : "퇴사하다", - "title_id" : "765124", -}, - -{ - "#url" : "https://comic.naver.com/bestChallenge/detail?titleId=620732&no=334", - "#comment" : "empty tags (#5120)", - "#category": ("", "naverwebtoon", "episode"), - "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, - "#count" : 9, - - "artist" : [], - "author" : ["안트로anthrokim"], - "comic" : "백일몽화원", - "count" : 9, - "episode" : "334", - "num" : range(1, 9), - "tags" : [], - "title" : "321화... 성(省)", - "title_id": "620732", -}, - -{ - "#url" : "https://comic.naver.com/bestChallenge/detail.nhn?titleId=771467&no=3", - "#category": ("", "naverwebtoon", "episode"), - "#class" : naverwebtoon.NaverwebtoonEpisodeExtractor, - "#pattern" : r"https://image-comic\.pstatic\.net/user_contents_data/challenge_comic/2021/04/28/345534/upload_3617293622396203109\.jpeg", - "#count" : 1, -}, - -{ - "#url" : "https://comic.naver.com/webtoon/list?titleId=22073", - "#category": ("", "naverwebtoon", "comic"), - "#class" : naverwebtoon.NaverwebtoonComicExtractor, - "#pattern" : naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, - "#count" : 32, -}, - -{ - "#url" : "https://comic.naver.com/webtoon/list?titleId=765124", - "#comment" : "/webtoon/ path for 'challenge' comic (#5123)", - "#category": ("", "naverwebtoon", "comic"), - "#class" : naverwebtoon.NaverwebtoonComicExtractor, - "#range" : "1", - "#urls" : "https://comic.naver.com/challenge/detail?titleId=765124&no=1", -}, - -{ - "#url" : "https://comic.naver.com/challenge/list?titleId=765124", - "#category": ("", "naverwebtoon", "comic"), - "#class" : naverwebtoon.NaverwebtoonComicExtractor, - "#pattern" : naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, - "#count" : 24, -}, - -{ - "#url" : "https://comic.naver.com/bestChallenge/list.nhn?titleId=789786", - "#category": ("", "naverwebtoon", "comic"), - "#class" : naverwebtoon.NaverwebtoonComicExtractor, - "#pattern" : naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, - "#count" : ">= 12", -}, - + { + "#url": "https://comic.naver.com/webtoon/detail?titleId=26458&no=1&weekday=tue", + "#category": ("", "naverwebtoon", "episode"), + "#class": naverwebtoon.NaverwebtoonEpisodeExtractor, + "#count": 14, + "#sha1_url": "47a956ba8c7a837213d5985f50c569fcff986f75", + "#sha1_content": "3806b6e8befbb1920048de9888dfce6220f69a60", + "author": ["김규삼"], + "artist": ["김규삼"], + "comic": "N의등대-눈의등대", + "count": 14, + "episode": "1", + "extension": "jpg", + "num": int, + "tags": [ + "스릴러", + "완결무료", + "완결스릴러", + ], + "title": "n의 등대 - 눈의 등대 1화", + "title_id": "26458", + }, + { + "#url": "https://comic.naver.com/challenge/detail?titleId=765124&no=1", + "#category": ("", "naverwebtoon", "episode"), + "#class": naverwebtoon.NaverwebtoonEpisodeExtractor, + "#pattern": r"https://image-comic\.pstatic\.net/user_contents_data/challenge_comic/2021/01/19/342586/upload_7149856273586337846\.jpeg", + "#count": 1, + "author": ["kemi****"], + "artist": [], + "comic": "우니 모두의 이야기", + "count": 1, + "episode": "1", + "extension": "jpeg", + "filename": "upload_7149856273586337846", + "num": 1, + "tags": [ + "일상툰", + "우니모두의이야기", + "퇴사", + "입사", + "신입사원", + "사회초년생", + "회사원", + "20대", + ], + "title": "퇴사하다", + "title_id": "765124", + }, + { + "#url": "https://comic.naver.com/bestChallenge/detail?titleId=620732&no=334", + "#comment": "empty tags (#5120)", + "#category": ("", "naverwebtoon", "episode"), + "#class": naverwebtoon.NaverwebtoonEpisodeExtractor, + "#count": 9, + "artist": [], + "author": ["안트로anthrokim"], + "comic": "백일몽화원", + "count": 9, + "episode": "334", + "num": range(1, 9), + "tags": [], + "title": "321화... 성(省)", + "title_id": "620732", + }, + { + "#url": "https://comic.naver.com/bestChallenge/detail.nhn?titleId=771467&no=3", + "#category": ("", "naverwebtoon", "episode"), + "#class": naverwebtoon.NaverwebtoonEpisodeExtractor, + "#pattern": r"https://image-comic\.pstatic\.net/user_contents_data/challenge_comic/2021/04/28/345534/upload_3617293622396203109\.jpeg", + "#count": 1, + }, + { + "#url": "https://comic.naver.com/webtoon/list?titleId=22073", + "#category": ("", "naverwebtoon", "comic"), + "#class": naverwebtoon.NaverwebtoonComicExtractor, + "#pattern": naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, + "#count": 32, + }, + { + "#url": "https://comic.naver.com/webtoon/list?titleId=765124", + "#comment": "/webtoon/ path for 'challenge' comic (#5123)", + "#category": ("", "naverwebtoon", "comic"), + "#class": naverwebtoon.NaverwebtoonComicExtractor, + "#range": "1", + "#urls": "https://comic.naver.com/challenge/detail?titleId=765124&no=1", + }, + { + "#url": "https://comic.naver.com/challenge/list?titleId=765124", + "#category": ("", "naverwebtoon", "comic"), + "#class": naverwebtoon.NaverwebtoonComicExtractor, + "#pattern": naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, + "#count": 24, + }, + { + "#url": "https://comic.naver.com/bestChallenge/list.nhn?titleId=789786", + "#category": ("", "naverwebtoon", "comic"), + "#class": naverwebtoon.NaverwebtoonComicExtractor, + "#pattern": naverwebtoon.NaverwebtoonEpisodeExtractor.pattern, + "#count": ">= 12", + }, ) diff --git a/test/results/newgrounds.py b/test/results/newgrounds.py index 8ff37b2d40..bb5e358f1f 100644 --- a/test/results/newgrounds.py +++ b/test/results/newgrounds.py @@ -1,405 +1,358 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import newgrounds - __tests__ = ( -{ - "#url" : "https://www.newgrounds.com/art/view/tomfulp/ryu-is-hawt", - "#category": ("", "newgrounds", "image"), - "#class" : newgrounds.NewgroundsImageExtractor, - "#urls" : "https://art.ngfiles.com/images/1993000/1993615_4474_tomfulp_ryu-is-hawt.44f81090378ae9c257a5e46a8e17cc4d.gif?f1695674895", - "#sha1_content": "8f395e08333eb2457ba8d8b715238f8910221365", - - "artist" : ["tomfulp"], - "comment" : "Consider this the bottom threshold for scouted artists.In fact consider it BELOW the bottom threshold.", - "date" : "dt:2009-06-04 14:44:05", - "description": "Consider this the bottom threshold for scouted artists. In fact consider it BELOW the bottom threshold. ", - "favorites" : int, - "filename" : "1993615_4474_tomfulp_ryu-is-hawt.44f81090378ae9c257a5e46a8e17cc4d", - "height" : 476, - "index" : 1993615, - "rating" : "e", - "score" : float, - "tags" : [ - "ryu", - "streetfighter", - ], - "title" : "Ryu is Hawt", - "type" : "article", - "user" : "tomfulp", - "width" : 447, -}, - -{ - "#url" : "https://art.ngfiles.com/images/0/94_tomfulp_ryu-is-hawt.gif", - "#category": ("", "newgrounds", "image"), - "#class" : newgrounds.NewgroundsImageExtractor, - "#urls" : "https://art.ngfiles.com/images/1993000/1993615_4474_tomfulp_ryu-is-hawt.44f81090378ae9c257a5e46a8e17cc4d.gif?f1695674895", -}, - -{ - "#url" : "https://www.newgrounds.com/art/view/sailoryon/yon-dream-buster", - "#comment" : "embedded file in 'comments' (#1033)", - "#category": ("", "newgrounds", "image"), - "#class" : newgrounds.NewgroundsImageExtractor, - "#urls" : ( - "https://art.ngfiles.com/images/1438000/1438673_sailoryon_yon-dream-buster.jpg?f1601058173", - "https://art.ngfiles.com/comments/172000/iu_172374_7112211.jpg", - ), -}, - -{ - "#url" : "https://www.newgrounds.com/art/view/zedrinbot/lewd-animation-tutorial", - "#comment" : "extra files in 'art-image-row' elements - WebP to GIF (#4642)", - "#category": ("", "newgrounds", "image"), - "#class" : newgrounds.NewgroundsImageExtractor, - "#urls" : ( - "https://art.ngfiles.com/images/5091000/5091275_45067_zedrinbot_untitled-5091275.0a9d27ed2bc265a7e89478ed6ad6f86f.gif?f1696187399", - "https://art.ngfiles.com/images/5091000/5091275_45071_zedrinbot_untitled-5091275.6fdc62eaef43528fb1c9bda624d30a3d.gif?f1696187437", - "https://art.ngfiles.com/images/5091000/5091275_45070_zedrinbot_untitled-5091275.0d7334746374465bd448908b88d1f810.gif?f1696187435", - "https://art.ngfiles.com/images/5091000/5091275_45072_zedrinbot_untitled-5091275.6fdc62eaef43528fb1c9bda624d30a3d.gif?f1696187438", - "https://art.ngfiles.com/images/5091000/5091275_45073_zedrinbot_untitled-5091275.20aa05c1cd22fd058e8c68ce58f5a302.gif?f1696187439", - ), -}, - -{ - "#url" : "https://www.newgrounds.com/art/view/zedrinbot/nazrin-tanlines", - "#comment" : "extra files in 'art-image-row' elements - native PNG files (#4642)", - "#category": ("", "newgrounds", "image"), - "#class" : newgrounds.NewgroundsImageExtractor, - "#auth" : True, - "#urls" : ( - "https://art.ngfiles.com/images/5009000/5009916_14628_zedrinbot_nazrin-tanlines.265f7b6beec5855a349e2646e90cbc01.png?f1695698131", - "https://art.ngfiles.com/images/5009000/5009916_14632_zedrinbot_nazrin-tanlines.40bd62fbf5875806cda6b004b348114a.png?f1695727318", - "https://art.ngfiles.com/images/5009000/5009916_14634_zedrinbot_nazrin-tanlines.40bd62fbf5875806cda6b004b348114a.png?f1695727321", - "https://art.ngfiles.com/images/5009000/5009916_14633_zedrinbot_nazrin-tanlines.40bd62fbf5875806cda6b004b348114a.png?f1695727318", - "https://art.ngfiles.com/images/5009000/5009916_14635_zedrinbot_nazrin-tanlines.6a7aa4fd63e5f8077ad29314568246cc.png?f1695727321", - "https://art.ngfiles.com/images/5009000/5009916_14636_zedrinbot_nazrin-tanlines.6a7aa4fd63e5f8077ad29314568246cc.png?f1695727322", - ), -}, - -{ - "#url" : "https://www.newgrounds.com/art/view/bacun/kill-la-kill-10th-anniversary", - "#comment" : "extra files in 'imageData' block (#4642)", - "#category": ("", "newgrounds", "image"), - "#class" : newgrounds.NewgroundsImageExtractor, - "#urls" : ( - "https://art.ngfiles.com/images/5127000/5127150_93307_bacun_kill-la-kill-10th-anniversary.61adfe309bec342f9db55fd44397235b.png?f1697310027", - "https://art.ngfiles.com/images/5127000/5127150_94250_bacun_kill-la-kill-10th-anniversary.64fdf525fa38c1ab34defac4b354bc7a.png?f1697332109", - ), -}, - -{ - "#url" : "https://www.newgrounds.com/art/view/sockdotclip/trickin-treats", - "#comment" : "extra files in comment section as '= 3", -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/art/page/3", - "#class" : newgrounds.NewgroundsArtExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/art?page=3", - "#class" : newgrounds.NewgroundsArtExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/audio", - "#class" : newgrounds.NewgroundsAudioExtractor, - "#pattern" : r"https://(audio\.ngfiles\.com/\d+/\d+_.+\.mp3|uploads\.ungrounded\.net/.+\.png)", - "#count" : ">= 10", -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/audio?page=3", - "#class" : newgrounds.NewgroundsAudioExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/movies", - "#class" : newgrounds.NewgroundsMoviesExtractor, - "#pattern" : r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/movies/?page=3", - "#class" : newgrounds.NewgroundsMoviesExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/games", - "#class" : newgrounds.NewgroundsGamesExtractor, - "#pattern" : r"https://uploads.ungrounded.net(/alternate)?/(\d+/\d+_.+|tmp/.+)", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/games?page=3", - "#class" : newgrounds.NewgroundsGamesExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com", - "#class" : newgrounds.NewgroundsUserExtractor, - "#urls" : "https://tomfulp.newgrounds.com/art", -}, - -{ - "#url" : "https://tomfulp.newgrounds.com", - "#class" : newgrounds.NewgroundsUserExtractor, - "#options" : {"include": "all"}, - "#urls" : ( - "https://tomfulp.newgrounds.com/art", - "https://tomfulp.newgrounds.com/audio", - "https://tomfulp.newgrounds.com/games", - "https://tomfulp.newgrounds.com/movies", - ), -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/art", - "#class" : newgrounds.NewgroundsFavoriteExtractor, - "#range" : "1-10", - "#count" : ">= 10", -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/art?page=3", - "#class" : newgrounds.NewgroundsFavoriteExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/audio", - "#class" : newgrounds.NewgroundsFavoriteExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/movies", - "#class" : newgrounds.NewgroundsFavoriteExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/", - "#class" : newgrounds.NewgroundsFavoriteExtractor, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/following", - "#class" : newgrounds.NewgroundsFollowingExtractor, - "#pattern" : newgrounds.NewgroundsUserExtractor.pattern, - "#range" : "76-125", - "#count" : 50, -}, - -{ - "#url" : "https://tomfulp.newgrounds.com/favorites/following?page=3", - "#class" : newgrounds.NewgroundsFollowingExtractor, -}, - - -{ - "#url" : "https://www.newgrounds.com/search/conduct/art?terms=tree", - "#class" : newgrounds.NewgroundsSearchExtractor, - "#pattern" : newgrounds.NewgroundsImageExtractor.pattern, - "#range" : "1-10", - "#count" : 10, - - "search_tags": "tree", -}, - -{ - "#url" : "https://www.newgrounds.com/search/conduct/movies?terms=tree", - "#class" : newgrounds.NewgroundsSearchExtractor, - "#pattern" : r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.newgrounds.com/search/conduct/audio?advanced=1&terms=tree+green+nature&match=tdtu&genre=5&suitabilities=e%2Cm", - "#class" : newgrounds.NewgroundsSearchExtractor, -}, - + "date": "dt:2015-02-23 19:31:59", + "description": "From The ZJ Report Show!", + "favorites": int, + "index": 609768, + "rating": "", + "score": float, + "tags": [ + "fulp", + "interview", + "tom", + "zj", + ], + "title": "ZJ Interviews Tom Fulp!", + "type": "music.song", + "user": "zj", + }, + { + "#url": "https://www.newgrounds.com/portal/view/161181/format/flash", + "#comment": "flash animation (#1257)", + "#category": ("", "newgrounds", "media"), + "#class": newgrounds.NewgroundsMediaExtractor, + "#urls": "https://uploads.ungrounded.net/161000/161181_ddautta_mask__550x281_.swf", + "type": "movie", + }, + { + "#url": "https://www.newgrounds.com/portal/view/758545", + "#comment": "format selection (#1729)", + "#category": ("", "newgrounds", "media"), + "#class": newgrounds.NewgroundsMediaExtractor, + "#options": {"format": "720p"}, + "#pattern": r"https://uploads\.ungrounded\.net/alternate/1482000/1482860_alternate_102516\.720p\.mp4\?\d+", + }, + { + "#url": "https://www.newgrounds.com/portal/view/717744", + "#comment": "'adult' rated (#2456)", + "#category": ("", "newgrounds", "media"), + "#class": newgrounds.NewgroundsMediaExtractor, + "#options": {"username": None}, + "#count": 1, + }, + { + "#url": "https://www.newgrounds.com/portal/view/829032", + "#comment": "flash game", + "#category": ("", "newgrounds", "media"), + "#class": newgrounds.NewgroundsMediaExtractor, + "#urls": ( + "https://uploads.ungrounded.net/829000/829032_picovsbeardx.swf", + "https://uploads.ungrounded.net/tmp/img/521000/iu_521265_5431202.gif", + ), + "artist": [ + "dungeonation", + "carpetbakery", + "animalspeakandrews", + "bill", + "chipollo", + "dylz49", + "gappyshamp", + "pinktophat", + "rad", + "shapeshiftingblob", + "tomfulp", + "voicesbycorey", + "psychogoldfish", + ], + "comment": r"re:The children are expendable. Take out the ", + "date": "dt:2022-01-10 23:00:57", + "description": "Bloodshed in The Big House that Blew...again!", + "favorites": int, + "index": 829032, + "post_url": "https://www.newgrounds.com/portal/view/829032", + "rating": "m", + "score": float, + "tags": [ + "assassin", + "boyfriend", + "darnell", + "nene", + "pico", + "picos-school", + ], + "title": "PICO VS BEAR DX", + "type": "game", + "url": "https://uploads.ungrounded.net/829000/829032_picovsbeardx.swf", + }, + { + "#url": "https://tomfulp.newgrounds.com/art", + "#class": newgrounds.NewgroundsArtExtractor, + "#pattern": newgrounds.NewgroundsImageExtractor.pattern, + "#count": ">= 3", + }, + { + "#url": "https://tomfulp.newgrounds.com/art/page/3", + "#class": newgrounds.NewgroundsArtExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/art?page=3", + "#class": newgrounds.NewgroundsArtExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/audio", + "#class": newgrounds.NewgroundsAudioExtractor, + "#pattern": r"https://(audio\.ngfiles\.com/\d+/\d+_.+\.mp3|uploads\.ungrounded\.net/.+\.png)", + "#count": ">= 10", + }, + { + "#url": "https://tomfulp.newgrounds.com/audio?page=3", + "#class": newgrounds.NewgroundsAudioExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/movies", + "#class": newgrounds.NewgroundsMoviesExtractor, + "#pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+_.+", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://tomfulp.newgrounds.com/movies/?page=3", + "#class": newgrounds.NewgroundsMoviesExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/games", + "#class": newgrounds.NewgroundsGamesExtractor, + "#pattern": r"https://uploads.ungrounded.net(/alternate)?/(\d+/\d+_.+|tmp/.+)", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://tomfulp.newgrounds.com/games?page=3", + "#class": newgrounds.NewgroundsGamesExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com", + "#class": newgrounds.NewgroundsUserExtractor, + "#urls": "https://tomfulp.newgrounds.com/art", + }, + { + "#url": "https://tomfulp.newgrounds.com", + "#class": newgrounds.NewgroundsUserExtractor, + "#options": {"include": "all"}, + "#urls": ( + "https://tomfulp.newgrounds.com/art", + "https://tomfulp.newgrounds.com/audio", + "https://tomfulp.newgrounds.com/games", + "https://tomfulp.newgrounds.com/movies", + ), + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/art", + "#class": newgrounds.NewgroundsFavoriteExtractor, + "#range": "1-10", + "#count": ">= 10", + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/art?page=3", + "#class": newgrounds.NewgroundsFavoriteExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/audio", + "#class": newgrounds.NewgroundsFavoriteExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/movies", + "#class": newgrounds.NewgroundsFavoriteExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/", + "#class": newgrounds.NewgroundsFavoriteExtractor, + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/following", + "#class": newgrounds.NewgroundsFollowingExtractor, + "#pattern": newgrounds.NewgroundsUserExtractor.pattern, + "#range": "76-125", + "#count": 50, + }, + { + "#url": "https://tomfulp.newgrounds.com/favorites/following?page=3", + "#class": newgrounds.NewgroundsFollowingExtractor, + }, + { + "#url": "https://www.newgrounds.com/search/conduct/art?terms=tree", + "#class": newgrounds.NewgroundsSearchExtractor, + "#pattern": newgrounds.NewgroundsImageExtractor.pattern, + "#range": "1-10", + "#count": 10, + "search_tags": "tree", + }, + { + "#url": "https://www.newgrounds.com/search/conduct/movies?terms=tree", + "#class": newgrounds.NewgroundsSearchExtractor, + "#pattern": r"https://uploads.ungrounded.net(/alternate)?/\d+/\d+", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.newgrounds.com/search/conduct/audio?advanced=1&terms=tree+green+nature&match=tdtu&genre=5&suitabilities=e%2Cm", + "#class": newgrounds.NewgroundsSearchExtractor, + }, ) diff --git a/test/results/nhentai.py b/test/results/nhentai.py index 02667ac436..9a4addc368 100644 --- a/test/results/nhentai.py +++ b/test/results/nhentai.py @@ -1,110 +1,95 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import nhentai - __tests__ = ( -{ - "#url" : "https://nhentai.net/g/147850/", - "#category": ("", "nhentai", "gallery"), - "#class" : nhentai.NhentaiGalleryExtractor, - "#sha1_url": "5179dbf0f96af44005a0ff705a0ad64ac26547d0", - - "title" : r"re:\[Morris\] Amazon no Hiyaku \| Amazon Elixir", - "title_en" : str, - "title_ja" : str, - "gallery_id": 147850, - "media_id" : 867789, - "count" : 16, - "date" : 1446050915, - "scanlator" : "", - "artist" : ["morris"], - "group" : list, - "parody" : list, - "characters": list, - "tags" : list, - "type" : "manga", - "lang" : "en", - "language" : "English", - "width" : int, - "height" : int, -}, - -{ - "#url" : "https://nhentai.net/g/538045/", - "#comment" : "webp (#6442)", - "#class" : nhentai.NhentaiGalleryExtractor, - "#range" : "4-7", - "#urls" : ( - "https://i.nhentai.net/galleries/3115523/4.jpg", - "https://i.nhentai.net/galleries/3115523/5.webp", - "https://i.nhentai.net/galleries/3115523/6.webp", - "https://i.nhentai.net/galleries/3115523/7.jpg", - ), -}, - -{ - "#url" : "https://nhentai.net/tag/sole-female/", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, - "#pattern" : nhentai.NhentaiGalleryExtractor.pattern, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://nhentai.net/artist/itou-life/", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, -}, - -{ - "#url" : "https://nhentai.net/group/itou-life/", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, -}, - -{ - "#url" : "https://nhentai.net/parody/touhou-project/", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, -}, - -{ - "#url" : "https://nhentai.net/character/patchouli-knowledge/popular", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, -}, - -{ - "#url" : "https://nhentai.net/category/doujinshi/popular-today", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, -}, - -{ - "#url" : "https://nhentai.net/language/english/popular-week", - "#category": ("", "nhentai", "tag"), - "#class" : nhentai.NhentaiTagExtractor, -}, - -{ - "#url" : "https://nhentai.net/search/?q=touhou", - "#category": ("", "nhentai", "search"), - "#class" : nhentai.NhentaiSearchExtractor, - "#pattern" : nhentai.NhentaiGalleryExtractor.pattern, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://nhentai.net/favorites/", - "#category": ("", "nhentai", "favorite"), - "#class" : nhentai.NhentaiFavoriteExtractor, -}, - + { + "#url": "https://nhentai.net/g/147850/", + "#category": ("", "nhentai", "gallery"), + "#class": nhentai.NhentaiGalleryExtractor, + "#sha1_url": "5179dbf0f96af44005a0ff705a0ad64ac26547d0", + "title": r"re:\[Morris\] Amazon no Hiyaku \| Amazon Elixir", + "title_en": str, + "title_ja": str, + "gallery_id": 147850, + "media_id": 867789, + "count": 16, + "date": 1446050915, + "scanlator": "", + "artist": ["morris"], + "group": list, + "parody": list, + "characters": list, + "tags": list, + "type": "manga", + "lang": "en", + "language": "English", + "width": int, + "height": int, + }, + { + "#url": "https://nhentai.net/g/538045/", + "#comment": "webp (#6442)", + "#class": nhentai.NhentaiGalleryExtractor, + "#range": "4-7", + "#urls": ( + "https://i.nhentai.net/galleries/3115523/4.jpg", + "https://i.nhentai.net/galleries/3115523/5.webp", + "https://i.nhentai.net/galleries/3115523/6.webp", + "https://i.nhentai.net/galleries/3115523/7.jpg", + ), + }, + { + "#url": "https://nhentai.net/tag/sole-female/", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + "#pattern": nhentai.NhentaiGalleryExtractor.pattern, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://nhentai.net/artist/itou-life/", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + }, + { + "#url": "https://nhentai.net/group/itou-life/", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + }, + { + "#url": "https://nhentai.net/parody/touhou-project/", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + }, + { + "#url": "https://nhentai.net/character/patchouli-knowledge/popular", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + }, + { + "#url": "https://nhentai.net/category/doujinshi/popular-today", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + }, + { + "#url": "https://nhentai.net/language/english/popular-week", + "#category": ("", "nhentai", "tag"), + "#class": nhentai.NhentaiTagExtractor, + }, + { + "#url": "https://nhentai.net/search/?q=touhou", + "#category": ("", "nhentai", "search"), + "#class": nhentai.NhentaiSearchExtractor, + "#pattern": nhentai.NhentaiGalleryExtractor.pattern, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://nhentai.net/favorites/", + "#category": ("", "nhentai", "favorite"), + "#class": nhentai.NhentaiFavoriteExtractor, + }, ) diff --git a/test/results/nijie.py b/test/results/nijie.py index 829b62015f..4d7280d699 100644 --- a/test/results/nijie.py +++ b/test/results/nijie.py @@ -1,193 +1,171 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import nijie import datetime -from gallery_dl import exception +from gallery_dl import exception +from gallery_dl.extractor import nijie __tests__ = ( -{ - "#url" : "https://nijie.info/members.php?id=44", - "#category": ("Nijie", "nijie", "user"), - "#class" : nijie.NijieUserExtractor, - "#urls" : ( - "https://nijie.info/members_illust.php?id=44", - "https://nijie.info/members_dojin.php?id=44", - ), -}, - -{ - "#url" : "https://nijie.info/members_illust.php?id=44", - "#category": ("Nijie", "nijie", "illustration"), - "#class" : nijie.NijieIllustrationExtractor, - "#urls": ( - "https://pic.nijie.net/04/nijie/14/44/44/illust/0_0_f46c08462568c2f1_be95d7.jpg", - "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg", - ), - - "artist_id" : 44, - "artist_name": "ED", - "count" : 1, - "date" : datetime.datetime, - "description": str, - "extension" : "jpg", - "filename" : str, - "image_id" : int, - "num" : 0, - "tags" : list, - "title" : str, - "url" : r"re:https://pic.nijie.net/\d+/nijie/.*jpg$", - "user_id" : 44, - "user_name" : "ED", -}, - -{ - "#url" : "https://nijie.info/members_illust.php?id=43", - "#category": ("Nijie", "nijie", "illustration"), - "#class" : nijie.NijieIllustrationExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://nijie.info/members_dojin.php?id=6782", - "#category": ("Nijie", "nijie", "doujin"), - "#class" : nijie.NijieDoujinExtractor, - "#count" : ">= 18", - - "user_id" : 6782, - "user_name": "ジョニー＠アビオン村", -}, - -{ - "#url" : "https://nijie.info/user_like_illust_view.php?id=44", - "#category": ("Nijie", "nijie", "favorite"), - "#class" : nijie.NijieFavoriteExtractor, - "#count" : ">= 16", - - "user_id" : 44, - "user_name": "ED", -}, - -{ - "#url" : "https://nijie.info/history_nuita.php?id=728995", - "#category": ("Nijie", "nijie", "nuita"), - "#class" : nijie.NijieNuitaExtractor, - "#range" : "1-10", - "#count" : 10, - - "user_id" : 728995, - "user_name": "莚", -}, - -{ - "#url" : "https://nijie.info/like_user_view.php", - "#category": ("Nijie", "nijie", "feed"), - "#class" : nijie.NijieFeedExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://nijie.info/like_my.php", - "#category": ("Nijie", "nijie", "followed"), - "#class" : nijie.NijieFollowedExtractor, -}, - -{ - "#url" : "https://nijie.info/view.php?id=70720", - "#category": ("Nijie", "nijie", "image"), - "#class" : nijie.NijieImageExtractor, - "#urls" : "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg", - "#sha1_url" : "3d654e890212ba823c9647754767336aebc0a743", - "#sha1_metadata": "58e716bcb03b431cae901178c198c787908e1c0c", - "#sha1_content" : "d85e3ea896ed5e4da0bca2390ad310a4df716ca6", - - "artist_id" : 44, - "artist_name": "ED", - "count" : 1, - "date" : "dt:2014-01-18 19:58:21", - "description": "租絵にてお邪魔いたし候\r\n是非ともこの”おっぱい”をご高覧賜りたく馳せ参じた次第\r\n長文にて失礼仕る\r\n\r\nまず全景でありますが、首を右に傾けてみて頂きたい\r\nこの絵図は茶碗を眺めていた私が思わぬ美しさにて昇天したときのものを、筆をとり、したためたものである（トレースではない）\r\n筆は疾風の如く走り、半刻過ぎには私好みの”おっぱい”になっていたのである！\r\n次に細部をみて頂きたい\r\n絵図を正面から見直して頂くと、なんとはんなりと美しいお椀型をしたおっぱいであろうか　　右手から緩やかに生まれる曲線は左手に進むにつれて、穏やかな歪みを含み流れる　　これは所謂轆轤目であるが三重の紐でおっぱいをぐるぐると巻きつけた情景そのままであり、この歪みから茶碗の均整は崩れ、たぷんたぷんのおっぱいの重量感を醸し出している！\r\nさらに左手に進めば梅花皮（カイラギ）を孕んだ高大が現れる　今回は点線にて表現するが、その姿は乳首から母乳が噴出するが如く　或は精子をぶっかけられたが如く　白くとろっとした釉薬の凝固が素晴しい景色をつくりだしているのである！\r\n最後には極めつけ、すくっと螺旋を帯びながらそそり立つ兜巾（ときん）！この情景はまさしく乳首である！　　全体をふんわりと盛り上げさせる乳輪にちょこっと存在する乳頭はぺろりと舌で確かめ勃起させたくなる風情がある！\r\n\r\nこれを”おっぱい”と呼ばずなんと呼ぼうや！？\r\n\r\n興奮のあまり失礼致した\r\n御免", - "extension" : "jpg", - "filename" : "0_0_28e8c02d921bee33_9222d3", - "image_id" : 70720, - "num" : 0, - "tags" : ["おっぱい"], - "title" : "俺好高麗井戸茶碗　銘おっぱい", - "url" : "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg", - "user_id" : 44, - "user_name" : "ED", -}, - -{ - "#url" : "https://nijie.info/view.php?id=594044", - "#category": ("Nijie", "nijie", "image"), - "#class" : nijie.NijieImageExtractor, - "#urls": ( - "https://pic.nijie.net/02/nijie/23m12/09/49509/illust/0_0_63568cc428259d50_45ca51.jpg", - "https://pic.nijie.net/01/nijie/23m12/09/49509/illust/594044_0_1c94b7cc4503589f_79c66c.jpg", - "https://pic.nijie.net/02/nijie/23m12/09/49509/illust/594044_1_9f4737ad48bf43c7_8f1e8e.jpg", - "https://pic.nijie.net/01/nijie/23m12/09/49509/illust/594044_2_a162861fac970a45_38c5f8.jpg", - ), - - "artist_id" : 49509, - "artist_name": "黒川　竜", - "count" : 4, - "date" : "dt:2023-12-02 04:19:29", - "description": "【DLサイトコム】ウィンターセール　30%OFF\r\n期間：2024年2月14日まで\r\n【toloveるドリンク】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ042727.html\r\n【toloveるドリンク２】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ043289.html\r\n【クランクランBIG】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ043564.html", - "image_id" : 594044, - "num" : range(0, 3), - "tags" : [ - "オリジナル", - "漫画", - "中出し", - "爆乳", - "巨乳", - "ToLOVEる", - "宣伝", - "クラン・クラン", - "マクロスF", - ], - "title" : "【DLサイトコム】ウィンターセール", - "url" : str, - "user_id" : 49509, - "user_name" : "黒川　竜", -}, - -{ - "#url" : "https://nijie.info/view.php?id=37078", - "#comment" : "'view_side_dojin' thumbnails (#5049)", - "#category": ("Nijie", "nijie", "image"), - "#class" : nijie.NijieImageExtractor, - "#urls" : "https://pic.nijie.net/03/nijie/13/98/498/illust/0_0_703023d18ca8d058_bca943.jpg", -}, - -{ - "#url" : "https://nijie.info/view.php?id=385585", - "#comment" : "video (#5707)", - "#category": ("Nijie", "nijie", "image"), - "#class" : nijie.NijieImageExtractor, - "#urls" : ( - "https://pic.nijie.net/01/nijie/20/82/59182/illust/0_0_162270ef49e2ee28_fab5ae.mp4", - "https://pic.nijie.net/04/nijie/20/82/59182/illust/385585_0_ff2d5d19129530d5_b2821e.jpg", - "https://pic.nijie.net/01/nijie/20/82/59182/illust/385585_1_7ee1a2a67bed2f84_212d67.jpg", - ), -}, - -{ - "#url" : "https://nijie.info/view.php?id=70724", - "#category": ("Nijie", "nijie", "image"), - "#class" : nijie.NijieImageExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://nijie.info/view_popup.php?id=70720", - "#category": ("Nijie", "nijie", "image"), - "#class" : nijie.NijieImageExtractor, -}, - + { + "#url": "https://nijie.info/members.php?id=44", + "#category": ("Nijie", "nijie", "user"), + "#class": nijie.NijieUserExtractor, + "#urls": ( + "https://nijie.info/members_illust.php?id=44", + "https://nijie.info/members_dojin.php?id=44", + ), + }, + { + "#url": "https://nijie.info/members_illust.php?id=44", + "#category": ("Nijie", "nijie", "illustration"), + "#class": nijie.NijieIllustrationExtractor, + "#urls": ( + "https://pic.nijie.net/04/nijie/14/44/44/illust/0_0_f46c08462568c2f1_be95d7.jpg", + "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg", + ), + "artist_id": 44, + "artist_name": "ED", + "count": 1, + "date": datetime.datetime, + "description": str, + "extension": "jpg", + "filename": str, + "image_id": int, + "num": 0, + "tags": list, + "title": str, + "url": r"re:https://pic.nijie.net/\d+/nijie/.*jpg$", + "user_id": 44, + "user_name": "ED", + }, + { + "#url": "https://nijie.info/members_illust.php?id=43", + "#category": ("Nijie", "nijie", "illustration"), + "#class": nijie.NijieIllustrationExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://nijie.info/members_dojin.php?id=6782", + "#category": ("Nijie", "nijie", "doujin"), + "#class": nijie.NijieDoujinExtractor, + "#count": ">= 18", + "user_id": 6782, + "user_name": "ジョニー＠アビオン村", + }, + { + "#url": "https://nijie.info/user_like_illust_view.php?id=44", + "#category": ("Nijie", "nijie", "favorite"), + "#class": nijie.NijieFavoriteExtractor, + "#count": ">= 16", + "user_id": 44, + "user_name": "ED", + }, + { + "#url": "https://nijie.info/history_nuita.php?id=728995", + "#category": ("Nijie", "nijie", "nuita"), + "#class": nijie.NijieNuitaExtractor, + "#range": "1-10", + "#count": 10, + "user_id": 728995, + "user_name": "莚", + }, + { + "#url": "https://nijie.info/like_user_view.php", + "#category": ("Nijie", "nijie", "feed"), + "#class": nijie.NijieFeedExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://nijie.info/like_my.php", + "#category": ("Nijie", "nijie", "followed"), + "#class": nijie.NijieFollowedExtractor, + }, + { + "#url": "https://nijie.info/view.php?id=70720", + "#category": ("Nijie", "nijie", "image"), + "#class": nijie.NijieImageExtractor, + "#urls": "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg", + "#sha1_url": "3d654e890212ba823c9647754767336aebc0a743", + "#sha1_metadata": "58e716bcb03b431cae901178c198c787908e1c0c", + "#sha1_content": "d85e3ea896ed5e4da0bca2390ad310a4df716ca6", + "artist_id": 44, + "artist_name": "ED", + "count": 1, + "date": "dt:2014-01-18 19:58:21", + "description": "租絵にてお邪魔いたし候\r\n是非ともこの”おっぱい”をご高覧賜りたく馳せ参じた次第\r\n長文にて失礼仕る\r\n\r\nまず全景でありますが、首を右に傾けてみて頂きたい\r\nこの絵図は茶碗を眺めていた私が思わぬ美しさにて昇天したときのものを、筆をとり、したためたものである（トレースではない）\r\n筆は疾風の如く走り、半刻過ぎには私好みの”おっぱい”になっていたのである！\r\n次に細部をみて頂きたい\r\n絵図を正面から見直して頂くと、なんとはんなりと美しいお椀型をしたおっぱいであろうか　　右手から緩やかに生まれる曲線は左手に進むにつれて、穏やかな歪みを含み流れる　　これは所謂轆轤目であるが三重の紐でおっぱいをぐるぐると巻きつけた情景そのままであり、この歪みから茶碗の均整は崩れ、たぷんたぷんのおっぱいの重量感を醸し出している！\r\nさらに左手に進めば梅花皮（カイラギ）を孕んだ高大が現れる　今回は点線にて表現するが、その姿は乳首から母乳が噴出するが如く　或は精子をぶっかけられたが如く　白くとろっとした釉薬の凝固が素晴しい景色をつくりだしているのである！\r\n最後には極めつけ、すくっと螺旋を帯びながらそそり立つ兜巾（ときん）！この情景はまさしく乳首である！　　全体をふんわりと盛り上げさせる乳輪にちょこっと存在する乳頭はぺろりと舌で確かめ勃起させたくなる風情がある！\r\n\r\nこれを”おっぱい”と呼ばずなんと呼ぼうや！？\r\n\r\n興奮のあまり失礼致した\r\n御免", + "extension": "jpg", + "filename": "0_0_28e8c02d921bee33_9222d3", + "image_id": 70720, + "num": 0, + "tags": ["おっぱい"], + "title": "俺好高麗井戸茶碗　銘おっぱい", + "url": "https://pic.nijie.net/06/nijie/14/44/44/illust/0_0_28e8c02d921bee33_9222d3.jpg", + "user_id": 44, + "user_name": "ED", + }, + { + "#url": "https://nijie.info/view.php?id=594044", + "#category": ("Nijie", "nijie", "image"), + "#class": nijie.NijieImageExtractor, + "#urls": ( + "https://pic.nijie.net/02/nijie/23m12/09/49509/illust/0_0_63568cc428259d50_45ca51.jpg", + "https://pic.nijie.net/01/nijie/23m12/09/49509/illust/594044_0_1c94b7cc4503589f_79c66c.jpg", + "https://pic.nijie.net/02/nijie/23m12/09/49509/illust/594044_1_9f4737ad48bf43c7_8f1e8e.jpg", + "https://pic.nijie.net/01/nijie/23m12/09/49509/illust/594044_2_a162861fac970a45_38c5f8.jpg", + ), + "artist_id": 49509, + "artist_name": "黒川　竜", + "count": 4, + "date": "dt:2023-12-02 04:19:29", + "description": "【DLサイトコム】ウィンターセール　30%OFF\r\n期間：2024年2月14日まで\r\n【toloveるドリンク】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ042727.html\r\n【toloveるドリンク２】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ043289.html\r\n【クランクランBIG】\r\nhttps://www.dlsite.com/maniax/work/=/product_id/RJ043564.html", + "image_id": 594044, + "num": range(3), + "tags": [ + "オリジナル", + "漫画", + "中出し", + "爆乳", + "巨乳", + "ToLOVEる", + "宣伝", + "クラン・クラン", + "マクロスF", + ], + "title": "【DLサイトコム】ウィンターセール", + "url": str, + "user_id": 49509, + "user_name": "黒川　竜", + }, + { + "#url": "https://nijie.info/view.php?id=37078", + "#comment": "'view_side_dojin' thumbnails (#5049)", + "#category": ("Nijie", "nijie", "image"), + "#class": nijie.NijieImageExtractor, + "#urls": "https://pic.nijie.net/03/nijie/13/98/498/illust/0_0_703023d18ca8d058_bca943.jpg", + }, + { + "#url": "https://nijie.info/view.php?id=385585", + "#comment": "video (#5707)", + "#category": ("Nijie", "nijie", "image"), + "#class": nijie.NijieImageExtractor, + "#urls": ( + "https://pic.nijie.net/01/nijie/20/82/59182/illust/0_0_162270ef49e2ee28_fab5ae.mp4", + "https://pic.nijie.net/04/nijie/20/82/59182/illust/385585_0_ff2d5d19129530d5_b2821e.jpg", + "https://pic.nijie.net/01/nijie/20/82/59182/illust/385585_1_7ee1a2a67bed2f84_212d67.jpg", + ), + }, + { + "#url": "https://nijie.info/view.php?id=70724", + "#category": ("Nijie", "nijie", "image"), + "#class": nijie.NijieImageExtractor, + "#count": 0, + }, + { + "#url": "https://nijie.info/view_popup.php?id=70720", + "#category": ("Nijie", "nijie", "image"), + "#class": nijie.NijieImageExtractor, + }, ) diff --git a/test/results/noop.py b/test/results/noop.py index 9355681776..b381952e8a 100644 --- a/test/results/noop.py +++ b/test/results/noop.py @@ -1,28 +1,22 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import noop - __tests__ = ( -{ - "#url" : "noop", - "#class" : noop.NoopExtractor, - "#urls" : (), - "#count" : 0, -}, - -{ - "#url" : "nop", - "#class" : noop.NoopExtractor, -}, - -{ - "#url" : "NOOP", - "#class" : noop.NoopExtractor, -}, - + { + "#url": "noop", + "#class": noop.NoopExtractor, + "#urls": (), + "#count": 0, + }, + { + "#url": "nop", + "#class": noop.NoopExtractor, + }, + { + "#url": "NOOP", + "#class": noop.NoopExtractor, + }, ) diff --git a/test/results/nozomi.py b/test/results/nozomi.py index 042c792098..e21fc54395 100644 --- a/test/results/nozomi.py +++ b/test/results/nozomi.py @@ -1,106 +1,91 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import nozomi - __tests__ = ( -{ - "#url" : "https://nozomi.la/post/3649262.html", - "#category": ("", "nozomi", "post"), - "#class" : nozomi.NozomiPostExtractor, - "#pattern" : r"https://w\.nozomi\.la/2/15/aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5cf5a358caf604bf73152\.webp", - "#sha1_url" : "e5525e717aec712843be8b88592d6406ae9e60ba", - "#sha1_content": "6d62c4a7fea50c0a89d499603c4e7a2b4b9bffa8", - - "artist" : ["hammer (sunset beach)"], - "character": ["patchouli knowledge"], - "copyright": ["touhou"], - "dataid" : r"re:aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5", - "date" : "dt:2016-07-26 02:32:03", - "extension": "webp", - "filename" : str, - "height" : 768, - "is_video" : False, - "postid" : 3649262, - "tags" : list, - "type" : "jpg", - "url" : str, - "width" : 1024, -}, - -{ - "#url" : "https://nozomi.la/post/25588032.html", - "#comment" : "multiple images per post", - "#category": ("", "nozomi", "post"), - "#class" : nozomi.NozomiPostExtractor, - "#count" : 7, - "#sha1_url" : "fb956ccedcf2cf509739d26e2609e910244aa56c", - "#sha1_metadata": "516ca5cbd0d2a46a8ce26679d6e08de5ac42184b", -}, - -{ - "#url" : "https://nozomi.la/post/130309.html", - "#comment" : "empty 'date' (#1163)", - "#category": ("", "nozomi", "post"), - "#class" : nozomi.NozomiPostExtractor, - - "date": None, -}, - -{ - "#url" : "https://nozomi.la/post/1647.html", - "#comment" : "gif", - "#category": ("", "nozomi", "post"), - "#class" : nozomi.NozomiPostExtractor, - "#pattern" : r"https://g\.nozomi\.la/a/f0/d1b06469e00d72e4f6346209c149db459d76b58a074416c260ed93cc31fa9f0a\.gif", - "#sha1_content": "952efb78252bbc9fb56df2e8fafb68d5e6364181", -}, - -{ - "#url" : "https://nozomi.la/post/2269847.html", - "#comment" : "video", - "#category": ("", "nozomi", "post"), - "#class" : nozomi.NozomiPostExtractor, - "#pattern" : r"https://v\.nozomi\.la/d/0e/ff88398862669783691b31519f2bea3a35c24b6e62e3ba2d89b4409e41c660ed\.webm", - "#sha1_content": "57065e6c16da7b1c7098a63b36fb0c6c6f1b9bca", -}, - -{ - "#url" : "https://nozomi.la/", - "#category": ("", "nozomi", "index"), - "#class" : nozomi.NozomiIndexExtractor, -}, - -{ - "#url" : "https://nozomi.la/index-2.html", - "#category": ("", "nozomi", "index"), - "#class" : nozomi.NozomiIndexExtractor, -}, - -{ - "#url" : "https://nozomi.la/index-Popular-33.html", - "#category": ("", "nozomi", "index"), - "#class" : nozomi.NozomiIndexExtractor, -}, - -{ - "#url" : "https://nozomi.la/tag/3:1_aspect_ratio-1.html", - "#category": ("", "nozomi", "tag"), - "#class" : nozomi.NozomiTagExtractor, - "#pattern" : r"^https://[wgv]\.nozomi\.la/\w/\w\w/\w+\.\w+$", - "#range" : "1-25", - "#count" : ">= 25", -}, - -{ - "#url" : "https://nozomi.la/search.html?q=hibiscus%203:4_ratio#1", - "#category": ("", "nozomi", "search"), - "#class" : nozomi.NozomiSearchExtractor, - "#count" : ">= 5", -}, - + { + "#url": "https://nozomi.la/post/3649262.html", + "#category": ("", "nozomi", "post"), + "#class": nozomi.NozomiPostExtractor, + "#pattern": r"https://w\.nozomi\.la/2/15/aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5cf5a358caf604bf73152\.webp", + "#sha1_url": "e5525e717aec712843be8b88592d6406ae9e60ba", + "#sha1_content": "6d62c4a7fea50c0a89d499603c4e7a2b4b9bffa8", + "artist": ["hammer (sunset beach)"], + "character": ["patchouli knowledge"], + "copyright": ["touhou"], + "dataid": r"re:aaa9f7c632cde1e1a5baaff3fb6a6d857ec73df7fdc5", + "date": "dt:2016-07-26 02:32:03", + "extension": "webp", + "filename": str, + "height": 768, + "is_video": False, + "postid": 3649262, + "tags": list, + "type": "jpg", + "url": str, + "width": 1024, + }, + { + "#url": "https://nozomi.la/post/25588032.html", + "#comment": "multiple images per post", + "#category": ("", "nozomi", "post"), + "#class": nozomi.NozomiPostExtractor, + "#count": 7, + "#sha1_url": "fb956ccedcf2cf509739d26e2609e910244aa56c", + "#sha1_metadata": "516ca5cbd0d2a46a8ce26679d6e08de5ac42184b", + }, + { + "#url": "https://nozomi.la/post/130309.html", + "#comment": "empty 'date' (#1163)", + "#category": ("", "nozomi", "post"), + "#class": nozomi.NozomiPostExtractor, + "date": None, + }, + { + "#url": "https://nozomi.la/post/1647.html", + "#comment": "gif", + "#category": ("", "nozomi", "post"), + "#class": nozomi.NozomiPostExtractor, + "#pattern": r"https://g\.nozomi\.la/a/f0/d1b06469e00d72e4f6346209c149db459d76b58a074416c260ed93cc31fa9f0a\.gif", + "#sha1_content": "952efb78252bbc9fb56df2e8fafb68d5e6364181", + }, + { + "#url": "https://nozomi.la/post/2269847.html", + "#comment": "video", + "#category": ("", "nozomi", "post"), + "#class": nozomi.NozomiPostExtractor, + "#pattern": r"https://v\.nozomi\.la/d/0e/ff88398862669783691b31519f2bea3a35c24b6e62e3ba2d89b4409e41c660ed\.webm", + "#sha1_content": "57065e6c16da7b1c7098a63b36fb0c6c6f1b9bca", + }, + { + "#url": "https://nozomi.la/", + "#category": ("", "nozomi", "index"), + "#class": nozomi.NozomiIndexExtractor, + }, + { + "#url": "https://nozomi.la/index-2.html", + "#category": ("", "nozomi", "index"), + "#class": nozomi.NozomiIndexExtractor, + }, + { + "#url": "https://nozomi.la/index-Popular-33.html", + "#category": ("", "nozomi", "index"), + "#class": nozomi.NozomiIndexExtractor, + }, + { + "#url": "https://nozomi.la/tag/3:1_aspect_ratio-1.html", + "#category": ("", "nozomi", "tag"), + "#class": nozomi.NozomiTagExtractor, + "#pattern": r"^https://[wgv]\.nozomi\.la/\w/\w\w/\w+\.\w+$", + "#range": "1-25", + "#count": ">= 25", + }, + { + "#url": "https://nozomi.la/search.html?q=hibiscus%203:4_ratio#1", + "#category": ("", "nozomi", "search"), + "#class": nozomi.NozomiSearchExtractor, + "#count": ">= 5", + }, ) diff --git a/test/results/nsfwalbum.py b/test/results/nsfwalbum.py index cabd3e104c..d4e88709ac 100644 --- a/test/results/nsfwalbum.py +++ b/test/results/nsfwalbum.py @@ -1,36 +1,31 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import nsfwalbum - __tests__ = ( -{ - "#url" : "https://nsfwalbum.com/album/401611", - "#category": ("", "nsfwalbum", "album"), - "#class" : nsfwalbum.NsfwalbumAlbumExtractor, - "#range" : "1-5", - "#urls" : ( - "https://img70.imgspice.com/i/05457/mio2bu5xbrxe.jpg", - "https://img70.imgspice.com/i/05457/zgpxa8kr4h1d.jpg", - "https://img70.imgspice.com/i/05457/3379nxsm9lx8.jpg", - "https://img70.imgspice.com/i/05457/pncrkhspuoa3.jpg", - "https://img70.imgspice.com/i/05457/128b2odt216a.jpg", - ), - - "album_id" : 401611, - "extension": "jpg", - "filename" : str, - "height" : range(1365, 2048), - "id" : int, - "models" : [], - "num" : range(1, 5), - "studio" : "Met-Art", - "title" : "Met-Art - Katherine A - Difuza 25.05.2014 (134 photos)(4368 X 2912)", - "width" : range(1365, 2048), -}, - + { + "#url": "https://nsfwalbum.com/album/401611", + "#category": ("", "nsfwalbum", "album"), + "#class": nsfwalbum.NsfwalbumAlbumExtractor, + "#range": "1-5", + "#urls": ( + "https://img70.imgspice.com/i/05457/mio2bu5xbrxe.jpg", + "https://img70.imgspice.com/i/05457/zgpxa8kr4h1d.jpg", + "https://img70.imgspice.com/i/05457/3379nxsm9lx8.jpg", + "https://img70.imgspice.com/i/05457/pncrkhspuoa3.jpg", + "https://img70.imgspice.com/i/05457/128b2odt216a.jpg", + ), + "album_id": 401611, + "extension": "jpg", + "filename": str, + "height": range(1365, 2048), + "id": int, + "models": [], + "num": range(1, 5), + "studio": "Met-Art", + "title": "Met-Art - Katherine A - Difuza 25.05.2014 (134 photos)(4368 X 2912)", + "width": range(1365, 2048), + }, ) diff --git a/test/results/ohpolly.py b/test/results/ohpolly.py index 3ce1968b9c..2430908cca 100644 --- a/test/results/ohpolly.py +++ b/test/results/ohpolly.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://www.ohpolly.com/collections/dresses-mini-dresses", - "#category": ("shopify", "ohpolly", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.ohpolly.com/products/edonia-ruched-triangle-cup-a-line-mini-dress-brown", - "#category": ("shopify", "ohpolly", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://www.ohpolly.com/collections/dresses-mini-dresses", + "#category": ("shopify", "ohpolly", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.ohpolly.com/products/edonia-ruched-triangle-cup-a-line-mini-dress-brown", + "#category": ("shopify", "ohpolly", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/omgmiamiswimwear.py b/test/results/omgmiamiswimwear.py index e92228da3d..7deaf0fbcc 100644 --- a/test/results/omgmiamiswimwear.py +++ b/test/results/omgmiamiswimwear.py @@ -1,25 +1,20 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://www.omgmiamiswimwear.com/collections/fajas", - "#category": ("shopify", "omgmiamiswimwear", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.omgmiamiswimwear.com/products/snatch-me-waist-belt", - "#category": ("shopify", "omgmiamiswimwear", "product"), - "#class" : shopify.ShopifyProductExtractor, - "#pattern" : r"https://cdn\.shopify\.com/s/files/1/1819/6171/", - "#count" : 3, -}, - + { + "#url": "https://www.omgmiamiswimwear.com/collections/fajas", + "#category": ("shopify", "omgmiamiswimwear", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.omgmiamiswimwear.com/products/snatch-me-waist-belt", + "#category": ("shopify", "omgmiamiswimwear", "product"), + "#class": shopify.ShopifyProductExtractor, + "#pattern": r"https://cdn\.shopify\.com/s/files/1/1819/6171/", + "#count": 3, + }, ) diff --git a/test/results/paheal.py b/test/results/paheal.py index 0d8262978d..5f0f1e5c8c 100644 --- a/test/results/paheal.py +++ b/test/results/paheal.py @@ -1,110 +1,96 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import paheal - __tests__ = ( -{ - "#url" : "https://rule34.paheal.net/post/list/Ayane_Suzuki/1", - "#category": ("shimmie2", "paheal", "tag"), - "#class" : paheal.PahealTagExtractor, - "#pattern" : r"https://[^.]+\.paheal\.net/_images/\w+/\d+%20-%20|https://r34i\.paheal-cdn\.net/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}$", - "#count" : range(70, 200), - - "date" : "type:datetime", - "extension": r"re:jpg|png", - "filename" : r"re:\d+ - \w+", - "duration" : float, - "height" : int, - "id" : int, - "md5" : r"re:[0-9a-f]{32}", - "search_tags": "Ayane_Suzuki", - "size" : int, - "tags" : str, - "width" : int, - -}, - -{ - "#url" : "https://rule34.paheal.net/post/list/Ayane_Suzuki/1", - "#category": ("shimmie2", "paheal", "tag"), - "#class" : paheal.PahealTagExtractor, - "#options" : {"metadata": True}, - "#range" : "1", - - "date" : "dt:2018-01-07 07:04:05", - "duration" : 0.0, - "extension" : "jpg", - "filename" : "2446128 - Ayane_Suzuki Idolmaster idolmaster_dearly_stars Zanzi", - "height" : 768, - "id" : 2446128, - "md5" : "b0ceda9d860df1d15b60293a7eb465c1", - "search_tags": "Ayane_Suzuki", - "size" : 204800, - "source" : "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=19957280", - "tags" : "Ayane_Suzuki Idolmaster idolmaster_dearly_stars Zanzi", - "uploader" : "XXXname", - "width" : 1024, -}, - -{ - "#url" : "https://rule34.paheal.net/post/view/481609", - "#category": ("shimmie2", "paheal", "post"), - "#class" : paheal.PahealPostExtractor, - "#urls" : "https://r34i.paheal-cdn.net/bb/dc/bbdc1c33410c2cdce7556c7990be26b7", - "#sha1_content": "7b924bcf150b352ac75c9d281d061e174c851a11", - - "date" : "dt:2010-06-17 15:40:23", - "extension": "jpg", - "file_url" : "https://r34i.paheal-cdn.net/bb/dc/bbdc1c33410c2cdce7556c7990be26b7", - "filename" : "481609 - Ayumu_Kasuga Azumanga_Daioh inanimate Vuvuzela", - "height" : 660, - "id" : 481609, - "md5" : "bbdc1c33410c2cdce7556c7990be26b7", - "size" : 157696, - "source" : "", - "tags" : "Ayumu_Kasuga Azumanga_Daioh inanimate Vuvuzela", - "uploader" : "CaptainButtface", - "width" : 614, -}, - -{ - "#url" : "https://rule34.paheal.net/post/view/488534", - "#category": ("shimmie2", "paheal", "post"), - "#class" : paheal.PahealPostExtractor, - - "date" : "dt:2010-06-25 13:51:17", - "height" : 800, - "md5" : "b39edfe455a0381110c710d6ed2ef57d", - "size" : 758784, - "source" : "http://www.furaffinity.net/view/4057821/", - "tags" : "inanimate thelost-dragon Vuvuzela", - "uploader": "leacheate_soup", - "width" : 1200, -}, - -{ - "#url" : "https://rule34.paheal.net/post/view/3864982", - "#comment" : "video", - "#category": ("shimmie2", "paheal", "post"), - "#class" : paheal.PahealPostExtractor, - "#urls" : "https://r34i.paheal-cdn.net/76/29/7629fc0ff77e32637dde5bf4f992b2cb", - - "date" : "dt:2020-09-06 01:59:03", - "duration" : 30.0, - "extension": "webm", - "height" : 2500, - "id" : 3864982, - "md5" : "7629fc0ff77e32637dde5bf4f992b2cb", - "size" : 18874368, - "source" : "https://twitter.com/VG_Worklog/status/1302407696294055936", - "tags" : "animated Metal_Gear Metal_Gear_Solid_V Quiet Vg_erotica webm", - "uploader" : "justausername", - "width" : 1768, -}, - + { + "#url": "https://rule34.paheal.net/post/list/Ayane_Suzuki/1", + "#category": ("shimmie2", "paheal", "tag"), + "#class": paheal.PahealTagExtractor, + "#pattern": r"https://[^.]+\.paheal\.net/_images/\w+/\d+%20-%20|https://r34i\.paheal-cdn\.net/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}$", + "#count": range(70, 200), + "date": "type:datetime", + "extension": r"re:jpg|png", + "filename": r"re:\d+ - \w+", + "duration": float, + "height": int, + "id": int, + "md5": r"re:[0-9a-f]{32}", + "search_tags": "Ayane_Suzuki", + "size": int, + "tags": str, + "width": int, + }, + { + "#url": "https://rule34.paheal.net/post/list/Ayane_Suzuki/1", + "#category": ("shimmie2", "paheal", "tag"), + "#class": paheal.PahealTagExtractor, + "#options": {"metadata": True}, + "#range": "1", + "date": "dt:2018-01-07 07:04:05", + "duration": 0.0, + "extension": "jpg", + "filename": "2446128 - Ayane_Suzuki Idolmaster idolmaster_dearly_stars Zanzi", + "height": 768, + "id": 2446128, + "md5": "b0ceda9d860df1d15b60293a7eb465c1", + "search_tags": "Ayane_Suzuki", + "size": 204800, + "source": "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=19957280", + "tags": "Ayane_Suzuki Idolmaster idolmaster_dearly_stars Zanzi", + "uploader": "XXXname", + "width": 1024, + }, + { + "#url": "https://rule34.paheal.net/post/view/481609", + "#category": ("shimmie2", "paheal", "post"), + "#class": paheal.PahealPostExtractor, + "#urls": "https://r34i.paheal-cdn.net/bb/dc/bbdc1c33410c2cdce7556c7990be26b7", + "#sha1_content": "7b924bcf150b352ac75c9d281d061e174c851a11", + "date": "dt:2010-06-17 15:40:23", + "extension": "jpg", + "file_url": "https://r34i.paheal-cdn.net/bb/dc/bbdc1c33410c2cdce7556c7990be26b7", + "filename": "481609 - Ayumu_Kasuga Azumanga_Daioh inanimate Vuvuzela", + "height": 660, + "id": 481609, + "md5": "bbdc1c33410c2cdce7556c7990be26b7", + "size": 157696, + "source": "", + "tags": "Ayumu_Kasuga Azumanga_Daioh inanimate Vuvuzela", + "uploader": "CaptainButtface", + "width": 614, + }, + { + "#url": "https://rule34.paheal.net/post/view/488534", + "#category": ("shimmie2", "paheal", "post"), + "#class": paheal.PahealPostExtractor, + "date": "dt:2010-06-25 13:51:17", + "height": 800, + "md5": "b39edfe455a0381110c710d6ed2ef57d", + "size": 758784, + "source": "http://www.furaffinity.net/view/4057821/", + "tags": "inanimate thelost-dragon Vuvuzela", + "uploader": "leacheate_soup", + "width": 1200, + }, + { + "#url": "https://rule34.paheal.net/post/view/3864982", + "#comment": "video", + "#category": ("shimmie2", "paheal", "post"), + "#class": paheal.PahealPostExtractor, + "#urls": "https://r34i.paheal-cdn.net/76/29/7629fc0ff77e32637dde5bf4f992b2cb", + "date": "dt:2020-09-06 01:59:03", + "duration": 30.0, + "extension": "webm", + "height": 2500, + "id": 3864982, + "md5": "7629fc0ff77e32637dde5bf4f992b2cb", + "size": 18874368, + "source": "https://twitter.com/VG_Worklog/status/1302407696294055936", + "tags": "animated Metal_Gear Metal_Gear_Solid_V Quiet Vg_erotica webm", + "uploader": "justausername", + "width": 1768, + }, ) diff --git a/test/results/palanq.py b/test/results/palanq.py index 046e6e8823..eb6c9acefe 100644 --- a/test/results/palanq.py +++ b/test/results/palanq.py @@ -1,36 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://archive.palanq.win/c/thread/4209598/", - "#category": ("foolfuuka", "palanq", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "1f9b5570d228f1f2991c827a6631030bc0e5933c", -}, - -{ - "#url" : "https://archive.palanq.win/c/", - "#category": ("foolfuuka", "palanq", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://archive.palanq.win/_/search/text/test/", - "#category": ("foolfuuka", "palanq", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://archive.palanq.win/c/gallery", - "#category": ("foolfuuka", "palanq", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://archive.palanq.win/c/thread/4209598/", + "#category": ("foolfuuka", "palanq", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "1f9b5570d228f1f2991c827a6631030bc0e5933c", + }, + { + "#url": "https://archive.palanq.win/c/", + "#category": ("foolfuuka", "palanq", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://archive.palanq.win/_/search/text/test/", + "#category": ("foolfuuka", "palanq", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://archive.palanq.win/c/gallery", + "#category": ("foolfuuka", "palanq", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/patreon.py b/test/results/patreon.py index f8cdaf10f9..d980757400 100644 --- a/test/results/patreon.py +++ b/test/results/patreon.py @@ -1,124 +1,104 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import patreon import datetime -from gallery_dl import exception +from gallery_dl import exception +from gallery_dl.extractor import patreon __tests__ = ( -{ - "#url" : "https://www.patreon.com/koveliana", - "#class" : patreon.PatreonCreatorExtractor, - "#range" : "1-15", - "#count" : 15, - - "attachments" : list, - "comment_count": int, - "content" : str, - "creator" : dict, - "date" : datetime.datetime, - "id" : int, - "images" : list, - "like_count" : int, - "post_type" : str, - "published_at" : str, - "title" : str, -}, - -{ - "#url" : "https://www.patreon.com/koveliana/posts?filters[month]=2020-3", - "#class" : patreon.PatreonCreatorExtractor, - "#count" : 1, - - "date": "dt:2020-03-30 21:21:44", -}, - -{ - "#url" : "https://www.patreon.com/kovelianot", - "#class" : patreon.PatreonCreatorExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.patreon.com/c/koveliana", - "#class" : patreon.PatreonCreatorExtractor, -}, - -{ - "#url" : "https://www.patreon.com/user?u=2931440", - "#class" : patreon.PatreonCreatorExtractor, -}, - -{ - "#url" : "https://www.patreon.com/user/posts/?u=2931440", - "#class" : patreon.PatreonCreatorExtractor, -}, - -{ - "#url" : "https://www.patreon.com/user?c=369707", - "#class" : patreon.PatreonCreatorExtractor, -}, - -{ - "#url" : "https://www.patreon.com/id:369707", - "#class" : patreon.PatreonCreatorExtractor, -}, - -{ - "#url" : "https://www.patreon.com/home", - "#class" : patreon.PatreonUserExtractor, -}, - -{ - "#url" : "https://www.patreon.com/posts/precious-metal-23563293", - "#comment" : "postfile + attachments", - "#class" : patreon.PatreonPostExtractor, - "#count" : 4, -}, - -{ - "#url" : "https://www.patreon.com/posts/free-mari-8s-113049301", - "#comment" : "postfile + attachments_media (#6241)", - "#class" : patreon.PatreonPostExtractor, - "#pattern" : [ - r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/113049301/7ae4fd78d3374d849a80863f3d8eee89/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.jpg", - r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/113049301/b6ea96b18cbc47f78f9334d50d0877ea/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.mp4", - r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/113049301/62dc1d4194db4245aca31c56f71234ed/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.mp4", - ], -}, - -{ - "#url" : "https://www.patreon.com/posts/56127163", - "#comment" : "account suspended", - "#class" : patreon.PatreonPostExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.patreon.com/posts/free-post-12497641", - "#comment" : "tags (#1539)", - "#class" : patreon.PatreonPostExtractor, - - "tags": ["AWMedia"], -}, - -{ - "#url" : "https://www.patreon.com/posts/m3u8-94714289", - "#class" : patreon.PatreonPostExtractor, - "#pattern" : [ - r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/94714289/be3d8eb994ae44eca4baffcdc6dd25fc/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.png", - r"ytdl:https://www.patreon\.com/api/video/255859412/video\.m3u8", - ] -}, - -{ - "#url" : "https://www.patreon.com/posts/not-found-123", - "#class" : patreon.PatreonPostExtractor, - "#exception": exception.NotFoundError, -}, - + { + "#url": "https://www.patreon.com/koveliana", + "#class": patreon.PatreonCreatorExtractor, + "#range": "1-15", + "#count": 15, + "attachments": list, + "comment_count": int, + "content": str, + "creator": dict, + "date": datetime.datetime, + "id": int, + "images": list, + "like_count": int, + "post_type": str, + "published_at": str, + "title": str, + }, + { + "#url": "https://www.patreon.com/koveliana/posts?filters[month]=2020-3", + "#class": patreon.PatreonCreatorExtractor, + "#count": 1, + "date": "dt:2020-03-30 21:21:44", + }, + { + "#url": "https://www.patreon.com/kovelianot", + "#class": patreon.PatreonCreatorExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.patreon.com/c/koveliana", + "#class": patreon.PatreonCreatorExtractor, + }, + { + "#url": "https://www.patreon.com/user?u=2931440", + "#class": patreon.PatreonCreatorExtractor, + }, + { + "#url": "https://www.patreon.com/user/posts/?u=2931440", + "#class": patreon.PatreonCreatorExtractor, + }, + { + "#url": "https://www.patreon.com/user?c=369707", + "#class": patreon.PatreonCreatorExtractor, + }, + { + "#url": "https://www.patreon.com/id:369707", + "#class": patreon.PatreonCreatorExtractor, + }, + { + "#url": "https://www.patreon.com/home", + "#class": patreon.PatreonUserExtractor, + }, + { + "#url": "https://www.patreon.com/posts/precious-metal-23563293", + "#comment": "postfile + attachments", + "#class": patreon.PatreonPostExtractor, + "#count": 4, + }, + { + "#url": "https://www.patreon.com/posts/free-mari-8s-113049301", + "#comment": "postfile + attachments_media (#6241)", + "#class": patreon.PatreonPostExtractor, + "#pattern": [ + r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/113049301/7ae4fd78d3374d849a80863f3d8eee89/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.jpg", + r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/113049301/b6ea96b18cbc47f78f9334d50d0877ea/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.mp4", + r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/113049301/62dc1d4194db4245aca31c56f71234ed/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.mp4", + ], + }, + { + "#url": "https://www.patreon.com/posts/56127163", + "#comment": "account suspended", + "#class": patreon.PatreonPostExtractor, + "#count": 0, + }, + { + "#url": "https://www.patreon.com/posts/free-post-12497641", + "#comment": "tags (#1539)", + "#class": patreon.PatreonPostExtractor, + "tags": ["AWMedia"], + }, + { + "#url": "https://www.patreon.com/posts/m3u8-94714289", + "#class": patreon.PatreonPostExtractor, + "#pattern": [ + r"https://c10\.patreonusercontent\.com/4/patreon-media/p/post/94714289/be3d8eb994ae44eca4baffcdc6dd25fc/eyJhIjoxLCJwIjoxfQ%3D%3D/1\.png", + r"ytdl:https://www.patreon\.com/api/video/255859412/video\.m3u8", + ], + }, + { + "#url": "https://www.patreon.com/posts/not-found-123", + "#class": patreon.PatreonPostExtractor, + "#exception": exception.NotFoundError, + }, ) diff --git a/test/results/pawoo.py b/test/results/pawoo.py index 5a9bfcaa2d..18857a9833 100644 --- a/test/results/pawoo.py +++ b/test/results/pawoo.py @@ -1,38 +1,31 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import mastodon - __tests__ = ( -{ - "#url" : "https://pawoo.net/@yoru_nine/", - "#category": ("mastodon", "pawoo", "user"), - "#class" : mastodon.MastodonUserExtractor, - "#range" : "1-60", - "#count" : 60, -}, - -{ - "#url" : "https://pawoo.net/bookmarks", - "#category": ("mastodon", "pawoo", "bookmark"), - "#class" : mastodon.MastodonBookmarkExtractor, -}, - -{ - "#url" : "https://pawoo.net/users/yoru_nine/following", - "#category": ("mastodon", "pawoo", "following"), - "#class" : mastodon.MastodonFollowingExtractor, -}, - -{ - "#url" : "https://pawoo.net/@yoru_nine/105038878897832922", - "#category": ("mastodon", "pawoo", "status"), - "#class" : mastodon.MastodonStatusExtractor, - "#sha1_content": "b52e807f8ab548d6f896b09218ece01eba83987a", -}, - + { + "#url": "https://pawoo.net/@yoru_nine/", + "#category": ("mastodon", "pawoo", "user"), + "#class": mastodon.MastodonUserExtractor, + "#range": "1-60", + "#count": 60, + }, + { + "#url": "https://pawoo.net/bookmarks", + "#category": ("mastodon", "pawoo", "bookmark"), + "#class": mastodon.MastodonBookmarkExtractor, + }, + { + "#url": "https://pawoo.net/users/yoru_nine/following", + "#category": ("mastodon", "pawoo", "following"), + "#class": mastodon.MastodonFollowingExtractor, + }, + { + "#url": "https://pawoo.net/@yoru_nine/105038878897832922", + "#category": ("mastodon", "pawoo", "status"), + "#class": mastodon.MastodonStatusExtractor, + "#sha1_content": "b52e807f8ab548d6f896b09218ece01eba83987a", + }, ) diff --git a/test/results/photovogue.py b/test/results/photovogue.py index 6898f5d9e4..902d358b25 100644 --- a/test/results/photovogue.py +++ b/test/results/photovogue.py @@ -1,50 +1,45 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import photovogue import datetime +from gallery_dl.extractor import photovogue __tests__ = ( -{ - "#url" : "https://www.vogue.com/photovogue/photographers/221252", - "#category": ("", "photovogue", "user"), - "#class" : photovogue.PhotovogueUserExtractor, -}, - -{ - "#url" : "https://vogue.com/photovogue/photographers/221252", - "#category": ("", "photovogue", "user"), - "#class" : photovogue.PhotovogueUserExtractor, - "#pattern" : "https://images.vogue.it/Photovogue/[^/]+_gallery.jpg", - - "date" : datetime.datetime, - "favorite_count" : int, - "favorited" : list, - "id" : int, - "image_id" : str, - "is_favorite" : False, - "orientation" : r"re:portrait|landscape", - "photographer" : { - "biography" : "Born in 1995. Live in Bologna.", - "city" : "Bologna", - "country_id" : 106, - "favoritedCount": int, - "id" : 221252, - "isGold" : bool, - "isPro" : bool, - "latitude" : str, - "longitude" : str, - "name" : "Arianna Mattarozzi", - "user_id" : "38cb0601-4a85-453c-b7dc-7650a037f2ab", - "websites" : list, + { + "#url": "https://www.vogue.com/photovogue/photographers/221252", + "#category": ("", "photovogue", "user"), + "#class": photovogue.PhotovogueUserExtractor, + }, + { + "#url": "https://vogue.com/photovogue/photographers/221252", + "#category": ("", "photovogue", "user"), + "#class": photovogue.PhotovogueUserExtractor, + "#pattern": "https://images.vogue.it/Photovogue/[^/]+_gallery.jpg", + "date": datetime.datetime, + "favorite_count": int, + "favorited": list, + "id": int, + "image_id": str, + "is_favorite": False, + "orientation": r"re:portrait|landscape", + "photographer": { + "biography": "Born in 1995. Live in Bologna.", + "city": "Bologna", + "country_id": 106, + "favoritedCount": int, + "id": 221252, + "isGold": bool, + "isPro": bool, + "latitude": str, + "longitude": str, + "name": "Arianna Mattarozzi", + "user_id": "38cb0601-4a85-453c-b7dc-7650a037f2ab", + "websites": list, + }, + "photographer_id": 221252, + "tags": list, + "title": str, }, - "photographer_id": 221252, - "tags" : list, - "title" : str, -}, - ) diff --git a/test/results/picarto.py b/test/results/picarto.py index 0745626019..0a994ab6b8 100644 --- a/test/results/picarto.py +++ b/test/results/picarto.py @@ -1,22 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import picarto import datetime +from gallery_dl.extractor import picarto __tests__ = ( -{ - "#url" : "https://picarto.tv/fnook/gallery/default/", - "#category": ("", "picarto", "gallery"), - "#class" : picarto.PicartoGalleryExtractor, - "#pattern" : r"https://images\.picarto\.tv/gallery/\d/\d\d/\d+/artwork/[0-9a-f-]+/large-[0-9a-f]+\.(jpg|png|gif)", - "#count" : ">= 7", - - "date": datetime.datetime, -}, - + { + "#url": "https://picarto.tv/fnook/gallery/default/", + "#category": ("", "picarto", "gallery"), + "#class": picarto.PicartoGalleryExtractor, + "#pattern": r"https://images\.picarto\.tv/gallery/\d/\d\d/\d+/artwork/[0-9a-f-]+/large-[0-9a-f]+\.(jpg|png|gif)", + "#count": ">= 7", + "date": datetime.datetime, + }, ) diff --git a/test/results/piczel.py b/test/results/piczel.py index 84d1d01b82..20f7b3c12d 100644 --- a/test/results/piczel.py +++ b/test/results/piczel.py @@ -1,57 +1,50 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import piczel - __tests__ = ( -{ - "#url" : "https://piczel.tv/gallery/Bikupan", - "#category": ("", "piczel", "user"), - "#class" : piczel.PiczelUserExtractor, - "#range" : "1-100", - "#count" : ">= 100", -}, - -{ - "#url" : "https://piczel.tv/gallery/Lulena/1114", - "#category": ("", "piczel", "folder"), - "#class" : piczel.PiczelFolderExtractor, - "#count" : ">= 4", -}, - -{ - "#url" : "https://piczel.tv/gallery/image/7807", - "#category": ("", "piczel", "image"), - "#class" : piczel.PiczelImageExtractor, - "#pattern" : r"https://(\w+\.)?piczel\.tv/static/uploads/gallery_image/32920/image/7807/1532236438-Lulena\.png", - "#sha1_content": "df9a053a24234474a19bce2b7e27e0dec23bff87", - - "created_at" : "2018-07-22T05:13:58.000Z", - "date" : "dt:2018-07-22 05:13:58", - "description" : None, - "extension" : "png", - "favorites_count" : int, - "folder_id" : 1113, - "id" : 7807, - "is_flash" : False, - "is_video" : False, - "multi" : False, - "nsfw" : False, - "num" : 0, - "password_protected": False, - "tags" : [ - "fanart", - "commission", - "altair", - "recreators", - ], - "title" : "Altair", - "user" : dict, - "views" : int, -}, - + { + "#url": "https://piczel.tv/gallery/Bikupan", + "#category": ("", "piczel", "user"), + "#class": piczel.PiczelUserExtractor, + "#range": "1-100", + "#count": ">= 100", + }, + { + "#url": "https://piczel.tv/gallery/Lulena/1114", + "#category": ("", "piczel", "folder"), + "#class": piczel.PiczelFolderExtractor, + "#count": ">= 4", + }, + { + "#url": "https://piczel.tv/gallery/image/7807", + "#category": ("", "piczel", "image"), + "#class": piczel.PiczelImageExtractor, + "#pattern": r"https://(\w+\.)?piczel\.tv/static/uploads/gallery_image/32920/image/7807/1532236438-Lulena\.png", + "#sha1_content": "df9a053a24234474a19bce2b7e27e0dec23bff87", + "created_at": "2018-07-22T05:13:58.000Z", + "date": "dt:2018-07-22 05:13:58", + "description": None, + "extension": "png", + "favorites_count": int, + "folder_id": 1113, + "id": 7807, + "is_flash": False, + "is_video": False, + "multi": False, + "nsfw": False, + "num": 0, + "password_protected": False, + "tags": [ + "fanart", + "commission", + "altair", + "recreators", + ], + "title": "Altair", + "user": dict, + "views": int, + }, ) diff --git a/test/results/pidgiwiki.py b/test/results/pidgiwiki.py index fc837d4c70..7738d26754 100644 --- a/test/results/pidgiwiki.py +++ b/test/results/pidgiwiki.py @@ -1,24 +1,19 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.pidgi.net/wiki/File:Key_art_-_Fight_Knight.png", - "#category": ("wikimedia", "pidgiwiki", "file"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#urls" : "https://cdn.pidgi.net/images/0/0c/Key_art_-_Fight_Knight.png", -}, - -{ - "#url" : "https://pidgi.net/wiki/File:Key_art_-_Fight_Knight.png", - "#category": ("wikimedia", "pidgiwiki", "file"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.pidgi.net/wiki/File:Key_art_-_Fight_Knight.png", + "#category": ("wikimedia", "pidgiwiki", "file"), + "#class": wikimedia.WikimediaArticleExtractor, + "#urls": "https://cdn.pidgi.net/images/0/0c/Key_art_-_Fight_Knight.png", + }, + { + "#url": "https://pidgi.net/wiki/File:Key_art_-_Fight_Knight.png", + "#category": ("wikimedia", "pidgiwiki", "file"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/pillowfort.py b/test/results/pillowfort.py index 514697be74..485bbfc885 100644 --- a/test/results/pillowfort.py +++ b/test/results/pillowfort.py @@ -1,185 +1,175 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import pillowfort import datetime +from gallery_dl.extractor import pillowfort __tests__ = ( -{ - "#url" : "https://www.pillowfort.social/posts/27510", - "#category": ("", "pillowfort", "post"), - "#class" : pillowfort.PillowfortPostExtractor, - "#pattern" : r"https://img\d+\.pillowfort\.social/posts/\w+_out\d+\.png", - "#count" : 4, - - "avatar_url" : str, - "col" : 0, - "commentable" : True, - "comments_count" : int, - "community_id" : None, - "content" : str, - "count" : 4, - "created_at" : str, - "date" : datetime.datetime, - "deleted" : None, - "deleted_at" : None, - "deleted_by_mod" : None, - "deleted_for_flag_id": None, - "embed_code" : None, - "id" : int, - "last_activity" : str, - "last_activity_elapsed": str, - "last_edited_at" : str, - "likes_count" : int, - "media_type" : "picture", - "nsfw" : False, - "num" : range(1, 4), - "original_post_id": None, - "original_post_user_id": None, - "picture_content_type": None, - "picture_file_name": None, - "picture_file_size": None, - "picture_updated_at": None, - "post_id" : 27510, - "post_type" : "picture", - "privacy" : "public", - "reblog_copy_info": list, - "rebloggable" : True, - "reblogged_from_post_id": None, - "reblogged_from_user_id": None, - "reblogs_count" : int, - "row" : int, - "small_image_url" : None, - "tags" : list, - "time_elapsed" : str, - "timestamp" : str, - "title" : "What is Pillowfort.social?", - "updated_at" : str, - "url" : r"re:https://img3.pillowfort.social/posts/.*\.png", - "user_id" : 5, - "username" : "Staff", -}, - -{ - "#url" : "https://www.pillowfort.social/posts/1124584", - "#comment" : "'b2_lg_url' media URL (#4570)", - "#category": ("", "pillowfort", "post"), - "#class" : pillowfort.PillowfortPostExtractor, - "#pattern" : r"https://img2\.pillowfort\.social/posts/c8e834bc09e6_Brandee\.png", - "#count" : 1, - - "avatar_frame" : None, - "avatar_id" : None, - "avatar_url" : "https://img3.pillowfort.social/avatars/000/037/139/original/437.jpg?1545015697", - "b2_lg_url" : "https://img2.pillowfort.social/posts/c8e834bc09e6_Brandee.png", - "b2_sm_url" : "https://img2.pillowfort.social/posts/c8e834bc09e6_Brandee_small.png", - "cached_tag_list": "art, digital art, mermaid, mermaids, underwater, seaweed, illustration, speed paint", - "col" : 0, - "comm_screening_status": "not_applicable", - "commentable" : True, - "comments_count": 0, - "community_id" : None, - "concealed_comment_warning": None, - "content" : "

          Sea Bed

          ", - "count" : 1, - "created_at" : r"re:2020-02-.+", - "currentuser_default_avatar_url": None, - "currentuser_multi_avi": None, - "date" : "dt:2020-02-29 17:09:03", - "deleted" : None, - "deleted_at" : None, - "deleted_by_mod": None, - "deleted_for_flag_id": None, - "embed_code" : None, - "extension" : "png", - "filename" : "Brandee", - "hash" : "c8e834bc09e6", - "id" : 720167, - "last_activity" : r"re:2020-02-.+", - "last_activity_elapsed": r"re:\d+ months", - "last_edited_at": None, - "likes_count" : 8, - "media_type" : "picture", - "nsfw" : False, - "num" : 1, - "original_post_id": None, - "original_post_user_id": None, - "pic_row_last" : 1, - "picture_content_type": None, - "picture_file_name": None, - "picture_file_size": None, - "picture_updated_at": None, - "post_id" : 1124584, - "post_type" : "picture", - "privacy" : "public", - "reblog_copy_info": [], - "rebloggable" : True, - "reblogged_from_post_id": None, - "reblogged_from_user_id": None, - "reblogs_count" : int, - "row" : 1, - "small_image_url": None, - "tag_list" : None, - "tags" : [ - "art", - "digital art", - "mermaid", - "mermaids", - "underwater", - "seaweed", - "illustration", - "speed paint", - ], - "time_elapsed" : r"re:\d+ months", - "timestamp" : str, - "title" : "", - "updated_at" : r"re:2020-02-.+", - "url" : "", - "user_concealed": None, - "user_id" : 37201, - "username" : "Maclanahan", -}, - -{ - "#url" : "https://www.pillowfort.social/posts/1557500", - "#comment" : "'external' option", - "#category": ("", "pillowfort", "post"), - "#class" : pillowfort.PillowfortPostExtractor, - "#options" : { - "external": True, - "inline" : False, + { + "#url": "https://www.pillowfort.social/posts/27510", + "#category": ("", "pillowfort", "post"), + "#class": pillowfort.PillowfortPostExtractor, + "#pattern": r"https://img\d+\.pillowfort\.social/posts/\w+_out\d+\.png", + "#count": 4, + "avatar_url": str, + "col": 0, + "commentable": True, + "comments_count": int, + "community_id": None, + "content": str, + "count": 4, + "created_at": str, + "date": datetime.datetime, + "deleted": None, + "deleted_at": None, + "deleted_by_mod": None, + "deleted_for_flag_id": None, + "embed_code": None, + "id": int, + "last_activity": str, + "last_activity_elapsed": str, + "last_edited_at": str, + "likes_count": int, + "media_type": "picture", + "nsfw": False, + "num": range(1, 4), + "original_post_id": None, + "original_post_user_id": None, + "picture_content_type": None, + "picture_file_name": None, + "picture_file_size": None, + "picture_updated_at": None, + "post_id": 27510, + "post_type": "picture", + "privacy": "public", + "reblog_copy_info": list, + "rebloggable": True, + "reblogged_from_post_id": None, + "reblogged_from_user_id": None, + "reblogs_count": int, + "row": int, + "small_image_url": None, + "tags": list, + "time_elapsed": str, + "timestamp": str, + "title": "What is Pillowfort.social?", + "updated_at": str, + "url": r"re:https://img3.pillowfort.social/posts/.*\.png", + "user_id": 5, + "username": "Staff", + }, + { + "#url": "https://www.pillowfort.social/posts/1124584", + "#comment": "'b2_lg_url' media URL (#4570)", + "#category": ("", "pillowfort", "post"), + "#class": pillowfort.PillowfortPostExtractor, + "#pattern": r"https://img2\.pillowfort\.social/posts/c8e834bc09e6_Brandee\.png", + "#count": 1, + "avatar_frame": None, + "avatar_id": None, + "avatar_url": "https://img3.pillowfort.social/avatars/000/037/139/original/437.jpg?1545015697", + "b2_lg_url": "https://img2.pillowfort.social/posts/c8e834bc09e6_Brandee.png", + "b2_sm_url": "https://img2.pillowfort.social/posts/c8e834bc09e6_Brandee_small.png", + "cached_tag_list": "art, digital art, mermaid, mermaids, underwater, seaweed, illustration, speed paint", + "col": 0, + "comm_screening_status": "not_applicable", + "commentable": True, + "comments_count": 0, + "community_id": None, + "concealed_comment_warning": None, + "content": "

          Sea Bed

          ", + "count": 1, + "created_at": r"re:2020-02-.+", + "currentuser_default_avatar_url": None, + "currentuser_multi_avi": None, + "date": "dt:2020-02-29 17:09:03", + "deleted": None, + "deleted_at": None, + "deleted_by_mod": None, + "deleted_for_flag_id": None, + "embed_code": None, + "extension": "png", + "filename": "Brandee", + "hash": "c8e834bc09e6", + "id": 720167, + "last_activity": r"re:2020-02-.+", + "last_activity_elapsed": r"re:\d+ months", + "last_edited_at": None, + "likes_count": 8, + "media_type": "picture", + "nsfw": False, + "num": 1, + "original_post_id": None, + "original_post_user_id": None, + "pic_row_last": 1, + "picture_content_type": None, + "picture_file_name": None, + "picture_file_size": None, + "picture_updated_at": None, + "post_id": 1124584, + "post_type": "picture", + "privacy": "public", + "reblog_copy_info": [], + "rebloggable": True, + "reblogged_from_post_id": None, + "reblogged_from_user_id": None, + "reblogs_count": int, + "row": 1, + "small_image_url": None, + "tag_list": None, + "tags": [ + "art", + "digital art", + "mermaid", + "mermaids", + "underwater", + "seaweed", + "illustration", + "speed paint", + ], + "time_elapsed": r"re:\d+ months", + "timestamp": str, + "title": "", + "updated_at": r"re:2020-02-.+", + "url": "", + "user_concealed": None, + "user_id": 37201, + "username": "Maclanahan", + }, + { + "#url": "https://www.pillowfort.social/posts/1557500", + "#comment": "'external' option", + "#category": ("", "pillowfort", "post"), + "#class": pillowfort.PillowfortPostExtractor, + "#options": { + "external": True, + "inline": False, + }, + "#pattern": r"https://twitter\.com/Aliciawitdaart/status/1282862493841457152", + }, + { + "#url": "https://www.pillowfort.social/posts/1672518", + "#comment": "'inline' option", + "#category": ("", "pillowfort", "post"), + "#class": pillowfort.PillowfortPostExtractor, + "#options": {"inline": True}, + "#count": 3, + }, + { + "#url": "https://www.pillowfort.social/Pome", + "#category": ("", "pillowfort", "user"), + "#class": pillowfort.PillowfortUserExtractor, + "#pattern": r"https://img\d+\.pillowfort\.social/posts/", + "#range": "1-15", + "#count": 15, + }, + { + "#url": "https://www.pillowfort.social/Staff/tagged/funding", + "#category": ("", "pillowfort", "user"), + "#class": pillowfort.PillowfortUserExtractor, + "#pattern": r"https://img\d+\.pillowfort\.social/posts/", + "#count": range(30, 50), }, - "#pattern" : r"https://twitter\.com/Aliciawitdaart/status/1282862493841457152", -}, - -{ - "#url" : "https://www.pillowfort.social/posts/1672518", - "#comment" : "'inline' option", - "#category": ("", "pillowfort", "post"), - "#class" : pillowfort.PillowfortPostExtractor, - "#options" : {"inline": True}, - "#count" : 3, -}, - -{ - "#url" : "https://www.pillowfort.social/Pome", - "#category": ("", "pillowfort", "user"), - "#class" : pillowfort.PillowfortUserExtractor, - "#pattern" : r"https://img\d+\.pillowfort\.social/posts/", - "#range" : "1-15", - "#count" : 15, -}, - -{ - "#url" : "https://www.pillowfort.social/Staff/tagged/funding", - "#category": ("", "pillowfort", "user"), - "#class" : pillowfort.PillowfortUserExtractor, - "#pattern" : r"https://img\d+\.pillowfort\.social/posts/", - "#count" : range(30, 50), -}, - ) diff --git a/test/results/pinterest.py b/test/results/pinterest.py index 3ab5ebb3a2..84408428f4 100644 --- a/test/results/pinterest.py +++ b/test/results/pinterest.py @@ -1,202 +1,174 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import pinterest from gallery_dl import exception - +from gallery_dl.extractor import pinterest __tests__ = ( -{ - "#url" : "https://www.pinterest.com/pin/858146903966145189/", - "#category": ("", "pinterest", "pin"), - "#class" : pinterest.PinterestPinExtractor, - "#sha1_url" : "afb3c26719e3a530bb0e871c480882a801a4e8a5", - "#sha1_content": [ - "4c435a66f6bb82bb681db2ecc888f76cf6c5f9ca", - "d3e24bc9f7af585e8c23b9136956bd45a4d9b947", - ], -}, - -{ - "#url" : "https://www.pinterest.com/pin/422564377542934214/", - "#comment" : "video pin (#1189)", - "#class" : pinterest.PinterestPinExtractor, - "#pattern" : r"https://v\d*\.pinimg\.com/videos/mc/hls/d7/22/ff/d722ff00ab2352981b89974b37909de8.m3u8", - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://jp.pinterest.com/pin/858146904010573850/", - "#comment" : "story pin with images", - "#class" : pinterest.PinterestPinExtractor, - "#urls" : ( - "https://i.pinimg.com/originals/0f/b0/8c/0fb08c519067dd263a1fcfecea775450.jpg", - "https://i.pinimg.com/originals/2f/27/f3/2f27f3eb781b107ce58bf588c12a12b7.jpg", - "https://i.pinimg.com/originals/55/fd/df/55fddf8d26aa0d96071af52ac6a0c25f.jpg", - ), -}, - -{ - "#url" : "https://www.pinterest.com/pin/63824519713049795/", - "#comment" : "story pin with video (#6188)", - "#class" : pinterest.PinterestPinExtractor, - "#urls" : "ytdl:https://v1.pinimg.com/videos/iht/hls/7a/b0/cc/7ab0cc56dcbfc1508b8d650af7b0a593.m3u8", - - "extension" : "mp4", - "_ytdl_manifest": "hls", -}, - -{ - "#url" : "https://www.pinterest.com/pin/606508274845593025/", - "#comment" : "story pin with audio (#6188)", - "#class" : pinterest.PinterestPinExtractor, - "#range" : "2", - "#urls" : "https://v1.pinimg.com/audios/mp3/5d/37/74/5d37749bde03855c1292f8869c8d9387.mp3", - - "extension": "mp3", -}, - -{ - "#url" : "https://jp.pinterest.com/pin/851532242064221228/", - "#comment" : "story pin with text", - "#class" : pinterest.PinterestPinExtractor, - "#range" : "2", - "#urls" : "text:Everskies character+outfits i made", -}, - -{ - "#url" : "https://www.pinterest.com/pin/858146903966145188/", - "#category": ("", "pinterest", "pin"), - "#class" : pinterest.PinterestPinExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/test-/", - "#class" : pinterest.PinterestBoardExtractor, - "#urls" : "https://i.pinimg.com/originals/d4/f4/7f/d4f47fa2fce4c4c28475af5d94972904.jpg", -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/stuff/", - "#comment" : "board with sections (#835)", - "#category": ("", "pinterest", "board"), - "#class" : pinterest.PinterestBoardExtractor, - "#options" : {"sections": True}, - "#count" : 4, -}, - -{ - "#url" : "https://www.pinterest.jp/gdldev/bname/", - "#comment" : "board & section with /?# in name (#5104)", - "#category": ("", "pinterest", "board"), - "#class" : pinterest.PinterestBoardExtractor, - "#options" : {"sections": True}, - "#urls" : "https://www.pinterest.jp/gdldev/bname/id:5345901183739414095", -}, - -{ - "#url" : "https://www.pinterest.de/g1952849/secret/", - "#comment" : "secret board (#1055)", - "#category": ("", "pinterest", "board"), - "#class" : pinterest.PinterestBoardExtractor, - "#auth" : True, - "#count" : 2, -}, - -{ - "#url" : "https://www.pinterest.com/g1952848/test/", - "#category": ("", "pinterest", "board"), - "#class" : pinterest.PinterestBoardExtractor, - "#exception": exception.GalleryDLException, -}, - -{ - "#url" : "https://www.pinterest.co.uk/hextra7519/based-animals/", - "#comment" : ".co.uk TLD (#914)", - "#category": ("", "pinterest", "board"), - "#class" : pinterest.PinterestBoardExtractor, -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/", - "#category": ("", "pinterest", "user"), - "#class" : pinterest.PinterestUserExtractor, - "#pattern" : pinterest.PinterestBoardExtractor.pattern, - "#count" : ">= 2", -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/_saved/", - "#category": ("", "pinterest", "user"), - "#class" : pinterest.PinterestUserExtractor, -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/pins/", - "#category": ("", "pinterest", "allpins"), - "#class" : pinterest.PinterestAllpinsExtractor, - "#pattern" : r"https://i\.pinimg\.com/originals/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w{3}", - "#count" : 9, -}, - -{ - "#url" : "https://www.pinterest.de/digitalmomblog/_created/", - "#category": ("", "pinterest", "created"), - "#class" : pinterest.PinterestCreatedExtractor, - "#pattern" : r"ytdl:|https://i\.pinimg\.com/originals/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.(jpg|png|webp)", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/stuff/section", - "#category": ("", "pinterest", "section"), - "#class" : pinterest.PinterestSectionExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://www.pinterest.com/search/pins/?q=nature", - "#category": ("", "pinterest", "search"), - "#class" : pinterest.PinterestSearchExtractor, - "#range" : "1-50", - "#count" : ">= 50", -}, - -{ - "#url" : "https://www.pinterest.com/pin/858146903966145189/#related", - "#category": ("", "pinterest", "related-pin"), - "#class" : pinterest.PinterestRelatedPinExtractor, - "#range" : "31-70", - "#count" : 40, - "#archive" : False, -}, - -{ - "#url" : "https://www.pinterest.com/g1952849/test-/#related", - "#category": ("", "pinterest", "related-board"), - "#class" : pinterest.PinterestRelatedBoardExtractor, - "#range" : "31-70", - "#count" : 40, - "#archive" : False, -}, - -{ - "#url" : "https://pin.it/Hvt8hgT", - "#category": ("", "pinterest", "pinit"), - "#class" : pinterest.PinterestPinitExtractor, - "#sha1_url": "8daad8558382c68f0868bdbd17d05205184632fa", -}, - -{ - "#url" : "https://pin.it/Hvt8hgS", - "#category": ("", "pinterest", "pinit"), - "#class" : pinterest.PinterestPinitExtractor, - "#exception": exception.NotFoundError, -}, - + { + "#url": "https://www.pinterest.com/pin/858146903966145189/", + "#category": ("", "pinterest", "pin"), + "#class": pinterest.PinterestPinExtractor, + "#sha1_url": "afb3c26719e3a530bb0e871c480882a801a4e8a5", + "#sha1_content": [ + "4c435a66f6bb82bb681db2ecc888f76cf6c5f9ca", + "d3e24bc9f7af585e8c23b9136956bd45a4d9b947", + ], + }, + { + "#url": "https://www.pinterest.com/pin/422564377542934214/", + "#comment": "video pin (#1189)", + "#class": pinterest.PinterestPinExtractor, + "#pattern": r"https://v\d*\.pinimg\.com/videos/mc/hls/d7/22/ff/d722ff00ab2352981b89974b37909de8.m3u8", + "#exception": exception.NotFoundError, + }, + { + "#url": "https://jp.pinterest.com/pin/858146904010573850/", + "#comment": "story pin with images", + "#class": pinterest.PinterestPinExtractor, + "#urls": ( + "https://i.pinimg.com/originals/0f/b0/8c/0fb08c519067dd263a1fcfecea775450.jpg", + "https://i.pinimg.com/originals/2f/27/f3/2f27f3eb781b107ce58bf588c12a12b7.jpg", + "https://i.pinimg.com/originals/55/fd/df/55fddf8d26aa0d96071af52ac6a0c25f.jpg", + ), + }, + { + "#url": "https://www.pinterest.com/pin/63824519713049795/", + "#comment": "story pin with video (#6188)", + "#class": pinterest.PinterestPinExtractor, + "#urls": "ytdl:https://v1.pinimg.com/videos/iht/hls/7a/b0/cc/7ab0cc56dcbfc1508b8d650af7b0a593.m3u8", + "extension": "mp4", + "_ytdl_manifest": "hls", + }, + { + "#url": "https://www.pinterest.com/pin/606508274845593025/", + "#comment": "story pin with audio (#6188)", + "#class": pinterest.PinterestPinExtractor, + "#range": "2", + "#urls": "https://v1.pinimg.com/audios/mp3/5d/37/74/5d37749bde03855c1292f8869c8d9387.mp3", + "extension": "mp3", + }, + { + "#url": "https://jp.pinterest.com/pin/851532242064221228/", + "#comment": "story pin with text", + "#class": pinterest.PinterestPinExtractor, + "#range": "2", + "#urls": "text:Everskies character+outfits i made", + }, + { + "#url": "https://www.pinterest.com/pin/858146903966145188/", + "#category": ("", "pinterest", "pin"), + "#class": pinterest.PinterestPinExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.pinterest.com/g1952849/test-/", + "#class": pinterest.PinterestBoardExtractor, + "#urls": "https://i.pinimg.com/originals/d4/f4/7f/d4f47fa2fce4c4c28475af5d94972904.jpg", + }, + { + "#url": "https://www.pinterest.com/g1952849/stuff/", + "#comment": "board with sections (#835)", + "#category": ("", "pinterest", "board"), + "#class": pinterest.PinterestBoardExtractor, + "#options": {"sections": True}, + "#count": 4, + }, + { + "#url": "https://www.pinterest.jp/gdldev/bname/", + "#comment": "board & section with /?# in name (#5104)", + "#category": ("", "pinterest", "board"), + "#class": pinterest.PinterestBoardExtractor, + "#options": {"sections": True}, + "#urls": "https://www.pinterest.jp/gdldev/bname/id:5345901183739414095", + }, + { + "#url": "https://www.pinterest.de/g1952849/secret/", + "#comment": "secret board (#1055)", + "#category": ("", "pinterest", "board"), + "#class": pinterest.PinterestBoardExtractor, + "#auth": True, + "#count": 2, + }, + { + "#url": "https://www.pinterest.com/g1952848/test/", + "#category": ("", "pinterest", "board"), + "#class": pinterest.PinterestBoardExtractor, + "#exception": exception.GalleryDLException, + }, + { + "#url": "https://www.pinterest.co.uk/hextra7519/based-animals/", + "#comment": ".co.uk TLD (#914)", + "#category": ("", "pinterest", "board"), + "#class": pinterest.PinterestBoardExtractor, + }, + { + "#url": "https://www.pinterest.com/g1952849/", + "#category": ("", "pinterest", "user"), + "#class": pinterest.PinterestUserExtractor, + "#pattern": pinterest.PinterestBoardExtractor.pattern, + "#count": ">= 2", + }, + { + "#url": "https://www.pinterest.com/g1952849/_saved/", + "#category": ("", "pinterest", "user"), + "#class": pinterest.PinterestUserExtractor, + }, + { + "#url": "https://www.pinterest.com/g1952849/pins/", + "#category": ("", "pinterest", "allpins"), + "#class": pinterest.PinterestAllpinsExtractor, + "#pattern": r"https://i\.pinimg\.com/originals/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.\w{3}", + "#count": 9, + }, + { + "#url": "https://www.pinterest.de/digitalmomblog/_created/", + "#category": ("", "pinterest", "created"), + "#class": pinterest.PinterestCreatedExtractor, + "#pattern": r"ytdl:|https://i\.pinimg\.com/originals/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{32}\.(jpg|png|webp)", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.pinterest.com/g1952849/stuff/section", + "#category": ("", "pinterest", "section"), + "#class": pinterest.PinterestSectionExtractor, + "#count": 2, + }, + { + "#url": "https://www.pinterest.com/search/pins/?q=nature", + "#category": ("", "pinterest", "search"), + "#class": pinterest.PinterestSearchExtractor, + "#range": "1-50", + "#count": ">= 50", + }, + { + "#url": "https://www.pinterest.com/pin/858146903966145189/#related", + "#category": ("", "pinterest", "related-pin"), + "#class": pinterest.PinterestRelatedPinExtractor, + "#range": "31-70", + "#count": 40, + "#archive": False, + }, + { + "#url": "https://www.pinterest.com/g1952849/test-/#related", + "#category": ("", "pinterest", "related-board"), + "#class": pinterest.PinterestRelatedBoardExtractor, + "#range": "31-70", + "#count": 40, + "#archive": False, + }, + { + "#url": "https://pin.it/Hvt8hgT", + "#category": ("", "pinterest", "pinit"), + "#class": pinterest.PinterestPinitExtractor, + "#sha1_url": "8daad8558382c68f0868bdbd17d05205184632fa", + }, + { + "#url": "https://pin.it/Hvt8hgS", + "#category": ("", "pinterest", "pinit"), + "#class": pinterest.PinterestPinitExtractor, + "#exception": exception.NotFoundError, + }, ) diff --git a/test/results/pinupgirlclothing.py b/test/results/pinupgirlclothing.py index ac82ad53ea..e93826652a 100644 --- a/test/results/pinupgirlclothing.py +++ b/test/results/pinupgirlclothing.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://pinupgirlclothing.com/collections/evening", - "#category": ("shopify", "pinupgirlclothing", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://pinupgirlclothing.com/collections/evening/products/clarice-coat-dress-in-olive-green-poly-crepe-laura-byrnes-design", - "#category": ("shopify", "pinupgirlclothing", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://pinupgirlclothing.com/collections/evening", + "#category": ("shopify", "pinupgirlclothing", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://pinupgirlclothing.com/collections/evening/products/clarice-coat-dress-in-olive-green-poly-crepe-laura-byrnes-design", + "#category": ("shopify", "pinupgirlclothing", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/pixeldrain.py b/test/results/pixeldrain.py index ed9448856f..dde71144ea 100644 --- a/test/results/pixeldrain.py +++ b/test/results/pixeldrain.py @@ -1,100 +1,92 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import pixeldrain import datetime -__tests__ = ( -{ - "#url" : "https://pixeldrain.com/u/jW9E6s4h", - "#category": ("", "pixeldrain", "file"), - "#class" : pixeldrain.PixeldrainFileExtractor, - "#urls" : "https://pixeldrain.com/api/file/jW9E6s4h?download", - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", - - "abuse_reporter_name" : "", - "abuse_type" : "", - "allow_video_player" : True, - "availability" : "", - "availability_message": "", - "bandwidth_used" : int, - "bandwidth_used_paid" : 0, - "can_download" : True, - "can_edit" : False, - "date" : "dt:2023-11-22 16:33:27", - "date_last_view" : r"re:\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z", - "date_upload" : "2023-11-22T16:33:27.744Z", - "delete_after_date" : "0001-01-01T00:00:00Z", - "delete_after_downloads": 0, - "download_speed_limit": 0, - "downloads" : int, - "extension" : "png", - "filename" : "test-テスト-\"&>", - "hash_sha256" : "eb359cd8f02a7d6762f9863798297ff6a22569c5c87a9d38c55bdb3a3e26003f", - "id" : "jW9E6s4h", - "mime_type" : "image/png", - "name" : "test-テスト-\"&>.png", - "show_ads" : True, - "size" : 182, - "success" : True, - "thumbnail_href" : "/file/jW9E6s4h/thumbnail", - "url" : "https://pixeldrain.com/api/file/jW9E6s4h?download", - "views" : int, -}, - -{ - "#url" : "https://pixeldrain.com/u/yEK1n2Qc", - "#category": ("", "pixeldrain", "file"), - "#class" : pixeldrain.PixeldrainFileExtractor, - "#urls" : "https://pixeldrain.com/api/file/yEK1n2Qc?download", - "#sha1_content": "08463261191d403de2133d829060050d8b04609f", - - "date" : "dt:2023-11-22 16:38:04", - "date_upload": "2023-11-22T16:38:04.928Z", - "extension" : "txt", - "filename" : '"&>', - "hash_sha256": "4c1e2bbcbe1dea8b6f895f5cdd8461c37c561bce4f1b3556ba58392d95964294", - "id" : "yEK1n2Qc", - "mime_type" : "text/plain; charset=utf-8", - "name" : '"&>.txt', - "size" : 14, -}, - -{ - "#url" : "https://pixeldrain.com/l/zQ7XpWfM", - "#category": ("", "pixeldrain", "album"), - "#class" : pixeldrain.PixeldrainAlbumExtractor, - "#urls" : ( - "https://pixeldrain.com/api/file/yEK1n2Qc?download", - "https://pixeldrain.com/api/file/jW9E6s4h?download", - ), +from gallery_dl.extractor import pixeldrain - "album" : { - "can_edit" : False, - "count" : 2, - "date" : "dt:2023-11-22 16:40:39", - "date_created": "2023-11-22T16:40:39.218Z", - "id" : "zQ7XpWfM", - "success" : True, - "title" : "アルバム", +__tests__ = ( + { + "#url": "https://pixeldrain.com/u/jW9E6s4h", + "#category": ("", "pixeldrain", "file"), + "#class": pixeldrain.PixeldrainFileExtractor, + "#urls": "https://pixeldrain.com/api/file/jW9E6s4h?download", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + "abuse_reporter_name": "", + "abuse_type": "", + "allow_video_player": True, + "availability": "", + "availability_message": "", + "bandwidth_used": int, + "bandwidth_used_paid": 0, + "can_download": True, + "can_edit": False, + "date": "dt:2023-11-22 16:33:27", + "date_last_view": r"re:\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z", + "date_upload": "2023-11-22T16:33:27.744Z", + "delete_after_date": "0001-01-01T00:00:00Z", + "delete_after_downloads": 0, + "download_speed_limit": 0, + "downloads": int, + "extension": "png", + "filename": 'test-テスト-"&>', + "hash_sha256": "eb359cd8f02a7d6762f9863798297ff6a22569c5c87a9d38c55bdb3a3e26003f", + "id": "jW9E6s4h", + "mime_type": "image/png", + "name": 'test-テスト-"&>.png', + "show_ads": True, + "size": 182, + "success": True, + "thumbnail_href": "/file/jW9E6s4h/thumbnail", + "url": "https://pixeldrain.com/api/file/jW9E6s4h?download", + "views": int, + }, + { + "#url": "https://pixeldrain.com/u/yEK1n2Qc", + "#category": ("", "pixeldrain", "file"), + "#class": pixeldrain.PixeldrainFileExtractor, + "#urls": "https://pixeldrain.com/api/file/yEK1n2Qc?download", + "#sha1_content": "08463261191d403de2133d829060050d8b04609f", + "date": "dt:2023-11-22 16:38:04", + "date_upload": "2023-11-22T16:38:04.928Z", + "extension": "txt", + "filename": '"&>', + "hash_sha256": "4c1e2bbcbe1dea8b6f895f5cdd8461c37c561bce4f1b3556ba58392d95964294", + "id": "yEK1n2Qc", + "mime_type": "text/plain; charset=utf-8", + "name": '"&>.txt', + "size": 14, + }, + { + "#url": "https://pixeldrain.com/l/zQ7XpWfM", + "#category": ("", "pixeldrain", "album"), + "#class": pixeldrain.PixeldrainAlbumExtractor, + "#urls": ( + "https://pixeldrain.com/api/file/yEK1n2Qc?download", + "https://pixeldrain.com/api/file/jW9E6s4h?download", + ), + "album": { + "can_edit": False, + "count": 2, + "date": "dt:2023-11-22 16:40:39", + "date_created": "2023-11-22T16:40:39.218Z", + "id": "zQ7XpWfM", + "success": True, + "title": "アルバム", + }, + "date": datetime.datetime, + "description": "", + "detail_href": r"re:/file/(yEK1n2Qc|jW9E6s4h)/info", + "hash_sha256": r"re:\w{64}", + "id": r"re:yEK1n2Qc|jW9E6s4h", + "mime_type": str, + }, + { + "#url": "https://pixeldrain.com/l/zQ7XpWfM#item=0", + "#category": ("", "pixeldrain", "album"), + "#class": pixeldrain.PixeldrainAlbumExtractor, + "#urls": "https://pixeldrain.com/api/file/jW9E6s4h?download", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", }, - "date" : datetime.datetime, - "description": "", - "detail_href": r"re:/file/(yEK1n2Qc|jW9E6s4h)/info", - "hash_sha256": r"re:\w{64}", - "id" : r"re:yEK1n2Qc|jW9E6s4h", - "mime_type" : str, -}, - -{ - "#url" : "https://pixeldrain.com/l/zQ7XpWfM#item=0", - "#category": ("", "pixeldrain", "album"), - "#class" : pixeldrain.PixeldrainAlbumExtractor, - "#urls" : "https://pixeldrain.com/api/file/jW9E6s4h?download", - "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", -}, - ) diff --git a/test/results/pixhost.py b/test/results/pixhost.py index 82fbea9854..790379a605 100644 --- a/test/results/pixhost.py +++ b/test/results/pixhost.py @@ -1,28 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://pixhost.to/show/190/130327671_test-.png", - "#category": ("imagehost", "pixhost", "image"), - "#class" : imagehosts.PixhostImageExtractor, - "#sha1_url" : "4e5470dcf6513944773044d40d883221bbc46cff", - "#sha1_metadata": "3bad6d59db42a5ebbd7842c2307e1c3ebd35e6b0", - "#sha1_content" : "0c8768055e4e20e7c7259608b67799171b691140", -}, - -{ - "#url" : "https://pixhost.to/gallery/jSMFq", - "#category": ("imagehost", "pixhost", "gallery"), - "#class" : imagehosts.PixhostGalleryExtractor, - "#pattern" : imagehosts.PixhostImageExtractor.pattern, - "#count" : 3, -}, - + { + "#url": "https://pixhost.to/show/190/130327671_test-.png", + "#category": ("imagehost", "pixhost", "image"), + "#class": imagehosts.PixhostImageExtractor, + "#sha1_url": "4e5470dcf6513944773044d40d883221bbc46cff", + "#sha1_metadata": "3bad6d59db42a5ebbd7842c2307e1c3ebd35e6b0", + "#sha1_content": "0c8768055e4e20e7c7259608b67799171b691140", + }, + { + "#url": "https://pixhost.to/gallery/jSMFq", + "#category": ("imagehost", "pixhost", "gallery"), + "#class": imagehosts.PixhostGalleryExtractor, + "#pattern": imagehosts.PixhostImageExtractor.pattern, + "#count": 3, + }, ) diff --git a/test/results/pixiv.py b/test/results/pixiv.py index d3f5bde496..b8fabc406b 100644 --- a/test/results/pixiv.py +++ b/test/results/pixiv.py @@ -1,676 +1,592 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import pixiv from gallery_dl import exception - +from gallery_dl.extractor import pixiv __tests__ = ( -{ - "#url" : "https://www.pixiv.net/en/users/173530", - "#category": ("", "pixiv", "user"), - "#class" : pixiv.PixivUserExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/u/173530", - "#category": ("", "pixiv", "user"), - "#class" : pixiv.PixivUserExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/member.php?id=173530", - "#category": ("", "pixiv", "user"), - "#class" : pixiv.PixivUserExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/mypage.php#id=173530", - "#category": ("", "pixiv", "user"), - "#class" : pixiv.PixivUserExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/#id=173530", - "#category": ("", "pixiv", "user"), - "#class" : pixiv.PixivUserExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/artworks", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, - "#sha1_url": "852c31ad83b6840bacbce824d85f2a997889efb7", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/artworks/%E6%89%8B%E3%81%B6%E3%82%8D", - "#comment" : "illusts with specific tag", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, - "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", -}, - -{ - "#url" : "https://www.pixiv.net/member_illust.php?id=173530&tag=%E6%89%8B%E3%81%B6%E3%82%8D", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, - "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", -}, - -{ - "#url" : "http://www.pixiv.net/member_illust.php?id=173531", - "#comment" : "deleted account", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, - "#options" : {"metadata": True}, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/56514424/artworks", - "#comment" : "limit_sanity_level_360.png in artworks results (#5435, #6339)", - "#class" : pixiv.PixivArtworksExtractor, - "#count" : ">= 39", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/manga", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/illustrations", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/member_illust.php?id=173530", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, -}, - -{ - "#url" : "https://touch.pixiv.net/member_illust.php?id=173530", - "#category": ("", "pixiv", "artworks"), - "#class" : pixiv.PixivArtworksExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/avatar", - "#category": ("", "pixiv", "avatar"), - "#class" : pixiv.PixivAvatarExtractor, - "#sha1_content": "4e57544480cc2036ea9608103e8f024fa737fe66", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/194921/background", - "#category": ("", "pixiv", "background"), - "#class" : pixiv.PixivBackgroundExtractor, - "#pattern" : r"https://i\.pximg\.net/background/img/2021/01/30/16/12/02/194921_af1f71e557a42f499213d4b9eaccc0f8\.jpg", -}, - -{ - "#url" : "https://pixiv.me/del_shannon", - "#category": ("", "pixiv", "me"), - "#class" : pixiv.PixivMeExtractor, - "#sha1_url": "29c295ce75150177e6b0a09089a949804c708fbf", -}, - -{ - "#url" : "https://pixiv.me/del_shanno", - "#category": ("", "pixiv", "me"), - "#class" : pixiv.PixivMeExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.pixiv.net/artworks/966412", - "#comment" : "related works (#1237)", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, - "#sha1_url" : "90c1715b07b0d1aad300bce256a0bc71f42540ba", - "#sha1_content": "69a8edfb717400d1c2e146ab2b30d2c235440c5a", - - "date" : "dt:2008-06-12 15:29:13", - "date_url": "dt:2008-06-12 15:29:13", -}, - -{ - "#url" : "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=966411", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=66806629", - "#comment" : "ugoira", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, - "#urls" : "https://i.pximg.net/img-zip-ugoira/img/2018/01/15/13/24/48/66806629_ugoira1920x1080.zip", - - "frames" : list, - "date" : "dt:2018-01-14 15:06:08", - "date_url": "dt:2018-01-15 04:24:48", -}, - -{ - "#url" : "https://www.pixiv.net/artworks/101003492", - "#comment" : "original ugoira frames (#6056)", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, - "#options" : {"ugoira": "original"}, - "#urls" : [ - "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira0.png", - "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira1.png", - "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira2.png", - "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira3.png", - "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira4.png", - "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira5.png", - ], - - "frames": list, -}, - -{ - "#url" : "https://www.pixiv.net/artworks/966412", - "#comment" : "related works (#1237)", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, - "#options" : {"related": True}, - "#range" : "1-10", - "#count" : ">= 10", -}, - -{ - "#url" : "https://www.pixiv.net/artworks/85960783", - "#comment" : "limit_sanity_level_360.png (#4327, #5180)", - "#class" : pixiv.PixivWorkExtractor, - "#options" : {"sanity": False}, - "#count" : 0, -}, - -{ - "#url" : "https://www.pixiv.net/en/artworks/102932581", - "#comment" : "limit_sanity_level_360.png (#4327, #5180)", - "#class" : pixiv.PixivWorkExtractor, - "#options" : {"sanity": True}, - "#urls" : "https://i.pximg.net/img-original/img/2022/11/20/00/00/49/102932581_p0.jpg", - - "caption" : "Meet a deer .", - "comment_access_control": 0, - "create_date" : "2022-11-19T15:00:00+00:00", - "date" : "dt:2022-11-19 15:00:00", - "date_url" : "dt:2022-11-19 15:00:49", - "extension" : "jpg", - "filename" : "102932581_p0", - "height" : 3840, - "id" : 102932581, - "illust_ai_type": 1, - "illust_book_style": 0, - "is_bookmarked" : False, - "is_muted" : False, - "num" : 0, - "page_count" : 1, - "rating" : "General", - "restrict" : 0, - "sanity_level" : 2, - "series" : None, - "suffix" : "", - "title" : "《 Bridge and Deer 》", - "tools" : [], - "total_bookmarks": range(1900, 3000), - "total_comments": range(3, 10), - "total_view" : range(11000, 20000), - "type" : "illust", - "url" : "https://i.pximg.net/img-original/img/2022/11/20/00/00/49/102932581_p0.jpg", - "visible" : False, - "width" : 2160, - "x_restrict" : 0, - "image_urls" : { - "mini" : "https://i.pximg.net/c/48x48/custom-thumb/img/2022/11/20/00/00/49/102932581_p0_custom1200.jpg", - "original": "https://i.pximg.net/img-original/img/2022/11/20/00/00/49/102932581_p0.jpg", - "regular" : "https://i.pximg.net/img-master/img/2022/11/20/00/00/49/102932581_p0_master1200.jpg", - "small" : "https://i.pximg.net/c/540x540_70/img-master/img/2022/11/20/00/00/49/102932581_p0_master1200.jpg", - "thumb" : "https://i.pximg.net/c/250x250_80_a2/custom-thumb/img/2022/11/20/00/00/49/102932581_p0_custom1200.jpg", - }, - "tags" : [ - "オリジナル", - "風景", - "イラスト", - "illustration", - "美しい", - "女の子", - "少女", - "deer", - "flower", - "spring", - ], - "user" : { - "account" : "805482263", - "id" : 7386235, - "is_followed": False, - "name" : "岛的鲸", - "profile_image_urls": {}, - }, -}, - -{ - "#url" : "https://www.pixiv.net/en/artworks/109487939", - "#comment" : "R-18 limit_sanity_level_360.png (#4327, #5180)", - "#class" : pixiv.PixivWorkExtractor, - "#urls" : [ - "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p0.png", - "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p1.png", - "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p2.png", - "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p3.png", - ], -}, - -{ - "#url" : "https://www.pixiv.net/en/artworks/104582860", - "#comment" : "deleted limit_sanity_level_360.png work (#6339)", - "#class" : pixiv.PixivWorkExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.pixiv.net/en/artworks/103983466", - "#comment" : "empty 'caption' in App API response (#4327, #5191)", - "#class" : pixiv.PixivWorkExtractor, - "#options" : {"captions": True}, - - "caption": r"re:Either she doesn't know how to pose or she can't move with that much clothing on her, in any case she's very well dressed for a holiday trip around town. Lots of stuff to see and a perfect day to grab some sweet pastries at the bakery.
          ...", -}, - -{ - "#url" : "https://www.pixiv.net/en/artworks/966412", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=96641", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "http://i1.pixiv.net/c/600x600/img-master/img/2008/06/13/00/29/13/966412_p0_master1200.jpg", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "https://i.pximg.net/img-original/img/2017/04/25/07/33/29/62568267_p0.png", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/i/966412", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "http://img.pixiv.net/img/soundcross/42626136.jpg", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "http://i2.pixiv.net/img76/img/snailrin/42672235.jpg", - "#category": ("", "pixiv", "work"), - "#class" : pixiv.PixivWorkExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/en/artworks/unlisted/eE3fTYaROT9IsZmep386", - "#class" : pixiv.PixivUnlistedExtractor, - "#urls" : "https://i.pximg.net/img-original/img/2020/10/15/00/46/12/85017704-149014193e4d3e23a6b8bd5e38b51ed4_p0.png", - - "id" : 85017704, - "id_unlisted": "eE3fTYaROT9IsZmep386", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/bookmarks/artworks", - "#category": ("", "pixiv", "favorite"), - "#class" : pixiv.PixivFavoriteExtractor, - "#urls" : [ - "https://i.pximg.net/img-original/img/2008/10/31/17/54/01/2005108_p0.jpg", - "https://i.pximg.net/img-original/img/2008/09/27/12/22/40/1719386_p0.jpg", - "https://i.pximg.net/img-original/img/2008/04/15/01/43/46/669358_p0.jpg", - "https://i.pximg.net/img-original/img/2008/06/19/21/52/15/1005851_p0.jpg", - "https://i.pximg.net/img-original/img/2008/06/17/22/16/54/994965_p0.jpg", - ], -}, - -{ - "#url" : "https://www.pixiv.net/bookmark.php?id=173530", - "#category": ("", "pixiv", "favorite"), - "#class" : pixiv.PixivFavoriteExtractor, - "#urls" : [ - "https://i.pximg.net/img-original/img/2008/10/31/17/54/01/2005108_p0.jpg", - "https://i.pximg.net/img-original/img/2008/09/27/12/22/40/1719386_p0.jpg", - "https://i.pximg.net/img-original/img/2008/04/15/01/43/46/669358_p0.jpg", - "https://i.pximg.net/img-original/img/2008/06/19/21/52/15/1005851_p0.jpg", - "https://i.pximg.net/img-original/img/2008/06/17/22/16/54/994965_p0.jpg", - ], -}, - -{ - "#url" : "https://www.pixiv.net/en/users/3137110/bookmarks/artworks/%E3%81%AF%E3%82%93%E3%82%82%E3%82%93", - "#comment" : "bookmarks with specific tag", - "#category": ("", "pixiv", "favorite"), - "#class" : pixiv.PixivFavoriteExtractor, - "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", -}, - -{ - "#url" : "https://www.pixiv.net/bookmark.php?id=3137110&tag=%E3%81%AF%E3%82%93%E3%82%82%E3%82%93&p=1", - "#comment" : "bookmarks with specific tag (legacy url)", - "#category": ("", "pixiv", "favorite"), - "#class" : pixiv.PixivFavoriteExtractor, - "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", -}, - -{ - "#url" : "https://www.pixiv.net/bookmark.php", - "#comment" : "own bookmarks", - "#category": ("", "pixiv", "bookmark"), - "#class" : pixiv.PixivFavoriteExtractor, - "#options" : {"metadata-bookmark": True}, - "#sha1_url": "90c1715b07b0d1aad300bce256a0bc71f42540ba", - - "tags_bookmark": [ - "47", - "hitman", - ], -}, - -{ - "#url" : "https://www.pixiv.net/bookmark.php?tag=foobar", - "#comment" : "own bookmarks with tag (#596)", - "#category": ("", "pixiv", "bookmark"), - "#class" : pixiv.PixivFavoriteExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/173530/following", - "#comment" : "followed users (#515)", - "#category": ("", "pixiv", "following"), - "#class" : pixiv.PixivFavoriteExtractor, - "#pattern" : pixiv.PixivUserExtractor.pattern, - "#count" : ">= 12", -}, - -{ - "#url" : "https://www.pixiv.net/bookmark.php?id=173530&type=user", - "#comment" : "followed users (legacy url) (#515)", - "#category": ("", "pixiv", "following"), - "#class" : pixiv.PixivFavoriteExtractor, - "#pattern" : pixiv.PixivUserExtractor.pattern, - "#count" : ">= 12", -}, - -{ - "#url" : "https://touch.pixiv.net/bookmark.php?id=173530", - "#comment" : "touch URLs", - "#category": ("", "pixiv", "favorite"), - "#class" : pixiv.PixivFavoriteExtractor, -}, - -{ - "#url" : "https://touch.pixiv.net/bookmark.php", - "#category": ("", "pixiv", "bookmark"), - "#class" : pixiv.PixivFavoriteExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/ranking.php?mode=daily&date=20170818", - "#category": ("", "pixiv", "ranking"), - "#class" : pixiv.PixivRankingExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/ranking.php", - "#category": ("", "pixiv", "ranking"), - "#class" : pixiv.PixivRankingExtractor, -}, - -{ - "#url" : "https://touch.pixiv.net/ranking.php", - "#category": ("", "pixiv", "ranking"), - "#class" : pixiv.PixivRankingExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/ranking.php?mode=unknown", - "#category": ("", "pixiv", "ranking"), - "#class" : pixiv.PixivRankingExtractor, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://www.pixiv.net/en/tags/Original", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://pixiv.net/en/tags/foo/artworks?order=week&s_mode=s_tag", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://pixiv.net/en/tags/foo/artworks?order=date&s_mode=tag", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://www.pixiv.net/search.php?s_mode=s_tag&name=Original", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://www.pixiv.net/en/tags/foo/artworks?order=date&s_mode=s_tag", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/search.php?s_mode=s_tag&word=Original", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, -}, - -{ - "#url" : "https://touch.pixiv.net/search.php?word=Original", - "#category": ("", "pixiv", "search"), - "#class" : pixiv.PixivSearchExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/bookmark_new_illust.php", - "#category": ("", "pixiv", "follow"), - "#class" : pixiv.PixivFollowExtractor, -}, - -{ - "#url" : "https://touch.pixiv.net/bookmark_new_illust.php", - "#category": ("", "pixiv", "follow"), - "#class" : pixiv.PixivFollowExtractor, -}, - -{ - "#url" : "https://www.pixivision.net/en/a/2791", - "#category": ("", "pixiv", "pixivision"), - "#class" : pixiv.PixivPixivisionExtractor, -}, - -{ - "#url" : "https://pixivision.net/a/2791", - "#category": ("", "pixiv", "pixivision"), - "#class" : pixiv.PixivPixivisionExtractor, - "#count" : 7, - - "pixivision_id" : "2791", - "pixivision_title": "What's your favorite music? Editor’s picks featuring: “CD Covers”!", -}, - -{ - "#url" : "https://www.pixiv.net/user/10509347/series/21859", - "#category": ("", "pixiv", "series"), - "#class" : pixiv.PixivSeriesExtractor, - "#range" : "1-10", - "#count" : 10, - - "num_series": int, - "series" : { - "create_date": "2017-10-22T14:07:42+09:00", - "width" : 4250, - "height": 3009, - "id" : 21859, - "title" : "先輩がうざい後輩の話", - "total" : range(100, 500), - "user" : dict, - "watchlist_added": False, - }, -}, - -{ - "#url" : "https://www.pixiv.net/novel/show.php?id=12101012", - "#category": ("", "pixiv", "novel"), - "#class" : pixiv.PixivNovelExtractor, - "#count" : 1, - "#sha1_content": "20f4a62f0e87ae2cb9f5a787b6c641bfa4eabf93", - - "caption" : "
          第一印象から決めてました！

          素敵な表紙はいもこは妹さん(illust/53802907)からお借りしました。

          たくさんのコメント、タグありがとうございます、本当に嬉しいです。お返事できていませんが、一つ一つ目を通させていただいてます。タイトルも込みで読んでくださってすごく嬉しいです。ありがとうございます……！！

          ■12/19付けルキラン20位を頂きました…！大変混乱していますがすごく嬉しいです。ありがとうございます！　

          ■2019/12/20デイリー15位、女子に人気8位をを頂きました…！？！？！？！？て、手が震える…。ありがとうございます…ひえええ。感謝してもしきれないです…！", - "create_date" : "2019-12-19T23:14:36+09:00", - "date" : "dt:2019-12-19 14:14:36", - "extension" : "txt", - "id" : 12101012, - "image_urls" : dict, - "is_bookmarked" : False, - "is_muted" : False, - "is_mypixiv_only": False, - "is_original" : False, - "is_x_restricted": False, - "novel_ai_type" : 0, - "page_count" : 1, - "rating" : "General", - "restrict" : 0, - "series" : { - "id" : 1479656, - "title": "一目惚れした彼らの話", - }, - "tags" : [ - "鬼滅の夢", - "女主人公", - "煉獄杏寿郎", - "涙腺崩壊", - "なにこれすごい", - "来世で幸せになって欲しい", - "キメ学世界線できっと幸せになってる!!", - "あなたが神か!!", - "キメ学編を·····", - "鬼滅の夢小説10000users入り", - ], - "text_length" : 9569, - "title" : "本当は、一目惚れだった", - "total_bookmarks": range(17900, 20000), - "total_comments" : range(200, 400), - "total_view" : range(158000, 300000), - "user" : { - "account": "46_maru", - "id" : 888268, - }, - "visible" : True, - "x_restrict" : 0, -}, - -{ - "#url" : "https://www.pixiv.net/novel/show.php?id=16422450", - "#comment" : "embeds // covers (#5373)", - "#category": ("", "pixiv", "novel"), - "#class" : pixiv.PixivNovelExtractor, - "#options" : { - "embeds": True, - "covers": True, - }, - "#count" : 4, -}, - -{ - "#url" : "https://www.pixiv.net/novel/show.php?id=12101012", - "#comment" : "full series", - "#category": ("", "pixiv", "novel"), - "#class" : pixiv.PixivNovelExtractor, - "#options" : {"full-series": True}, - "#count" : 2, -}, - -{ - "#url" : "https://www.pixiv.net/n/19612040", - "#comment" : "short URL", - "#category": ("", "pixiv", "novel"), - "#class" : pixiv.PixivNovelExtractor, -}, - -{ - "#url" : "https://www.pixiv.net/en/users/77055466/novels", - "#category": ("", "pixiv", "novel-user"), - "#class" : pixiv.PixivNovelUserExtractor, - "#pattern" : "^text:", - "#range" : "1-5", - "#count" : 5, -}, - -{ - "#url" : "https://www.pixiv.net/novel/series/1479656", - "#category": ("", "pixiv", "novel-series"), - "#class" : pixiv.PixivNovelSeriesExtractor, - "#count" : 2, - "#sha1_content": "243ce593333bbfe26e255e3372d9c9d8cea22d5b", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/77055466/bookmarks/novels", - "#category": ("", "pixiv", "novel-bookmark"), - "#class" : pixiv.PixivNovelBookmarkExtractor, - "#count" : 1, - "#sha1_content": "7194e8faa876b2b536f185ee271a2b6e46c69089", -}, - -{ - "#url" : "https://www.pixiv.net/en/users/11/bookmarks/novels/TAG?rest=hide", - "#category": ("", "pixiv", "novel-bookmark"), - "#class" : pixiv.PixivNovelBookmarkExtractor, -}, - -{ - "#url" : "https://sketch.pixiv.net/@nicoby", - "#category": ("", "pixiv", "sketch"), - "#class" : pixiv.PixivSketchExtractor, - "#pattern" : r"https://img\-sketch\.pixiv\.net/uploads/medium/file/\d+/\d+\.(jpg|png)", - "#count" : ">= 35", -}, - + { + "#url": "https://www.pixiv.net/en/users/173530", + "#category": ("", "pixiv", "user"), + "#class": pixiv.PixivUserExtractor, + }, + { + "#url": "https://www.pixiv.net/u/173530", + "#category": ("", "pixiv", "user"), + "#class": pixiv.PixivUserExtractor, + }, + { + "#url": "https://www.pixiv.net/member.php?id=173530", + "#category": ("", "pixiv", "user"), + "#class": pixiv.PixivUserExtractor, + }, + { + "#url": "https://www.pixiv.net/mypage.php#id=173530", + "#category": ("", "pixiv", "user"), + "#class": pixiv.PixivUserExtractor, + }, + { + "#url": "https://www.pixiv.net/#id=173530", + "#category": ("", "pixiv", "user"), + "#class": pixiv.PixivUserExtractor, + }, + { + "#url": "https://www.pixiv.net/en/users/173530/artworks", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + "#sha1_url": "852c31ad83b6840bacbce824d85f2a997889efb7", + }, + { + "#url": "https://www.pixiv.net/en/users/173530/artworks/%E6%89%8B%E3%81%B6%E3%82%8D", + "#comment": "illusts with specific tag", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", + }, + { + "#url": "https://www.pixiv.net/member_illust.php?id=173530&tag=%E6%89%8B%E3%81%B6%E3%82%8D", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + "#sha1_url": "25b1cd81153a8ff82eec440dd9f20a4a22079658", + }, + { + "#url": "http://www.pixiv.net/member_illust.php?id=173531", + "#comment": "deleted account", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + "#options": {"metadata": True}, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.pixiv.net/en/users/56514424/artworks", + "#comment": "limit_sanity_level_360.png in artworks results (#5435, #6339)", + "#class": pixiv.PixivArtworksExtractor, + "#count": ">= 39", + }, + { + "#url": "https://www.pixiv.net/en/users/173530/manga", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + }, + { + "#url": "https://www.pixiv.net/en/users/173530/illustrations", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + }, + { + "#url": "https://www.pixiv.net/member_illust.php?id=173530", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + }, + { + "#url": "https://touch.pixiv.net/member_illust.php?id=173530", + "#category": ("", "pixiv", "artworks"), + "#class": pixiv.PixivArtworksExtractor, + }, + { + "#url": "https://www.pixiv.net/en/users/173530/avatar", + "#category": ("", "pixiv", "avatar"), + "#class": pixiv.PixivAvatarExtractor, + "#sha1_content": "4e57544480cc2036ea9608103e8f024fa737fe66", + }, + { + "#url": "https://www.pixiv.net/en/users/194921/background", + "#category": ("", "pixiv", "background"), + "#class": pixiv.PixivBackgroundExtractor, + "#pattern": r"https://i\.pximg\.net/background/img/2021/01/30/16/12/02/194921_af1f71e557a42f499213d4b9eaccc0f8\.jpg", + }, + { + "#url": "https://pixiv.me/del_shannon", + "#category": ("", "pixiv", "me"), + "#class": pixiv.PixivMeExtractor, + "#sha1_url": "29c295ce75150177e6b0a09089a949804c708fbf", + }, + { + "#url": "https://pixiv.me/del_shanno", + "#category": ("", "pixiv", "me"), + "#class": pixiv.PixivMeExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.pixiv.net/artworks/966412", + "#comment": "related works (#1237)", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + "#sha1_url": "90c1715b07b0d1aad300bce256a0bc71f42540ba", + "#sha1_content": "69a8edfb717400d1c2e146ab2b30d2c235440c5a", + "date": "dt:2008-06-12 15:29:13", + "date_url": "dt:2008-06-12 15:29:13", + }, + { + "#url": "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=966411", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=66806629", + "#comment": "ugoira", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + "#urls": "https://i.pximg.net/img-zip-ugoira/img/2018/01/15/13/24/48/66806629_ugoira1920x1080.zip", + "frames": list, + "date": "dt:2018-01-14 15:06:08", + "date_url": "dt:2018-01-15 04:24:48", + }, + { + "#url": "https://www.pixiv.net/artworks/101003492", + "#comment": "original ugoira frames (#6056)", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + "#options": {"ugoira": "original"}, + "#urls": [ + "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira0.png", + "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira1.png", + "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira2.png", + "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira3.png", + "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira4.png", + "https://i.pximg.net/img-original/img/2022/09/04/23/54/19/101003492_ugoira5.png", + ], + "frames": list, + }, + { + "#url": "https://www.pixiv.net/artworks/966412", + "#comment": "related works (#1237)", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + "#options": {"related": True}, + "#range": "1-10", + "#count": ">= 10", + }, + { + "#url": "https://www.pixiv.net/artworks/85960783", + "#comment": "limit_sanity_level_360.png (#4327, #5180)", + "#class": pixiv.PixivWorkExtractor, + "#options": {"sanity": False}, + "#count": 0, + }, + { + "#url": "https://www.pixiv.net/en/artworks/102932581", + "#comment": "limit_sanity_level_360.png (#4327, #5180)", + "#class": pixiv.PixivWorkExtractor, + "#options": {"sanity": True}, + "#urls": "https://i.pximg.net/img-original/img/2022/11/20/00/00/49/102932581_p0.jpg", + "caption": "Meet a deer .", + "comment_access_control": 0, + "create_date": "2022-11-19T15:00:00+00:00", + "date": "dt:2022-11-19 15:00:00", + "date_url": "dt:2022-11-19 15:00:49", + "extension": "jpg", + "filename": "102932581_p0", + "height": 3840, + "id": 102932581, + "illust_ai_type": 1, + "illust_book_style": 0, + "is_bookmarked": False, + "is_muted": False, + "num": 0, + "page_count": 1, + "rating": "General", + "restrict": 0, + "sanity_level": 2, + "series": None, + "suffix": "", + "title": "《 Bridge and Deer 》", + "tools": [], + "total_bookmarks": range(1900, 3000), + "total_comments": range(3, 10), + "total_view": range(11000, 20000), + "type": "illust", + "url": "https://i.pximg.net/img-original/img/2022/11/20/00/00/49/102932581_p0.jpg", + "visible": False, + "width": 2160, + "x_restrict": 0, + "image_urls": { + "mini": "https://i.pximg.net/c/48x48/custom-thumb/img/2022/11/20/00/00/49/102932581_p0_custom1200.jpg", + "original": "https://i.pximg.net/img-original/img/2022/11/20/00/00/49/102932581_p0.jpg", + "regular": "https://i.pximg.net/img-master/img/2022/11/20/00/00/49/102932581_p0_master1200.jpg", + "small": "https://i.pximg.net/c/540x540_70/img-master/img/2022/11/20/00/00/49/102932581_p0_master1200.jpg", + "thumb": "https://i.pximg.net/c/250x250_80_a2/custom-thumb/img/2022/11/20/00/00/49/102932581_p0_custom1200.jpg", + }, + "tags": [ + "オリジナル", + "風景", + "イラスト", + "illustration", + "美しい", + "女の子", + "少女", + "deer", + "flower", + "spring", + ], + "user": { + "account": "805482263", + "id": 7386235, + "is_followed": False, + "name": "岛的鲸", + "profile_image_urls": {}, + }, + }, + { + "#url": "https://www.pixiv.net/en/artworks/109487939", + "#comment": "R-18 limit_sanity_level_360.png (#4327, #5180)", + "#class": pixiv.PixivWorkExtractor, + "#urls": [ + "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p0.png", + "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p1.png", + "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p2.png", + "https://i.pximg.net/img-original/img/2023/07/01/00/06/28/109487939_p3.png", + ], + }, + { + "#url": "https://www.pixiv.net/en/artworks/104582860", + "#comment": "deleted limit_sanity_level_360.png work (#6339)", + "#class": pixiv.PixivWorkExtractor, + "#count": 0, + }, + { + "#url": "https://www.pixiv.net/en/artworks/103983466", + "#comment": "empty 'caption' in App API response (#4327, #5191)", + "#class": pixiv.PixivWorkExtractor, + "#options": {"captions": True}, + "caption": r"re:Either she doesn't know how to pose or she can't move with that much clothing on her, in any case she's very well dressed for a holiday trip around town. Lots of stuff to see and a perfect day to grab some sweet pastries at the bakery.
          ...", + }, + { + "#url": "https://www.pixiv.net/en/artworks/966412", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "http://www.pixiv.net/member_illust.php?mode=medium&illust_id=96641", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "http://i1.pixiv.net/c/600x600/img-master/img/2008/06/13/00/29/13/966412_p0_master1200.jpg", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "https://i.pximg.net/img-original/img/2017/04/25/07/33/29/62568267_p0.png", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "https://www.pixiv.net/i/966412", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "http://img.pixiv.net/img/soundcross/42626136.jpg", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "http://i2.pixiv.net/img76/img/snailrin/42672235.jpg", + "#category": ("", "pixiv", "work"), + "#class": pixiv.PixivWorkExtractor, + }, + { + "#url": "https://www.pixiv.net/en/artworks/unlisted/eE3fTYaROT9IsZmep386", + "#class": pixiv.PixivUnlistedExtractor, + "#urls": "https://i.pximg.net/img-original/img/2020/10/15/00/46/12/85017704-149014193e4d3e23a6b8bd5e38b51ed4_p0.png", + "id": 85017704, + "id_unlisted": "eE3fTYaROT9IsZmep386", + }, + { + "#url": "https://www.pixiv.net/en/users/173530/bookmarks/artworks", + "#category": ("", "pixiv", "favorite"), + "#class": pixiv.PixivFavoriteExtractor, + "#urls": [ + "https://i.pximg.net/img-original/img/2008/10/31/17/54/01/2005108_p0.jpg", + "https://i.pximg.net/img-original/img/2008/09/27/12/22/40/1719386_p0.jpg", + "https://i.pximg.net/img-original/img/2008/04/15/01/43/46/669358_p0.jpg", + "https://i.pximg.net/img-original/img/2008/06/19/21/52/15/1005851_p0.jpg", + "https://i.pximg.net/img-original/img/2008/06/17/22/16/54/994965_p0.jpg", + ], + }, + { + "#url": "https://www.pixiv.net/bookmark.php?id=173530", + "#category": ("", "pixiv", "favorite"), + "#class": pixiv.PixivFavoriteExtractor, + "#urls": [ + "https://i.pximg.net/img-original/img/2008/10/31/17/54/01/2005108_p0.jpg", + "https://i.pximg.net/img-original/img/2008/09/27/12/22/40/1719386_p0.jpg", + "https://i.pximg.net/img-original/img/2008/04/15/01/43/46/669358_p0.jpg", + "https://i.pximg.net/img-original/img/2008/06/19/21/52/15/1005851_p0.jpg", + "https://i.pximg.net/img-original/img/2008/06/17/22/16/54/994965_p0.jpg", + ], + }, + { + "#url": "https://www.pixiv.net/en/users/3137110/bookmarks/artworks/%E3%81%AF%E3%82%93%E3%82%82%E3%82%93", + "#comment": "bookmarks with specific tag", + "#category": ("", "pixiv", "favorite"), + "#class": pixiv.PixivFavoriteExtractor, + "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", + }, + { + "#url": "https://www.pixiv.net/bookmark.php?id=3137110&tag=%E3%81%AF%E3%82%93%E3%82%82%E3%82%93&p=1", + "#comment": "bookmarks with specific tag (legacy url)", + "#category": ("", "pixiv", "favorite"), + "#class": pixiv.PixivFavoriteExtractor, + "#sha1_url": "379b28275f786d946e01f721e54afe346c148a8c", + }, + { + "#url": "https://www.pixiv.net/bookmark.php", + "#comment": "own bookmarks", + "#category": ("", "pixiv", "bookmark"), + "#class": pixiv.PixivFavoriteExtractor, + "#options": {"metadata-bookmark": True}, + "#sha1_url": "90c1715b07b0d1aad300bce256a0bc71f42540ba", + "tags_bookmark": [ + "47", + "hitman", + ], + }, + { + "#url": "https://www.pixiv.net/bookmark.php?tag=foobar", + "#comment": "own bookmarks with tag (#596)", + "#category": ("", "pixiv", "bookmark"), + "#class": pixiv.PixivFavoriteExtractor, + "#count": 0, + }, + { + "#url": "https://www.pixiv.net/en/users/173530/following", + "#comment": "followed users (#515)", + "#category": ("", "pixiv", "following"), + "#class": pixiv.PixivFavoriteExtractor, + "#pattern": pixiv.PixivUserExtractor.pattern, + "#count": ">= 12", + }, + { + "#url": "https://www.pixiv.net/bookmark.php?id=173530&type=user", + "#comment": "followed users (legacy url) (#515)", + "#category": ("", "pixiv", "following"), + "#class": pixiv.PixivFavoriteExtractor, + "#pattern": pixiv.PixivUserExtractor.pattern, + "#count": ">= 12", + }, + { + "#url": "https://touch.pixiv.net/bookmark.php?id=173530", + "#comment": "touch URLs", + "#category": ("", "pixiv", "favorite"), + "#class": pixiv.PixivFavoriteExtractor, + }, + { + "#url": "https://touch.pixiv.net/bookmark.php", + "#category": ("", "pixiv", "bookmark"), + "#class": pixiv.PixivFavoriteExtractor, + }, + { + "#url": "https://www.pixiv.net/ranking.php?mode=daily&date=20170818", + "#category": ("", "pixiv", "ranking"), + "#class": pixiv.PixivRankingExtractor, + }, + { + "#url": "https://www.pixiv.net/ranking.php", + "#category": ("", "pixiv", "ranking"), + "#class": pixiv.PixivRankingExtractor, + }, + { + "#url": "https://touch.pixiv.net/ranking.php", + "#category": ("", "pixiv", "ranking"), + "#class": pixiv.PixivRankingExtractor, + }, + { + "#url": "https://www.pixiv.net/ranking.php?mode=unknown", + "#category": ("", "pixiv", "ranking"), + "#class": pixiv.PixivRankingExtractor, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://www.pixiv.net/en/tags/Original", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://pixiv.net/en/tags/foo/artworks?order=week&s_mode=s_tag", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://pixiv.net/en/tags/foo/artworks?order=date&s_mode=tag", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://www.pixiv.net/search.php?s_mode=s_tag&name=Original", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://www.pixiv.net/en/tags/foo/artworks?order=date&s_mode=s_tag", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + }, + { + "#url": "https://www.pixiv.net/search.php?s_mode=s_tag&word=Original", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + }, + { + "#url": "https://touch.pixiv.net/search.php?word=Original", + "#category": ("", "pixiv", "search"), + "#class": pixiv.PixivSearchExtractor, + }, + { + "#url": "https://www.pixiv.net/bookmark_new_illust.php", + "#category": ("", "pixiv", "follow"), + "#class": pixiv.PixivFollowExtractor, + }, + { + "#url": "https://touch.pixiv.net/bookmark_new_illust.php", + "#category": ("", "pixiv", "follow"), + "#class": pixiv.PixivFollowExtractor, + }, + { + "#url": "https://www.pixivision.net/en/a/2791", + "#category": ("", "pixiv", "pixivision"), + "#class": pixiv.PixivPixivisionExtractor, + }, + { + "#url": "https://pixivision.net/a/2791", + "#category": ("", "pixiv", "pixivision"), + "#class": pixiv.PixivPixivisionExtractor, + "#count": 7, + "pixivision_id": "2791", + "pixivision_title": "What's your favorite music? Editor’s picks featuring: “CD Covers”!", + }, + { + "#url": "https://www.pixiv.net/user/10509347/series/21859", + "#category": ("", "pixiv", "series"), + "#class": pixiv.PixivSeriesExtractor, + "#range": "1-10", + "#count": 10, + "num_series": int, + "series": { + "create_date": "2017-10-22T14:07:42+09:00", + "width": 4250, + "height": 3009, + "id": 21859, + "title": "先輩がうざい後輩の話", + "total": range(100, 500), + "user": dict, + "watchlist_added": False, + }, + }, + { + "#url": "https://www.pixiv.net/novel/show.php?id=12101012", + "#category": ("", "pixiv", "novel"), + "#class": pixiv.PixivNovelExtractor, + "#count": 1, + "#sha1_content": "20f4a62f0e87ae2cb9f5a787b6c641bfa4eabf93", + "caption": '
          第一印象から決めてました！

          素敵な表紙はいもこは妹さん(illust/53802907)からお借りしました。

          たくさんのコメント、タグありがとうございます、本当に嬉しいです。お返事できていませんが、一つ一つ目を通させていただいてます。タイトルも込みで読んでくださってすごく嬉しいです。ありがとうございます……！！

          ■12/19付けルキラン20位を頂きました…！大変混乱していますがすごく嬉しいです。ありがとうございます！　

          ■2019/12/20デイリー15位、女子に人気8位をを頂きました…！？！？！？！？て、手が震える…。ありがとうございます…ひえええ。感謝してもしきれないです…！', + "create_date": "2019-12-19T23:14:36+09:00", + "date": "dt:2019-12-19 14:14:36", + "extension": "txt", + "id": 12101012, + "image_urls": dict, + "is_bookmarked": False, + "is_muted": False, + "is_mypixiv_only": False, + "is_original": False, + "is_x_restricted": False, + "novel_ai_type": 0, + "page_count": 1, + "rating": "General", + "restrict": 0, + "series": { + "id": 1479656, + "title": "一目惚れした彼らの話", + }, + "tags": [ + "鬼滅の夢", + "女主人公", + "煉獄杏寿郎", + "涙腺崩壊", + "なにこれすごい", + "来世で幸せになって欲しい", + "キメ学世界線できっと幸せになってる!!", + "あなたが神か!!", + "キメ学編を·····", + "鬼滅の夢小説10000users入り", + ], + "text_length": 9569, + "title": "本当は、一目惚れだった", + "total_bookmarks": range(17900, 20000), + "total_comments": range(200, 400), + "total_view": range(158000, 300000), + "user": { + "account": "46_maru", + "id": 888268, + }, + "visible": True, + "x_restrict": 0, + }, + { + "#url": "https://www.pixiv.net/novel/show.php?id=16422450", + "#comment": "embeds // covers (#5373)", + "#category": ("", "pixiv", "novel"), + "#class": pixiv.PixivNovelExtractor, + "#options": { + "embeds": True, + "covers": True, + }, + "#count": 4, + }, + { + "#url": "https://www.pixiv.net/novel/show.php?id=12101012", + "#comment": "full series", + "#category": ("", "pixiv", "novel"), + "#class": pixiv.PixivNovelExtractor, + "#options": {"full-series": True}, + "#count": 2, + }, + { + "#url": "https://www.pixiv.net/n/19612040", + "#comment": "short URL", + "#category": ("", "pixiv", "novel"), + "#class": pixiv.PixivNovelExtractor, + }, + { + "#url": "https://www.pixiv.net/en/users/77055466/novels", + "#category": ("", "pixiv", "novel-user"), + "#class": pixiv.PixivNovelUserExtractor, + "#pattern": "^text:", + "#range": "1-5", + "#count": 5, + }, + { + "#url": "https://www.pixiv.net/novel/series/1479656", + "#category": ("", "pixiv", "novel-series"), + "#class": pixiv.PixivNovelSeriesExtractor, + "#count": 2, + "#sha1_content": "243ce593333bbfe26e255e3372d9c9d8cea22d5b", + }, + { + "#url": "https://www.pixiv.net/en/users/77055466/bookmarks/novels", + "#category": ("", "pixiv", "novel-bookmark"), + "#class": pixiv.PixivNovelBookmarkExtractor, + "#count": 1, + "#sha1_content": "7194e8faa876b2b536f185ee271a2b6e46c69089", + }, + { + "#url": "https://www.pixiv.net/en/users/11/bookmarks/novels/TAG?rest=hide", + "#category": ("", "pixiv", "novel-bookmark"), + "#class": pixiv.PixivNovelBookmarkExtractor, + }, + { + "#url": "https://sketch.pixiv.net/@nicoby", + "#category": ("", "pixiv", "sketch"), + "#class": pixiv.PixivSketchExtractor, + "#pattern": r"https://img\-sketch\.pixiv\.net/uploads/medium/file/\d+/\d+\.(jpg|png)", + "#count": ">= 35", + }, ) diff --git a/test/results/pixnet.py b/test/results/pixnet.py index 9c086526e3..ad1fd0ca5f 100644 --- a/test/results/pixnet.py +++ b/test/results/pixnet.py @@ -1,86 +1,73 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import pixnet - __tests__ = ( -{ - "#url" : "https://albertayu773.pixnet.net/album/photo/159443828", - "#category": ("", "pixnet", "image"), - "#class" : pixnet.PixnetImageExtractor, - "#sha1_url" : "156564c422138914c9fa5b42191677b45c414af4", - "#sha1_metadata": "19971bcd056dfef5593f4328a723a9602be0f087", - "#sha1_content" : "0e097bdf49e76dd9b9d57a016b08b16fa6a33280", -}, - -{ - "#url" : "https://albertayu773.pixnet.net/album/set/15078995", - "#category": ("", "pixnet", "set"), - "#class" : pixnet.PixnetSetExtractor, - "#sha1_url" : "6535712801af47af51110542f4938a7cef44557f", - "#sha1_metadata": "bf25d59e5b0959cb1f53e7fd2e2a25f2f67e5925", -}, - -{ - "#url" : "https://anrine910070.pixnet.net/album/set/5917493", - "#category": ("", "pixnet", "set"), - "#class" : pixnet.PixnetSetExtractor, - "#sha1_url" : "b3eb6431aea0bcf5003432a4a0f3a3232084fc13", - "#sha1_metadata": "bf7004faa1cea18cf9bd856f0955a69be51b1ec6", -}, - -{ - "#url" : "https://sky92100.pixnet.net/album/set/17492544", - "#comment" : "password-protected", - "#category": ("", "pixnet", "set"), - "#class" : pixnet.PixnetSetExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://albertayu773.pixnet.net/album/folder/1405768", - "#category": ("", "pixnet", "folder"), - "#class" : pixnet.PixnetFolderExtractor, - "#pattern" : pixnet.PixnetSetExtractor.pattern, - "#count" : ">= 15", -}, - -{ - "#url" : "https://albertayu773.pixnet.net/", - "#category": ("", "pixnet", "user"), - "#class" : pixnet.PixnetUserExtractor, -}, - -{ - "#url" : "https://albertayu773.pixnet.net/blog", - "#category": ("", "pixnet", "user"), - "#class" : pixnet.PixnetUserExtractor, -}, - -{ - "#url" : "https://albertayu773.pixnet.net/album", - "#category": ("", "pixnet", "user"), - "#class" : pixnet.PixnetUserExtractor, -}, - -{ - "#url" : "https://albertayu773.pixnet.net/album/list", - "#category": ("", "pixnet", "user"), - "#class" : pixnet.PixnetUserExtractor, - "#pattern" : pixnet.PixnetFolderExtractor.pattern, - "#count" : ">= 30", -}, - -{ - "#url" : "https://anrine910070.pixnet.net/album/list", - "#category": ("", "pixnet", "user"), - "#class" : pixnet.PixnetUserExtractor, - "#pattern" : pixnet.PixnetSetExtractor.pattern, - "#count" : ">= 14", -}, - + { + "#url": "https://albertayu773.pixnet.net/album/photo/159443828", + "#category": ("", "pixnet", "image"), + "#class": pixnet.PixnetImageExtractor, + "#sha1_url": "156564c422138914c9fa5b42191677b45c414af4", + "#sha1_metadata": "19971bcd056dfef5593f4328a723a9602be0f087", + "#sha1_content": "0e097bdf49e76dd9b9d57a016b08b16fa6a33280", + }, + { + "#url": "https://albertayu773.pixnet.net/album/set/15078995", + "#category": ("", "pixnet", "set"), + "#class": pixnet.PixnetSetExtractor, + "#sha1_url": "6535712801af47af51110542f4938a7cef44557f", + "#sha1_metadata": "bf25d59e5b0959cb1f53e7fd2e2a25f2f67e5925", + }, + { + "#url": "https://anrine910070.pixnet.net/album/set/5917493", + "#category": ("", "pixnet", "set"), + "#class": pixnet.PixnetSetExtractor, + "#sha1_url": "b3eb6431aea0bcf5003432a4a0f3a3232084fc13", + "#sha1_metadata": "bf7004faa1cea18cf9bd856f0955a69be51b1ec6", + }, + { + "#url": "https://sky92100.pixnet.net/album/set/17492544", + "#comment": "password-protected", + "#category": ("", "pixnet", "set"), + "#class": pixnet.PixnetSetExtractor, + "#count": 0, + }, + { + "#url": "https://albertayu773.pixnet.net/album/folder/1405768", + "#category": ("", "pixnet", "folder"), + "#class": pixnet.PixnetFolderExtractor, + "#pattern": pixnet.PixnetSetExtractor.pattern, + "#count": ">= 15", + }, + { + "#url": "https://albertayu773.pixnet.net/", + "#category": ("", "pixnet", "user"), + "#class": pixnet.PixnetUserExtractor, + }, + { + "#url": "https://albertayu773.pixnet.net/blog", + "#category": ("", "pixnet", "user"), + "#class": pixnet.PixnetUserExtractor, + }, + { + "#url": "https://albertayu773.pixnet.net/album", + "#category": ("", "pixnet", "user"), + "#class": pixnet.PixnetUserExtractor, + }, + { + "#url": "https://albertayu773.pixnet.net/album/list", + "#category": ("", "pixnet", "user"), + "#class": pixnet.PixnetUserExtractor, + "#pattern": pixnet.PixnetFolderExtractor.pattern, + "#count": ">= 30", + }, + { + "#url": "https://anrine910070.pixnet.net/album/list", + "#category": ("", "pixnet", "user"), + "#class": pixnet.PixnetUserExtractor, + "#pattern": pixnet.PixnetSetExtractor.pattern, + "#count": ">= 14", + }, ) diff --git a/test/results/plurk.py b/test/results/plurk.py index 11600baed2..23a628a9fa 100644 --- a/test/results/plurk.py +++ b/test/results/plurk.py @@ -1,35 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import plurk - __tests__ = ( -{ - "#url" : "https://www.plurk.com/plurkapi", - "#category": ("", "plurk", "timeline"), - "#class" : plurk.PlurkTimelineExtractor, - "#pattern" : "https?://.+", - "#count" : ">= 23", -}, - -{ - "#url" : "https://www.plurk.com/p/i701j1", - "#category": ("", "plurk", "post"), - "#class" : plurk.PlurkPostExtractor, - "#count" : 3, - "#sha1_url": "2115f208564591b8748525c2807a84596aaaaa5f", -}, - -{ - "#url" : "https://www.plurk.com/p/i701j1", - "#category": ("", "plurk", "post"), - "#class" : plurk.PlurkPostExtractor, - "#options" : {"comments": True}, - "#count" : ">= 210", -}, - + { + "#url": "https://www.plurk.com/plurkapi", + "#category": ("", "plurk", "timeline"), + "#class": plurk.PlurkTimelineExtractor, + "#pattern": "https?://.+", + "#count": ">= 23", + }, + { + "#url": "https://www.plurk.com/p/i701j1", + "#category": ("", "plurk", "post"), + "#class": plurk.PlurkPostExtractor, + "#count": 3, + "#sha1_url": "2115f208564591b8748525c2807a84596aaaaa5f", + }, + { + "#url": "https://www.plurk.com/p/i701j1", + "#category": ("", "plurk", "post"), + "#class": plurk.PlurkPostExtractor, + "#options": {"comments": True}, + "#count": ">= 210", + }, ) diff --git a/test/results/poipiku.py b/test/results/poipiku.py index dd9e744d48..114c46c395 100644 --- a/test/results/poipiku.py +++ b/test/results/poipiku.py @@ -1,76 +1,65 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import poipiku - __tests__ = ( -{ - "#url" : "https://poipiku.com/25049/", - "#category": ("", "poipiku", "user"), - "#class" : poipiku.PoipikuUserExtractor, - "#pattern" : r"https://img-org\.poipiku\.com/user_img\d+/000025049/\d+_\w+\.(jpe?g|png)$", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://poipiku.com/IllustListPcV.jsp?PG=1&ID=25049&KWD=", - "#category": ("", "poipiku", "user"), - "#class" : poipiku.PoipikuUserExtractor, -}, - -{ - "#url" : "https://poipiku.com/25049/5864576.html", - "#category": ("", "poipiku", "post"), - "#class" : poipiku.PoipikuPostExtractor, - "#pattern" : r"https://img-org\.poipiku\.com/user_img\d+/000025049/005864576_EWN1Y65gQ\.png$", - - "count" : 1, - "description" : "", - "extension" : "png", - "filename" : "005864576_EWN1Y65gQ", - "num" : 1, - "post_category": "DOODLE", - "post_id" : "5864576", - "user_id" : "25049", - "user_name" : "ユキウサギ", -}, - -{ - "#url" : "https://poipiku.com/2166245/6411749.html", - "#category": ("", "poipiku", "post"), - "#class" : poipiku.PoipikuPostExtractor, - "#pattern" : r"https://img-org\.poipiku\.com/user_img\d+/002166245/006411749_\w+\.jpeg$", - "#count" : 4, - - "count" : 4, - "description" : "絵茶の産物ネタバレあるやつ", - "num" : int, - "post_category": "SPOILER", - "post_id" : "6411749", - "user_id" : "2166245", - "user_name" : "wadahito", -}, - -{ - "#url" : "https://poipiku.com/3572553/5776587.html", - "#comment" : "different warning button style", - "#category": ("", "poipiku", "post"), - "#class" : poipiku.PoipikuPostExtractor, - "#pattern" : r"https://img-org\.poipiku.com/user_img\d+/003572553/005776587_(\d+_)?\w+\.jpeg$", - "#count" : 3, - - "count" : 3, - "description" : "ORANGE OASISボスネタバレ
          曲も大好き
          2枚目以降はほとんど見えなかった1枚目背景のヒエログリフ小ネタです𓀀", - "num" : int, - "post_category": "SPOILER", - "post_id" : "5776587", - "user_id" : "3572553", - "user_name" : "nagakun", -}, - + { + "#url": "https://poipiku.com/25049/", + "#category": ("", "poipiku", "user"), + "#class": poipiku.PoipikuUserExtractor, + "#pattern": r"https://img-org\.poipiku\.com/user_img\d+/000025049/\d+_\w+\.(jpe?g|png)$", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://poipiku.com/IllustListPcV.jsp?PG=1&ID=25049&KWD=", + "#category": ("", "poipiku", "user"), + "#class": poipiku.PoipikuUserExtractor, + }, + { + "#url": "https://poipiku.com/25049/5864576.html", + "#category": ("", "poipiku", "post"), + "#class": poipiku.PoipikuPostExtractor, + "#pattern": r"https://img-org\.poipiku\.com/user_img\d+/000025049/005864576_EWN1Y65gQ\.png$", + "count": 1, + "description": "", + "extension": "png", + "filename": "005864576_EWN1Y65gQ", + "num": 1, + "post_category": "DOODLE", + "post_id": "5864576", + "user_id": "25049", + "user_name": "ユキウサギ", + }, + { + "#url": "https://poipiku.com/2166245/6411749.html", + "#category": ("", "poipiku", "post"), + "#class": poipiku.PoipikuPostExtractor, + "#pattern": r"https://img-org\.poipiku\.com/user_img\d+/002166245/006411749_\w+\.jpeg$", + "#count": 4, + "count": 4, + "description": "絵茶の産物ネタバレあるやつ", + "num": int, + "post_category": "SPOILER", + "post_id": "6411749", + "user_id": "2166245", + "user_name": "wadahito", + }, + { + "#url": "https://poipiku.com/3572553/5776587.html", + "#comment": "different warning button style", + "#category": ("", "poipiku", "post"), + "#class": poipiku.PoipikuPostExtractor, + "#pattern": r"https://img-org\.poipiku.com/user_img\d+/003572553/005776587_(\d+_)?\w+\.jpeg$", + "#count": 3, + "count": 3, + "description": "ORANGE OASISボスネタバレ
          曲も大好き
          2枚目以降はほとんど見えなかった1枚目背景のヒエログリフ小ネタです𓀀", + "num": int, + "post_category": "SPOILER", + "post_id": "5776587", + "user_id": "3572553", + "user_name": "nagakun", + }, ) diff --git a/test/results/ponybooru.py b/test/results/ponybooru.py index 6955bb4908..2419856857 100644 --- a/test/results/ponybooru.py +++ b/test/results/ponybooru.py @@ -1,39 +1,32 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import philomena - __tests__ = ( -{ - "#url" : "https://ponybooru.org/images/1", - "#category": ("philomena", "ponybooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, - "#sha1_content": "bca26f58fafd791fe07adcd2a28efd7751824605", -}, - -{ - "#url" : "https://www.ponybooru.org/images/1", - "#category": ("philomena", "ponybooru", "post"), - "#class" : philomena.PhilomenaPostExtractor, -}, - -{ - "#url" : "https://ponybooru.org/search?q=cute", - "#category": ("philomena", "ponybooru", "search"), - "#class" : philomena.PhilomenaSearchExtractor, - "#range" : "40-60", - "#count" : 21, -}, - -{ - "#url" : "https://ponybooru.org/galleries/27", - "#category": ("philomena", "ponybooru", "gallery"), - "#class" : philomena.PhilomenaGalleryExtractor, - "#count" : ">= 24", -}, - + { + "#url": "https://ponybooru.org/images/1", + "#category": ("philomena", "ponybooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + "#sha1_content": "bca26f58fafd791fe07adcd2a28efd7751824605", + }, + { + "#url": "https://www.ponybooru.org/images/1", + "#category": ("philomena", "ponybooru", "post"), + "#class": philomena.PhilomenaPostExtractor, + }, + { + "#url": "https://ponybooru.org/search?q=cute", + "#category": ("philomena", "ponybooru", "search"), + "#class": philomena.PhilomenaSearchExtractor, + "#range": "40-60", + "#count": 21, + }, + { + "#url": "https://ponybooru.org/galleries/27", + "#category": ("philomena", "ponybooru", "gallery"), + "#class": philomena.PhilomenaGalleryExtractor, + "#count": ">= 24", + }, ) diff --git a/test/results/poringa.py b/test/results/poringa.py index 16c27e42bb..db41098d5a 100644 --- a/test/results/poringa.py +++ b/test/results/poringa.py @@ -1,54 +1,45 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import poringa - __tests__ = ( -{ - "#url" : "http://www.poringa.net/posts/imagenes/3051081/Turrita-alto-ojete.html", - "#category": ("", "poringa", "post"), - "#class" : poringa.PoringaPostExtractor, - "#count" : 26, - - "count" : 26, - "num" : range(1, 26), - "post_id" : "3051081", - "title" : "turrita alto ojete...", - "user" : "vipower1top", -}, - -{ - "#url" : "http://www.poringa.net/posts/imagenes/3095554/Otra-culona-de-instagram.html", - "#category": ("", "poringa", "post"), - "#class" : poringa.PoringaPostExtractor, - "#count" : 15, - - "count" : 15, - "num" : range(1, 15), - "post_id" : "3095554", - "title" : "Otra culona de instagram", - "user" : "Expectro007", -}, - -{ - "#url" : "http://www.poringa.net/Expectro007", - "#category": ("", "poringa", "user"), - "#class" : poringa.PoringaUserExtractor, - "#pattern" : r"https?://img-\d+\.poringa\.net/poringa/img/././././././Expectro007/\w{3}\.(jpg|png|gif)", - "#count" : range(500, 600), -}, - -{ - "#url" : "http://www.poringa.net/buscar/?&q=yuslopez", - "#category": ("", "poringa", "search"), - "#class" : poringa.PoringaSearchExtractor, - "#pattern" : r"https?://img-\d+\.poringa\.net/poringa/img/././././././\w+/\w{3}\.(jpg|png|gif)", - "#range" : "1-50", - "#count" : 50, -}, - + { + "#url": "http://www.poringa.net/posts/imagenes/3051081/Turrita-alto-ojete.html", + "#category": ("", "poringa", "post"), + "#class": poringa.PoringaPostExtractor, + "#count": 26, + "count": 26, + "num": range(1, 26), + "post_id": "3051081", + "title": "turrita alto ojete...", + "user": "vipower1top", + }, + { + "#url": "http://www.poringa.net/posts/imagenes/3095554/Otra-culona-de-instagram.html", + "#category": ("", "poringa", "post"), + "#class": poringa.PoringaPostExtractor, + "#count": 15, + "count": 15, + "num": range(1, 15), + "post_id": "3095554", + "title": "Otra culona de instagram", + "user": "Expectro007", + }, + { + "#url": "http://www.poringa.net/Expectro007", + "#category": ("", "poringa", "user"), + "#class": poringa.PoringaUserExtractor, + "#pattern": r"https?://img-\d+\.poringa\.net/poringa/img/././././././Expectro007/\w{3}\.(jpg|png|gif)", + "#count": range(500, 600), + }, + { + "#url": "http://www.poringa.net/buscar/?&q=yuslopez", + "#category": ("", "poringa", "search"), + "#class": poringa.PoringaSearchExtractor, + "#pattern": r"https?://img-\d+\.poringa\.net/poringa/img/././././././\w+/\w{3}\.(jpg|png|gif)", + "#range": "1-50", + "#count": 50, + }, ) diff --git a/test/results/pornhub.py b/test/results/pornhub.py index c60dc0e4eb..2b1d527ed3 100644 --- a/test/results/pornhub.py +++ b/test/results/pornhub.py @@ -1,137 +1,119 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import pornhub from gallery_dl import exception - +from gallery_dl.extractor import pornhub __tests__ = ( -{ - "#url" : "https://www.pornhub.com/album/19289801", - "#category": ("", "pornhub", "gallery"), - "#class" : pornhub.PornhubGalleryExtractor, - "#pattern" : r"https://\w+.phncdn.com/pics/albums/\d+/\d+/\d+/\d+/", - "#count" : ">= 300", - - "id" : int, - "num" : int, - "score" : int, - "views" : int, - "caption": str, - "user" : "Danika Mori", - "gallery": { - "id" : 19289801, + { + "#url": "https://www.pornhub.com/album/19289801", + "#category": ("", "pornhub", "gallery"), + "#class": pornhub.PornhubGalleryExtractor, + "#pattern": r"https://\w+.phncdn.com/pics/albums/\d+/\d+/\d+/\d+/", + "#count": ">= 300", + "id": int, + "num": int, "score": int, "views": int, - "tags" : list, - "title": "Danika Mori Best Moments", + "caption": str, + "user": "Danika Mori", + "gallery": { + "id": 19289801, + "score": int, + "views": int, + "tags": list, + "title": "Danika Mori Best Moments", + }, + }, + { + "#url": "https://www.pornhub.com/album/69606532", + "#comment": "KeyError due to missing image entry (#6299)", + "#class": pornhub.PornhubGalleryExtractor, + "#count": 6, + }, + { + "#url": "https://www.pornhub.com/album/69040172", + "#category": ("", "pornhub", "gallery"), + "#class": pornhub.PornhubGalleryExtractor, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://www.pornhub.com/gif/43726891", + "#category": ("", "pornhub", "gif"), + "#class": pornhub.PornhubGifExtractor, + "#pattern": r"https://\w+\.phncdn\.com/pics/gifs/043/726/891/43726891a\.webm", + "date": "dt:2023-04-20 00:00:00", + "extension": "webm", + "filename": "43726891a", + "id": "43726891", + "tags": [ + "sloppy deepthroat", + "perfect body", + "petite brunette", + "mouth fuck", + "big dick", + "natural big tits", + "deepthroat swallow", + "amateur couple", + "homemade", + "girls wanking boys", + "hardcore sex", + "babes 18 year", + ], + "timestamp": "5:07", + "title": "Intense sloppy blowjob of Danika Mori", + "url": "https://el.phncdn.com/pics/gifs/043/726/891/43726891a.webm", + "user": "Danika Mori", + "viewkey": "64367c8c78a4a", + }, + { + "#url": "https://www.pornhub.com/pornstar/danika-mori", + "#category": ("", "pornhub", "user"), + "#class": pornhub.PornhubUserExtractor, + }, + { + "#url": "https://www.pornhub.com/pornstar/danika-mori/photos", + "#category": ("", "pornhub", "photos"), + "#class": pornhub.PornhubPhotosExtractor, + "#pattern": pornhub.PornhubGalleryExtractor.pattern, + "#count": ">= 6", + }, + { + "#url": "https://www.pornhub.com/users/flyings0l0/photos/public", + "#category": ("", "pornhub", "photos"), + "#class": pornhub.PornhubPhotosExtractor, + }, + { + "#url": "https://www.pornhub.com/users/flyings0l0/photos/private", + "#category": ("", "pornhub", "photos"), + "#class": pornhub.PornhubPhotosExtractor, + }, + { + "#url": "https://www.pornhub.com/users/flyings0l0/photos/favorites", + "#category": ("", "pornhub", "photos"), + "#class": pornhub.PornhubPhotosExtractor, + }, + { + "#url": "https://www.pornhub.com/model/bossgirl/photos", + "#category": ("", "pornhub", "photos"), + "#class": pornhub.PornhubPhotosExtractor, + }, + { + "#url": "https://www.pornhub.com/pornstar/danika-mori/gifs", + "#category": ("", "pornhub", "gifs"), + "#class": pornhub.PornhubGifsExtractor, + "#pattern": pornhub.PornhubGifExtractor.pattern, + "#count": ">= 30", + }, + { + "#url": "https://www.pornhub.com/users/flyings0l0/gifs", + "#category": ("", "pornhub", "gifs"), + "#class": pornhub.PornhubGifsExtractor, + }, + { + "#url": "https://www.pornhub.com/model/bossgirl/gifs/video", + "#category": ("", "pornhub", "gifs"), + "#class": pornhub.PornhubGifsExtractor, }, -}, - -{ - "#url" : "https://www.pornhub.com/album/69606532", - "#comment" : "KeyError due to missing image entry (#6299)", - "#class" : pornhub.PornhubGalleryExtractor, - "#count" : 6, -}, - -{ - "#url" : "https://www.pornhub.com/album/69040172", - "#category": ("", "pornhub", "gallery"), - "#class" : pornhub.PornhubGalleryExtractor, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://www.pornhub.com/gif/43726891", - "#category": ("", "pornhub", "gif"), - "#class" : pornhub.PornhubGifExtractor, - "#pattern" : r"https://\w+\.phncdn\.com/pics/gifs/043/726/891/43726891a\.webm", - - "date" : "dt:2023-04-20 00:00:00", - "extension": "webm", - "filename" : "43726891a", - "id" : "43726891", - "tags" : [ - "sloppy deepthroat", - "perfect body", - "petite brunette", - "mouth fuck", - "big dick", - "natural big tits", - "deepthroat swallow", - "amateur couple", - "homemade", - "girls wanking boys", - "hardcore sex", - "babes 18 year", - ], - "timestamp": "5:07", - "title" : "Intense sloppy blowjob of Danika Mori", - "url" : "https://el.phncdn.com/pics/gifs/043/726/891/43726891a.webm", - "user" : "Danika Mori", - "viewkey" : "64367c8c78a4a", -}, - -{ - "#url" : "https://www.pornhub.com/pornstar/danika-mori", - "#category": ("", "pornhub", "user"), - "#class" : pornhub.PornhubUserExtractor, -}, - -{ - "#url" : "https://www.pornhub.com/pornstar/danika-mori/photos", - "#category": ("", "pornhub", "photos"), - "#class" : pornhub.PornhubPhotosExtractor, - "#pattern" : pornhub.PornhubGalleryExtractor.pattern, - "#count" : ">= 6", -}, - -{ - "#url" : "https://www.pornhub.com/users/flyings0l0/photos/public", - "#category": ("", "pornhub", "photos"), - "#class" : pornhub.PornhubPhotosExtractor, -}, - -{ - "#url" : "https://www.pornhub.com/users/flyings0l0/photos/private", - "#category": ("", "pornhub", "photos"), - "#class" : pornhub.PornhubPhotosExtractor, -}, - -{ - "#url" : "https://www.pornhub.com/users/flyings0l0/photos/favorites", - "#category": ("", "pornhub", "photos"), - "#class" : pornhub.PornhubPhotosExtractor, -}, - -{ - "#url" : "https://www.pornhub.com/model/bossgirl/photos", - "#category": ("", "pornhub", "photos"), - "#class" : pornhub.PornhubPhotosExtractor, -}, - -{ - "#url" : "https://www.pornhub.com/pornstar/danika-mori/gifs", - "#category": ("", "pornhub", "gifs"), - "#class" : pornhub.PornhubGifsExtractor, - "#pattern" : pornhub.PornhubGifExtractor.pattern, - "#count" : ">= 30", -}, - -{ - "#url" : "https://www.pornhub.com/users/flyings0l0/gifs", - "#category": ("", "pornhub", "gifs"), - "#class" : pornhub.PornhubGifsExtractor, -}, - -{ - "#url" : "https://www.pornhub.com/model/bossgirl/gifs/video", - "#category": ("", "pornhub", "gifs"), - "#class" : pornhub.PornhubGifsExtractor, -}, - ) diff --git a/test/results/pornpics.py b/test/results/pornpics.py index 6caf5dd11f..644d83369d 100644 --- a/test/results/pornpics.py +++ b/test/results/pornpics.py @@ -1,145 +1,128 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import pornpics - __tests__ = ( -{ - "#url" : "https://www.pornpics.com/galleries/british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest-62610699/", - "#category": ("", "pornpics", "gallery"), - "#class" : pornpics.PornpicsGalleryExtractor, - "#pattern" : r"https://cdni\.pornpics\.com/1280/7/160/62610699/62610699_\d+_[0-9a-f]{4}\.jpg", - - "categories": [ - "Outdoor", - "MILF", - "Boots", - "Amateur", - "Sexy", - ], - "channel" : ["FTV MILFs"], - "count" : 17, - "gallery_id": 62610699, - "models" : ["Danielle"], - "num" : int, - "slug" : "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", - "tags" : [ - "MILF Outdoor", - "Amateur MILF", - "Nature", - "Amateur Outdoor", - "First Time", - "Sexy MILF", - ], - "title" : "British beauty Danielle flashes hot breasts, ass and snatch in the forest", - "views" : int, -}, - -{ - "#url" : "https://pornpics.com/es/galleries/62610699", - "#category": ("", "pornpics", "gallery"), - "#class" : pornpics.PornpicsGalleryExtractor, - - "slug": "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", -}, - -{ - "#url" : "https://www.pornpics.com/galleries/four-american-hotties-in-swimsuit-showing-off-their-sexy-booty-and-bare-legs-59500405/", - "#comment" : "more than one 'channel' (#5195)", - "#category": ("", "pornpics", "gallery"), - "#class" : pornpics.PornpicsGalleryExtractor, - - "count" : 16, - "num" : range(1, 16), - "gallery_id": 59500405, - "slug" : "four-american-hotties-in-swimsuit-showing-off-their-sexy-booty-and-bare-legs", - "title" : "Four American hotties in swimsuit showing off their sexy booty and bare legs", - "views" : range(50000, 100000), - "models" : [ - "Kayla West", - "Layla Price", - "Marley Blaze", - "Mena Mason", - ], - "categories": [ - "Outdoor", - "Asian", - "Pornstar", - "Brunette", - "Blonde", - ], - "channel" : [ - "Adult Time", - "Fame Digital", - ], - "tags" : [ - "Nature", - "Asian Outdoor", - ], -}, - -{ - "#url" : "https://www.pornpics.com/tags/summer-dress/", - "#category": ("", "pornpics", "tag"), - "#class" : pornpics.PornpicsTagExtractor, - "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://pornpics.com/fr/tags/summer-dress", - "#category": ("", "pornpics", "tag"), - "#class" : pornpics.PornpicsTagExtractor, -}, - -{ - "#url" : "https://www.pornpics.com/?q=nature", - "#category": ("", "pornpics", "search"), - "#class" : pornpics.PornpicsSearchExtractor, - "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://www.pornpics.com/channels/femjoy/", - "#category": ("", "pornpics", "search"), - "#class" : pornpics.PornpicsSearchExtractor, - "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://www.pornpics.com/pornstars/emma-brown/", - "#category": ("", "pornpics", "search"), - "#class" : pornpics.PornpicsSearchExtractor, - "#pattern" : pornpics.PornpicsGalleryExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://pornpics.com/jp/?q=nature", - "#category": ("", "pornpics", "search"), - "#class" : pornpics.PornpicsSearchExtractor, -}, - -{ - "#url" : "https://pornpics.com/it/channels/femjoy", - "#category": ("", "pornpics", "search"), - "#class" : pornpics.PornpicsSearchExtractor, -}, - -{ - "#url" : "https://pornpics.com/pt/pornstars/emma-brown", - "#category": ("", "pornpics", "search"), - "#class" : pornpics.PornpicsSearchExtractor, -}, - + { + "#url": "https://www.pornpics.com/galleries/british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest-62610699/", + "#category": ("", "pornpics", "gallery"), + "#class": pornpics.PornpicsGalleryExtractor, + "#pattern": r"https://cdni\.pornpics\.com/1280/7/160/62610699/62610699_\d+_[0-9a-f]{4}\.jpg", + "categories": [ + "Outdoor", + "MILF", + "Boots", + "Amateur", + "Sexy", + ], + "channel": ["FTV MILFs"], + "count": 17, + "gallery_id": 62610699, + "models": ["Danielle"], + "num": int, + "slug": "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", + "tags": [ + "MILF Outdoor", + "Amateur MILF", + "Nature", + "Amateur Outdoor", + "First Time", + "Sexy MILF", + ], + "title": "British beauty Danielle flashes hot breasts, ass and snatch in the forest", + "views": int, + }, + { + "#url": "https://pornpics.com/es/galleries/62610699", + "#category": ("", "pornpics", "gallery"), + "#class": pornpics.PornpicsGalleryExtractor, + "slug": "british-beauty-danielle-flashes-hot-breasts-ass-and-snatch-in-the-forest", + }, + { + "#url": "https://www.pornpics.com/galleries/four-american-hotties-in-swimsuit-showing-off-their-sexy-booty-and-bare-legs-59500405/", + "#comment": "more than one 'channel' (#5195)", + "#category": ("", "pornpics", "gallery"), + "#class": pornpics.PornpicsGalleryExtractor, + "count": 16, + "num": range(1, 16), + "gallery_id": 59500405, + "slug": "four-american-hotties-in-swimsuit-showing-off-their-sexy-booty-and-bare-legs", + "title": "Four American hotties in swimsuit showing off their sexy booty and bare legs", + "views": range(50000, 100000), + "models": [ + "Kayla West", + "Layla Price", + "Marley Blaze", + "Mena Mason", + ], + "categories": [ + "Outdoor", + "Asian", + "Pornstar", + "Brunette", + "Blonde", + ], + "channel": [ + "Adult Time", + "Fame Digital", + ], + "tags": [ + "Nature", + "Asian Outdoor", + ], + }, + { + "#url": "https://www.pornpics.com/tags/summer-dress/", + "#category": ("", "pornpics", "tag"), + "#class": pornpics.PornpicsTagExtractor, + "#pattern": pornpics.PornpicsGalleryExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://pornpics.com/fr/tags/summer-dress", + "#category": ("", "pornpics", "tag"), + "#class": pornpics.PornpicsTagExtractor, + }, + { + "#url": "https://www.pornpics.com/?q=nature", + "#category": ("", "pornpics", "search"), + "#class": pornpics.PornpicsSearchExtractor, + "#pattern": pornpics.PornpicsGalleryExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://www.pornpics.com/channels/femjoy/", + "#category": ("", "pornpics", "search"), + "#class": pornpics.PornpicsSearchExtractor, + "#pattern": pornpics.PornpicsGalleryExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://www.pornpics.com/pornstars/emma-brown/", + "#category": ("", "pornpics", "search"), + "#class": pornpics.PornpicsSearchExtractor, + "#pattern": pornpics.PornpicsGalleryExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://pornpics.com/jp/?q=nature", + "#category": ("", "pornpics", "search"), + "#class": pornpics.PornpicsSearchExtractor, + }, + { + "#url": "https://pornpics.com/it/channels/femjoy", + "#category": ("", "pornpics", "search"), + "#class": pornpics.PornpicsSearchExtractor, + }, + { + "#url": "https://pornpics.com/pt/pornstars/emma-brown", + "#category": ("", "pornpics", "search"), + "#class": pornpics.PornpicsSearchExtractor, + }, ) diff --git a/test/results/pornreactor.py b/test/results/pornreactor.py index 0062d19aa0..e4b37d2e82 100644 --- a/test/results/pornreactor.py +++ b/test/results/pornreactor.py @@ -1,66 +1,55 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import reactor - __tests__ = ( -{ - "#url" : "http://pornreactor.cc/tag/RiceGnat", - "#category": ("reactor", "pornreactor", "tag"), - "#class" : reactor.ReactorTagExtractor, - "#range" : "1-25", - "#count" : ">= 25", -}, - -{ - "#url" : "http://fapreactor.com/tag/RiceGnat", - "#category": ("reactor", "pornreactor", "tag"), - "#class" : reactor.ReactorTagExtractor, -}, - -{ - "#url" : "http://pornreactor.cc/search?q=ecchi+hentai", - "#category": ("reactor", "pornreactor", "search"), - "#class" : reactor.ReactorSearchExtractor, -}, - -{ - "#url" : "http://fapreactor.com/search/ecchi+hentai", - "#category": ("reactor", "pornreactor", "search"), - "#class" : reactor.ReactorSearchExtractor, -}, - -{ - "#url" : "http://pornreactor.cc/user/Disillusion", - "#category": ("reactor", "pornreactor", "user"), - "#class" : reactor.ReactorUserExtractor, - "#range" : "1-25", - "#count" : ">= 20", -}, - -{ - "#url" : "http://fapreactor.com/user/Disillusion", - "#category": ("reactor", "pornreactor", "user"), - "#class" : reactor.ReactorUserExtractor, -}, - -{ - "#url" : "http://pornreactor.cc/post/863166", - "#category": ("reactor", "pornreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#sha1_url" : "a09fb0577489e1f9564c25d0ad576f81b19c2ef3", - "#sha1_content": "ec6b0568bfb1803648744077da082d14de844340", -}, - -{ - "#url" : "http://fapreactor.com/post/863166", - "#category": ("reactor", "pornreactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#sha1_url": "2a956ce0c90e8bc47b4392db4fa25ad1342f3e54", -}, - + { + "#url": "http://pornreactor.cc/tag/RiceGnat", + "#category": ("reactor", "pornreactor", "tag"), + "#class": reactor.ReactorTagExtractor, + "#range": "1-25", + "#count": ">= 25", + }, + { + "#url": "http://fapreactor.com/tag/RiceGnat", + "#category": ("reactor", "pornreactor", "tag"), + "#class": reactor.ReactorTagExtractor, + }, + { + "#url": "http://pornreactor.cc/search?q=ecchi+hentai", + "#category": ("reactor", "pornreactor", "search"), + "#class": reactor.ReactorSearchExtractor, + }, + { + "#url": "http://fapreactor.com/search/ecchi+hentai", + "#category": ("reactor", "pornreactor", "search"), + "#class": reactor.ReactorSearchExtractor, + }, + { + "#url": "http://pornreactor.cc/user/Disillusion", + "#category": ("reactor", "pornreactor", "user"), + "#class": reactor.ReactorUserExtractor, + "#range": "1-25", + "#count": ">= 20", + }, + { + "#url": "http://fapreactor.com/user/Disillusion", + "#category": ("reactor", "pornreactor", "user"), + "#class": reactor.ReactorUserExtractor, + }, + { + "#url": "http://pornreactor.cc/post/863166", + "#category": ("reactor", "pornreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#sha1_url": "a09fb0577489e1f9564c25d0ad576f81b19c2ef3", + "#sha1_content": "ec6b0568bfb1803648744077da082d14de844340", + }, + { + "#url": "http://fapreactor.com/post/863166", + "#category": ("reactor", "pornreactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#sha1_url": "2a956ce0c90e8bc47b4392db4fa25ad1342f3e54", + }, ) diff --git a/test/results/postimg.py b/test/results/postimg.py index ba4b8a8667..a045f7c4b2 100644 --- a/test/results/postimg.py +++ b/test/results/postimg.py @@ -1,46 +1,38 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://postimages.org/Wtn2b3hC", - "#category": ("imagehost", "postimg", "image"), - "#class" : imagehosts.PostimgImageExtractor, -}, - -{ - "#url" : "https://www.postimages.org/Wtn2b3hC", - "#category": ("imagehost", "postimg", "image"), - "#class" : imagehosts.PostimgImageExtractor, -}, - -{ - "#url" : "https://pixxxels.cc/Wtn2b3hC", - "#category": ("imagehost", "postimg", "image"), - "#class" : imagehosts.PostimgImageExtractor, -}, - -{ - "#url" : "https://postimg.cc/Wtn2b3hC", - "#category": ("imagehost", "postimg", "image"), - "#class" : imagehosts.PostimgImageExtractor, - "#sha1_url" : "72f3c8b1d6c6601a20ad58f35635494b4891a99e", - "#sha1_metadata": "2d05808d04e4e83e33200db83521af06e3147a84", - "#sha1_content" : "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", -}, - -{ - "#url" : "https://postimg.cc/gallery/wxpDLgX", - "#category": ("imagehost", "postimg", "gallery"), - "#class" : imagehosts.PostimgGalleryExtractor, - "#pattern" : imagehosts.PostimgImageExtractor.pattern, - "#count" : 22, -}, - + { + "#url": "https://postimages.org/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class": imagehosts.PostimgImageExtractor, + }, + { + "#url": "https://www.postimages.org/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class": imagehosts.PostimgImageExtractor, + }, + { + "#url": "https://pixxxels.cc/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class": imagehosts.PostimgImageExtractor, + }, + { + "#url": "https://postimg.cc/Wtn2b3hC", + "#category": ("imagehost", "postimg", "image"), + "#class": imagehosts.PostimgImageExtractor, + "#sha1_url": "72f3c8b1d6c6601a20ad58f35635494b4891a99e", + "#sha1_metadata": "2d05808d04e4e83e33200db83521af06e3147a84", + "#sha1_content": "cfaa8def53ed1a575e0c665c9d6d8cf2aac7a0ee", + }, + { + "#url": "https://postimg.cc/gallery/wxpDLgX", + "#category": ("imagehost", "postimg", "gallery"), + "#class": imagehosts.PostimgGalleryExtractor, + "#pattern": imagehosts.PostimgImageExtractor.pattern, + "#count": 22, + }, ) diff --git a/test/results/raddle.py b/test/results/raddle.py index 6b3ccde82a..7ee955a079 100644 --- a/test/results/raddle.py +++ b/test/results/raddle.py @@ -1,111 +1,96 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import postmill - __tests__ = ( -{ - "#url" : "https://raddle.me/", - "#category": ("postmill", "raddle", "home"), - "#class" : postmill.PostmillHomeExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://raddle.me/f/traa", - "#category": ("postmill", "raddle", "forum"), - "#class" : postmill.PostmillForumExtractor, - "#count" : 1, - "#pattern" : r"^https://raddle\.me/f/traa/156646/click-here-to-go-to-f-traaaaaaannnnnnnnnns$", -}, - -{ - "#url" : "https://raddle.me/user/Sam_the_enby/submissions", - "#category": ("postmill", "raddle", "usersubmissions"), - "#class" : postmill.PostmillUserSubmissionsExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://raddle.me/tag/Trans", - "#category": ("postmill", "raddle", "tag"), - "#class" : postmill.PostmillTagExtractor, -}, - -{ - "#url" : "https://raddle.me/search?q=tw", - "#category": ("postmill", "raddle", "search"), - "#class" : postmill.PostmillSearchExtractor, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://raddle.me/160845", - "#category": ("postmill", "raddle", "shorturl"), - "#class" : postmill.PostmillShortURLExtractor, - "#pattern" : r"^https://raddle\.me/f/egg_irl/160845/egg_irl$", -}, - -{ - "#url" : "https://raddle.me/f/NonBinary/179017/scattered-thoughts-would-appreciate-advice-immensely-tw", - "#comment" : "Text post", - "#category": ("postmill", "raddle", "post"), - "#class" : postmill.PostmillPostExtractor, - "#sha1_url" : "99277f815820810d9d7e219d455f818601858378", - "#sha1_content": "7a1159e1e45f2ce8e2c8b5959f6d66b042776f3b", - "#count" : 1, -}, - -{ - "#url" : "https://raddle.me/f/egg_irl/160845", - "#comment" : "Image post", - "#category": ("postmill", "raddle", "post"), - "#class" : postmill.PostmillPostExtractor, - "#sha1_content": "431e938082c2b59c44888a83cfc711cd1f0e910a", - "#urls" : "https://uploads-cdn.raddle.me/submission_images/30f4cf7d235d40c1daebf6dc2e58bef2a80bec2b5b2dab10f2021ea8e3f29e11.png", -}, - -{ - "#url" : "https://raddle.me/f/trans/177042/tw-vent-nsfw-suicide-i-lost-no-nut-november-tw-trauma", - "#comment" : "Image + text post (with text enabled)", - "#category": ("postmill", "raddle", "post"), - "#class" : postmill.PostmillPostExtractor, - "#options" : {"save-link-post-body": True}, - "#pattern" : r"^(text:[\s\S]+|https://(uploads-cdn\.)?raddle\.me/submission_images/[0-9a-f]+\.png)$", - "#count" : 2, -}, - -{ - "#url" : "https://raddle.me/f/videos/179541/raisins-and-sprite", - "#comment" : "Link post", - "#category": ("postmill", "raddle", "post"), - "#class" : postmill.PostmillPostExtractor, - "#urls" : "https://m.youtube.com/watch?v=RFJCA5zcZxI", - "#count" : 1, -}, - -{ - "#url" : "https://raddle.me/f/Anime/150698/neo-tokyo-1987-link-to-the-english-dub-version-last-link", - "#comment" : "Link + text post (with text disabled)", - "#category": ("postmill", "raddle", "post"), - "#class" : postmill.PostmillPostExtractor, - "#pattern" : r"^https://fantasyanime\.com/anime/neo-tokyo-dub$", - "#count" : 1, -}, - -{ - "#url" : "https://raddle.me/f/egg_irl/166855/4th-wall-breaking-please-let-this-be-a-flair-egg-irl", - "#comment" : "Post with multiple flairs", - "#category": ("postmill", "raddle", "post"), - "#class" : postmill.PostmillPostExtractor, - "flair" : ["Gender non-specific", "4th wall breaking"], -}, - + { + "#url": "https://raddle.me/", + "#category": ("postmill", "raddle", "home"), + "#class": postmill.PostmillHomeExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://raddle.me/f/traa", + "#category": ("postmill", "raddle", "forum"), + "#class": postmill.PostmillForumExtractor, + "#count": 1, + "#pattern": r"^https://raddle\.me/f/traa/156646/click-here-to-go-to-f-traaaaaaannnnnnnnnns$", + }, + { + "#url": "https://raddle.me/user/Sam_the_enby/submissions", + "#category": ("postmill", "raddle", "usersubmissions"), + "#class": postmill.PostmillUserSubmissionsExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://raddle.me/tag/Trans", + "#category": ("postmill", "raddle", "tag"), + "#class": postmill.PostmillTagExtractor, + }, + { + "#url": "https://raddle.me/search?q=tw", + "#category": ("postmill", "raddle", "search"), + "#class": postmill.PostmillSearchExtractor, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://raddle.me/160845", + "#category": ("postmill", "raddle", "shorturl"), + "#class": postmill.PostmillShortURLExtractor, + "#pattern": r"^https://raddle\.me/f/egg_irl/160845/egg_irl$", + }, + { + "#url": "https://raddle.me/f/NonBinary/179017/scattered-thoughts-would-appreciate-advice-immensely-tw", + "#comment": "Text post", + "#category": ("postmill", "raddle", "post"), + "#class": postmill.PostmillPostExtractor, + "#sha1_url": "99277f815820810d9d7e219d455f818601858378", + "#sha1_content": "7a1159e1e45f2ce8e2c8b5959f6d66b042776f3b", + "#count": 1, + }, + { + "#url": "https://raddle.me/f/egg_irl/160845", + "#comment": "Image post", + "#category": ("postmill", "raddle", "post"), + "#class": postmill.PostmillPostExtractor, + "#sha1_content": "431e938082c2b59c44888a83cfc711cd1f0e910a", + "#urls": "https://uploads-cdn.raddle.me/submission_images/30f4cf7d235d40c1daebf6dc2e58bef2a80bec2b5b2dab10f2021ea8e3f29e11.png", + }, + { + "#url": "https://raddle.me/f/trans/177042/tw-vent-nsfw-suicide-i-lost-no-nut-november-tw-trauma", + "#comment": "Image + text post (with text enabled)", + "#category": ("postmill", "raddle", "post"), + "#class": postmill.PostmillPostExtractor, + "#options": {"save-link-post-body": True}, + "#pattern": r"^(text:[\s\S]+|https://(uploads-cdn\.)?raddle\.me/submission_images/[0-9a-f]+\.png)$", + "#count": 2, + }, + { + "#url": "https://raddle.me/f/videos/179541/raisins-and-sprite", + "#comment": "Link post", + "#category": ("postmill", "raddle", "post"), + "#class": postmill.PostmillPostExtractor, + "#urls": "https://m.youtube.com/watch?v=RFJCA5zcZxI", + "#count": 1, + }, + { + "#url": "https://raddle.me/f/Anime/150698/neo-tokyo-1987-link-to-the-english-dub-version-last-link", + "#comment": "Link + text post (with text disabled)", + "#category": ("postmill", "raddle", "post"), + "#class": postmill.PostmillPostExtractor, + "#pattern": r"^https://fantasyanime\.com/anime/neo-tokyo-dub$", + "#count": 1, + }, + { + "#url": "https://raddle.me/f/egg_irl/166855/4th-wall-breaking-please-let-this-be-a-flair-egg-irl", + "#comment": "Post with multiple flairs", + "#category": ("postmill", "raddle", "post"), + "#class": postmill.PostmillPostExtractor, + "flair": ["Gender non-specific", "4th wall breaking"], + }, ) diff --git a/test/results/raidlondon.py b/test/results/raidlondon.py index 93aab265f7..89e32ce0f7 100644 --- a/test/results/raidlondon.py +++ b/test/results/raidlondon.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://www.raidlondon.com/collections/flats", - "#category": ("shopify", "raidlondon", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.raidlondon.com/collections/flats/products/raid-addyson-chunky-flat-shoe-in-white", - "#category": ("shopify", "raidlondon", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://www.raidlondon.com/collections/flats", + "#category": ("shopify", "raidlondon", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.raidlondon.com/collections/flats/products/raid-addyson-chunky-flat-shoe-in-white", + "#category": ("shopify", "raidlondon", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/rbt.py b/test/results/rbt.py index 5a81ad1af7..2bf6eae83f 100644 --- a/test/results/rbt.py +++ b/test/results/rbt.py @@ -1,43 +1,35 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://rbt.asia/g/thread/61487650/", - "#category": ("foolfuuka", "rbt", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", -}, - -{ - "#url" : "https://archive.rebeccablacktech.com/g/thread/61487650/", - "#category": ("foolfuuka", "rbt", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", -}, - -{ - "#url" : "https://rbt.asia/g/", - "#category": ("foolfuuka", "rbt", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://rbt.asia/_/search/text/test/", - "#category": ("foolfuuka", "rbt", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://rbt.asia/g/gallery/8", - "#category": ("foolfuuka", "rbt", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://rbt.asia/g/thread/61487650/", + "#category": ("foolfuuka", "rbt", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", + }, + { + "#url": "https://archive.rebeccablacktech.com/g/thread/61487650/", + "#category": ("foolfuuka", "rbt", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "fadd274b25150a1bdf03a40c58db320fa3b617c4", + }, + { + "#url": "https://rbt.asia/g/", + "#category": ("foolfuuka", "rbt", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://rbt.asia/_/search/text/test/", + "#category": ("foolfuuka", "rbt", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://rbt.asia/g/gallery/8", + "#category": ("foolfuuka", "rbt", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/reactor.py b/test/results/reactor.py index 7ff6dcced2..01741c9e27 100644 --- a/test/results/reactor.py +++ b/test/results/reactor.py @@ -1,36 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import reactor - __tests__ = ( -{ - "#url" : "http://reactor.cc/tag/gif", - "#category": ("reactor", "reactor", "tag"), - "#class" : reactor.ReactorTagExtractor, -}, - -{ - "#url" : "http://reactor.cc/search?q=Art", - "#category": ("reactor", "reactor", "search"), - "#class" : reactor.ReactorSearchExtractor, -}, - -{ - "#url" : "http://reactor.cc/user/Dioklet", - "#category": ("reactor", "reactor", "user"), - "#class" : reactor.ReactorUserExtractor, -}, - -{ - "#url" : "http://reactor.cc/post/4999736", - "#category": ("reactor", "reactor", "post"), - "#class" : reactor.ReactorPostExtractor, - "#sha1_url": "dfc74d150d7267384d8c229c4b82aa210755daa0", -}, - + { + "#url": "http://reactor.cc/tag/gif", + "#category": ("reactor", "reactor", "tag"), + "#class": reactor.ReactorTagExtractor, + }, + { + "#url": "http://reactor.cc/search?q=Art", + "#category": ("reactor", "reactor", "search"), + "#class": reactor.ReactorSearchExtractor, + }, + { + "#url": "http://reactor.cc/user/Dioklet", + "#category": ("reactor", "reactor", "user"), + "#class": reactor.ReactorUserExtractor, + }, + { + "#url": "http://reactor.cc/post/4999736", + "#category": ("reactor", "reactor", "post"), + "#class": reactor.ReactorPostExtractor, + "#sha1_url": "dfc74d150d7267384d8c229c4b82aa210755daa0", + }, ) diff --git a/test/results/readcomiconline.py b/test/results/readcomiconline.py index 883ed32100..b92d710c16 100644 --- a/test/results/readcomiconline.py +++ b/test/results/readcomiconline.py @@ -1,36 +1,30 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import readcomiconline - __tests__ = ( -{ - "#url" : "https://readcomiconline.li/Comic/W-i-t-c-h/Issue-130?id=22289", - "#category": ("", "readcomiconline", "issue"), - "#class" : readcomiconline.ReadcomiconlineIssueExtractor, - "#pattern" : r"https://2\.bp\.blogspot\.com/[\w-]+=s0\?.+", - "#count" : 36, - "#sha1_metadata": "2d9ec81ce1b11fac06ebf96ce33cdbfca0e85eb5", -}, - -{ - "#url" : "https://readcomiconline.li/Comic/W-i-t-c-h", - "#category": ("", "readcomiconline", "comic"), - "#class" : readcomiconline.ReadcomiconlineComicExtractor, - "#sha1_url" : "74eb8b9504b4084fcc9367b341300b2c52260918", - "#sha1_metadata": "3986248e4458fa44a201ec073c3684917f48ee0c", -}, - -{ - "#url" : "https://readcomiconline.to/Comic/Bazooka-Jules", - "#category": ("", "readcomiconline", "comic"), - "#class" : readcomiconline.ReadcomiconlineComicExtractor, - "#sha1_url" : "2f66a467a772df4d4592e97a059ddbc3e8991799", - "#sha1_metadata": "f5ba5246cd787bb750924d9690cb1549199bd516", -}, - + { + "#url": "https://readcomiconline.li/Comic/W-i-t-c-h/Issue-130?id=22289", + "#category": ("", "readcomiconline", "issue"), + "#class": readcomiconline.ReadcomiconlineIssueExtractor, + "#pattern": r"https://2\.bp\.blogspot\.com/[\w-]+=s0\?.+", + "#count": 36, + "#sha1_metadata": "2d9ec81ce1b11fac06ebf96ce33cdbfca0e85eb5", + }, + { + "#url": "https://readcomiconline.li/Comic/W-i-t-c-h", + "#category": ("", "readcomiconline", "comic"), + "#class": readcomiconline.ReadcomiconlineComicExtractor, + "#sha1_url": "74eb8b9504b4084fcc9367b341300b2c52260918", + "#sha1_metadata": "3986248e4458fa44a201ec073c3684917f48ee0c", + }, + { + "#url": "https://readcomiconline.to/Comic/Bazooka-Jules", + "#category": ("", "readcomiconline", "comic"), + "#class": readcomiconline.ReadcomiconlineComicExtractor, + "#sha1_url": "2f66a467a772df4d4592e97a059ddbc3e8991799", + "#sha1_metadata": "f5ba5246cd787bb750924d9690cb1549199bd516", + }, ) diff --git a/test/results/realbooru.py b/test/results/realbooru.py index 1ff069e86a..fbca6382f1 100644 --- a/test/results/realbooru.py +++ b/test/results/realbooru.py @@ -1,85 +1,76 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v02 - __tests__ = ( -{ - "#url" : "https://realbooru.com/index.php?page=post&s=list&tags=wine", - "#category": ("gelbooru_v02", "realbooru", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#count" : ">= 64", -}, - -{ - "#url" : "https://realbooru.com/index.php?page=pool&s=show&id=1", - "#category": ("gelbooru_v02", "realbooru", "pool"), - "#class" : gelbooru_v02.GelbooruV02PoolExtractor, - "#urls" : ( - "https://realbooru.com//images/bf/d6/bfd682f338691e5254de796040fcba21.webm", - "https://realbooru.com//images/cb/7d/cb7d921673ba99f688031ac554777695.webm", - "https://realbooru.com//images/9e/14/9e140edc1cb2e4cc734ba5bdc4870955.webm", - ), -}, - -{ - "#url" : "https://realbooru.com/index.php?page=favorites&s=view&id=274", - "#category": ("gelbooru_v02", "realbooru", "favorite"), - "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, - "#urls" : ( - "https://realbooru.com//images/20/3e/0c2c4d8c978355c053602dc963eb13136c1614c1.jpeg", - ), -}, - -{ - "#url" : "https://realbooru.com/index.php?page=post&s=view&id=862054", - "#comment" : "regular post", - "#category": ("gelbooru_v02", "realbooru", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#options" : {"tags": True}, - "#urls" : "https://realbooru.com//images/8a/34/8a345820da989637c21ac013d522bf69.jpeg", - "#sha1_content": "f6213e6f25c3cb9e3cfefa6d4b3a78e44b9dea5b", - - "change" : "1705562002", - "created_at" : "Thu Jan 18 01:12:50 -0600 2024", - "creator_id" : "32011", - "date" : "dt:2024-01-18 07:12:50", - "file_url" : "https://realbooru.com//images/8a/34/8a345820da989637c21ac013d522bf69.jpeg", - "filename" : "8a345820da989637c21ac013d522bf69", - "has_children" : "false", - "has_comments" : "false", - "has_notes" : "false", - "height" : "1800", - "id" : "862054", - "md5" : "8a345820da989637c21ac013d522bf69", - "parent_id" : "", - "preview_height": "150", - "preview_url" : "https://realbooru.com/thumbnails/8a/34/thumbnail_8a345820da989637c21ac013d522bf69.jpg", - "preview_width" : "120", - "rating" : "e", - "sample_height" : "1063", - "sample_url" : "https://realbooru.com/samples/8a/34/sample_8a345820da989637c21ac013d522bf69.jpg", - "sample_width" : "850", - "score" : "", - "source" : "https://www.instagram.com/p/CwAO1UyJBnw", - "status" : "active", - "tags" : "1girl asian bikini black_hair breasts cleavage female female_only floral_print instagram japanese kurita_emi large_breasts looking_at_viewer navel sauna short_hair side-tie_bikini sitting solo", - "tags_copyright": "instagram", - "tags_general" : "1girl asian bikini black_hair breasts cleavage female female_only floral_print japanese large_breasts looking_at_viewer navel sauna short_hair side-tie_bikini sitting solo", - "tags_model" : "kurita_emi", - "width" : "1440", -}, - -{ - "#url" : "https://realbooru.com/index.php?page=post&s=view&id=568145", - "#comment" : "older post", - "#category": ("gelbooru_v02", "realbooru", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#sha1_content": "4a7424810f5f846c161b5d3b7c8b0a85a03368c8", -}, - + { + "#url": "https://realbooru.com/index.php?page=post&s=list&tags=wine", + "#category": ("gelbooru_v02", "realbooru", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#count": ">= 64", + }, + { + "#url": "https://realbooru.com/index.php?page=pool&s=show&id=1", + "#category": ("gelbooru_v02", "realbooru", "pool"), + "#class": gelbooru_v02.GelbooruV02PoolExtractor, + "#urls": ( + "https://realbooru.com//images/bf/d6/bfd682f338691e5254de796040fcba21.webm", + "https://realbooru.com//images/cb/7d/cb7d921673ba99f688031ac554777695.webm", + "https://realbooru.com//images/9e/14/9e140edc1cb2e4cc734ba5bdc4870955.webm", + ), + }, + { + "#url": "https://realbooru.com/index.php?page=favorites&s=view&id=274", + "#category": ("gelbooru_v02", "realbooru", "favorite"), + "#class": gelbooru_v02.GelbooruV02FavoriteExtractor, + "#urls": ( + "https://realbooru.com//images/20/3e/0c2c4d8c978355c053602dc963eb13136c1614c1.jpeg", + ), + }, + { + "#url": "https://realbooru.com/index.php?page=post&s=view&id=862054", + "#comment": "regular post", + "#category": ("gelbooru_v02", "realbooru", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#options": {"tags": True}, + "#urls": "https://realbooru.com//images/8a/34/8a345820da989637c21ac013d522bf69.jpeg", + "#sha1_content": "f6213e6f25c3cb9e3cfefa6d4b3a78e44b9dea5b", + "change": "1705562002", + "created_at": "Thu Jan 18 01:12:50 -0600 2024", + "creator_id": "32011", + "date": "dt:2024-01-18 07:12:50", + "file_url": "https://realbooru.com//images/8a/34/8a345820da989637c21ac013d522bf69.jpeg", + "filename": "8a345820da989637c21ac013d522bf69", + "has_children": "false", + "has_comments": "false", + "has_notes": "false", + "height": "1800", + "id": "862054", + "md5": "8a345820da989637c21ac013d522bf69", + "parent_id": "", + "preview_height": "150", + "preview_url": "https://realbooru.com/thumbnails/8a/34/thumbnail_8a345820da989637c21ac013d522bf69.jpg", + "preview_width": "120", + "rating": "e", + "sample_height": "1063", + "sample_url": "https://realbooru.com/samples/8a/34/sample_8a345820da989637c21ac013d522bf69.jpg", + "sample_width": "850", + "score": "", + "source": "https://www.instagram.com/p/CwAO1UyJBnw", + "status": "active", + "tags": "1girl asian bikini black_hair breasts cleavage female female_only floral_print instagram japanese kurita_emi large_breasts looking_at_viewer navel sauna short_hair side-tie_bikini sitting solo", + "tags_copyright": "instagram", + "tags_general": "1girl asian bikini black_hair breasts cleavage female female_only floral_print japanese large_breasts looking_at_viewer navel sauna short_hair side-tie_bikini sitting solo", + "tags_model": "kurita_emi", + "width": "1440", + }, + { + "#url": "https://realbooru.com/index.php?page=post&s=view&id=568145", + "#comment": "older post", + "#category": ("gelbooru_v02", "realbooru", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#sha1_content": "4a7424810f5f846c161b5d3b7c8b0a85a03368c8", + }, ) diff --git a/test/results/recursive.py b/test/results/recursive.py index 2ad31dd46d..647e50faed 100644 --- a/test/results/recursive.py +++ b/test/results/recursive.py @@ -1,18 +1,14 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import recursive - __tests__ = ( -{ - "#url" : "recursive:https://pastebin.com/raw/FLwrCYsT", - "#category": ("", "recursive", ""), - "#class" : recursive.RecursiveExtractor, - "#sha1_url": "eee86d65c346361b818e8f4b2b307d9429f136a2", -}, - + { + "#url": "recursive:https://pastebin.com/raw/FLwrCYsT", + "#category": ("", "recursive", ""), + "#class": recursive.RecursiveExtractor, + "#sha1_url": "eee86d65c346361b818e8f4b2b307d9429f136a2", + }, ) diff --git a/test/results/reddit.py b/test/results/reddit.py index 457aaf4bef..5982cb80fe 100644 --- a/test/results/reddit.py +++ b/test/results/reddit.py @@ -1,287 +1,248 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import reddit - __tests__ = ( -{ - "#url" : "https://www.reddit.com/r/lavaporn/", - "#category": ("", "reddit", "subreddit"), - "#class" : reddit.RedditSubredditExtractor, - "#range" : "1-20", - "#count" : ">= 20", -}, - -{ - "#url" : "https://www.reddit.com/r/lavaporn/top/?sort=top&t=month", - "#category": ("", "reddit", "subreddit-top"), - "#class" : reddit.RedditSubredditExtractor, -}, - -{ - "#url" : "https://old.reddit.com/r/lavaporn/", - "#category": ("", "reddit", "subreddit"), - "#class" : reddit.RedditSubredditExtractor, -}, - -{ - "#url" : "https://np.reddit.com/r/lavaporn/", - "#category": ("", "reddit", "subreddit"), - "#class" : reddit.RedditSubredditExtractor, -}, - -{ - "#url" : "https://m.reddit.com/r/lavaporn/", - "#category": ("", "reddit", "subreddit"), - "#class" : reddit.RedditSubredditExtractor, -}, - -{ - "#url" : "https://www.reddit.com/", - "#category": ("", "reddit", "home"), - "#class" : reddit.RedditHomeExtractor, - "#range" : "1-20", - "#count" : ">= 20", - "#archive" : False, -}, - -{ - "#url" : "https://old.reddit.com/top/?sort=top&t=month", - "#category": ("", "reddit", "home-top"), - "#class" : reddit.RedditHomeExtractor, -}, - -{ - "#url" : "https://www.reddit.com/user/username/", - "#category": ("", "reddit", "user"), - "#class" : reddit.RedditUserExtractor, - "#count" : ">= 2", -}, - -{ - "#url" : "https://www.reddit.com/user/username/gilded/?sort=top&t=month", - "#category": ("", "reddit", "user-gilded"), - "#class" : reddit.RedditUserExtractor, -}, - -{ - "#url" : "https://old.reddit.com/user/username/", - "#category": ("", "reddit", "user"), - "#class" : reddit.RedditUserExtractor, -}, - -{ - "#url" : "https://www.reddit.com/u/username/", - "#category": ("", "reddit", "user"), - "#class" : reddit.RedditUserExtractor, -}, - -{ - "#url" : "https://www.reddit.com/r/lavaporn/comments/8cqhub/", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#pattern" : r"https://c2.staticflickr.com/8/7272/\w+_k.jpg", - "#count" : 1, -}, - -{ - "#url" : "https://www.reddit.com/r/lavaporn/comments/8cqhub/", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#options" : {"comments": 500}, - "#pattern" : "https://", - "#count" : 3, -}, - -{ - "#url" : "https://www.reddit.com/gallery/hrrh23", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#count" : 3, - "#sha1_url" : "25b91ede15459470274dd17291424b037ed8b0ae", - "#sha1_content": "1e7dde4ee7d5f4c4b45749abfd15b2dbfa27df3f", -}, - -{ - "#url" : "https://www.reddit.com/r/aww/comments/90bu6w/", - "#comment" : "video (dash)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#pattern" : "ytdl:https://v.redd.it/gyh95hiqc0b11", - "#count" : 1, -}, - -{ - "#url" : "https://www.reddit.com/r/aww/comments/90bu6w/", - "#comment" : "video (dash)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#options" : {"videos": "ytdl"}, - "#pattern" : "ytdl:https://www.reddit.com/r/aww/comments/90bu6w/heat_index_was_110_degrees_so_we_offered_him_a/", - "#count" : 1, -}, - -{ - "#url" : "https://www.reddit.com/r/aww/comments/90bu6w/", - "#comment" : "video (dash)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#options" : {"videos": "dash"}, - "#pattern" : r"ytdl:https://v.redd.it/gyh95hiqc0b11/DASHPlaylist.mpd\?a=", - "#count" : 1, -}, - -{ - "#url" : "https://www.reddit.com/gallery/icfgzv", - "#comment" : "deleted gallery (#953)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.reddit.com/r/araragi/comments/ib32hm", - "#comment" : "animated gallery items (#955)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#pattern" : r"https://i\.redd\.it/\w+\.gif", - "#count" : 2, -}, - -{ - "#url" : "https://www.reddit.com/r/cosplay/comments/jvwaqr", - "#comment" : "'failed' gallery item (#1127)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#count" : 1, -}, - -{ - "#url" : "https://www.reddit.com/r/kpopfap/comments/qjj04q/", - "#comment" : "gallery with no 'media_metadata' (#2001)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://www.reddit.com/r/RobloxArt/comments/15ko0qu/", - "#comment" : "comment embeds (#5366)", - "#class" : reddit.RedditSubmissionExtractor, - "#options" : {"comments": 10}, - "#urls" : ( - "https://i.redd.it/ppt5yciyipgb1.jpg", - "https://i.redd.it/u0ojzd69kpgb1.png", - ), -}, - -{ - "#url" : "https://www.reddit.com/r/RobloxArt/comments/15ko0qu/", - "#comment" : "disabled comment embeds (#6357)", - "#class" : reddit.RedditSubmissionExtractor, - "#options" : {"comments": 10, "embeds": False}, - "#urls" : "https://i.redd.it/ppt5yciyipgb1.jpg", -}, - -{ - "#url" : "https://www.reddit.com/user/TheSpiritTree/comments/srilyf/", - "#comment" : "user page submission (#2301)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#pattern" : "https://i.redd.it/8fpgv17yqlh81.jpg", - "#count" : 1, -}, - -{ - "#url" : "https://www.reddit.com/r/kittengifs/comments/12m0b8d", - "#comment" : "cross-posted video (#887, #3586, #3976)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#pattern" : r"ytdl:https://v\.redd\.it/cvabpjacrvta1", -}, - -{ - "#url" : "https://www.reddit.com/r/europe/comments/pm4531/the_name_of/", - "#comment" : "preview.redd.it (#4470)", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, - "#urls" : "https://preview.redd.it/u9ud4k6xaf271.jpg?auto=webp&s=19b1334cb4409111cda136c01f7b44c2c42bf9fb", -}, - -{ - "#url" : "https://old.reddit.com/r/lavaporn/comments/2a00np/", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, -}, - -{ - "#url" : "https://np.reddit.com/r/lavaporn/comments/2a00np/", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, -}, - -{ - "#url" : "https://m.reddit.com/r/lavaporn/comments/2a00np/", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, -}, - -{ - "#url" : "https://redd.it/2a00np/", - "#category": ("", "reddit", "submission"), - "#class" : reddit.RedditSubmissionExtractor, -}, - -{ - "#url" : "https://i.redd.it/upjtjcx2npzz.jpg", - "#category": ("", "reddit", "image"), - "#class" : reddit.RedditImageExtractor, - "#sha1_url" : "0de614900feef103e580b632190458c0b62b641a", - "#sha1_content": "cc9a68cf286708d5ce23c68e79cd9cf7826db6a3", -}, - -{ - "#url" : "https://i.reddituploads.com/0f44f1b1fca2461f957c713d9592617d?fit=max&h=1536&w=1536&s=e96ce7846b3c8e1f921d2ce2671fb5e2", - "#category": ("", "reddit", "image"), - "#class" : reddit.RedditImageExtractor, - "#sha1_url" : "f24f25efcedaddeec802e46c60d77ef975dc52a5", - "#sha1_content": "541dbcc3ad77aa01ee21ca49843c5e382371fae7", -}, - -{ - "#url" : "https://preview.redd.it/00af44lpn0u51.jpg?width=960&crop=smart&auto=webp&v=enabled&s=dbca8ab84033f4a433772d9c15dbe0429c74e8ac", - "#comment" : "preview.redd.it -> i.redd.it", - "#category": ("", "reddit", "image"), - "#class" : reddit.RedditImageExtractor, - "#pattern" : r"^https://i\.redd\.it/00af44lpn0u51\.jpg$", -}, - -{ - "#url" : "https://www.reddit.com/r/analog/s/hKrTTvFVwZ", - "#comment" : "Mobile share URL", - "#category": ("", "reddit", "redirect"), - "#class" : reddit.RedditRedirectExtractor, - "#pattern" : r"^https://www\.reddit\.com/r/analog/comments/179exao/photographing_the_recent_annular_eclipse_with_a", -}, - -{ - "#url" : "https://www.reddit.com/u/Tailhook91/s/w4yAMbtOYm", - "#comment" : "Mobile share URL, user submission", - "#category": ("", "reddit", "redirect"), - "#class" : reddit.RedditRedirectExtractor, - "#pattern" : r"^https://www.reddit.com/user/Tailhook91/comments/znfxbr/prove_it/", -}, - -{ - "#url" : "https://www.reddit.com/user/Tailhook91/s/w4yAMbtOYm", - "#comment" : "Mobile share URL, user submission", - "#category": ("", "reddit", "redirect"), - "#class" : reddit.RedditRedirectExtractor, - "#pattern" : r"^https://www.reddit.com/user/Tailhook91/comments/znfxbr/prove_it/", -}, - + { + "#url": "https://www.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class": reddit.RedditSubredditExtractor, + "#range": "1-20", + "#count": ">= 20", + }, + { + "#url": "https://www.reddit.com/r/lavaporn/top/?sort=top&t=month", + "#category": ("", "reddit", "subreddit-top"), + "#class": reddit.RedditSubredditExtractor, + }, + { + "#url": "https://old.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class": reddit.RedditSubredditExtractor, + }, + { + "#url": "https://np.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class": reddit.RedditSubredditExtractor, + }, + { + "#url": "https://m.reddit.com/r/lavaporn/", + "#category": ("", "reddit", "subreddit"), + "#class": reddit.RedditSubredditExtractor, + }, + { + "#url": "https://www.reddit.com/", + "#category": ("", "reddit", "home"), + "#class": reddit.RedditHomeExtractor, + "#range": "1-20", + "#count": ">= 20", + "#archive": False, + }, + { + "#url": "https://old.reddit.com/top/?sort=top&t=month", + "#category": ("", "reddit", "home-top"), + "#class": reddit.RedditHomeExtractor, + }, + { + "#url": "https://www.reddit.com/user/username/", + "#category": ("", "reddit", "user"), + "#class": reddit.RedditUserExtractor, + "#count": ">= 2", + }, + { + "#url": "https://www.reddit.com/user/username/gilded/?sort=top&t=month", + "#category": ("", "reddit", "user-gilded"), + "#class": reddit.RedditUserExtractor, + }, + { + "#url": "https://old.reddit.com/user/username/", + "#category": ("", "reddit", "user"), + "#class": reddit.RedditUserExtractor, + }, + { + "#url": "https://www.reddit.com/u/username/", + "#category": ("", "reddit", "user"), + "#class": reddit.RedditUserExtractor, + }, + { + "#url": "https://www.reddit.com/r/lavaporn/comments/8cqhub/", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#pattern": r"https://c2.staticflickr.com/8/7272/\w+_k.jpg", + "#count": 1, + }, + { + "#url": "https://www.reddit.com/r/lavaporn/comments/8cqhub/", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#options": {"comments": 500}, + "#pattern": "https://", + "#count": 3, + }, + { + "#url": "https://www.reddit.com/gallery/hrrh23", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#count": 3, + "#sha1_url": "25b91ede15459470274dd17291424b037ed8b0ae", + "#sha1_content": "1e7dde4ee7d5f4c4b45749abfd15b2dbfa27df3f", + }, + { + "#url": "https://www.reddit.com/r/aww/comments/90bu6w/", + "#comment": "video (dash)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#pattern": "ytdl:https://v.redd.it/gyh95hiqc0b11", + "#count": 1, + }, + { + "#url": "https://www.reddit.com/r/aww/comments/90bu6w/", + "#comment": "video (dash)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#options": {"videos": "ytdl"}, + "#pattern": "ytdl:https://www.reddit.com/r/aww/comments/90bu6w/heat_index_was_110_degrees_so_we_offered_him_a/", + "#count": 1, + }, + { + "#url": "https://www.reddit.com/r/aww/comments/90bu6w/", + "#comment": "video (dash)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#options": {"videos": "dash"}, + "#pattern": r"ytdl:https://v.redd.it/gyh95hiqc0b11/DASHPlaylist.mpd\?a=", + "#count": 1, + }, + { + "#url": "https://www.reddit.com/gallery/icfgzv", + "#comment": "deleted gallery (#953)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#count": 0, + }, + { + "#url": "https://www.reddit.com/r/araragi/comments/ib32hm", + "#comment": "animated gallery items (#955)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#pattern": r"https://i\.redd\.it/\w+\.gif", + "#count": 2, + }, + { + "#url": "https://www.reddit.com/r/cosplay/comments/jvwaqr", + "#comment": "'failed' gallery item (#1127)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#count": 1, + }, + { + "#url": "https://www.reddit.com/r/kpopfap/comments/qjj04q/", + "#comment": "gallery with no 'media_metadata' (#2001)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#count": 0, + }, + { + "#url": "https://www.reddit.com/r/RobloxArt/comments/15ko0qu/", + "#comment": "comment embeds (#5366)", + "#class": reddit.RedditSubmissionExtractor, + "#options": {"comments": 10}, + "#urls": ( + "https://i.redd.it/ppt5yciyipgb1.jpg", + "https://i.redd.it/u0ojzd69kpgb1.png", + ), + }, + { + "#url": "https://www.reddit.com/r/RobloxArt/comments/15ko0qu/", + "#comment": "disabled comment embeds (#6357)", + "#class": reddit.RedditSubmissionExtractor, + "#options": {"comments": 10, "embeds": False}, + "#urls": "https://i.redd.it/ppt5yciyipgb1.jpg", + }, + { + "#url": "https://www.reddit.com/user/TheSpiritTree/comments/srilyf/", + "#comment": "user page submission (#2301)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#pattern": "https://i.redd.it/8fpgv17yqlh81.jpg", + "#count": 1, + }, + { + "#url": "https://www.reddit.com/r/kittengifs/comments/12m0b8d", + "#comment": "cross-posted video (#887, #3586, #3976)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#pattern": r"ytdl:https://v\.redd\.it/cvabpjacrvta1", + }, + { + "#url": "https://www.reddit.com/r/europe/comments/pm4531/the_name_of/", + "#comment": "preview.redd.it (#4470)", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + "#urls": "https://preview.redd.it/u9ud4k6xaf271.jpg?auto=webp&s=19b1334cb4409111cda136c01f7b44c2c42bf9fb", + }, + { + "#url": "https://old.reddit.com/r/lavaporn/comments/2a00np/", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + }, + { + "#url": "https://np.reddit.com/r/lavaporn/comments/2a00np/", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + }, + { + "#url": "https://m.reddit.com/r/lavaporn/comments/2a00np/", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + }, + { + "#url": "https://redd.it/2a00np/", + "#category": ("", "reddit", "submission"), + "#class": reddit.RedditSubmissionExtractor, + }, + { + "#url": "https://i.redd.it/upjtjcx2npzz.jpg", + "#category": ("", "reddit", "image"), + "#class": reddit.RedditImageExtractor, + "#sha1_url": "0de614900feef103e580b632190458c0b62b641a", + "#sha1_content": "cc9a68cf286708d5ce23c68e79cd9cf7826db6a3", + }, + { + "#url": "https://i.reddituploads.com/0f44f1b1fca2461f957c713d9592617d?fit=max&h=1536&w=1536&s=e96ce7846b3c8e1f921d2ce2671fb5e2", + "#category": ("", "reddit", "image"), + "#class": reddit.RedditImageExtractor, + "#sha1_url": "f24f25efcedaddeec802e46c60d77ef975dc52a5", + "#sha1_content": "541dbcc3ad77aa01ee21ca49843c5e382371fae7", + }, + { + "#url": "https://preview.redd.it/00af44lpn0u51.jpg?width=960&crop=smart&auto=webp&v=enabled&s=dbca8ab84033f4a433772d9c15dbe0429c74e8ac", + "#comment": "preview.redd.it -> i.redd.it", + "#category": ("", "reddit", "image"), + "#class": reddit.RedditImageExtractor, + "#pattern": r"^https://i\.redd\.it/00af44lpn0u51\.jpg$", + }, + { + "#url": "https://www.reddit.com/r/analog/s/hKrTTvFVwZ", + "#comment": "Mobile share URL", + "#category": ("", "reddit", "redirect"), + "#class": reddit.RedditRedirectExtractor, + "#pattern": r"^https://www\.reddit\.com/r/analog/comments/179exao/photographing_the_recent_annular_eclipse_with_a", + }, + { + "#url": "https://www.reddit.com/u/Tailhook91/s/w4yAMbtOYm", + "#comment": "Mobile share URL, user submission", + "#category": ("", "reddit", "redirect"), + "#class": reddit.RedditRedirectExtractor, + "#pattern": r"^https://www.reddit.com/user/Tailhook91/comments/znfxbr/prove_it/", + }, + { + "#url": "https://www.reddit.com/user/Tailhook91/s/w4yAMbtOYm", + "#comment": "Mobile share URL, user submission", + "#category": ("", "reddit", "redirect"), + "#class": reddit.RedditRedirectExtractor, + "#pattern": r"^https://www.reddit.com/user/Tailhook91/comments/znfxbr/prove_it/", + }, ) diff --git a/test/results/redgifs.py b/test/results/redgifs.py index 610e32ebfe..a17f4bc0cb 100644 --- a/test/results/redgifs.py +++ b/test/results/redgifs.py @@ -1,182 +1,156 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import redgifs - __tests__ = ( -{ - "#url" : "https://www.redgifs.com/users/mmj", - "#category": ("", "redgifs", "user"), - "#class" : redgifs.RedgifsUserExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "#count" : range(50, 60), -}, - -{ - "#url" : "https://www.redgifs.com/users/mmj?order=old", - "#comment" : "'order' URL parameter (#4583)", - "#category": ("", "redgifs", "user"), - "#class" : redgifs.RedgifsUserExtractor, - "#range" : "1-5", - "#patterns": ( - r"https://thumbs\d+\.redgifs\.com/ShoddyOilyHarlequinbug\.mp4", - r"https://thumbs\d+\.redgifs\.com/UnevenPrestigiousKilldeer\.mp4", - r"https://thumbs\d+\.redgifs\.com/EveryShockingFlickertailsquirrel\.mp4", - r"https://thumbs\d+\.redgifs\.com/NegativeWarlikeAmericancurl\.mp4", - r"https://thumbs\d+\.redgifs\.com/PopularTerribleFritillarybutterfly\.mp4", - ), -}, - -{ - "#url" : "https://v3.redgifs.com/users/lamsinka89", - "#comment" : "'v3' subdomain (#3588, #3589)", - "#category": ("", "redgifs", "user"), - "#class" : redgifs.RedgifsUserExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", - "#count" : ">= 100", -}, - -{ - "#url" : "https://www.redgifs.com/users/boombah123/collections/2631326bbd", - "#category": ("", "redgifs", "collection"), - "#class" : redgifs.RedgifsCollectionExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.redgifs.com/users/boombah123/collections/9e6f7dd41f", - "#category": ("", "redgifs", "collection"), - "#class" : redgifs.RedgifsCollectionExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.mp4", - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.redgifs.com/users/boombah123/collections", - "#category": ("", "redgifs", "collections"), - "#class" : redgifs.RedgifsCollectionsExtractor, - "#pattern" : r"https://www\.redgifs\.com/users/boombah123/collections/\w+", - "#count" : ">= 3", -}, - -{ - "#url" : "https://www.redgifs.com/niches/just-boobs", - "#category": ("", "redgifs", "niches"), - "#class" : redgifs.RedgifsNichesExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.redgifs.com/niches/thick-booty", - "#category": ("", "redgifs", "niches"), - "#class" : redgifs.RedgifsNichesExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://www.redgifs.com/gifs/jav", - "#category": ("", "redgifs", "search"), - "#class" : redgifs.RedgifsSearchExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.redgifs.com/browse?tags=JAV", - "#category": ("", "redgifs", "search"), - "#class" : redgifs.RedgifsSearchExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.redgifs.com/gifs/jav?order=best&verified=1", - "#category": ("", "redgifs", "search"), - "#class" : redgifs.RedgifsSearchExtractor, -}, - -{ - "#url" : "https://www.redgifs.com/browse?type=i&verified=y&order=top7", - "#category": ("", "redgifs", "search"), - "#class" : redgifs.RedgifsSearchExtractor, -}, - -{ - "#url" : "https://v3.redgifs.com/browse?tags=JAV", - "#category": ("", "redgifs", "search"), - "#class" : redgifs.RedgifsSearchExtractor, -}, - -{ - "#url" : "https://redgifs.com/watch/foolishforkedabyssiniancat", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/FoolishForkedAbyssiniancat\.mp4", - "#sha1_content": "f6e03f1df9a2ff2a74092f53ee7580d2fb943533", -}, - -{ - "#url" : "https://www.redgifs.com/watch/desertedbaregraywolf", - "#comment" : "gallery (#4021)", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, - "#pattern" : r"https://\w+\.redgifs\.com/[A-Za-z-]+\.jpg", - "#count" : 4, - - "num" : int, - "count" : 4, - "gallery": "187ad979693-1922-fc66-0000-a96fb07b8a5d", -}, - -{ - "#url" : "https://redgifs.com/ifr/FoolishForkedAbyssiniancat", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, -}, - -{ - "#url" : "https://i.redgifs.com/i/FoolishForkedAbyssiniancat", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, -}, - -{ - "#url" : "https://www.gifdeliverynetwork.com/foolishforkedabyssiniancat", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, -}, - -{ - "#url" : "https://v3.redgifs.com/watch/FoolishForkedAbyssiniancat", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, -}, - -{ - "#url" : "https://v3.redgifs.com/watch/605025947780972895", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, - - "id": "humblegrippingmole", -}, - -{ - "#url" : "https://www.gfycat.com/foolishforkedabyssiniancat", - "#category": ("", "redgifs", "image"), - "#class" : redgifs.RedgifsImageExtractor, -}, - + { + "#url": "https://www.redgifs.com/users/mmj", + "#category": ("", "redgifs", "user"), + "#class": redgifs.RedgifsUserExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#count": range(50, 60), + }, + { + "#url": "https://www.redgifs.com/users/mmj?order=old", + "#comment": "'order' URL parameter (#4583)", + "#category": ("", "redgifs", "user"), + "#class": redgifs.RedgifsUserExtractor, + "#range": "1-5", + "#patterns": ( + r"https://thumbs\d+\.redgifs\.com/ShoddyOilyHarlequinbug\.mp4", + r"https://thumbs\d+\.redgifs\.com/UnevenPrestigiousKilldeer\.mp4", + r"https://thumbs\d+\.redgifs\.com/EveryShockingFlickertailsquirrel\.mp4", + r"https://thumbs\d+\.redgifs\.com/NegativeWarlikeAmericancurl\.mp4", + r"https://thumbs\d+\.redgifs\.com/PopularTerribleFritillarybutterfly\.mp4", + ), + }, + { + "#url": "https://v3.redgifs.com/users/lamsinka89", + "#comment": "'v3' subdomain (#3588, #3589)", + "#category": ("", "redgifs", "user"), + "#class": redgifs.RedgifsUserExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", + "#count": ">= 100", + }, + { + "#url": "https://www.redgifs.com/users/boombah123/collections/2631326bbd", + "#category": ("", "redgifs", "collection"), + "#class": redgifs.RedgifsCollectionExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.redgifs.com/users/boombah123/collections/9e6f7dd41f", + "#category": ("", "redgifs", "collection"), + "#class": redgifs.RedgifsCollectionExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[\w-]+\.mp4", + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.redgifs.com/users/boombah123/collections", + "#category": ("", "redgifs", "collections"), + "#class": redgifs.RedgifsCollectionsExtractor, + "#pattern": r"https://www\.redgifs\.com/users/boombah123/collections/\w+", + "#count": ">= 3", + }, + { + "#url": "https://www.redgifs.com/niches/just-boobs", + "#category": ("", "redgifs", "niches"), + "#class": redgifs.RedgifsNichesExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.redgifs.com/niches/thick-booty", + "#category": ("", "redgifs", "niches"), + "#class": redgifs.RedgifsNichesExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[\w-]+\.(mp4|jpg)", + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://www.redgifs.com/gifs/jav", + "#category": ("", "redgifs", "search"), + "#class": redgifs.RedgifsSearchExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.redgifs.com/browse?tags=JAV", + "#category": ("", "redgifs", "search"), + "#class": redgifs.RedgifsSearchExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.(mp4|jpg)", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.redgifs.com/gifs/jav?order=best&verified=1", + "#category": ("", "redgifs", "search"), + "#class": redgifs.RedgifsSearchExtractor, + }, + { + "#url": "https://www.redgifs.com/browse?type=i&verified=y&order=top7", + "#category": ("", "redgifs", "search"), + "#class": redgifs.RedgifsSearchExtractor, + }, + { + "#url": "https://v3.redgifs.com/browse?tags=JAV", + "#category": ("", "redgifs", "search"), + "#class": redgifs.RedgifsSearchExtractor, + }, + { + "#url": "https://redgifs.com/watch/foolishforkedabyssiniancat", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + "#pattern": r"https://\w+\.redgifs\.com/FoolishForkedAbyssiniancat\.mp4", + "#sha1_content": "f6e03f1df9a2ff2a74092f53ee7580d2fb943533", + }, + { + "#url": "https://www.redgifs.com/watch/desertedbaregraywolf", + "#comment": "gallery (#4021)", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + "#pattern": r"https://\w+\.redgifs\.com/[A-Za-z-]+\.jpg", + "#count": 4, + "num": int, + "count": 4, + "gallery": "187ad979693-1922-fc66-0000-a96fb07b8a5d", + }, + { + "#url": "https://redgifs.com/ifr/FoolishForkedAbyssiniancat", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + }, + { + "#url": "https://i.redgifs.com/i/FoolishForkedAbyssiniancat", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + }, + { + "#url": "https://www.gifdeliverynetwork.com/foolishforkedabyssiniancat", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + }, + { + "#url": "https://v3.redgifs.com/watch/FoolishForkedAbyssiniancat", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + }, + { + "#url": "https://v3.redgifs.com/watch/605025947780972895", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + "id": "humblegrippingmole", + }, + { + "#url": "https://www.gfycat.com/foolishforkedabyssiniancat", + "#category": ("", "redgifs", "image"), + "#class": redgifs.RedgifsImageExtractor, + }, ) diff --git a/test/results/rule34.py b/test/results/rule34.py index f6b9b27413..73b9025c3c 100644 --- a/test/results/rule34.py +++ b/test/results/rule34.py @@ -1,84 +1,75 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v02 - __tests__ = ( -{ - "#url" : "https://rule34.xxx/index.php?page=post&s=list&tags=danraku", - "#category": ("gelbooru_v02", "rule34", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#pattern" : r"https?://.*rule34\.xxx/images/\d+/[0-9a-f]+\.jpg", - "#count" : 2, - "#sha1_content": [ - "5c6ae9ee13e6d4bc9cb8bdce224c84e67fbfa36c", - "622e80be3f496672c44aab5c47fbc6941c61bc79", - "1e0dced55bcb5eefe5cc32f69c7a8df35547b459", - ], -}, - -{ - "#url" : "https://rule34.xxx/index.php?page=pool&s=show&id=179", - "#category": ("gelbooru_v02", "rule34", "pool"), - "#class" : gelbooru_v02.GelbooruV02PoolExtractor, - "#count" : 3, -}, - -{ - "#url" : "https://rule34.xxx/index.php?page=favorites&s=view&id=1030218", - "#category": ("gelbooru_v02", "rule34", "favorite"), - "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, - "#count" : 3, -}, - -{ - "#url" : "https://www.rule34.xxx/index.php?page=post&s=view&id=863", - "#comment" : "www subdomain", - "#category": ("gelbooru_v02", "rule34", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, -}, - -{ - "#url" : "https://rule34.xxx/index.php?page=post&s=view&id=863", - "#category": ("gelbooru_v02", "rule34", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#options" : { - "tags" : True, - "notes": True, + { + "#url": "https://rule34.xxx/index.php?page=post&s=list&tags=danraku", + "#category": ("gelbooru_v02", "rule34", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#pattern": r"https?://.*rule34\.xxx/images/\d+/[0-9a-f]+\.jpg", + "#count": 2, + "#sha1_content": [ + "5c6ae9ee13e6d4bc9cb8bdce224c84e67fbfa36c", + "622e80be3f496672c44aab5c47fbc6941c61bc79", + "1e0dced55bcb5eefe5cc32f69c7a8df35547b459", + ], }, - "#pattern" : r"https://api-cdn\.rule34\.xxx/images/1/6aafbdb3e22f3f3b412ea2cf53321317a37063f3\.jpg", - "#sha1_content": [ - "a43f418aa350039af0d11cae501396a33bbe2201", - "67b516295950867e1c1ab6bc13b35d3b762ed2a3", - ], - - "tags_artist" : "reverse_noise yamu_(reverse_noise)", - "tags_character": "hong_meiling", - "tags_copyright": "team_shanghai_alice touhou", - "tags_general" : str, - "tags_metadata" : "censored translated", - "notes" : [ - { - "body" : "It feels angry, I'm losing myself... It won't calm down!", - "height": 65, - "id" : 93586, - "width" : 116, - "x" : 22, - "y" : 333, - }, - { - "body" : "REPUTATION OF RAGE", - "height": 272, - "id" : 93587, - "width" : 199, - "x" : 78, - "y" : 442, + { + "#url": "https://rule34.xxx/index.php?page=pool&s=show&id=179", + "#category": ("gelbooru_v02", "rule34", "pool"), + "#class": gelbooru_v02.GelbooruV02PoolExtractor, + "#count": 3, + }, + { + "#url": "https://rule34.xxx/index.php?page=favorites&s=view&id=1030218", + "#category": ("gelbooru_v02", "rule34", "favorite"), + "#class": gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count": 3, + }, + { + "#url": "https://www.rule34.xxx/index.php?page=post&s=view&id=863", + "#comment": "www subdomain", + "#category": ("gelbooru_v02", "rule34", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + }, + { + "#url": "https://rule34.xxx/index.php?page=post&s=view&id=863", + "#category": ("gelbooru_v02", "rule34", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#options": { + "tags": True, + "notes": True, }, - ], -}, - + "#pattern": r"https://api-cdn\.rule34\.xxx/images/1/6aafbdb3e22f3f3b412ea2cf53321317a37063f3\.jpg", + "#sha1_content": [ + "a43f418aa350039af0d11cae501396a33bbe2201", + "67b516295950867e1c1ab6bc13b35d3b762ed2a3", + ], + "tags_artist": "reverse_noise yamu_(reverse_noise)", + "tags_character": "hong_meiling", + "tags_copyright": "team_shanghai_alice touhou", + "tags_general": str, + "tags_metadata": "censored translated", + "notes": [ + { + "body": "It feels angry, I'm losing myself... It won't calm down!", + "height": 65, + "id": 93586, + "width": 116, + "x": 22, + "y": 333, + }, + { + "body": "REPUTATION OF RAGE", + "height": 272, + "id": 93587, + "width": 199, + "x": 78, + "y": 442, + }, + ], + }, ) diff --git a/test/results/rule34hentai.py b/test/results/rule34hentai.py index 1d3cb29122..f391db10e0 100644 --- a/test/results/rule34hentai.py +++ b/test/results/rule34hentai.py @@ -1,51 +1,44 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shimmie2 - __tests__ = ( -{ - "#url" : "https://rule34hentai.net/post/list/mizuki_kotora/1", - "#category": ("shimmie2", "rule34hentai", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, - "#urls" : ( - "https://rule34hentai.net/_images/7f3a411263d0f6de936e47ae8f9d35fb/332%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", - "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", - "https://rule34hentai.net/_images/09511511c4c9e9e1f9b795e059a60832/259%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", - ), - - "extension" : "jpeg", - "file_url" : r"re:https://rule34hentai.net/_images/.+\.jpeg", - "filename" : r"re:\d+ - \w+", - "height" : range(496, 875), - "id" : range(259, 332), - "md5" : r"re:^[0-9a-f]{32}$", - "search_tags": "mizuki_kotora", - "size" : int, - "tags" : str, - "width" : range(500, 850), -}, - -{ - "#url" : "https://rule34hentai.net/post/view/264", - "#category": ("shimmie2", "rule34hentai", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, - "#urls" : "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpg", - "#sha1_content": "6c23780bb78673cbff1bca9accb77ea11ec734f3", - - "extension": "jpg", - "file_url" : "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpg", - "filename" : "264 - Darkstalkers Felicia mizuki_kotora", - "height" : 875, - "id" : 264, - "md5" : "1a8eca7c04f8bf325bc993c5751a91c4", - "size" : 0, - "tags" : "Darkstalkers Felicia mizuki_kotora", - "width" : 657, -}, - + { + "#url": "https://rule34hentai.net/post/list/mizuki_kotora/1", + "#category": ("shimmie2", "rule34hentai", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + "#urls": ( + "https://rule34hentai.net/_images/7f3a411263d0f6de936e47ae8f9d35fb/332%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", + "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", + "https://rule34hentai.net/_images/09511511c4c9e9e1f9b795e059a60832/259%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpeg", + ), + "extension": "jpeg", + "file_url": r"re:https://rule34hentai.net/_images/.+\.jpeg", + "filename": r"re:\d+ - \w+", + "height": range(496, 875), + "id": range(259, 332), + "md5": r"re:^[0-9a-f]{32}$", + "search_tags": "mizuki_kotora", + "size": int, + "tags": str, + "width": range(500, 850), + }, + { + "#url": "https://rule34hentai.net/post/view/264", + "#category": ("shimmie2", "rule34hentai", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + "#urls": "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpg", + "#sha1_content": "6c23780bb78673cbff1bca9accb77ea11ec734f3", + "extension": "jpg", + "file_url": "https://rule34hentai.net/_images/1a8eca7c04f8bf325bc993c5751a91c4/264%20-%20Darkstalkers%20Felicia%20mizuki_kotora.jpg", + "filename": "264 - Darkstalkers Felicia mizuki_kotora", + "height": 875, + "id": 264, + "md5": "1a8eca7c04f8bf325bc993c5751a91c4", + "size": 0, + "tags": "Darkstalkers Felicia mizuki_kotora", + "width": 657, + }, ) diff --git a/test/results/rule34us.py b/test/results/rule34us.py index c3e193babe..2c620fa309 100644 --- a/test/results/rule34us.py +++ b/test/results/rule34us.py @@ -1,47 +1,40 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import rule34us - __tests__ = ( -{ - "#url" : "https://rule34.us/index.php?r=posts/index&q=[terios]_elysion", - "#category": ("booru", "rule34us", "tag"), - "#class" : rule34us.Rule34usTagExtractor, - "#pattern" : r"https://img\d*\.rule34\.us/images/../../[0-9a-f]{32}\.\w+", - "#count" : 10, -}, - -{ - "#url" : "https://rule34.us/index.php?r=posts/view&id=3709005", - "#category": ("booru", "rule34us", "post"), - "#class" : rule34us.Rule34usPostExtractor, - "#pattern" : r"https://img\d*\.rule34\.us/images/14/7b/147bee6fc2e13f73f5f9bac9d4930b13\.png", - "#sha1_content": "d714342ea84050f82dda5f0c194d677337abafc5", -}, - -{ - "#url" : "https://rule34.us/index.php?r=posts/view&id=4576310", - "#category": ("booru", "rule34us", "post"), - "#class" : rule34us.Rule34usPostExtractor, - "#pattern" : r"https://video-cdn\d\.rule34\.us/images/a2/94/a294ff8e1f8e0efa041e5dc9d1480011\.mp4", - - "extension" : "mp4", - "file_url" : str, - "filename" : "a294ff8e1f8e0efa041e5dc9d1480011", - "height" : "3982", - "id" : "4576310", - "md5" : "a294ff8e1f8e0efa041e5dc9d1480011", - "score" : r"re:\d+", - "tags" : "tagme, video", - "tags_general" : "video", - "tags_metadata": "tagme", - "uploader" : "Anonymous", - "width" : "3184", -}, - + { + "#url": "https://rule34.us/index.php?r=posts/index&q=[terios]_elysion", + "#category": ("booru", "rule34us", "tag"), + "#class": rule34us.Rule34usTagExtractor, + "#pattern": r"https://img\d*\.rule34\.us/images/../../[0-9a-f]{32}\.\w+", + "#count": 10, + }, + { + "#url": "https://rule34.us/index.php?r=posts/view&id=3709005", + "#category": ("booru", "rule34us", "post"), + "#class": rule34us.Rule34usPostExtractor, + "#pattern": r"https://img\d*\.rule34\.us/images/14/7b/147bee6fc2e13f73f5f9bac9d4930b13\.png", + "#sha1_content": "d714342ea84050f82dda5f0c194d677337abafc5", + }, + { + "#url": "https://rule34.us/index.php?r=posts/view&id=4576310", + "#category": ("booru", "rule34us", "post"), + "#class": rule34us.Rule34usPostExtractor, + "#pattern": r"https://video-cdn\d\.rule34\.us/images/a2/94/a294ff8e1f8e0efa041e5dc9d1480011\.mp4", + "extension": "mp4", + "file_url": str, + "filename": "a294ff8e1f8e0efa041e5dc9d1480011", + "height": "3982", + "id": "4576310", + "md5": "a294ff8e1f8e0efa041e5dc9d1480011", + "score": r"re:\d+", + "tags": "tagme, video", + "tags_general": "video", + "tags_metadata": "tagme", + "uploader": "Anonymous", + "width": "3184", + }, ) diff --git a/test/results/rule34vault.py b/test/results/rule34vault.py index 425ef7ed86..c52f142536 100644 --- a/test/results/rule34vault.py +++ b/test/results/rule34vault.py @@ -1,90 +1,82 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import rule34vault - __tests__ = ( -{ - "#url" : "https://rule34vault.com/sfw", - "#class": rule34vault.Rule34vaultTagExtractor, - "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://rule34vault.com/playlists/view/20164", - "#class": rule34vault.Rule34vaultPlaylistExtractor, - "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", - "#count" : 55, -}, - -{ - "#url" : "https://rule34vault.com/post/280517", - "#comment": "image", - "#class" : rule34vault.Rule34vaultPostExtractor, - "#options": {"tags": True}, - "#pattern" : "https://r34xyz.b-cdn.net/posts/280/280517/280517.jpg", - "#sha1_content": "1e19d601b4a79c06e6f885a83a5003e7e2a17057", - - "created" : "2023-09-01T11:57:57.317331Z", - "date" : "dt:2023-09-01 11:57:57", - "extension" : "jpg", - "file_url" : "https://r34xyz.b-cdn.net/posts/280/280517/280517.jpg", - "filename" : "280517", - "height" : 1152, - "id" : 280517, - "likes" : range(3, 100), - "posted" : "2023-09-01T12:01:41.008547Z", - "status" : 2, - "type" : 0, - "uploaderId": 20678, - "views" : range(90, 999), - "width" : 768, - "data": { - "sources": [ - "https://trynectar.ai/view/87c98fc8-e4f3-447c-a0d3-024b1890580a", + { + "#url": "https://rule34vault.com/sfw", + "#class": rule34vault.Rule34vaultTagExtractor, + "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://rule34vault.com/playlists/view/20164", + "#class": rule34vault.Rule34vaultPlaylistExtractor, + "#pattern": r"https://r34xyz\.b-cdn\.net/posts/\d+/\d+/\d+\.(jpg|mp4)", + "#count": 55, + }, + { + "#url": "https://rule34vault.com/post/280517", + "#comment": "image", + "#class": rule34vault.Rule34vaultPostExtractor, + "#options": {"tags": True}, + "#pattern": "https://r34xyz.b-cdn.net/posts/280/280517/280517.jpg", + "#sha1_content": "1e19d601b4a79c06e6f885a83a5003e7e2a17057", + "created": "2023-09-01T11:57:57.317331Z", + "date": "dt:2023-09-01 11:57:57", + "extension": "jpg", + "file_url": "https://r34xyz.b-cdn.net/posts/280/280517/280517.jpg", + "filename": "280517", + "height": 1152, + "id": 280517, + "likes": range(3, 100), + "posted": "2023-09-01T12:01:41.008547Z", + "status": 2, + "type": 0, + "uploaderId": 20678, + "views": range(90, 999), + "width": 768, + "data": { + "sources": [ + "https://trynectar.ai/view/87c98fc8-e4f3-447c-a0d3-024b1890580a", + ], + }, + "tags": [ + "ai generated", + "demon slayer", + "kamado nezuko", + "school uniform", + "sfw", + ], + "tags_character": [ + "kamado nezuko", + ], + "tags_copyright": [ + "demon slayer", ], + "tags_general": [ + "ai generated", + "school uniform", + "sfw", + ], + "uploader": { + "created": "2023-07-24T04:33:36.734495Z", + "data": None, + "displayName": "quick1e", + "emailVerified": False, + "id": 20678, + "role": 1, + "userName": "quick1e", + }, }, - "tags": [ - "ai generated", - "demon slayer", - "kamado nezuko", - "school uniform", - "sfw", - ], - "tags_character": [ - "kamado nezuko", - ], - "tags_copyright": [ - "demon slayer", - ], - "tags_general": [ - "ai generated", - "school uniform", - "sfw", - ], - "uploader": { - "created" : "2023-07-24T04:33:36.734495Z", - "data" : None, - "displayName" : "quick1e", - "emailVerified": False, - "id" : 20678, - "role" : 1, - "userName" : "quick1e", + { + "#url": "https://rule34vault.com/post/382937", + "#comment": "video", + "#class": rule34vault.Rule34vaultPostExtractor, + "#urls": "https://r34xyz.b-cdn.net/posts/382/382937/382937.mp4", + "#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af", }, -}, - -{ - "#url" : "https://rule34vault.com/post/382937", - "#comment": "video", - "#class" : rule34vault.Rule34vaultPostExtractor, - "#urls" : "https://r34xyz.b-cdn.net/posts/382/382937/382937.mp4", - "#sha1_content": "b962e3e2304139767c3792508353e6e83a85a2af", -}, - ) diff --git a/test/results/rule34xyz.py b/test/results/rule34xyz.py index 8a07c5dc31..58fe9bd5b8 100644 --- a/test/results/rule34xyz.py +++ b/test/results/rule34xyz.py @@ -8,135 +8,125 @@ __tests__ = ( -{ - "#url" : "https://rule34.xyz/sfw", - "#class": rule34xyz.Rule34xyzTagExtractor, - "#pattern": r"https://rule34(\.xyz|xyz\.b-cdn\.net)/posts/\d+/\d+/\d+\.(pic|mov\d*)\.(jpg|mp4)", - "#range" : "1-150", - "#count" : 150, - - "search_tags": "sfw", -}, - -{ - "#url" : "https://rule34.xyz/playlists/view/119", - "#class": rule34xyz.Rule34xyzPlaylistExtractor, - "#pattern": r"https://rule34(\.xyz|xyz\.b-cdn\.net)/posts/\d+/\d+/\d+\.(pic|mov\d*)\.(jpg|mp4)", - "#count" : 64, - - "playlist_id": "119", -}, - -{ - "#url" : "https://rule34.xyz/post/3613851", - "#comment": "image", - "#class" : rule34xyz.Rule34xyzPostExtractor, - "#options" : {"tags": True}, - "#urls" : "https://rule34xyz.b-cdn.net/posts/3613/3613851/3613851.pic.jpg", - "#sha1_content": "4d7146db258fd5b1645a1a5fc01550d102f495e1", - - "attributes": 1, - "comments" : 0, - "created" : "2023-03-29T06:00:59.136819", - "date" : "dt:2023-03-29 06:00:59", - "duration" : None, - "error" : None, - "extension" : "jpg", - "file_url" : "https://rule34xyz.b-cdn.net/posts/3613/3613851/3613851.pic.jpg", - "filename" : "3613851.pic", - "format" : "pic", - "format_id" : "2", - "id" : 3613851, - "likes" : range(3, 100), - "posted" : "2023-03-29T06:01:07.900161", - "type" : 0, - "uploaderId": 9741, - "views" : range(200, 2000), - "status" : 2, - "files" : dict, - "sources": [ - "https://twitter.com/DesireDelta13/status/1636502494292373505?t=OrmlnC85cELyY5BPmBy9Hw&s=19", - ], - "tags": [ - "doki doki literature club", - "doki doki takeover", - "friday night funkin", - "friday night funkin mod", - "yuri (doki doki literature club)", - "desiredelta", - "1girls", - "big breasts", - "clothed", - "clothed female", - "female", - "female focus", - "female only", - "holding microphone", - "holding object", - "long hair", - "long purple hair", - "looking at viewer", - "microphone", - "open hand", - "open mouth", - "purple background", - "purple hair", - "solo", - "solo female", - "solo focus", - "sweater", - "white outline", - "jpeg", - "safe for work", - "sfw", - ], - "tags_artist": [ - "desiredelta", - ], - "tags_character": [ - "yuri (doki doki literature club)", - ], - "tags_copyright": [ - "doki doki literature club", - "friday night funkin", - "friday night funkin mod", - ], - "tags_general": list, - "uploader": { - "avatarUrl" : None, - "bookmarks" : 0, - "certified" : True, - "created" : "2021-04-03T08:29:51.373823", - "email" : "agent.rulexxx-uploader@z.com", - "id" : 9741, - "isSystemAccount": True, - "name" : "agent.rulexxx-uploader", - "role" : 2, - "uploadedPosts" : range(100000, 999999), - "webId" : None, + { + "#url": "https://rule34.xyz/sfw", + "#class": rule34xyz.Rule34xyzTagExtractor, + "#pattern": r"https://rule34(\.xyz|xyz\.b-cdn\.net)/posts/\d+/\d+/\d+\.(pic|mov\d*)\.(jpg|mp4)", + "#range": "1-150", + "#count": 150, + "search_tags": "sfw", + }, + { + "#url": "https://rule34.xyz/playlists/view/119", + "#class": rule34xyz.Rule34xyzPlaylistExtractor, + "#pattern": r"https://rule34(\.xyz|xyz\.b-cdn\.net)/posts/\d+/\d+/\d+\.(pic|mov\d*)\.(jpg|mp4)", + "#count": 64, + "playlist_id": "119", + }, + { + "#url": "https://rule34.xyz/post/3613851", + "#comment": "image", + "#class": rule34xyz.Rule34xyzPostExtractor, + "#options": {"tags": True}, + "#urls": "https://rule34xyz.b-cdn.net/posts/3613/3613851/3613851.pic.jpg", + "#sha1_content": "4d7146db258fd5b1645a1a5fc01550d102f495e1", + "attributes": 1, + "comments": 0, + "created": "2023-03-29T06:00:59.136819", + "date": "dt:2023-03-29 06:00:59", + "duration": None, + "error": None, + "extension": "jpg", + "file_url": "https://rule34xyz.b-cdn.net/posts/3613/3613851/3613851.pic.jpg", + "filename": "3613851.pic", + "format": "pic", + "format_id": "2", + "id": 3613851, + "likes": range(3, 100), + "posted": "2023-03-29T06:01:07.900161", + "type": 0, + "uploaderId": 9741, + "views": range(200, 2000), + "status": 2, + "files": dict, + "sources": [ + "https://twitter.com/DesireDelta13/status/1636502494292373505?t=OrmlnC85cELyY5BPmBy9Hw&s=19", + ], + "tags": [ + "doki doki literature club", + "doki doki takeover", + "friday night funkin", + "friday night funkin mod", + "yuri (doki doki literature club)", + "desiredelta", + "1girls", + "big breasts", + "clothed", + "clothed female", + "female", + "female focus", + "female only", + "holding microphone", + "holding object", + "long hair", + "long purple hair", + "looking at viewer", + "microphone", + "open hand", + "open mouth", + "purple background", + "purple hair", + "solo", + "solo female", + "solo focus", + "sweater", + "white outline", + "jpeg", + "safe for work", + "sfw", + ], + "tags_artist": [ + "desiredelta", + ], + "tags_character": [ + "yuri (doki doki literature club)", + ], + "tags_copyright": [ + "doki doki literature club", + "friday night funkin", + "friday night funkin mod", + ], + "tags_general": list, + "uploader": { + "avatarUrl": None, + "bookmarks": 0, + "certified": True, + "created": "2021-04-03T08:29:51.373823", + "email": "agent.rulexxx-uploader@z.com", + "id": 9741, + "isSystemAccount": True, + "name": "agent.rulexxx-uploader", + "role": 2, + "uploadedPosts": range(100000, 999999), + "webId": None, + }, + }, + { + "#url": "https://rule34.xyz/post/3571567", + "#comment": "video", + "#class": rule34xyz.Rule34xyzPostExtractor, + "#urls": "https://rule34xyz.b-cdn.net/posts/3571/3571567/3571567.mov720.mp4", + "#sha1_content": "c0a5e7e887774f91527f00e6142c435a3c482c1f", + "format": "mov720", + "format_id": "40", + }, + { + "#url": "https://rule34.xyz/post/3571567", + "#comment": "'format' option", + "#class": rule34xyz.Rule34xyzPostExtractor, + "#options": {"format": "10,33"}, + "#urls": "https://rule34xyz.b-cdn.net/posts/3571/3571567/3571567.pic256avif.avif", + "format": "pic256avif", + "format_id": "33", }, -}, - -{ - "#url" : "https://rule34.xyz/post/3571567", - "#comment": "video", - "#class" : rule34xyz.Rule34xyzPostExtractor, - "#urls" : "https://rule34xyz.b-cdn.net/posts/3571/3571567/3571567.mov720.mp4", - "#sha1_content": "c0a5e7e887774f91527f00e6142c435a3c482c1f", - - "format" : "mov720", - "format_id" : "40", -}, - -{ - "#url" : "https://rule34.xyz/post/3571567", - "#comment": "'format' option", - "#class" : rule34xyz.Rule34xyzPostExtractor, - "#options": {"format": "10,33"}, - "#urls" : "https://rule34xyz.b-cdn.net/posts/3571/3571567/3571567.pic256avif.avif", - - "format" : "pic256avif", - "format_id" : "33", -}, - ) diff --git a/test/results/safebooru.py b/test/results/safebooru.py index ec92cc0732..662056eb6e 100644 --- a/test/results/safebooru.py +++ b/test/results/safebooru.py @@ -1,61 +1,51 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v02 - __tests__ = ( -{ - "#url" : "https://safebooru.org/index.php?page=post&s=list&tags=bonocho", - "#category": ("gelbooru_v02", "safebooru", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#sha1_url" : "17c61b386530cf4c30842c9f580d15ef1cd09586", - "#sha1_content": "e5ad4c5bf241b1def154958535bef6c2f6b733eb", -}, - -{ - "#url" : "https://safebooru.org/index.php?page=post&s=list&tags=all", - "#category": ("gelbooru_v02", "safebooru", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#range" : "1-3", - "#count" : 3, -}, - -{ - "#url" : "https://safebooru.org/index.php?page=post&s=list&tags=", - "#category": ("gelbooru_v02", "safebooru", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, -}, - -{ - "#url" : "https://safebooru.org/index.php?page=pool&s=show&id=11", - "#category": ("gelbooru_v02", "safebooru", "pool"), - "#class" : gelbooru_v02.GelbooruV02PoolExtractor, - "#count" : 5, -}, - -{ - "#url" : "https://safebooru.org/index.php?page=favorites&s=view&id=17567", - "#category": ("gelbooru_v02", "safebooru", "favorite"), - "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://safebooru.org/index.php?page=post&s=view&id=1169132", - "#category": ("gelbooru_v02", "safebooru", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#options" : {"tags": True}, - "#sha1_url" : "cf05e37a3c62b2d55788e2080b8eabedb00f999b", - "#sha1_content": "93b293b27dabd198afafabbaf87c49863ac82f27", - - "tags_artist" : "kawanakajima", - "tags_character": "heath_ledger ronald_mcdonald the_joker", - "tags_copyright": "dc_comics mcdonald's the_dark_knight", - "tags_metadata" : "parody tagme", -}, - + { + "#url": "https://safebooru.org/index.php?page=post&s=list&tags=bonocho", + "#category": ("gelbooru_v02", "safebooru", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#sha1_url": "17c61b386530cf4c30842c9f580d15ef1cd09586", + "#sha1_content": "e5ad4c5bf241b1def154958535bef6c2f6b733eb", + }, + { + "#url": "https://safebooru.org/index.php?page=post&s=list&tags=all", + "#category": ("gelbooru_v02", "safebooru", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#range": "1-3", + "#count": 3, + }, + { + "#url": "https://safebooru.org/index.php?page=post&s=list&tags=", + "#category": ("gelbooru_v02", "safebooru", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + }, + { + "#url": "https://safebooru.org/index.php?page=pool&s=show&id=11", + "#category": ("gelbooru_v02", "safebooru", "pool"), + "#class": gelbooru_v02.GelbooruV02PoolExtractor, + "#count": 5, + }, + { + "#url": "https://safebooru.org/index.php?page=favorites&s=view&id=17567", + "#category": ("gelbooru_v02", "safebooru", "favorite"), + "#class": gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count": 2, + }, + { + "#url": "https://safebooru.org/index.php?page=post&s=view&id=1169132", + "#category": ("gelbooru_v02", "safebooru", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#options": {"tags": True}, + "#sha1_url": "cf05e37a3c62b2d55788e2080b8eabedb00f999b", + "#sha1_content": "93b293b27dabd198afafabbaf87c49863ac82f27", + "tags_artist": "kawanakajima", + "tags_character": "heath_ledger ronald_mcdonald the_joker", + "tags_copyright": "dc_comics mcdonald's the_dark_knight", + "tags_metadata": "parody tagme", + }, ) diff --git a/test/results/saint.py b/test/results/saint.py index 7f14b317e8..0fef239a04 100644 --- a/test/results/saint.py +++ b/test/results/saint.py @@ -1,83 +1,73 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import saint - __tests__ = ( -{ - "#url" : "https://saint2.su/a/2c5iuWHTumH", - "#class": saint.SaintAlbumExtractor, - "#urls" : ( - "https://cold1.saint2.cr/videos/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", - "https://cold1.saint2.cr/videos/3b125e3fb4b98693f17d85cb53590215.mp4", - ), - - "album_id" : "2c5iuWHTumH", - "album_name" : "animations", - "album_size" : 37083862, - "count" : 2, - "date" : "type:datetime", - "description": "Descriptions can contain only alphanumeric ASCII characters", - "extension" : "mp4", - "file" : r"re:https://...", - "filename" : {"3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", - "3b125e3fb4b98693f17d85cb53590215-ze10Ohbpoy5"}, - "id" : {"6lC7mKrJst8", - "ze10Ohbpoy5"}, - "id2" : {"6712834015d67", - "671284a627e0e"}, - "id_dl" : {"M2IxY2NlYmYzNTc2ZjhkNWFhYzNlZTBlNWExMmRhOTUubXA0", - "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0"}, - "name" : {"3b1ccebf3576f8d5aac3ee0e5a12da95", - "3b125e3fb4b98693f17d85cb53590215"}, - "num" : {1, 2}, -}, - -{ - "#url" : "https://saint2.su/embed/6lC7mKrJst8", - "#class": saint.SaintMediaExtractor, - "#urls" : "https://cold1.saint2.cr/videos/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", - "#sha1_content": "39037a029b3fe96f838b4545316caaa545c84075", - - "count" : 1, - "date" : "dt:2024-10-18 15:48:16", - "extension": "mp4", - "file" : "https://cold1.saint2.cr/videos/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", - "filename" : "3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", - "id" : "6lC7mKrJst8", - "id2" : "6712834015d67", - "id_dl" : "M2IxY2NlYmYzNTc2ZjhkNWFhYzNlZTBlNWExMmRhOTUubXA0", - "name" : "3b1ccebf3576f8d5aac3ee0e5a12da95", - "num" : 1, -}, - -{ - "#url" : "https://saint2.su/d/M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "#class": saint.SaintMediaExtractor, - "#urls" : "https://cold1.saint2.cr/api/download.php?file=M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - - "count" : 1, - "extension": "mp4", - "file" : "https://cold1.saint2.cr/api/download.php?file=M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "filename" : "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "id" : "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "id_dl" : "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "name" : "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", - "num" : 1, -}, - -{ - "#url" : "https://saint2.pk/embed/6lC7mKrJst8", - "#class": saint.SaintMediaExtractor, -}, - -{ - "#url" : "https://saint.to/embed/6lC7mKrJst8", - "#class": saint.SaintMediaExtractor, -}, - + { + "#url": "https://saint2.su/a/2c5iuWHTumH", + "#class": saint.SaintAlbumExtractor, + "#urls": ( + "https://cold1.saint2.cr/videos/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", + "https://cold1.saint2.cr/videos/3b125e3fb4b98693f17d85cb53590215.mp4", + ), + "album_id": "2c5iuWHTumH", + "album_name": "animations", + "album_size": 37083862, + "count": 2, + "date": "type:datetime", + "description": "Descriptions can contain only alphanumeric ASCII characters", + "extension": "mp4", + "file": r"re:https://...", + "filename": { + "3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", + "3b125e3fb4b98693f17d85cb53590215-ze10Ohbpoy5", + }, + "id": {"6lC7mKrJst8", "ze10Ohbpoy5"}, + "id2": {"6712834015d67", "671284a627e0e"}, + "id_dl": { + "M2IxY2NlYmYzNTc2ZjhkNWFhYzNlZTBlNWExMmRhOTUubXA0", + "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + }, + "name": {"3b1ccebf3576f8d5aac3ee0e5a12da95", "3b125e3fb4b98693f17d85cb53590215"}, + "num": {1, 2}, + }, + { + "#url": "https://saint2.su/embed/6lC7mKrJst8", + "#class": saint.SaintMediaExtractor, + "#urls": "https://cold1.saint2.cr/videos/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", + "#sha1_content": "39037a029b3fe96f838b4545316caaa545c84075", + "count": 1, + "date": "dt:2024-10-18 15:48:16", + "extension": "mp4", + "file": "https://cold1.saint2.cr/videos/3b1ccebf3576f8d5aac3ee0e5a12da95.mp4", + "filename": "3b1ccebf3576f8d5aac3ee0e5a12da95-6lC7mKrJst8", + "id": "6lC7mKrJst8", + "id2": "6712834015d67", + "id_dl": "M2IxY2NlYmYzNTc2ZjhkNWFhYzNlZTBlNWExMmRhOTUubXA0", + "name": "3b1ccebf3576f8d5aac3ee0e5a12da95", + "num": 1, + }, + { + "#url": "https://saint2.su/d/M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "#class": saint.SaintMediaExtractor, + "#urls": "https://cold1.saint2.cr/api/download.php?file=M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "count": 1, + "extension": "mp4", + "file": "https://cold1.saint2.cr/api/download.php?file=M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "filename": "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "id": "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "id_dl": "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "name": "M2IxMjVlM2ZiNGI5ODY5M2YxN2Q4NWNiNTM1OTAyMTUubXA0", + "num": 1, + }, + { + "#url": "https://saint2.pk/embed/6lC7mKrJst8", + "#class": saint.SaintMediaExtractor, + }, + { + "#url": "https://saint.to/embed/6lC7mKrJst8", + "#class": saint.SaintMediaExtractor, + }, ) diff --git a/test/results/sakugabooru.py b/test/results/sakugabooru.py index 73d3831688..5e6568bf01 100644 --- a/test/results/sakugabooru.py +++ b/test/results/sakugabooru.py @@ -1,35 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import moebooru - __tests__ = ( -{ - "#url" : "https://www.sakugabooru.com/post/show/125570", - "#category": ("moebooru", "sakugabooru", "post"), - "#class" : moebooru.MoebooruPostExtractor, -}, - -{ - "#url" : "https://www.sakugabooru.com/post?tags=nichijou", - "#category": ("moebooru", "sakugabooru", "tag"), - "#class" : moebooru.MoebooruTagExtractor, -}, - -{ - "#url" : "https://www.sakugabooru.com/pool/show/54", - "#category": ("moebooru", "sakugabooru", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, -}, - -{ - "#url" : "https://www.sakugabooru.com/post/popular_recent", - "#category": ("moebooru", "sakugabooru", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, -}, - + { + "#url": "https://www.sakugabooru.com/post/show/125570", + "#category": ("moebooru", "sakugabooru", "post"), + "#class": moebooru.MoebooruPostExtractor, + }, + { + "#url": "https://www.sakugabooru.com/post?tags=nichijou", + "#category": ("moebooru", "sakugabooru", "tag"), + "#class": moebooru.MoebooruTagExtractor, + }, + { + "#url": "https://www.sakugabooru.com/pool/show/54", + "#category": ("moebooru", "sakugabooru", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + }, + { + "#url": "https://www.sakugabooru.com/post/popular_recent", + "#category": ("moebooru", "sakugabooru", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + }, ) diff --git a/test/results/sankaku.py b/test/results/sankaku.py index e19ea67bef..6f31b52ea8 100644 --- a/test/results/sankaku.py +++ b/test/results/sankaku.py @@ -1,309 +1,266 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import sankaku from gallery_dl import exception - +from gallery_dl.extractor import sankaku __tests__ = ( -{ - "#url" : "https://sankaku.app/?tags=bonocho", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, - "#pattern" : r"https://s\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[0-9a-f]{32}\.\w+\?e=\d+&(expires=\d+&)?m=[^&#]+", - "#count" : 5, -}, - -{ - "#url" : "https://www.sankakucomplex.com/?tags=bonocho", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://beta.sankakucomplex.com/?tags=bonocho", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/?tags=bonocho", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://black.sankakucomplex.com/?tags=bonocho", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://white.sankakucomplex.com/?tags=bonocho", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://sankaku.app/ja?tags=order%3Apopularity", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://sankaku.app/no/?tags=order%3Apopularity", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/posts?tags=TAG", - "#comment" : "'/posts' in tag search URL (#4740)", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/ja/posts/?tags=あえいおう", - "#comment" : "'/posts' in tag search URL (#4740)", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/?tags=bonocho+a+b+c+d", - "#comment" : "error on five or more tags", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, - "#options" : {"username": None}, - "#exception": exception.StopExtraction, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/?tags=marie_rose&page=98&next=3874906&commit=Search", - "#comment" : "match arbitrary query parameters", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/?tags=date:2023-03-20", - "#comment" : "'date:' tags (#1790)", - "#category": ("booru", "sankaku", "tag"), - "#class" : sankaku.SankakuTagExtractor, - "#range" : "1", - "#count" : 1, -}, - -{ - "#url" : "https://sankaku.app/books/90", - "#category": ("booru", "sankaku", "pool"), - "#class" : sankaku.SankakuPoolExtractor, - "#count" : 5, -}, - -{ - "#url" : "https://www.sankakucomplex.com/books/90", - "#category": ("booru", "sankaku", "pool"), - "#class" : sankaku.SankakuPoolExtractor, -}, - -{ - "#url" : "https://beta.sankakucomplex.com/books/90", - "#category": ("booru", "sankaku", "pool"), - "#class" : sankaku.SankakuPoolExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/pool/show/90", - "#category": ("booru", "sankaku", "pool"), - "#class" : sankaku.SankakuPoolExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/pools/show/90", - "#category": ("booru", "sankaku", "pool"), - "#class" : sankaku.SankakuPoolExtractor, -}, - -{ - "#url" : "https://sankaku.app/posts/y0abGlDOr2o", - "#comment" : "extended tag categories; alphanumeric ID (#5073)", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#options" : { - "tags" : True, - "notes" : True, - "id-format": "alphanumeric", - }, - "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", - - "id": "y0abGlDOr2o", - "notes": (), - "tags_artist": [ - "bonocho", - ], - "tags_character": [ - "batman", - "letty_whiterock", - "bruce_wayne", - "the_joker", - "heath_ledger", - ], - "tags_copyright": [ - "batman_(series)", - "the_dark_knight", - ], - "tags_studio": [ - "dc_comics", - ], - "tags_general": list, -}, - -{ - "#url" : "https://sankaku.app/posts/VAr2mjLJ2av", - "#comment" : "notes (#5073)", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#options" : {"notes": True}, - - "notes": [ - { - "body" : "A lonely person, is a lonely person, because he or she is lonely.", - "created_at": 1643733759, - "creator_id": 1370766, - "height" : 871, - "id" : 1832643, - "is_active" : True, - "post_id" : 23688624, - "updated_at": 1643733759, - "width" : 108, - "x" : 703, - "y" : 83, + { + "#url": "https://sankaku.app/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + "#pattern": r"https://s\.sankakucomplex\.com/data/[^/]{2}/[^/]{2}/[0-9a-f]{32}\.\w+\?e=\d+&(expires=\d+&)?m=[^&#]+", + "#count": 5, + }, + { + "#url": "https://www.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://beta.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://black.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://white.sankakucomplex.com/?tags=bonocho", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://sankaku.app/ja?tags=order%3Apopularity", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://sankaku.app/no/?tags=order%3Apopularity", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/posts?tags=TAG", + "#comment": "'/posts' in tag search URL (#4740)", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/ja/posts/?tags=あえいおう", + "#comment": "'/posts' in tag search URL (#4740)", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/?tags=bonocho+a+b+c+d", + "#comment": "error on five or more tags", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + "#options": {"username": None}, + "#exception": exception.StopExtraction, + }, + { + "#url": "https://chan.sankakucomplex.com/?tags=marie_rose&page=98&next=3874906&commit=Search", + "#comment": "match arbitrary query parameters", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/?tags=date:2023-03-20", + "#comment": "'date:' tags (#1790)", + "#category": ("booru", "sankaku", "tag"), + "#class": sankaku.SankakuTagExtractor, + "#range": "1", + "#count": 1, + }, + { + "#url": "https://sankaku.app/books/90", + "#category": ("booru", "sankaku", "pool"), + "#class": sankaku.SankakuPoolExtractor, + "#count": 5, + }, + { + "#url": "https://www.sankakucomplex.com/books/90", + "#category": ("booru", "sankaku", "pool"), + "#class": sankaku.SankakuPoolExtractor, + }, + { + "#url": "https://beta.sankakucomplex.com/books/90", + "#category": ("booru", "sankaku", "pool"), + "#class": sankaku.SankakuPoolExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/pool/show/90", + "#category": ("booru", "sankaku", "pool"), + "#class": sankaku.SankakuPoolExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/pools/show/90", + "#category": ("booru", "sankaku", "pool"), + "#class": sankaku.SankakuPoolExtractor, + }, + { + "#url": "https://sankaku.app/posts/y0abGlDOr2o", + "#comment": "extended tag categories; alphanumeric ID (#5073)", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#options": { + "tags": True, + "notes": True, + "id-format": "alphanumeric", }, - ], -}, - -{ - "#url" : "https://sankaku.app/post/show/360451", - "#comment" : "legacy post URL", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#pattern" : r"https://s\.sankakucomplex\.com/data/ac/8e/ac8e3b92ea328ce9cf7211e69c905bf9\.jpg\?e=.+", - - "id": 360451, -}, - -{ - "#url" : "https://sankaku.app/post/show/21418978", - "#comment" : "'contentious_content'", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#auth" : True, - "#pattern" : r"https://s\.sankakucomplex\.com/data/13/3c/133cda3bfde249c504284493903fb985\.jpg", -}, - -{ - "#url" : "https://sankaku.app/post/show/20758561", - "#comment" : "empty tags (#1617)", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#options" : {"tags": True}, - "#count" : 1, - - "id" : 20758561, - "tags" : list, - "tags_general": [ - "key(mangaka)", - "key(mangaka)", - "english_language", - "english_language", - "high_resolution", - "tagme", - "very_high_resolution", - "large_filesize", - ], -}, - -{ - "#url" : "https://chan.sankakucomplex.com/post/show/f8ba89043078f0e4be2d9c46550b840a", - "#comment" : "md5 hexdigest instead of ID (#3952)", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#pattern" : r"https://s\.sankakucomplex\.com/data/f8/ba/f8ba89043078f0e4be2d9c46550b840a\.jpg", - "#count" : 1, - - "id" : 33195194, - "md5": "f8ba89043078f0e4be2d9c46550b840a", -}, - -{ - "#url" : "https://chan.sankakucomplex.com/posts/f8ba89043078f0e4be2d9c46550b840a", - "#comment" : "/posts/ instead of /post/show/ (#4688)", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - "#pattern" : r"https://s\.sankakucomplex\.com/data/f8/ba/f8ba89043078f0e4be2d9c46550b840a\.jpg", - "#count" : 1, - - "id" : 33195194, - "md5": "f8ba89043078f0e4be2d9c46550b840a", -}, - -{ - "#url" : "https://chan.sankakucomplex.com/en/posts/show/ac8e3b92ea328ce9cf7211e69c905bf9", - "#comment" : "/en/posts/show/HEX", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, - - "id" : 360451, - "md5": "ac8e3b92ea328ce9cf7211e69c905bf9", -}, - -{ - "#url" : "https://chan.sankakucomplex.com/post/show/360451", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, -}, - -{ - "#url" : "https://chan.sankakucomplex.com/ja/post/show/360451", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, -}, - -{ - "#url" : "https://beta.sankakucomplex.com/post/show/360451", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, -}, - -{ - "#url" : "https://white.sankakucomplex.com/post/show/360451", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, -}, - -{ - "#url" : "https://black.sankakucomplex.com/post/show/360451", - "#category": ("booru", "sankaku", "post"), - "#class" : sankaku.SankakuPostExtractor, -}, - -{ - "#url" : "https://sankaku.app/books?tags=aiue_oka", - "#category": ("booru", "sankaku", "books"), - "#class" : sankaku.SankakuBooksExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://beta.sankakucomplex.com/books?tags=aiue_oka", - "#category": ("booru", "sankaku", "books"), - "#class" : sankaku.SankakuBooksExtractor, -}, - + "#sha1_content": "5e255713cbf0a8e0801dc423563c34d896bb9229", + "id": "y0abGlDOr2o", + "notes": (), + "tags_artist": [ + "bonocho", + ], + "tags_character": [ + "batman", + "letty_whiterock", + "bruce_wayne", + "the_joker", + "heath_ledger", + ], + "tags_copyright": [ + "batman_(series)", + "the_dark_knight", + ], + "tags_studio": [ + "dc_comics", + ], + "tags_general": list, + }, + { + "#url": "https://sankaku.app/posts/VAr2mjLJ2av", + "#comment": "notes (#5073)", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#options": {"notes": True}, + "notes": [ + { + "body": "A lonely person, is a lonely person, because he or she is lonely.", + "created_at": 1643733759, + "creator_id": 1370766, + "height": 871, + "id": 1832643, + "is_active": True, + "post_id": 23688624, + "updated_at": 1643733759, + "width": 108, + "x": 703, + "y": 83, + }, + ], + }, + { + "#url": "https://sankaku.app/post/show/360451", + "#comment": "legacy post URL", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#pattern": r"https://s\.sankakucomplex\.com/data/ac/8e/ac8e3b92ea328ce9cf7211e69c905bf9\.jpg\?e=.+", + "id": 360451, + }, + { + "#url": "https://sankaku.app/post/show/21418978", + "#comment": "'contentious_content'", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#auth": True, + "#pattern": r"https://s\.sankakucomplex\.com/data/13/3c/133cda3bfde249c504284493903fb985\.jpg", + }, + { + "#url": "https://sankaku.app/post/show/20758561", + "#comment": "empty tags (#1617)", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#options": {"tags": True}, + "#count": 1, + "id": 20758561, + "tags": list, + "tags_general": [ + "key(mangaka)", + "key(mangaka)", + "english_language", + "english_language", + "high_resolution", + "tagme", + "very_high_resolution", + "large_filesize", + ], + }, + { + "#url": "https://chan.sankakucomplex.com/post/show/f8ba89043078f0e4be2d9c46550b840a", + "#comment": "md5 hexdigest instead of ID (#3952)", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#pattern": r"https://s\.sankakucomplex\.com/data/f8/ba/f8ba89043078f0e4be2d9c46550b840a\.jpg", + "#count": 1, + "id": 33195194, + "md5": "f8ba89043078f0e4be2d9c46550b840a", + }, + { + "#url": "https://chan.sankakucomplex.com/posts/f8ba89043078f0e4be2d9c46550b840a", + "#comment": "/posts/ instead of /post/show/ (#4688)", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "#pattern": r"https://s\.sankakucomplex\.com/data/f8/ba/f8ba89043078f0e4be2d9c46550b840a\.jpg", + "#count": 1, + "id": 33195194, + "md5": "f8ba89043078f0e4be2d9c46550b840a", + }, + { + "#url": "https://chan.sankakucomplex.com/en/posts/show/ac8e3b92ea328ce9cf7211e69c905bf9", + "#comment": "/en/posts/show/HEX", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + "id": 360451, + "md5": "ac8e3b92ea328ce9cf7211e69c905bf9", + }, + { + "#url": "https://chan.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + }, + { + "#url": "https://chan.sankakucomplex.com/ja/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + }, + { + "#url": "https://beta.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + }, + { + "#url": "https://white.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + }, + { + "#url": "https://black.sankakucomplex.com/post/show/360451", + "#category": ("booru", "sankaku", "post"), + "#class": sankaku.SankakuPostExtractor, + }, + { + "#url": "https://sankaku.app/books?tags=aiue_oka", + "#category": ("booru", "sankaku", "books"), + "#class": sankaku.SankakuBooksExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://beta.sankakucomplex.com/books?tags=aiue_oka", + "#category": ("booru", "sankaku", "books"), + "#class": sankaku.SankakuBooksExtractor, + }, ) diff --git a/test/results/sankakucomplex.py b/test/results/sankakucomplex.py index bcf44d9585..d67343ab68 100644 --- a/test/results/sankakucomplex.py +++ b/test/results/sankakucomplex.py @@ -1,69 +1,59 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import sankakucomplex - __tests__ = ( -{ - "#url" : "https://news.sankakucomplex.com/2019/05/11/twitter-cosplayers", - "#category": ("", "sankakucomplex", "article"), - "#class" : sankakucomplex.SankakucomplexArticleExtractor, - "#pattern" : r"https://news\.sankakucomplex\.com/wp-content/uploads/2019/05/maid-day-cosplay-\d+\.jpg", - "#sha1_metadata": "21bf106150913a1398860031f06d6e1e6423e518", -}, - -{ - "#url" : "https://www.sankakucomplex.com/2009/12/01/sexy-goddesses-of-2ch", - "#category": ("", "sankakucomplex", "article"), - "#class" : sankakucomplex.SankakucomplexArticleExtractor, - "#pattern" : r"https://news\.sankakucomplex\.com/wp-content/uploads/2009/12/Goddesses-of-2ch-amateur-internet-idol-\d+\.jpe?g", - "#sha1_metadata": "651e4ee79ecab1771b43df467b5ab32249d69b2a", -}, - -{ - "#url" : "https://www.sankakucomplex.com/2019/06/11/darling-ol-goddess-shows-off-her-plump-lower-area/", - "#comment" : "videos (#308)", - "#category": ("", "sankakucomplex", "article"), - "#class" : sankakucomplex.SankakucomplexArticleExtractor, - "#pattern" : r"/wp-content/uploads/2019/06/[^/]+\d\.mp4", - "#range" : "26-", - "#count" : 5, -}, - -{ - "#url" : "https://www.sankakucomplex.com/2015/02/12/snow-miku-2015-live-magical-indeed/", - "#comment" : "youtube embeds (#308)", - "#category": ("", "sankakucomplex", "article"), - "#class" : sankakucomplex.SankakucomplexArticleExtractor, - "#options" : {"embeds": True}, - "#pattern" : "https://www.youtube.com/embed/", - "#range" : "2-", - "#count" : 2, -}, - -{ - "#url" : "https://www.sankakucomplex.com/tag/cosplay/", - "#category": ("", "sankakucomplex", "tag"), - "#class" : sankakucomplex.SankakucomplexTagExtractor, - "#pattern" : sankakucomplex.SankakucomplexArticleExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://www.sankakucomplex.com/category/anime/", - "#category": ("", "sankakucomplex", "tag"), - "#class" : sankakucomplex.SankakucomplexTagExtractor, -}, - -{ - "#url" : "https://www.sankakucomplex.com/author/rift/page/5/", - "#category": ("", "sankakucomplex", "tag"), - "#class" : sankakucomplex.SankakucomplexTagExtractor, -}, - + { + "#url": "https://news.sankakucomplex.com/2019/05/11/twitter-cosplayers", + "#category": ("", "sankakucomplex", "article"), + "#class": sankakucomplex.SankakucomplexArticleExtractor, + "#pattern": r"https://news\.sankakucomplex\.com/wp-content/uploads/2019/05/maid-day-cosplay-\d+\.jpg", + "#sha1_metadata": "21bf106150913a1398860031f06d6e1e6423e518", + }, + { + "#url": "https://www.sankakucomplex.com/2009/12/01/sexy-goddesses-of-2ch", + "#category": ("", "sankakucomplex", "article"), + "#class": sankakucomplex.SankakucomplexArticleExtractor, + "#pattern": r"https://news\.sankakucomplex\.com/wp-content/uploads/2009/12/Goddesses-of-2ch-amateur-internet-idol-\d+\.jpe?g", + "#sha1_metadata": "651e4ee79ecab1771b43df467b5ab32249d69b2a", + }, + { + "#url": "https://www.sankakucomplex.com/2019/06/11/darling-ol-goddess-shows-off-her-plump-lower-area/", + "#comment": "videos (#308)", + "#category": ("", "sankakucomplex", "article"), + "#class": sankakucomplex.SankakucomplexArticleExtractor, + "#pattern": r"/wp-content/uploads/2019/06/[^/]+\d\.mp4", + "#range": "26-", + "#count": 5, + }, + { + "#url": "https://www.sankakucomplex.com/2015/02/12/snow-miku-2015-live-magical-indeed/", + "#comment": "youtube embeds (#308)", + "#category": ("", "sankakucomplex", "article"), + "#class": sankakucomplex.SankakucomplexArticleExtractor, + "#options": {"embeds": True}, + "#pattern": "https://www.youtube.com/embed/", + "#range": "2-", + "#count": 2, + }, + { + "#url": "https://www.sankakucomplex.com/tag/cosplay/", + "#category": ("", "sankakucomplex", "tag"), + "#class": sankakucomplex.SankakucomplexTagExtractor, + "#pattern": sankakucomplex.SankakucomplexArticleExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://www.sankakucomplex.com/category/anime/", + "#category": ("", "sankakucomplex", "tag"), + "#class": sankakucomplex.SankakucomplexTagExtractor, + }, + { + "#url": "https://www.sankakucomplex.com/author/rift/page/5/", + "#category": ("", "sankakucomplex", "tag"), + "#class": sankakucomplex.SankakucomplexTagExtractor, + }, ) diff --git a/test/results/scrolller.py b/test/results/scrolller.py index 6d34fa1f75..cc2a08d0e8 100644 --- a/test/results/scrolller.py +++ b/test/results/scrolller.py @@ -1,81 +1,73 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import scrolller - __tests__ = ( -{ - "#url" : "https://scrolller.com/r/AmateurPhotography", - "#class" : scrolller.ScrolllerSubredditExtractor, - "#pattern": r"https://\w+\.scrolller\.com/(\w+/)?[\w-]+-\w+\.(jpg|png)", - "#range" : "1-100", - "#count" : 100, - - "albumUrl" : None, - "displayName" : None, - "fullLengthSource": None, - "gfycatSource" : None, - "hasAudio" : None, - "height" : int, - "id" : int, - "isFavorite" : False, - "isNsfw" : False, - "isOptimized" : bool, - "isPaid" : None, - "mediaSources" : list, - "ownerAvatar" : None, - "redditPath" : r"re:/r/AmateurPhotography/comments/...", - "redgifsSource" : None, - "subredditId" : {0, 413}, - "subredditTitle" : "AmateurPhotography", - "subredditUrl" : "/r/AmateurPhotography", - "tags" : None, - "title" : str, - "url" : str, - "username" : None, - "width" : int, -}, - -{ - "#url" : "https://scrolller.com/cabin-in-northern-finland-7nagf1929p", - "#class": scrolller.ScrolllerPostExtractor, - "#urls" : "https://yocto.scrolller.com/cabin-in-northern-finland-93vjsuxmcz.jpg", - - "albumUrl" : None, - "displayName" : None, - "extension" : "jpg", - "filename" : "cabin-in-northern-finland-93vjsuxmcz", - "fullLengthSource": None, - "gfycatSource" : None, - "hasAudio" : None, - "height" : 1350, - "id" : 10478722, - "isNsfw" : False, - "isOptimized" : False, - "isPaid" : None, - "mediaSources" : list, - "ownerAvatar" : None, - "redditPath" : "/r/AmateurPhotography/comments/jj048q/cabin_in_northern_finland/", - "redgifsSource" : None, - "subredditId" : 0, - "subredditTitle" : "AmateurPhotography", - "subredditUrl" : "/r/AmateurPhotography", - "tags" : None, - "title" : "Cabin in northern Finland", - "url" : "https://yocto.scrolller.com/cabin-in-northern-finland-93vjsuxmcz.jpg", - "username" : None, - "width" : 1080, -}, - -{ - "#url" : "https://scrolller.com/following", - "#class" : scrolller.ScrolllerFollowingExtractor, - "#pattern": scrolller.ScrolllerSubredditExtractor.pattern, - "#auth" : True, -}, - + { + "#url": "https://scrolller.com/r/AmateurPhotography", + "#class": scrolller.ScrolllerSubredditExtractor, + "#pattern": r"https://\w+\.scrolller\.com/(\w+/)?[\w-]+-\w+\.(jpg|png)", + "#range": "1-100", + "#count": 100, + "albumUrl": None, + "displayName": None, + "fullLengthSource": None, + "gfycatSource": None, + "hasAudio": None, + "height": int, + "id": int, + "isFavorite": False, + "isNsfw": False, + "isOptimized": bool, + "isPaid": None, + "mediaSources": list, + "ownerAvatar": None, + "redditPath": r"re:/r/AmateurPhotography/comments/...", + "redgifsSource": None, + "subredditId": {0, 413}, + "subredditTitle": "AmateurPhotography", + "subredditUrl": "/r/AmateurPhotography", + "tags": None, + "title": str, + "url": str, + "username": None, + "width": int, + }, + { + "#url": "https://scrolller.com/cabin-in-northern-finland-7nagf1929p", + "#class": scrolller.ScrolllerPostExtractor, + "#urls": "https://yocto.scrolller.com/cabin-in-northern-finland-93vjsuxmcz.jpg", + "albumUrl": None, + "displayName": None, + "extension": "jpg", + "filename": "cabin-in-northern-finland-93vjsuxmcz", + "fullLengthSource": None, + "gfycatSource": None, + "hasAudio": None, + "height": 1350, + "id": 10478722, + "isNsfw": False, + "isOptimized": False, + "isPaid": None, + "mediaSources": list, + "ownerAvatar": None, + "redditPath": "/r/AmateurPhotography/comments/jj048q/cabin_in_northern_finland/", + "redgifsSource": None, + "subredditId": 0, + "subredditTitle": "AmateurPhotography", + "subredditUrl": "/r/AmateurPhotography", + "tags": None, + "title": "Cabin in northern Finland", + "url": "https://yocto.scrolller.com/cabin-in-northern-finland-93vjsuxmcz.jpg", + "username": None, + "width": 1080, + }, + { + "#url": "https://scrolller.com/following", + "#class": scrolller.ScrolllerFollowingExtractor, + "#pattern": scrolller.ScrolllerSubredditExtractor.pattern, + "#auth": True, + }, ) diff --git a/test/results/seiga.py b/test/results/seiga.py index 3749adc4a5..954dd75299 100644 --- a/test/results/seiga.py +++ b/test/results/seiga.py @@ -1,104 +1,88 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import seiga from gallery_dl import exception - +from gallery_dl.extractor import seiga __tests__ = ( -{ - "#url" : "https://seiga.nicovideo.jp/user/illust/39537793", - "#category": ("", "seiga", "user"), - "#class" : seiga.SeigaUserExtractor, - "#pattern" : r"https://lohas\.nicoseiga\.jp/priv/[0-9a-f]+/\d+/\d+", - "#count" : ">= 4", - - "user" : { - "id" : 39537793, - "message": str, - "name" : str, + { + "#url": "https://seiga.nicovideo.jp/user/illust/39537793", + "#category": ("", "seiga", "user"), + "#class": seiga.SeigaUserExtractor, + "#pattern": r"https://lohas\.nicoseiga\.jp/priv/[0-9a-f]+/\d+/\d+", + "#count": ">= 4", + "user": { + "id": 39537793, + "message": str, + "name": str, + }, + "clips": int, + "comments": int, + "count": int, + "extension": None, + "image_id": int, + "title": str, + "views": int, + }, + { + "#url": "https://seiga.nicovideo.jp/user/illust/79433", + "#category": ("", "seiga", "user"), + "#class": seiga.SeigaUserExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://seiga.nicovideo.jp/user/illust/39537793?sort=image_view&target=illust_all", + "#category": ("", "seiga", "user"), + "#class": seiga.SeigaUserExtractor, + }, + { + "#url": "https://sp.seiga.nicovideo.jp/user/illust/39537793", + "#category": ("", "seiga", "user"), + "#class": seiga.SeigaUserExtractor, + }, + { + "#url": "https://seiga.nicovideo.jp/seiga/im5977527", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + "#sha1_metadata": "c8339781da260f7fc44894ad9ada016f53e3b12a", + "#sha1_content": "d9202292012178374d57fb0126f6124387265297", + }, + { + "#url": "https://seiga.nicovideo.jp/seiga/im123", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://seiga.nicovideo.jp/seiga/im10877923", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + "#pattern": r"https://lohas\.nicoseiga\.jp/priv/5936a2a6c860a600e465e0411c0822e0b510e286/1688757110/10877923", + }, + { + "#url": "https://seiga.nicovideo.jp/image/source/5977527", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + }, + { + "#url": "https://sp.seiga.nicovideo.jp/seiga/#!/im5977527", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + }, + { + "#url": "https://lohas.nicoseiga.jp/thumb/5977527i", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + }, + { + "#url": "https://lohas.nicoseiga.jp/priv/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, + }, + { + "#url": "https://lohas.nicoseiga.jp/o/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527", + "#category": ("", "seiga", "image"), + "#class": seiga.SeigaImageExtractor, }, - "clips" : int, - "comments" : int, - "count" : int, - "extension": None, - "image_id" : int, - "title" : str, - "views" : int, -}, - -{ - "#url" : "https://seiga.nicovideo.jp/user/illust/79433", - "#category": ("", "seiga", "user"), - "#class" : seiga.SeigaUserExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://seiga.nicovideo.jp/user/illust/39537793?sort=image_view&target=illust_all", - "#category": ("", "seiga", "user"), - "#class" : seiga.SeigaUserExtractor, -}, - -{ - "#url" : "https://sp.seiga.nicovideo.jp/user/illust/39537793", - "#category": ("", "seiga", "user"), - "#class" : seiga.SeigaUserExtractor, -}, - -{ - "#url" : "https://seiga.nicovideo.jp/seiga/im5977527", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, - "#sha1_metadata": "c8339781da260f7fc44894ad9ada016f53e3b12a", - "#sha1_content" : "d9202292012178374d57fb0126f6124387265297", -}, - -{ - "#url" : "https://seiga.nicovideo.jp/seiga/im123", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://seiga.nicovideo.jp/seiga/im10877923", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, - "#pattern" : r"https://lohas\.nicoseiga\.jp/priv/5936a2a6c860a600e465e0411c0822e0b510e286/1688757110/10877923", -}, - -{ - "#url" : "https://seiga.nicovideo.jp/image/source/5977527", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, -}, - -{ - "#url" : "https://sp.seiga.nicovideo.jp/seiga/#!/im5977527", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, -}, - -{ - "#url" : "https://lohas.nicoseiga.jp/thumb/5977527i", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, -}, - -{ - "#url" : "https://lohas.nicoseiga.jp/priv/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, -}, - -{ - "#url" : "https://lohas.nicoseiga.jp/o/759a4ef1c639106ba4d665ee6333832e647d0e4e/1549727594/5977527", - "#category": ("", "seiga", "image"), - "#class" : seiga.SeigaImageExtractor, -}, - ) diff --git a/test/results/senmanga.py b/test/results/senmanga.py index 8192cfad78..ef161aa334 100644 --- a/test/results/senmanga.py +++ b/test/results/senmanga.py @@ -1,65 +1,55 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import senmanga - __tests__ = ( -{ - "#url" : "https://raw.senmanga.com/Bokura-wa-Minna-Kawaisou/37A/1", - "#category": ("", "senmanga", "chapter"), - "#class" : senmanga.SenmangaChapterExtractor, - "#pattern" : r"https://raw\.senmanga\.com/viewer/Bokura-wa-Minna-Kawaisou/37A/[12]", - "#sha1_url" : "5f95140ff511d8497e2ec08fa7267c6bb231faec", - "#sha1_content": "556a16d5ca3441d7a5807b6b5ac06ec458a3e4ba", - - "chapter" : "37A", - "count" : 2, - "extension": "", - "filename" : r"re:[12]", - "lang" : "ja", - "language" : "Japanese", - "manga" : "Bokura wa Minna Kawaisou", - "page" : int, -}, - -{ - "#url" : "http://raw.senmanga.com/Love-Lab/2016-03/1", - "#category": ("", "senmanga", "chapter"), - "#class" : senmanga.SenmangaChapterExtractor, - "#pattern" : r"https://raw\.senmanga\.com/viewer/Love-Lab/2016-03/\d", - "#sha1_url": "8347b9f00c14b864dd3c19a1f5ae52adb2ef00de", - - "chapter" : "2016-03", - "count" : 9, - "extension": "", - "filename" : r"re:\d", - "manga" : "Renai Lab 恋愛ラボ", -}, - -{ - "#url" : "https://raw.senmanga.com/akabane-honeko-no-bodyguard/1", - "#category": ("", "senmanga", "chapter"), - "#class" : senmanga.SenmangaChapterExtractor, - "#pattern" : r"https://i\d\.wp\.com/kumacdn.club/image-new-2/a/akabane-honeko-no-bodyguard/chapter-1/\d+-[0-9a-f]{13}\.jpg", - - "chapter" : "1", - "count" : 65, - "extension": "jpg", - "filename" : r"re:\d+-\w+", - "manga" : "Akabane Honeko no Bodyguard", -}, - -{ - "#url" : "https://raw.senmanga.com/amama-cinderella/3", - "#comment" : "no http scheme ()", - "#category": ("", "senmanga", "chapter"), - "#class" : senmanga.SenmangaChapterExtractor, - "#pattern" : r"^https://kumacdn.club/image-new-2/a/amama-cinderella/chapter-3/.+\.jpg", - "#count" : 30, -}, - + { + "#url": "https://raw.senmanga.com/Bokura-wa-Minna-Kawaisou/37A/1", + "#category": ("", "senmanga", "chapter"), + "#class": senmanga.SenmangaChapterExtractor, + "#pattern": r"https://raw\.senmanga\.com/viewer/Bokura-wa-Minna-Kawaisou/37A/[12]", + "#sha1_url": "5f95140ff511d8497e2ec08fa7267c6bb231faec", + "#sha1_content": "556a16d5ca3441d7a5807b6b5ac06ec458a3e4ba", + "chapter": "37A", + "count": 2, + "extension": "", + "filename": r"re:[12]", + "lang": "ja", + "language": "Japanese", + "manga": "Bokura wa Minna Kawaisou", + "page": int, + }, + { + "#url": "http://raw.senmanga.com/Love-Lab/2016-03/1", + "#category": ("", "senmanga", "chapter"), + "#class": senmanga.SenmangaChapterExtractor, + "#pattern": r"https://raw\.senmanga\.com/viewer/Love-Lab/2016-03/\d", + "#sha1_url": "8347b9f00c14b864dd3c19a1f5ae52adb2ef00de", + "chapter": "2016-03", + "count": 9, + "extension": "", + "filename": r"re:\d", + "manga": "Renai Lab 恋愛ラボ", + }, + { + "#url": "https://raw.senmanga.com/akabane-honeko-no-bodyguard/1", + "#category": ("", "senmanga", "chapter"), + "#class": senmanga.SenmangaChapterExtractor, + "#pattern": r"https://i\d\.wp\.com/kumacdn.club/image-new-2/a/akabane-honeko-no-bodyguard/chapter-1/\d+-[0-9a-f]{13}\.jpg", + "chapter": "1", + "count": 65, + "extension": "jpg", + "filename": r"re:\d+-\w+", + "manga": "Akabane Honeko no Bodyguard", + }, + { + "#url": "https://raw.senmanga.com/amama-cinderella/3", + "#comment": "no http scheme ()", + "#category": ("", "senmanga", "chapter"), + "#class": senmanga.SenmangaChapterExtractor, + "#pattern": r"^https://kumacdn.club/image-new-2/a/amama-cinderella/chapter-3/.+\.jpg", + "#count": 30, + }, ) diff --git a/test/results/sexcom.py b/test/results/sexcom.py index e51f801c82..c1bd2ddea2 100644 --- a/test/results/sexcom.py +++ b/test/results/sexcom.py @@ -1,110 +1,95 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import sexcom - __tests__ = ( -{ - "#url" : "https://www.sex.com/pin/21241874-sexy-ecchi-girls-166/", - "#comment" : "picture", - "#category": ("", "sexcom", "pin"), - "#class" : sexcom.SexcomPinExtractor, - "#pattern" : "https://cdn.sex.com/images/.+/2014/08/26/7637609.jpg", - "#sha1_content": "ebe1814dadfebf15d11c6af4f6afb1a50d6c2a1c", - - "comments" : int, - "date" : "dt:2014-10-19 15:45:44", - "extension": "jpg", - "filename" : "7637609", - "likes" : int, - "pin_id" : 21241874, - "repins" : int, - "tags" : list, - "thumbnail": str, - "title" : "Sexy Ecchi Girls 166", - "type" : "picture", - "uploader" : "mangazeta", - "url" : str, -}, - -{ - "#url" : "https://www.sex.com/pin/55435122-ecchi/", - "#comment" : "gif", - "#category": ("", "sexcom", "pin"), - "#class" : sexcom.SexcomPinExtractor, - "#pattern" : "https://cdn.sex.com/images/.+/2017/12/07/18760842.gif", - "#sha1_content": "176cc63fa05182cb0438c648230c0f324a5965fe", -}, - -{ - "#url" : "https://www.sex.com/pin/55748341/", - "#comment" : "video", - "#category": ("", "sexcom", "pin"), - "#class" : sexcom.SexcomPinExtractor, - "#pattern" : r"https://cdn\.sex\.com/videos/pinporn/2018/02/10/776229_hd\.mp4", - "#sha1_content": "e1a5834869163e2c4d1ca2677f5b7b367cf8cfff", -}, - -{ - "#url" : "https://www.sex.com/pin/55847384-very-nicely-animated/", - "#comment" : "pornhub embed (404 gone)", - "#category": ("", "sexcom", "pin"), - "#class" : sexcom.SexcomPinExtractor, -}, - -{ - "#url" : "https://www.sex.com/pin/21241874/#related", - "#category": ("", "sexcom", "related-pin"), - "#class" : sexcom.SexcomRelatedPinExtractor, - "#count" : ">= 20", -}, - -{ - "#url" : "https://www.sex.com/user/sirjuan79/pins/", - "#category": ("", "sexcom", "pins"), - "#class" : sexcom.SexcomPinsExtractor, - "#count" : ">= 4", -}, - -{ - "#url" : "https://www.sex.com/user/sirjuan79/likes/", - "#category": ("", "sexcom", "likes"), - "#class" : sexcom.SexcomLikesExtractor, - "#range" : "1-30", - "#count" : ">= 25", -}, - -{ - "#url" : "https://www.sex.com/user/ronin17/exciting-hentai/", - "#category": ("", "sexcom", "board"), - "#class" : sexcom.SexcomBoardExtractor, - "#count" : ">= 10", -}, - -{ - "#url" : "https://www.sex.com/search/pics?query=ecchi", - "#category": ("", "sexcom", "search"), - "#class" : sexcom.SexcomSearchExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.sex.com/videos/hentai/", - "#category": ("", "sexcom", "search"), - "#class" : sexcom.SexcomSearchExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.sex.com/pics/?sort=popular&sub=all&page=1", - "#category": ("", "sexcom", "search"), - "#class" : sexcom.SexcomSearchExtractor, -}, - + { + "#url": "https://www.sex.com/pin/21241874-sexy-ecchi-girls-166/", + "#comment": "picture", + "#category": ("", "sexcom", "pin"), + "#class": sexcom.SexcomPinExtractor, + "#pattern": "https://cdn.sex.com/images/.+/2014/08/26/7637609.jpg", + "#sha1_content": "ebe1814dadfebf15d11c6af4f6afb1a50d6c2a1c", + "comments": int, + "date": "dt:2014-10-19 15:45:44", + "extension": "jpg", + "filename": "7637609", + "likes": int, + "pin_id": 21241874, + "repins": int, + "tags": list, + "thumbnail": str, + "title": "Sexy Ecchi Girls 166", + "type": "picture", + "uploader": "mangazeta", + "url": str, + }, + { + "#url": "https://www.sex.com/pin/55435122-ecchi/", + "#comment": "gif", + "#category": ("", "sexcom", "pin"), + "#class": sexcom.SexcomPinExtractor, + "#pattern": "https://cdn.sex.com/images/.+/2017/12/07/18760842.gif", + "#sha1_content": "176cc63fa05182cb0438c648230c0f324a5965fe", + }, + { + "#url": "https://www.sex.com/pin/55748341/", + "#comment": "video", + "#category": ("", "sexcom", "pin"), + "#class": sexcom.SexcomPinExtractor, + "#pattern": r"https://cdn\.sex\.com/videos/pinporn/2018/02/10/776229_hd\.mp4", + "#sha1_content": "e1a5834869163e2c4d1ca2677f5b7b367cf8cfff", + }, + { + "#url": "https://www.sex.com/pin/55847384-very-nicely-animated/", + "#comment": "pornhub embed (404 gone)", + "#category": ("", "sexcom", "pin"), + "#class": sexcom.SexcomPinExtractor, + }, + { + "#url": "https://www.sex.com/pin/21241874/#related", + "#category": ("", "sexcom", "related-pin"), + "#class": sexcom.SexcomRelatedPinExtractor, + "#count": ">= 20", + }, + { + "#url": "https://www.sex.com/user/sirjuan79/pins/", + "#category": ("", "sexcom", "pins"), + "#class": sexcom.SexcomPinsExtractor, + "#count": ">= 4", + }, + { + "#url": "https://www.sex.com/user/sirjuan79/likes/", + "#category": ("", "sexcom", "likes"), + "#class": sexcom.SexcomLikesExtractor, + "#range": "1-30", + "#count": ">= 25", + }, + { + "#url": "https://www.sex.com/user/ronin17/exciting-hentai/", + "#category": ("", "sexcom", "board"), + "#class": sexcom.SexcomBoardExtractor, + "#count": ">= 10", + }, + { + "#url": "https://www.sex.com/search/pics?query=ecchi", + "#category": ("", "sexcom", "search"), + "#class": sexcom.SexcomSearchExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.sex.com/videos/hentai/", + "#category": ("", "sexcom", "search"), + "#class": sexcom.SexcomSearchExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.sex.com/pics/?sort=popular&sub=all&page=1", + "#category": ("", "sexcom", "search"), + "#class": sexcom.SexcomSearchExtractor, + }, ) diff --git a/test/results/simplyhentai.py b/test/results/simplyhentai.py index 561ce42642..db453c9867 100644 --- a/test/results/simplyhentai.py +++ b/test/results/simplyhentai.py @@ -1,75 +1,64 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import simplyhentai from gallery_dl import exception - +from gallery_dl.extractor import simplyhentai __tests__ = ( -{ - "#url" : "https://original-work.simply-hentai.com/amazon-no-hiyaku-amazon-elixir", - "#category": ("", "simplyhentai", "gallery"), - "#class" : simplyhentai.SimplyhentaiGalleryExtractor, - "#sha1_url" : "21613585ae5ec2f69ea579e9713f536fceab5bd5", - "#sha1_metadata": "9e87a0973553b2922ddee37958b8f5d87910af72", -}, - -{ - "#url" : "https://www.simply-hentai.com/notfound", - "#category": ("", "simplyhentai", "gallery"), - "#class" : simplyhentai.SimplyhentaiGalleryExtractor, - "#exception": exception.GalleryDLException, -}, - -{ - "#url" : "https://pokemon.simply-hentai.com/mao-friends-9bc39", - "#comment" : "custom subdomain", - "#category": ("", "simplyhentai", "gallery"), - "#class" : simplyhentai.SimplyhentaiGalleryExtractor, -}, - -{ - "#url" : "https://www.simply-hentai.com/vocaloid/black-magnet", - "#comment" : "www subdomain, two path segments", - "#category": ("", "simplyhentai", "gallery"), - "#class" : simplyhentai.SimplyhentaiGalleryExtractor, -}, - -{ - "#url" : "https://www.simply-hentai.com/image/pheromomania-vol-1-kanzenban-isao-3949d8b3-400c-4b6", - "#category": ("", "simplyhentai", "image"), - "#class" : simplyhentai.SimplyhentaiImageExtractor, - "#sha1_url" : "3d8eb55240a960134891bd77fe1df7988fcdc455", - "#sha1_metadata": "e10e5588481cab68329ef6ec1e5325206b2079a2", -}, - -{ - "#url" : "https://www.simply-hentai.com/gif/8915dfcf-0b6a-47c", - "#category": ("", "simplyhentai", "image"), - "#class" : simplyhentai.SimplyhentaiImageExtractor, - "#sha1_url" : "f73916527211b4a40f26568ee26cd8999f5f4f30", - "#sha1_metadata": "f94d775177fed918759c8a78a50976f867425b48", -}, - -{ - "#url" : "https://videos.simply-hentai.com/creamy-pie-episode-02", - "#category": ("", "simplyhentai", "video"), - "#class" : simplyhentai.SimplyhentaiVideoExtractor, - "#options" : {"verify": False}, - "#pattern" : r"https://www\.googleapis\.com/drive/v3/files/0B1ecQ8ZVLm3JcHZzQzBnVy1ZUmc\?alt=media&key=[\w-]+", - "#count" : 1, - "#sha1_metadata": "706790708b14773efc1e075ddd3b738a375348a5", -}, - -{ - "#url" : "https://videos.simply-hentai.com/1715-tifa-in-hentai-gang-bang-3d-movie", - "#category": ("", "simplyhentai", "video"), - "#class" : simplyhentai.SimplyhentaiVideoExtractor, - "#sha1_url" : "ad9a36ae06c601b6490e3c401834b4949d947eb0", - "#sha1_metadata": "f9dad94fbde9c95859e631ff4f07297a9567b874", -}, - + { + "#url": "https://original-work.simply-hentai.com/amazon-no-hiyaku-amazon-elixir", + "#category": ("", "simplyhentai", "gallery"), + "#class": simplyhentai.SimplyhentaiGalleryExtractor, + "#sha1_url": "21613585ae5ec2f69ea579e9713f536fceab5bd5", + "#sha1_metadata": "9e87a0973553b2922ddee37958b8f5d87910af72", + }, + { + "#url": "https://www.simply-hentai.com/notfound", + "#category": ("", "simplyhentai", "gallery"), + "#class": simplyhentai.SimplyhentaiGalleryExtractor, + "#exception": exception.GalleryDLException, + }, + { + "#url": "https://pokemon.simply-hentai.com/mao-friends-9bc39", + "#comment": "custom subdomain", + "#category": ("", "simplyhentai", "gallery"), + "#class": simplyhentai.SimplyhentaiGalleryExtractor, + }, + { + "#url": "https://www.simply-hentai.com/vocaloid/black-magnet", + "#comment": "www subdomain, two path segments", + "#category": ("", "simplyhentai", "gallery"), + "#class": simplyhentai.SimplyhentaiGalleryExtractor, + }, + { + "#url": "https://www.simply-hentai.com/image/pheromomania-vol-1-kanzenban-isao-3949d8b3-400c-4b6", + "#category": ("", "simplyhentai", "image"), + "#class": simplyhentai.SimplyhentaiImageExtractor, + "#sha1_url": "3d8eb55240a960134891bd77fe1df7988fcdc455", + "#sha1_metadata": "e10e5588481cab68329ef6ec1e5325206b2079a2", + }, + { + "#url": "https://www.simply-hentai.com/gif/8915dfcf-0b6a-47c", + "#category": ("", "simplyhentai", "image"), + "#class": simplyhentai.SimplyhentaiImageExtractor, + "#sha1_url": "f73916527211b4a40f26568ee26cd8999f5f4f30", + "#sha1_metadata": "f94d775177fed918759c8a78a50976f867425b48", + }, + { + "#url": "https://videos.simply-hentai.com/creamy-pie-episode-02", + "#category": ("", "simplyhentai", "video"), + "#class": simplyhentai.SimplyhentaiVideoExtractor, + "#options": {"verify": False}, + "#pattern": r"https://www\.googleapis\.com/drive/v3/files/0B1ecQ8ZVLm3JcHZzQzBnVy1ZUmc\?alt=media&key=[\w-]+", + "#count": 1, + "#sha1_metadata": "706790708b14773efc1e075ddd3b738a375348a5", + }, + { + "#url": "https://videos.simply-hentai.com/1715-tifa-in-hentai-gang-bang-3d-movie", + "#category": ("", "simplyhentai", "video"), + "#class": simplyhentai.SimplyhentaiVideoExtractor, + "#sha1_url": "ad9a36ae06c601b6490e3c401834b4949d947eb0", + "#sha1_metadata": "f9dad94fbde9c95859e631ff4f07297a9567b874", + }, ) diff --git a/test/results/skeb.py b/test/results/skeb.py index 4aa8691d05..70c030da0e 100644 --- a/test/results/skeb.py +++ b/test/results/skeb.py @@ -1,93 +1,82 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import skeb - __tests__ = ( -{ - "#url" : "https://skeb.jp/@kanade_cocotte/works/38", - "#category": ("", "skeb", "post"), - "#class" : skeb.SkebPostExtractor, - "#count" : 2, - - "anonymous" : False, - "body" : r"re:はじめまして。私はYouTubeにてVTuberとして活動をしている湊ラ", - "count" : 2, - "num" : range(1, 2), - "client" : { - "avatar_url" : r"re:https://pbs.twimg.com/profile_images/\d+/\w+\.jpg", - "header_url" : r"re:https://pbs.twimg.com/profile_banners/1375007870291300358/\d+/1500x500", - "id" : 1196514, - "name" : str, - "screen_name": "minato_ragi", + { + "#url": "https://skeb.jp/@kanade_cocotte/works/38", + "#category": ("", "skeb", "post"), + "#class": skeb.SkebPostExtractor, + "#count": 2, + "anonymous": False, + "body": r"re:はじめまして。私はYouTubeにてVTuberとして活動をしている湊ラ", + "count": 2, + "num": range(1, 2), + "client": { + "avatar_url": r"re:https://pbs.twimg.com/profile_images/\d+/\w+\.jpg", + "header_url": r"re:https://pbs.twimg.com/profile_banners/1375007870291300358/\d+/1500x500", + "id": 1196514, + "name": str, + "screen_name": "minato_ragi", + }, + "content_category": "preview", + "creator": { + "avatar_url": "https://pbs.twimg.com/profile_images/1225470417063645184/P8_SiB0V.jpg", + "header_url": "https://pbs.twimg.com/profile_banners/71243217/1647958329/1500x500", + "id": 159273, + "name": "イチノセ奏", + "screen_name": "kanade_cocotte", + }, + "file_id": int, + "file_url": str, + "genre": "art", + "nsfw": False, + "original": { + "byte_size": int, + "duration": None, + "extension": r"re:psd|png", + "frame_rate": None, + "height": 3727, + "is_movie": False, + "width": 2810, + }, + "post_num": "38", + "post_url": "https://skeb.jp/@kanade_cocotte/works/38", + "source_body": None, + "source_thanks": None, + "tags": list, + "thanks": None, + "translated_body": False, + "translated_thanks": None, }, - "content_category": "preview", - "creator" : { - "avatar_url" : "https://pbs.twimg.com/profile_images/1225470417063645184/P8_SiB0V.jpg", - "header_url" : "https://pbs.twimg.com/profile_banners/71243217/1647958329/1500x500", - "id" : 159273, - "name" : "イチノセ奏", - "screen_name": "kanade_cocotte", + { + "#url": "https://skeb.jp/@kanade_cocotte", + "#category": ("", "skeb", "user"), + "#class": skeb.SkebUserExtractor, + "#pattern": r"https://si\.imgix\.net/\w+/uploads/origins/[\w-]+", + "#range": "1-5", + "count": int, + "num": int, }, - "file_id" : int, - "file_url" : str, - "genre" : "art", - "nsfw" : False, - "original" : { - "byte_size" : int, - "duration" : None, - "extension" : r"re:psd|png", - "frame_rate": None, - "height" : 3727, - "is_movie" : False, - "width" : 2810, + { + "#url": "https://skeb.jp/search?q=bunny%20tree&t=works", + "#category": ("", "skeb", "search"), + "#class": skeb.SkebSearchExtractor, + "#count": ">= 18", + "search_tags": "bunny tree", + }, + { + "#url": "https://skeb.jp/@user/following_creators", + "#category": ("", "skeb", "following"), + "#class": skeb.SkebFollowingExtractor, + }, + { + "#url": "https://skeb.jp/following_users", + "#category": ("", "skeb", "following-users"), + "#class": skeb.SkebFollowingUsersExtractor, + "#pattern": skeb.SkebUserExtractor.pattern, + "#auth": True, }, - "post_num" : "38", - "post_url" : "https://skeb.jp/@kanade_cocotte/works/38", - "source_body" : None, - "source_thanks" : None, - "tags" : list, - "thanks" : None, - "translated_body" : False, - "translated_thanks": None, -}, - -{ - "#url" : "https://skeb.jp/@kanade_cocotte", - "#category": ("", "skeb", "user"), - "#class" : skeb.SkebUserExtractor, - "#pattern" : r"https://si\.imgix\.net/\w+/uploads/origins/[\w-]+", - "#range" : "1-5", - - "count": int, - "num" : int, -}, - -{ - "#url" : "https://skeb.jp/search?q=bunny%20tree&t=works", - "#category": ("", "skeb", "search"), - "#class" : skeb.SkebSearchExtractor, - "#count" : ">= 18", - - "search_tags": "bunny tree", -}, - -{ - "#url" : "https://skeb.jp/@user/following_creators", - "#category": ("", "skeb", "following"), - "#class" : skeb.SkebFollowingExtractor, -}, - -{ - "#url" : "https://skeb.jp/following_users", - "#category": ("", "skeb", "following-users"), - "#class" : skeb.SkebFollowingUsersExtractor, - "#pattern" : skeb.SkebUserExtractor.pattern, - "#auth" : True, -}, - ) diff --git a/test/results/slickpic.py b/test/results/slickpic.py index ff1eb1ff02..a9fdf2302a 100644 --- a/test/results/slickpic.py +++ b/test/results/slickpic.py @@ -1,48 +1,41 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import slickpic - __tests__ = ( -{ - "#url" : "https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", - "#category": ("", "slickpic", "album"), - "#class" : slickpic.SlickpicAlbumExtractor, - "#pattern" : r"https://stored-cf\.slickpic\.com/NDk5MjNmYTc1MzU0MQ,,/20160807/\w+/p/o/JSBFSS-\d+\.jpg", - "#count" : 102, - "#sha1_metadata": "c37c4ce9c54c09abc6abdf295855d46f11529cbf", -}, - -{ - "#url" : "https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", - "#category": ("", "slickpic", "album"), - "#class" : slickpic.SlickpicAlbumExtractor, - "#range" : "34", - "#sha1_content": [ - "276eb2c902187bb177ae8013e310e1d6641fba9a", - "52b5a310587de1048030ab13a912f6a3a9cc7dab", - "cec6630e659dc72db1ee1a9a6f3b525189261988", - "6f81e1e74c6cd6db36844e7211eef8e7cd30055d", - "22e83645fc242bc3584eca7ec982c8a53a4d8a44", - ], -}, - -{ - "#url" : "https://mattcrandall.slickpic.com/gallery/", - "#category": ("", "slickpic", "user"), - "#class" : slickpic.SlickpicUserExtractor, - "#pattern" : slickpic.SlickpicAlbumExtractor.pattern, - "#count" : ">= 358", -}, - -{ - "#url" : "https://mattcrandall.slickpic.com/", - "#category": ("", "slickpic", "user"), - "#class" : slickpic.SlickpicUserExtractor, -}, - + { + "#url": "https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", + "#category": ("", "slickpic", "album"), + "#class": slickpic.SlickpicAlbumExtractor, + "#pattern": r"https://stored-cf\.slickpic\.com/NDk5MjNmYTc1MzU0MQ,,/20160807/\w+/p/o/JSBFSS-\d+\.jpg", + "#count": 102, + "#sha1_metadata": "c37c4ce9c54c09abc6abdf295855d46f11529cbf", + }, + { + "#url": "https://mattcrandall.slickpic.com/albums/LamborghiniMurcielago/", + "#category": ("", "slickpic", "album"), + "#class": slickpic.SlickpicAlbumExtractor, + "#range": "34", + "#sha1_content": [ + "276eb2c902187bb177ae8013e310e1d6641fba9a", + "52b5a310587de1048030ab13a912f6a3a9cc7dab", + "cec6630e659dc72db1ee1a9a6f3b525189261988", + "6f81e1e74c6cd6db36844e7211eef8e7cd30055d", + "22e83645fc242bc3584eca7ec982c8a53a4d8a44", + ], + }, + { + "#url": "https://mattcrandall.slickpic.com/gallery/", + "#category": ("", "slickpic", "user"), + "#class": slickpic.SlickpicUserExtractor, + "#pattern": slickpic.SlickpicAlbumExtractor.pattern, + "#count": ">= 358", + }, + { + "#url": "https://mattcrandall.slickpic.com/", + "#category": ("", "slickpic", "user"), + "#class": slickpic.SlickpicUserExtractor, + }, ) diff --git a/test/results/slideshare.py b/test/results/slideshare.py index 66dd480be7..1f58681e81 100644 --- a/test/results/slideshare.py +++ b/test/results/slideshare.py @@ -1,48 +1,40 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import slideshare - __tests__ = ( -{ - "#url" : "https://www.slideshare.net/Slideshare/get-started-with-slide-share", - "#category": ("", "slideshare", "presentation"), - "#class" : slideshare.SlidesharePresentationExtractor, - "#pattern" : r"https://image\.slidesharecdn\.com/getstartedwithslideshare-150520173821-lva1-app6892/95/Getting-Started-With-SlideShare-\d+-1024\.jpg", - "#count" : 19, - "#sha1_content": "2b6a191eab60b3978fdacfecf2da302dd45bc108", - - "description" : "Get Started with SlideShare - A Beginngers Guide for Creators", - "likes" : int, - "presentation": "get-started-with-slide-share", - "date" : "dt:2015-05-20 17:38:21", - "title" : "Getting Started With SlideShare", - "user" : "Slideshare", - "views" : int, -}, - -{ - "#url" : "https://www.slideshare.net/pragmaticsolutions/warum-sie-nicht-ihren-mitarbeitenden-ndern-sollten-sondern-ihr-managementsystem", - "#comment" : "long title and description", - "#category": ("", "slideshare", "presentation"), - "#class" : slideshare.SlidesharePresentationExtractor, - "#sha1_url": "c2d0079cc3b05de0fd93b0d0b1f47ff2a32119b7", - - "title" : "Warum Sie nicht Ihren Mitarbeitenden ändern sollten, sondern Ihr Managementsystem", - "description": "Mitarbeitende verhalten sich mehrheitlich so, wie das System es ihnen vorgibt. Welche Voraussetzungen es braucht, damit Ihre Mitarbeitenden ihr ganzes Herzblut einsetzen, bespricht Fredi Schmidli in diesem Referat.", -}, - -{ - "#url" : "https://www.slideshare.net/mobile/uqudent/introduction-to-fixed-prosthodontics", - "#comment" : "mobile URL", - "#category": ("", "slideshare", "presentation"), - "#class" : slideshare.SlidesharePresentationExtractor, - "#pattern" : r"https://image\.slidesharecdn\.com/introductiontofixedprosthodonticsfinal-110427200948-phpapp02/95/Introduction-to-fixed-prosthodontics-\d+-1024\.jpg", - "#count" : 27, -}, - + { + "#url": "https://www.slideshare.net/Slideshare/get-started-with-slide-share", + "#category": ("", "slideshare", "presentation"), + "#class": slideshare.SlidesharePresentationExtractor, + "#pattern": r"https://image\.slidesharecdn\.com/getstartedwithslideshare-150520173821-lva1-app6892/95/Getting-Started-With-SlideShare-\d+-1024\.jpg", + "#count": 19, + "#sha1_content": "2b6a191eab60b3978fdacfecf2da302dd45bc108", + "description": "Get Started with SlideShare - A Beginngers Guide for Creators", + "likes": int, + "presentation": "get-started-with-slide-share", + "date": "dt:2015-05-20 17:38:21", + "title": "Getting Started With SlideShare", + "user": "Slideshare", + "views": int, + }, + { + "#url": "https://www.slideshare.net/pragmaticsolutions/warum-sie-nicht-ihren-mitarbeitenden-ndern-sollten-sondern-ihr-managementsystem", + "#comment": "long title and description", + "#category": ("", "slideshare", "presentation"), + "#class": slideshare.SlidesharePresentationExtractor, + "#sha1_url": "c2d0079cc3b05de0fd93b0d0b1f47ff2a32119b7", + "title": "Warum Sie nicht Ihren Mitarbeitenden ändern sollten, sondern Ihr Managementsystem", + "description": "Mitarbeitende verhalten sich mehrheitlich so, wie das System es ihnen vorgibt. Welche Voraussetzungen es braucht, damit Ihre Mitarbeitenden ihr ganzes Herzblut einsetzen, bespricht Fredi Schmidli in diesem Referat.", + }, + { + "#url": "https://www.slideshare.net/mobile/uqudent/introduction-to-fixed-prosthodontics", + "#comment": "mobile URL", + "#category": ("", "slideshare", "presentation"), + "#class": slideshare.SlidesharePresentationExtractor, + "#pattern": r"https://image\.slidesharecdn\.com/introductiontofixedprosthodonticsfinal-110427200948-phpapp02/95/Introduction-to-fixed-prosthodontics-\d+-1024\.jpg", + "#count": 27, + }, ) diff --git a/test/results/smugloli.py b/test/results/smugloli.py index c71d054e42..ad51761719 100644 --- a/test/results/smugloli.py +++ b/test/results/smugloli.py @@ -1,49 +1,40 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import vichan - __tests__ = ( -{ - "#url" : "https://smuglo.li/a/res/1143245.html", - "#category": ("vichan", "smugloli", "thread"), - "#class" : vichan.VichanThreadExtractor, - "#pattern" : r"https://smug.+/a/src/\d+(-\d)?\.\w+", - "#count" : ">= 50", - - "board" : "a", - "thread": "1143245", - "title": "Rabbit Rabbit Thread #4", -}, - -{ - "#url" : "https://smugloli.net/a/res/1145409.html", - "#category": ("vichan", "smugloli", "thread"), - "#class" : vichan.VichanThreadExtractor, -}, - -{ - "#url" : "https://smuglo.li/a", - "#category": ("vichan", "smugloli", "board"), - "#class" : vichan.VichanBoardExtractor, - "#pattern" : vichan.VichanThreadExtractor.pattern, - "#count" : ">= 100", -}, - -{ - "#url" : "https://smuglo.li/a/1.html", - "#category": ("vichan", "smugloli", "board"), - "#class" : vichan.VichanBoardExtractor, -}, - -{ - "#url" : "https://smugloli.net/cute/catalog.html", - "#category": ("vichan", "smugloli", "board"), - "#class" : vichan.VichanBoardExtractor, -}, - + { + "#url": "https://smuglo.li/a/res/1143245.html", + "#category": ("vichan", "smugloli", "thread"), + "#class": vichan.VichanThreadExtractor, + "#pattern": r"https://smug.+/a/src/\d+(-\d)?\.\w+", + "#count": ">= 50", + "board": "a", + "thread": "1143245", + "title": "Rabbit Rabbit Thread #4", + }, + { + "#url": "https://smugloli.net/a/res/1145409.html", + "#category": ("vichan", "smugloli", "thread"), + "#class": vichan.VichanThreadExtractor, + }, + { + "#url": "https://smuglo.li/a", + "#category": ("vichan", "smugloli", "board"), + "#class": vichan.VichanBoardExtractor, + "#pattern": vichan.VichanThreadExtractor.pattern, + "#count": ">= 100", + }, + { + "#url": "https://smuglo.li/a/1.html", + "#category": ("vichan", "smugloli", "board"), + "#class": vichan.VichanBoardExtractor, + }, + { + "#url": "https://smugloli.net/cute/catalog.html", + "#category": ("vichan", "smugloli", "board"), + "#class": vichan.VichanBoardExtractor, + }, ) diff --git a/test/results/smugmug.py b/test/results/smugmug.py index dda8d3a281..08fc3fddee 100644 --- a/test/results/smugmug.py +++ b/test/results/smugmug.py @@ -1,175 +1,159 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import smugmug - __tests__ = ( -{ - "#url" : "smugmug:album:cr4C7f", - "#category": ("", "smugmug", "album"), - "#class" : smugmug.SmugmugAlbumExtractor, - "#urls": ( - "https://photos.smugmug.com/Nature/Dove/i-XvZFJFG/0/DMk7cm6qRBSFPvQgT9C4t4jtBJKF7JSK9jszgHZnr/O/Dual%20Suicide_20070721-DSC_4804.jpg", - "https://photos.smugmug.com/Nature/Dove/i-2wVPqHf/0/DBXmTSTqVWzTLZxL3JPVK7hGT9zp8tzsFdhtWm68v/O/Morning%20Dove2_20070621-DSC_3222.jpg", - "https://photos.smugmug.com/Nature/Dove/i-QHFnmb8/0/GKLvnm7zFQWX2G2VcJRprx8WZqTfFJkn8C5nRnCk/O/Speed%20Skater_03082008_POR7728.jpg", - "https://photos.smugmug.com/Nature/Dove/i-MXQZKws/0/D6XCS9xnncDVtZ9NtVq66ZK9xjL4D2H9KSbpFMjfM/O/Airing%20it%20Out0_5142008_DSC_8166.jpg", - "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/FfB6gSx8X6MS7Hvww7GK7tWsrfdtwCx79hCVzwSm/O/Fluff_20090521-_DSC1542.jpg", - "https://photos.smugmug.com/Nature/Dove/i-T9Qv5Pm/0/CFT4MB9hg7rKwWmbFhGQTCnmxdpnGBKPDbHTPLSgV/O/D2F_D300_20090827-_TDM5650.jpg", - ), -}, - -{ - "#url" : "smugmug:album:Fb7hMs", - "#comment" : "empty", - "#category": ("", "smugmug", "album"), - "#class" : smugmug.SmugmugAlbumExtractor, - "#count" : 0, -}, - -{ - "#url" : "smugmug:album:6VRT8G", - "#comment" : "no 'User'", - "#category": ("", "smugmug", "album"), - "#class" : smugmug.SmugmugAlbumExtractor, - "#sha1_url": "17837ff2c78a6e2335291666f43d620d82f2926a", - - "User": { - "Name" : "", - "NickName" : "", - "QuickShare" : False, - "RefTag" : "", - "ResponseLevel": "Public", - "Uri" : "", - "ViewPassHint" : "", - "WebUri" : "", + { + "#url": "smugmug:album:cr4C7f", + "#category": ("", "smugmug", "album"), + "#class": smugmug.SmugmugAlbumExtractor, + "#urls": ( + "https://photos.smugmug.com/Nature/Dove/i-XvZFJFG/0/DMk7cm6qRBSFPvQgT9C4t4jtBJKF7JSK9jszgHZnr/O/Dual%20Suicide_20070721-DSC_4804.jpg", + "https://photos.smugmug.com/Nature/Dove/i-2wVPqHf/0/DBXmTSTqVWzTLZxL3JPVK7hGT9zp8tzsFdhtWm68v/O/Morning%20Dove2_20070621-DSC_3222.jpg", + "https://photos.smugmug.com/Nature/Dove/i-QHFnmb8/0/GKLvnm7zFQWX2G2VcJRprx8WZqTfFJkn8C5nRnCk/O/Speed%20Skater_03082008_POR7728.jpg", + "https://photos.smugmug.com/Nature/Dove/i-MXQZKws/0/D6XCS9xnncDVtZ9NtVq66ZK9xjL4D2H9KSbpFMjfM/O/Airing%20it%20Out0_5142008_DSC_8166.jpg", + "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/FfB6gSx8X6MS7Hvww7GK7tWsrfdtwCx79hCVzwSm/O/Fluff_20090521-_DSC1542.jpg", + "https://photos.smugmug.com/Nature/Dove/i-T9Qv5Pm/0/CFT4MB9hg7rKwWmbFhGQTCnmxdpnGBKPDbHTPLSgV/O/D2F_D300_20090827-_TDM5650.jpg", + ), }, -}, - -{ - "#url" : "https://tdm.smugmug.com/Nature/Dove/i-kCsLJT6", - "#category": ("", "smugmug", "image"), - "#class" : smugmug.SmugmugImageExtractor, - "#urls" : "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/FfB6gSx8X6MS7Hvww7GK7tWsrfdtwCx79hCVzwSm/O/Fluff_20090521-_DSC1542.jpg", - "#sha1_content": "ecbd9d7b4f75a637abc8d35319be9ec065a44eb0", - - "Image": { - "Altitude" : 0, - "CanBuy" : True, - "CanEdit" : False, - "CanShare" : True, - "Caption" : "White Wing Dove", - "Collectable": False, - "Comments" : True, - "ComponentFileTypes": [], - "Date" : "2009-08-01T23:00:56+00:00", - "DateTimeOriginal": "2009-05-22T00:05:36+00:00", - "DateTimeUploaded": "2009-08-01T23:00:56+00:00", - "EZProject" : False, - "FileName" : "Fluff_20090521-_DSC1542.jpg", - "Format" : "JPG", - "FormattedValues": { - "Caption": { - "html": "White Wing Dove", - "text": "White Wing Dove", - }, - "FileName": { - "html": "Fluff_20090521-_DSC1542.jpg", - "text": "Fluff_20090521-_DSC1542.jpg", + { + "#url": "smugmug:album:Fb7hMs", + "#comment": "empty", + "#category": ("", "smugmug", "album"), + "#class": smugmug.SmugmugAlbumExtractor, + "#count": 0, + }, + { + "#url": "smugmug:album:6VRT8G", + "#comment": "no 'User'", + "#category": ("", "smugmug", "album"), + "#class": smugmug.SmugmugAlbumExtractor, + "#sha1_url": "17837ff2c78a6e2335291666f43d620d82f2926a", + "User": { + "Name": "", + "NickName": "", + "QuickShare": False, + "RefTag": "", + "ResponseLevel": "Public", + "Uri": "", + "ViewPassHint": "", + "WebUri": "", + }, + }, + { + "#url": "https://tdm.smugmug.com/Nature/Dove/i-kCsLJT6", + "#category": ("", "smugmug", "image"), + "#class": smugmug.SmugmugImageExtractor, + "#urls": "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/FfB6gSx8X6MS7Hvww7GK7tWsrfdtwCx79hCVzwSm/O/Fluff_20090521-_DSC1542.jpg", + "#sha1_content": "ecbd9d7b4f75a637abc8d35319be9ec065a44eb0", + "Image": { + "Altitude": 0, + "CanBuy": True, + "CanEdit": False, + "CanShare": True, + "Caption": "White Wing Dove", + "Collectable": False, + "Comments": True, + "ComponentFileTypes": [], + "Date": "2009-08-01T23:00:56+00:00", + "DateTimeOriginal": "2009-05-22T00:05:36+00:00", + "DateTimeUploaded": "2009-08-01T23:00:56+00:00", + "EZProject": False, + "FileName": "Fluff_20090521-_DSC1542.jpg", + "Format": "JPG", + "FormattedValues": { + "Caption": { + "html": "White Wing Dove", + "text": "White Wing Dove", + }, + "FileName": { + "html": "Fluff_20090521-_DSC1542.jpg", + "text": "Fluff_20090521-_DSC1542.jpg", + }, }, + "Height": 1008, + "Hidden": False, + "ImageKey": "kCsLJT6", + "IsArchive": False, + "IsVideo": False, + "KeywordArray": [ + "Birds", + "Dove", + "White Wing Dove", + ], + "Keywords": "Birds; Dove; White Wing Dove", + "LastUpdated": "2012-11-03T20:01:15+00:00", + "Latitude": "0", + "Longitude": "0", + "OriginalHeight": 1008, + "OriginalSize": 381297, + "OriginalWidth": 1024, + "PreferredDisplayFileExtension": "JPG", + "Processing": False, + "Protected": True, + "Serial": 0, + "ShowKeywords": True, + "Size": 381297, + "Status": "Open", + "SubStatus": "NFS", + "ThumbnailUrl": "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/Df2nQXwHWSmmh4W2CjhJJdxDcZWbhkKTG86JXp9x2/Th/Fluff_20090521-_DSC1542-Th.jpg", + "Title": "", + "UploadKey": "608043804", + "Uri": "/api/v2/image/kCsLJT6-0", + "Url": "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/FfB6gSx8X6MS7Hvww7GK7tWsrfdtwCx79hCVzwSm/O/Fluff_20090521-_DSC1542.jpg", + "Watermark": "No", + "Watermarked": False, + "Width": 1024, }, - "Height" : 1008, - "Hidden" : False, - "ImageKey" : "kCsLJT6", - "IsArchive" : False, - "IsVideo" : False, - "KeywordArray": [ - "Birds", - "Dove", - "White Wing Dove", - ], - "Keywords" : "Birds; Dove; White Wing Dove", - "LastUpdated": "2012-11-03T20:01:15+00:00", - "Latitude" : "0", - "Longitude" : "0", - "OriginalHeight": 1008, - "OriginalSize": 381297, - "OriginalWidth": 1024, - "PreferredDisplayFileExtension": "JPG", - "Processing" : False, - "Protected" : True, - "Serial" : 0, - "ShowKeywords": True, - "Size" : 381297, - "Status" : "Open", - "SubStatus" : "NFS", - "ThumbnailUrl": "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/Df2nQXwHWSmmh4W2CjhJJdxDcZWbhkKTG86JXp9x2/Th/Fluff_20090521-_DSC1542-Th.jpg", - "Title" : "", - "UploadKey" : "608043804", - "Uri" : "/api/v2/image/kCsLJT6-0", - "Url" : "https://photos.smugmug.com/Nature/Dove/i-kCsLJT6/0/FfB6gSx8X6MS7Hvww7GK7tWsrfdtwCx79hCVzwSm/O/Fluff_20090521-_DSC1542.jpg", - "Watermark" : "No", - "Watermarked": False, - "Width" : 1024, + "extension": "jpg", + "filename": "Fluff_20090521-_DSC1542", + }, + { + "#url": "https://tstravels.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB", + "#comment": "video", + "#category": ("", "smugmug", "image"), + "#class": smugmug.SmugmugImageExtractor, + "#urls": "https://photos.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB/0/Q4Qg6kt4SqVcKsSLWM4PnhMhSTS2r5BkmBMd9Dx4/1920/657%20WS3-1920.mp4", + }, + { + "#url": "https://tdm.smugmug.com/Nature/Dove", + "#category": ("", "smugmug", "path"), + "#class": smugmug.SmugmugPathExtractor, + "#pattern": "smugmug:album:cr4C7f$", + }, + { + "#url": "https://tdm.smugmug.com/", + "#category": ("", "smugmug", "path"), + "#class": smugmug.SmugmugPathExtractor, + "#pattern": smugmug.SmugmugAlbumExtractor.pattern, + "#sha1_url": "1640028712875b90974e5aecd91b60e6de6138c7", + }, + { + "#url": "https://www.smugmug.com/gallery/n-GLCjnD/", + "#comment": "gallery node without owner", + "#category": ("", "smugmug", "path"), + "#class": smugmug.SmugmugPathExtractor, + "#pattern": "smugmug:album:6VRT8G$", + }, + { + "#url": "smugmug:www.sitkapics.com/TREES-and-TRAILS/", + "#comment": "custom domain", + "#category": ("", "smugmug", "path"), + "#class": smugmug.SmugmugPathExtractor, + "#pattern": "smugmug:album:ct8Nds$", + }, + { + "#url": "smugmug:www.sitkapics.com/", + "#category": ("", "smugmug", "path"), + "#class": smugmug.SmugmugPathExtractor, + "#pattern": r"smugmug:album:\w{6}$", + "#count": ">= 14", + }, + { + "#url": "smugmug:https://www.sitkapics.com/", + "#category": ("", "smugmug", "path"), + "#class": smugmug.SmugmugPathExtractor, }, - "extension": "jpg", - "filename": "Fluff_20090521-_DSC1542", -}, - -{ - "#url" : "https://tstravels.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB", - "#comment" : "video", - "#category": ("", "smugmug", "image"), - "#class" : smugmug.SmugmugImageExtractor, - "#urls" : "https://photos.smugmug.com/Dailies/Daily-Dose-2015/i-39JFNzB/0/Q4Qg6kt4SqVcKsSLWM4PnhMhSTS2r5BkmBMd9Dx4/1920/657%20WS3-1920.mp4", -}, - -{ - "#url" : "https://tdm.smugmug.com/Nature/Dove", - "#category": ("", "smugmug", "path"), - "#class" : smugmug.SmugmugPathExtractor, - "#pattern" : "smugmug:album:cr4C7f$", -}, - -{ - "#url" : "https://tdm.smugmug.com/", - "#category": ("", "smugmug", "path"), - "#class" : smugmug.SmugmugPathExtractor, - "#pattern" : smugmug.SmugmugAlbumExtractor.pattern, - "#sha1_url": "1640028712875b90974e5aecd91b60e6de6138c7", -}, - -{ - "#url" : "https://www.smugmug.com/gallery/n-GLCjnD/", - "#comment" : "gallery node without owner", - "#category": ("", "smugmug", "path"), - "#class" : smugmug.SmugmugPathExtractor, - "#pattern" : "smugmug:album:6VRT8G$", -}, - -{ - "#url" : "smugmug:www.sitkapics.com/TREES-and-TRAILS/", - "#comment" : "custom domain", - "#category": ("", "smugmug", "path"), - "#class" : smugmug.SmugmugPathExtractor, - "#pattern" : "smugmug:album:ct8Nds$", -}, - -{ - "#url" : "smugmug:www.sitkapics.com/", - "#category": ("", "smugmug", "path"), - "#class" : smugmug.SmugmugPathExtractor, - "#pattern" : r"smugmug:album:\w{6}$", - "#count" : ">= 14", -}, - -{ - "#url" : "smugmug:https://www.sitkapics.com/", - "#category": ("", "smugmug", "path"), - "#class" : smugmug.SmugmugPathExtractor, -}, - ) diff --git a/test/results/snootbooru.py b/test/results/snootbooru.py index 25a294ebb4..4732ef26f6 100644 --- a/test/results/snootbooru.py +++ b/test/results/snootbooru.py @@ -1,76 +1,64 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import szurubooru - __tests__ = ( -{ - "#url" : "https://snootbooru.com/posts/query=sport", - "#category": ("szurubooru", "snootbooru", "tag"), - "#class" : szurubooru.SzurubooruTagExtractor, - "#pattern" : r"https://snootbooru\.com/data/posts/\d+_[0-9a-f]{16}\.\w+", - "#count" : range(35, 50), -}, - -{ - "#url" : "https://snootbooru.com/post/14511", - "#category": ("szurubooru", "snootbooru", "post"), - "#class" : szurubooru.SzurubooruPostExtractor, - "#urls" : "https://snootbooru.com/data/posts/14511_e753313112755da6.png", - "#sha1_content": "e69e61e61c5372514808480aae3a8e355c9cd6fb", - - "canvasHeight" : 1000, - "canvasWidth" : 1414, - "checksum" : "e69e61e61c5372514808480aae3a8e355c9cd6fb", - "checksumMD5" : "f4f4ddfcbdf367f466ede0980acb3d7d", - "commentCount" : int, - "comments" : list, - "contentUrl" : "data/posts/14511_e753313112755da6.png", - "creationTime" : "2023-12-02T01:11:01.433664Z", - "date" : "dt:2023-12-02 01:11:01", - "extension" : "png", - "favoriteCount": int, - "favoritedBy" : list, - "featureCount" : int, - "fileSize" : 270639, - "filename" : "14511_e753313112755da6", - "flags" : [], - "hasCustomThumbnail": False, - "id" : 14511, - "lastEditTime" : "2023-12-02T01:12:09.500217Z", - "lastFeatureTime": None, - "mimeType" : "image/png", - "noteCount" : 0, - "notes" : [], - "ownFavorite" : False, - "ownScore" : 0, - "pools" : [], - "relationCount": 0, - "relations" : [], - "safety" : "safe", - "score" : 0, - "source" : None, - "tagCount" : 3, - "tags" : [ - "transparent", - "sport", - "text", - ], - "tags_default" : [ - "sport", - "text" - ], - "thumbnailUrl" : "data/generated-thumbnails/14511_e753313112755da6.jpg", - "type" : "image", - "user" : { - "avatarUrl": "data/avatars/komp.png", - "name": "komp" + { + "#url": "https://snootbooru.com/posts/query=sport", + "#category": ("szurubooru", "snootbooru", "tag"), + "#class": szurubooru.SzurubooruTagExtractor, + "#pattern": r"https://snootbooru\.com/data/posts/\d+_[0-9a-f]{16}\.\w+", + "#count": range(35, 50), + }, + { + "#url": "https://snootbooru.com/post/14511", + "#category": ("szurubooru", "snootbooru", "post"), + "#class": szurubooru.SzurubooruPostExtractor, + "#urls": "https://snootbooru.com/data/posts/14511_e753313112755da6.png", + "#sha1_content": "e69e61e61c5372514808480aae3a8e355c9cd6fb", + "canvasHeight": 1000, + "canvasWidth": 1414, + "checksum": "e69e61e61c5372514808480aae3a8e355c9cd6fb", + "checksumMD5": "f4f4ddfcbdf367f466ede0980acb3d7d", + "commentCount": int, + "comments": list, + "contentUrl": "data/posts/14511_e753313112755da6.png", + "creationTime": "2023-12-02T01:11:01.433664Z", + "date": "dt:2023-12-02 01:11:01", + "extension": "png", + "favoriteCount": int, + "favoritedBy": list, + "featureCount": int, + "fileSize": 270639, + "filename": "14511_e753313112755da6", + "flags": [], + "hasCustomThumbnail": False, + "id": 14511, + "lastEditTime": "2023-12-02T01:12:09.500217Z", + "lastFeatureTime": None, + "mimeType": "image/png", + "noteCount": 0, + "notes": [], + "ownFavorite": False, + "ownScore": 0, + "pools": [], + "relationCount": 0, + "relations": [], + "safety": "safe", + "score": 0, + "source": None, + "tagCount": 3, + "tags": [ + "transparent", + "sport", + "text", + ], + "tags_default": ["sport", "text"], + "thumbnailUrl": "data/generated-thumbnails/14511_e753313112755da6.jpg", + "type": "image", + "user": {"avatarUrl": "data/avatars/komp.png", "name": "komp"}, + "version": 2, }, - "version" : 2, -}, - ) diff --git a/test/results/soundgasm.py b/test/results/soundgasm.py index 16f0a17cca..19b96f6a74 100644 --- a/test/results/soundgasm.py +++ b/test/results/soundgasm.py @@ -1,47 +1,39 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import soundgasm - __tests__ = ( -{ - "#url" : "https://soundgasm.net/u/ClassWarAndPuppies2/687-Otto-von-Toontown-12822", - "#category": ("", "soundgasm", "audio"), - "#class" : soundgasm.SoundgasmAudioExtractor, - "#pattern" : r"https://media\.soundgasm\.net/sounds/26cb2b23b2f2c6094b40ee3a9167271e274b570a\.m4a", - - "description": "We celebrate today’s important prisoner swap, and finally bring the 2022 mid-terms to a close with Raphael Warnock’s defeat of Herschel Walker in Georgia. Then, we take a look at the Qanon-addled attempt to overthrow the German government and install Heinrich XIII Prince of Reuss as kaiser.", - "extension" : "m4a", - "filename" : "26cb2b23b2f2c6094b40ee3a9167271e274b570a", - "slug" : "687-Otto-von-Toontown-12822", - "title" : "687 - Otto von Toontown (12/8/22)", - "user" : "ClassWarAndPuppies2", -}, - -{ - "#url" : "https://www.soundgasm.net/user/ClassWarAndPuppies2/687-Otto-von-Toontown-12822", - "#category": ("", "soundgasm", "audio"), - "#class" : soundgasm.SoundgasmAudioExtractor, -}, - -{ - "#url" : "https://soundgasm.net/u/fierce-aphrodite", - "#category": ("", "soundgasm", "user"), - "#class" : soundgasm.SoundgasmUserExtractor, - "#pattern" : r"https://media\.soundgasm\.net/sounds/[0-9a-f]{40}\.m4a", - "#count" : ">= 15", - - "description": str, - "extension" : "m4a", - "filename" : r"re:^[0-9a-f]{40}$", - "slug" : str, - "title" : str, - "url" : str, - "user" : "fierce-aphrodite", -}, - + { + "#url": "https://soundgasm.net/u/ClassWarAndPuppies2/687-Otto-von-Toontown-12822", + "#category": ("", "soundgasm", "audio"), + "#class": soundgasm.SoundgasmAudioExtractor, + "#pattern": r"https://media\.soundgasm\.net/sounds/26cb2b23b2f2c6094b40ee3a9167271e274b570a\.m4a", + "description": "We celebrate today’s important prisoner swap, and finally bring the 2022 mid-terms to a close with Raphael Warnock’s defeat of Herschel Walker in Georgia. Then, we take a look at the Qanon-addled attempt to overthrow the German government and install Heinrich XIII Prince of Reuss as kaiser.", + "extension": "m4a", + "filename": "26cb2b23b2f2c6094b40ee3a9167271e274b570a", + "slug": "687-Otto-von-Toontown-12822", + "title": "687 - Otto von Toontown (12/8/22)", + "user": "ClassWarAndPuppies2", + }, + { + "#url": "https://www.soundgasm.net/user/ClassWarAndPuppies2/687-Otto-von-Toontown-12822", + "#category": ("", "soundgasm", "audio"), + "#class": soundgasm.SoundgasmAudioExtractor, + }, + { + "#url": "https://soundgasm.net/u/fierce-aphrodite", + "#category": ("", "soundgasm", "user"), + "#class": soundgasm.SoundgasmUserExtractor, + "#pattern": r"https://media\.soundgasm\.net/sounds/[0-9a-f]{40}\.m4a", + "#count": ">= 15", + "description": str, + "extension": "m4a", + "filename": r"re:^[0-9a-f]{40}$", + "slug": str, + "title": str, + "url": str, + "user": "fierce-aphrodite", + }, ) diff --git a/test/results/speakerdeck.py b/test/results/speakerdeck.py index b089323398..7baa7d43f1 100644 --- a/test/results/speakerdeck.py +++ b/test/results/speakerdeck.py @@ -1,28 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import speakerdeck - __tests__ = ( -{ - "#url" : "https://speakerdeck.com/speakerdeck/introduction-to-speakerdeck", - "#category": ("", "speakerdeck", "presentation"), - "#class" : speakerdeck.SpeakerdeckPresentationExtractor, - "#pattern" : r"https://files.speakerdeck.com/presentations/50021f75cf1db900020005e7/slide_\d+.jpg", - "#count" : 6, - "#sha1_content": "75c7abf0969b0bcab23e0da9712c95ee5113db3a", - - "author" : "Speaker Deck", - "count" : 6, - "num" : range(1, 6), - "presentation" : "introduction-to-speakerdeck", - "presentation_id": "50021f75cf1db900020005e7", - "title" : "Introduction to SpeakerDeck", - "user" : "speakerdeck", -}, - + { + "#url": "https://speakerdeck.com/speakerdeck/introduction-to-speakerdeck", + "#category": ("", "speakerdeck", "presentation"), + "#class": speakerdeck.SpeakerdeckPresentationExtractor, + "#pattern": r"https://files.speakerdeck.com/presentations/50021f75cf1db900020005e7/slide_\d+.jpg", + "#count": 6, + "#sha1_content": "75c7abf0969b0bcab23e0da9712c95ee5113db3a", + "author": "Speaker Deck", + "count": 6, + "num": range(1, 6), + "presentation": "introduction-to-speakerdeck", + "presentation_id": "50021f75cf1db900020005e7", + "title": "Introduction to SpeakerDeck", + "user": "speakerdeck", + }, ) diff --git a/test/results/steamgriddb.py b/test/results/steamgriddb.py index 8cb39d172d..8b20686197 100644 --- a/test/results/steamgriddb.py +++ b/test/results/steamgriddb.py @@ -1,118 +1,100 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import steamgriddb from gallery_dl import exception - +from gallery_dl.extractor import steamgriddb __tests__ = ( -{ - "#url" : "https://www.steamgriddb.com/grid/368023", - "#comment" : "deleted", - "#category": ("", "steamgriddb", "asset"), - "#class" : steamgriddb.SteamgriddbAssetExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://www.steamgriddb.com/grid/132605", - "#category": ("", "steamgriddb", "asset"), - "#class" : steamgriddb.SteamgriddbAssetExtractor, - "#count" : 2, - "#sha1_url" : "4ff9158c008a1f01921d7553bcabf5e6204cdc79", - "#sha1_content": "bc16c5eebf71463abdb33cfbf4b45a2fe092a2b2", - - "game": { - "id" : 5247997, - "name": "OMORI", + { + "#url": "https://www.steamgriddb.com/grid/368023", + "#comment": "deleted", + "#category": ("", "steamgriddb", "asset"), + "#class": steamgriddb.SteamgriddbAssetExtractor, + "#exception": exception.NotFoundError, }, -}, - -{ - "#url" : "https://www.steamgriddb.com/grid/132605", - "#category": ("", "steamgriddb", "asset"), - "#class" : steamgriddb.SteamgriddbAssetExtractor, - "#options" : {"download-fake-png": False}, - "#count" : 1, - "#sha1_url" : "f6819c593ff65f15864796fb89581f05d21adddb", - "#sha1_content": "0d9e6114dd8bb9699182fbb7c6bd9064d8b0b6cd", - - "game": { - "id" : 5247997, - "name": "OMORI", + { + "#url": "https://www.steamgriddb.com/grid/132605", + "#category": ("", "steamgriddb", "asset"), + "#class": steamgriddb.SteamgriddbAssetExtractor, + "#count": 2, + "#sha1_url": "4ff9158c008a1f01921d7553bcabf5e6204cdc79", + "#sha1_content": "bc16c5eebf71463abdb33cfbf4b45a2fe092a2b2", + "game": { + "id": 5247997, + "name": "OMORI", + }, + }, + { + "#url": "https://www.steamgriddb.com/grid/132605", + "#category": ("", "steamgriddb", "asset"), + "#class": steamgriddb.SteamgriddbAssetExtractor, + "#options": {"download-fake-png": False}, + "#count": 1, + "#sha1_url": "f6819c593ff65f15864796fb89581f05d21adddb", + "#sha1_content": "0d9e6114dd8bb9699182fbb7c6bd9064d8b0b6cd", + "game": { + "id": 5247997, + "name": "OMORI", + }, + }, + { + "#url": "https://www.steamgriddb.com/hero/61104", + "#category": ("", "steamgriddb", "asset"), + "#class": steamgriddb.SteamgriddbAssetExtractor, + }, + { + "#url": "https://www.steamgriddb.com/logo/9610", + "#category": ("", "steamgriddb", "asset"), + "#class": steamgriddb.SteamgriddbAssetExtractor, + }, + { + "#url": "https://www.steamgriddb.com/icon/173", + "#category": ("", "steamgriddb", "asset"), + "#class": steamgriddb.SteamgriddbAssetExtractor, + }, + { + "#url": "https://www.steamgriddb.com/game/5259324/grids", + "#category": ("", "steamgriddb", "grids"), + "#class": steamgriddb.SteamgriddbGridsExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.steamgriddb.com/game/5259324/grids", + "#category": ("", "steamgriddb", "grids"), + "#class": steamgriddb.SteamgriddbGridsExtractor, + "#options": {"humor": False, "epilepsy": False, "untagged": False}, + "#range": "1-33", + "#count": 33, + }, + { + "#url": "https://www.steamgriddb.com/game/5331605/heroes", + "#category": ("", "steamgriddb", "heroes"), + "#class": steamgriddb.SteamgriddbHeroesExtractor, + }, + { + "#url": "https://www.steamgriddb.com/game/5255394/logos", + "#category": ("", "steamgriddb", "logos"), + "#class": steamgriddb.SteamgriddbLogosExtractor, + }, + { + "#url": "https://www.steamgriddb.com/game/5279790/icons", + "#category": ("", "steamgriddb", "icons"), + "#class": steamgriddb.SteamgriddbIconsExtractor, + }, + { + "#url": "https://www.steamgriddb.com/collection/332/grids", + "#category": ("", "steamgriddb", "grids"), + "#class": steamgriddb.SteamgriddbGridsExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.steamgriddb.com/collection/332/heroes", + "#category": ("", "steamgriddb", "heroes"), + "#class": steamgriddb.SteamgriddbHeroesExtractor, + "#options": {"animated": False}, + "#count": 0, }, -}, - -{ - "#url" : "https://www.steamgriddb.com/hero/61104", - "#category": ("", "steamgriddb", "asset"), - "#class" : steamgriddb.SteamgriddbAssetExtractor, -}, - -{ - "#url" : "https://www.steamgriddb.com/logo/9610", - "#category": ("", "steamgriddb", "asset"), - "#class" : steamgriddb.SteamgriddbAssetExtractor, -}, - -{ - "#url" : "https://www.steamgriddb.com/icon/173", - "#category": ("", "steamgriddb", "asset"), - "#class" : steamgriddb.SteamgriddbAssetExtractor, -}, - -{ - "#url" : "https://www.steamgriddb.com/game/5259324/grids", - "#category": ("", "steamgriddb", "grids"), - "#class" : steamgriddb.SteamgriddbGridsExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.steamgriddb.com/game/5259324/grids", - "#category": ("", "steamgriddb", "grids"), - "#class" : steamgriddb.SteamgriddbGridsExtractor, - "#options" : {"humor": False, "epilepsy": False, "untagged": False}, - "#range" : "1-33", - "#count" : 33, -}, - -{ - "#url" : "https://www.steamgriddb.com/game/5331605/heroes", - "#category": ("", "steamgriddb", "heroes"), - "#class" : steamgriddb.SteamgriddbHeroesExtractor, -}, - -{ - "#url" : "https://www.steamgriddb.com/game/5255394/logos", - "#category": ("", "steamgriddb", "logos"), - "#class" : steamgriddb.SteamgriddbLogosExtractor, -}, - -{ - "#url" : "https://www.steamgriddb.com/game/5279790/icons", - "#category": ("", "steamgriddb", "icons"), - "#class" : steamgriddb.SteamgriddbIconsExtractor, -}, - -{ - "#url" : "https://www.steamgriddb.com/collection/332/grids", - "#category": ("", "steamgriddb", "grids"), - "#class" : steamgriddb.SteamgriddbGridsExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.steamgriddb.com/collection/332/heroes", - "#category": ("", "steamgriddb", "heroes"), - "#class" : steamgriddb.SteamgriddbHeroesExtractor, - "#options" : {"animated": False}, - "#count" : 0, -}, - ) diff --git a/test/results/subscribestar.py b/test/results/subscribestar.py index 8fe6d8dc86..48c2b3eca4 100644 --- a/test/results/subscribestar.py +++ b/test/results/subscribestar.py @@ -1,82 +1,71 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import subscribestar import datetime +from gallery_dl.extractor import subscribestar __tests__ = ( -{ - "#url" : "https://www.subscribestar.com/subscribestar", - "#category": ("", "subscribestar", "user"), - "#class" : subscribestar.SubscribestarUserExtractor, - "#pattern" : r"https://(ss-uploads-prod\.b-cdn|\w+\.cloudfront)\.net/uploads(_v2)?/users/11/", - "#count" : ">= 20", - - "author_id" : 11, - "author_name": "subscribestar", - "author_nick": "SubscribeStar", - "content" : str, - "date" : datetime.datetime, - "id" : int, - "num" : int, - "post_id" : int, - "type" : r"re:image|video|attachment", - "url" : str, - "?pinned" : bool, -}, - -{ - "#url" : "https://www.subscribestar.com/subscribestar", - "#category": ("", "subscribestar", "user"), - "#class" : subscribestar.SubscribestarUserExtractor, - "#options" : {"metadata": True}, - "#range" : "1", - - "date": datetime.datetime, -}, - -{ - "#url" : "https://subscribestar.adult/kanashiipanda", - "#category": ("", "subscribestar", "user-adult"), - "#class" : subscribestar.SubscribestarUserExtractor, - "#range" : "1-10", - "#count" : 10, -}, - -{ - "#url" : "https://www.subscribestar.com/posts/102468", - "#category": ("", "subscribestar", "post"), - "#class" : subscribestar.SubscribestarPostExtractor, - "#count" : 1, - - "author_id" : 11, - "author_name": "subscribestar", - "author_nick": "SubscribeStar", - "content" : r"re:

          Brand Guidelines and Assets

          ", - "date" : "dt:2020-05-07 12:33:00", - "extension" : "jpg", - "filename" : "8ff61299-b249-47dc-880a-cdacc9081c62", - "group" : "imgs_and_videos", - "height" : 291, - "id" : 203885, - "num" : 1, - "pinned" : False, - "post_id" : 102468, - "type" : "image", - "width" : 700, -}, - -{ - "#url" : "https://subscribestar.adult/posts/22950", - "#category": ("", "subscribestar", "post-adult"), - "#class" : subscribestar.SubscribestarPostExtractor, - "#count" : 1, - - "date": "dt:2019-04-28 07:32:00", -}, - + { + "#url": "https://www.subscribestar.com/subscribestar", + "#category": ("", "subscribestar", "user"), + "#class": subscribestar.SubscribestarUserExtractor, + "#pattern": r"https://(ss-uploads-prod\.b-cdn|\w+\.cloudfront)\.net/uploads(_v2)?/users/11/", + "#count": ">= 20", + "author_id": 11, + "author_name": "subscribestar", + "author_nick": "SubscribeStar", + "content": str, + "date": datetime.datetime, + "id": int, + "num": int, + "post_id": int, + "type": r"re:image|video|attachment", + "url": str, + "?pinned": bool, + }, + { + "#url": "https://www.subscribestar.com/subscribestar", + "#category": ("", "subscribestar", "user"), + "#class": subscribestar.SubscribestarUserExtractor, + "#options": {"metadata": True}, + "#range": "1", + "date": datetime.datetime, + }, + { + "#url": "https://subscribestar.adult/kanashiipanda", + "#category": ("", "subscribestar", "user-adult"), + "#class": subscribestar.SubscribestarUserExtractor, + "#range": "1-10", + "#count": 10, + }, + { + "#url": "https://www.subscribestar.com/posts/102468", + "#category": ("", "subscribestar", "post"), + "#class": subscribestar.SubscribestarPostExtractor, + "#count": 1, + "author_id": 11, + "author_name": "subscribestar", + "author_nick": "SubscribeStar", + "content": r"re:

          Brand Guidelines and Assets

          ", + "date": "dt:2020-05-07 12:33:00", + "extension": "jpg", + "filename": "8ff61299-b249-47dc-880a-cdacc9081c62", + "group": "imgs_and_videos", + "height": 291, + "id": 203885, + "num": 1, + "pinned": False, + "post_id": 102468, + "type": "image", + "width": 700, + }, + { + "#url": "https://subscribestar.adult/posts/22950", + "#category": ("", "subscribestar", "post-adult"), + "#class": subscribestar.SubscribestarPostExtractor, + "#count": 1, + "date": "dt:2019-04-28 07:32:00", + }, ) diff --git a/test/results/sushiski.py b/test/results/sushiski.py index e1bbd20a5f..4e77ea5fdd 100644 --- a/test/results/sushiski.py +++ b/test/results/sushiski.py @@ -1,37 +1,30 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import misskey - __tests__ = ( -{ - "#url" : "https://sushi.ski/@ui@misskey.04.si", - "#category": ("misskey", "sushi.ski", "user"), - "#class" : misskey.MisskeyUserExtractor, -}, - -{ - "#url" : "https://sushi.ski/@hatusimo_sigure/following", - "#category": ("misskey", "sushi.ski", "following"), - "#class" : misskey.MisskeyFollowingExtractor, -}, - -{ - "#url" : "https://sushi.ski/notes/9bm3x4ksqw", - "#category": ("misskey", "sushi.ski", "note"), - "#class" : misskey.MisskeyNoteExtractor, - "#pattern" : r"https://media\.sushi\.ski/files/[\w-]+\.png", - "#count" : 1, -}, - -{ - "#url" : "https://sushi.ski/my/favorites", - "#category": ("misskey", "sushi.ski", "favorite"), - "#class" : misskey.MisskeyFavoriteExtractor, -}, - + { + "#url": "https://sushi.ski/@ui@misskey.04.si", + "#category": ("misskey", "sushi.ski", "user"), + "#class": misskey.MisskeyUserExtractor, + }, + { + "#url": "https://sushi.ski/@hatusimo_sigure/following", + "#category": ("misskey", "sushi.ski", "following"), + "#class": misskey.MisskeyFollowingExtractor, + }, + { + "#url": "https://sushi.ski/notes/9bm3x4ksqw", + "#category": ("misskey", "sushi.ski", "note"), + "#class": misskey.MisskeyNoteExtractor, + "#pattern": r"https://media\.sushi\.ski/files/[\w-]+\.png", + "#count": 1, + }, + { + "#url": "https://sushi.ski/my/favorites", + "#category": ("misskey", "sushi.ski", "favorite"), + "#class": misskey.MisskeyFavoriteExtractor, + }, ) diff --git a/test/results/tapas.py b/test/results/tapas.py index d4289383f6..0a608d5e1d 100644 --- a/test/results/tapas.py +++ b/test/results/tapas.py @@ -1,89 +1,81 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import tapas - __tests__ = ( -{ - "#url" : "https://tapas.io/series/just-leave-me-be", - "#category": ("", "tapas", "series"), - "#class" : tapas.TapasSeriesExtractor, - "#pattern" : r"https://us-a\.tapas\.io/pc/\w\w/[0-9a-f-]+\.jpg", - "#count" : 132, -}, - -{ - "#url" : "https://tapas.io/series/yona", - "#comment" : "mature", - "#category": ("", "tapas", "series"), - "#class" : tapas.TapasSeriesExtractor, - "#count" : 26, -}, - -{ - "#url" : "https://tapas.io/episode/2068651", - "#category": ("", "tapas", "episode"), - "#class" : tapas.TapasEpisodeExtractor, - "#pattern" : "^text:", - "#sha1_url": "0b53644c864a0a097f65accea6bb620be9671078", - - "book" : True, - "comment_cnt" : int, - "date" : "dt:2021-02-23 16:02:07", - "early_access" : False, - "escape_title" : "You are a Tomb Raider (2)", - "free" : True, - "id" : 2068651, - "like_cnt" : int, - "liked" : bool, - "mature" : False, - "next_ep_id" : 2068652, - "nsfw" : False, - "nu" : False, - "num" : 1, - "open_comments" : True, - "pending_scene" : 2, - "prev_ep_id" : 2068650, - "publish_date" : "2021-02-23T16:02:07Z", - "read" : bool, - "related_ep_id" : None, - "relative_publish_date": "Feb 23, 2021", - "scene" : 2, - "scheduled" : False, - "title" : "You are a Tomb Raider (2)", - "unlock_cnt" : 0, - "unlocked" : False, - "view_cnt" : int, - "series" : { - "genre" : dict, - "has_book_cover": True, - "has_top_banner": True, - "id" : 199931, - "premium" : True, - "sale_type" : "WAIT_OR_MUST_PAY", - "subscribed" : bool, - "thumbsup_cnt" : int, - "title" : "Tomb Raider King", - "type" : "BOOKS", - "url" : "tomb-raider-king-novel", + { + "#url": "https://tapas.io/series/just-leave-me-be", + "#category": ("", "tapas", "series"), + "#class": tapas.TapasSeriesExtractor, + "#pattern": r"https://us-a\.tapas\.io/pc/\w\w/[0-9a-f-]+\.jpg", + "#count": 132, + }, + { + "#url": "https://tapas.io/series/yona", + "#comment": "mature", + "#category": ("", "tapas", "series"), + "#class": tapas.TapasSeriesExtractor, + "#count": 26, + }, + { + "#url": "https://tapas.io/episode/2068651", + "#category": ("", "tapas", "episode"), + "#class": tapas.TapasEpisodeExtractor, + "#pattern": "^text:", + "#sha1_url": "0b53644c864a0a097f65accea6bb620be9671078", + "book": True, + "comment_cnt": int, + "date": "dt:2021-02-23 16:02:07", + "early_access": False, + "escape_title": "You are a Tomb Raider (2)", + "free": True, + "id": 2068651, + "like_cnt": int, + "liked": bool, + "mature": False, + "next_ep_id": 2068652, + "nsfw": False, + "nu": False, + "num": 1, + "open_comments": True, + "pending_scene": 2, + "prev_ep_id": 2068650, + "publish_date": "2021-02-23T16:02:07Z", + "read": bool, + "related_ep_id": None, + "relative_publish_date": "Feb 23, 2021", + "scene": 2, + "scheduled": False, + "title": "You are a Tomb Raider (2)", + "unlock_cnt": 0, + "unlocked": False, + "view_cnt": int, + "series": { + "genre": dict, + "has_book_cover": True, + "has_top_banner": True, + "id": 199931, + "premium": True, + "sale_type": "WAIT_OR_MUST_PAY", + "subscribed": bool, + "thumbsup_cnt": int, + "title": "Tomb Raider King", + "type": "BOOKS", + "url": "tomb-raider-king-novel", + }, + }, + { + "#url": "https://tapas.io/SANG123/series", + "#comment": "#5306", + "#category": ("", "tapas", "creator"), + "#class": tapas.TapasCreatorExtractor, + "#urls": ( + "https://tapas.io/series/the-return-of-the-disaster-class-hero-novel", + "https://tapas.io/series/the-return-of-the-disaster-class-hero", + "https://tapas.io/series/tomb-raider-king", + "https://tapas.io/series/tomb-raider-king-novel", + ), }, -}, - -{ - "#url" : "https://tapas.io/SANG123/series", - "#comment" : "#5306", - "#category": ("", "tapas", "creator"), - "#class" : tapas.TapasCreatorExtractor, - "#urls" : ( - "https://tapas.io/series/the-return-of-the-disaster-class-hero-novel", - "https://tapas.io/series/the-return-of-the-disaster-class-hero", - "https://tapas.io/series/tomb-raider-king", - "https://tapas.io/series/tomb-raider-king-novel", - ), -}, - ) diff --git a/test/results/tbib.py b/test/results/tbib.py index e4481da5ed..2faa53cd89 100644 --- a/test/results/tbib.py +++ b/test/results/tbib.py @@ -1,33 +1,27 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v02 - __tests__ = ( -{ - "#url" : "https://tbib.org/index.php?page=post&s=list&tags=yuyaiyaui", - "#category": ("gelbooru_v02", "tbib", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#count" : ">= 120", -}, - -{ - "#url" : "https://tbib.org/index.php?page=favorites&s=view&id=7881", - "#category": ("gelbooru_v02", "tbib", "favorite"), - "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, - "#count" : 3, -}, - -{ - "#url" : "https://tbib.org/index.php?page=post&s=view&id=9233957", - "#category": ("gelbooru_v02", "tbib", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#sha1_url" : "5a6ebe07bfff8e6d27f7c30b5480f27abcb577d2", - "#sha1_content": "1c3831b6fbaa4686e3c79035b5d98460b1c85c43", -}, - + { + "#url": "https://tbib.org/index.php?page=post&s=list&tags=yuyaiyaui", + "#category": ("gelbooru_v02", "tbib", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#count": ">= 120", + }, + { + "#url": "https://tbib.org/index.php?page=favorites&s=view&id=7881", + "#category": ("gelbooru_v02", "tbib", "favorite"), + "#class": gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count": 3, + }, + { + "#url": "https://tbib.org/index.php?page=post&s=view&id=9233957", + "#category": ("gelbooru_v02", "tbib", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#sha1_url": "5a6ebe07bfff8e6d27f7c30b5480f27abcb577d2", + "#sha1_content": "1c3831b6fbaa4686e3c79035b5d98460b1c85c43", + }, ) diff --git a/test/results/tcbscans.py b/test/results/tcbscans.py index b3d0cb5455..96baed871d 100644 --- a/test/results/tcbscans.py +++ b/test/results/tcbscans.py @@ -1,94 +1,80 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import tcbscans from gallery_dl import exception - +from gallery_dl.extractor import tcbscans __tests__ = ( -{ - "#url" : "https://tcbscans.com/chapters/4708/chainsaw-man-chapter-108", - "#category": ("", "tcbscans", "chapter"), - "#class" : tcbscans.TcbscansChapterExtractor, - "#pattern" : r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", - "#count" : 17, - - "manga" : "Chainsaw Man", - "chapter" : 108, - "chapter_minor": "", - "lang" : "en", - "language" : "English", -}, - -{ - "#url" : "https://onepiecechapters.com/chapters/4716/one-piece-chapter-1065", - "#category": ("", "tcbscans", "chapter"), - "#class" : tcbscans.TcbscansChapterExtractor, - "#pattern" : r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", - "#count" : 18, - - "manga" : "One Piece", - "chapter" : 1065, - "chapter_minor": "", - "lang" : "en", - "language" : "English", - "#exception" : exception.HttpError, -}, - -{ - "#url" : "https://onepiecechapters.com/chapters/44/ace-novel-manga-adaptation-chapter-1", - "#category": ("", "tcbscans", "chapter"), - "#class" : tcbscans.TcbscansChapterExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://tcb-backup.bihar-mirchi.com/chapters/7719/jujutsu-kaisen-chapter-258", - "#category": ("", "tcbscans", "chapter"), - "#class" : tcbscans.TcbscansChapterExtractor, - "#pattern" : r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", - "#count" : 15, - - "manga" : "Jujutsu Kaisen", - "chapter" : 258, - "chapter_minor": "", - "lang" : "en", - "language" : "English", -}, - -{ - "#url" : "https://tcbscans.com/mangas/13/chainsaw-man", - "#category": ("", "tcbscans", "manga"), - "#class" : tcbscans.TcbscansMangaExtractor, - "#pattern" : tcbscans.TcbscansChapterExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://onepiecechapters.com/mangas/4/jujutsu-kaisen", - "#category": ("", "tcbscans", "manga"), - "#class" : tcbscans.TcbscansMangaExtractor, - "#pattern" : tcbscans.TcbscansChapterExtractor.pattern, - "#range" : "1-50", - "#count" : 50, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://onepiecechapters.com/mangas/15/hunter-x-hunter", - "#category": ("", "tcbscans", "manga"), - "#class" : tcbscans.TcbscansMangaExtractor, - "#exception": exception.HttpError, -}, - -{ - "#url" : "https://tcbscans.me/mangas/4/jujutsu-kaisen", - "#category": ("", "tcbscans", "manga"), - "#class" : tcbscans.TcbscansMangaExtractor, -}, - + { + "#url": "https://tcbscans.com/chapters/4708/chainsaw-man-chapter-108", + "#category": ("", "tcbscans", "chapter"), + "#class": tcbscans.TcbscansChapterExtractor, + "#pattern": r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", + "#count": 17, + "manga": "Chainsaw Man", + "chapter": 108, + "chapter_minor": "", + "lang": "en", + "language": "English", + }, + { + "#url": "https://onepiecechapters.com/chapters/4716/one-piece-chapter-1065", + "#category": ("", "tcbscans", "chapter"), + "#class": tcbscans.TcbscansChapterExtractor, + "#pattern": r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", + "#count": 18, + "manga": "One Piece", + "chapter": 1065, + "chapter_minor": "", + "lang": "en", + "language": "English", + "#exception": exception.HttpError, + }, + { + "#url": "https://onepiecechapters.com/chapters/44/ace-novel-manga-adaptation-chapter-1", + "#category": ("", "tcbscans", "chapter"), + "#class": tcbscans.TcbscansChapterExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://tcb-backup.bihar-mirchi.com/chapters/7719/jujutsu-kaisen-chapter-258", + "#category": ("", "tcbscans", "chapter"), + "#class": tcbscans.TcbscansChapterExtractor, + "#pattern": r"https://cdn\.[^/]+/(file|attachments/[^/]+)/[^/]+/[^.]+\.\w+", + "#count": 15, + "manga": "Jujutsu Kaisen", + "chapter": 258, + "chapter_minor": "", + "lang": "en", + "language": "English", + }, + { + "#url": "https://tcbscans.com/mangas/13/chainsaw-man", + "#category": ("", "tcbscans", "manga"), + "#class": tcbscans.TcbscansMangaExtractor, + "#pattern": tcbscans.TcbscansChapterExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://onepiecechapters.com/mangas/4/jujutsu-kaisen", + "#category": ("", "tcbscans", "manga"), + "#class": tcbscans.TcbscansMangaExtractor, + "#pattern": tcbscans.TcbscansChapterExtractor.pattern, + "#range": "1-50", + "#count": 50, + "#exception": exception.HttpError, + }, + { + "#url": "https://onepiecechapters.com/mangas/15/hunter-x-hunter", + "#category": ("", "tcbscans", "manga"), + "#class": tcbscans.TcbscansMangaExtractor, + "#exception": exception.HttpError, + }, + { + "#url": "https://tcbscans.me/mangas/4/jujutsu-kaisen", + "#category": ("", "tcbscans", "manga"), + "#class": tcbscans.TcbscansMangaExtractor, + }, ) diff --git a/test/results/tco.py b/test/results/tco.py index 37a1b106ac..1106af43e4 100644 --- a/test/results/tco.py +++ b/test/results/tco.py @@ -1,27 +1,22 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import urlshortener from gallery_dl import exception - +from gallery_dl.extractor import urlshortener __tests__ = ( -{ - "#url" : "https://t.co/bCgBY8Iv5n", - "#category": ("urlshortener", "tco", "link"), - "#class" : urlshortener.UrlshortenerLinkExtractor, - "#pattern" : "^https://twitter.com/elonmusk/status/1421395561324896257/photo/1", - "#count" : 1, -}, - -{ - "#url" : "https://t.co/abcdefghij", - "#category": ("urlshortener", "tco", "link"), - "#class" : urlshortener.UrlshortenerLinkExtractor, - "#exception": exception.NotFoundError, -}, - + { + "#url": "https://t.co/bCgBY8Iv5n", + "#category": ("urlshortener", "tco", "link"), + "#class": urlshortener.UrlshortenerLinkExtractor, + "#pattern": "^https://twitter.com/elonmusk/status/1421395561324896257/photo/1", + "#count": 1, + }, + { + "#url": "https://t.co/abcdefghij", + "#category": ("urlshortener", "tco", "link"), + "#class": urlshortener.UrlshortenerLinkExtractor, + "#exception": exception.NotFoundError, + }, ) diff --git a/test/results/telegraph.py b/test/results/telegraph.py index 726981b794..ff1ae0d6dd 100644 --- a/test/results/telegraph.py +++ b/test/results/telegraph.py @@ -1,84 +1,73 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import telegraph - __tests__ = ( -{ - "#url" : "https://telegra.ph/Telegraph-Test-03-28", - "#category": ("", "telegraph", "gallery"), - "#class" : telegraph.TelegraphGalleryExtractor, - "#pattern" : r"https://telegra\.ph/file/[0-9a-f]+\.png", - - "author" : "mikf", - "caption" : r"re:test|", - "count" : 2, - "date" : "dt:2022-03-28 16:01:36", - "description": "Just a test", - "post_url" : "https://telegra.ph/Telegraph-Test-03-28", - "slug" : "Telegraph-Test-03-28", - "title" : "Telegra.ph Test", -}, - -{ - "#url" : "https://telegra.ph/森-03-28", - "#category": ("", "telegraph", "gallery"), - "#class" : telegraph.TelegraphGalleryExtractor, - "#pattern" : "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", - "#count" : 1, - - "author" : "&", - "caption" : "kokiri", - "count" : 1, - "date" : "dt:2022-03-28 16:31:26", - "description" : "コキリの森", - "extension" : "jpg", - "filename" : "3ea79d23b0dd0889f215a", - "num" : 1, - "num_formatted": "1", - "post_url" : "https://telegra.ph/森-03-28", - "slug" : "森-03-28", - "title" : "\"森\"", - "url" : "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", -}, - -{ - "#url" : "https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", - "#category": ("", "telegraph", "gallery"), - "#class" : telegraph.TelegraphGalleryExtractor, - "#pattern" : r"^https://pith1\.ru/uploads/posts/2019-12/\d+_\d+\.jpg$", - "#sha1_url": "c1f3048e5d94bee53af30a8c27f70b0d3b15438e", - - "author" : "Shotacon - заходи сюда", - "caption" : "", - "count" : 19, - "date" : "dt:2022-05-27 16:17:27", - "description" : "", - "num_formatted": r"re:^\d{2}$", - "post_url" : "https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", - "slug" : "Vsyo-o-druzyah-moej-sestricy-05-27", - "title" : "Всё о друзьях моей сестрицы", -}, - -{ - "#url" : "https://telegra.ph/Disharmonica---Saber-Nero-02-21", - "#category": ("", "telegraph", "gallery"), - "#class" : telegraph.TelegraphGalleryExtractor, - "#pattern" : r"https://telegra\.ph/file/[0-9a-f]+\.(jpg|png)", - - "author" : "cosmos", - "caption" : "", - "count" : 89, - "date" : "dt:2022-02-21 05:57:39", - "description" : "", - "num_formatted": r"re:^\d{2}$", - "post_url" : "https://telegra.ph/Disharmonica---Saber-Nero-02-21", - "slug" : "Disharmonica---Saber-Nero-02-21", - "title" : "Disharmonica - Saber Nero", -}, - + { + "#url": "https://telegra.ph/Telegraph-Test-03-28", + "#category": ("", "telegraph", "gallery"), + "#class": telegraph.TelegraphGalleryExtractor, + "#pattern": r"https://telegra\.ph/file/[0-9a-f]+\.png", + "author": "mikf", + "caption": r"re:test|", + "count": 2, + "date": "dt:2022-03-28 16:01:36", + "description": "Just a test", + "post_url": "https://telegra.ph/Telegraph-Test-03-28", + "slug": "Telegraph-Test-03-28", + "title": "Telegra.ph Test", + }, + { + "#url": "https://telegra.ph/森-03-28", + "#category": ("", "telegraph", "gallery"), + "#class": telegraph.TelegraphGalleryExtractor, + "#pattern": "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", + "#count": 1, + "author": "&", + "caption": "kokiri", + "count": 1, + "date": "dt:2022-03-28 16:31:26", + "description": "コキリの森", + "extension": "jpg", + "filename": "3ea79d23b0dd0889f215a", + "num": 1, + "num_formatted": "1", + "post_url": "https://telegra.ph/森-03-28", + "slug": "森-03-28", + "title": '"森"', + "url": "https://telegra.ph/file/3ea79d23b0dd0889f215a.jpg", + }, + { + "#url": "https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", + "#category": ("", "telegraph", "gallery"), + "#class": telegraph.TelegraphGalleryExtractor, + "#pattern": r"^https://pith1\.ru/uploads/posts/2019-12/\d+_\d+\.jpg$", + "#sha1_url": "c1f3048e5d94bee53af30a8c27f70b0d3b15438e", + "author": "Shotacon - заходи сюда", + "caption": "", + "count": 19, + "date": "dt:2022-05-27 16:17:27", + "description": "", + "num_formatted": r"re:^\d{2}$", + "post_url": "https://telegra.ph/Vsyo-o-druzyah-moej-sestricy-05-27", + "slug": "Vsyo-o-druzyah-moej-sestricy-05-27", + "title": "Всё о друзьях моей сестрицы", + }, + { + "#url": "https://telegra.ph/Disharmonica---Saber-Nero-02-21", + "#category": ("", "telegraph", "gallery"), + "#class": telegraph.TelegraphGalleryExtractor, + "#pattern": r"https://telegra\.ph/file/[0-9a-f]+\.(jpg|png)", + "author": "cosmos", + "caption": "", + "count": 89, + "date": "dt:2022-02-21 05:57:39", + "description": "", + "num_formatted": r"re:^\d{2}$", + "post_url": "https://telegra.ph/Disharmonica---Saber-Nero-02-21", + "slug": "Disharmonica---Saber-Nero-02-21", + "title": "Disharmonica - Saber Nero", + }, ) diff --git a/test/results/tentaclerape.py b/test/results/tentaclerape.py index 247067f6db..45e7e712bd 100644 --- a/test/results/tentaclerape.py +++ b/test/results/tentaclerape.py @@ -1,47 +1,40 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shimmie2 - __tests__ = ( -{ - "#url" : "https://tentaclerape.net/post/list/comic/1", - "#category": ("shimmie2", "tentaclerape", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, - "#pattern" : r"https://tentaclerape\.net/_images/[0-9a-f]{32}/\d+", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://tentaclerape.net/post/view/10", - "#category": ("shimmie2", "tentaclerape", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, - "#pattern" : r"https://tentaclerape\.net/\./index\.php\?q=/image/10\.jpg", - "#sha1_content": "d0fd8f0f6517a76cb5e23ba09f3844950bf2c516", - - "extension" : "jpg", - "file_url" : "https://tentaclerape.net/./index.php?q=/image/10.jpg", - "filename" : "10", - "height" : 427, - "id" : 10, - "md5" : "945db71eeccaef82ce44b77564260c0b", - "size" : 0, - "subcategory": "post", - "tags" : "Deviant_Art Pet Tentacle artist_sche blonde_hair blouse boots green_eyes highheels leash miniskirt octopus schoolgirl white_skin willing", - "width" : 300, -}, - -{ - "#url" : "https://tentaclerape.net/post/view/91267", - "#comment" : "video", - "#category": ("shimmie2", "tentaclerape", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, - "#pattern" : r"https://tentaclerape\.net/\./index\.php\?q=/image/91267\.mp4", -}, - + { + "#url": "https://tentaclerape.net/post/list/comic/1", + "#category": ("shimmie2", "tentaclerape", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + "#pattern": r"https://tentaclerape\.net/_images/[0-9a-f]{32}/\d+", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://tentaclerape.net/post/view/10", + "#category": ("shimmie2", "tentaclerape", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + "#pattern": r"https://tentaclerape\.net/\./index\.php\?q=/image/10\.jpg", + "#sha1_content": "d0fd8f0f6517a76cb5e23ba09f3844950bf2c516", + "extension": "jpg", + "file_url": "https://tentaclerape.net/./index.php?q=/image/10.jpg", + "filename": "10", + "height": 427, + "id": 10, + "md5": "945db71eeccaef82ce44b77564260c0b", + "size": 0, + "subcategory": "post", + "tags": "Deviant_Art Pet Tentacle artist_sche blonde_hair blouse boots green_eyes highheels leash miniskirt octopus schoolgirl white_skin willing", + "width": 300, + }, + { + "#url": "https://tentaclerape.net/post/view/91267", + "#comment": "video", + "#category": ("shimmie2", "tentaclerape", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + "#pattern": r"https://tentaclerape\.net/\./index\.php\?q=/image/91267\.mp4", + }, ) diff --git a/test/results/thebarchive.py b/test/results/thebarchive.py index 5fd6e22666..204405952c 100644 --- a/test/results/thebarchive.py +++ b/test/results/thebarchive.py @@ -1,36 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import foolfuuka - __tests__ = ( -{ - "#url" : "https://thebarchive.com/b/thread/739772332/", - "#category": ("foolfuuka", "thebarchive", "thread"), - "#class" : foolfuuka.FoolfuukaThreadExtractor, - "#sha1_url": "e8b18001307d130d67db31740ce57c8561b5d80c", -}, - -{ - "#url" : "https://thebarchive.com/b/", - "#category": ("foolfuuka", "thebarchive", "board"), - "#class" : foolfuuka.FoolfuukaBoardExtractor, -}, - -{ - "#url" : "https://thebarchive.com/_/search/text/test/", - "#category": ("foolfuuka", "thebarchive", "search"), - "#class" : foolfuuka.FoolfuukaSearchExtractor, -}, - -{ - "#url" : "https://thebarchive.com/b/gallery/9", - "#category": ("foolfuuka", "thebarchive", "gallery"), - "#class" : foolfuuka.FoolfuukaGalleryExtractor, -}, - + { + "#url": "https://thebarchive.com/b/thread/739772332/", + "#category": ("foolfuuka", "thebarchive", "thread"), + "#class": foolfuuka.FoolfuukaThreadExtractor, + "#sha1_url": "e8b18001307d130d67db31740ce57c8561b5d80c", + }, + { + "#url": "https://thebarchive.com/b/", + "#category": ("foolfuuka", "thebarchive", "board"), + "#class": foolfuuka.FoolfuukaBoardExtractor, + }, + { + "#url": "https://thebarchive.com/_/search/text/test/", + "#category": ("foolfuuka", "thebarchive", "search"), + "#class": foolfuuka.FoolfuukaSearchExtractor, + }, + { + "#url": "https://thebarchive.com/b/gallery/9", + "#category": ("foolfuuka", "thebarchive", "gallery"), + "#class": foolfuuka.FoolfuukaGalleryExtractor, + }, ) diff --git a/test/results/thecollection.py b/test/results/thecollection.py index abe47380b4..23e9925039 100644 --- a/test/results/thecollection.py +++ b/test/results/thecollection.py @@ -1,34 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v01 - __tests__ = ( -{ - "#url" : "https://the-collection.booru.org/index.php?page=post&s=list&tags=parody", - "#category": ("gelbooru_v01", "thecollection", "tag"), - "#class" : gelbooru_v01.GelbooruV01TagExtractor, - "#range" : "1-25", - "#count" : 25, -}, - -{ - "#url" : "https://the-collection.booru.org/index.php?page=favorites&s=view&id=1166", - "#category": ("gelbooru_v01", "thecollection", "favorite"), - "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://the-collection.booru.org/index.php?page=post&s=view&id=100520", - "#category": ("gelbooru_v01", "thecollection", "post"), - "#class" : gelbooru_v01.GelbooruV01PostExtractor, - "#sha1_url" : "0329ac8588bb93cf242ca0edbe3e995b4ba554e8", - "#sha1_content": "1e585874e7b874f7937df1060dd1517fef2f4dfb", -}, - + { + "#url": "https://the-collection.booru.org/index.php?page=post&s=list&tags=parody", + "#category": ("gelbooru_v01", "thecollection", "tag"), + "#class": gelbooru_v01.GelbooruV01TagExtractor, + "#range": "1-25", + "#count": 25, + }, + { + "#url": "https://the-collection.booru.org/index.php?page=favorites&s=view&id=1166", + "#category": ("gelbooru_v01", "thecollection", "favorite"), + "#class": gelbooru_v01.GelbooruV01FavoriteExtractor, + "#count": 2, + }, + { + "#url": "https://the-collection.booru.org/index.php?page=post&s=view&id=100520", + "#category": ("gelbooru_v01", "thecollection", "post"), + "#class": gelbooru_v01.GelbooruV01PostExtractor, + "#sha1_url": "0329ac8588bb93cf242ca0edbe3e995b4ba554e8", + "#sha1_content": "1e585874e7b874f7937df1060dd1517fef2f4dfb", + }, ) diff --git a/test/results/tmohentai.py b/test/results/tmohentai.py index b565ae5e4f..a150bcf36c 100644 --- a/test/results/tmohentai.py +++ b/test/results/tmohentai.py @@ -1,54 +1,48 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import tmohentai - __tests__ = ( -{ - "#url" : "https://tmohentai.com/contents/653c2aeaa693c", - "#category": ("", "tmohentai", "gallery"), - "#class" : tmohentai.TmohentaiGalleryExtractor, - "#pattern" : r"https://imgrojo\.tmohentai\.com/contents/653c2aeaa693c/\d\d\d\.webp", - "#count" : 46, - - "artists" : ["Andoryu"], - "genres" : [ - "Big Breasts", - "BlowJob", - "Cheating", - "Mature", - "Milf", - "Student", - ], - "count" : 46, - "extension" : "webp", - "gallery_id": "653c2aeaa693c", - "language" : "Español", - "num" : int, - "tags" : [ - "milf", - "Madre", - "enormes pechos", - "Peluda", - "nakadashi", - "cheating", - "madura", - "sexo a escondidas", - "Ama de casa", - "mamada", - ], - "title" : "La Mama de mi Novia es tan Pervertida que no Pude Soportarlo mas", - "uploader" : "NekoCreme Fansub", -}, - -{ - "#url" : "https://tmohentai.com/reader/653c2aeaa693c/paginated/1", - "#category": ("", "tmohentai", "gallery"), - "#class" : tmohentai.TmohentaiGalleryExtractor, -}, - + { + "#url": "https://tmohentai.com/contents/653c2aeaa693c", + "#category": ("", "tmohentai", "gallery"), + "#class": tmohentai.TmohentaiGalleryExtractor, + "#pattern": r"https://imgrojo\.tmohentai\.com/contents/653c2aeaa693c/\d\d\d\.webp", + "#count": 46, + "artists": ["Andoryu"], + "genres": [ + "Big Breasts", + "BlowJob", + "Cheating", + "Mature", + "Milf", + "Student", + ], + "count": 46, + "extension": "webp", + "gallery_id": "653c2aeaa693c", + "language": "Español", + "num": int, + "tags": [ + "milf", + "Madre", + "enormes pechos", + "Peluda", + "nakadashi", + "cheating", + "madura", + "sexo a escondidas", + "Ama de casa", + "mamada", + ], + "title": "La Mama de mi Novia es tan Pervertida que no Pude Soportarlo mas", + "uploader": "NekoCreme Fansub", + }, + { + "#url": "https://tmohentai.com/reader/653c2aeaa693c/paginated/1", + "#category": ("", "tmohentai", "gallery"), + "#class": tmohentai.TmohentaiGalleryExtractor, + }, ) diff --git a/test/results/toyhouse.py b/test/results/toyhouse.py index 21d13ee114..6b9fbfd2e2 100644 --- a/test/results/toyhouse.py +++ b/test/results/toyhouse.py @@ -1,75 +1,65 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import toyhouse import datetime +from gallery_dl.extractor import toyhouse __tests__ = ( -{ - "#url" : "https://www.toyhou.se/d-floe/art", - "#category": ("", "toyhouse", "art"), - "#class" : toyhouse.ToyhouseArtExtractor, - "#pattern" : r"https://f\d+\.toyhou\.se/file/f\d+-toyhou-se/images/\d+_\w+\.\w+$", - "#range" : "1-30", - "#count" : 30, - - "artists" : list, - "characters": list, - "date" : datetime.datetime, - "hash" : r"re:\w+", - "id" : r"re:\d+", - "url" : str, - "user" : "d-floe", -}, - -{ - "#url" : "https://www.toyhou.se/kroksoc/art", - "#comment" : "protected by Content Warning", - "#category": ("", "toyhouse", "art"), - "#class" : toyhouse.ToyhouseArtExtractor, - "#count" : ">= 19", -}, - -{ - "#url" : "https://toyhou.se/~images/40587320", - "#category": ("", "toyhouse", "image"), - "#class" : toyhouse.ToyhouseImageExtractor, - "#sha1_content": "058ec8427977ab432c4cc5be5a6dd39ce18713ef", - - "artists" : ["d-floe"], - "characters": ["Sumi"], - "date" : "dt:2021-10-08 01:32:47", - "extension" : "png", - "filename" : "40587320_TT1NaBUr3FLkS1p", - "hash" : "TT1NaBUr3FLkS1p", - "id" : "40587320", - "url" : "https://f2.toyhou.se/file/f2-toyhou-se/images/40587320_TT1NaBUr3FLkS1p.png", -}, - -{ - "#url" : "https://f2.toyhou.se/file/f2-toyhou-se/watermarks/36817425_bqhGcwcnU.png?1625561467", - "#comment" : "direct link, multiple artists", - "#category": ("", "toyhouse", "image"), - "#class" : toyhouse.ToyhouseImageExtractor, - - "artists" : [ - "http://aminoapps.com/p/92sf3z", - "kroksoc (Color)", - ], - "characters": ["Reiichi❀"], - "date" : "dt:2021-07-03 20:02:02", - "hash" : "bqhGcwcnU", - "id" : "36817425", -}, - -{ - "#url" : "https://f2.toyhou.se/file/f2-toyhou-se/images/40587320_TT1NaBUr3FLkS1p.png", - "#category": ("", "toyhouse", "image"), - "#class" : toyhouse.ToyhouseImageExtractor, -}, - + { + "#url": "https://www.toyhou.se/d-floe/art", + "#category": ("", "toyhouse", "art"), + "#class": toyhouse.ToyhouseArtExtractor, + "#pattern": r"https://f\d+\.toyhou\.se/file/f\d+-toyhou-se/images/\d+_\w+\.\w+$", + "#range": "1-30", + "#count": 30, + "artists": list, + "characters": list, + "date": datetime.datetime, + "hash": r"re:\w+", + "id": r"re:\d+", + "url": str, + "user": "d-floe", + }, + { + "#url": "https://www.toyhou.se/kroksoc/art", + "#comment": "protected by Content Warning", + "#category": ("", "toyhouse", "art"), + "#class": toyhouse.ToyhouseArtExtractor, + "#count": ">= 19", + }, + { + "#url": "https://toyhou.se/~images/40587320", + "#category": ("", "toyhouse", "image"), + "#class": toyhouse.ToyhouseImageExtractor, + "#sha1_content": "058ec8427977ab432c4cc5be5a6dd39ce18713ef", + "artists": ["d-floe"], + "characters": ["Sumi"], + "date": "dt:2021-10-08 01:32:47", + "extension": "png", + "filename": "40587320_TT1NaBUr3FLkS1p", + "hash": "TT1NaBUr3FLkS1p", + "id": "40587320", + "url": "https://f2.toyhou.se/file/f2-toyhou-se/images/40587320_TT1NaBUr3FLkS1p.png", + }, + { + "#url": "https://f2.toyhou.se/file/f2-toyhou-se/watermarks/36817425_bqhGcwcnU.png?1625561467", + "#comment": "direct link, multiple artists", + "#category": ("", "toyhouse", "image"), + "#class": toyhouse.ToyhouseImageExtractor, + "artists": [ + "http://aminoapps.com/p/92sf3z", + "kroksoc (Color)", + ], + "characters": ["Reiichi❀"], + "date": "dt:2021-07-03 20:02:02", + "hash": "bqhGcwcnU", + "id": "36817425", + }, + { + "#url": "https://f2.toyhou.se/file/f2-toyhou-se/images/40587320_TT1NaBUr3FLkS1p.png", + "#category": ("", "toyhouse", "image"), + "#class": toyhouse.ToyhouseImageExtractor, + }, ) diff --git a/test/results/tsumino.py b/test/results/tsumino.py index cbe2c77a6f..49a526577e 100644 --- a/test/results/tsumino.py +++ b/test/results/tsumino.py @@ -1,72 +1,62 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import tsumino - __tests__ = ( -{ - "#url" : "https://www.tsumino.com/entry/40996", - "#category": ("", "tsumino", "gallery"), - "#class" : tsumino.TsuminoGalleryExtractor, - "#pattern" : r"https://content.tsumino.com/parts/40996/\d+\?key=\w+", - - "title" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", - "title_en" : r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", - "title_jp" : "シコシコ大好きナイチンゲール + 会場限定おまけ本", - "gallery_id": 40996, - "date" : "dt:2018-06-29 00:00:00", - "count" : 42, - "collection": "", - "artist" : ["Itou Life"], - "group" : ["Itou Life"], - "parody" : list, - "characters": list, - "tags" : list, - "type" : "Doujinshi", - "rating" : float, - "uploader" : "sehki", - "lang" : "en", - "language" : "English", - "thumbnail" : "https://content.tsumino.com/thumbs/40996/1", -}, - -{ - "#url" : "https://www.tsumino.com/Book/Info/40996", - "#category": ("", "tsumino", "gallery"), - "#class" : tsumino.TsuminoGalleryExtractor, -}, - -{ - "#url" : "https://www.tsumino.com/Read/View/45834", - "#category": ("", "tsumino", "gallery"), - "#class" : tsumino.TsuminoGalleryExtractor, -}, - -{ - "#url" : "https://www.tsumino.com/Read/Index/45834", - "#category": ("", "tsumino", "gallery"), - "#class" : tsumino.TsuminoGalleryExtractor, -}, - -{ - "#url" : "https://www.tsumino.com/Books#?Character=Reimu+Hakurei", - "#category": ("", "tsumino", "search"), - "#class" : tsumino.TsuminoSearchExtractor, - "#pattern" : tsumino.TsuminoGalleryExtractor.pattern, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "http://www.tsumino.com/Books#~(Tags~(~(Type~7~Text~'Reimu*20Hakurei~Exclude~false)~(Type~'1~Text~'Pantyhose~Exclude~false)))#", - "#category": ("", "tsumino", "search"), - "#class" : tsumino.TsuminoSearchExtractor, - "#pattern" : tsumino.TsuminoGalleryExtractor.pattern, - "#count" : ">= 3", -}, - + { + "#url": "https://www.tsumino.com/entry/40996", + "#category": ("", "tsumino", "gallery"), + "#class": tsumino.TsuminoGalleryExtractor, + "#pattern": r"https://content.tsumino.com/parts/40996/\d+\?key=\w+", + "title": r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", + "title_en": r"re:Shikoshiko Daisuki Nightingale \+ Kaijou", + "title_jp": "シコシコ大好きナイチンゲール + 会場限定おまけ本", + "gallery_id": 40996, + "date": "dt:2018-06-29 00:00:00", + "count": 42, + "collection": "", + "artist": ["Itou Life"], + "group": ["Itou Life"], + "parody": list, + "characters": list, + "tags": list, + "type": "Doujinshi", + "rating": float, + "uploader": "sehki", + "lang": "en", + "language": "English", + "thumbnail": "https://content.tsumino.com/thumbs/40996/1", + }, + { + "#url": "https://www.tsumino.com/Book/Info/40996", + "#category": ("", "tsumino", "gallery"), + "#class": tsumino.TsuminoGalleryExtractor, + }, + { + "#url": "https://www.tsumino.com/Read/View/45834", + "#category": ("", "tsumino", "gallery"), + "#class": tsumino.TsuminoGalleryExtractor, + }, + { + "#url": "https://www.tsumino.com/Read/Index/45834", + "#category": ("", "tsumino", "gallery"), + "#class": tsumino.TsuminoGalleryExtractor, + }, + { + "#url": "https://www.tsumino.com/Books#?Character=Reimu+Hakurei", + "#category": ("", "tsumino", "search"), + "#class": tsumino.TsuminoSearchExtractor, + "#pattern": tsumino.TsuminoGalleryExtractor.pattern, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "http://www.tsumino.com/Books#~(Tags~(~(Type~7~Text~'Reimu*20Hakurei~Exclude~false)~(Type~'1~Text~'Pantyhose~Exclude~false)))#", + "#category": ("", "tsumino", "search"), + "#class": tsumino.TsuminoSearchExtractor, + "#pattern": tsumino.TsuminoGalleryExtractor.pattern, + "#count": ">= 3", + }, ) diff --git a/test/results/tumblr.py b/test/results/tumblr.py index 50b6767618..1e6e6f95fd 100644 --- a/test/results/tumblr.py +++ b/test/results/tumblr.py @@ -4,377 +4,325 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import tumblr from gallery_dl import exception - +from gallery_dl.extractor import tumblr __tests__ = ( -{ - "#url" : "http://demo.tumblr.com/", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#options" : {"posts": "photo"}, - "#pattern" : r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_\d+\.jpg", - "#count" : 1, -}, - -{ - "#url" : "http://demo.tumblr.com/", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#options" : { - "posts" : "all", - "external": True, - }, - "#pattern" : r"https?://(?:$|\d+\.media\.tumblr\.com/.+\.(jpg|png|gif|mp3|mp4)|v?a\.(media\.)?tumblr\.com/tumblr_\w+)", - "#count" : 27, -}, - -{ - "#url" : "https://mikf123-hidden.tumblr.com/", - "#comment" : "dashboard-only", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#options" : {"access-token": None}, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://mikf123-hidden.tumblr.com/", - "#comment" : "dashboard-only", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#count" : 2, - - "tags": [ - "test", - "hidden", - ], -}, - -{ - "#url" : "https://mikf123-private.tumblr.com/", - "#comment" : "password protected", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#count" : 2, - - "tags": [ - "test", - "private", - ], -}, - -{ - "#url" : "https://mikf123-private-hidden.tumblr.com/", - "#comment" : "dashboard-only & password protected", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#count" : 2, - - "tags": [ - "test", - "private", - "hidden", - ], -}, - -{ - "#url" : "https://mikf123.tumblr.com/", - "#comment" : "date-min/-max/-format (#337)", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#options" : { - "date-min" : "201804", - "date-max" : "201805", - "date-format": "%Y%m", - }, - "#count" : 4, -}, - -{ - "#url" : "https://donttrustthetits.tumblr.com/", - "#comment" : "pagination with 'date-max' (#2191) and 'api-key'", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, - "#options" : { - "access-token": None, - "original" : False, - "date-max" : "2015-04-25T00:00:00", - "date-min" : "2015-04-01T00:00:00", - }, - "#count" : 192, -}, - -{ - "#url" : "https://demo.tumblr.com/page/2", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "https://demo.tumblr.com/archive", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "tumblr:http://www.b-authentique.com/", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "tumblr:www.b-authentique.com", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/view/smarties-art", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/smarties-art", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/smarties-art", - "#category": ("", "tumblr", "user"), - "#class" : tumblr.TumblrUserExtractor, -}, - -{ - "#url" : "http://demo.tumblr.com/post/459265350", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#pattern" : r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_1280.jpg", - "#count" : 1, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/167770226574/text-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/181022561719/quote-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 1, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/167623351559/link-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/167633596145/video-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/167770026604/audio-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 2, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/172687798174/photo-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 4, -}, - -{ - "#url" : "https://mikf123.tumblr.com/post/181022380064/chat-post", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://kichatundk.tumblr.com/post/654953419288821760", - "#comment" : "high-quality images (#1846)", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#count" : 2, - "#sha1_content": "d6fcc7b6f750d835d55c7f31fa3b63be26c9f89b", -}, - -{ - "#url" : "https://hameru-is-cool.tumblr.com/post/639261855227002880", - "#comment" : "high-quality images (#1344)", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#exception" : exception.NotFoundError, - "#count" : 2, - "#sha1_content": "6bc19a42787e46e1bba2ef4aeef5ca28fcd3cd34", -}, - -{ - "#url" : "https://k-eke.tumblr.com/post/185341184856", - "#comment" : "wrong extension returned by api (#3095)", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#options" : {"retries": 0}, - "#urls" : "https://64.media.tumblr.com/5e9d760aba24c65beaf0e72de5aae4dd/tumblr_psj5yaqV871t1ig6no1_1280.gif", - "#sha1_content": "3508d894b6cc25e364d182a8e1ff370d706965fb", -}, - -{ - "#url" : "https://mikf123.tumblr.com/image/689860196535762944", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#pattern" : r"^https://\d+\.media\.tumblr\.com/134791621559a79793563b636b5fe2c6/8f1131551cef6e74-bc/s99999x99999/188cf9b8915b0d0911c6c743d152fc62e8f38491\.png$", -}, - -{ - "#url" : "http://ziemniax.tumblr.com/post/109697912859/", - "#comment" : "HTML response (#297)", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "http://demo.tumblr.com/image/459265350", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/view/smarties-art/686047436641353728", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/smarties-art/686047436641353728", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/smarties-art/686047436641353728", - "#category": ("", "tumblr", "post"), - "#class" : tumblr.TumblrPostExtractor, -}, - -{ - "#url" : "http://demo.tumblr.com/tagged/Times%20Square", - "#category": ("", "tumblr", "tag"), - "#class" : tumblr.TumblrTagExtractor, - "#pattern" : r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_1280.jpg", - "#count" : 1, -}, - -{ - "#url" : "https://www.tumblr.com/blog/view/smarties-art/tagged/undertale", - "#category": ("", "tumblr", "tag"), - "#class" : tumblr.TumblrTagExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/smarties-art/tagged/undertale", - "#category": ("", "tumblr", "tag"), - "#class" : tumblr.TumblrTagExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/smarties-art/tagged/undertale", - "#category": ("", "tumblr", "tag"), - "#class" : tumblr.TumblrTagExtractor, -}, - -{ - "#url" : "https://mikf123.tumblr.com/day/2018/01/05", - "#category": ("", "tumblr", "day"), - "#class" : tumblr.TumblrDayExtractor, - "#pattern" : r"https://64\.media\.tumblr\.com/1a2be8c63f1df58abd2622861696c72a/tumblr_ozm9nqst9t1wgha4yo1_1280\.jpg", - "#count" : 1, - - "id": 169341068404, -}, - -{ - "#url" : "https://www.tumblr.com/blog/view/mikf123/day/2018/01/05", - "#category": ("", "tumblr", "day"), - "#class" : tumblr.TumblrDayExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/mikf123/day/2018/01/05", - "#category": ("", "tumblr", "day"), - "#class" : tumblr.TumblrDayExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/mikf123/day/2018/01/05", - "#category": ("", "tumblr", "day"), - "#class" : tumblr.TumblrDayExtractor, -}, - -{ - "#url" : "http://mikf123.tumblr.com/likes", - "#category": ("", "tumblr", "likes"), - "#class" : tumblr.TumblrLikesExtractor, - "#count" : 1, -}, - -{ - "#url" : "http://mikf123.tumblr.com/likes", - "#category": ("", "tumblr", "likes"), - "#class" : tumblr.TumblrLikesExtractor, - "#options" : {"api-secret": None}, - "#count" : 1, -}, - -{ - "#url" : "https://www.tumblr.com/blog/view/mikf123/likes", - "#category": ("", "tumblr", "likes"), - "#class" : tumblr.TumblrLikesExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/blog/mikf123/likes", - "#category": ("", "tumblr", "likes"), - "#class" : tumblr.TumblrLikesExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/mikf123/likes", - "#category": ("", "tumblr", "likes"), - "#class" : tumblr.TumblrLikesExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/search/nathan fielder", - "#category": ("", "tumblr", "search"), - "#class" : tumblr.TumblrSearchExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/search/nathan fielder/recent/quote?src=typed_query", - "#category": ("", "tumblr", "search"), - "#class" : tumblr.TumblrSearchExtractor, -}, - -{ - "#url" : "https://www.tumblr.com/search/nathan%20fielder?t=90", - "#category": ("", "tumblr", "search"), - "#class" : tumblr.TumblrSearchExtractor, -}, + { + "#url": "http://demo.tumblr.com/", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#options": {"posts": "photo"}, + "#pattern": r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_\d+\.jpg", + "#count": 1, + }, + { + "#url": "http://demo.tumblr.com/", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#options": { + "posts": "all", + "external": True, + }, + "#pattern": r"https?://(?:$|\d+\.media\.tumblr\.com/.+\.(jpg|png|gif|mp3|mp4)|v?a\.(media\.)?tumblr\.com/tumblr_\w+)", + "#count": 27, + }, + { + "#url": "https://mikf123-hidden.tumblr.com/", + "#comment": "dashboard-only", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#options": {"access-token": None}, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://mikf123-hidden.tumblr.com/", + "#comment": "dashboard-only", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#count": 2, + "tags": [ + "test", + "hidden", + ], + }, + { + "#url": "https://mikf123-private.tumblr.com/", + "#comment": "password protected", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#count": 2, + "tags": [ + "test", + "private", + ], + }, + { + "#url": "https://mikf123-private-hidden.tumblr.com/", + "#comment": "dashboard-only & password protected", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#count": 2, + "tags": [ + "test", + "private", + "hidden", + ], + }, + { + "#url": "https://mikf123.tumblr.com/", + "#comment": "date-min/-max/-format (#337)", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#options": { + "date-min": "201804", + "date-max": "201805", + "date-format": "%Y%m", + }, + "#count": 4, + }, + { + "#url": "https://donttrustthetits.tumblr.com/", + "#comment": "pagination with 'date-max' (#2191) and 'api-key'", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + "#options": { + "access-token": None, + "original": False, + "date-max": "2015-04-25T00:00:00", + "date-min": "2015-04-01T00:00:00", + }, + "#count": 192, + }, + { + "#url": "https://demo.tumblr.com/page/2", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "https://demo.tumblr.com/archive", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "tumblr:http://www.b-authentique.com/", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "tumblr:www.b-authentique.com", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/view/smarties-art", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/smarties-art", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "https://www.tumblr.com/smarties-art", + "#category": ("", "tumblr", "user"), + "#class": tumblr.TumblrUserExtractor, + }, + { + "#url": "http://demo.tumblr.com/post/459265350", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#pattern": r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_1280.jpg", + "#count": 1, + }, + { + "#url": "https://mikf123.tumblr.com/post/167770226574/text-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 2, + }, + { + "#url": "https://mikf123.tumblr.com/post/181022561719/quote-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 1, + }, + { + "#url": "https://mikf123.tumblr.com/post/167623351559/link-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 2, + }, + { + "#url": "https://mikf123.tumblr.com/post/167633596145/video-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 2, + }, + { + "#url": "https://mikf123.tumblr.com/post/167770026604/audio-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 2, + }, + { + "#url": "https://mikf123.tumblr.com/post/172687798174/photo-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 4, + }, + { + "#url": "https://mikf123.tumblr.com/post/181022380064/chat-post", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 0, + }, + { + "#url": "https://kichatundk.tumblr.com/post/654953419288821760", + "#comment": "high-quality images (#1846)", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#count": 2, + "#sha1_content": "d6fcc7b6f750d835d55c7f31fa3b63be26c9f89b", + }, + { + "#url": "https://hameru-is-cool.tumblr.com/post/639261855227002880", + "#comment": "high-quality images (#1344)", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#exception": exception.NotFoundError, + "#count": 2, + "#sha1_content": "6bc19a42787e46e1bba2ef4aeef5ca28fcd3cd34", + }, + { + "#url": "https://k-eke.tumblr.com/post/185341184856", + "#comment": "wrong extension returned by api (#3095)", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#options": {"retries": 0}, + "#urls": "https://64.media.tumblr.com/5e9d760aba24c65beaf0e72de5aae4dd/tumblr_psj5yaqV871t1ig6no1_1280.gif", + "#sha1_content": "3508d894b6cc25e364d182a8e1ff370d706965fb", + }, + { + "#url": "https://mikf123.tumblr.com/image/689860196535762944", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#pattern": r"^https://\d+\.media\.tumblr\.com/134791621559a79793563b636b5fe2c6/8f1131551cef6e74-bc/s99999x99999/188cf9b8915b0d0911c6c743d152fc62e8f38491\.png$", + }, + { + "#url": "http://ziemniax.tumblr.com/post/109697912859/", + "#comment": "HTML response (#297)", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "http://demo.tumblr.com/image/459265350", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/view/smarties-art/686047436641353728", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/smarties-art/686047436641353728", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + }, + { + "#url": "https://www.tumblr.com/smarties-art/686047436641353728", + "#category": ("", "tumblr", "post"), + "#class": tumblr.TumblrPostExtractor, + }, + { + "#url": "http://demo.tumblr.com/tagged/Times%20Square", + "#category": ("", "tumblr", "tag"), + "#class": tumblr.TumblrTagExtractor, + "#pattern": r"https://\d+\.media\.tumblr\.com/tumblr_[^/_]+_1280.jpg", + "#count": 1, + }, + { + "#url": "https://www.tumblr.com/blog/view/smarties-art/tagged/undertale", + "#category": ("", "tumblr", "tag"), + "#class": tumblr.TumblrTagExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/smarties-art/tagged/undertale", + "#category": ("", "tumblr", "tag"), + "#class": tumblr.TumblrTagExtractor, + }, + { + "#url": "https://www.tumblr.com/smarties-art/tagged/undertale", + "#category": ("", "tumblr", "tag"), + "#class": tumblr.TumblrTagExtractor, + }, + { + "#url": "https://mikf123.tumblr.com/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class": tumblr.TumblrDayExtractor, + "#pattern": r"https://64\.media\.tumblr\.com/1a2be8c63f1df58abd2622861696c72a/tumblr_ozm9nqst9t1wgha4yo1_1280\.jpg", + "#count": 1, + "id": 169341068404, + }, + { + "#url": "https://www.tumblr.com/blog/view/mikf123/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class": tumblr.TumblrDayExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/mikf123/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class": tumblr.TumblrDayExtractor, + }, + { + "#url": "https://www.tumblr.com/mikf123/day/2018/01/05", + "#category": ("", "tumblr", "day"), + "#class": tumblr.TumblrDayExtractor, + }, + { + "#url": "http://mikf123.tumblr.com/likes", + "#category": ("", "tumblr", "likes"), + "#class": tumblr.TumblrLikesExtractor, + "#count": 1, + }, + { + "#url": "http://mikf123.tumblr.com/likes", + "#category": ("", "tumblr", "likes"), + "#class": tumblr.TumblrLikesExtractor, + "#options": {"api-secret": None}, + "#count": 1, + }, + { + "#url": "https://www.tumblr.com/blog/view/mikf123/likes", + "#category": ("", "tumblr", "likes"), + "#class": tumblr.TumblrLikesExtractor, + }, + { + "#url": "https://www.tumblr.com/blog/mikf123/likes", + "#category": ("", "tumblr", "likes"), + "#class": tumblr.TumblrLikesExtractor, + }, + { + "#url": "https://www.tumblr.com/mikf123/likes", + "#category": ("", "tumblr", "likes"), + "#class": tumblr.TumblrLikesExtractor, + }, + { + "#url": "https://www.tumblr.com/search/nathan fielder", + "#category": ("", "tumblr", "search"), + "#class": tumblr.TumblrSearchExtractor, + }, + { + "#url": "https://www.tumblr.com/search/nathan fielder/recent/quote?src=typed_query", + "#category": ("", "tumblr", "search"), + "#class": tumblr.TumblrSearchExtractor, + }, + { + "#url": "https://www.tumblr.com/search/nathan%20fielder?t=90", + "#category": ("", "tumblr", "search"), + "#class": tumblr.TumblrSearchExtractor, + }, ) diff --git a/test/results/tumblrgallery.py b/test/results/tumblrgallery.py index a1a6eeb596..c3f881c206 100644 --- a/test/results/tumblrgallery.py +++ b/test/results/tumblrgallery.py @@ -1,33 +1,27 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import tumblrgallery - __tests__ = ( -{ - "#url" : "https://tumblrgallery.xyz/tumblrblog/gallery/103975.html", - "#category": ("", "tumblrgallery", "tumblrblog"), - "#class" : tumblrgallery.TumblrgalleryTumblrblogExtractor, -}, - -{ - "#url" : "https://tumblrgallery.xyz/post/405674.html", - "#category": ("", "tumblrgallery", "post"), - "#class" : tumblrgallery.TumblrgalleryPostExtractor, - "#pattern" : r"https://78\.media\.tumblr\.com/bec67072219c1f3bc04fd9711dec42ef/tumblr_p51qq1XCHS1txhgk3o1_1280\.jpg", - "#count" : 3, -}, - -{ - "#url" : "https://tumblrgallery.xyz/s.php?q=everyday-life", - "#category": ("", "tumblrgallery", "search"), - "#class" : tumblrgallery.TumblrgallerySearchExtractor, - "#pattern" : r"https://\d+\.media\.tumblr\.com/.+", - "#count" : "< 1000", -}, - + { + "#url": "https://tumblrgallery.xyz/tumblrblog/gallery/103975.html", + "#category": ("", "tumblrgallery", "tumblrblog"), + "#class": tumblrgallery.TumblrgalleryTumblrblogExtractor, + }, + { + "#url": "https://tumblrgallery.xyz/post/405674.html", + "#category": ("", "tumblrgallery", "post"), + "#class": tumblrgallery.TumblrgalleryPostExtractor, + "#pattern": r"https://78\.media\.tumblr\.com/bec67072219c1f3bc04fd9711dec42ef/tumblr_p51qq1XCHS1txhgk3o1_1280\.jpg", + "#count": 3, + }, + { + "#url": "https://tumblrgallery.xyz/s.php?q=everyday-life", + "#category": ("", "tumblrgallery", "search"), + "#class": tumblrgallery.TumblrgallerySearchExtractor, + "#pattern": r"https://\d+\.media\.tumblr\.com/.+", + "#count": "< 1000", + }, ) diff --git a/test/results/turboimagehost.py b/test/results/turboimagehost.py index 642d932193..b3f45d5202 100644 --- a/test/results/turboimagehost.py +++ b/test/results/turboimagehost.py @@ -1,23 +1,19 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://www.turboimagehost.com/p/39078423/test--.png.html", - "#category": ("imagehost", "turboimagehost", "image"), - "#class" : imagehosts.TurboimagehostImageExtractor, - "#sha1_url" : "b94de43612318771ced924cb5085976f13b3b90e", - "#sha1_metadata": "704757ca8825f51cec516ec44c1e627c1f2058ca", - "#sha1_content" : ( - "f38b54b17cd7462e687b58d83f00fca88b1b105a", - "0c8768055e4e20e7c7259608b67799171b691140", - ), -}, - + { + "#url": "https://www.turboimagehost.com/p/39078423/test--.png.html", + "#category": ("imagehost", "turboimagehost", "image"), + "#class": imagehosts.TurboimagehostImageExtractor, + "#sha1_url": "b94de43612318771ced924cb5085976f13b3b90e", + "#sha1_metadata": "704757ca8825f51cec516ec44c1e627c1f2058ca", + "#sha1_content": ( + "f38b54b17cd7462e687b58d83f00fca88b1b105a", + "0c8768055e4e20e7c7259608b67799171b691140", + ), + }, ) diff --git a/test/results/twibooru.py b/test/results/twibooru.py index 4720309f1e..89ef0c4261 100644 --- a/test/results/twibooru.py +++ b/test/results/twibooru.py @@ -1,110 +1,97 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import twibooru - __tests__ = ( -{ - "#url" : "https://twibooru.org/1", - "#category": ("philomena", "twibooru", "post"), - "#class" : twibooru.TwibooruPostExtractor, - "#pattern" : "https://cdn.twibooru.org/img/2020/7/8/1/full.png", - "#sha1_content": "aac4d1dba611883ac701aaa8f0b2b322590517ae", - - "animated" : False, - "aspect_ratio" : 1.0, - "comment_count" : int, - "created_at" : "2020-07-08T22:26:55.743Z", - "date" : "dt:2020-07-08 22:26:55", - "description" : "Why have I done this?", - "downvotes" : 0, - "duration" : 0.0, - "faves" : int, - "first_seen_at" : "2020-07-08T22:26:55.743Z", - "format" : "png", - "height" : 576, - "hidden_from_users": False, - "id" : 1, - "intensities" : dict, - "locations" : [], - "media_type" : "image", - "mime_type" : "image/png", - "name" : "1676547__safe_artist-colon-scraggleman_oc_oc-colon-floor+bored_oc+only_bags+under+eyes_bust_earth+pony_female_goggles_helmet_mare_meme_neet_neet+home+g.png", - "orig_sha512_hash": r"re:8b4c00d2[0-9a-f]{120}", - "processed" : True, - "representations" : dict, - "score" : int, - "sha512_hash" : "8b4c00d2eff52d51ad9647e14738944ab306fd1d8e1bf634fbb181b32f44070aa588938e26c4eb072b1eb61489aaf3062fb644a76c79f936b97723a2c3e0e5d3", - "size" : 70910, - "source_url" : "", - "tag_ids" : list, - "tags" : list, - "thumbnails_generated": True, - "updated_at" : str, - "upvotes" : int, - "view_url" : "https://cdn.twibooru.org/img/2020/7/8/1/full.png", - "width" : 576, - "wilson_score" : float, -}, - -{ - "#url" : "https://twibooru.org/523964", - "#comment" : "svg (#5643)", - "#category": ("philomena", "twibooru", "post"), - "#class" : twibooru.TwibooruPostExtractor, - "#urls" : "https://cdn.twibooru.org/img/2020/7/13/523964/full.svg", - "#sha1_content": "15590fe151ff65ef767b409e46dfdf708b339f4d", - - "extension": "svg", - "format" : "svg", -}, - -{ - "#url" : "https://twibooru.org/523964", - "#comment" : "svg (#5643)", - "#category": ("philomena", "twibooru", "post"), - "#class" : twibooru.TwibooruPostExtractor, - "#options" : {"svg": False}, - "#urls" : "https://cdn.twibooru.org/img/2020/7/13/523964/full.png", - "#sha1_content": "f8ff78e6a929a024f8529199f9a600617898d03c", - - "extension": "png", - "format" : "svg", -}, - -{ - "#url" : "https://twibooru.org/search?q=cute", - "#category": ("philomena", "twibooru", "search"), - "#class" : twibooru.TwibooruSearchExtractor, - "#range" : "40-60", - "#count" : 21, -}, - -{ - "#url" : "https://twibooru.org/tags/cute", - "#category": ("philomena", "twibooru", "search"), - "#class" : twibooru.TwibooruSearchExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://twibooru.org/galleries/1", - "#category": ("philomena", "twibooru", "gallery"), - "#class" : twibooru.TwibooruGalleryExtractor, - "#range" : "1-20", - - "gallery": { - "description" : "Best nation pone and russian related pics.", - "id" : 1, - "spoiler_warning": "Russia", - "thumbnail_id" : 694923, - "title" : "Marussiaverse", + { + "#url": "https://twibooru.org/1", + "#category": ("philomena", "twibooru", "post"), + "#class": twibooru.TwibooruPostExtractor, + "#pattern": "https://cdn.twibooru.org/img/2020/7/8/1/full.png", + "#sha1_content": "aac4d1dba611883ac701aaa8f0b2b322590517ae", + "animated": False, + "aspect_ratio": 1.0, + "comment_count": int, + "created_at": "2020-07-08T22:26:55.743Z", + "date": "dt:2020-07-08 22:26:55", + "description": "Why have I done this?", + "downvotes": 0, + "duration": 0.0, + "faves": int, + "first_seen_at": "2020-07-08T22:26:55.743Z", + "format": "png", + "height": 576, + "hidden_from_users": False, + "id": 1, + "intensities": dict, + "locations": [], + "media_type": "image", + "mime_type": "image/png", + "name": "1676547__safe_artist-colon-scraggleman_oc_oc-colon-floor+bored_oc+only_bags+under+eyes_bust_earth+pony_female_goggles_helmet_mare_meme_neet_neet+home+g.png", + "orig_sha512_hash": r"re:8b4c00d2[0-9a-f]{120}", + "processed": True, + "representations": dict, + "score": int, + "sha512_hash": "8b4c00d2eff52d51ad9647e14738944ab306fd1d8e1bf634fbb181b32f44070aa588938e26c4eb072b1eb61489aaf3062fb644a76c79f936b97723a2c3e0e5d3", + "size": 70910, + "source_url": "", + "tag_ids": list, + "tags": list, + "thumbnails_generated": True, + "updated_at": str, + "upvotes": int, + "view_url": "https://cdn.twibooru.org/img/2020/7/8/1/full.png", + "width": 576, + "wilson_score": float, + }, + { + "#url": "https://twibooru.org/523964", + "#comment": "svg (#5643)", + "#category": ("philomena", "twibooru", "post"), + "#class": twibooru.TwibooruPostExtractor, + "#urls": "https://cdn.twibooru.org/img/2020/7/13/523964/full.svg", + "#sha1_content": "15590fe151ff65ef767b409e46dfdf708b339f4d", + "extension": "svg", + "format": "svg", + }, + { + "#url": "https://twibooru.org/523964", + "#comment": "svg (#5643)", + "#category": ("philomena", "twibooru", "post"), + "#class": twibooru.TwibooruPostExtractor, + "#options": {"svg": False}, + "#urls": "https://cdn.twibooru.org/img/2020/7/13/523964/full.png", + "#sha1_content": "f8ff78e6a929a024f8529199f9a600617898d03c", + "extension": "png", + "format": "svg", + }, + { + "#url": "https://twibooru.org/search?q=cute", + "#category": ("philomena", "twibooru", "search"), + "#class": twibooru.TwibooruSearchExtractor, + "#range": "40-60", + "#count": 21, + }, + { + "#url": "https://twibooru.org/tags/cute", + "#category": ("philomena", "twibooru", "search"), + "#class": twibooru.TwibooruSearchExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://twibooru.org/galleries/1", + "#category": ("philomena", "twibooru", "gallery"), + "#class": twibooru.TwibooruGalleryExtractor, + "#range": "1-20", + "gallery": { + "description": "Best nation pone and russian related pics.", + "id": 1, + "spoiler_warning": "Russia", + "thumbnail_id": 694923, + "title": "Marussiaverse", + }, }, -}, - ) diff --git a/test/results/twitter.py b/test/results/twitter.py index 6da5d68e42..0961c31ca1 100644 --- a/test/results/twitter.py +++ b/test/results/twitter.py @@ -1,721 +1,628 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +from gallery_dl import exception +from gallery_dl import util from gallery_dl.extractor import twitter -from gallery_dl import util, exception - __tests__ = ( -{ - "#url" : "https://twitter.com/supernaturepics", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, - "#options" : {"include": "all"}, - "#urls" : [ - "https://x.com/supernaturepics/info", - "https://x.com/supernaturepics/photo", - "https://x.com/supernaturepics/header_photo", - "https://x.com/supernaturepics/timeline", - "https://x.com/supernaturepics/tweets", - "https://x.com/supernaturepics/media", - "https://x.com/supernaturepics/with_replies", - "https://x.com/supernaturepics/likes", - ], -}, - -{ - "#url" : "https://mobile.twitter.com/supernaturepics?p=i", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://www.twitter.com/id:2976459548", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://twitter.com/i/user/2976459548", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://twitter.com/intent/user?user_id=2976459548", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://fxtwitter.com/supernaturepics", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://vxtwitter.com/supernaturepics", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://fixupx.com/supernaturepics", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://fixvx.com/supernaturepics", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://x.com/supernaturepics", - "#category": ("", "twitter", "user"), - "#class" : twitter.TwitterUserExtractor, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/timeline", - "#category": ("", "twitter", "timeline"), - "#class" : twitter.TwitterTimelineExtractor, - "#range" : "1-40", - "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", - - "author": { - "date" : "dt:2015-01-12 10:25:22", - "description" : "The very best nature pictures.", - "favourites_count": int, - "followers_count" : int, - "friends_count" : int, - "listed_count" : int, - "media_count" : int, - "statuses_count" : int, - "id" : 2976459548, - "location" : "Earth", - "name" : "supernaturepics", - "nick" : "Nature Pictures", - "profile_banner" : "https://pbs.twimg.com/profile_banners/2976459548/1421058583", - "profile_image" : "https://pbs.twimg.com/profile_images/554585280938659841/FLVAlX18.jpeg", - "protected" : False, - "verified" : False, - }, - "user": { - "date" : "dt:2015-01-12 10:25:22", - "description" : "The very best nature pictures.", - "favourites_count": int, - "followers_count" : int, - "friends_count" : int, - "listed_count" : int, - "media_count" : int, - "statuses_count" : int, - "id" : 2976459548, - "location" : "Earth", - "name" : "supernaturepics", - "nick" : "Nature Pictures", - "profile_banner" : "https://pbs.twimg.com/profile_banners/2976459548/1421058583", - "profile_image" : "https://pbs.twimg.com/profile_images/554585280938659841/FLVAlX18.jpeg", - "protected" : False, - "verified" : False, - }, - "tweet_id" : range(400000000000000000, 800000000000000000), - "conversation_id": range(400000000000000000, 800000000000000000), - "quote_id" : 0, - "reply_id" : 0, - "retweet_id" : 0, - "count" : range(1, 4), - "num" : range(1, 4), - "favorite_count" : int, - "quote_count" : int, - "reply_count" : int, - "retweet_count" : int, - "content" : str, - "lang" : str, - "date" : "type:datetime", - "sensitive" : False, - "source" : "nature_pics", -}, - -{ - "#url" : "https://twitter.com/OptionalTypo/timeline", - "#comment" : "suspended account (#2216)", - "#category": ("", "twitter", "timeline"), - "#class" : twitter.TwitterTimelineExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://twitter.com/id:772949683521978368/timeline", - "#comment" : "suspended account user ID", - "#category": ("", "twitter", "timeline"), - "#class" : twitter.TwitterTimelineExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://mobile.twitter.com/supernaturepics/timeline#t", - "#category": ("", "twitter", "timeline"), - "#class" : twitter.TwitterTimelineExtractor, -}, - -{ - "#url" : "https://www.twitter.com/id:2976459548/timeline", - "#category": ("", "twitter", "timeline"), - "#class" : twitter.TwitterTimelineExtractor, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/tweets", - "#category": ("", "twitter", "tweets"), - "#class" : twitter.TwitterTweetsExtractor, - "#range" : "1-40", - "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", -}, - -{ - "#url" : "https://mobile.twitter.com/supernaturepics/tweets#t", - "#category": ("", "twitter", "tweets"), - "#class" : twitter.TwitterTweetsExtractor, -}, - -{ - "#url" : "https://www.twitter.com/id:2976459548/tweets", - "#category": ("", "twitter", "tweets"), - "#class" : twitter.TwitterTweetsExtractor, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/with_replies", - "#category": ("", "twitter", "replies"), - "#class" : twitter.TwitterRepliesExtractor, - "#range" : "1-40", - "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", -}, - -{ - "#url" : "https://mobile.twitter.com/supernaturepics/with_replies#t", - "#category": ("", "twitter", "replies"), - "#class" : twitter.TwitterRepliesExtractor, -}, - -{ - "#url" : "https://www.twitter.com/id:2976459548/with_replies", - "#category": ("", "twitter", "replies"), - "#class" : twitter.TwitterRepliesExtractor, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/media", - "#category": ("", "twitter", "media"), - "#class" : twitter.TwitterMediaExtractor, - "#range" : "1-40", - "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", -}, - -{ - "#url" : "https://mobile.twitter.com/supernaturepics/media#t", - "#category": ("", "twitter", "media"), - "#class" : twitter.TwitterMediaExtractor, -}, - -{ - "#url" : "https://www.twitter.com/id:2976459548/media", - "#category": ("", "twitter", "media"), - "#class" : twitter.TwitterMediaExtractor, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/likes", - "#category": ("", "twitter", "likes"), - "#class" : twitter.TwitterLikesExtractor, -}, - -{ - "#url" : "https://twitter.com/i/bookmarks", - "#category": ("", "twitter", "bookmark"), - "#class" : twitter.TwitterBookmarkExtractor, -}, - -{ - "#url" : "https://twitter.com/i/lists/784214683683127296", - "#category": ("", "twitter", "list"), - "#class" : twitter.TwitterListExtractor, - "#range" : "1-40", - "#count" : 40, - "#archive" : False, -}, - -{ - "#url" : "https://twitter.com/i/lists/784214683683127296/members", - "#category": ("", "twitter", "list-members"), - "#class" : twitter.TwitterListMembersExtractor, - "#pattern" : twitter.TwitterUserExtractor.pattern, - "#range" : "1-40", - "#count" : 40, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/following", - "#category": ("", "twitter", "following"), - "#class" : twitter.TwitterFollowingExtractor, -}, - -{ - "#url" : "https://www.twitter.com/id:2976459548/following", - "#category": ("", "twitter", "following"), - "#class" : twitter.TwitterFollowingExtractor, -}, - -{ - "#url" : "https://twitter.com/search?q=nature", - "#category": ("", "twitter", "search"), - "#class" : twitter.TwitterSearchExtractor, - "#range" : "1-20", - "#count" : 20, - "#archive" : False, -}, - -{ - "#url" : "https://twitter.com/hashtag/nature", - "#category": ("", "twitter", "hashtag"), - "#class" : twitter.TwitterHashtagExtractor, - "#pattern" : twitter.TwitterSearchExtractor.pattern, - "#urls" : "https://x.com/search?q=%23nature", -}, - -{ - "#url" : "https://twitter.com/i/events/1484669206993903616", - "#category": ("", "twitter", "event"), - "#class" : twitter.TwitterEventExtractor, - "#range" : "1-20", - "#count" : ">=1", -}, - -{ - "#url" : "https://twitter.com/i/communities", - "#category": ("", "twitter", "communities"), - "#class" : twitter.TwitterCommunitiesExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://twitter.com/i/communities/1651515740753735697", - "#category": ("", "twitter", "community"), - "#class" : twitter.TwitterCommunityExtractor, - "#range" : "1-20", - "#count" : 20, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/status/604341487988576256", - "#comment" : "all Tweets from a 'conversation' (#1319)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#sha1_url" : "88a40f7d25529c2501c46f2218f9e0de9aa634b4", - "#sha1_content": "ab05e1d8d21f8d43496df284d31e8b362cd3bcab", -}, - -{ - "#url" : "https://twitter.com/perrypumas/status/894001459754180609", - "#comment" : "4 images", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#sha1_url": "3a2a43dc5fb79dd5432c701d8e55e87c4e551f47", - - "type": "photo", -}, - -{ - "#url" : "https://twitter.com/perrypumas/status/1065692031626829824?s=20", - "#comment" : "video", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#pattern" : r"https://video.twimg.com/ext_tw_video/.+\.mp4\?tag=5", - - "type": "video", -}, - -{ - "#url" : "https://twitter.com/playpokemon/status/1263832915173048321/", - "#comment" : "content with emoji, newlines, hashtags (#338)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - - "source" : "Sprinklr", - "content": r"""re:Gear up for #PokemonSwordShieldEX with special Mystery Gifts! \n + { + "#url": "https://twitter.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + "#options": {"include": "all"}, + "#urls": [ + "https://x.com/supernaturepics/info", + "https://x.com/supernaturepics/photo", + "https://x.com/supernaturepics/header_photo", + "https://x.com/supernaturepics/timeline", + "https://x.com/supernaturepics/tweets", + "https://x.com/supernaturepics/media", + "https://x.com/supernaturepics/with_replies", + "https://x.com/supernaturepics/likes", + ], + }, + { + "#url": "https://mobile.twitter.com/supernaturepics?p=i", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://www.twitter.com/id:2976459548", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://twitter.com/i/user/2976459548", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://twitter.com/intent/user?user_id=2976459548", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://fxtwitter.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://vxtwitter.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://fixupx.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://fixvx.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://x.com/supernaturepics", + "#category": ("", "twitter", "user"), + "#class": twitter.TwitterUserExtractor, + }, + { + "#url": "https://twitter.com/supernaturepics/timeline", + "#category": ("", "twitter", "timeline"), + "#class": twitter.TwitterTimelineExtractor, + "#range": "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", + "author": { + "date": "dt:2015-01-12 10:25:22", + "description": "The very best nature pictures.", + "favourites_count": int, + "followers_count": int, + "friends_count": int, + "listed_count": int, + "media_count": int, + "statuses_count": int, + "id": 2976459548, + "location": "Earth", + "name": "supernaturepics", + "nick": "Nature Pictures", + "profile_banner": "https://pbs.twimg.com/profile_banners/2976459548/1421058583", + "profile_image": "https://pbs.twimg.com/profile_images/554585280938659841/FLVAlX18.jpeg", + "protected": False, + "verified": False, + }, + "user": { + "date": "dt:2015-01-12 10:25:22", + "description": "The very best nature pictures.", + "favourites_count": int, + "followers_count": int, + "friends_count": int, + "listed_count": int, + "media_count": int, + "statuses_count": int, + "id": 2976459548, + "location": "Earth", + "name": "supernaturepics", + "nick": "Nature Pictures", + "profile_banner": "https://pbs.twimg.com/profile_banners/2976459548/1421058583", + "profile_image": "https://pbs.twimg.com/profile_images/554585280938659841/FLVAlX18.jpeg", + "protected": False, + "verified": False, + }, + "tweet_id": range(400000000000000000, 800000000000000000), + "conversation_id": range(400000000000000000, 800000000000000000), + "quote_id": 0, + "reply_id": 0, + "retweet_id": 0, + "count": range(1, 4), + "num": range(1, 4), + "favorite_count": int, + "quote_count": int, + "reply_count": int, + "retweet_count": int, + "content": str, + "lang": str, + "date": "type:datetime", + "sensitive": False, + "source": "nature_pics", + }, + { + "#url": "https://twitter.com/OptionalTypo/timeline", + "#comment": "suspended account (#2216)", + "#category": ("", "twitter", "timeline"), + "#class": twitter.TwitterTimelineExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://twitter.com/id:772949683521978368/timeline", + "#comment": "suspended account user ID", + "#category": ("", "twitter", "timeline"), + "#class": twitter.TwitterTimelineExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://mobile.twitter.com/supernaturepics/timeline#t", + "#category": ("", "twitter", "timeline"), + "#class": twitter.TwitterTimelineExtractor, + }, + { + "#url": "https://www.twitter.com/id:2976459548/timeline", + "#category": ("", "twitter", "timeline"), + "#class": twitter.TwitterTimelineExtractor, + }, + { + "#url": "https://twitter.com/supernaturepics/tweets", + "#category": ("", "twitter", "tweets"), + "#class": twitter.TwitterTweetsExtractor, + "#range": "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", + }, + { + "#url": "https://mobile.twitter.com/supernaturepics/tweets#t", + "#category": ("", "twitter", "tweets"), + "#class": twitter.TwitterTweetsExtractor, + }, + { + "#url": "https://www.twitter.com/id:2976459548/tweets", + "#category": ("", "twitter", "tweets"), + "#class": twitter.TwitterTweetsExtractor, + }, + { + "#url": "https://twitter.com/supernaturepics/with_replies", + "#category": ("", "twitter", "replies"), + "#class": twitter.TwitterRepliesExtractor, + "#range": "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", + }, + { + "#url": "https://mobile.twitter.com/supernaturepics/with_replies#t", + "#category": ("", "twitter", "replies"), + "#class": twitter.TwitterRepliesExtractor, + }, + { + "#url": "https://www.twitter.com/id:2976459548/with_replies", + "#category": ("", "twitter", "replies"), + "#class": twitter.TwitterRepliesExtractor, + }, + { + "#url": "https://twitter.com/supernaturepics/media", + "#category": ("", "twitter", "media"), + "#class": twitter.TwitterMediaExtractor, + "#range": "1-40", + "#sha1_url": "c570ac1aae38ed1463be726cc46f31cac3d82a40", + }, + { + "#url": "https://mobile.twitter.com/supernaturepics/media#t", + "#category": ("", "twitter", "media"), + "#class": twitter.TwitterMediaExtractor, + }, + { + "#url": "https://www.twitter.com/id:2976459548/media", + "#category": ("", "twitter", "media"), + "#class": twitter.TwitterMediaExtractor, + }, + { + "#url": "https://twitter.com/supernaturepics/likes", + "#category": ("", "twitter", "likes"), + "#class": twitter.TwitterLikesExtractor, + }, + { + "#url": "https://twitter.com/i/bookmarks", + "#category": ("", "twitter", "bookmark"), + "#class": twitter.TwitterBookmarkExtractor, + }, + { + "#url": "https://twitter.com/i/lists/784214683683127296", + "#category": ("", "twitter", "list"), + "#class": twitter.TwitterListExtractor, + "#range": "1-40", + "#count": 40, + "#archive": False, + }, + { + "#url": "https://twitter.com/i/lists/784214683683127296/members", + "#category": ("", "twitter", "list-members"), + "#class": twitter.TwitterListMembersExtractor, + "#pattern": twitter.TwitterUserExtractor.pattern, + "#range": "1-40", + "#count": 40, + }, + { + "#url": "https://twitter.com/supernaturepics/following", + "#category": ("", "twitter", "following"), + "#class": twitter.TwitterFollowingExtractor, + }, + { + "#url": "https://www.twitter.com/id:2976459548/following", + "#category": ("", "twitter", "following"), + "#class": twitter.TwitterFollowingExtractor, + }, + { + "#url": "https://twitter.com/search?q=nature", + "#category": ("", "twitter", "search"), + "#class": twitter.TwitterSearchExtractor, + "#range": "1-20", + "#count": 20, + "#archive": False, + }, + { + "#url": "https://twitter.com/hashtag/nature", + "#category": ("", "twitter", "hashtag"), + "#class": twitter.TwitterHashtagExtractor, + "#pattern": twitter.TwitterSearchExtractor.pattern, + "#urls": "https://x.com/search?q=%23nature", + }, + { + "#url": "https://twitter.com/i/events/1484669206993903616", + "#category": ("", "twitter", "event"), + "#class": twitter.TwitterEventExtractor, + "#range": "1-20", + "#count": ">=1", + }, + { + "#url": "https://twitter.com/i/communities", + "#category": ("", "twitter", "communities"), + "#class": twitter.TwitterCommunitiesExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://twitter.com/i/communities/1651515740753735697", + "#category": ("", "twitter", "community"), + "#class": twitter.TwitterCommunityExtractor, + "#range": "1-20", + "#count": 20, + }, + { + "#url": "https://twitter.com/supernaturepics/status/604341487988576256", + "#comment": "all Tweets from a 'conversation' (#1319)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#sha1_url": "88a40f7d25529c2501c46f2218f9e0de9aa634b4", + "#sha1_content": "ab05e1d8d21f8d43496df284d31e8b362cd3bcab", + }, + { + "#url": "https://twitter.com/perrypumas/status/894001459754180609", + "#comment": "4 images", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#sha1_url": "3a2a43dc5fb79dd5432c701d8e55e87c4e551f47", + "type": "photo", + }, + { + "#url": "https://twitter.com/perrypumas/status/1065692031626829824?s=20", + "#comment": "video", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#pattern": r"https://video.twimg.com/ext_tw_video/.+\.mp4\?tag=5", + "type": "video", + }, + { + "#url": "https://twitter.com/playpokemon/status/1263832915173048321/", + "#comment": "content with emoji, newlines, hashtags (#338)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "source": "Sprinklr", + "content": r"""re:Gear up for #PokemonSwordShieldEX with special Mystery Gifts! \n You’ll be able to receive four Galarian form Pokémon with Hidden Abilities, plus some very useful items. It’s our \(Mystery\) Gift to you, Trainers! \n ❓🎁➡️ """, -}, - -{ - "#url" : "https://twitter.com/i/web/status/1170041925560258560", - "#comment" : "'replies' option (#705)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#pattern" : "https://pbs.twimg.com/media/EDzS7VrU0AAFL4_", -}, - -{ - "#url" : "https://twitter.com/i/web/status/1170041925560258560", - "#comment" : "'replies' option (#705)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"replies": False}, - "#count" : 0, -}, - -{ - "#url" : "https://twitter.com/i/web/status/1424882930803908612", - "#comment" : "'replies' to self (#1254)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"replies": "self"}, - "#count" : 4, - - "user": { - "description": r"re:business email-- rhettaro.bloom@gmail.com patreon- http://patreon.com/Princecanary", - "url" : "http://princecanary.tumblr.com", }, -}, - -{ - "#url" : "https://twitter.com/i/web/status/1424898916156284928", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"replies": "self"}, - "#count" : 1, -}, - -{ - "#url" : "https://twitter.com/StobiesGalaxy/status/1270755918330896395", - "#comment" : "quoted tweet (#526, #854)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"quoted": True}, - "#pattern" : r"https://pbs\.twimg\.com/media/Ea[KG].+=jpg", - "#count" : 8, -}, - -{ - "#url" : "https://twitter.com/StobiesGalaxy/status/1270755918330896395", - "#comment" : "quoted tweet (#526, #854)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#pattern" : r"https://pbs\.twimg\.com/media/EaK.+=jpg", - "#count" : 4, -}, - -{ - "#url" : "https://twitter.com/web/status/1644907989109751810", - "#comment" : "different 'user' and 'author' in quoted Tweet (#3922)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - - "author": { - "id" : 321629993, - "name": "Cakes_Comics", + { + "#url": "https://twitter.com/i/web/status/1170041925560258560", + "#comment": "'replies' option (#705)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#pattern": "https://pbs.twimg.com/media/EDzS7VrU0AAFL4_", }, - "user" : { - "id" : 718928225360080897, - "name": "StobiesGalaxy", + { + "#url": "https://twitter.com/i/web/status/1170041925560258560", + "#comment": "'replies' option (#705)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"replies": False}, + "#count": 0, }, -}, - -{ - "#url" : "https://twitter.com/i/web/status/112900228289540096", - "#comment" : "TwitPic embeds (#579)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : { - "twitpic": True, - "cards" : False, - }, - "#pattern" : r"https://\w+.cloudfront.net/photos/large/\d+.jpg", - "#count" : 2, -}, - -{ - "#url" : "https://twitter.com/shimoigusaP/status/8138669971", - "#comment" : "TwitPic URL not in 'urls' (#3792)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"twitpic": True}, - "#pattern" : r"https://\w+.cloudfront.net/photos/large/\d+.png", - "#count" : 1, -}, - -{ - "#url" : "https://twitter.com/billboard/status/1306599586602135555", - "#comment" : "Twitter card (#1005)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"cards": True}, - "#pattern" : r"https://pbs.twimg.com/card_img/\d+/", -}, - -{ - "#url" : "https://twitter.com/i/web/status/1561674543323910144", - "#comment" : "unified_card image_website (#2875)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"cards": True}, - "#pattern" : r"https://pbs\.twimg\.com/media/F.+=jpg", -}, - -{ - "#url" : "https://twitter.com/doax_vv_staff/status/1479438945662685184", - "#comment" : "unified_card image_carousel_website", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"cards": True}, - "#pattern" : r"https://pbs\.twimg\.com/media/F.+=png", - "#count" : 6, -}, - -{ - "#url" : "https://twitter.com/bang_dream_1242/status/1561548715348746241", - "#comment" : "unified_card video_website (#2875)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"cards": True}, - "#pattern" : r"https://video\.twimg\.com/amplify_video/1560607284333449216/vid/720x720/\w+\.mp4", -}, - -{ - "#url" : "https://twitter.com/i/web/status/1466183847628865544", - "#comment" : "unified_card without type", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://twitter.com/i/web/status/1571141912295243776", - "#comment" : "'cards-blacklist' option", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : { - "cards" : "ytdl", - "cards-blacklist": ("twitch.tv",), - }, - "#count" : 0, -}, - -{ - "#url" : "https://twitter.com/jessica_3978/status/1296304589591810048", - "#comment" : "original retweets (#1026)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"retweets": True}, - "#count" : 2, - - "tweet_id" : 1296304589591810048, - "retweet_id" : 1296296016002547713, - "date" : "dt:2020-08-20 04:34:32", - "date_original": "dt:2020-08-20 04:00:28", -}, - -{ - "#url" : "https://twitter.com/jessica_3978/status/1296304589591810048", - "#comment" : "original retweets (#1026)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"retweets": "original"}, - "#count" : 2, - - "tweet_id" : 1296296016002547713, - "retweet_id" : 1296296016002547713, - "date" : "dt:2020-08-20 04:00:28", - "date_original": "dt:2020-08-20 04:00:28", -}, - -{ - "#url" : "https://twitter.com/supernaturepics/status/604341487988576256", - "#comment" : "all Tweets from a 'conversation' (#1319)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"conversations": True}, - "#count" : 5, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/status/604341487988576256/photo/1", - "#comment" : "/photo/ URL (#5443)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, -}, - -{ - "#url" : "https://twitter.com/perrypumas/status/1065692031626829824/video/1", - "#comment" : "/video/ URL", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, -}, - -{ - "#url" : "https://twitter.com/morino_ya/status/1392763691599237121", - "#comment" : "retweet with missing media entities (#1555)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"retweets": True}, - "#count" : 4, -}, - -{ - "#url" : "https://twitter.com/i/web/status/1460044411165888515", - "#comment" : "deleted quote tweet (#2225)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://twitter.com/i/web/status/1486373748911575046", - "#comment" : "'Misleading' content", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#count" : 4, -}, - -{ - "#url" : "https://twitter.com/mightbecursed/status/1492954264909479936", - "#comment" : "age-restricted (#2354)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#auth" : False, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://twitter.com/my0nruri/status/1528379296041299968", - "#comment" : "media alt texts / descriptions (#2617)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - - "description": "oc", - "type" : "photo", -}, - -{ - "#url" : "https://twitter.com/poco_dandy/status/1150646424461176832", - "#comment" : "'?format=...&name=...'-style URLs", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"cards": True}, - "#pattern" : r"https://pbs.twimg.com/card_img/17\d+/[\w-]+\?format=(jpg|png)&name=orig$", - "#range" : "1,3", -}, - -{ - "#url" : "https://twitter.com/i/web/status/1629193457112686592", - "#comment" : "note tweet with long 'content'", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - - "content": """BREAKING - DEADLY LIES: Independent researchers at Texas A&M University have just contradicted federal government regulators, saying that toxic air pollutants in East Palestine, Ohio, could pose long-term risks. \n + { + "#url": "https://twitter.com/i/web/status/1424882930803908612", + "#comment": "'replies' to self (#1254)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"replies": "self"}, + "#count": 4, + "user": { + "description": r"re:business email-- rhettaro.bloom@gmail.com patreon- http://patreon.com/Princecanary", + "url": "http://princecanary.tumblr.com", + }, + }, + { + "#url": "https://twitter.com/i/web/status/1424898916156284928", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"replies": "self"}, + "#count": 1, + }, + { + "#url": "https://twitter.com/StobiesGalaxy/status/1270755918330896395", + "#comment": "quoted tweet (#526, #854)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"quoted": True}, + "#pattern": r"https://pbs\.twimg\.com/media/Ea[KG].+=jpg", + "#count": 8, + }, + { + "#url": "https://twitter.com/StobiesGalaxy/status/1270755918330896395", + "#comment": "quoted tweet (#526, #854)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#pattern": r"https://pbs\.twimg\.com/media/EaK.+=jpg", + "#count": 4, + }, + { + "#url": "https://twitter.com/web/status/1644907989109751810", + "#comment": "different 'user' and 'author' in quoted Tweet (#3922)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "author": { + "id": 321629993, + "name": "Cakes_Comics", + }, + "user": { + "id": 718928225360080897, + "name": "StobiesGalaxy", + }, + }, + { + "#url": "https://twitter.com/i/web/status/112900228289540096", + "#comment": "TwitPic embeds (#579)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": { + "twitpic": True, + "cards": False, + }, + "#pattern": r"https://\w+.cloudfront.net/photos/large/\d+.jpg", + "#count": 2, + }, + { + "#url": "https://twitter.com/shimoigusaP/status/8138669971", + "#comment": "TwitPic URL not in 'urls' (#3792)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"twitpic": True}, + "#pattern": r"https://\w+.cloudfront.net/photos/large/\d+.png", + "#count": 1, + }, + { + "#url": "https://twitter.com/billboard/status/1306599586602135555", + "#comment": "Twitter card (#1005)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"cards": True}, + "#pattern": r"https://pbs.twimg.com/card_img/\d+/", + }, + { + "#url": "https://twitter.com/i/web/status/1561674543323910144", + "#comment": "unified_card image_website (#2875)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"cards": True}, + "#pattern": r"https://pbs\.twimg\.com/media/F.+=jpg", + }, + { + "#url": "https://twitter.com/doax_vv_staff/status/1479438945662685184", + "#comment": "unified_card image_carousel_website", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"cards": True}, + "#pattern": r"https://pbs\.twimg\.com/media/F.+=png", + "#count": 6, + }, + { + "#url": "https://twitter.com/bang_dream_1242/status/1561548715348746241", + "#comment": "unified_card video_website (#2875)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"cards": True}, + "#pattern": r"https://video\.twimg\.com/amplify_video/1560607284333449216/vid/720x720/\w+\.mp4", + }, + { + "#url": "https://twitter.com/i/web/status/1466183847628865544", + "#comment": "unified_card without type", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#count": 0, + }, + { + "#url": "https://twitter.com/i/web/status/1571141912295243776", + "#comment": "'cards-blacklist' option", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": { + "cards": "ytdl", + "cards-blacklist": ("twitch.tv",), + }, + "#count": 0, + }, + { + "#url": "https://twitter.com/jessica_3978/status/1296304589591810048", + "#comment": "original retweets (#1026)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"retweets": True}, + "#count": 2, + "tweet_id": 1296304589591810048, + "retweet_id": 1296296016002547713, + "date": "dt:2020-08-20 04:34:32", + "date_original": "dt:2020-08-20 04:00:28", + }, + { + "#url": "https://twitter.com/jessica_3978/status/1296304589591810048", + "#comment": "original retweets (#1026)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"retweets": "original"}, + "#count": 2, + "tweet_id": 1296296016002547713, + "retweet_id": 1296296016002547713, + "date": "dt:2020-08-20 04:00:28", + "date_original": "dt:2020-08-20 04:00:28", + }, + { + "#url": "https://twitter.com/supernaturepics/status/604341487988576256", + "#comment": "all Tweets from a 'conversation' (#1319)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"conversations": True}, + "#count": 5, + }, + { + "#url": "https://twitter.com/supernaturepics/status/604341487988576256/photo/1", + "#comment": "/photo/ URL (#5443)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + }, + { + "#url": "https://twitter.com/perrypumas/status/1065692031626829824/video/1", + "#comment": "/video/ URL", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + }, + { + "#url": "https://twitter.com/morino_ya/status/1392763691599237121", + "#comment": "retweet with missing media entities (#1555)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"retweets": True}, + "#count": 4, + }, + { + "#url": "https://twitter.com/i/web/status/1460044411165888515", + "#comment": "deleted quote tweet (#2225)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#count": 0, + }, + { + "#url": "https://twitter.com/i/web/status/1486373748911575046", + "#comment": "'Misleading' content", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#count": 4, + }, + { + "#url": "https://twitter.com/mightbecursed/status/1492954264909479936", + "#comment": "age-restricted (#2354)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#auth": False, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://twitter.com/my0nruri/status/1528379296041299968", + "#comment": "media alt texts / descriptions (#2617)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "description": "oc", + "type": "photo", + }, + { + "#url": "https://twitter.com/poco_dandy/status/1150646424461176832", + "#comment": "'?format=...&name=...'-style URLs", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"cards": True}, + "#pattern": r"https://pbs.twimg.com/card_img/17\d+/[\w-]+\?format=(jpg|png)&name=orig$", + "#range": "1,3", + }, + { + "#url": "https://twitter.com/i/web/status/1629193457112686592", + "#comment": "note tweet with long 'content'", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "content": """BREAKING - DEADLY LIES: Independent researchers at Texas A&M University have just contradicted federal government regulators, saying that toxic air pollutants in East Palestine, Ohio, could pose long-term risks. \n The Washington Post writes, "Three weeks after the toxic train derailment in Ohio, an analysis of Environmental Protection Agency data has found nine air pollutants at levels that could raise long-term health concerns in and around East Palestine, according to an independent analysis. \n "The analysis by Texas A&M University seems to contradict statements by state and federal regulators that air near the crash site is completely safe, despite residents complaining about rashes, breathing problems and other health effects." Your reaction.""", -}, - -{ - "#url" : "https://twitter.com/KrisKobach1787/status/1765935595702919299", - "#comment" : "'birdwatch' note (#5317)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#options" : {"text-tweets": True}, - - "birdwatch": "In addition to the known harm of lead exposure, especially to children, Mr. Kobach is incorrect when he states the mandate is unfunded. In fact, the BIPARTISAN Infrastructure Law Joe Biden signed into law in Nov 2021 provides $15B toward lead service line replacement projects. epa.gov/ground-water-a…", - "content" : "Biden wants to replace lead pipes. He failed to mention that the unfunded mandate sets an almost impossible timeline, will cost billions, infringe on the rights of the States and their residents – all for benefits that may be entirely speculative. #sotu https://ag.ks.gov/media-center/news-releases/2024/02/09/kobach-leads-coalition-demanding-biden-drop-unnecessary-epa-rule", -}, - -{ - "#url" : "https://x.com/jsports_motor/status/1801338077618524583", - "#comment" : "geo-restricted video (#5736)", - "#category": ("", "twitter", "tweet"), - "#class" : twitter.TwitterTweetExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://twitter.com/playpokemon/status/1263832915173048321/quotes", - "#category": ("", "twitter", "quotes"), - "#class" : twitter.TwitterQuotesExtractor, - "#pattern" : twitter.TwitterSearchExtractor.pattern, - "#urls" : "https://x.com/search?q=quoted_tweet_id:1263832915173048321", -}, - -{ - "#url" : "https://twitter.com/supernaturepics/info", - "#category": ("", "twitter", "info"), - "#class" : twitter.TwitterInfoExtractor, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/photo", - "#category": ("", "twitter", "avatar"), - "#class" : twitter.TwitterAvatarExtractor, - "#urls" : "https://pbs.twimg.com/profile_images/554585280938659841/FLVAlX18.jpeg", - - "date" : "dt:2015-01-12 10:26:49", - "extension": "jpeg", - "filename" : "FLVAlX18", - "tweet_id" : 554585280938659841, -}, - -{ - "#url" : "https://twitter.com/User16/photo", - "#category": ("", "twitter", "avatar"), - "#class" : twitter.TwitterAvatarExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://twitter.com/i_n_u/photo", - "#comment" : "old avatar with small ID and no valid 'date' (#4696)", - "#category": ("", "twitter", "avatar"), - "#class" : twitter.TwitterAvatarExtractor, - "#urls" : "https://pbs.twimg.com/profile_images/2946444489/32028c6affdab425e037ff5a6bf77c1d.jpeg", - - "date" : util.NONE, - "tweet_id" : 2946444489, -}, - -{ - "#url" : "https://twitter.com/supernaturepics/header_photo", - "#category": ("", "twitter", "background"), - "#class" : twitter.TwitterBackgroundExtractor, - "#pattern" : r"https://pbs\.twimg\.com/profile_banners/2976459548/1421058583", - - "date" : "dt:2015-01-12 10:29:43", - "filename": "1421058583", - "tweet_id": 554586009367478272, -}, - -{ - "#url" : "https://twitter.com/User16/header_photo", - "#category": ("", "twitter", "background"), - "#class" : twitter.TwitterBackgroundExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://pbs.twimg.com/media/EqcpviCVoAAG-QG?format=jpg&name=orig", - "#category": ("", "twitter", "image"), - "#class" : twitter.TwitterImageExtractor, - "#options" : {"size": "4096x4096,orig"}, - "#sha1_url": "cb3042a6f6826923da98f0d2b66c427e9385114c", -}, - -{ - "#url" : "https://pbs.twimg.com/media/EqcpviCVoAAG-QG.jpg:orig", - "#category": ("", "twitter", "image"), - "#class" : twitter.TwitterImageExtractor, -}, - + }, + { + "#url": "https://twitter.com/KrisKobach1787/status/1765935595702919299", + "#comment": "'birdwatch' note (#5317)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#options": {"text-tweets": True}, + "birdwatch": "In addition to the known harm of lead exposure, especially to children, Mr. Kobach is incorrect when he states the mandate is unfunded. In fact, the BIPARTISAN Infrastructure Law Joe Biden signed into law in Nov 2021 provides $15B toward lead service line replacement projects. epa.gov/ground-water-a…", + "content": "Biden wants to replace lead pipes. He failed to mention that the unfunded mandate sets an almost impossible timeline, will cost billions, infringe on the rights of the States and their residents – all for benefits that may be entirely speculative. #sotu https://ag.ks.gov/media-center/news-releases/2024/02/09/kobach-leads-coalition-demanding-biden-drop-unnecessary-epa-rule", + }, + { + "#url": "https://x.com/jsports_motor/status/1801338077618524583", + "#comment": "geo-restricted video (#5736)", + "#category": ("", "twitter", "tweet"), + "#class": twitter.TwitterTweetExtractor, + "#count": 0, + }, + { + "#url": "https://twitter.com/playpokemon/status/1263832915173048321/quotes", + "#category": ("", "twitter", "quotes"), + "#class": twitter.TwitterQuotesExtractor, + "#pattern": twitter.TwitterSearchExtractor.pattern, + "#urls": "https://x.com/search?q=quoted_tweet_id:1263832915173048321", + }, + { + "#url": "https://twitter.com/supernaturepics/info", + "#category": ("", "twitter", "info"), + "#class": twitter.TwitterInfoExtractor, + }, + { + "#url": "https://twitter.com/supernaturepics/photo", + "#category": ("", "twitter", "avatar"), + "#class": twitter.TwitterAvatarExtractor, + "#urls": "https://pbs.twimg.com/profile_images/554585280938659841/FLVAlX18.jpeg", + "date": "dt:2015-01-12 10:26:49", + "extension": "jpeg", + "filename": "FLVAlX18", + "tweet_id": 554585280938659841, + }, + { + "#url": "https://twitter.com/User16/photo", + "#category": ("", "twitter", "avatar"), + "#class": twitter.TwitterAvatarExtractor, + "#count": 0, + }, + { + "#url": "https://twitter.com/i_n_u/photo", + "#comment": "old avatar with small ID and no valid 'date' (#4696)", + "#category": ("", "twitter", "avatar"), + "#class": twitter.TwitterAvatarExtractor, + "#urls": "https://pbs.twimg.com/profile_images/2946444489/32028c6affdab425e037ff5a6bf77c1d.jpeg", + "date": util.NONE, + "tweet_id": 2946444489, + }, + { + "#url": "https://twitter.com/supernaturepics/header_photo", + "#category": ("", "twitter", "background"), + "#class": twitter.TwitterBackgroundExtractor, + "#pattern": r"https://pbs\.twimg\.com/profile_banners/2976459548/1421058583", + "date": "dt:2015-01-12 10:29:43", + "filename": "1421058583", + "tweet_id": 554586009367478272, + }, + { + "#url": "https://twitter.com/User16/header_photo", + "#category": ("", "twitter", "background"), + "#class": twitter.TwitterBackgroundExtractor, + "#count": 0, + }, + { + "#url": "https://pbs.twimg.com/media/EqcpviCVoAAG-QG?format=jpg&name=orig", + "#category": ("", "twitter", "image"), + "#class": twitter.TwitterImageExtractor, + "#options": {"size": "4096x4096,orig"}, + "#sha1_url": "cb3042a6f6826923da98f0d2b66c427e9385114c", + }, + { + "#url": "https://pbs.twimg.com/media/EqcpviCVoAAG-QG.jpg:orig", + "#category": ("", "twitter", "image"), + "#class": twitter.TwitterImageExtractor, + }, ) diff --git a/test/results/unique-vintage.py b/test/results/unique-vintage.py index 8fe37a3be1..a164be3792 100644 --- a/test/results/unique-vintage.py +++ b/test/results/unique-vintage.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://www.unique-vintage.com/collections/flapper-1920s", - "#category": ("shopify", "unique-vintage", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.unique-vintage.com/collections/flapper-1920s/products/unique-vintage-plus-size-black-silver-beaded-troyes-flapper-dress", - "#category": ("shopify", "unique-vintage", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://www.unique-vintage.com/collections/flapper-1920s", + "#category": ("shopify", "unique-vintage", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.unique-vintage.com/collections/flapper-1920s/products/unique-vintage-plus-size-black-silver-beaded-troyes-flapper-dress", + "#category": ("shopify", "unique-vintage", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/unsplash.py b/test/results/unsplash.py index 8c5297430a..f5f65a500a 100644 --- a/test/results/unsplash.py +++ b/test/results/unsplash.py @@ -1,159 +1,147 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import unsplash - __tests__ = ( -{ - "#url" : "https://unsplash.com/photos/red-wooden-cross-on-gray-concrete-pathway-between-green-trees-during-daytime-kaoHI0iHJPM", - "#category": ("", "unsplash", "image"), - "#class" : unsplash.UnsplashImageExtractor, - "#pattern" : r"https://images\.unsplash\.com/photo-1601823984263-b87b59798b", - - "alt_description": "red wooden cross on gray concrete pathway between green trees during daytime", - "blur_hash" : "LIAwhq%e4TRjXAIBMyt89GRj%fj[", - "breadcrumbs": list, - "color" : "#0c2626", - "created_at" : "2020-10-04T15:13:59Z", - "date" : "dt:2020-10-04 15:13:59", - "description": None, - "downloads" : range(50000, 300000), - "exif" : { - "aperture" : "9", - "exposure_time": "1/125", - "focal_length" : "35.0", - "iso" : 800, - "make" : "SONY", - "model" : "ILCE-7M3", - "name" : "SONY, ILCE-7M3", - }, - "extension" : "jpg", - "filename" : "photo-1601823984263-b87b59798b70", - "height" : 5371, - "id" : "kaoHI0iHJPM", - "liked_by_user": False, - "likes" : range(1000, 10000), - "links" : dict, - "location" : { - "city" : "箱根町", - "country" : "日本", - "name" : "Hakone, 神奈川県 日本", - "position": { - "latitude" : 35.232383, - "longitude": 139.106936, + { + "#url": "https://unsplash.com/photos/red-wooden-cross-on-gray-concrete-pathway-between-green-trees-during-daytime-kaoHI0iHJPM", + "#category": ("", "unsplash", "image"), + "#class": unsplash.UnsplashImageExtractor, + "#pattern": r"https://images\.unsplash\.com/photo-1601823984263-b87b59798b", + "alt_description": "red wooden cross on gray concrete pathway between green trees during daytime", + "blur_hash": "LIAwhq%e4TRjXAIBMyt89GRj%fj[", + "breadcrumbs": list, + "color": "#0c2626", + "created_at": "2020-10-04T15:13:59Z", + "date": "dt:2020-10-04 15:13:59", + "description": None, + "downloads": range(50000, 300000), + "exif": { + "aperture": "9", + "exposure_time": "1/125", + "focal_length": "35.0", + "iso": 800, + "make": "SONY", + "model": "ILCE-7M3", + "name": "SONY, ILCE-7M3", }, - }, - "meta" : { - "index": True, - }, - "plus" : False, - "premium" : False, - "promoted_at": "2020-10-05T13:04:43Z", - "public_domain": False, - "slug" : "red-wooden-cross-on-gray-concrete-pathway-between-green-trees-during-daytime-kaoHI0iHJPM", - "sponsorship": None, - "subcategory": "image", - "tags" : list, - "tags_preview": list, - "topic_submissions": {}, - "topics" : [], - "updated_at" : str, - "urls": dict, - "user": { - "accepted_tos" : True, - "bio" : "Professional photographer.\r\nBased in Japan.", - "first_name" : "Syuhei", - "for_hire" : True, - "id" : "F4HO358YSeo", - "instagram_username": "_______life_", - "last_name" : "Inoue", - "links": { - "followers": "https://api.unsplash.com/users/_______life_/followers", - "following": "https://api.unsplash.com/users/_______life_/following", - "html" : "https://unsplash.com/@_______life_", - "likes" : "https://api.unsplash.com/users/_______life_/likes", - "photos" : "https://api.unsplash.com/users/_______life_/photos", - "portfolio": "https://api.unsplash.com/users/_______life_/portfolio", - "self" : "https://api.unsplash.com/users/_______life_", + "extension": "jpg", + "filename": "photo-1601823984263-b87b59798b70", + "height": 5371, + "id": "kaoHI0iHJPM", + "liked_by_user": False, + "likes": range(1000, 10000), + "links": dict, + "location": { + "city": "箱根町", + "country": "日本", + "name": "Hakone, 神奈川県 日本", + "position": { + "latitude": 35.232383, + "longitude": 139.106936, + }, }, - "location" : "Yokohama, Japan", - "name" : "Syuhei Inoue", - "portfolio_url" : "https://syuheiinoue.life/", - "profile_image" : { - "large" : "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=128&h=128", - "medium": "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=64&h=64", - "small" : "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=32&h=32", + "meta": { + "index": True, }, - "social" : { + "plus": False, + "premium": False, + "promoted_at": "2020-10-05T13:04:43Z", + "public_domain": False, + "slug": "red-wooden-cross-on-gray-concrete-pathway-between-green-trees-during-daytime-kaoHI0iHJPM", + "sponsorship": None, + "subcategory": "image", + "tags": list, + "tags_preview": list, + "topic_submissions": {}, + "topics": [], + "updated_at": str, + "urls": dict, + "user": { + "accepted_tos": True, + "bio": "Professional photographer.\r\nBased in Japan.", + "first_name": "Syuhei", + "for_hire": True, + "id": "F4HO358YSeo", "instagram_username": "_______life_", - "paypal_email" : None, - "portfolio_url" : "https://syuheiinoue.life/", - "twitter_username" : None, + "last_name": "Inoue", + "links": { + "followers": "https://api.unsplash.com/users/_______life_/followers", + "following": "https://api.unsplash.com/users/_______life_/following", + "html": "https://unsplash.com/@_______life_", + "likes": "https://api.unsplash.com/users/_______life_/likes", + "photos": "https://api.unsplash.com/users/_______life_/photos", + "portfolio": "https://api.unsplash.com/users/_______life_/portfolio", + "self": "https://api.unsplash.com/users/_______life_", + }, + "location": "Yokohama, Japan", + "name": "Syuhei Inoue", + "portfolio_url": "https://syuheiinoue.life/", + "profile_image": { + "large": "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=128&h=128", + "medium": "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=64&h=64", + "small": "https://images.unsplash.com/profile-1601689368522-8855bbd61be6image?ixlib=rb-4.0.3&crop=faces&fit=crop&w=32&h=32", + }, + "social": { + "instagram_username": "_______life_", + "paypal_email": None, + "portfolio_url": "https://syuheiinoue.life/", + "twitter_username": None, + }, + "total_collections": 2, + "total_likes": 32, + "total_photos": 86, + "total_promoted_photos": 24, + "twitter_username": None, + "updated_at": str, + "username": "_______life_", }, - "total_collections" : 2, - "total_likes" : 32, - "total_photos" : 86, - "total_promoted_photos": 24, - "twitter_username" : None, - "updated_at" : str, - "username" : "_______life_" + "views": range(2000000, 10000000), + "width": 3581, + }, + { + "#url": "https://unsplash.com/@_______life_", + "#category": ("", "unsplash", "user"), + "#class": unsplash.UnsplashUserExtractor, + "#pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://unsplash.com/@_______life_/likes", + "#category": ("", "unsplash", "favorite"), + "#class": unsplash.UnsplashFavoriteExtractor, + "#pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#count": range(25, 35), + }, + { + "#url": "https://unsplash.com/collections/3178572/winter", + "#category": ("", "unsplash", "collection"), + "#class": unsplash.UnsplashCollectionExtractor, + "#pattern": r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range": "1-30", + "#count": 30, + "collection_id": "3178572", + "collection_title": "winter", + }, + { + "#url": "https://unsplash.com/collections/3178572/", + "#category": ("", "unsplash", "collection"), + "#class": unsplash.UnsplashCollectionExtractor, + }, + { + "#url": "https://unsplash.com/collections/_8qJQ2bCMWE/2021.05", + "#category": ("", "unsplash", "collection"), + "#class": unsplash.UnsplashCollectionExtractor, + }, + { + "#url": "https://unsplash.com/s/photos/hair-style", + "#category": ("", "unsplash", "search"), + "#class": unsplash.UnsplashSearchExtractor, + "#pattern": r"https://(images|plus)\.unsplash\.com/((flagged/|premium_)?photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", + "#range": "1-30", + "#count": 30, }, - "views": range(2000000, 10000000), - "width": 3581, -}, - -{ - "#url" : "https://unsplash.com/@_______life_", - "#category": ("", "unsplash", "user"), - "#class" : unsplash.UnsplashUserExtractor, - "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://unsplash.com/@_______life_/likes", - "#category": ("", "unsplash", "favorite"), - "#class" : unsplash.UnsplashFavoriteExtractor, - "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "#count" : range(25, 35), -}, - -{ - "#url" : "https://unsplash.com/collections/3178572/winter", - "#category": ("", "unsplash", "collection"), - "#class" : unsplash.UnsplashCollectionExtractor, - "#pattern" : r"https://images\.unsplash\.com/(photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "#range" : "1-30", - "#count" : 30, - - "collection_id" : "3178572", - "collection_title": "winter", -}, - -{ - "#url" : "https://unsplash.com/collections/3178572/", - "#category": ("", "unsplash", "collection"), - "#class" : unsplash.UnsplashCollectionExtractor, -}, - -{ - "#url" : "https://unsplash.com/collections/_8qJQ2bCMWE/2021.05", - "#category": ("", "unsplash", "collection"), - "#class" : unsplash.UnsplashCollectionExtractor, -}, - -{ - "#url" : "https://unsplash.com/s/photos/hair-style", - "#category": ("", "unsplash", "search"), - "#class" : unsplash.UnsplashSearchExtractor, - "#pattern" : r"https://(images|plus)\.unsplash\.com/((flagged/|premium_)?photo-\d+-\w+|reserve/[^/?#]+)\?ixid=\w+&ixlib=rb-4\.0\.3$", - "#range" : "1-30", - "#count" : 30, -}, - ) diff --git a/test/results/uploadir.py b/test/results/uploadir.py index 18d6a204e1..e0767d8ac7 100644 --- a/test/results/uploadir.py +++ b/test/results/uploadir.py @@ -1,62 +1,51 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import uploadir - __tests__ = ( -{ - "#url" : "https://uploadir.com/u/rd3t46ry", - "#comment" : "image", - "#category": ("", "uploadir", "file"), - "#class" : uploadir.UploadirFileExtractor, - "#pattern" : r"https://uploadir\.com/u/rd3t46ry", - "#count" : 1, - - "extension": "jpg", - "filename" : "Chloe and Rachel 4K jpg", - "id" : "rd3t46ry", -}, - -{ - "#url" : "https://uploadir.com/uploads/gxe8ti9v/downloads/new", - "#comment" : "archive", - "#category": ("", "uploadir", "file"), - "#class" : uploadir.UploadirFileExtractor, - "#pattern" : r"https://uploadir\.com/uploads/gxe8ti9v/downloads", - "#count" : 1, - - "extension": "zip", - "filename" : "NYAN-Mods-Pack#1", - "id" : "gxe8ti9v", -}, - -{ - "#url" : "https://uploadir.com/u/fllda6xl", - "#comment" : "utf-8 filename", - "#category": ("", "uploadir", "file"), - "#class" : uploadir.UploadirFileExtractor, - "#pattern" : r"https://uploadir\.com/u/fllda6xl", - "#count" : 1, - - "extension": "png", - "filename" : "_圖片_🖼_image_", - "id" : "fllda6xl", -}, - -{ - "#url" : "https://uploadir.com/uploads/rd3t46ry", - "#category": ("", "uploadir", "file"), - "#class" : uploadir.UploadirFileExtractor, -}, - -{ - "#url" : "https://uploadir.com/user/uploads/rd3t46ry", - "#category": ("", "uploadir", "file"), - "#class" : uploadir.UploadirFileExtractor, -}, - + { + "#url": "https://uploadir.com/u/rd3t46ry", + "#comment": "image", + "#category": ("", "uploadir", "file"), + "#class": uploadir.UploadirFileExtractor, + "#pattern": r"https://uploadir\.com/u/rd3t46ry", + "#count": 1, + "extension": "jpg", + "filename": "Chloe and Rachel 4K jpg", + "id": "rd3t46ry", + }, + { + "#url": "https://uploadir.com/uploads/gxe8ti9v/downloads/new", + "#comment": "archive", + "#category": ("", "uploadir", "file"), + "#class": uploadir.UploadirFileExtractor, + "#pattern": r"https://uploadir\.com/uploads/gxe8ti9v/downloads", + "#count": 1, + "extension": "zip", + "filename": "NYAN-Mods-Pack#1", + "id": "gxe8ti9v", + }, + { + "#url": "https://uploadir.com/u/fllda6xl", + "#comment": "utf-8 filename", + "#category": ("", "uploadir", "file"), + "#class": uploadir.UploadirFileExtractor, + "#pattern": r"https://uploadir\.com/u/fllda6xl", + "#count": 1, + "extension": "png", + "filename": "_圖片_🖼_image_", + "id": "fllda6xl", + }, + { + "#url": "https://uploadir.com/uploads/rd3t46ry", + "#category": ("", "uploadir", "file"), + "#class": uploadir.UploadirFileExtractor, + }, + { + "#url": "https://uploadir.com/user/uploads/rd3t46ry", + "#category": ("", "uploadir", "file"), + "#class": uploadir.UploadirFileExtractor, + }, ) diff --git a/test/results/urlgalleries.py b/test/results/urlgalleries.py index 88a321e769..8811e52c66 100644 --- a/test/results/urlgalleries.py +++ b/test/results/urlgalleries.py @@ -1,49 +1,42 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import urlgalleries - __tests__ = ( -{ - "#url" : "https://photos2q.urlgalleries.net/porn-gallery-7851311/clarice-window-8", - "#category": ("", "urlgalleries", "gallery"), - "#class" : urlgalleries.UrlgalleriesGalleryExtractor, - "#range" : "1-3", - "#urls" : ( - "https://fappic.com/x207mqkn2463/4gq1yv.jpg", - "https://fappic.com/q684ua2rp0j9/4gq1xv.jpg", - "https://fappic.com/8vf3n8fgz9po/4gq1ya.jpg", - ), - - "blog" : "photos2q", - "count" : 39, - "date" : "dt:2023-12-08 13:59:00", - "gallery_id": "7851311", - "num" : range(1, 3), - "title" : "Clarice window 8", -}, - -{ - "#url" : "https://dreamer.urlgalleries.net/7645840", - "#category": ("", "urlgalleries", "gallery"), - "#class" : urlgalleries.UrlgalleriesGalleryExtractor, - "#range" : "1-3", - "#urls" : ( - "https://www.fappic.com/vj7up04ny487/AmourAngels-0001.jpg", - "https://www.fappic.com/zfgsmpm36iyv/AmourAngels-0002.jpg", - "https://www.fappic.com/rqpt37rdbwa5/AmourAngels-0003.jpg", - ), - - "blog" : "Dreamer", - "count" : 105, - "date" : "dt:2020-03-10 21:17:00", - "gallery_id": "7645840", - "num" : range(1, 3), - "title" : "Angelika - Rustic Charm - AmourAngels 2016-09-27", -}, - + { + "#url": "https://photos2q.urlgalleries.net/porn-gallery-7851311/clarice-window-8", + "#category": ("", "urlgalleries", "gallery"), + "#class": urlgalleries.UrlgalleriesGalleryExtractor, + "#range": "1-3", + "#urls": ( + "https://fappic.com/x207mqkn2463/4gq1yv.jpg", + "https://fappic.com/q684ua2rp0j9/4gq1xv.jpg", + "https://fappic.com/8vf3n8fgz9po/4gq1ya.jpg", + ), + "blog": "photos2q", + "count": 39, + "date": "dt:2023-12-08 13:59:00", + "gallery_id": "7851311", + "num": range(1, 3), + "title": "Clarice window 8", + }, + { + "#url": "https://dreamer.urlgalleries.net/7645840", + "#category": ("", "urlgalleries", "gallery"), + "#class": urlgalleries.UrlgalleriesGalleryExtractor, + "#range": "1-3", + "#urls": ( + "https://www.fappic.com/vj7up04ny487/AmourAngels-0001.jpg", + "https://www.fappic.com/zfgsmpm36iyv/AmourAngels-0002.jpg", + "https://www.fappic.com/rqpt37rdbwa5/AmourAngels-0003.jpg", + ), + "blog": "Dreamer", + "count": 105, + "date": "dt:2020-03-10 21:17:00", + "gallery_id": "7645840", + "num": range(1, 3), + "title": "Angelika - Rustic Charm - AmourAngels 2016-09-27", + }, ) diff --git a/test/results/vanillarock.py b/test/results/vanillarock.py index f0cf3e1c31..b1cb674777 100644 --- a/test/results/vanillarock.py +++ b/test/results/vanillarock.py @@ -1,35 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import vanillarock - __tests__ = ( -{ - "#url" : "https://vanilla-rock.com/mizuhashi_parsee-5", - "#category": ("", "vanillarock", "post"), - "#class" : vanillarock.VanillarockPostExtractor, - "#sha1_url" : "7fb9a4d18d9fa22d7295fee8d94ab5a7a52265dd", - "#sha1_metadata": "b91df99b714e1958d9636748b1c81a07c3ef52c9", -}, - -{ - "#url" : "https://vanilla-rock.com/tag/%e5%b0%84%e5%91%bd%e4%b8%b8%e6%96%87", - "#category": ("", "vanillarock", "tag"), - "#class" : vanillarock.VanillarockTagExtractor, - "#pattern" : vanillarock.VanillarockPostExtractor.pattern, - "#count" : ">= 12", -}, - -{ - "#url" : "https://vanilla-rock.com/category/%e4%ba%8c%e6%ac%a1%e3%82%a8%e3%83%ad%e7%94%bb%e5%83%8f/%e8%90%8c%e3%81%88%e3%83%bb%e3%82%bd%e3%83%95%e3%83%88%e3%82%a8%e3%83%ad", - "#category": ("", "vanillarock", "tag"), - "#class" : vanillarock.VanillarockTagExtractor, - "#pattern" : vanillarock.VanillarockPostExtractor.pattern, - "#count" : ">= 5", -}, - + { + "#url": "https://vanilla-rock.com/mizuhashi_parsee-5", + "#category": ("", "vanillarock", "post"), + "#class": vanillarock.VanillarockPostExtractor, + "#sha1_url": "7fb9a4d18d9fa22d7295fee8d94ab5a7a52265dd", + "#sha1_metadata": "b91df99b714e1958d9636748b1c81a07c3ef52c9", + }, + { + "#url": "https://vanilla-rock.com/tag/%e5%b0%84%e5%91%bd%e4%b8%b8%e6%96%87", + "#category": ("", "vanillarock", "tag"), + "#class": vanillarock.VanillarockTagExtractor, + "#pattern": vanillarock.VanillarockPostExtractor.pattern, + "#count": ">= 12", + }, + { + "#url": "https://vanilla-rock.com/category/%e4%ba%8c%e6%ac%a1%e3%82%a8%e3%83%ad%e7%94%bb%e5%83%8f/%e8%90%8c%e3%81%88%e3%83%bb%e3%82%bd%e3%83%95%e3%83%88%e3%82%a8%e3%83%ad", + "#category": ("", "vanillarock", "tag"), + "#class": vanillarock.VanillarockTagExtractor, + "#pattern": vanillarock.VanillarockPostExtractor.pattern, + "#count": ">= 5", + }, ) diff --git a/test/results/vidyapics.py b/test/results/vidyapics.py index bb81760a96..af0c64ad99 100644 --- a/test/results/vidyapics.py +++ b/test/results/vidyapics.py @@ -1,38 +1,32 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shimmie2 - __tests__ = ( -{ - "#url" : "https://vidya.pics/post/list/kirby/1", - "#category": ("shimmie2", "vidyapics", "tag"), - "#class" : shimmie2.Shimmie2TagExtractor, - "#pattern" : r"https://vidya.pics/_images/[0-9a-f]{32}/\d+", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://vidya.pics/post/view/108820", - "#category": ("shimmie2", "vidyapics", "post"), - "#class" : shimmie2.Shimmie2PostExtractor, - "#pattern" : r"https://vidya\.pics/_images/277ecdb90285bfa6e0c4cd46d9515b11/108820.+\.png", - "#sha1_content": "7d2fe9327759c231ff17f6e341df749b70b191ce", - - "extension": "png", - "file_url" : "https://vidya.pics/_images/277ecdb90285bfa6e0c4cd46d9515b11/108820%20-%201boy%20artist%3Aunknown%20flag%20kirby%20kirby_%28series%29.png", - "filename" : "108820 - 1boy artist:unknown flag kirby kirby_(series)", - "height" : 700, - "id" : 108820, - "md5" : "277ecdb90285bfa6e0c4cd46d9515b11", - "size" : 0, - "tags" : "1boy artist:unknown flag kirby kirby_(series", - "width" : 700, -}, - + { + "#url": "https://vidya.pics/post/list/kirby/1", + "#category": ("shimmie2", "vidyapics", "tag"), + "#class": shimmie2.Shimmie2TagExtractor, + "#pattern": r"https://vidya.pics/_images/[0-9a-f]{32}/\d+", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://vidya.pics/post/view/108820", + "#category": ("shimmie2", "vidyapics", "post"), + "#class": shimmie2.Shimmie2PostExtractor, + "#pattern": r"https://vidya\.pics/_images/277ecdb90285bfa6e0c4cd46d9515b11/108820.+\.png", + "#sha1_content": "7d2fe9327759c231ff17f6e341df749b70b191ce", + "extension": "png", + "file_url": "https://vidya.pics/_images/277ecdb90285bfa6e0c4cd46d9515b11/108820%20-%201boy%20artist%3Aunknown%20flag%20kirby%20kirby_%28series%29.png", + "filename": "108820 - 1boy artist:unknown flag kirby kirby_(series)", + "height": 700, + "id": 108820, + "md5": "277ecdb90285bfa6e0c4cd46d9515b11", + "size": 0, + "tags": "1boy artist:unknown flag kirby kirby_(series", + "width": 700, + }, ) diff --git a/test/results/vidyart2.py b/test/results/vidyart2.py index 6c11dcbd5e..4ae419edcc 100644 --- a/test/results/vidyart2.py +++ b/test/results/vidyart2.py @@ -1,29 +1,23 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v01 - __tests__ = ( -{ - "#url" : "https://vidyart2.booru.org/index.php?page=post&s=list&tags=all", - "#category": ("gelbooru_v01", "vidyart2", "tag"), - "#class" : gelbooru_v01.GelbooruV01TagExtractor, -}, - -{ - "#url" : "https://vidyart2.booru.org/index.php?page=favorites&s=view&id=1", - "#category": ("gelbooru_v01", "vidyart2", "favorite"), - "#class" : gelbooru_v01.GelbooruV01FavoriteExtractor, -}, - -{ - "#url" : "https://vidyart2.booru.org/index.php?page=post&s=view&id=39168", - "#category": ("gelbooru_v01", "vidyart2", "post"), - "#class" : gelbooru_v01.GelbooruV01PostExtractor, -}, - + { + "#url": "https://vidyart2.booru.org/index.php?page=post&s=list&tags=all", + "#category": ("gelbooru_v01", "vidyart2", "tag"), + "#class": gelbooru_v01.GelbooruV01TagExtractor, + }, + { + "#url": "https://vidyart2.booru.org/index.php?page=favorites&s=view&id=1", + "#category": ("gelbooru_v01", "vidyart2", "favorite"), + "#class": gelbooru_v01.GelbooruV01FavoriteExtractor, + }, + { + "#url": "https://vidyart2.booru.org/index.php?page=post&s=view&id=39168", + "#category": ("gelbooru_v01", "vidyart2", "post"), + "#class": gelbooru_v01.GelbooruV01PostExtractor, + }, ) diff --git a/test/results/vipergirls.py b/test/results/vipergirls.py index cd6adac0f2..95a679f384 100644 --- a/test/results/vipergirls.py +++ b/test/results/vipergirls.py @@ -1,59 +1,49 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import vipergirls - __tests__ = ( -{ - "#url" : "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000", - "#category": ("", "vipergirls", "thread"), - "#class" : vipergirls.VipergirlsThreadExtractor, - "#count" : 225, - "#sha1_url": "0d75cb42777f5bebc0d284d1d38cb90c750c61d9", -}, - -{ - "#url" : "https://vipergirls.to/threads/6858916-Karina/page4", - "#category": ("", "vipergirls", "thread"), - "#class" : vipergirls.VipergirlsThreadExtractor, - "#count" : 1279, -}, - -{ - "#url" : "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000?highlight=foobar", - "#category": ("", "vipergirls", "thread"), - "#class" : vipergirls.VipergirlsThreadExtractor, -}, - -{ - "#url" : "https://vipergirls.to/threads/4328304?foo=bar", - "#category": ("", "vipergirls", "thread"), - "#class" : vipergirls.VipergirlsThreadExtractor, -}, - -{ - "#url" : "https://vipergirls.to/threads/4328304", - "#category": ("", "vipergirls", "thread"), - "#class" : vipergirls.VipergirlsThreadExtractor, -}, - -{ - "#url" : "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000?p=116038081&viewfull=1#post116038081", - "#category": ("", "vipergirls", "post"), - "#class" : vipergirls.VipergirlsPostExtractor, - "#pattern" : r"https://vipr\.im/\w{12}$", - "#range" : "2-113", - "#count" : 112, - - "id" : "116038081", - "imagecount": "113", - "number" : "116038081", - "thread_id" : "4328304", - "title" : "FemJoy Danica - Simply Beautiful (x112) 3000x4500", -}, - + { + "#url": "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000", + "#category": ("", "vipergirls", "thread"), + "#class": vipergirls.VipergirlsThreadExtractor, + "#count": 225, + "#sha1_url": "0d75cb42777f5bebc0d284d1d38cb90c750c61d9", + }, + { + "#url": "https://vipergirls.to/threads/6858916-Karina/page4", + "#category": ("", "vipergirls", "thread"), + "#class": vipergirls.VipergirlsThreadExtractor, + "#count": 1279, + }, + { + "#url": "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000?highlight=foobar", + "#category": ("", "vipergirls", "thread"), + "#class": vipergirls.VipergirlsThreadExtractor, + }, + { + "#url": "https://vipergirls.to/threads/4328304?foo=bar", + "#category": ("", "vipergirls", "thread"), + "#class": vipergirls.VipergirlsThreadExtractor, + }, + { + "#url": "https://vipergirls.to/threads/4328304", + "#category": ("", "vipergirls", "thread"), + "#class": vipergirls.VipergirlsThreadExtractor, + }, + { + "#url": "https://vipergirls.to/threads/4328304-2011-05-28-Danica-Simply-Beautiful-x112-4500x3000?p=116038081&viewfull=1#post116038081", + "#category": ("", "vipergirls", "post"), + "#class": vipergirls.VipergirlsPostExtractor, + "#pattern": r"https://vipr\.im/\w{12}$", + "#range": "2-113", + "#count": 112, + "id": "116038081", + "imagecount": "113", + "number": "116038081", + "thread_id": "4328304", + "title": "FemJoy Danica - Simply Beautiful (x112) 3000x4500", + }, ) diff --git a/test/results/vipr.py b/test/results/vipr.py index f5e00269fd..535eb1e18b 100644 --- a/test/results/vipr.py +++ b/test/results/vipr.py @@ -1,19 +1,15 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import imagehosts - __tests__ = ( -{ - "#url" : "https://vipr.im/kcd5jcuhgs3v.html", - "#category": ("imagehost", "vipr", "image"), - "#class" : imagehosts.ViprImageExtractor, - "#sha1_url" : "88f6a3ecbf3356a11ae0868b518c60800e070202", - "#sha1_metadata": "c432e8a1836b0d97045195b745731c2b1bb0e771", -}, - + { + "#url": "https://vipr.im/kcd5jcuhgs3v.html", + "#category": ("imagehost", "vipr", "image"), + "#class": imagehosts.ViprImageExtractor, + "#sha1_url": "88f6a3ecbf3356a11ae0868b518c60800e070202", + "#sha1_metadata": "c432e8a1836b0d97045195b745731c2b1bb0e771", + }, ) diff --git a/test/results/vk.py b/test/results/vk.py index b681bdf950..4cc7442a71 100644 --- a/test/results/vk.py +++ b/test/results/vk.py @@ -1,100 +1,85 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import vk from gallery_dl import exception - +from gallery_dl.extractor import vk __tests__ = ( -{ - "#url" : "https://vk.com/id398982326", - "#category": ("", "vk", "photos"), - "#class" : vk.VkPhotosExtractor, - "#pattern" : r"https://sun\d+-\d+\.userapi\.com/s/v1/if1/[\w-]+\.jpg\?size=\d+x\d+&quality=96&type=album", - "#count" : ">= 35", - - "id" : r"re:\d+", - "user": { - "id" : "398982326", - "info": "Мы за Движуху! – m1ni SounD #4 [EROmusic]", - "name": "", - "nick": "Dobrov Kurva", + { + "#url": "https://vk.com/id398982326", + "#category": ("", "vk", "photos"), + "#class": vk.VkPhotosExtractor, + "#pattern": r"https://sun\d+-\d+\.userapi\.com/s/v1/if1/[\w-]+\.jpg\?size=\d+x\d+&quality=96&type=album", + "#count": ">= 35", + "id": r"re:\d+", + "user": { + "id": "398982326", + "info": "Мы за Движуху! – m1ni SounD #4 [EROmusic]", + "name": "", + "nick": "Dobrov Kurva", + }, }, -}, - -{ - "#url" : "https://vk.com/cosplayinrussia", - "#category": ("", "vk", "photos"), - "#class" : vk.VkPhotosExtractor, - "#range" : "15-25", - - "id" : r"re:\d+", - "user": { - "id" : "-165740836", - "info": str, - "name": "cosplayinrussia", - "nick": "Косплей | Cosplay 18+", + { + "#url": "https://vk.com/cosplayinrussia", + "#category": ("", "vk", "photos"), + "#class": vk.VkPhotosExtractor, + "#range": "15-25", + "id": r"re:\d+", + "user": { + "id": "-165740836", + "info": str, + "name": "cosplayinrussia", + "nick": "Косплей | Cosplay 18+", + }, + }, + { + "#url": "https://vk.com/id76957806", + "#comment": "photos without width/height (#2535)", + "#category": ("", "vk", "photos"), + "#class": vk.VkPhotosExtractor, + "#pattern": r"https://sun\d+-\d+\.userapi\.com/", + "#range": "1-9", + "#count": 9, + }, + { + "#url": "https://m.vk.com/albums398982326", + "#category": ("", "vk", "photos"), + "#class": vk.VkPhotosExtractor, + }, + { + "#url": "https://www.vk.com/id398982326?profile=1", + "#category": ("", "vk", "photos"), + "#class": vk.VkPhotosExtractor, + }, + { + "#url": "https://vk.com/albums-165740836", + "#category": ("", "vk", "photos"), + "#class": vk.VkPhotosExtractor, + }, + { + "#url": "https://vk.com/album-165740836_281339889", + "#category": ("", "vk", "album"), + "#class": vk.VkAlbumExtractor, + "#count": 12, + }, + { + "#url": "https://vk.com/album-53775183_00", + "#comment": "'Access denied' (#2556)", + "#category": ("", "vk", "album"), + "#class": vk.VkAlbumExtractor, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://vk.com/album232175027_00", + "#category": ("", "vk", "album"), + "#class": vk.VkAlbumExtractor, + "#exception": exception.AuthorizationError, + }, + { + "#url": "https://vk.com/tag304303884", + "#category": ("", "vk", "tagged"), + "#class": vk.VkTaggedExtractor, + "#count": 44, }, -}, - -{ - "#url" : "https://vk.com/id76957806", - "#comment" : "photos without width/height (#2535)", - "#category": ("", "vk", "photos"), - "#class" : vk.VkPhotosExtractor, - "#pattern" : r"https://sun\d+-\d+\.userapi\.com/", - "#range" : "1-9", - "#count" : 9, -}, - -{ - "#url" : "https://m.vk.com/albums398982326", - "#category": ("", "vk", "photos"), - "#class" : vk.VkPhotosExtractor, -}, - -{ - "#url" : "https://www.vk.com/id398982326?profile=1", - "#category": ("", "vk", "photos"), - "#class" : vk.VkPhotosExtractor, -}, - -{ - "#url" : "https://vk.com/albums-165740836", - "#category": ("", "vk", "photos"), - "#class" : vk.VkPhotosExtractor, -}, - -{ - "#url" : "https://vk.com/album-165740836_281339889", - "#category": ("", "vk", "album"), - "#class" : vk.VkAlbumExtractor, - "#count" : 12, -}, - -{ - "#url" : "https://vk.com/album-53775183_00", - "#comment" : "'Access denied' (#2556)", - "#category": ("", "vk", "album"), - "#class" : vk.VkAlbumExtractor, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://vk.com/album232175027_00", - "#category": ("", "vk", "album"), - "#class" : vk.VkAlbumExtractor, - "#exception": exception.AuthorizationError, -}, - -{ - "#url" : "https://vk.com/tag304303884", - "#category": ("", "vk", "tagged"), - "#class" : vk.VkTaggedExtractor, - "#count" : 44, -}, - ) diff --git a/test/results/vsco.py b/test/results/vsco.py index c0b32c42b8..ef0dbe56a9 100644 --- a/test/results/vsco.py +++ b/test/results/vsco.py @@ -1,111 +1,93 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import vsco - __tests__ = ( -{ - "#url" : "https://vsco.co/missuri", - "#category": ("", "vsco", "user"), - "#class" : vsco.VscoUserExtractor, - "#urls" : "https://vsco.co/missuri/gallery", -}, - -{ - "#url" : "https://vsco.co/missuri", - "#category": ("", "vsco", "user"), - "#class" : vsco.VscoUserExtractor, - "#options" : {"include": "all"}, - "#urls" : [ - "https://vsco.co/missuri/avatar", - "https://vsco.co/missuri/gallery", - "https://vsco.co/missuri/spaces", - "https://vsco.co/missuri/collection", - ], -}, - -{ - "#url" : "https://vsco.co/missuri/gallery", - "#category": ("", "vsco", "gallery"), - "#class" : vsco.VscoGalleryExtractor, - "#pattern" : r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w-]+\.\w+", - "#range" : "1-80", - "#count" : 80, -}, - -{ - "#url" : "https://vsco.co/missuri/images/1", - "#category": ("", "vsco", "gallery"), - "#class" : vsco.VscoGalleryExtractor, -}, - -{ - "#url" : "https://vsco.co/vsco/collection/1", - "#category": ("", "vsco", "collection"), - "#class" : vsco.VscoCollectionExtractor, - "#pattern" : r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w\s-]+\.\w+", - "#range" : "1-80", - "#count" : 80, -}, - -{ - "#url" : "https://vsco.co/spaces/6320a3e1e0338d1350b33fea", - "#category": ("", "vsco", "space"), - "#class" : vsco.VscoSpaceExtractor, - "#pattern" : r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w\s-]+\.\w+", - "#count" : range(100, 150), -}, - -{ - "#url" : "https://vsco.co/missuri/spaces", - "#category": ("", "vsco", "spaces"), - "#class" : vsco.VscoSpacesExtractor, - "#urls" : ( - "https://vsco.co/spaces/62e4934e6920440801d19f05", - ), -}, - -{ - "#url" : "https://vsco.co/vsco/avatar", - "#category": ("", "vsco", "avatar"), - "#class" : vsco.VscoAvatarExtractor, - "#urls" : "https://image-aws-us-west-2.vsco.co/3c69ae/304128/652d9f3b39a6007526dda683/vscoprofile-avatar.jpg", - "#sha1_content" : "57cd648759e34a6daefc5c79542ddb4595b9b677", - - "id": "652d9f3b39a6007526dda683", -}, - -{ - "#url" : "https://vsco.co/erenyildiz/media/5d34b93ef632433030707ce2", - "#category": ("", "vsco", "image"), - "#class" : vsco.VscoImageExtractor, - "#sha1_url" : "a45f9712325b42742324b330c348b72477996031", - "#sha1_content": "1394d070828d82078035f19a92f404557b56b83f", - - "id" : "5d34b93ef632433030707ce2", - "user" : "erenyildiz", - "grid" : "erenyildiz", - "meta" : dict, - "tags" : list, - "date" : "dt:2019-07-21 19:12:11", - "video" : False, - "width" : 1537, - "height" : 1537, - "description": r"re:Ni seviyorum. #vsco #vscox #vscochallenges", -}, - -{ - "#url" : "https://vsco.co/jimenalazof/media/5b4feec558f6c45c18c040fd", - "#category": ("", "vsco", "image"), - "#class" : vsco.VscoImageExtractor, - "#sha1_url" : "08e7eef3301756ce81206c0b47c1e9373756a74a", - "#sha1_content": "e739f058d726ee42c51c180a505747972a7dfa47", - - "video": True, -}, - + { + "#url": "https://vsco.co/missuri", + "#category": ("", "vsco", "user"), + "#class": vsco.VscoUserExtractor, + "#urls": "https://vsco.co/missuri/gallery", + }, + { + "#url": "https://vsco.co/missuri", + "#category": ("", "vsco", "user"), + "#class": vsco.VscoUserExtractor, + "#options": {"include": "all"}, + "#urls": [ + "https://vsco.co/missuri/avatar", + "https://vsco.co/missuri/gallery", + "https://vsco.co/missuri/spaces", + "https://vsco.co/missuri/collection", + ], + }, + { + "#url": "https://vsco.co/missuri/gallery", + "#category": ("", "vsco", "gallery"), + "#class": vsco.VscoGalleryExtractor, + "#pattern": r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w-]+\.\w+", + "#range": "1-80", + "#count": 80, + }, + { + "#url": "https://vsco.co/missuri/images/1", + "#category": ("", "vsco", "gallery"), + "#class": vsco.VscoGalleryExtractor, + }, + { + "#url": "https://vsco.co/vsco/collection/1", + "#category": ("", "vsco", "collection"), + "#class": vsco.VscoCollectionExtractor, + "#pattern": r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w\s-]+\.\w+", + "#range": "1-80", + "#count": 80, + }, + { + "#url": "https://vsco.co/spaces/6320a3e1e0338d1350b33fea", + "#category": ("", "vsco", "space"), + "#class": vsco.VscoSpaceExtractor, + "#pattern": r"https://image(-aws.+)?\.vsco\.co/[0-9a-f/]+/[\w\s-]+\.\w+", + "#count": range(100, 150), + }, + { + "#url": "https://vsco.co/missuri/spaces", + "#category": ("", "vsco", "spaces"), + "#class": vsco.VscoSpacesExtractor, + "#urls": ("https://vsco.co/spaces/62e4934e6920440801d19f05",), + }, + { + "#url": "https://vsco.co/vsco/avatar", + "#category": ("", "vsco", "avatar"), + "#class": vsco.VscoAvatarExtractor, + "#urls": "https://image-aws-us-west-2.vsco.co/3c69ae/304128/652d9f3b39a6007526dda683/vscoprofile-avatar.jpg", + "#sha1_content": "57cd648759e34a6daefc5c79542ddb4595b9b677", + "id": "652d9f3b39a6007526dda683", + }, + { + "#url": "https://vsco.co/erenyildiz/media/5d34b93ef632433030707ce2", + "#category": ("", "vsco", "image"), + "#class": vsco.VscoImageExtractor, + "#sha1_url": "a45f9712325b42742324b330c348b72477996031", + "#sha1_content": "1394d070828d82078035f19a92f404557b56b83f", + "id": "5d34b93ef632433030707ce2", + "user": "erenyildiz", + "grid": "erenyildiz", + "meta": dict, + "tags": list, + "date": "dt:2019-07-21 19:12:11", + "video": False, + "width": 1537, + "height": 1537, + "description": r"re:Ni seviyorum. #vsco #vscox #vscochallenges", + }, + { + "#url": "https://vsco.co/jimenalazof/media/5b4feec558f6c45c18c040fd", + "#category": ("", "vsco", "image"), + "#class": vsco.VscoImageExtractor, + "#sha1_url": "08e7eef3301756ce81206c0b47c1e9373756a74a", + "#sha1_content": "e739f058d726ee42c51c180a505747972a7dfa47", + "video": True, + }, ) diff --git a/test/results/wallhaven.py b/test/results/wallhaven.py index 47a8ba777c..deb87fe271 100644 --- a/test/results/wallhaven.py +++ b/test/results/wallhaven.py @@ -1,104 +1,90 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wallhaven - __tests__ = ( -{ - "#url" : "https://wallhaven.cc/search?q=touhou", - "#category": ("", "wallhaven", "search"), - "#class" : wallhaven.WallhavenSearchExtractor, -}, - -{ - "#url" : "https://wallhaven.cc/search?q=id%3A87&categories=111&purity=100&sorting=date_added&order=asc&page=3", - "#category": ("", "wallhaven", "search"), - "#class" : wallhaven.WallhavenSearchExtractor, - "#pattern" : r"https://w\.wallhaven\.cc/full/\w\w/wallhaven-\w+\.\w+", - "#count" : "<= 30", -}, - -{ - "#url" : "https://wallhaven.cc/user/AksumkA/favorites/74", - "#category": ("", "wallhaven", "collection"), - "#class" : wallhaven.WallhavenCollectionExtractor, - "#count" : ">= 50", -}, - -{ - "#url" : "https://wallhaven.cc/user/AksumkA/", - "#category": ("", "wallhaven", "user"), - "#class" : wallhaven.WallhavenUserExtractor, -}, - -{ - "#url" : "https://wallhaven.cc/user/AksumkA/favorites", - "#category": ("", "wallhaven", "collections"), - "#class" : wallhaven.WallhavenCollectionsExtractor, - "#pattern" : wallhaven.WallhavenCollectionExtractor.pattern, - "#count" : 4, -}, - -{ - "#url" : "https://wallhaven.cc/user/AksumkA/uploads", - "#category": ("", "wallhaven", "uploads"), - "#class" : wallhaven.WallhavenUploadsExtractor, - "#pattern" : r"https://[^.]+\.wallhaven\.cc/full/\w\w/wallhaven-\w+\.\w+", - "#range" : "1-100", - "#count" : 100, -}, - -{ - "#url" : "https://wallhaven.cc/w/01w334", - "#category": ("", "wallhaven", "image"), - "#class" : wallhaven.WallhavenImageExtractor, - "#pattern" : r"https://[^.]+\.wallhaven\.cc/full/01/wallhaven-01w334\.jpg", - "#sha1_content": "497212679383a465da1e35bd75873240435085a2", - - "id" : "01w334", - "width" : 1920, - "height" : 1200, - "resolution" : "1920x1200", - "ratio" : "1.6", - "colors" : list, - "tags" : list, - "file_size" : 278799, - "file_type" : "image/jpeg", - "purity" : "sfw", - "short_url" : "https://whvn.cc/01w334", - "source" : str, - "uploader" : { - "group" : "Owner/Developer", - "username": "AksumkA", + { + "#url": "https://wallhaven.cc/search?q=touhou", + "#category": ("", "wallhaven", "search"), + "#class": wallhaven.WallhavenSearchExtractor, + }, + { + "#url": "https://wallhaven.cc/search?q=id%3A87&categories=111&purity=100&sorting=date_added&order=asc&page=3", + "#category": ("", "wallhaven", "search"), + "#class": wallhaven.WallhavenSearchExtractor, + "#pattern": r"https://w\.wallhaven\.cc/full/\w\w/wallhaven-\w+\.\w+", + "#count": "<= 30", + }, + { + "#url": "https://wallhaven.cc/user/AksumkA/favorites/74", + "#category": ("", "wallhaven", "collection"), + "#class": wallhaven.WallhavenCollectionExtractor, + "#count": ">= 50", + }, + { + "#url": "https://wallhaven.cc/user/AksumkA/", + "#category": ("", "wallhaven", "user"), + "#class": wallhaven.WallhavenUserExtractor, + }, + { + "#url": "https://wallhaven.cc/user/AksumkA/favorites", + "#category": ("", "wallhaven", "collections"), + "#class": wallhaven.WallhavenCollectionsExtractor, + "#pattern": wallhaven.WallhavenCollectionExtractor.pattern, + "#count": 4, + }, + { + "#url": "https://wallhaven.cc/user/AksumkA/uploads", + "#category": ("", "wallhaven", "uploads"), + "#class": wallhaven.WallhavenUploadsExtractor, + "#pattern": r"https://[^.]+\.wallhaven\.cc/full/\w\w/wallhaven-\w+\.\w+", + "#range": "1-100", + "#count": 100, + }, + { + "#url": "https://wallhaven.cc/w/01w334", + "#category": ("", "wallhaven", "image"), + "#class": wallhaven.WallhavenImageExtractor, + "#pattern": r"https://[^.]+\.wallhaven\.cc/full/01/wallhaven-01w334\.jpg", + "#sha1_content": "497212679383a465da1e35bd75873240435085a2", + "id": "01w334", + "width": 1920, + "height": 1200, + "resolution": "1920x1200", + "ratio": "1.6", + "colors": list, + "tags": list, + "file_size": 278799, + "file_type": "image/jpeg", + "purity": "sfw", + "short_url": "https://whvn.cc/01w334", + "source": str, + "uploader": { + "group": "Owner/Developer", + "username": "AksumkA", + }, + "date": "dt:2014-08-31 06:17:19", + "wh_category": "anime", + "views": int, + "favorites": int, + }, + { + "#url": "https://wallhaven.cc/w/dge6v3", + "#comment": "NSFW", + "#category": ("", "wallhaven", "image"), + "#class": wallhaven.WallhavenImageExtractor, + "#sha1_url": "e4b802e70483f659d790ad5d0bd316245badf2ec", + }, + { + "#url": "https://whvn.cc/01w334", + "#category": ("", "wallhaven", "image"), + "#class": wallhaven.WallhavenImageExtractor, + }, + { + "#url": "https://w.wallhaven.cc/full/01/wallhaven-01w334.jpg", + "#category": ("", "wallhaven", "image"), + "#class": wallhaven.WallhavenImageExtractor, }, - "date" : "dt:2014-08-31 06:17:19", - "wh_category": "anime", - "views" : int, - "favorites" : int, -}, - -{ - "#url" : "https://wallhaven.cc/w/dge6v3", - "#comment" : "NSFW", - "#category": ("", "wallhaven", "image"), - "#class" : wallhaven.WallhavenImageExtractor, - "#sha1_url": "e4b802e70483f659d790ad5d0bd316245badf2ec", -}, - -{ - "#url" : "https://whvn.cc/01w334", - "#category": ("", "wallhaven", "image"), - "#class" : wallhaven.WallhavenImageExtractor, -}, - -{ - "#url" : "https://w.wallhaven.cc/full/01/wallhaven-01w334.jpg", - "#category": ("", "wallhaven", "image"), - "#class" : wallhaven.WallhavenImageExtractor, -}, - ) diff --git a/test/results/wallpapercave.py b/test/results/wallpapercave.py index 80ade86d54..e14c232bc3 100644 --- a/test/results/wallpapercave.py +++ b/test/results/wallpapercave.py @@ -1,33 +1,28 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wallpapercave - __tests__ = ( -{ - "#url" : "https://wallpapercave.com/w/wp10270355", - "#category": ("", "wallpapercave", "image"), - "#class" : wallpapercave.WallpapercaveImageExtractor, - "#urls" : "https://wallpapercave.com/download/sekai-saikou-no-ansatsusha-isekai-kizoku-ni-tensei-suru-wallpapers-wp10270355", - "#sha1_content": "58b088aaa1cf1a60e347015019eb0c5a22b263a6", -}, - -{ - "#url" : "https://wallpapercave.com/apple-wwdc-2024-wallpapers", - "#comment" : "album listing", - "#category": ("", "wallpapercave", "image"), - "#class" : wallpapercave.WallpapercaveImageExtractor, - "#archive" : False, - "#urls" : [ - "https://wallpapercave.com/wp/wp13775438.jpg", - "https://wallpapercave.com/wp/wp13775439.jpg", - "https://wallpapercave.com/wp/wp13775440.jpg", - "https://wallpapercave.com/wp/wp13775441.jpg", - ], -}, - + { + "#url": "https://wallpapercave.com/w/wp10270355", + "#category": ("", "wallpapercave", "image"), + "#class": wallpapercave.WallpapercaveImageExtractor, + "#urls": "https://wallpapercave.com/download/sekai-saikou-no-ansatsusha-isekai-kizoku-ni-tensei-suru-wallpapers-wp10270355", + "#sha1_content": "58b088aaa1cf1a60e347015019eb0c5a22b263a6", + }, + { + "#url": "https://wallpapercave.com/apple-wwdc-2024-wallpapers", + "#comment": "album listing", + "#category": ("", "wallpapercave", "image"), + "#class": wallpapercave.WallpapercaveImageExtractor, + "#archive": False, + "#urls": [ + "https://wallpapercave.com/wp/wp13775438.jpg", + "https://wallpapercave.com/wp/wp13775439.jpg", + "https://wallpapercave.com/wp/wp13775440.jpg", + "https://wallpapercave.com/wp/wp13775441.jpg", + ], + }, ) diff --git a/test/results/warosu.py b/test/results/warosu.py index fd095183d4..72ff6b0bf6 100644 --- a/test/results/warosu.py +++ b/test/results/warosu.py @@ -1,94 +1,84 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import warosu - __tests__ = ( -{ - "#url" : "https://warosu.org/jp/thread/16656025", - "#category": ("", "warosu", "thread"), - "#class" : warosu.WarosuThreadExtractor, - "#urls" : ( - "https://i.warosu.org/data/jp/img/0166/56/1488487280004.png", - "https://i.warosu.org/data/jp/img/0166/56/1488493239417.png", - "https://i.warosu.org/data/jp/img/0166/56/1488493636725.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1488493700040.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1488499585168.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1488530851199.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1488536072155.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1488603426484.png", - "https://i.warosu.org/data/jp/img/0166/56/1488647021253.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1488866825031.jpg", - "https://i.warosu.org/data/jp/img/0166/56/1489094956868.jpg", - ), -}, - -{ - "#url" : "https://warosu.org/jp/thread/16658073", - "#category": ("", "warosu", "thread"), - "#class" : warosu.WarosuThreadExtractor, - "#sha1_content" : "d48df0a701e6599312bfff8674f4aa5d4fb8db1c", - "#urls" : "https://i.warosu.org/data/jp/img/0166/58/1488521824388.jpg", - "#count" : 1, - - "board" : "jp", - "board_name": "Otaku Culture", - "com" : "Is this canon?", - "ext" : ".jpg", - "extension" : "jpg", - "filename" : "sadako-vs-kayako-movie-review", - "fsize" : "55 KB", - "h" : 675, - "image" : "https://i.warosu.org/data/jp/img/0166/58/1488521824388.jpg", - "name" : "Anonymous", - "no" : 16658073, - "now" : "Fri Mar 3 01:17:04 2017", - "thread" : "16658073", - "tim" : 1488521824388, - "time" : 1488503824, - "title" : "Is this canon?", - "w" : 450, -}, - -{ - "#url" : "https://warosu.org/jp/thread/45886210", - "#comment" : "deleted post (#5289)", - "#category": ("", "warosu", "thread"), - "#class" : warosu.WarosuThreadExtractor, - "#count" : "> 150", - - "board" : "jp", - "board_name": "Otaku Culture", - "title" : "/07/th Expansion Thread", -}, - -{ - "#url" : "https://warosu.org/ic/thread/4604652", - "#category": ("", "warosu", "thread"), - "#class" : warosu.WarosuThreadExtractor, - "#pattern" : r"https://i.warosu\.org/data/ic/img/0046/04/1590\d{9}\.jpg", - "#count" : 133, - - "board" : "ic", - "board_name": "Artwork/Critique", - "com" : str, - "ext" : ".jpg", - "filename" : str, - "fsize" : str, - "h" : range(200, 3507), - "image" : r"re:https://i.warosu\.org/data/ic/img/0046/04/1590\d+\.jpg", - "name" : "re:Anonymous|Dhe Specky Spider-Man", - "no" : range(4604652, 4620000), - "now" : r"re:\w\w\w \w\w\w \d\d \d\d:\d\d:\d\d 2020", - "thread" : "4604652", - "tim" : range(1590430159651, 1590755510488), - "time" : range(1590415759, 1590755510), - "title" : "American Classic Comic Artists", - "w" : range(200, 3000), -}, - + { + "#url": "https://warosu.org/jp/thread/16656025", + "#category": ("", "warosu", "thread"), + "#class": warosu.WarosuThreadExtractor, + "#urls": ( + "https://i.warosu.org/data/jp/img/0166/56/1488487280004.png", + "https://i.warosu.org/data/jp/img/0166/56/1488493239417.png", + "https://i.warosu.org/data/jp/img/0166/56/1488493636725.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1488493700040.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1488499585168.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1488530851199.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1488536072155.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1488603426484.png", + "https://i.warosu.org/data/jp/img/0166/56/1488647021253.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1488866825031.jpg", + "https://i.warosu.org/data/jp/img/0166/56/1489094956868.jpg", + ), + }, + { + "#url": "https://warosu.org/jp/thread/16658073", + "#category": ("", "warosu", "thread"), + "#class": warosu.WarosuThreadExtractor, + "#sha1_content": "d48df0a701e6599312bfff8674f4aa5d4fb8db1c", + "#urls": "https://i.warosu.org/data/jp/img/0166/58/1488521824388.jpg", + "#count": 1, + "board": "jp", + "board_name": "Otaku Culture", + "com": "Is this canon?", + "ext": ".jpg", + "extension": "jpg", + "filename": "sadako-vs-kayako-movie-review", + "fsize": "55 KB", + "h": 675, + "image": "https://i.warosu.org/data/jp/img/0166/58/1488521824388.jpg", + "name": "Anonymous", + "no": 16658073, + "now": "Fri Mar 3 01:17:04 2017", + "thread": "16658073", + "tim": 1488521824388, + "time": 1488503824, + "title": "Is this canon?", + "w": 450, + }, + { + "#url": "https://warosu.org/jp/thread/45886210", + "#comment": "deleted post (#5289)", + "#category": ("", "warosu", "thread"), + "#class": warosu.WarosuThreadExtractor, + "#count": "> 150", + "board": "jp", + "board_name": "Otaku Culture", + "title": "/07/th Expansion Thread", + }, + { + "#url": "https://warosu.org/ic/thread/4604652", + "#category": ("", "warosu", "thread"), + "#class": warosu.WarosuThreadExtractor, + "#pattern": r"https://i.warosu\.org/data/ic/img/0046/04/1590\d{9}\.jpg", + "#count": 133, + "board": "ic", + "board_name": "Artwork/Critique", + "com": str, + "ext": ".jpg", + "filename": str, + "fsize": str, + "h": range(200, 3507), + "image": r"re:https://i.warosu\.org/data/ic/img/0046/04/1590\d+\.jpg", + "name": "re:Anonymous|Dhe Specky Spider-Man", + "no": range(4604652, 4620000), + "now": r"re:\w\w\w \w\w\w \d\d \d\d:\d\d:\d\d 2020", + "thread": "4604652", + "tim": range(1590430159651, 1590755510488), + "time": range(1590415759, 1590755510), + "title": "American Classic Comic Artists", + "w": range(200, 3000), + }, ) diff --git a/test/results/weasyl.py b/test/results/weasyl.py index 0c652644a5..dc8a932b03 100644 --- a/test/results/weasyl.py +++ b/test/results/weasyl.py @@ -1,102 +1,87 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import weasyl - __tests__ = ( -{ - "#url" : "https://www.weasyl.com/~fiz/submissions/2031/a-wesley", - "#category": ("", "weasyl", "submission"), - "#class" : weasyl.WeasylSubmissionExtractor, - "#pattern" : "https://cdn.weasyl.com/~fiz/submissions/2031/41ebc1c2940be928532785dfbf35c37622664d2fbb8114c3b063df969562fc51/fiz-a-wesley.png", - - "comments" : int, - "date" : "dt:2012-04-20 00:38:04", - "description" : """

          (flex)

          + { + "#url": "https://www.weasyl.com/~fiz/submissions/2031/a-wesley", + "#category": ("", "weasyl", "submission"), + "#class": weasyl.WeasylSubmissionExtractor, + "#pattern": "https://cdn.weasyl.com/~fiz/submissions/2031/41ebc1c2940be928532785dfbf35c37622664d2fbb8114c3b063df969562fc51/fiz-a-wesley.png", + "comments": int, + "date": "dt:2012-04-20 00:38:04", + "description": """

          (flex)

          """, - "favorites" : int, - "folder_name" : "Wesley Stuff", - "folderid" : 2081, - "friends_only": False, - "owner" : "Fiz", - "owner_login" : "fiz", - "rating" : "general", - "submitid" : 2031, - "subtype" : "visual", - "tags" : list, - "title" : "A Wesley!", - "type" : "submission", - "views" : int, -}, - -{ - "#url" : "https://www.weasyl.com/submission/2031/a-wesley", - "#category": ("", "weasyl", "submission"), - "#class" : weasyl.WeasylSubmissionExtractor, -}, - -{ - "#url" : "https://www.weasyl.com/~tanidareal", - "#category": ("", "weasyl", "submissions"), - "#class" : weasyl.WeasylSubmissionsExtractor, - "#count" : ">= 200", -}, - -{ - "#url" : "https://www.weasyl.com/submissions/tanidareal", - "#category": ("", "weasyl", "submissions"), - "#class" : weasyl.WeasylSubmissionsExtractor, -}, - -{ - "#url" : "https://www.weasyl.com/~aro~so", - "#category": ("", "weasyl", "submissions"), - "#class" : weasyl.WeasylSubmissionsExtractor, -}, - -{ - "#url" : "https://www.weasyl.com/submissions/tanidareal?folderid=7403", - "#category": ("", "weasyl", "folder"), - "#class" : weasyl.WeasylFolderExtractor, - "#count" : ">= 12", -}, - -{ - "#url" : "https://www.weasyl.com/journal/17647/bbcode", - "#category": ("", "weasyl", "journal"), - "#class" : weasyl.WeasylJournalExtractor, - - "title" : "BBCode", - "date" : "dt:2013-09-19 23:11:23", - "content": """

          javascript:alert(42);

          + "favorites": int, + "folder_name": "Wesley Stuff", + "folderid": 2081, + "friends_only": False, + "owner": "Fiz", + "owner_login": "fiz", + "rating": "general", + "submitid": 2031, + "subtype": "visual", + "tags": list, + "title": "A Wesley!", + "type": "submission", + "views": int, + }, + { + "#url": "https://www.weasyl.com/submission/2031/a-wesley", + "#category": ("", "weasyl", "submission"), + "#class": weasyl.WeasylSubmissionExtractor, + }, + { + "#url": "https://www.weasyl.com/~tanidareal", + "#category": ("", "weasyl", "submissions"), + "#class": weasyl.WeasylSubmissionsExtractor, + "#count": ">= 200", + }, + { + "#url": "https://www.weasyl.com/submissions/tanidareal", + "#category": ("", "weasyl", "submissions"), + "#class": weasyl.WeasylSubmissionsExtractor, + }, + { + "#url": "https://www.weasyl.com/~aro~so", + "#category": ("", "weasyl", "submissions"), + "#class": weasyl.WeasylSubmissionsExtractor, + }, + { + "#url": "https://www.weasyl.com/submissions/tanidareal?folderid=7403", + "#category": ("", "weasyl", "folder"), + "#class": weasyl.WeasylFolderExtractor, + "#count": ">= 12", + }, + { + "#url": "https://www.weasyl.com/journal/17647/bbcode", + "#category": ("", "weasyl", "journal"), + "#class": weasyl.WeasylJournalExtractor, + "title": "BBCode", + "date": "dt:2013-09-19 23:11:23", + "content": """

          javascript:alert(42);

          No more of that!

          """, -}, - -{ - "#url" : "https://www.weasyl.com/journals/charmander", - "#category": ("", "weasyl", "journals"), - "#class" : weasyl.WeasylJournalsExtractor, - "#count" : ">= 2", -}, - -{ - "#url" : "https://www.weasyl.com/favorites?userid=184616&feature=submit", - "#category": ("", "weasyl", "favorite"), - "#class" : weasyl.WeasylFavoriteExtractor, - "#count" : ">= 5", -}, - -{ - "#url" : "https://www.weasyl.com/favorites/furoferre", - "#category": ("", "weasyl", "favorite"), - "#class" : weasyl.WeasylFavoriteExtractor, - "#count" : ">= 5", -} - + }, + { + "#url": "https://www.weasyl.com/journals/charmander", + "#category": ("", "weasyl", "journals"), + "#class": weasyl.WeasylJournalsExtractor, + "#count": ">= 2", + }, + { + "#url": "https://www.weasyl.com/favorites?userid=184616&feature=submit", + "#category": ("", "weasyl", "favorite"), + "#class": weasyl.WeasylFavoriteExtractor, + "#count": ">= 5", + }, + { + "#url": "https://www.weasyl.com/favorites/furoferre", + "#category": ("", "weasyl", "favorite"), + "#class": weasyl.WeasylFavoriteExtractor, + "#count": ">= 5", + }, ) diff --git a/test/results/webmshare.py b/test/results/webmshare.py index 007a12b4b2..7710e5e67c 100644 --- a/test/results/webmshare.py +++ b/test/results/webmshare.py @@ -1,55 +1,46 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import webmshare - __tests__ = ( -{ - "#url" : "https://webmshare.com/O9mWY", - "#category": ("", "webmshare", "video"), - "#class" : webmshare.WebmshareVideoExtractor, - - "date" : "dt:2022-12-04 00:00:00", - "extension": "webm", - "filename" : "O9mWY", - "height" : 568, - "id" : "O9mWY", - "thumb" : "https://s1.webmshare.com/t/O9mWY.jpg", - "title" : "Yeah buddy over here", - "url" : "https://s1.webmshare.com/O9mWY.webm", - "views" : int, - "width" : 320, -}, - -{ - "#url" : "https://s1.webmshare.com/zBGAg.webm", - "#category": ("", "webmshare", "video"), - "#class" : webmshare.WebmshareVideoExtractor, - - "date" : "dt:2018-12-07 00:00:00", - "height": 1080, - "id" : "zBGAg", - "thumb" : "https://s1.webmshare.com/t/zBGAg.jpg", - "title" : "", - "url" : "https://s1.webmshare.com/zBGAg.webm", - "views" : int, - "width" : 1920, -}, - -{ - "#url" : "https://webmshare.com/play/zBGAg", - "#category": ("", "webmshare", "video"), - "#class" : webmshare.WebmshareVideoExtractor, -}, - -{ - "#url" : "https://webmshare.com/download-webm/zBGAg", - "#category": ("", "webmshare", "video"), - "#class" : webmshare.WebmshareVideoExtractor, -}, - + { + "#url": "https://webmshare.com/O9mWY", + "#category": ("", "webmshare", "video"), + "#class": webmshare.WebmshareVideoExtractor, + "date": "dt:2022-12-04 00:00:00", + "extension": "webm", + "filename": "O9mWY", + "height": 568, + "id": "O9mWY", + "thumb": "https://s1.webmshare.com/t/O9mWY.jpg", + "title": "Yeah buddy over here", + "url": "https://s1.webmshare.com/O9mWY.webm", + "views": int, + "width": 320, + }, + { + "#url": "https://s1.webmshare.com/zBGAg.webm", + "#category": ("", "webmshare", "video"), + "#class": webmshare.WebmshareVideoExtractor, + "date": "dt:2018-12-07 00:00:00", + "height": 1080, + "id": "zBGAg", + "thumb": "https://s1.webmshare.com/t/zBGAg.jpg", + "title": "", + "url": "https://s1.webmshare.com/zBGAg.webm", + "views": int, + "width": 1920, + }, + { + "#url": "https://webmshare.com/play/zBGAg", + "#category": ("", "webmshare", "video"), + "#class": webmshare.WebmshareVideoExtractor, + }, + { + "#url": "https://webmshare.com/download-webm/zBGAg", + "#category": ("", "webmshare", "video"), + "#class": webmshare.WebmshareVideoExtractor, + }, ) diff --git a/test/results/webtoons.py b/test/results/webtoons.py index 4574fd482d..769dcf9c0f 100644 --- a/test/results/webtoons.py +++ b/test/results/webtoons.py @@ -1,140 +1,120 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import webtoons from gallery_dl import exception - +from gallery_dl.extractor import webtoons __tests__ = ( -{ - "#url" : "https://www.webtoons.com/en/comedy/safely-endangered/ep-572-earth/viewer?title_no=352&episode_no=572", - "#category": ("", "webtoons", "episode"), - "#class" : webtoons.WebtoonsEpisodeExtractor, - "#count" : 5, - "#sha1_url" : "55bec5d7c42aba19e3d0d56db25fdf0b0b13be38", - "#sha1_content": [ - "1748c7e82b6db910fa179f6dc7c4281b0f680fa7", - "42055e44659f6ffc410b3fb6557346dfbb993df3", - "49e1f2def04c6f7a6a3dacf245a1cd9abe77a6a9", - ], - - "author_name" : "Chris McCoy", - "comic" : "safely-endangered", - "comic_name" : "Safely Endangered", - "count" : 5, - "description" : "Silly comics for silly people.", - "episode" : "572", - "episode_name": "Ep. 572 - Earth", - "episode_no" : "572", - "genre" : "comedy", - "lang" : "en", - "language" : "English", - "num" : range(1, 5), - "title" : "Safely Endangered - Ep. 572 - Earth", - "title_no" : "352", - "username" : "safelyendangered", -}, - -{ - "#url" : "https://www.webtoons.com/en/challenge/punderworld/happy-earth-day-/viewer?title_no=312584&episode_no=40", - "#category": ("", "webtoons", "episode"), - "#class" : webtoons.WebtoonsEpisodeExtractor, - "#exception": exception.NotFoundError, - - "comic" : "punderworld", - "description": str, - "episode" : "36", - "episode_no" : "40", - "genre" : "challenge", - "title" : r"re:^Punderworld - .+", - "title_no" : "312584", -}, - -{ - "#url" : "https://www.webtoons.com/en/canvas/i-want-to-be-a-cute-anime-girl/209-the-storys-story/viewer?title_no=349416&episode_no=214", - "#category": ("", "webtoons", "episode"), - "#class" : webtoons.WebtoonsEpisodeExtractor, - "#count" : 4, - - "comic_name" : "I want to be a cute anime girl", - "episode_name": "209 - The story's story", - "episode" : "214", - "username" : "m9huj", - "author_name" : "Azul Crescent", -}, - -{ - "#url" : "https://www.webtoons.com/en/canvas/i-want-to-be-a-cute-anime-girl/174-not-194-it-was-a-typo-later/viewer?title_no=349416&episode_no=179", - "#category": ("", "webtoons", "episode"), - "#class" : webtoons.WebtoonsEpisodeExtractor, - "#count" : 4, - - "comic_name" : "I want to be a cute anime girl", - "episode_name": "174 (not 194, it was a typo) - Later", - "episode" : "179", - "username" : "m9huj", - "author_name" : "Azul Crescent", -}, - -{ - "#url" : "https://www.webtoons.com/en/canvas/us-over-here/1-the-wheel/viewer?title_no=919536&episode_no=1", - "#category": ("", "webtoons", "episode"), - "#class" : webtoons.WebtoonsEpisodeExtractor, - "#count" : 60, - - "comic_name" : "Us, over here", - "episode_name": "1. The Wheel", - "episode" : "1", - "username" : "i94q8", - "author_name" : "spin.ani", -}, - -{ - "#url" : "https://www.webtoons.com/en/comedy/live-with-yourself/list?title_no=919", - "#comment" : "english", - "#category": ("", "webtoons", "comic"), - "#class" : webtoons.WebtoonsComicExtractor, - "#pattern" : webtoons.WebtoonsEpisodeExtractor.pattern, - "#range" : "1-15", - "#count" : ">= 14", - - "page" : range(1, 2), - "title_no" : 919, - "episode_no": range(1, 14), -}, - -{ - "#url" : "https://www.webtoons.com/fr/romance/subzero/list?title_no=1845&page=7", - "#comment" : "french", - "#category": ("", "webtoons", "comic"), - "#class" : webtoons.WebtoonsComicExtractor, - "#count" : ">= 15", - - "page" : range(7, 25), - "title_no" : 1845, - "episode_no": int, -}, - -{ - "#url" : "https://www.webtoons.com/en/challenge/scoob-and-shag/list?title_no=210827&page=9", - "#comment" : "(#820)", - "#category": ("", "webtoons", "comic"), - "#class" : webtoons.WebtoonsComicExtractor, - "#count" : ">= 18", - - "page" : int, - "title_no" : 210827, - "episode_no": int, -}, - -{ - "#url" : "https://www.webtoons.com/es/romance/lore-olympus/list?title_no=1725", - "#comment" : "(#1643)", - "#category": ("", "webtoons", "comic"), - "#class" : webtoons.WebtoonsComicExtractor, -}, - + { + "#url": "https://www.webtoons.com/en/comedy/safely-endangered/ep-572-earth/viewer?title_no=352&episode_no=572", + "#category": ("", "webtoons", "episode"), + "#class": webtoons.WebtoonsEpisodeExtractor, + "#count": 5, + "#sha1_url": "55bec5d7c42aba19e3d0d56db25fdf0b0b13be38", + "#sha1_content": [ + "1748c7e82b6db910fa179f6dc7c4281b0f680fa7", + "42055e44659f6ffc410b3fb6557346dfbb993df3", + "49e1f2def04c6f7a6a3dacf245a1cd9abe77a6a9", + ], + "author_name": "Chris McCoy", + "comic": "safely-endangered", + "comic_name": "Safely Endangered", + "count": 5, + "description": "Silly comics for silly people.", + "episode": "572", + "episode_name": "Ep. 572 - Earth", + "episode_no": "572", + "genre": "comedy", + "lang": "en", + "language": "English", + "num": range(1, 5), + "title": "Safely Endangered - Ep. 572 - Earth", + "title_no": "352", + "username": "safelyendangered", + }, + { + "#url": "https://www.webtoons.com/en/challenge/punderworld/happy-earth-day-/viewer?title_no=312584&episode_no=40", + "#category": ("", "webtoons", "episode"), + "#class": webtoons.WebtoonsEpisodeExtractor, + "#exception": exception.NotFoundError, + "comic": "punderworld", + "description": str, + "episode": "36", + "episode_no": "40", + "genre": "challenge", + "title": r"re:^Punderworld - .+", + "title_no": "312584", + }, + { + "#url": "https://www.webtoons.com/en/canvas/i-want-to-be-a-cute-anime-girl/209-the-storys-story/viewer?title_no=349416&episode_no=214", + "#category": ("", "webtoons", "episode"), + "#class": webtoons.WebtoonsEpisodeExtractor, + "#count": 4, + "comic_name": "I want to be a cute anime girl", + "episode_name": "209 - The story's story", + "episode": "214", + "username": "m9huj", + "author_name": "Azul Crescent", + }, + { + "#url": "https://www.webtoons.com/en/canvas/i-want-to-be-a-cute-anime-girl/174-not-194-it-was-a-typo-later/viewer?title_no=349416&episode_no=179", + "#category": ("", "webtoons", "episode"), + "#class": webtoons.WebtoonsEpisodeExtractor, + "#count": 4, + "comic_name": "I want to be a cute anime girl", + "episode_name": "174 (not 194, it was a typo) - Later", + "episode": "179", + "username": "m9huj", + "author_name": "Azul Crescent", + }, + { + "#url": "https://www.webtoons.com/en/canvas/us-over-here/1-the-wheel/viewer?title_no=919536&episode_no=1", + "#category": ("", "webtoons", "episode"), + "#class": webtoons.WebtoonsEpisodeExtractor, + "#count": 60, + "comic_name": "Us, over here", + "episode_name": "1. The Wheel", + "episode": "1", + "username": "i94q8", + "author_name": "spin.ani", + }, + { + "#url": "https://www.webtoons.com/en/comedy/live-with-yourself/list?title_no=919", + "#comment": "english", + "#category": ("", "webtoons", "comic"), + "#class": webtoons.WebtoonsComicExtractor, + "#pattern": webtoons.WebtoonsEpisodeExtractor.pattern, + "#range": "1-15", + "#count": ">= 14", + "page": range(1, 2), + "title_no": 919, + "episode_no": range(1, 14), + }, + { + "#url": "https://www.webtoons.com/fr/romance/subzero/list?title_no=1845&page=7", + "#comment": "french", + "#category": ("", "webtoons", "comic"), + "#class": webtoons.WebtoonsComicExtractor, + "#count": ">= 15", + "page": range(7, 25), + "title_no": 1845, + "episode_no": int, + }, + { + "#url": "https://www.webtoons.com/en/challenge/scoob-and-shag/list?title_no=210827&page=9", + "#comment": "(#820)", + "#category": ("", "webtoons", "comic"), + "#class": webtoons.WebtoonsComicExtractor, + "#count": ">= 18", + "page": int, + "title_no": 210827, + "episode_no": int, + }, + { + "#url": "https://www.webtoons.com/es/romance/lore-olympus/list?title_no=1725", + "#comment": "(#1643)", + "#category": ("", "webtoons", "comic"), + "#class": webtoons.WebtoonsComicExtractor, + }, ) diff --git a/test/results/weibo.py b/test/results/weibo.py index e9d8d64471..6b62ebafc8 100644 --- a/test/results/weibo.py +++ b/test/results/weibo.py @@ -4,294 +4,255 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -from gallery_dl.extractor import weibo from gallery_dl import exception - +from gallery_dl.extractor import weibo __tests__ = ( -{ - "#url" : "https://weibo.com/1758989602", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, - "#urls" : "https://weibo.com/u/1758989602?tabtype=feed", -}, - -{ - "#url" : "https://weibo.com/1758989602", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, - "#options" : {"include": "all"}, - "#urls" : ( - "https://weibo.com/u/1758989602?tabtype=home", - "https://weibo.com/u/1758989602?tabtype=feed", - "https://weibo.com/u/1758989602?tabtype=video", - "https://weibo.com/u/1758989602?tabtype=newVideo", - "https://weibo.com/u/1758989602?tabtype=album", - ), -}, - -{ - "#url" : "https://weibo.com/zhouyuxi77", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, - "#urls" : "https://weibo.com/u/7488709788?tabtype=feed", -}, - -{ - "#url" : "https://www.weibo.com/n/周于希Sally", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, - "#urls" : "https://weibo.com/u/7488709788?tabtype=feed", -}, - -{ - "#url" : "https://weibo.com/u/1758989602", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, -}, - -{ - "#url" : "https://weibo.com/p/1758989602", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, -}, - -{ - "#url" : "https://m.weibo.cn/profile/2314621010", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, -}, - -{ - "#url" : "https://m.weibo.cn/p/2304132314621010_-_WEIBO_SECOND_PROFILE_WEIBO", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, -}, - -{ - "#url" : "https://www.weibo.com/p/1003062314621010/home", - "#category": ("", "weibo", "user"), - "#class" : weibo.WeiboUserExtractor, -}, - -{ - "#url" : "https://weibo.com/1758989602?tabtype=home", - "#comment" : "'tabtype=home' is broken on website itself", - "#category": ("", "weibo", "home"), - "#class" : weibo.WeiboHomeExtractor, - "#range" : "1-30", - "#count" : 0, -}, - -{ - "#url" : "https://weibo.com/2553930725?tabtype=feed", - "#category": ("", "weibo", "feed"), - "#class" : weibo.WeiboFeedExtractor, - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://weibo.com/zhouyuxi77?tabtype=feed", - "#category": ("", "weibo", "feed"), - "#class" : weibo.WeiboFeedExtractor, - "#range" : "1", - - "status": { - "user": { - "id": 7488709788, + { + "#url": "https://weibo.com/1758989602", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + "#urls": "https://weibo.com/u/1758989602?tabtype=feed", + }, + { + "#url": "https://weibo.com/1758989602", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + "#options": {"include": "all"}, + "#urls": ( + "https://weibo.com/u/1758989602?tabtype=home", + "https://weibo.com/u/1758989602?tabtype=feed", + "https://weibo.com/u/1758989602?tabtype=video", + "https://weibo.com/u/1758989602?tabtype=newVideo", + "https://weibo.com/u/1758989602?tabtype=album", + ), + }, + { + "#url": "https://weibo.com/zhouyuxi77", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + "#urls": "https://weibo.com/u/7488709788?tabtype=feed", + }, + { + "#url": "https://www.weibo.com/n/周于希Sally", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + "#urls": "https://weibo.com/u/7488709788?tabtype=feed", + }, + { + "#url": "https://weibo.com/u/1758989602", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + }, + { + "#url": "https://weibo.com/p/1758989602", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + }, + { + "#url": "https://m.weibo.cn/profile/2314621010", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + }, + { + "#url": "https://m.weibo.cn/p/2304132314621010_-_WEIBO_SECOND_PROFILE_WEIBO", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + }, + { + "#url": "https://www.weibo.com/p/1003062314621010/home", + "#category": ("", "weibo", "user"), + "#class": weibo.WeiboUserExtractor, + }, + { + "#url": "https://weibo.com/1758989602?tabtype=home", + "#comment": "'tabtype=home' is broken on website itself", + "#category": ("", "weibo", "home"), + "#class": weibo.WeiboHomeExtractor, + "#range": "1-30", + "#count": 0, + }, + { + "#url": "https://weibo.com/2553930725?tabtype=feed", + "#category": ("", "weibo", "feed"), + "#class": weibo.WeiboFeedExtractor, + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://weibo.com/zhouyuxi77?tabtype=feed", + "#category": ("", "weibo", "feed"), + "#class": weibo.WeiboFeedExtractor, + "#range": "1", + "status": { + "user": { + "id": 7488709788, + }, }, }, -}, - -{ - "#url" : "https://www.weibo.com/n/周于希Sally?tabtype=feed", - "#category": ("", "weibo", "feed"), - "#class" : weibo.WeiboFeedExtractor, - "#range" : "1", - - - "status": { - "user": { - "id": 7488709788, + { + "#url": "https://www.weibo.com/n/周于希Sally?tabtype=feed", + "#category": ("", "weibo", "feed"), + "#class": weibo.WeiboFeedExtractor, + "#range": "1", + "status": { + "user": { + "id": 7488709788, + }, }, }, -}, - -{ - "#url" : "https://weibo.com/u/7500315942?tabtype=feed", - "#comment" : "deleted (#2521)", - "#category": ("", "weibo", "feed"), - "#class" : weibo.WeiboFeedExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://weibo.com/1758989602?tabtype=video", - "#category": ("", "weibo", "videos"), - "#class" : weibo.WeiboVideosExtractor, - "#pattern" : r"https://f\.(video\.weibocdn\.com|us\.sinaimg\.cn)/(../)?\w+\.mp4\?label=mp", - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://weibo.com/1758989602?tabtype=newVideo", - "#category": ("", "weibo", "newvideo"), - "#class" : weibo.WeiboNewvideoExtractor, - "#pattern" : r"https://f\.video\.weibocdn\.com/(../)?\w+\.mp4\?label=mp", - "#range" : "1-30", - "#count" : 30, -}, - -{ - "#url" : "https://weibo.com/1758989602?tabtype=article", - "#category": ("", "weibo", "article"), - "#class" : weibo.WeiboArticleExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://weibo.com/1758989602?tabtype=album", - "#category": ("", "weibo", "album"), - "#class" : weibo.WeiboAlbumExtractor, - "#pattern" : r"https://(wx\d+\.sinaimg\.cn/large/\w{32}\.(jpg|png|gif)|g\.us\.sinaimg\.cn/../\w+\.mp4)", - "#range" : "1-3", - "#count" : 3, -}, - -{ - "#url" : "https://m.weibo.cn/detail/4323047042991618", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#pattern" : r"https?://wx\d+.sinaimg.cn/large/\w+.jpg", - - "status": { - "count": 1, - "date" : "dt:2018-12-30 13:56:36", + { + "#url": "https://weibo.com/u/7500315942?tabtype=feed", + "#comment": "deleted (#2521)", + "#category": ("", "weibo", "feed"), + "#class": weibo.WeiboFeedExtractor, + "#count": 0, }, -}, - -{ - "#url" : "https://m.weibo.cn/detail/4339748116375525", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#pattern" : r"https?://f.us.sinaimg.cn/\w+\.mp4\?label=mp4_1080p", -}, - -{ - "#url" : "https://m.weibo.cn/status/4268682979207023", - "#comment" : "unavailable video (#427)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#exception": exception.NotFoundError, -}, - -{ - "#url" : "https://weibo.com/3314883543/Iy7fj4qVg", - "#comment" : "non-numeric status ID (#664)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, -}, - -{ - "#url" : "https://weibo.cn/detail/4600272267522211", - "#comment" : "retweet", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#count" : 0, -}, - -{ - "#url" : "https://weibo.cn/detail/4600272267522211", - "#comment" : "retweet", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#options" : {"retweets": True}, - "#count" : 2, - - "status": { - "id" : 4600272267522211, - "retweeted_status": {"id": 4600167083287033}, + { + "#url": "https://weibo.com/1758989602?tabtype=video", + "#category": ("", "weibo", "videos"), + "#class": weibo.WeiboVideosExtractor, + "#pattern": r"https://f\.(video\.weibocdn\.com|us\.sinaimg\.cn)/(../)?\w+\.mp4\?label=mp", + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://weibo.com/1758989602?tabtype=newVideo", + "#category": ("", "weibo", "newvideo"), + "#class": weibo.WeiboNewvideoExtractor, + "#pattern": r"https://f\.video\.weibocdn\.com/(../)?\w+\.mp4\?label=mp", + "#range": "1-30", + "#count": 30, + }, + { + "#url": "https://weibo.com/1758989602?tabtype=article", + "#category": ("", "weibo", "article"), + "#class": weibo.WeiboArticleExtractor, + "#count": 0, + }, + { + "#url": "https://weibo.com/1758989602?tabtype=album", + "#category": ("", "weibo", "album"), + "#class": weibo.WeiboAlbumExtractor, + "#pattern": r"https://(wx\d+\.sinaimg\.cn/large/\w{32}\.(jpg|png|gif)|g\.us\.sinaimg\.cn/../\w+\.mp4)", + "#range": "1-3", + "#count": 3, + }, + { + "#url": "https://m.weibo.cn/detail/4323047042991618", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#pattern": r"https?://wx\d+.sinaimg.cn/large/\w+.jpg", + "status": { + "count": 1, + "date": "dt:2018-12-30 13:56:36", + }, + }, + { + "#url": "https://m.weibo.cn/detail/4339748116375525", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#pattern": r"https?://f.us.sinaimg.cn/\w+\.mp4\?label=mp4_1080p", + }, + { + "#url": "https://m.weibo.cn/status/4268682979207023", + "#comment": "unavailable video (#427)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#exception": exception.NotFoundError, + }, + { + "#url": "https://weibo.com/3314883543/Iy7fj4qVg", + "#comment": "non-numeric status ID (#664)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + }, + { + "#url": "https://weibo.cn/detail/4600272267522211", + "#comment": "retweet", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#count": 0, + }, + { + "#url": "https://weibo.cn/detail/4600272267522211", + "#comment": "retweet", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#options": {"retweets": True}, + "#count": 2, + "status": { + "id": 4600272267522211, + "retweeted_status": {"id": 4600167083287033}, + }, + }, + { + "#url": "https://m.weibo.cn/detail/4600272267522211", + "#comment": "original retweets (#1542)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#options": {"retweets": "original"}, + "status": {"id": 4600167083287033}, + }, + { + "#url": "https://weibo.com/3194672795/OuxSwgUrC", + "#comment": "type == livephoto (#2146, #6471)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#pattern": r"https://livephoto\.us\.sinaimg\.cn/\w+\.mov\?Expires=\d+&ssig=[^&#]+&KID=unistore,video", + "#range": "2,4", + "filename": { + "000yfKhRjx08hBAXxdZ60f0f0100tBPr0k01", + "000GEYrCjx08hBAXUFo40f0f0100vS5G0k01", + }, + "extension": "mov", + }, + { + "#url": "https://weibo.com/1758989602/LvBhm5DiP", + "#comment": "type == gif", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#urls": "https://wx4.sinaimg.cn/large/68d80d22gy1h2ryfa8k0kg208w06o7wh.gif", + "extension": "gif", + }, + { + "#url": "https://weibo.com/1758989602/LvBhm5DiP", + "#comment": "type == gif as video (#5183)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#options": {"gifs": "video"}, + "#pattern": r"https://g\.us\.sinaimg.cn/o0/qNZcaAAglx07Wuf921CM0104120005tc0E010\.mp4\?label=gif_mp4", + }, + { + "#url": "https://weibo.com/2909128931/4409545658754086", + "#comment": "missing 'playback_list' (#2792)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#count": 10, + }, + { + "#url": "https://weibo.com/1501933722/4142890299009993", + "#comment": "empty 'playback_list' (#3301)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#pattern": r"https://f\.us\.sinaimg\.cn/004zstGKlx07dAHg4ZVu010f01000OOl0k01\.mp4\?label=mp4_hd&template=template_7&ori=0&ps=1CwnkDw1GXwCQx.+&KID=unistore,video", + "#count": 1, + }, + { + "#url": "https://weibo.com/2427303621/MxojLlLgQ", + "#comment": "mix_media_info (#3793)", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + "#count": 9, + }, + { + "#url": "https://m.weibo.cn/status/4339748116375525", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, + }, + { + "#url": "https://m.weibo.cn/5746766133/4339748116375525", + "#category": ("", "weibo", "status"), + "#class": weibo.WeiboStatusExtractor, }, -}, - -{ - "#url" : "https://m.weibo.cn/detail/4600272267522211", - "#comment" : "original retweets (#1542)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#options" : {"retweets": "original"}, - - "status": {"id": 4600167083287033}, -}, - -{ - "#url" : "https://weibo.com/3194672795/OuxSwgUrC", - "#comment" : "type == livephoto (#2146, #6471)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#pattern" : r"https://livephoto\.us\.sinaimg\.cn/\w+\.mov\?Expires=\d+&ssig=[^&#]+&KID=unistore,video", - "#range" : "2,4", - - "filename" : {"000yfKhRjx08hBAXxdZ60f0f0100tBPr0k01", "000GEYrCjx08hBAXUFo40f0f0100vS5G0k01"}, - "extension": "mov", -}, - -{ - "#url" : "https://weibo.com/1758989602/LvBhm5DiP", - "#comment" : "type == gif", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#urls" : "https://wx4.sinaimg.cn/large/68d80d22gy1h2ryfa8k0kg208w06o7wh.gif", - - "extension": "gif", -}, - -{ - "#url" : "https://weibo.com/1758989602/LvBhm5DiP", - "#comment" : "type == gif as video (#5183)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#options" : {"gifs": "video"}, - "#pattern" : r"https://g\.us\.sinaimg.cn/o0/qNZcaAAglx07Wuf921CM0104120005tc0E010\.mp4\?label=gif_mp4", -}, - -{ - "#url" : "https://weibo.com/2909128931/4409545658754086", - "#comment" : "missing 'playback_list' (#2792)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#count" : 10, -}, - -{ - "#url" : "https://weibo.com/1501933722/4142890299009993", - "#comment" : "empty 'playback_list' (#3301)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#pattern" : r"https://f\.us\.sinaimg\.cn/004zstGKlx07dAHg4ZVu010f01000OOl0k01\.mp4\?label=mp4_hd&template=template_7&ori=0&ps=1CwnkDw1GXwCQx.+&KID=unistore,video", - "#count" : 1, -}, - -{ - "#url" : "https://weibo.com/2427303621/MxojLlLgQ", - "#comment" : "mix_media_info (#3793)", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, - "#count" : 9, -}, - -{ - "#url" : "https://m.weibo.cn/status/4339748116375525", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, -}, - -{ - "#url" : "https://m.weibo.cn/5746766133/4339748116375525", - "#category": ("", "weibo", "status"), - "#class" : weibo.WeiboStatusExtractor, -}, - ) diff --git a/test/results/wikiart.py b/test/results/wikiart.py index 9ab131030c..10889e232b 100644 --- a/test/results/wikiart.py +++ b/test/results/wikiart.py @@ -1,108 +1,97 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikiart - __tests__ = ( -{ - "#url" : "https://www.wikiart.org/en/thomas-cole", - "#category": ("", "wikiart", "artist"), - "#class" : wikiart.WikiartArtistExtractor, - "#pattern" : r"https://uploads\d+\.wikiart\.org/(\d+/)?images/thomas-cole/[\w()-]+\.(jpg|png)", - "#count" : "> 100", - - "albums" : None, - "artist" : { - "OriginalArtistName": "Thomas Cole", - "activeYearsCompletion": None, - "activeYearsStart" : None, - "artistName" : "Thomas Cole", - "biography" : "Thomas Cole inspired the generation of American [url href=https://www.wikiart.org/en/paintings-by-genre/landscape]landscape[/url] painters that came to be known as the [url href=https://www.wikiart.org/en/artists-by-painting-school/hudson-river-school]Hudson River School[/url]. Born in Bolton-le-Moors, Lancashire, England, in 1801, at the age of seventeen he emigrated with his family to the United States, first working as a wood engraver in Philadelphia before going to Steubenville, Ohio, where his father had established a wallpaper manufacturing business. \n\nCole received rudimentary instruction from an itinerant artist, began painting portraits, genre scenes, and a few landscapes, and set out to seek his fortune through Ohio and Pennsylvania. He soon moved on to Philadelphia to pursue his art, inspired by paintings he saw at the Pennsylvania Academy of the Fine Arts. Moving to New York City in spring 1825, Cole made a trip up the Hudson River to the eastern Catskill Mountains. Based on his sketches there, he executed three landscapes that a city bookseller agreed to display in his window. Colonel [url href=https://www.wikiart.org/en/john-trumbull]John Trumbull[/url], already renowned as the painter of the American Revolution, saw Cole’s pictures and instantly purchased one, recommending the other two to his colleagues William Dunlap and [url href=https://www.wikiart.org/en/asher-brown-durand]Asher B. Durand[/url]. \n\nWhat Trumbull recognized in the work of the young painter was the perception of wildness inherent in American scenery that landscape artists had theretofore ignored. Trumbull brought Cole to the attention of various patrons, who began eagerly buying his work. Dunlap publicized the discovery of the new talent, and Cole was welcomed into New York’s cultural community, which included the poet and editor William Cullen Bryant and the author James Fenimore Cooper. Cole became one of the founding members of the National Academy of Design in 1825. Even as Cole expanded his travels and subjects to include scenes in the White Mountains of New Hampshire, he aspired to what he termed a “higher style of a landscape” that included narrative—some of the paintings in paired series—including biblical and literary subjects, such as Cooper’s popular [url href=https://www.wikiart.org/en/thomas-cole/scene-from-the-last-of-the-mohicans-by-james-fenimore-cooper-1827][i]Last of the Mohicans[/i][/url]. \n\nBy 1829, his success enabled him to take the Grand Tour of Europe and especially Italy, where he remained in 1831–32, visiting Florence, Rome, and Naples. Thereafter he painted many Italian subjects, like [url href=https://www.wikiart.org/en/thomas-cole/a-view-near-tivoli-morning-1832][i]View near Tivoli. Morning[/i][/url] (1832). The region around Rome, along with the classical myth, also inspired [url href=https://www.wikiart.org/en/thomas-cole/the-titan-s-goblet-1833][i]The Titan’s Goblet[/i][/url] (1833). Cole’s travels and the encouragement and patronage of the New York merchant Luman Reed culminated in his most ambitious historical landscape series, [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-course-of-empire,resultType:masonry][i]The Course of Empire[/i][/url] (1833–1836), five pictures dramatizing the rise and fall of an ancient classical state. \n\nCole also continued to paint, with ever-rising technical assurance, sublime American scenes such as the [url href=https://www.wikiart.org/en/thomas-cole/view-from-mount-holyoke-1836][i]View from Mount Holyoke[/i][/url] (1836), [url href=https://www.wikiart.org/en/thomas-cole/the-oxbow-the-connecticut-river-near-northampton-1836][i]The Oxbow[/i][/url] (1836), in which he included a portrait of himself painting the vista and [url href=https://www.wikiart.org/en/thomas-cole/view-on-the-catskill-early-autunm-1837][i]View on the Catskill—Early Autumn[/i][/url] (1836-1837), in which he pastorally interpreted the prospect of his beloved Catskill Mountains from the village of Catskill, where he had moved the year before and met his wife-to-be, Maria Bartow. \n\nThe artist’s marriage brought with it increasing religious piety manifested in the four-part series [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-voyage-of-life,resultType:masonry][i]The Voyage of Life[/i][/url] (1840). In it, a river journey represents the human passage through life to eternal reward. Cole painted and exhibited a replica of the series in Rome, where he returned in 1841–42, traveling south to Sicily. After his return, he lived and worked chiefly in Catskill, keeping up with art activity in New York primarily through Durand. He continued to produce American and foreign landscape subjects of incredible beauty, including the [url href=https://www.wikiart.org/en/thomas-cole/the-mountain-ford-1846][i]Mountain Ford[/i][/url] (1846). \n\nIn 1844, Cole welcomed into his Catskill studio the young [url href=https://www.wikiart.org/en/frederic-edwin-church]Frederic Church[/url], who studied with him until 1846 and went on to become the most renowned exponent of the generation that followed Cole. By 1846, Cole was at work on his largest and most ambitious series, [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-cross-and-the-world,resultType:masonry][i]The Cross and the World[/i][/url], but in February 1848 contracted pleurisy and died before completing it. \n\nThe paintings of Thomas Cole, like the writings of his contemporary Ralph Waldo Emerson, stand as monuments to the dreams and anxieties of the fledgling American nation during the mid-19th century; and they are also euphoric celebrations of its natural landscapes. Cole is considered the first artist to bring the eye of a European [url href=https://www.wikiart.org/en/artists-by-art-movement/romanticism]Romantic[/url] landscape painter to those environments, but also a figure whose idealism and religious sensibilities expressed a uniquely American spirit. In his works, we find the dramatic splendor of [url href=https://www.wikiart.org/en/caspar-david-friedrich]Caspar David Freidrich[/url] or [url href=https://www.wikiart.org/en/william-turner]J.M.W Turner[/url] transposed onto the Catskill and Adirondack Mountains. But whereas younger American painters such as [url href=https://www.wikiart.org/en/albert-bierstadt]Albert Bierstadt[/url] had come into direct contact with [url href=https://www.wikiart.org/en/artists-by-art-institution/kunstakademie-dusseldorf-dusseldorf-germany#!#resultType:masonry]The Düsseldorf School of painting[/url], and thus with the tradition in which they placed themselves, Cole was largely self-tutored, representing something of the archetypal American figure of the auto-didact.\n\nIn many ways, Cole's art epitomizes all contradictions of European settler culture in America. He was in love with the sublime wildness of the American landscape and sought to preserve it with his art, but his very presence in that landscape, and the development of his career, depended on the processes of urbanization and civilization which threatened it. From a modern perspective, Cole's Eurocentric gaze on seemingly empty wildernesses which had, in fact, been populated for centuries, also seems troubling; where Native Americans do appear in his work, as in [url href=https://www.wikiart.org/en/thomas-cole/falls-of-the-kaaterskill-1826][i]Falls of the Kaaterskill[/i][/url] (1826), it is as picturesque flecks rather than characterized participants in the scene.\n\nCole's legacy is evident in the work of future American artists who advanced the Hudson River style, including his student Frederic Edwin Church, Albert Bierstadt, Jasper Cropsey, Asher B. Durand, [url href=https://www.wikiart.org/en/george-inness]George Inness[/url], [url href=https://www.wikiart.org/en/john-frederick-kensett]John Kensett[/url], and [url href=https://www.wikiart.org/en/thomas-moran]Thomas Moran[/url]. Speaking more broadly, a whole sweep of 20th-century North-American art, from [url href=https://www.wikiart.org/en/artists-by-art-movement/precisionism]Precisionism[/url] to [url href=https://www.wikiart.org/en/artists-by-art-movement/environmental-art]Land Art[/url], might be seen to have inherited something of the grand scale and ambition of Cole's work. In this sense, his paintings capture not only the character of American culture during the mid-19th century but perhaps something more enduring about the open and expansive quality of that culture.", - "birthDay" : "/Date(-5330448000000)/", - "birthDayAsString" : "February 1, 1801", - "contentId" : 254330, - "deathDay" : "/Date(-3846441600000)/", - "deathDayAsString" : "February 11, 1848", - "dictonaries" : [ - 1368, - 11415, - 310, - ], - "gender" : "male", - "image" : "https://uploads8.wikiart.org/temp/19f6a140-59d2-4959-8d11-fd4ca582b7f2.jpg!Portrait.jpg", - "lastNameFirst" : "Cole Thomas", - "periodsOfWork" : "", - "relatedArtistsIds" : [], - "series" : "The Cross and the World\r\nThe Course of Empire\r\nThe Voyage of Life", - "story" : "http://en.wikipedia.org/wiki/Thomas_Cole", - "themes" : "", - "url" : "thomas-cole", - "wikipediaUrl" : "http://en.wikipedia.org/wiki/Thomas_Cole" + { + "#url": "https://www.wikiart.org/en/thomas-cole", + "#category": ("", "wikiart", "artist"), + "#class": wikiart.WikiartArtistExtractor, + "#pattern": r"https://uploads\d+\.wikiart\.org/(\d+/)?images/thomas-cole/[\w()-]+\.(jpg|png)", + "#count": "> 100", + "albums": None, + "artist": { + "OriginalArtistName": "Thomas Cole", + "activeYearsCompletion": None, + "activeYearsStart": None, + "artistName": "Thomas Cole", + "biography": "Thomas Cole inspired the generation of American [url href=https://www.wikiart.org/en/paintings-by-genre/landscape]landscape[/url] painters that came to be known as the [url href=https://www.wikiart.org/en/artists-by-painting-school/hudson-river-school]Hudson River School[/url]. Born in Bolton-le-Moors, Lancashire, England, in 1801, at the age of seventeen he emigrated with his family to the United States, first working as a wood engraver in Philadelphia before going to Steubenville, Ohio, where his father had established a wallpaper manufacturing business. \n\nCole received rudimentary instruction from an itinerant artist, began painting portraits, genre scenes, and a few landscapes, and set out to seek his fortune through Ohio and Pennsylvania. He soon moved on to Philadelphia to pursue his art, inspired by paintings he saw at the Pennsylvania Academy of the Fine Arts. Moving to New York City in spring 1825, Cole made a trip up the Hudson River to the eastern Catskill Mountains. Based on his sketches there, he executed three landscapes that a city bookseller agreed to display in his window. Colonel [url href=https://www.wikiart.org/en/john-trumbull]John Trumbull[/url], already renowned as the painter of the American Revolution, saw Cole’s pictures and instantly purchased one, recommending the other two to his colleagues William Dunlap and [url href=https://www.wikiart.org/en/asher-brown-durand]Asher B. Durand[/url]. \n\nWhat Trumbull recognized in the work of the young painter was the perception of wildness inherent in American scenery that landscape artists had theretofore ignored. Trumbull brought Cole to the attention of various patrons, who began eagerly buying his work. Dunlap publicized the discovery of the new talent, and Cole was welcomed into New York’s cultural community, which included the poet and editor William Cullen Bryant and the author James Fenimore Cooper. Cole became one of the founding members of the National Academy of Design in 1825. Even as Cole expanded his travels and subjects to include scenes in the White Mountains of New Hampshire, he aspired to what he termed a “higher style of a landscape” that included narrative—some of the paintings in paired series—including biblical and literary subjects, such as Cooper’s popular [url href=https://www.wikiart.org/en/thomas-cole/scene-from-the-last-of-the-mohicans-by-james-fenimore-cooper-1827][i]Last of the Mohicans[/i][/url]. \n\nBy 1829, his success enabled him to take the Grand Tour of Europe and especially Italy, where he remained in 1831–32, visiting Florence, Rome, and Naples. Thereafter he painted many Italian subjects, like [url href=https://www.wikiart.org/en/thomas-cole/a-view-near-tivoli-morning-1832][i]View near Tivoli. Morning[/i][/url] (1832). The region around Rome, along with the classical myth, also inspired [url href=https://www.wikiart.org/en/thomas-cole/the-titan-s-goblet-1833][i]The Titan’s Goblet[/i][/url] (1833). Cole’s travels and the encouragement and patronage of the New York merchant Luman Reed culminated in his most ambitious historical landscape series, [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-course-of-empire,resultType:masonry][i]The Course of Empire[/i][/url] (1833–1836), five pictures dramatizing the rise and fall of an ancient classical state. \n\nCole also continued to paint, with ever-rising technical assurance, sublime American scenes such as the [url href=https://www.wikiart.org/en/thomas-cole/view-from-mount-holyoke-1836][i]View from Mount Holyoke[/i][/url] (1836), [url href=https://www.wikiart.org/en/thomas-cole/the-oxbow-the-connecticut-river-near-northampton-1836][i]The Oxbow[/i][/url] (1836), in which he included a portrait of himself painting the vista and [url href=https://www.wikiart.org/en/thomas-cole/view-on-the-catskill-early-autunm-1837][i]View on the Catskill—Early Autumn[/i][/url] (1836-1837), in which he pastorally interpreted the prospect of his beloved Catskill Mountains from the village of Catskill, where he had moved the year before and met his wife-to-be, Maria Bartow. \n\nThe artist’s marriage brought with it increasing religious piety manifested in the four-part series [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-voyage-of-life,resultType:masonry][i]The Voyage of Life[/i][/url] (1840). In it, a river journey represents the human passage through life to eternal reward. Cole painted and exhibited a replica of the series in Rome, where he returned in 1841–42, traveling south to Sicily. After his return, he lived and worked chiefly in Catskill, keeping up with art activity in New York primarily through Durand. He continued to produce American and foreign landscape subjects of incredible beauty, including the [url href=https://www.wikiart.org/en/thomas-cole/the-mountain-ford-1846][i]Mountain Ford[/i][/url] (1846). \n\nIn 1844, Cole welcomed into his Catskill studio the young [url href=https://www.wikiart.org/en/frederic-edwin-church]Frederic Church[/url], who studied with him until 1846 and went on to become the most renowned exponent of the generation that followed Cole. By 1846, Cole was at work on his largest and most ambitious series, [url href=https://www.wikiart.org/en/thomas-cole/all-works#!#filterName:Series_the-cross-and-the-world,resultType:masonry][i]The Cross and the World[/i][/url], but in February 1848 contracted pleurisy and died before completing it. \n\nThe paintings of Thomas Cole, like the writings of his contemporary Ralph Waldo Emerson, stand as monuments to the dreams and anxieties of the fledgling American nation during the mid-19th century; and they are also euphoric celebrations of its natural landscapes. Cole is considered the first artist to bring the eye of a European [url href=https://www.wikiart.org/en/artists-by-art-movement/romanticism]Romantic[/url] landscape painter to those environments, but also a figure whose idealism and religious sensibilities expressed a uniquely American spirit. In his works, we find the dramatic splendor of [url href=https://www.wikiart.org/en/caspar-david-friedrich]Caspar David Freidrich[/url] or [url href=https://www.wikiart.org/en/william-turner]J.M.W Turner[/url] transposed onto the Catskill and Adirondack Mountains. But whereas younger American painters such as [url href=https://www.wikiart.org/en/albert-bierstadt]Albert Bierstadt[/url] had come into direct contact with [url href=https://www.wikiart.org/en/artists-by-art-institution/kunstakademie-dusseldorf-dusseldorf-germany#!#resultType:masonry]The Düsseldorf School of painting[/url], and thus with the tradition in which they placed themselves, Cole was largely self-tutored, representing something of the archetypal American figure of the auto-didact.\n\nIn many ways, Cole's art epitomizes all contradictions of European settler culture in America. He was in love with the sublime wildness of the American landscape and sought to preserve it with his art, but his very presence in that landscape, and the development of his career, depended on the processes of urbanization and civilization which threatened it. From a modern perspective, Cole's Eurocentric gaze on seemingly empty wildernesses which had, in fact, been populated for centuries, also seems troubling; where Native Americans do appear in his work, as in [url href=https://www.wikiart.org/en/thomas-cole/falls-of-the-kaaterskill-1826][i]Falls of the Kaaterskill[/i][/url] (1826), it is as picturesque flecks rather than characterized participants in the scene.\n\nCole's legacy is evident in the work of future American artists who advanced the Hudson River style, including his student Frederic Edwin Church, Albert Bierstadt, Jasper Cropsey, Asher B. Durand, [url href=https://www.wikiart.org/en/george-inness]George Inness[/url], [url href=https://www.wikiart.org/en/john-frederick-kensett]John Kensett[/url], and [url href=https://www.wikiart.org/en/thomas-moran]Thomas Moran[/url]. Speaking more broadly, a whole sweep of 20th-century North-American art, from [url href=https://www.wikiart.org/en/artists-by-art-movement/precisionism]Precisionism[/url] to [url href=https://www.wikiart.org/en/artists-by-art-movement/environmental-art]Land Art[/url], might be seen to have inherited something of the grand scale and ambition of Cole's work. In this sense, his paintings capture not only the character of American culture during the mid-19th century but perhaps something more enduring about the open and expansive quality of that culture.", + "birthDay": "/Date(-5330448000000)/", + "birthDayAsString": "February 1, 1801", + "contentId": 254330, + "deathDay": "/Date(-3846441600000)/", + "deathDayAsString": "February 11, 1848", + "dictonaries": [ + 1368, + 11415, + 310, + ], + "gender": "male", + "image": "https://uploads8.wikiart.org/temp/19f6a140-59d2-4959-8d11-fd4ca582b7f2.jpg!Portrait.jpg", + "lastNameFirst": "Cole Thomas", + "periodsOfWork": "", + "relatedArtistsIds": [], + "series": "The Cross and the World\r\nThe Course of Empire\r\nThe Voyage of Life", + "story": "http://en.wikipedia.org/wiki/Thomas_Cole", + "themes": "", + "url": "thomas-cole", + "wikipediaUrl": "http://en.wikipedia.org/wiki/Thomas_Cole", + }, + "artistName": "Thomas Cole", + "artistUrl": "/en/thomas-cole", + "extension": str, + "filename": str, + "flags": int, + "height": int, + "id": r"re:[0-9a-f]+", + "image": str, + "map": str, + "paintingUrl": r"re:/en/thomas-cole/.+", + "title": str, + "width": int, + "year": str, + }, + { + "#url": "https://www.wikiart.org/en/thomas-cole/the-departure-1838", + "#category": ("", "wikiart", "image"), + "#class": wikiart.WikiartImageExtractor, + "#sha1_url": "976cc2545f308a650b5dbb35c29d3cee0f4673b3", + "#sha1_metadata": "8e80cdcb01c1fedb934633d1c4c3ab0419cfbedf", + }, + { + "#url": "https://www.wikiart.org/en/huang-shen/summer", + "#comment": "no year or '-' in slug", + "#category": ("", "wikiart", "image"), + "#class": wikiart.WikiartImageExtractor, + "#sha1_url": "d7f60118c34067b2b37d9577e412dc1477b94207", + "#urls": ("https://uploads5.wikiart.org/images/huang-shen/summer.jpg",), + }, + { + "#url": "https://www.wikiart.org/en/paintings-by-media/grisaille", + "#category": ("", "wikiart", "artworks"), + "#class": wikiart.WikiartArtworksExtractor, + "#sha1_url": "36e054fcb3363b7f085c81f4778e6db3994e56a3", + "#urls": ( + "https://uploads4.wikiart.org/images/hieronymus-bosch/triptych-of-last-judgement.jpg", + "https://uploads6.wikiart.org/images/hieronymus-bosch/triptych-of-last-judgement-1.jpg", + "https://uploads0.wikiart.org/images/hieronymus-bosch/tiptych-of-temptation-of-st-anthony-1506.jpg", + "https://uploads7.wikiart.org/images/matthias-grünewald/st-elizabeth-and-a-saint-woman-with-palm-1511.jpg", + "https://uploads2.wikiart.org/images/matthias-grünewald/st-lawrence-and-st-cyricus-1511.jpg", + "https://uploads0.wikiart.org/images/pieter-bruegel-the-elder/the-death-of-the-virgin.jpg", + "https://uploads4.wikiart.org/images/pieter-bruegel-the-elder/christ-and-the-woman-taken-in-adultery-1565-1.jpg", + "https://uploads6.wikiart.org/images/giovanni-battista-tiepolo/not_detected_241014.jpg", + "https://uploads4.wikiart.org/images/edgar-degas/interior-the-rape-1869.jpg", + "https://uploads3.wikiart.org/00265/images/john-singer-sargent/1396294310-dame-alice-ellen-terry-by-john-singer-sargent.jpg", + "https://uploads0.wikiart.org/00293/images/hryhorii-havrylenko/1954-18-5-32-5.jpg", + ), + }, + { + "#url": "https://www.wikiart.org/en/artists-by-century/12", + "#category": ("", "wikiart", "artists"), + "#class": wikiart.WikiartArtistsExtractor, + "#pattern": wikiart.WikiartArtistExtractor.pattern, + "#count": ">= 8", }, - "artistName" : "Thomas Cole", - "artistUrl" : "/en/thomas-cole", - "extension" : str, - "filename" : str, - "flags" : int, - "height" : int, - "id" : r"re:[0-9a-f]+", - "image" : str, - "map" : str, - "paintingUrl": r"re:/en/thomas-cole/.+", - "title" : str, - "width" : int, - "year" : str, -}, - -{ - "#url" : "https://www.wikiart.org/en/thomas-cole/the-departure-1838", - "#category": ("", "wikiart", "image"), - "#class" : wikiart.WikiartImageExtractor, - "#sha1_url" : "976cc2545f308a650b5dbb35c29d3cee0f4673b3", - "#sha1_metadata": "8e80cdcb01c1fedb934633d1c4c3ab0419cfbedf", -}, - -{ - "#url" : "https://www.wikiart.org/en/huang-shen/summer", - "#comment" : "no year or '-' in slug", - "#category": ("", "wikiart", "image"), - "#class" : wikiart.WikiartImageExtractor, - "#sha1_url": "d7f60118c34067b2b37d9577e412dc1477b94207", - "#urls" : ( - "https://uploads5.wikiart.org/images/huang-shen/summer.jpg", - ), -}, - -{ - "#url" : "https://www.wikiart.org/en/paintings-by-media/grisaille", - "#category": ("", "wikiart", "artworks"), - "#class" : wikiart.WikiartArtworksExtractor, - "#sha1_url": "36e054fcb3363b7f085c81f4778e6db3994e56a3", - "#urls" : ( - "https://uploads4.wikiart.org/images/hieronymus-bosch/triptych-of-last-judgement.jpg", - "https://uploads6.wikiart.org/images/hieronymus-bosch/triptych-of-last-judgement-1.jpg", - "https://uploads0.wikiart.org/images/hieronymus-bosch/tiptych-of-temptation-of-st-anthony-1506.jpg", - "https://uploads7.wikiart.org/images/matthias-grünewald/st-elizabeth-and-a-saint-woman-with-palm-1511.jpg", - "https://uploads2.wikiart.org/images/matthias-grünewald/st-lawrence-and-st-cyricus-1511.jpg", - "https://uploads0.wikiart.org/images/pieter-bruegel-the-elder/the-death-of-the-virgin.jpg", - "https://uploads4.wikiart.org/images/pieter-bruegel-the-elder/christ-and-the-woman-taken-in-adultery-1565-1.jpg", - "https://uploads6.wikiart.org/images/giovanni-battista-tiepolo/not_detected_241014.jpg", - "https://uploads4.wikiart.org/images/edgar-degas/interior-the-rape-1869.jpg", - "https://uploads3.wikiart.org/00265/images/john-singer-sargent/1396294310-dame-alice-ellen-terry-by-john-singer-sargent.jpg", - "https://uploads0.wikiart.org/00293/images/hryhorii-havrylenko/1954-18-5-32-5.jpg", - ), -}, - -{ - "#url" : "https://www.wikiart.org/en/artists-by-century/12", - "#category": ("", "wikiart", "artists"), - "#class" : wikiart.WikiartArtistsExtractor, - "#pattern" : wikiart.WikiartArtistExtractor.pattern, - "#count" : ">= 8", -}, - ) diff --git a/test/results/wikibooks.py b/test/results/wikibooks.py index da4d761dba..15806977e5 100644 --- a/test/results/wikibooks.py +++ b/test/results/wikibooks.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikibooks.org/wiki/Title", - "#category": ("wikimedia", "wikibooks", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikibooks.org/wiki/Category:Title", - "#category": ("wikimedia", "wikibooks", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikibooks.org/wiki/Title", + "#category": ("wikimedia", "wikibooks", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikibooks.org/wiki/Category:Title", + "#category": ("wikimedia", "wikibooks", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikidata.py b/test/results/wikidata.py index c0e2eb6aef..36c664c017 100644 --- a/test/results/wikidata.py +++ b/test/results/wikidata.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikidata.org/wiki/Title", - "#category": ("wikimedia", "wikidata", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikidata.org/wiki/Category:Title", - "#category": ("wikimedia", "wikidata", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikidata.org/wiki/Title", + "#category": ("wikimedia", "wikidata", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikidata.org/wiki/Category:Title", + "#category": ("wikimedia", "wikidata", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikifeet.py b/test/results/wikifeet.py index 2a8b849b37..dce6f40a77 100644 --- a/test/results/wikifeet.py +++ b/test/results/wikifeet.py @@ -1,51 +1,44 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikifeet - __tests__ = ( -{ - "#url" : "https://www.wikifeet.com/Madison_Beer", - "#category": ("", "wikifeet", "gallery"), - "#class" : wikifeet.WikifeetGalleryExtractor, - "#pattern" : r"https://pics\.wikifeet\.com/Madison_Beer-Feet-\d+\.jpg", - "#count" : ">= 352", - - "celeb" : "Madison_Beer", - "celebrity" : "Madison Beer", - "birthday" : "dt:1999-03-05 00:00:00", - "birthplace": "United States", - "rating" : float, - "pid" : int, - "width" : int, - "height" : int, - "shoesize" : r"re:\d+ US", - "type" : "women", - "tags" : list, -}, - -{ - "#url" : "https://men.wikifeet.com/Chris_Hemsworth", - "#category": ("", "wikifeet", "gallery"), - "#class" : wikifeet.WikifeetGalleryExtractor, - "#pattern" : r"https://pics\.wikifeet\.com/Chris_Hemsworth-Feet-\d+\.jpg", - "#count" : ">= 860", - - "celeb" : "Chris_Hemsworth", - "celebrity" : "Chris Hemsworth", - "birthday" : "dt:1983-08-11 00:00:00", - "birthplace": "Australia", - "rating" : float, - "pid" : int, - "width" : int, - "height" : int, - "shoesize" : "12.5 US", - "type" : "men", - "tags" : list, -}, - + { + "#url": "https://www.wikifeet.com/Madison_Beer", + "#category": ("", "wikifeet", "gallery"), + "#class": wikifeet.WikifeetGalleryExtractor, + "#pattern": r"https://pics\.wikifeet\.com/Madison_Beer-Feet-\d+\.jpg", + "#count": ">= 352", + "celeb": "Madison_Beer", + "celebrity": "Madison Beer", + "birthday": "dt:1999-03-05 00:00:00", + "birthplace": "United States", + "rating": float, + "pid": int, + "width": int, + "height": int, + "shoesize": r"re:\d+ US", + "type": "women", + "tags": list, + }, + { + "#url": "https://men.wikifeet.com/Chris_Hemsworth", + "#category": ("", "wikifeet", "gallery"), + "#class": wikifeet.WikifeetGalleryExtractor, + "#pattern": r"https://pics\.wikifeet\.com/Chris_Hemsworth-Feet-\d+\.jpg", + "#count": ">= 860", + "celeb": "Chris_Hemsworth", + "celebrity": "Chris Hemsworth", + "birthday": "dt:1983-08-11 00:00:00", + "birthplace": "Australia", + "rating": float, + "pid": int, + "width": int, + "height": int, + "shoesize": "12.5 US", + "type": "men", + "tags": list, + }, ) diff --git a/test/results/wikifeetx.py b/test/results/wikifeetx.py index a3d2be6ed3..5e04d4c76a 100644 --- a/test/results/wikifeetx.py +++ b/test/results/wikifeetx.py @@ -1,31 +1,26 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikifeet - __tests__ = ( -{ - "#url" : "https://www.wikifeetx.com/Tifa_Quinn", - "#category": ("", "wikifeetx", "gallery"), - "#class" : wikifeet.WikifeetGalleryExtractor, - "#pattern" : r"https://pics\.wikifeet\.com/Tifa_Quinn-Feet-\d+\.jpg", - "#count" : ">= 9", - - "celeb" : "Tifa_Quinn", - "celebrity" : "Tifa Quinn", - "birthday" : "[NOT SET]", - "birthplace": "United States", - "rating" : float, - "pid" : int, - "width" : int, - "height" : int, - "shoesize" : "4 US", - "type" : "women", - "tags" : list, -}, - + { + "#url": "https://www.wikifeetx.com/Tifa_Quinn", + "#category": ("", "wikifeetx", "gallery"), + "#class": wikifeet.WikifeetGalleryExtractor, + "#pattern": r"https://pics\.wikifeet\.com/Tifa_Quinn-Feet-\d+\.jpg", + "#count": ">= 9", + "celeb": "Tifa_Quinn", + "celebrity": "Tifa Quinn", + "birthday": "[NOT SET]", + "birthplace": "United States", + "rating": float, + "pid": int, + "width": int, + "height": int, + "shoesize": "4 US", + "type": "women", + "tags": list, + }, ) diff --git a/test/results/wikigg.py b/test/results/wikigg.py index f64259fea2..ad6026006c 100644 --- a/test/results/wikigg.py +++ b/test/results/wikigg.py @@ -1,33 +1,27 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wiki.gg/wiki/Title", - "#comment" : "for scripts/supportedsites.py", - "#category": ("wikimedia", "wikigg-www", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://hearthstone.wiki.gg/wiki/Flame_Juggler", - "#category": ("wikimedia", "wikigg-hearthstone", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://terraria.wiki.gg/de/wiki/Golem", - "#comment" : "non-English language prefix (#6370)", - "#category": ("wikimedia", "wikigg-terraria", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#count" : "> 45", - "#archive" : False, -}, - + { + "#url": "https://www.wiki.gg/wiki/Title", + "#comment": "for scripts/supportedsites.py", + "#category": ("wikimedia", "wikigg-www", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://hearthstone.wiki.gg/wiki/Flame_Juggler", + "#category": ("wikimedia", "wikigg-hearthstone", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://terraria.wiki.gg/de/wiki/Golem", + "#comment": "non-English language prefix (#6370)", + "#category": ("wikimedia", "wikigg-terraria", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#count": "> 45", + "#archive": False, + }, ) diff --git a/test/results/wikimediacommons.py b/test/results/wikimediacommons.py index b61a9061ba..4fa308feb1 100644 --- a/test/results/wikimediacommons.py +++ b/test/results/wikimediacommons.py @@ -1,24 +1,19 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://commons.wikimedia.org/wiki/File:Starr-050516-1367-Pimenta_dioica-flowers-Maunaloa-Molokai_(24762757525).jpg", - "#category": ("wikimedia", "wikimediacommons", "file"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#urls" : "https://upload.wikimedia.org/wikipedia/commons/f/fa/Starr-050516-1367-Pimenta_dioica-flowers-Maunaloa-Molokai_%2824762757525%29.jpg", -}, - -{ - "#url" : "https://commons.wikimedia.org/wiki/Category:Network_maps_of_the_Paris_Metro", - "#category": ("wikimedia", "wikimediacommons", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://commons.wikimedia.org/wiki/File:Starr-050516-1367-Pimenta_dioica-flowers-Maunaloa-Molokai_(24762757525).jpg", + "#category": ("wikimedia", "wikimediacommons", "file"), + "#class": wikimedia.WikimediaArticleExtractor, + "#urls": "https://upload.wikimedia.org/wikipedia/commons/f/fa/Starr-050516-1367-Pimenta_dioica-flowers-Maunaloa-Molokai_%2824762757525%29.jpg", + }, + { + "#url": "https://commons.wikimedia.org/wiki/Category:Network_maps_of_the_Paris_Metro", + "#category": ("wikimedia", "wikimediacommons", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikinews.py b/test/results/wikinews.py index 79817fdbf1..3418a22de9 100644 --- a/test/results/wikinews.py +++ b/test/results/wikinews.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikinews.org/wiki/Title", - "#category": ("wikimedia", "wikinews", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikinews.org/wiki/Category:Title", - "#category": ("wikimedia", "wikinews", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikinews.org/wiki/Title", + "#category": ("wikimedia", "wikinews", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikinews.org/wiki/Category:Title", + "#category": ("wikimedia", "wikinews", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikipedia.py b/test/results/wikipedia.py index f478a49d6b..d67b22bf9b 100644 --- a/test/results/wikipedia.py +++ b/test/results/wikipedia.py @@ -1,61 +1,53 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikipedia.org/wiki/Title", - "#category": ("wikimedia", "wikipedia", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikipedia.org/wiki/Athena", - "#category": ("wikimedia", "wikipedia", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#pattern" : r"https://upload.wikimedia.org/wikipedia/.+", - "#count" : range(50, 100), - - "bitdepth" : int, - "canonicaltitle": str, - "comment" : str, - "commonmetadata": dict, - "date" : "type:datetime", - "descriptionshorturl": str, - "descriptionurl": str, - "extension" : str, - "extmetadata" : dict, - "filename" : str, - "height" : int, - "metadata" : dict, - "mime" : r"re:image/\w+", - "page" : "Athena", - "sha1" : r"re:^[0-9a-f]{40}$", - "size" : int, - "timestamp" : str, - "url" : str, - "user" : str, - "userid" : int, - "width" : int, -}, - -{ - "#url" : "https://en.wikipedia.org/wiki/Category:Physics", - "#category": ("wikimedia", "wikipedia", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikipedia.org", - "#category": ("wikimedia", "wikipedia", "wiki"), - "#class" : wikimedia.WikimediaWikiExtractor, - "#range" : "1-10", - "#count" : 10, -}, - + { + "#url": "https://www.wikipedia.org/wiki/Title", + "#category": ("wikimedia", "wikipedia", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikipedia.org/wiki/Athena", + "#category": ("wikimedia", "wikipedia", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#pattern": r"https://upload.wikimedia.org/wikipedia/.+", + "#count": range(50, 100), + "bitdepth": int, + "canonicaltitle": str, + "comment": str, + "commonmetadata": dict, + "date": "type:datetime", + "descriptionshorturl": str, + "descriptionurl": str, + "extension": str, + "extmetadata": dict, + "filename": str, + "height": int, + "metadata": dict, + "mime": r"re:image/\w+", + "page": "Athena", + "sha1": r"re:^[0-9a-f]{40}$", + "size": int, + "timestamp": str, + "url": str, + "user": str, + "userid": int, + "width": int, + }, + { + "#url": "https://en.wikipedia.org/wiki/Category:Physics", + "#category": ("wikimedia", "wikipedia", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikipedia.org", + "#category": ("wikimedia", "wikipedia", "wiki"), + "#class": wikimedia.WikimediaWikiExtractor, + "#range": "1-10", + "#count": 10, + }, ) diff --git a/test/results/wikiquote.py b/test/results/wikiquote.py index 8365e3b7de..9e7dd57343 100644 --- a/test/results/wikiquote.py +++ b/test/results/wikiquote.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikiquote.org/wiki/Title", - "#category": ("wikimedia", "wikiquote", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikiquote.org/wiki/Category:Title", - "#category": ("wikimedia", "wikiquote", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikiquote.org/wiki/Title", + "#category": ("wikimedia", "wikiquote", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikiquote.org/wiki/Category:Title", + "#category": ("wikimedia", "wikiquote", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikisource.py b/test/results/wikisource.py index 0ac1bb0fa6..1a172b3af6 100644 --- a/test/results/wikisource.py +++ b/test/results/wikisource.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikisource.org/wiki/Title", - "#category": ("wikimedia", "wikisource", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikisource.org/wiki/Category:Title", - "#category": ("wikimedia", "wikisource", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikisource.org/wiki/Title", + "#category": ("wikimedia", "wikisource", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikisource.org/wiki/Category:Title", + "#category": ("wikimedia", "wikisource", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikispecies.py b/test/results/wikispecies.py index 26aca84b1e..330f1079a7 100644 --- a/test/results/wikispecies.py +++ b/test/results/wikispecies.py @@ -1,25 +1,20 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://species.wikimedia.org/wiki/Geranospiza", - "#category": ("wikimedia", "wikispecies", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, - "#urls" : "https://upload.wikimedia.org/wikipedia/commons/0/01/Geranospiza_caerulescens.jpg", - "#sha1_content": "3a17c14b15489928e4154f826af1c42afb5a523e", -}, - -{ - "#url" : "https://species.wikimedia.org/wiki/Category:Names", - "#category": ("wikimedia", "wikispecies", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://species.wikimedia.org/wiki/Geranospiza", + "#category": ("wikimedia", "wikispecies", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + "#urls": "https://upload.wikimedia.org/wikipedia/commons/0/01/Geranospiza_caerulescens.jpg", + "#sha1_content": "3a17c14b15489928e4154f826af1c42afb5a523e", + }, + { + "#url": "https://species.wikimedia.org/wiki/Category:Names", + "#category": ("wikimedia", "wikispecies", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikiversity.py b/test/results/wikiversity.py index 2e64ca3140..de37e1e292 100644 --- a/test/results/wikiversity.py +++ b/test/results/wikiversity.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikiversity.org/wiki/Title", - "#category": ("wikimedia", "wikiversity", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikiversity.org/wiki/Category:Title", - "#category": ("wikimedia", "wikiversity", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikiversity.org/wiki/Title", + "#category": ("wikimedia", "wikiversity", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikiversity.org/wiki/Category:Title", + "#category": ("wikimedia", "wikiversity", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wikivoyage.py b/test/results/wikivoyage.py index 860e68dec1..6d5ec9286e 100644 --- a/test/results/wikivoyage.py +++ b/test/results/wikivoyage.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wikivoyage.org/wiki/Title", - "#category": ("wikimedia", "wikivoyage", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wikivoyage.org/wiki/Category:Title", - "#category": ("wikimedia", "wikivoyage", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wikivoyage.org/wiki/Title", + "#category": ("wikimedia", "wikivoyage", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wikivoyage.org/wiki/Category:Title", + "#category": ("wikimedia", "wikivoyage", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/wiktionary.py b/test/results/wiktionary.py index 4a643ab57f..d836aaf710 100644 --- a/test/results/wiktionary.py +++ b/test/results/wiktionary.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import wikimedia - __tests__ = ( -{ - "#url" : "https://www.wiktionary.org/wiki/Word", - "#category": ("wikimedia", "wiktionary", "article"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - -{ - "#url" : "https://en.wiktionary.org/wiki/Category:Words", - "#category": ("wikimedia", "wiktionary", "category"), - "#class" : wikimedia.WikimediaArticleExtractor, -}, - + { + "#url": "https://www.wiktionary.org/wiki/Word", + "#category": ("wikimedia", "wiktionary", "article"), + "#class": wikimedia.WikimediaArticleExtractor, + }, + { + "#url": "https://en.wiktionary.org/wiki/Category:Words", + "#category": ("wikimedia", "wiktionary", "category"), + "#class": wikimedia.WikimediaArticleExtractor, + }, ) diff --git a/test/results/windsorstore.py b/test/results/windsorstore.py index c7cfd69892..6b028ca7d2 100644 --- a/test/results/windsorstore.py +++ b/test/results/windsorstore.py @@ -1,23 +1,18 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import shopify - __tests__ = ( -{ - "#url" : "https://www.windsorstore.com/collections/dresses-ball-gowns", - "#category": ("shopify", "windsorstore", "collection"), - "#class" : shopify.ShopifyCollectionExtractor, -}, - -{ - "#url" : "https://www.windsorstore.com/collections/accessories-belts/products/rhine-buckle-dbl-o-ring-pu-strap-belt-073010158001", - "#category": ("shopify", "windsorstore", "product"), - "#class" : shopify.ShopifyProductExtractor, -}, - + { + "#url": "https://www.windsorstore.com/collections/dresses-ball-gowns", + "#category": ("shopify", "windsorstore", "collection"), + "#class": shopify.ShopifyCollectionExtractor, + }, + { + "#url": "https://www.windsorstore.com/collections/accessories-belts/products/rhine-buckle-dbl-o-ring-pu-strap-belt-073010158001", + "#category": ("shopify", "windsorstore", "product"), + "#class": shopify.ShopifyProductExtractor, + }, ) diff --git a/test/results/xbooru.py b/test/results/xbooru.py index 84cd7c7c15..32ac176b18 100644 --- a/test/results/xbooru.py +++ b/test/results/xbooru.py @@ -1,46 +1,39 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import gelbooru_v02 - __tests__ = ( -{ - "#url" : "https://xbooru.com/index.php?page=post&s=list&tags=konoyan", - "#category": ("gelbooru_v02", "xbooru", "tag"), - "#class" : gelbooru_v02.GelbooruV02TagExtractor, - "#count" : range(28, 40), -}, - -{ - "#url" : "https://xbooru.com/index.php?page=pool&s=show&id=757", - "#category": ("gelbooru_v02", "xbooru", "pool"), - "#class" : gelbooru_v02.GelbooruV02PoolExtractor, - "#urls": ( - "https://img.xbooru.com/images/154/aeca160f8c7131f6a93033adac5416d7.jpeg", - "https://img.xbooru.com/images/278/6185a8a71547568020e45e8319c02978.jpeg", - "https://img.xbooru.com/images/524/0fc2b1e2e3cc8be259e9712ca3f48b0b.jpeg", - "https://img.xbooru.com/images/253/74412b59a60fac5040c6cfe8efe7a625.jpeg", - "https://img.xbooru.com/images/590/2eacd900958a467fb053b8a92145b55b.jpeg", - ), -}, - -{ - "#url" : "https://xbooru.com/index.php?page=favorites&s=view&id=45206", - "#category": ("gelbooru_v02", "xbooru", "favorite"), - "#class" : gelbooru_v02.GelbooruV02FavoriteExtractor, - "#count" : 4, -}, - -{ - "#url" : "https://xbooru.com/index.php?page=post&s=view&id=1025649", - "#category": ("gelbooru_v02", "xbooru", "post"), - "#class" : gelbooru_v02.GelbooruV02PostExtractor, - "#pattern" : r"https://img\.xbooru\.com/images/444/f3eda549ad8b9db244ac335c7406c92f\.jpeg", - "#sha1_content": "086668afd445438d491ecc11cee3ac69b4d65530", -}, - + { + "#url": "https://xbooru.com/index.php?page=post&s=list&tags=konoyan", + "#category": ("gelbooru_v02", "xbooru", "tag"), + "#class": gelbooru_v02.GelbooruV02TagExtractor, + "#count": range(28, 40), + }, + { + "#url": "https://xbooru.com/index.php?page=pool&s=show&id=757", + "#category": ("gelbooru_v02", "xbooru", "pool"), + "#class": gelbooru_v02.GelbooruV02PoolExtractor, + "#urls": ( + "https://img.xbooru.com/images/154/aeca160f8c7131f6a93033adac5416d7.jpeg", + "https://img.xbooru.com/images/278/6185a8a71547568020e45e8319c02978.jpeg", + "https://img.xbooru.com/images/524/0fc2b1e2e3cc8be259e9712ca3f48b0b.jpeg", + "https://img.xbooru.com/images/253/74412b59a60fac5040c6cfe8efe7a625.jpeg", + "https://img.xbooru.com/images/590/2eacd900958a467fb053b8a92145b55b.jpeg", + ), + }, + { + "#url": "https://xbooru.com/index.php?page=favorites&s=view&id=45206", + "#category": ("gelbooru_v02", "xbooru", "favorite"), + "#class": gelbooru_v02.GelbooruV02FavoriteExtractor, + "#count": 4, + }, + { + "#url": "https://xbooru.com/index.php?page=post&s=view&id=1025649", + "#category": ("gelbooru_v02", "xbooru", "post"), + "#class": gelbooru_v02.GelbooruV02PostExtractor, + "#pattern": r"https://img\.xbooru\.com/images/444/f3eda549ad8b9db244ac335c7406c92f\.jpeg", + "#sha1_content": "086668afd445438d491ecc11cee3ac69b4d65530", + }, ) diff --git a/test/results/xhamster.py b/test/results/xhamster.py index 72634e1877..b84714a207 100644 --- a/test/results/xhamster.py +++ b/test/results/xhamster.py @@ -1,122 +1,106 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import xhamster - __tests__ = ( -{ - "#url" : "https://xhamster.com/photos/gallery/take-me-to-the-carwash-at-digitaldesire-15860946", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, - "#pattern" : r"https://ic-ph-\w+\.xhcdn\.com/a/\w+/webp/000/\d+/\d+/\d+_1000\.jpg$", - "#count" : 19, - - "comments": int, - "count" : int, - "favorite": bool, - "id" : int, - "num" : int, - "height" : int, - "width" : int, - "imageURL": str, - "pageURL" : str, - "thumbURL": str, - "gallery" : { - "date" : "dt:2022-02-02 06:30:09", - "description": "Alina Henessy loves to wash her car, and we love seeing every inch of her gorgeous body. More at DigitalDesire.com", - "dislikes" : int, - "id" : 15860946, - "likes" : int, - "tags" : [ - "Babe", - "Public Nudity", - "Take", - "Taking", - "Masturbation", - "Take Me", - ], - "thumbnail" : str, - "title" : "Take me to the carwash at DigitalDesire", - "views" : range(100000, 200000), - + { + "#url": "https://xhamster.com/photos/gallery/take-me-to-the-carwash-at-digitaldesire-15860946", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + "#pattern": r"https://ic-ph-\w+\.xhcdn\.com/a/\w+/webp/000/\d+/\d+/\d+_1000\.jpg$", + "#count": 19, + "comments": int, + "count": int, + "favorite": bool, + "id": int, + "num": int, + "height": int, + "width": int, + "imageURL": str, + "pageURL": str, + "thumbURL": str, + "gallery": { + "date": "dt:2022-02-02 06:30:09", + "description": "Alina Henessy loves to wash her car, and we love seeing every inch of her gorgeous body. More at DigitalDesire.com", + "dislikes": int, + "id": 15860946, + "likes": int, + "tags": [ + "Babe", + "Public Nudity", + "Take", + "Taking", + "Masturbation", + "Take Me", + ], + "thumbnail": str, + "title": "Take me to the carwash at DigitalDesire", + "views": range(100000, 200000), + }, + "user": { + "id": 4741860, + "name": "DaringSex", + "retired": False, + "subscribers": range(25000, 50000), + "url": "https://xhamster.com/users/daringsex", + "verified": False, + }, }, - "user" : { - "id" : 4741860, - "name" : "DaringSex", - "retired" : False, - "subscribers": range(25000, 50000), - "url" : "https://xhamster.com/users/daringsex", - "verified" : False, + { + "#url": "https://jp.xhamster2.com/photos/gallery/take-me-to-the-carwash-at-digitaldesire-15860946", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + "#pattern": r"https://ic-ph-\w+\.xhcdn\.com/a/\w+/webp/000/\d+/\d+/\d+_1000\.jpg$", + "#count": 19, + }, + { + "#url": "https://xhamster.com/photos/gallery/make-the-world-better-11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://xhamster.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://xhamster.one/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://xhamster.desi/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://xhamster2.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://en.xhamster.com/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://xhamster.porncache.net/photos/gallery/11748968", + "#category": ("", "xhamster", "gallery"), + "#class": xhamster.XhamsterGalleryExtractor, + }, + { + "#url": "https://xhamster.com/users/daringsex/photos", + "#category": ("", "xhamster", "user"), + "#class": xhamster.XhamsterUserExtractor, + "#pattern": xhamster.XhamsterGalleryExtractor.pattern, + "#range": "1-50", + "#count": 50, + }, + { + "#url": "https://xhamster.com/users/nickname68", + "#category": ("", "xhamster", "user"), + "#class": xhamster.XhamsterUserExtractor, }, -}, - -{ - "#url" : "https://jp.xhamster2.com/photos/gallery/take-me-to-the-carwash-at-digitaldesire-15860946", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, - "#pattern" : r"https://ic-ph-\w+\.xhcdn\.com/a/\w+/webp/000/\d+/\d+/\d+_1000\.jpg$", - "#count" : 19, -}, - -{ - "#url" : "https://xhamster.com/photos/gallery/make-the-world-better-11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://xhamster.com/photos/gallery/11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://xhamster.one/photos/gallery/11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://xhamster.desi/photos/gallery/11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://xhamster2.com/photos/gallery/11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://en.xhamster.com/photos/gallery/11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://xhamster.porncache.net/photos/gallery/11748968", - "#category": ("", "xhamster", "gallery"), - "#class" : xhamster.XhamsterGalleryExtractor, -}, - -{ - "#url" : "https://xhamster.com/users/daringsex/photos", - "#category": ("", "xhamster", "user"), - "#class" : xhamster.XhamsterUserExtractor, - "#pattern" : xhamster.XhamsterGalleryExtractor.pattern, - "#range" : "1-50", - "#count" : 50, -}, - -{ - "#url" : "https://xhamster.com/users/nickname68", - "#category": ("", "xhamster", "user"), - "#class" : xhamster.XhamsterUserExtractor, -}, - ) diff --git a/test/results/xvideos.py b/test/results/xvideos.py index 7cfa1dc014..45b064cb3f 100644 --- a/test/results/xvideos.py +++ b/test/results/xvideos.py @@ -1,83 +1,70 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import xvideos - __tests__ = ( -{ - "#url" : "https://www.xvideos.com/profiles/pervertedcouple/photos/751031", - "#category": ("", "xvideos", "gallery"), - "#class" : xvideos.XvideosGalleryExtractor, - "#pattern" : r"https://profile-pics-cdn\d+\.xvideos-cdn\.com/[^/]+\,\d+/videos/profiles/galleries/84/ca/37/pervertedcouple/gal751031/pic_\d+_big\.jpg", - "#count" : 8, - - "gallery": { - "id" : 751031, - "title": "Random Stuff", - "tags" : list, + { + "#url": "https://www.xvideos.com/profiles/pervertedcouple/photos/751031", + "#category": ("", "xvideos", "gallery"), + "#class": xvideos.XvideosGalleryExtractor, + "#pattern": r"https://profile-pics-cdn\d+\.xvideos-cdn\.com/[^/]+\,\d+/videos/profiles/galleries/84/ca/37/pervertedcouple/gal751031/pic_\d+_big\.jpg", + "#count": 8, + "gallery": { + "id": 751031, + "title": "Random Stuff", + "tags": list, + }, + "user": { + "id": 20245371, + "name": "pervertedcouple", + "display": "Pervertedcouple", + "sex": "Woman", + "description": str, + }, }, - "user" : { - "id" : 20245371, - "name" : "pervertedcouple", - "display" : "Pervertedcouple", - "sex" : "Woman", - "description": str, + { + "#url": "https://www.xvideos.com/amateur-channels/pervertedcouple/photos/12", + "#category": ("", "xvideos", "gallery"), + "#class": xvideos.XvideosGalleryExtractor, + }, + { + "#url": "https://www.xvideos.com/model-channels/pervertedcouple/photos/12", + "#category": ("", "xvideos", "gallery"), + "#class": xvideos.XvideosGalleryExtractor, + }, + { + "#url": "https://www.xvideos.com/channels/pervertedcouple/photos/12", + "#comment": "/channels/ URL (#5244)", + "#category": ("", "xvideos", "gallery"), + "#class": xvideos.XvideosGalleryExtractor, + }, + { + "#url": "https://www.xvideos.com/profiles/pervertedcouple", + "#category": ("", "xvideos", "user"), + "#class": xvideos.XvideosUserExtractor, + "#sha1_url": "a413f3e60d6d3a2de79bd44fa3b7a9c03db4336e", + "#sha1_metadata": "335a3304941ff2e666c0201e9122819b61b34adb", + }, + { + "#url": "https://www.xvideos.com/profiles/pervertedcouple#_tabPhotos", + "#category": ("", "xvideos", "user"), + "#class": xvideos.XvideosUserExtractor, + }, + { + "#url": "https://www.xvideos.com/channels/pervertedcouple", + "#category": ("", "xvideos", "user"), + "#class": xvideos.XvideosUserExtractor, + }, + { + "#url": "https://www.xvideos.com/amateur-channels/pervertedcouple", + "#category": ("", "xvideos", "user"), + "#class": xvideos.XvideosUserExtractor, + }, + { + "#url": "https://www.xvideos.com/model-channels/pervertedcouple", + "#category": ("", "xvideos", "user"), + "#class": xvideos.XvideosUserExtractor, }, -}, - -{ - "#url" : "https://www.xvideos.com/amateur-channels/pervertedcouple/photos/12", - "#category": ("", "xvideos", "gallery"), - "#class" : xvideos.XvideosGalleryExtractor, -}, - -{ - "#url" : "https://www.xvideos.com/model-channels/pervertedcouple/photos/12", - "#category": ("", "xvideos", "gallery"), - "#class" : xvideos.XvideosGalleryExtractor, -}, - -{ - "#url" : "https://www.xvideos.com/channels/pervertedcouple/photos/12", - "#comment" : "/channels/ URL (#5244)", - "#category": ("", "xvideos", "gallery"), - "#class" : xvideos.XvideosGalleryExtractor, -}, - -{ - "#url" : "https://www.xvideos.com/profiles/pervertedcouple", - "#category": ("", "xvideos", "user"), - "#class" : xvideos.XvideosUserExtractor, - "#sha1_url" : "a413f3e60d6d3a2de79bd44fa3b7a9c03db4336e", - "#sha1_metadata": "335a3304941ff2e666c0201e9122819b61b34adb", -}, - -{ - "#url" : "https://www.xvideos.com/profiles/pervertedcouple#_tabPhotos", - "#category": ("", "xvideos", "user"), - "#class" : xvideos.XvideosUserExtractor, -}, - -{ - "#url" : "https://www.xvideos.com/channels/pervertedcouple", - "#category": ("", "xvideos", "user"), - "#class" : xvideos.XvideosUserExtractor, -}, - -{ - "#url" : "https://www.xvideos.com/amateur-channels/pervertedcouple", - "#category": ("", "xvideos", "user"), - "#class" : xvideos.XvideosUserExtractor, -}, - -{ - "#url" : "https://www.xvideos.com/model-channels/pervertedcouple", - "#category": ("", "xvideos", "user"), - "#class" : xvideos.XvideosUserExtractor, -}, - ) diff --git a/test/results/yandere.py b/test/results/yandere.py index 74194bb4be..7220a9cbd7 100644 --- a/test/results/yandere.py +++ b/test/results/yandere.py @@ -1,99 +1,85 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import moebooru - __tests__ = ( -{ - "#url" : "https://yande.re/post/show/51824", - "#category": ("moebooru", "yandere", "post"), - "#class" : moebooru.MoebooruPostExtractor, - "#options" : {"tags": True}, - "#sha1_content": "59201811c728096b2d95ce6896fd0009235fe683", - - "tags_artist" : "sasaki_tamaru", - "tags_circle" : "softhouse_chara", - "tags_copyright": "ouzoku", - "tags_general" : str, -}, - -{ - "#url" : "https://yande.re/post/show/993156", - "#category": ("moebooru", "yandere", "post"), - "#class" : moebooru.MoebooruPostExtractor, - "#options" : {"notes": True}, - "#sha1_content": "fed722bd90f48de41ec163692befc701056e2b1e", - - "notes": [ - { - "id" : 7096, - "x" : 90, - "y" : 626, - "width" : 283, - "height": 529, - "body" : "Please keep this as a secret for me!!", - }, - { - "id" : 7095, - "x" : 900, - "y" : 438, - "width" : 314, - "height": 588, - "body" : "The facts that I love playing games", + { + "#url": "https://yande.re/post/show/51824", + "#category": ("moebooru", "yandere", "post"), + "#class": moebooru.MoebooruPostExtractor, + "#options": {"tags": True}, + "#sha1_content": "59201811c728096b2d95ce6896fd0009235fe683", + "tags_artist": "sasaki_tamaru", + "tags_circle": "softhouse_chara", + "tags_copyright": "ouzoku", + "tags_general": str, + }, + { + "#url": "https://yande.re/post/show/993156", + "#category": ("moebooru", "yandere", "post"), + "#class": moebooru.MoebooruPostExtractor, + "#options": {"notes": True}, + "#sha1_content": "fed722bd90f48de41ec163692befc701056e2b1e", + "notes": [ + { + "id": 7096, + "x": 90, + "y": 626, + "width": 283, + "height": 529, + "body": "Please keep this as a secret for me!!", + }, + { + "id": 7095, + "x": 900, + "y": 438, + "width": 314, + "height": 588, + "body": "The facts that I love playing games", + }, + ], + }, + { + "#url": "https://yande.re/post?tags=ouzoku+armor", + "#category": ("moebooru", "yandere", "tag"), + "#class": moebooru.MoebooruTagExtractor, + "#sha1_content": "59201811c728096b2d95ce6896fd0009235fe683", + }, + { + "#url": "https://yande.re/pool/show/318", + "#category": ("moebooru", "yandere", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + "#sha1_content": "2a35b9d6edecce11cc2918c6dce4de2198342b68", + }, + { + "#url": "https://yande.re/pool/show/318", + "#comment": "'metadata' option (#4646)", + "#category": ("moebooru", "yandere", "pool"), + "#class": moebooru.MoebooruPoolExtractor, + "#options": {"metadata": True}, + "#count": 3, + "pool": { + "created_at": "2008-12-13T15:56:10.728Z", + "description": "Dengeki Hime's posts are in pool #97.", + "id": 318, + "is_public": True, + "name": "Galgame_Mag_08", + "post_count": 3, + "updated_at": "2012-03-11T14:31:00.935Z", + "user_id": 1305, }, - ], -}, - -{ - "#url" : "https://yande.re/post?tags=ouzoku+armor", - "#category": ("moebooru", "yandere", "tag"), - "#class" : moebooru.MoebooruTagExtractor, - "#sha1_content": "59201811c728096b2d95ce6896fd0009235fe683", -}, - -{ - "#url" : "https://yande.re/pool/show/318", - "#category": ("moebooru", "yandere", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, - "#sha1_content": "2a35b9d6edecce11cc2918c6dce4de2198342b68", -}, - -{ - "#url" : "https://yande.re/pool/show/318", - "#comment" : "'metadata' option (#4646)", - "#category": ("moebooru", "yandere", "pool"), - "#class" : moebooru.MoebooruPoolExtractor, - "#options" : {"metadata": True}, - "#count" : 3, - - "pool": { - "created_at" : "2008-12-13T15:56:10.728Z", - "description": "Dengeki Hime's posts are in pool #97.", - "id" : 318, - "is_public" : True, - "name" : "Galgame_Mag_08", - "post_count" : 3, - "updated_at" : "2012-03-11T14:31:00.935Z", - "user_id" : 1305, }, - -}, - -{ - "#url" : "https://yande.re/post/popular_by_month?month=6&year=2014", - "#category": ("moebooru", "yandere", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, - "#count" : 40, -}, - -{ - "#url" : "https://yande.re/post/popular_recent", - "#category": ("moebooru", "yandere", "popular"), - "#class" : moebooru.MoebooruPopularExtractor, -}, - + { + "#url": "https://yande.re/post/popular_by_month?month=6&year=2014", + "#category": ("moebooru", "yandere", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + "#count": 40, + }, + { + "#url": "https://yande.re/post/popular_recent", + "#category": ("moebooru", "yandere", "popular"), + "#class": moebooru.MoebooruPopularExtractor, + }, ) diff --git a/test/results/ytdl.py b/test/results/ytdl.py index 1aecee91f0..e65171e23c 100644 --- a/test/results/ytdl.py +++ b/test/results/ytdl.py @@ -1,17 +1,13 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import ytdl - __tests__ = ( -{ - "#url" : "ytdl:https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9", - "#category": ("", "ytdl", "Youtube"), - "#class" : ytdl.YoutubeDLExtractor, -}, - + { + "#url": "ytdl:https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9", + "#category": ("", "ytdl", "Youtube"), + "#class": ytdl.YoutubeDLExtractor, + }, ) diff --git a/test/results/zerochan.py b/test/results/zerochan.py index ec8eddf619..09e31aecfe 100644 --- a/test/results/zerochan.py +++ b/test/results/zerochan.py @@ -1,191 +1,176 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import zerochan - __tests__ = ( -{ - "#url" : "https://www.zerochan.net/Perth+%28Kantai+Collection%29", - "#category": ("booru", "zerochan", "tag"), - "#class" : zerochan.ZerochanTagExtractor, - "#pattern" : r"https://static\.zerochan\.net/\.full\.\d+\.jpg", - "#count" : "> 50", - - "extension" : r"jpg", - "file_url" : r"re:https://static\.zerochan\.net/\.full\.\d+\.jpg", - "filename" : r"re:\.full\.\d+", - "height" : int, - "id" : int, - "search_tags": "Perth (Kantai Collection)", - "tag" : r"re:(Perth \(Kantai Collection\)|Kantai Collection)", - "tags" : list, - "width" : int, -}, - -{ - "#url" : "https://www.zerochan.net/Perth+%28Kantai+Collection%29", - "#category": ("booru", "zerochan", "tag"), - "#class" : zerochan.ZerochanTagExtractor, - "#options" : {"pagination": "html"}, - "#pattern" : r"https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", - "#count" : "> 45", - - "extension" : r"re:jpg|png", - "file_url" : r"re:https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", - "filename" : r"re:(Perth\.\(Kantai\.Collection\)|Kantai\.Collection)\.full\.\d+", - "height" : r"re:^\d+$", - "id" : r"re:^\d+$", - "name" : r"re:(Perth \(Kantai Collection\)|Kantai Collection)", - "search_tags": "Perth (Kantai Collection)", - "size" : r"re:^\d+k$", - "width" : r"re:^\d+$", -}, - -{ - "#url" : "https://www.zerochan.net/2920445", - "#category": ("booru", "zerochan", "image"), - "#class" : zerochan.ZerochanImageExtractor, - "#pattern" : r"https://static\.zerochan\.net/Perth\.%28Kantai\.Collection%29\.full.2920445\.jpg", - "#auth" : True, - - "author" : "YeFan 葉凡", - "date" : "dt:2020-04-24 21:33:44", - "file_url": "https://static.zerochan.net/Perth.%28Kantai.Collection%29.full.2920445.jpg", - "filename": "Perth.(Kantai.Collection).full.2920445", - "height" : 1366, - "id" : 2920445, - "path" : [ - "Kantai Collection", - "Perth (Kantai Collection)", - ], - "size" : 1975296, - "source" : "", - "tags" : [ - "Mangaka:YeFan 葉凡", - "Game:Kantai Collection", - "Character:Perth (Kantai Collection)", - "Theme:Blonde Hair", - "Theme:Braids", - "Theme:Coat", - "Theme:Female", - "Theme:Firefighter Outfit", - "Theme:Group", - "Theme:Long Sleeves", - "Theme:Personification", - "Theme:Pins", - "Theme:Ribbon", - "Theme:Short Hair", - "Theme:Top", - ], - "uploader": "YukinoTokisaki", - "width" : 1920, -}, - -{ - "#url" : "https://www.zerochan.net/2920445", - "#category": ("booru", "zerochan", "image"), - "#class" : zerochan.ZerochanImageExtractor, - "#pattern" : r"https://static\.zerochan\.net/Perth\.%28Kantai\.Collection%29\.full.2920445\.jpg", - "#auth" : False, - - "author" : "YeFan 葉凡", - "date" : "dt:2020-04-24 21:33:44", - "file_url": "https://static.zerochan.net/Perth.%28Kantai.Collection%29.full.2920445.jpg", - "filename": "Perth.(Kantai.Collection).full.2920445", - "height" : 1366, - "id" : 2920445, - "path" : [ - "Kantai Collection", - "Perth (Kantai Collection)", - ], - "size" : 1975296, - "source" : "", - "tags" : [ - "Mangaka:YeFan 葉凡", - "Game:Kantai Collection", - "Character:Perth (Kantai Collection)", - "Theme:Firefighter Outfit", - "Theme:Pins", - ], - "uploader": "YukinoTokisaki", - "width" : 1920, -}, - -{ - "#url" : "https://www.zerochan.net/4233756", - "#category": ("booru", "zerochan", "image"), - "#class" : zerochan.ZerochanImageExtractor, - "#urls" : "https://static.zerochan.net/DRAGON.BALL.full.4233756.jpg", - "#options" : {"tags": True}, - - "author" : "Raydash", - "date" : "dt:2024-07-23 00:10:51", - "extension": "jpg", - "file_url" : "https://static.zerochan.net/DRAGON.BALL.full.4233756.jpg", - "filename" : "DRAGON.BALL.full.4233756", - "height" : 1125, - "id" : 4233756, - "path" : [ - "Toriyama Akira", - "DRAGON BALL", - ], - "size" : 136192, - "source": "https://x.com/Raydash30/status/1766012730769862774", - "tags" : [ - "Mangaka:Raydash", - "Series:DRAGON BALL", - "Series:DRAGON BALL Z", - "Character:Piccolo", - "Character:Son Gohan", - "Theme:Duo", - "Theme:Green Skin", - "Theme:Male", - "Theme:Male Focus", - "Theme:Two Males", - "Source:Fanart", - "Source:Fanart from X (Twitter)", - "Source:X (Twitter)", - ], - "tags_character": [ - "Piccolo", - "Son Gohan", - ], - "tags_mangaka" : [ - "Raydash", - ], - "tags_series" : [ - "DRAGON BALL", - "DRAGON BALL Z", - ], - "tags_source" : [ - "Fanart", - "Fanart from X (Twitter)", - "X (Twitter)", - ], - "tags_theme" : [ - "Duo", - "Green Skin", - "Male", - "Male Focus", - "Two Males", - ], - "uploader" : "menotbug", - "width" : 750, -}, - -{ - "#url" : "https://www.zerochan.net/1395035", - "#comment" : "Invalid control character '\r' in 'source' field (#5892)", - "#category": ("booru", "zerochan", "image"), - "#class" : zerochan.ZerochanImageExtractor, - "#auth" : True, - "#options" : {"metadata": True}, - - "source": "http://www.youtube.com/watch?v=0vodqkGPxt8", -}, - + { + "#url": "https://www.zerochan.net/Perth+%28Kantai+Collection%29", + "#category": ("booru", "zerochan", "tag"), + "#class": zerochan.ZerochanTagExtractor, + "#pattern": r"https://static\.zerochan\.net/\.full\.\d+\.jpg", + "#count": "> 50", + "extension": r"jpg", + "file_url": r"re:https://static\.zerochan\.net/\.full\.\d+\.jpg", + "filename": r"re:\.full\.\d+", + "height": int, + "id": int, + "search_tags": "Perth (Kantai Collection)", + "tag": r"re:(Perth \(Kantai Collection\)|Kantai Collection)", + "tags": list, + "width": int, + }, + { + "#url": "https://www.zerochan.net/Perth+%28Kantai+Collection%29", + "#category": ("booru", "zerochan", "tag"), + "#class": zerochan.ZerochanTagExtractor, + "#options": {"pagination": "html"}, + "#pattern": r"https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", + "#count": "> 45", + "extension": r"re:jpg|png", + "file_url": r"re:https://static\.zerochan\.net/.+\.full\.\d+\.(jpg|png)", + "filename": r"re:(Perth\.\(Kantai\.Collection\)|Kantai\.Collection)\.full\.\d+", + "height": r"re:^\d+$", + "id": r"re:^\d+$", + "name": r"re:(Perth \(Kantai Collection\)|Kantai Collection)", + "search_tags": "Perth (Kantai Collection)", + "size": r"re:^\d+k$", + "width": r"re:^\d+$", + }, + { + "#url": "https://www.zerochan.net/2920445", + "#category": ("booru", "zerochan", "image"), + "#class": zerochan.ZerochanImageExtractor, + "#pattern": r"https://static\.zerochan\.net/Perth\.%28Kantai\.Collection%29\.full.2920445\.jpg", + "#auth": True, + "author": "YeFan 葉凡", + "date": "dt:2020-04-24 21:33:44", + "file_url": "https://static.zerochan.net/Perth.%28Kantai.Collection%29.full.2920445.jpg", + "filename": "Perth.(Kantai.Collection).full.2920445", + "height": 1366, + "id": 2920445, + "path": [ + "Kantai Collection", + "Perth (Kantai Collection)", + ], + "size": 1975296, + "source": "", + "tags": [ + "Mangaka:YeFan 葉凡", + "Game:Kantai Collection", + "Character:Perth (Kantai Collection)", + "Theme:Blonde Hair", + "Theme:Braids", + "Theme:Coat", + "Theme:Female", + "Theme:Firefighter Outfit", + "Theme:Group", + "Theme:Long Sleeves", + "Theme:Personification", + "Theme:Pins", + "Theme:Ribbon", + "Theme:Short Hair", + "Theme:Top", + ], + "uploader": "YukinoTokisaki", + "width": 1920, + }, + { + "#url": "https://www.zerochan.net/2920445", + "#category": ("booru", "zerochan", "image"), + "#class": zerochan.ZerochanImageExtractor, + "#pattern": r"https://static\.zerochan\.net/Perth\.%28Kantai\.Collection%29\.full.2920445\.jpg", + "#auth": False, + "author": "YeFan 葉凡", + "date": "dt:2020-04-24 21:33:44", + "file_url": "https://static.zerochan.net/Perth.%28Kantai.Collection%29.full.2920445.jpg", + "filename": "Perth.(Kantai.Collection).full.2920445", + "height": 1366, + "id": 2920445, + "path": [ + "Kantai Collection", + "Perth (Kantai Collection)", + ], + "size": 1975296, + "source": "", + "tags": [ + "Mangaka:YeFan 葉凡", + "Game:Kantai Collection", + "Character:Perth (Kantai Collection)", + "Theme:Firefighter Outfit", + "Theme:Pins", + ], + "uploader": "YukinoTokisaki", + "width": 1920, + }, + { + "#url": "https://www.zerochan.net/4233756", + "#category": ("booru", "zerochan", "image"), + "#class": zerochan.ZerochanImageExtractor, + "#urls": "https://static.zerochan.net/DRAGON.BALL.full.4233756.jpg", + "#options": {"tags": True}, + "author": "Raydash", + "date": "dt:2024-07-23 00:10:51", + "extension": "jpg", + "file_url": "https://static.zerochan.net/DRAGON.BALL.full.4233756.jpg", + "filename": "DRAGON.BALL.full.4233756", + "height": 1125, + "id": 4233756, + "path": [ + "Toriyama Akira", + "DRAGON BALL", + ], + "size": 136192, + "source": "https://x.com/Raydash30/status/1766012730769862774", + "tags": [ + "Mangaka:Raydash", + "Series:DRAGON BALL", + "Series:DRAGON BALL Z", + "Character:Piccolo", + "Character:Son Gohan", + "Theme:Duo", + "Theme:Green Skin", + "Theme:Male", + "Theme:Male Focus", + "Theme:Two Males", + "Source:Fanart", + "Source:Fanart from X (Twitter)", + "Source:X (Twitter)", + ], + "tags_character": [ + "Piccolo", + "Son Gohan", + ], + "tags_mangaka": [ + "Raydash", + ], + "tags_series": [ + "DRAGON BALL", + "DRAGON BALL Z", + ], + "tags_source": [ + "Fanart", + "Fanart from X (Twitter)", + "X (Twitter)", + ], + "tags_theme": [ + "Duo", + "Green Skin", + "Male", + "Male Focus", + "Two Males", + ], + "uploader": "menotbug", + "width": 750, + }, + { + "#url": "https://www.zerochan.net/1395035", + "#comment": "Invalid control character '\r' in 'source' field (#5892)", + "#category": ("booru", "zerochan", "image"), + "#class": zerochan.ZerochanImageExtractor, + "#auth": True, + "#options": {"metadata": True}, + "source": "http://www.youtube.com/watch?v=0vodqkGPxt8", + }, ) diff --git a/test/results/zzup.py b/test/results/zzup.py index 322a9601b6..c60a515257 100644 --- a/test/results/zzup.py +++ b/test/results/zzup.py @@ -1,36 +1,29 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import zzup - __tests__ = ( -{ - "#url" : "https://zzup.com/content/NjM=/MetArt_20080206_viki_c_sensazioni_by_ingret/OTE=/index.html", - "#category": ("", "zzup", "gallery"), - "#class" : zzup.ZzupGalleryExtractor, - "#pattern" : r"https://zzup\.com/MjAxNjc3OTIyMjE5Nzk=/showimage/zzup-8769086487/image00\d\d\d-5896498214-1-9689595623/MetArt-20080206_viki_c_sensazioni_by_ingret/9879560327/zzup.com.jpg", - - "slug" : "MetArt_20080206_viki_c_sensazioni_by_ingret", - "title" : "MetArt 20080206 viki c sensazioni by ingret", - "num" : int, - "count" : 135, -}, - -{ - "#url" : "https://zzup.com/content/MTc2MDYxMw==/Courtesan/NDA=/page-1.html", - "#category": ("", "zzup", "gallery"), - "#class" : zzup.ZzupGalleryExtractor, - "#pattern" : r"https://zzup.com/MjAxNjc3OTIyMjE5Nzk=/showimage/zzup-8769086487/image000\d\d-5896498214-40-9689595623/Courtesan/9879560327/zzup.com.jpg", -}, - -{ - "#url" : "https://up.zzup.com/viewalbum/TE9MQUxVWlogLSBMYWxsaSAtIFdhcm0gYW5kIENvenk=/NTM0MTk=/OTgz/index.html", - "#category": ("", "zzup", "gallery"), - "#class" : zzup.ZzupGalleryExtractor, -}, - + { + "#url": "https://zzup.com/content/NjM=/MetArt_20080206_viki_c_sensazioni_by_ingret/OTE=/index.html", + "#category": ("", "zzup", "gallery"), + "#class": zzup.ZzupGalleryExtractor, + "#pattern": r"https://zzup\.com/MjAxNjc3OTIyMjE5Nzk=/showimage/zzup-8769086487/image00\d\d\d-5896498214-1-9689595623/MetArt-20080206_viki_c_sensazioni_by_ingret/9879560327/zzup.com.jpg", + "slug": "MetArt_20080206_viki_c_sensazioni_by_ingret", + "title": "MetArt 20080206 viki c sensazioni by ingret", + "num": int, + "count": 135, + }, + { + "#url": "https://zzup.com/content/MTc2MDYxMw==/Courtesan/NDA=/page-1.html", + "#category": ("", "zzup", "gallery"), + "#class": zzup.ZzupGalleryExtractor, + "#pattern": r"https://zzup.com/MjAxNjc3OTIyMjE5Nzk=/showimage/zzup-8769086487/image000\d\d-5896498214-40-9689595623/Courtesan/9879560327/zzup.com.jpg", + }, + { + "#url": "https://up.zzup.com/viewalbum/TE9MQUxVWlogLSBMYWxsaSAtIFdhcm0gYW5kIENvenk=/NTM0MTk=/OTgz/index.html", + "#category": ("", "zzup", "gallery"), + "#class": zzup.ZzupGalleryExtractor, + }, ) diff --git a/test/test_cache.py b/test/test_cache.py index 9951ef2039..f3d5dd6568 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2020 Mike Fährmann # @@ -9,17 +8,17 @@ import os import sys +import tempfile import unittest from unittest.mock import patch -import tempfile - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import config, util # noqa E402 dbpath = tempfile.mkstemp()[1] config.set(("cache",), "file", dbpath) -from gallery_dl import cache # noqa E402 +from gallery_dl import cache + cache._init() @@ -28,9 +27,7 @@ class TestCache(unittest.TestCase): - def test_decorator(self): - @cache.memcache() def mc1(): pass @@ -50,7 +47,7 @@ def dbc(): def test_keyarg_mem_simple(self): @cache.memcache(keyarg=2) def ka(a, b, c): - return a+b+c + return a + b + c self.assertEqual(ka(1, 1, 1), 3) self.assertEqual(ka(2, 2, 2), 6) @@ -63,7 +60,7 @@ def ka(a, b, c): def test_keyarg_mem(self): @cache.memcache(keyarg=2, maxage=10) def ka(a, b, c): - return a+b+c + return a + b + c self.assertEqual(ka(1, 1, 1), 3) self.assertEqual(ka(2, 2, 2), 6) @@ -76,7 +73,7 @@ def ka(a, b, c): def test_keyarg_db(self): @cache.cache(keyarg=2, maxage=10) def ka(a, b, c): - return a+b+c + return a + b + c self.assertEqual(ka(1, 1, 1), 3) self.assertEqual(ka(2, 2, 2), 6) @@ -89,7 +86,7 @@ def ka(a, b, c): def test_expires_mem(self): @cache.memcache(maxage=2) def ex(a, b, c): - return a+b+c + return a + b + c with patch("time.time") as tmock: tmock.return_value = 0.001 @@ -112,7 +109,7 @@ def ex(a, b, c): def test_expires_db(self): @cache.cache(maxage=2) def ex(a, b, c): - return a+b+c + return a + b + c with patch("time.time") as tmock: tmock.return_value = 0.999 @@ -135,7 +132,7 @@ def ex(a, b, c): def test_update_mem_simple(self): @cache.memcache(keyarg=0) def up(a, b, c): - return a+b+c + return a + b + c self.assertEqual(up(1, 1, 1), 3) up.update(1, 0) @@ -146,7 +143,7 @@ def up(a, b, c): def test_update_mem(self): @cache.memcache(keyarg=0, maxage=10) def up(a, b, c): - return a+b+c + return a + b + c self.assertEqual(up(1, 1, 1), 3) up.update(1, 0) @@ -157,7 +154,7 @@ def up(a, b, c): def test_update_db(self): @cache.cache(keyarg=0, maxage=10) def up(a, b, c): - return a+b+c + return a + b + c self.assertEqual(up(1, 1, 1), 3) up.update(1, 0) @@ -168,7 +165,7 @@ def up(a, b, c): def test_invalidate_mem_simple(self): @cache.memcache(keyarg=0) def inv(a, b, c): - return a+b+c + return a + b + c self.assertEqual(inv(1, 1, 1), 3) inv.invalidate(1) @@ -179,7 +176,7 @@ def inv(a, b, c): def test_invalidate_mem(self): @cache.memcache(keyarg=0, maxage=10) def inv(a, b, c): - return a+b+c + return a + b + c self.assertEqual(inv(1, 1, 1), 3) inv.invalidate(1) @@ -190,7 +187,7 @@ def inv(a, b, c): def test_invalidate_db(self): @cache.cache(keyarg=0, maxage=10) def inv(a, b, c): - return a+b+c + return a + b + c self.assertEqual(inv(1, 1, 1), 3) inv.invalidate(1) @@ -201,7 +198,7 @@ def inv(a, b, c): def test_database_read(self): @cache.cache(keyarg=0, maxage=10) def db(a, b, c): - return a+b+c + return a + b + c # initialize cache self.assertEqual(db(1, 1, 1), 3) diff --git a/test/test_config.py b/test/test_config.py index bbe288ff32..0c25c1526f 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2015-2023 Mike Fährmann # @@ -9,9 +8,8 @@ import os import sys -import unittest - import tempfile +import unittest ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOTDIR) @@ -19,63 +17,64 @@ class TestConfig(unittest.TestCase): - def setUp(self): - config.set(() , "a", 1) - config.set(("b",) , "a", 2) + config.set((), "a", 1) + config.set(("b",), "a", 2) config.set(("b", "b"), "a", 3) - config.set(("b",) , "c", "text") + config.set(("b",), "c", "text") config.set(("b", "b"), "c", [8, 9]) def tearDown(self): config.clear() def test_get(self): - self.assertEqual(config.get(() , "a") , 1) - self.assertEqual(config.get(("b",) , "a") , 2) - self.assertEqual(config.get(("b", "b"), "a") , 3) + self.assertEqual(config.get((), "a"), 1) + self.assertEqual(config.get(("b",), "a"), 2) + self.assertEqual(config.get(("b", "b"), "a"), 3) - self.assertEqual(config.get(() , "c") , None) - self.assertEqual(config.get(("b",) , "c") , "text") - self.assertEqual(config.get(("b", "b"), "c") , [8, 9]) + self.assertEqual(config.get((), "c"), None) + self.assertEqual(config.get(("b",), "c"), "text") + self.assertEqual(config.get(("b", "b"), "c"), [8, 9]) - self.assertEqual(config.get(("a",) , "g") , None) - self.assertEqual(config.get(("a", "a"), "g") , None) - self.assertEqual(config.get(("e", "f"), "g") , None) + self.assertEqual(config.get(("a",), "g"), None) + self.assertEqual(config.get(("a", "a"), "g"), None) + self.assertEqual(config.get(("e", "f"), "g"), None) self.assertEqual(config.get(("e", "f"), "g", 4), 4) def test_interpolate(self): - self.assertEqual(config.interpolate(() , "a"), 1) - self.assertEqual(config.interpolate(("b",) , "a"), 1) + self.assertEqual(config.interpolate((), "a"), 1) + self.assertEqual(config.interpolate(("b",), "a"), 1) self.assertEqual(config.interpolate(("b", "b"), "a"), 1) - self.assertEqual(config.interpolate(() , "c"), None) - self.assertEqual(config.interpolate(("b",) , "c"), "text") + self.assertEqual(config.interpolate((), "c"), None) + self.assertEqual(config.interpolate(("b",), "c"), "text") self.assertEqual(config.interpolate(("b", "b"), "c"), [8, 9]) - self.assertEqual(config.interpolate(("a",) , "g") , None) - self.assertEqual(config.interpolate(("a", "a"), "g") , None) - self.assertEqual(config.interpolate(("e", "f"), "g") , None) + self.assertEqual(config.interpolate(("a",), "g"), None) + self.assertEqual(config.interpolate(("a", "a"), "g"), None) + self.assertEqual(config.interpolate(("e", "f"), "g"), None) self.assertEqual(config.interpolate(("e", "f"), "g", 4), 4) - self.assertEqual(config.interpolate(("b",), "d", 1) , 1) - self.assertEqual(config.interpolate(("d",), "d", 1) , 1) - config.set(() , "d", 2) - self.assertEqual(config.interpolate(("b",), "d", 1) , 2) - self.assertEqual(config.interpolate(("d",), "d", 1) , 2) + self.assertEqual(config.interpolate(("b",), "d", 1), 1) + self.assertEqual(config.interpolate(("d",), "d", 1), 1) + config.set((), "d", 2) + self.assertEqual(config.interpolate(("b",), "d", 1), 2) + self.assertEqual(config.interpolate(("d",), "d", 1), 2) config.set(("b",), "d", 3) - self.assertEqual(config.interpolate(("b",), "d", 1) , 2) - self.assertEqual(config.interpolate(("d",), "d", 1) , 2) + self.assertEqual(config.interpolate(("b",), "d", 1), 2) + self.assertEqual(config.interpolate(("d",), "d", 1), 2) def test_interpolate_common(self): - def lookup(): return config.interpolate_common( - ("Z1", "Z2"), ( + ("Z1", "Z2"), + ( ("A1", "A2"), ("B1",), ("C1", "C2", "C3"), - ), "KEY", "DEFAULT", + ), + "KEY", + "DEFAULT", ) def test(path, value, expected=None): @@ -98,48 +97,42 @@ def test(path, value, expected=None): def test_accumulate(self): self.assertEqual(config.accumulate((), "l"), []) - config.set(() , "l", [5, 6]) - config.set(("c",) , "l", [3, 4]) + config.set((), "l", [5, 6]) + config.set(("c",), "l", [3, 4]) config.set(("c", "c"), "l", [1, 2]) - self.assertEqual( - config.accumulate((), "l") , [5, 6]) - self.assertEqual( - config.accumulate(("c",), "l") , [3, 4, 5, 6]) - self.assertEqual( - config.accumulate(("c", "c"), "l"), [1, 2, 3, 4, 5, 6]) + self.assertEqual(config.accumulate((), "l"), [5, 6]) + self.assertEqual(config.accumulate(("c",), "l"), [3, 4, 5, 6]) + self.assertEqual(config.accumulate(("c", "c"), "l"), [1, 2, 3, 4, 5, 6]) config.set(("c",), "l", None) config.unset(("c", "c"), "l") - self.assertEqual( - config.accumulate((), "l") , [5, 6]) - self.assertEqual( - config.accumulate(("c",), "l") , [5, 6]) - self.assertEqual( - config.accumulate(("c", "c"), "l"), [5, 6]) + self.assertEqual(config.accumulate((), "l"), [5, 6]) + self.assertEqual(config.accumulate(("c",), "l"), [5, 6]) + self.assertEqual(config.accumulate(("c", "c"), "l"), [5, 6]) def test_set(self): - config.set(() , "c", [1, 2, 3]) - config.set(("b",) , "c", [1, 2, 3]) + config.set((), "c", [1, 2, 3]) + config.set(("b",), "c", [1, 2, 3]) config.set(("e", "f"), "g", value=234) - self.assertEqual(config.get(() , "c"), [1, 2, 3]) - self.assertEqual(config.get(("b",) , "c"), [1, 2, 3]) + self.assertEqual(config.get((), "c"), [1, 2, 3]) + self.assertEqual(config.get(("b",), "c"), [1, 2, 3]) self.assertEqual(config.get(("e", "f"), "g"), 234) def test_setdefault(self): - config.setdefault(() , "c", [1, 2, 3]) - config.setdefault(("b",) , "c", [1, 2, 3]) + config.setdefault((), "c", [1, 2, 3]) + config.setdefault(("b",), "c", [1, 2, 3]) config.setdefault(("e", "f"), "g", value=234) - self.assertEqual(config.get(() , "c"), [1, 2, 3]) - self.assertEqual(config.get(("b",) , "c"), "text") + self.assertEqual(config.get((), "c"), [1, 2, 3]) + self.assertEqual(config.get(("b",), "c"), "text") self.assertEqual(config.get(("e", "f"), "g"), 234) def test_unset(self): - config.unset(() , "a") + config.unset((), "a") config.unset(("b",), "c") config.unset(("a",), "d") config.unset(("b",), "d") config.unset(("c",), "d") - self.assertEqual(config.get(() , "a"), None) + self.assertEqual(config.get((), "a"), None) self.assertEqual(config.get(("b",), "a"), 2) self.assertEqual(config.get(("b",), "c"), None) self.assertEqual(config.get(("a",), "d"), None) @@ -148,18 +141,18 @@ def test_unset(self): def test_apply(self): options = ( - (("b",) , "c", [1, 2, 3]), + (("b",), "c", [1, 2, 3]), (("e", "f"), "g", 234), ) - self.assertEqual(config.get(("b",) , "c"), "text") + self.assertEqual(config.get(("b",), "c"), "text") self.assertEqual(config.get(("e", "f"), "g"), None) with config.apply(options): - self.assertEqual(config.get(("b",) , "c"), [1, 2, 3]) + self.assertEqual(config.get(("b",), "c"), [1, 2, 3]) self.assertEqual(config.get(("e", "f"), "g"), 234) - self.assertEqual(config.get(("b",) , "c"), "text") + self.assertEqual(config.get(("b",), "c"), "text") self.assertEqual(config.get(("e", "f"), "g"), None) def test_load(self): @@ -174,26 +167,25 @@ def test_load(self): config.clear() config.load((path1,)) - self.assertEqual(config.get(() , "a"), 1) + self.assertEqual(config.get((), "a"), 1) self.assertEqual(config.get(("b",), "a"), 2) self.assertEqual(config.get(("b",), "c"), "text") config.load((path2,)) - self.assertEqual(config.get(() , "a"), 7) + self.assertEqual(config.get((), "a"), 7) self.assertEqual(config.get(("b",), "a"), 8) self.assertEqual(config.get(("b",), "c"), "text") self.assertEqual(config.get(("b",), "e"), "foo") config.clear() config.load((path1, path2)) - self.assertEqual(config.get(() , "a"), 7) + self.assertEqual(config.get((), "a"), 7) self.assertEqual(config.get(("b",), "a"), 8) self.assertEqual(config.get(("b",), "c"), "text") self.assertEqual(config.get(("b",), "e"), "foo") class TestConfigFiles(unittest.TestCase): - def test_default_config(self): cfg = self._load("gallery-dl.conf") self.assertIsInstance(cfg, dict) diff --git a/test/test_cookies.py b/test/test_cookies.py index 60c83ffb1b..5443a36c27 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2017-2023 Mike Fährmann # @@ -7,22 +6,20 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import logging import os import sys -import unittest -from unittest import mock - -import time -import logging import tempfile +import time +import unittest from os.path import join +from unittest import mock sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import config, extractor # noqa E402 class TestCookiejar(unittest.TestCase): - @classmethod def setUpClass(cls): cls.path = tempfile.TemporaryDirectory() @@ -51,9 +48,9 @@ def test_cookiefile(self): cookie = next(iter(cookies)) self.assertEqual(cookie.domain, ".example.org") - self.assertEqual(cookie.path , "/") - self.assertEqual(cookie.name , "NAME") - self.assertEqual(cookie.value , "VALUE") + self.assertEqual(cookie.path, "/") + self.assertEqual(cookie.name, "NAME") + self.assertEqual(cookie.value, "VALUE") def test_invalid_cookiefile(self): self._test_warning(self.invalid_cookiefile, ValueError) @@ -75,7 +72,6 @@ def _test_warning(self, filename, exc): class TestCookiedict(unittest.TestCase): - def setUp(self): self.cdict = {"NAME1": "VALUE1", "NAME2": "VALUE2"} config.set((), "cookies", self.cdict) @@ -101,16 +97,15 @@ def test_domain(self): class TestCookieLogin(unittest.TestCase): - def tearDown(self): config.clear() def test_cookie_login(self): extr_cookies = { - "exhentai" : ("ipb_member_id", "ipb_pass_hash"), + "exhentai": ("ipb_member_id", "ipb_pass_hash"), "idolcomplex": ("login", "pass_hash"), - "nijie" : ("nijie_tok",), - "horne" : ("horne_tok",), + "nijie": ("nijie_tok",), + "horne": ("horne_tok",), } for category, cookienames in extr_cookies.items(): cookies = {name: "value" for name in cookienames} @@ -122,7 +117,6 @@ def test_cookie_login(self): class TestCookieUtils(unittest.TestCase): - def test_check_cookies(self): extr = _get_extractor("test") self.assertFalse(extr.cookies, "empty") @@ -175,27 +169,29 @@ def test_check_cookies_expires(self): now = int(time.time()) log = logging.getLogger("generic") - extr.cookies.set("a", "1", expires=now-100) + extr.cookies.set("a", "1", expires=now - 100) with mock.patch.object(log, "warning") as mw: self.assertFalse(extr.cookies_check(("a",))) self.assertEqual(mw.call_count, 1) self.assertEqual(mw.call_args[0], ("Cookie '%s' has expired", "a")) - extr.cookies.set("a", "1", expires=now+100) + extr.cookies.set("a", "1", expires=now + 100) with mock.patch.object(log, "warning") as mw: self.assertTrue(extr.cookies_check(("a",))) self.assertEqual(mw.call_count, 1) - self.assertEqual(mw.call_args[0], ( - "Cookie '%s' will expire in less than %s hour%s", "a", 1, "")) + self.assertEqual( + mw.call_args[0], ("Cookie '%s' will expire in less than %s hour%s", "a", 1, "") + ) - extr.cookies.set("a", "1", expires=now+100+7200) + extr.cookies.set("a", "1", expires=now + 100 + 7200) with mock.patch.object(log, "warning") as mw: self.assertTrue(extr.cookies_check(("a",))) self.assertEqual(mw.call_count, 1) - self.assertEqual(mw.call_args[0], ( - "Cookie '%s' will expire in less than %s hour%s", "a", 3, "s")) + self.assertEqual( + mw.call_args[0], ("Cookie '%s' will expire in less than %s hour%s", "a", 3, "s") + ) - extr.cookies.set("a", "1", expires=now+100+24*3600) + extr.cookies.set("a", "1", expires=now + 100 + 24 * 3600) with mock.patch.object(log, "warning") as mw: self.assertTrue(extr.cookies_check(("a",))) self.assertEqual(mw.call_count, 0) @@ -208,11 +204,11 @@ def _get_extractor(category): URLS = { - "exhentai" : "https://exhentai.org/g/1200119/d55c44d3d0/", + "exhentai": "https://exhentai.org/g/1200119/d55c44d3d0/", "idolcomplex": "https://idol.sankakucomplex.com/post/show/1", - "nijie" : "https://nijie.info/view.php?id=1", - "horne" : "https://horne.red/view.php?id=1", - "test" : "generic:https://example.org/", + "nijie": "https://nijie.info/view.php?id=1", + "horne": "https://horne.red/view.php?id=1", + "test": "generic:https://example.org/", } diff --git a/test/test_downloader.py b/test/test_downloader.py index 35cccc4027..490e87f191 100644 --- a/test/test_downloader.py +++ b/test/test_downloader.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2018-2022 Mike Fährmann # @@ -7,31 +6,30 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import os -import sys -import unittest -from unittest.mock import Mock, MagicMock, patch - -import re +import binascii +import http.server import logging +import os import os.path -import binascii +import re +import sys import tempfile import threading -import http.server - +import unittest +from unittest.mock import MagicMock +from unittest.mock import Mock +from unittest.mock import patch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import downloader, extractor, output, config, path # noqa E402 -from gallery_dl.downloader.http import MIME_TYPES, SIGNATURE_CHECKS # noqa E402 +from gallery_dl.downloader.http import MIME_TYPES, SIGNATURE_CHECKS # noqa E402 class MockDownloaderModule(Mock): __downloader__ = "mock" -class FakeJob(): - +class FakeJob: def __init__(self): self.extractor = extractor.find("generic:https://example.org/") self.extractor.initialize() @@ -41,7 +39,6 @@ def __init__(self): class TestDownloaderModule(unittest.TestCase): - @classmethod def setUpClass(cls): # allow import of ytdl downloader module without youtube_dl installed @@ -61,24 +58,24 @@ def tearDown(self): def test_find(self): cls = downloader.find("http") self.assertEqual(cls.__name__, "HttpDownloader") - self.assertEqual(cls.scheme , "http") + self.assertEqual(cls.scheme, "http") cls = downloader.find("https") self.assertEqual(cls.__name__, "HttpDownloader") - self.assertEqual(cls.scheme , "http") + self.assertEqual(cls.scheme, "http") cls = downloader.find("text") self.assertEqual(cls.__name__, "TextDownloader") - self.assertEqual(cls.scheme , "text") + self.assertEqual(cls.scheme, "text") cls = downloader.find("ytdl") self.assertEqual(cls.__name__, "YoutubeDLDownloader") - self.assertEqual(cls.scheme , "ytdl") + self.assertEqual(cls.scheme, "ytdl") self.assertEqual(downloader.find("ftp"), None) self.assertEqual(downloader.find("foo"), None) - self.assertEqual(downloader.find(1234) , None) - self.assertEqual(downloader.find(None) , None) + self.assertEqual(downloader.find(1234), None) + self.assertEqual(downloader.find(None), None) @patch("builtins.__import__") def test_cache(self, import_module): @@ -108,7 +105,6 @@ def test_cache_https(self, import_module): class TestDownloaderBase(unittest.TestCase): - @classmethod def setUpClass(cls): cls.dir = tempfile.TemporaryDirectory() @@ -123,14 +119,14 @@ def tearDownClass(cls): @classmethod def _prepare_destination(cls, content=None, part=True, extension=None): - name = "file-{}".format(cls.fnum) + name = f"file-{cls.fnum}" cls.fnum += 1 kwdict = { - "category" : "test", + "category": "test", "subcategory": "test", - "filename" : name, - "extension" : extension, + "filename": name, + "extension": extension, } pathfmt = cls.job.pathfmt @@ -145,13 +141,12 @@ def _prepare_destination(cls, content=None, part=True, extension=None): return pathfmt - def _run_test(self, url, input, output, - extension, expected_extension=None): + def _run_test(self, url, input, output, extension, expected_extension=None): pathfmt = self._prepare_destination(input, extension=extension) success = self.downloader.download(url, pathfmt) # test successful download - self.assertTrue(success, "downloading '{}' failed".format(url)) + self.assertTrue(success, f"downloading '{url}' failed") # test content mode = "r" + ("b" if isinstance(output, bytes) else "") @@ -172,7 +167,6 @@ def _run_test(self, url, input, output, class TestHTTPDownloader(TestDownloaderBase): - @classmethod def setUpClass(cls): TestDownloaderBase.setUpClass() @@ -184,18 +178,16 @@ def setUpClass(cls): try: server = http.server.HTTPServer((host, port), HttpRequestHandler) except OSError as exc: - raise unittest.SkipTest( - "cannot spawn local HTTP server ({})".format(exc)) + raise unittest.SkipTest(f"cannot spawn local HTTP server ({exc})") host, port = server.server_address - cls.address = "http://{}:{}".format(host, port) + cls.address = f"http://{host}:{port}" threading.Thread(target=server.serve_forever, daemon=True).start() - def _run_test(self, ext, input, output, - extension, expected_extension=None): + def _run_test(self, ext, input, output, extension, expected_extension=None): TestDownloaderBase._run_test( - self, self.address + "/" + ext, input, output, - extension, expected_extension) + self, self.address + "/" + ext, input, output, extension, expected_extension + ) def tearDown(self): self.downloader.minsize = self.downloader.maxsize = None @@ -207,8 +199,8 @@ def test_http_download(self): def test_http_offset(self): self._run_test("jpg", DATA["jpg"][:123], DATA["jpg"], "jpg", "jpg") - self._run_test("png", DATA["png"][:12] , DATA["png"], "png", "png") - self._run_test("gif", DATA["gif"][:1] , DATA["gif"], "gif", "gif") + self._run_test("png", DATA["png"][:12], DATA["png"], "png", "png") + self._run_test("gif", DATA["gif"][:1], DATA["gif"], "gif", "gif") def test_http_extension(self): self._run_test("jpg", None, DATA["jpg"], None, "jpg") @@ -240,7 +232,6 @@ def test_http_filesize_max(self): class TestTextDownloader(TestDownloaderBase): - @classmethod def setUpClass(cls): TestDownloaderBase.setUpClass() @@ -257,7 +248,6 @@ def test_text_empty(self): class HttpRequestHandler(http.server.BaseHTTPRequestHandler): - def do_GET(self): try: output = DATA[self.path[1:]] @@ -274,8 +264,7 @@ def do_GET(self): match = re.match(r"bytes=(\d+)-", self.headers["Range"]) start = int(match.group(1)) - headers["Content-Range"] = "bytes {}-{}/{}".format( - start, len(output)-1, len(output)) + headers["Content-Range"] = f"bytes {start}-{len(output) - 1}/{len(output)}" output = output[start:] else: status = 200 @@ -288,19 +277,26 @@ def do_GET(self): SAMPLES = { - ("jpg" , binascii.a2b_base64( - "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB" - "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEB" - "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB" - "AQEBAQEBAQEBAQEBAQH/wAARCAABAAEDAREAAhEBAxEB/8QAFAABAAAAAAAAAAAA" - "AAAAAAAACv/EABQQAQAAAAAAAAAAAAAAAAAAAAD/xAAUAQEAAAAAAAAAAAAAAAAA" - "AAAA/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAwDAQACEQMRAD8AfwD/2Q==")), - ("png" , binascii.a2b_base64( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVQIHWP4DwAB" - "AQEANl9ngAAAAABJRU5ErkJggg==")), - ("gif" , binascii.a2b_base64( - "R0lGODdhAQABAIAAAP///////ywAAAAAAQABAAACAkQBADs=")), - ("bmp" , b"BM"), + ( + "jpg", + binascii.a2b_base64( + "/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB" + "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEB" + "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB" + "AQEBAQEBAQEBAQEBAQH/wAARCAABAAEDAREAAhEBAxEB/8QAFAABAAAAAAAAAAAA" + "AAAAAAAACv/EABQQAQAAAAAAAAAAAAAAAAAAAAD/xAAUAQEAAAAAAAAAAAAAAAAA" + "AAAA/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAwDAQACEQMRAD8AfwD/2Q==" + ), + ), + ( + "png", + binascii.a2b_base64( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAAAAAA6fptVAAAACklEQVQIHWP4DwAB" + "AQEANl9ngAAAAABJRU5ErkJggg==" + ), + ), + ("gif", binascii.a2b_base64("R0lGODdhAQABAIAAAP///////ywAAAAAAQABAAACAkQBADs=")), + ("bmp", b"BM"), ("webp", b"RIFF????WEBP"), ("avif", b"????ftypavif"), ("avif", b"????ftypavis"), @@ -308,33 +304,33 @@ def do_GET(self): ("heic", b"????ftypheim"), ("heic", b"????ftypheis"), ("heic", b"????ftypheix"), - ("svg" , b"02}".format(idx)] = content + DATA[f"S{idx:>02}"] = content # reverse mime types mapping -MIME_TYPES = { - ext: mtype - for mtype, ext in MIME_TYPES.items() -} +MIME_TYPES = {ext: mtype for mtype, ext in MIME_TYPES.items()} def generate_tests(): def generate_test(idx, ext, content): def test(self): - self._run_test("S{:>02}".format(idx), None, content, "bin", ext) - test.__name__ = "test_http_ext_{:>02}_{}".format(idx, ext) + self._run_test(f"S{idx:>02}", None, content, "bin", ext) + + test.__name__ = f"test_http_ext_{idx:>02}_{ext}" return test for idx, (ext, content) in enumerate(SAMPLES): diff --git a/test/test_extractor.py b/test/test_extractor.py index cc85fb2b23..bfa18cbb8f 100644 --- a/test/test_extractor.py +++ b/test/test_extractor.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2018-2023 Mike Fährmann # @@ -8,19 +7,19 @@ # published by the Free Software Foundation. import os +import string import sys +import time import unittest +from datetime import datetime +from datetime import timedelta from unittest.mock import patch -import time -import string -from datetime import datetime, timedelta - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import extractor, util # noqa E402 from gallery_dl.extractor import mastodon # noqa E402 -from gallery_dl.extractor.common import Extractor, Message # noqa E402 -from gallery_dl.extractor.directlink import DirectlinkExtractor # noqa E402 +from gallery_dl.extractor.common import Extractor, Message +from gallery_dl.extractor.directlink import DirectlinkExtractor _list_classes = extractor._list_classes @@ -110,7 +109,7 @@ def test_categories(self): extr = cls.from_url(url) except ImportError as exc: if exc.name in ("youtube_dl", "yt_dlp"): - print("Skipping '{}' category checks".format(cls.category)) + print(f"Skipping '{cls.category}' category checks") continue raise self.assertTrue(extr, url) @@ -141,10 +140,8 @@ def test_unique_pattern_matches(self): # ... and apply all regex patterns to each one for extr2 in _list_classes(): - # skip DirectlinkExtractor pattern if it isn't tested - if extr1 != DirectlinkExtractor and \ - extr2 == DirectlinkExtractor: + if extr1 != DirectlinkExtractor and extr2 == DirectlinkExtractor: continue match = extr2.pattern.match(url) @@ -153,14 +150,13 @@ def test_unique_pattern_matches(self): # fail if more or less than 1 match happened if len(matches) > 1: - msg = "'{}' gets matched by more than one pattern:".format(url) + msg = f"'{url}' gets matched by more than one pattern:" for match, extr in matches: - msg += "\n\n- {}:\n{}".format( - extr.__name__, match.re.pattern) + msg += f"\n\n- {extr.__name__}:\n{match.re.pattern}" self.fail(msg) elif len(matches) < 1: - msg = "'{}' isn't matched by any pattern".format(url) + msg = f"'{url}' isn't matched by any pattern" self.fail(msg) else: @@ -168,6 +164,7 @@ def test_unique_pattern_matches(self): def test_init(self): """Test for exceptions in Extractor.initialize() and .finalize()""" + def fail_request(*args, **kwargs): self.fail("called 'request() during initialization") @@ -190,8 +187,7 @@ def test_init_ytdl(self): extr.finalize() except ImportError as exc: if exc.name in ("youtube_dl", "yt_dlp"): - raise unittest.SkipTest("cannot import module '{}'".format( - exc.name)) + raise unittest.SkipTest(f"cannot import module '{exc.name}'") raise def test_docstrings(self): @@ -202,11 +198,12 @@ def test_docstrings(self): self.assertNotEqual( extr1.__doc__, extr2.__doc__, - "{} <-> {}".format(extr1, extr2), + f"{extr1} <-> {extr2}", ) def test_names(self): """Ensure extractor classes are named CategorySubcategoryExtractor""" + def capitalize(c): if "-" in c: return string.capwords(c.replace("-", " ")).replace(" ", "") @@ -214,17 +211,13 @@ def capitalize(c): for extr in extractor.extractors(): if extr.category not in ("", "oauth", "ytdl"): - expected = "{}{}Extractor".format( - capitalize(extr.category), - capitalize(extr.subcategory), - ) + expected = f"{capitalize(extr.category)}{capitalize(extr.subcategory)}Extractor" if expected[0].isdigit(): expected = "_" + expected self.assertEqual(expected, extr.__name__) class TestExtractorWait(unittest.TestCase): - def test_wait_seconds(self): extr = extractor.find("generic:https://example.org/") seconds = 5 @@ -278,7 +271,7 @@ def _assert_isotime(self, output, until): until = datetime.fromtimestamp(until) o = self._isotime_to_seconds(output) u = self._isotime_to_seconds(until.time().isoformat()[:8]) - self.assertLessEqual(o-u, 1.0) + self.assertLessEqual(o - u, 1.0) @staticmethod def _isotime_to_seconds(isotime): @@ -287,7 +280,6 @@ def _isotime_to_seconds(isotime): class TextExtractorOAuth(unittest.TestCase): - def test_oauth1(self): for category in ("flickr", "smugmug", "tumblr"): extr = extractor.find("oauth:" + category) @@ -309,8 +301,10 @@ def test_oauth2(self): def test_oauth2_mastodon(self): extr = extractor.find("oauth:mastodon:pawoo.net") - with patch.object(extr, "_oauth2_authorization_code_grant") as m, \ - patch.object(extr, "_register") as r: + with ( + patch.object(extr, "_oauth2_authorization_code_grant") as m, + patch.object(extr, "_register") as r, + ): for msg in extr: pass self.assertEqual(len(r.mock_calls), 0) @@ -319,10 +313,12 @@ def test_oauth2_mastodon(self): def test_oauth2_mastodon_unknown(self): extr = extractor.find("oauth:mastodon:example.com") - with patch.object(extr, "_oauth2_authorization_code_grant") as m, \ - patch.object(extr, "_register") as r: + with ( + patch.object(extr, "_oauth2_authorization_code_grant") as m, + patch.object(extr, "_register") as r, + ): r.return_value = { - "client-id" : "foo", + "client-id": "foo", "client-secret": "bar", } diff --git a/test/test_formatter.py b/test/test_formatter.py index c0b504da87..19a619aba1 100644 --- a/test/test_formatter.py +++ b/test/test_formatter.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2021-2023 Mike Fährmann # @@ -7,19 +6,19 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import datetime import os import sys +import tempfile import time import unittest -import datetime -import tempfile +from time import sleep sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import formatter, text, util # noqa E402 class TestFormatter(unittest.TestCase): - kwdict = { "a": "hElLo wOrLd", "b": "äöü", @@ -106,28 +105,27 @@ def test_missing(self): def test_missing_custom_default(self): replacement = default = "foobar" - self._run_test("{missing}" , replacement, default) + self._run_test("{missing}", replacement, default) self._run_test("{missing.attr}", replacement, default) self._run_test("{missing[key]}", replacement, default) self._run_test("{missing:?a//}", "a" + default, default) def test_fmt_func(self): - self._run_test("{t}" , self.kwdict["t"] , None, int) - self._run_test("{t}" , self.kwdict["t"] , None, util.identity) + self._run_test("{t}", self.kwdict["t"], None, int) + self._run_test("{t}", self.kwdict["t"], None, util.identity) self._run_test("{dt}", self.kwdict["dt"], None, util.identity) self._run_test("{ds}", self.kwdict["dt"], None, text.parse_datetime) - self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", self.kwdict["dt"], - None, util.identity) + self._run_test("{ds:D%Y-%m-%dT%H:%M:%S%z}", self.kwdict["dt"], None, util.identity) def test_alternative(self): - self._run_test("{a|z}" , "hElLo wOrLd") - self._run_test("{z|a}" , "hElLo wOrLd") - self._run_test("{z|y|a}" , "hElLo wOrLd") + self._run_test("{a|z}", "hElLo wOrLd") + self._run_test("{z|a}", "hElLo wOrLd") + self._run_test("{z|y|a}", "hElLo wOrLd") self._run_test("{z|y|x|a}", "hElLo wOrLd") self._run_test("{z|n|a|y}", "hElLo wOrLd") - self._run_test("{z|a!C}" , "Hello World") - self._run_test("{z|a:Rh/C/}" , "CElLo wOrLd") + self._run_test("{z|a!C}", "Hello World") + self._run_test("{z|a:Rh/C/}", "CElLo wOrLd") self._run_test("{z|a!C:RH/C/}", "Cello World") self._run_test("{z|y|x:?/}", "") @@ -136,93 +134,93 @@ def test_alternative(self): self._run_test("{d[z]|d[y]|d[x]}", "None") def test_indexing(self): - self._run_test("{l[0]}" , "a") - self._run_test("{a[6]}" , "w") + self._run_test("{l[0]}", "a") + self._run_test("{a[6]}", "w") def test_dict_access(self): - self._run_test("{d[a]}" , "foo") + self._run_test("{d[a]}", "foo") self._run_test("{d['a']}", "foo") self._run_test('{d["a"]}', "foo") def test_slice_str(self): v = self.kwdict["a"] - self._run_test("{a[1:10]}" , v[1:10]) + self._run_test("{a[1:10]}", v[1:10]) self._run_test("{a[-10:-1]}", v[-10:-1]) - self._run_test("{a[5:]}" , v[5:]) + self._run_test("{a[5:]}", v[5:]) self._run_test("{a[50:]}", v[50:]) - self._run_test("{a[:5]}" , v[:5]) + self._run_test("{a[:5]}", v[:5]) self._run_test("{a[:50]}", v[:50]) - self._run_test("{a[:]}" , v) - self._run_test("{a[1:10:2]}" , v[1:10:2]) + self._run_test("{a[:]}", v) + self._run_test("{a[1:10:2]}", v[1:10:2]) self._run_test("{a[-10:-1:2]}", v[-10:-1:2]) - self._run_test("{a[5::2]}" , v[5::2]) + self._run_test("{a[5::2]}", v[5::2]) self._run_test("{a[50::2]}", v[50::2]) - self._run_test("{a[:5:2]}" , v[:5:2]) + self._run_test("{a[:5:2]}", v[:5:2]) self._run_test("{a[:50:2]}", v[:50:2]) - self._run_test("{a[::]}" , v) + self._run_test("{a[::]}", v) - self._run_test("{a:[1:10]}" , v[1:10]) + self._run_test("{a:[1:10]}", v[1:10]) self._run_test("{a:[-10:-1]}", v[-10:-1]) - self._run_test("{a:[5:]}" , v[5:]) + self._run_test("{a:[5:]}", v[5:]) self._run_test("{a:[50:]}", v[50:]) - self._run_test("{a:[:5]}" , v[:5]) + self._run_test("{a:[:5]}", v[:5]) self._run_test("{a:[:50]}", v[:50]) - self._run_test("{a:[:]}" , v) - self._run_test("{a:[1:10:2]}" , v[1:10:2]) + self._run_test("{a:[:]}", v) + self._run_test("{a:[1:10:2]}", v[1:10:2]) self._run_test("{a:[-10:-1:2]}", v[-10:-1:2]) - self._run_test("{a:[5::2]}" , v[5::2]) + self._run_test("{a:[5::2]}", v[5::2]) self._run_test("{a:[50::2]}", v[50::2]) - self._run_test("{a:[:5:2]}" , v[:5:2]) + self._run_test("{a:[:5:2]}", v[:5:2]) self._run_test("{a:[:50:2]}", v[:50:2]) - self._run_test("{a:[::]}" , v) + self._run_test("{a:[::]}", v) def test_slice_bytes(self): v = self.kwdict["j"] - self._run_test("{j[b1:10]}" , v[1:3]) + self._run_test("{j[b1:10]}", v[1:3]) self._run_test("{j[b-10:-1]}", v[-3:-1]) - self._run_test("{j[b5:]}" , v[2:]) - self._run_test("{j[b50:]}" , v[50:]) - self._run_test("{j[b:5]}" , v[:1]) - self._run_test("{j[b:50]}" , v[:50]) - self._run_test("{j[b:]}" , v) - self._run_test("{j[b::]}" , v) - - self._run_test("{j:[b1:10]}" , v[1:3]) + self._run_test("{j[b5:]}", v[2:]) + self._run_test("{j[b50:]}", v[50:]) + self._run_test("{j[b:5]}", v[:1]) + self._run_test("{j[b:50]}", v[:50]) + self._run_test("{j[b:]}", v) + self._run_test("{j[b::]}", v) + + self._run_test("{j:[b1:10]}", v[1:3]) self._run_test("{j:[b-10:-1]}", v[-3:-1]) - self._run_test("{j:[b5:]}" , v[2:]) - self._run_test("{j:[b50:]}" , v[50:]) - self._run_test("{j:[b:5]}" , v[:1]) - self._run_test("{j:[b:50]}" , v[:50]) - self._run_test("{j:[b:]}" , v) - self._run_test("{j:[b::]}" , v) + self._run_test("{j:[b5:]}", v[2:]) + self._run_test("{j:[b50:]}", v[50:]) + self._run_test("{j:[b:5]}", v[:1]) + self._run_test("{j:[b:50]}", v[:50]) + self._run_test("{j:[b:]}", v) + self._run_test("{j:[b::]}", v) def test_maxlen(self): v = self.kwdict["a"] - self._run_test("{a:L5/foo/}" , "foo") + self._run_test("{a:L5/foo/}", "foo") self._run_test("{a:L50/foo/}", v) self._run_test("{a:L50/foo/>50}", " " * 39 + v) self._run_test("{a:L50/foo/>51}", "foo") self._run_test("{a:Lab/foo/}", "foo") def test_join(self): - self._run_test("{l:J}" , "abc") - self._run_test("{l:J,}" , "a,b,c") - self._run_test("{l:J,/}" , "a,b,c") - self._run_test("{l:J,/>20}" , " a,b,c") - self._run_test("{l:J - }" , "a - b - c") - self._run_test("{l:J - /}" , "a - b - c") + self._run_test("{l:J}", "abc") + self._run_test("{l:J,}", "a,b,c") + self._run_test("{l:J,/}", "a,b,c") + self._run_test("{l:J,/>20}", " a,b,c") + self._run_test("{l:J - }", "a - b - c") + self._run_test("{l:J - /}", "a - b - c") self._run_test("{l:J - />20}", " a - b - c") - self._run_test("{a:J/}" , self.kwdict["a"]) - self._run_test("{a:J, /}" , self.kwdict["a"]) + self._run_test("{a:J/}", self.kwdict["a"]) + self._run_test("{a:J, /}", self.kwdict["a"]) def test_replace(self): - self._run_test("{a:Rh/C/}" , "CElLo wOrLd") + self._run_test("{a:Rh/C/}", "CElLo wOrLd") self._run_test("{a!l:Rh/C/}", "Cello world") self._run_test("{a!u:Rh/C/}", "HELLO WORLD") self._run_test("{a!l:Rl/_/}", "he__o wor_d") - self._run_test("{a!l:Rl//}" , "heo word") + self._run_test("{a!l:Rl//}", "heo word") self._run_test("{name:Rame/othing/}", "Nothing") def test_datetime(self): @@ -242,34 +240,33 @@ def test_offset(self): self._run_test("{t!d:O2}", "2010-01-01 02:00:00") def test_offset_local(self): - ts = self.kwdict["dt"].replace( - tzinfo=datetime.timezone.utc).timestamp() + ts = self.kwdict["dt"].replace(tzinfo=datetime.timezone.utc).timestamp() offset = time.localtime(ts).tm_gmtoff dt = self.kwdict["dt"] + datetime.timedelta(seconds=offset) self._run_test("{dt:O}", str(dt)) self._run_test("{dt:Olocal}", str(dt)) - ts = self.kwdict["dt_dst"].replace( - tzinfo=datetime.timezone.utc).timestamp() + ts = self.kwdict["dt_dst"].replace(tzinfo=datetime.timezone.utc).timestamp() offset = time.localtime(ts).tm_gmtoff dt = self.kwdict["dt_dst"] + datetime.timedelta(seconds=offset) self._run_test("{dt_dst:O}", str(dt)) self._run_test("{dt_dst:Olocal}", str(dt)) def test_sort(self): - self._run_test("{l:S}" , "['a', 'b', 'c']") + self._run_test("{l:S}", "['a', 'b', 'c']") self._run_test("{l:Sa}", "['a', 'b', 'c']") self._run_test("{l:Sd}", "['c', 'b', 'a']") self._run_test("{l:Sr}", "['c', 'b', 'a']") - self._run_test( - "{a:S}", "[' ', 'E', 'L', 'L', 'O', 'd', 'h', 'l', 'o', 'r', 'w']") + self._run_test("{a:S}", "[' ', 'E', 'L', 'L', 'O', 'd', 'h', 'l', 'o', 'r', 'w']") self._run_test( "{a:S-asc}", # starts with 'S', contains 'a' - "[' ', 'E', 'L', 'L', 'O', 'd', 'h', 'l', 'o', 'r', 'w']") + "[' ', 'E', 'L', 'L', 'O', 'd', 'h', 'l', 'o', 'r', 'w']", + ) self._run_test( "{a:Sort-reverse}", # starts with 'S', contains 'r' - "['w', 'r', 'o', 'l', 'h', 'd', 'O', 'L', 'L', 'E', ' ']") + "['w', 'r', 'o', 'l', 'h', 'd', 'O', 'L', 'L', 'E', ' ']", + ) def test_specifier_arithmetic(self): self._run_test("{i:A+1}", "3") @@ -277,8 +274,8 @@ def test_specifier_arithmetic(self): self._run_test("{i:A*3}", "6") def test_specifier_conversions(self): - self._run_test("{a:Cl}" , "hello world") - self._run_test("{h:CHC}" , "Foo & Bar") + self._run_test("{a:Cl}", "hello world") + self._run_test("{h:CHC}", "Foo & Bar") self._run_test("{l:CSulc}", "A, b, c") def test_specifier_limit(self): @@ -332,7 +329,7 @@ def test_separator(self): def test_globals_env(self): os.environ["FORMATTER_TEST"] = value = self.kwdict["a"] - self._run_test("{_env[FORMATTER_TEST]}" , value) + self._run_test("{_env[FORMATTER_TEST]}", value) self._run_test("{_env[FORMATTER_TEST]!l}", value.lower()) self._run_test("{z|_env[FORMATTER_TEST]}", value) @@ -350,6 +347,10 @@ def test_globals_now(self): self.assertRegex(out, r"^\d{4}$") self.assertEqual(out, format(now, "%Y")) + # Sleep briefly to ensure `now` is actually different + # Without this, the following assertion can fail if the instructions are executed quickly enough + sleep(0.01) + out2 = fmt.format_map(self.kwdict) self.assertRegex(out1, r"^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d(\.\d+)?$") self.assertNotEqual(out1, out2) @@ -357,38 +358,38 @@ def test_globals_now(self): def test_globals_nul(self): value = "None" - self._run_test("{_nul}" , value) - self._run_test("{_nul[key]}" , value) - self._run_test("{z|_nul}" , value) + self._run_test("{_nul}", value) + self._run_test("{_nul[key]}", value) + self._run_test("{z|_nul}", value) self._run_test("{z|_nul:%Y%m%s}", value) def test_literals(self): value = "foo" - self._run_test("{'foo'}" , value) - self._run_test("{'foo'!u}" , value.upper()) + self._run_test("{'foo'}", value) + self._run_test("{'foo'!u}", value.upper()) self._run_test("{'f00':R0/o/}", value) - self._run_test("{z|'foo'}" , value) - self._run_test("{z|''|'foo'}" , value) - self._run_test("{z|'foo'!u}" , value.upper()) + self._run_test("{z|'foo'}", value) + self._run_test("{z|''|'foo'}", value) + self._run_test("{z|'foo'!u}", value.upper()) self._run_test("{z|'f00':R0/o/}", value) - self._run_test("{_lit[foo]}" , value) - self._run_test("{_lit[foo]!u}" , value.upper()) - self._run_test("{_lit[f00]:R0/o/}" , value) + self._run_test("{_lit[foo]}", value) + self._run_test("{_lit[foo]!u}", value.upper()) + self._run_test("{_lit[f00]:R0/o/}", value) self._run_test("{_lit[foobar][:3]}", value) - self._run_test("{z|_lit[foo]}" , value) + self._run_test("{z|_lit[foo]}", value) # empty (#4492) - self._run_test("{z|''}" , "") + self._run_test("{z|''}", "") self._run_test("{''|''}", "") # special characters (dots, brackets, singlee quotes) (#5539) - self._run_test("{'f.o.o'}" , "f.o.o") + self._run_test("{'f.o.o'}", "f.o.o") self._run_test("{_lit[f.o.o]}", "f.o.o") self._run_test("{_lit[f'o'o]}", "f'o'o") - self._run_test("{'f.[].[]'}" , "f.[].[]") + self._run_test("{'f.[].[]'}", "f.[].[]") self._run_test("{z|'f.[].[]'}", "f.[].[]") def test_template(self): @@ -412,16 +413,21 @@ def test_template(self): def test_expression(self): self._run_test("\fE a", self.kwdict["a"]) - self._run_test("\fE name * 2 + ' ' + a", "{}{} {}".format( - self.kwdict["name"], self.kwdict["name"], self.kwdict["a"])) + self._run_test( + "\fE name * 2 + ' ' + a", + "{}{} {}".format(self.kwdict["name"], self.kwdict["name"], self.kwdict["a"]), + ) @unittest.skipIf(sys.hexversion < 0x3060000, "no fstring support") def test_fstring(self): self._run_test("\fF {a}", self.kwdict["a"]) - self._run_test("\fF {name}{name} {a}", "{}{} {}".format( - self.kwdict["name"], self.kwdict["name"], self.kwdict["a"])) - self._run_test("\fF foo-'\"{a.upper()}\"'-bar", - """foo-'"{}"'-bar""".format(self.kwdict["a"].upper())) + self._run_test( + "\fF {name}{name} {a}", + "{}{} {}".format(self.kwdict["name"], self.kwdict["name"], self.kwdict["a"]), + ) + self._run_test( + "\fF foo-'\"{a.upper()}\"'-bar", """foo-'"{}"'-bar""".format(self.kwdict["a"].upper()) + ) @unittest.skipIf(sys.hexversion < 0x3060000, "no fstring support") def test_template_fstring(self): @@ -438,8 +444,9 @@ def test_template_fstring(self): fmt2 = formatter.parse("\fTF " + path2) self.assertEqual(fmt1.format_map(self.kwdict), self.kwdict["a"]) - self.assertEqual(fmt2.format_map(self.kwdict), - """foo-'"{}"'-bar""".format(self.kwdict["a"].upper())) + self.assertEqual( + fmt2.format_map(self.kwdict), """foo-'"{}"'-bar""".format(self.kwdict["a"].upper()) + ) with self.assertRaises(OSError): formatter.parse("\fTF /") diff --git a/test/test_job.py b/test/test_job.py index 3e6f85be65..f206026617 100644 --- a/test/test_job.py +++ b/test/test_job.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2021-2023 Mike Fährmann # @@ -7,20 +6,18 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import io import os import sys import unittest from unittest.mock import patch -import io - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import job, config, text # noqa E402 -from gallery_dl.extractor.common import Extractor, Message # noqa E402 +from gallery_dl.extractor.common import Extractor, Message class TestJob(unittest.TestCase): - def tearDown(self): config.clear() @@ -49,21 +46,21 @@ def test_extractor_filter(self): tjob = self.jobclass(extr) func = tjob._build_extractor_filter() - self.assertEqual(func(TestExtractor) , False) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), False) - self.assertEqual(func(TestExtractorAlt) , True) + self.assertEqual(func(TestExtractorAlt), True) config.set((), "blacklist", ":test_subcategory") func = tjob._build_extractor_filter() - self.assertEqual(func(TestExtractor) , False) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), True) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) config.set((), "whitelist", "test_category:test_subcategory") func = tjob._build_extractor_filter() - self.assertEqual(func(TestExtractor) , True) + self.assertEqual(func(TestExtractor), True) self.assertEqual(func(TestExtractorParent), False) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) class TestKeywordJob(TestJob): @@ -72,7 +69,9 @@ class TestKeywordJob(TestJob): def test_default(self): self.maxDiff = None extr = TestExtractor.from_url("test:self") - self.assertEqual(self._capture_stdout(extr), """\ + self.assertEqual( + self._capture_stdout(extr), + """\ Keywords for directory names: ----------------------------- author['id'] @@ -120,7 +119,8 @@ def test_default(self): test user['self'] -""") +""", + ) class TestUrlJob(TestJob): @@ -128,42 +128,55 @@ class TestUrlJob(TestJob): def test_default(self): extr = TestExtractor.from_url("test:") - self.assertEqual(self._capture_stdout(extr), """\ + self.assertEqual( + self._capture_stdout(extr), + """\ https://example.org/1.jpg https://example.org/2.jpg https://example.org/3.jpg -""") +""", + ) def test_fallback(self): extr = TestExtractor.from_url("test:") tjob = self.jobclass(extr) tjob.handle_url = tjob.handle_url_fallback - self.assertEqual(self._capture_stdout(tjob), """\ + self.assertEqual( + self._capture_stdout(tjob), + """\ https://example.org/1.jpg | https://example.org/alt/1.jpg https://example.org/2.jpg | https://example.org/alt/2.jpg https://example.org/3.jpg | https://example.org/alt/3.jpg -""") +""", + ) def test_parent(self): extr = TestExtractorParent.from_url("test:parent") - self.assertEqual(self._capture_stdout(extr), """\ + self.assertEqual( + self._capture_stdout(extr), + """\ test:child test:child test:child -""") +""", + ) def test_child(self): extr = TestExtractorParent.from_url("test:parent") tjob = job.UrlJob(extr, depth=0) - self.assertEqual(self._capture_stdout(tjob), 3 * """\ + self.assertEqual( + self._capture_stdout(tjob), + 3 + * """\ https://example.org/1.jpg https://example.org/2.jpg https://example.org/3.jpg -""") +""", + ) class TestInfoJob(TestJob): @@ -171,7 +184,9 @@ class TestInfoJob(TestJob): def test_default(self): extr = TestExtractor.from_url("test:") - self.assertEqual(self._capture_stdout(extr), """\ + self.assertEqual( + self._capture_stdout(extr), + """\ Category / Subcategory "test_category" / "test_subcategory" @@ -181,7 +196,8 @@ def test_default(self): Directory format (default): ["{category}"] -""") +""", + ) def test_custom(self): config.set((), "filename", "custom") @@ -190,7 +206,9 @@ def test_custom(self): extr = TestExtractor.from_url("test:") extr.request_interval = 123.456 - self.assertEqual(self._capture_stdout(extr), """\ + self.assertEqual( + self._capture_stdout(extr), + """\ Category / Subcategory "test_category" / "test_subcategory" @@ -209,13 +227,16 @@ def test_custom(self): Request interval (default): 123.456 -""") +""", + ) def test_base_category(self): extr = TestExtractor.from_url("test:") extr.basecategory = "test_basecategory" - self.assertEqual(self._capture_stdout(extr), """\ + self.assertEqual( + self._capture_stdout(extr), + """\ Category / Subcategory / Basecategory "test_category" / "test_subcategory" / "test_basecategory" @@ -225,7 +246,8 @@ def test_base_category(self): Directory format (default): ["{category}"] -""") +""", + ) class TestDataJob(TestJob): @@ -238,51 +260,68 @@ def test_default(self): tjob.run() - self.assertEqual(tjob.data, [ - (Message.Directory, { - "category" : "test_category", - "subcategory": "test_subcategory", - "user" : user, - "author" : user, - }), - (Message.Url, "https://example.org/1.jpg", { - "category" : "test_category", - "subcategory": "test_subcategory", - "filename" : "1", - "extension" : "jpg", - "num" : 1, - "tags" : ["foo", "bar", "テスト"], - "user" : user, - "author" : user, - }), - (Message.Url, "https://example.org/2.jpg", { - "category" : "test_category", - "subcategory": "test_subcategory", - "filename" : "2", - "extension" : "jpg", - "num" : 2, - "tags" : ["foo", "bar", "テスト"], - "user" : user, - "author" : user, - }), - (Message.Url, "https://example.org/3.jpg", { - "category" : "test_category", - "subcategory": "test_subcategory", - "filename" : "3", - "extension" : "jpg", - "num" : 3, - "tags" : ["foo", "bar", "テスト"], - "user" : user, - "author" : user, - }), - ]) + self.assertEqual( + tjob.data, + [ + ( + Message.Directory, + { + "category": "test_category", + "subcategory": "test_subcategory", + "user": user, + "author": user, + }, + ), + ( + Message.Url, + "https://example.org/1.jpg", + { + "category": "test_category", + "subcategory": "test_subcategory", + "filename": "1", + "extension": "jpg", + "num": 1, + "tags": ["foo", "bar", "テスト"], + "user": user, + "author": user, + }, + ), + ( + Message.Url, + "https://example.org/2.jpg", + { + "category": "test_category", + "subcategory": "test_subcategory", + "filename": "2", + "extension": "jpg", + "num": 2, + "tags": ["foo", "bar", "テスト"], + "user": user, + "author": user, + }, + ), + ( + Message.Url, + "https://example.org/3.jpg", + { + "category": "test_category", + "subcategory": "test_subcategory", + "filename": "3", + "extension": "jpg", + "num": 3, + "tags": ["foo", "bar", "テスト"], + "user": user, + "author": user, + }, + ), + ], + ) def test_exception(self): extr = TestExtractorException.from_url("test:exception") tjob = self.jobclass(extr, file=io.StringIO()) tjob.run() - self.assertEqual( - tjob.data[-1], ("ZeroDivisionError", "division by zero")) + self.assertEqual(tjob.data[-1], ("ZeroDivisionError", "division by zero")) def test_private(self): config.set(("output",), "private", True) @@ -294,7 +333,7 @@ def test_private(self): for i in range(1, 4): self.assertEqual( tjob.data[i][2]["_fallback"], - ("https://example.org/alt/{}.jpg".format(i),), + (f"https://example.org/alt/{i}.jpg",), ) def test_sleep(self): @@ -317,24 +356,30 @@ def test_ascii(self): tjob.file = buffer = io.StringIO() tjob.run() - self.assertIn("""\ + self.assertIn( + """\ "tags": [ "foo", "bar", "\\u30c6\\u30b9\\u30c8" ], -""", buffer.getvalue()) +""", + buffer.getvalue(), + ) tjob.file = buffer = io.StringIO() tjob.ascii = False tjob.run() - self.assertIn("""\ + self.assertIn( + """\ "tags": [ "foo", "bar", "テスト" ], -""", buffer.getvalue()) +""", + buffer.getvalue(), + ) def test_num_string(self): extr = TestExtractor.from_url("test:") @@ -371,20 +416,30 @@ def items(self): root = "https://example.org" user = self.user - yield Message.Directory, { - "user": user, - "author": user, - } - - for i in range(1, 4): - url = "{}/{}.jpg".format(root, i) - yield Message.Url, url, text.nameext_from_url(url, { - "num" : i, - "tags": ["foo", "bar", "テスト"], + yield ( + Message.Directory, + { "user": user, "author": user, - "_fallback": ("{}/alt/{}.jpg".format(root, i),), - }) + }, + ) + + for i in range(1, 4): + url = f"{root}/{i}.jpg" + yield ( + Message.Url, + url, + text.nameext_from_url( + url, + { + "num": i, + "tags": ["foo", "bar", "テスト"], + "user": user, + "author": user, + "_fallback": (f"{root}/alt/{i}.jpg",), + }, + ), + ) class TestExtractorParent(Extractor): @@ -396,11 +451,15 @@ def items(self): url = "test:child" for i in range(11, 14): - yield Message.Queue, url, { - "num" : i, - "tags": ["abc", "def"], - "_extractor": TestExtractor, - } + yield ( + Message.Queue, + url, + { + "num": i, + "tags": ["abc", "def"], + "_extractor": TestExtractor, + }, + ) class TestExtractorException(Extractor): @@ -409,7 +468,7 @@ class TestExtractorException(Extractor): pattern = r"test:exception$" def items(self): - return 1/0 + return 1 / 0 class TestExtractorAlt(Extractor): diff --git a/test/test_oauth.py b/test/test_oauth.py index 0082419d3d..f85b0a6273 100644 --- a/test/test_oauth.py +++ b/test/test_oauth.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2018-2023 Mike Fährmann # @@ -25,7 +24,6 @@ class TestOAuthSession(unittest.TestCase): - def test_concat(self): concat = oauth.concat @@ -36,7 +34,7 @@ def test_concat(self): self.assertEqual(concat("&", "?/"), "%26&%3F%2F") self.assertEqual( concat("GET", "http://example.org/", "foo=bar&baz=a"), - "GET&http%3A%2F%2Fexample.org%2F&foo%3Dbar%26baz%3Da" + "GET&http%3A%2F%2Fexample.org%2F&foo%3Dbar%26baz%3Da", ) def test_nonce(self, size=16): @@ -53,9 +51,7 @@ def test_quote(self): quote = oauth.quote reserved = ",;:!\"§$%&/(){}[]=?`´+*'äöü" - unreserved = ("ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789-._~") + unreserved = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789-._~" for char in unreserved: self.assertEqual(quote(char), char) @@ -69,34 +65,36 @@ def test_quote(self): def test_generate_signature(self): client = oauth.OAuth1Client( - CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET) + CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET + ) request = MockRequest() params = [] self.assertEqual( - client.generate_signature(request, params), - "Wt2xo49dM5pkL4gsnCakNdHaVUo%3D") + client.generate_signature(request, params), "Wt2xo49dM5pkL4gsnCakNdHaVUo%3D" + ) request = MockRequest("https://example.org/") params = [("hello", "world"), ("foo", "bar")] self.assertEqual( - client.generate_signature(request, params), - "ay2269%2F8uKpZqKJR1doTtpv%2Bzn0%3D") + client.generate_signature(request, params), "ay2269%2F8uKpZqKJR1doTtpv%2Bzn0%3D" + ) - request = MockRequest("https://example.org/index.html" - "?hello=world&foo=bar", method="POST") + request = MockRequest( + "https://example.org/index.html" "?hello=world&foo=bar", method="POST" + ) params = [("oauth_signature_method", "HMAC-SHA1")] self.assertEqual( - client.generate_signature(request, params), - "yVZWb1ts4smdMmXxMlhaXrkoOng%3D") + client.generate_signature(request, params), "yVZWb1ts4smdMmXxMlhaXrkoOng%3D" + ) def test_dunder_call(self): client = oauth.OAuth1Client( - CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET) + CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET + ) request = MockRequest("https://example.org/") - with patch("time.time") as tmock, \ - patch("gallery_dl.oauth.nonce") as nmock: + with patch("time.time") as tmock, patch("gallery_dl.oauth.nonce") as nmock: tmock.return_value = 123456789.123 nmock.return_value = "abcdefghijklmno" @@ -112,11 +110,11 @@ def test_dunder_call(self): oauth_version="1.0",\ oauth_token="accesskey",\ oauth_signature="DjtTk5j5P3BDZFnstZ%2FtEYcwD6c%3D"\ -""") +""", + ) def test_request_token(self): - response = self._oauth_request( - "/request_token.php", {}) + response = self._oauth_request("/request_token.php", {}) expected = "oauth_token=requestkey&oauth_token_secret=requestsecret" self.assertEqual(response, expected, msg=response) @@ -125,8 +123,7 @@ def test_request_token(self): self.assertTrue(data["oauth_token_secret"], REQUEST_TOKEN_SECRET) def test_access_token(self): - response = self._oauth_request( - "/access_token.php", {}, REQUEST_TOKEN, REQUEST_TOKEN_SECRET) + response = self._oauth_request("/access_token.php", {}, REQUEST_TOKEN, REQUEST_TOKEN_SECRET) expected = "oauth_token=accesskey&oauth_token_secret=accesssecret" self.assertEqual(response, expected, msg=response) @@ -136,19 +133,19 @@ def test_access_token(self): def test_authenticated_call(self): params = {"method": "foo", "a": "äöüß/?&#", "äöüß/?&#": "a"} - response = self._oauth_request( - "/echo_api.php", params, ACCESS_TOKEN, ACCESS_TOKEN_SECRET) + response = self._oauth_request("/echo_api.php", params, ACCESS_TOKEN, ACCESS_TOKEN_SECRET) self.assertEqual(text.parse_query(response), params) - def _oauth_request(self, endpoint, params=None, - oauth_token=None, oauth_token_secret=None): + def _oauth_request(self, endpoint, params=None, oauth_token=None, oauth_token_secret=None): # the test server at 'term.ie' is unreachable raise unittest.SkipTest() session = oauth.OAuth1Session( - CONSUMER_KEY, CONSUMER_SECRET, - oauth_token, oauth_token_secret, + CONSUMER_KEY, + CONSUMER_SECRET, + oauth_token, + oauth_token_secret, ) try: response = session.get(TESTSERVER + endpoint, params=params) @@ -158,8 +155,7 @@ def _oauth_request(self, endpoint, params=None, raise unittest.SkipTest() -class MockRequest(): - +class MockRequest: def __init__(self, url="", method="GET"): self.url = url self.method = method diff --git a/test/test_output.py b/test/test_output.py index e81f7681c1..83e41876d6 100644 --- a/test/test_output.py +++ b/test/test_output.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2021 Mike Fährmann # @@ -12,13 +11,12 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import output # noqa E402 +from gallery_dl import output class TestShorten(unittest.TestCase): - def test_shorten_noop(self, f=output.shorten_string): - self.assertEqual(f("" , 10), "") + self.assertEqual(f("", 10), "") self.assertEqual(f("foobar", 10), "foobar") def test_shorten(self, f=output.shorten_string): @@ -36,9 +34,9 @@ def test_shorten(self, f=output.shorten_string): self.assertEqual(f(s, 12), "01234…456789") self.assertEqual(f(s, 11), "01234…56789") self.assertEqual(f(s, 10), "0123…56789") - self.assertEqual(f(s, 9) , "0123…6789") - self.assertEqual(f(s, 3) , "0…9") - self.assertEqual(f(s, 2) , "…9") + self.assertEqual(f(s, 9), "0123…6789") + self.assertEqual(f(s, 3), "0…9") + self.assertEqual(f(s, 2), "…9") def test_shorten_separator(self, f=output.shorten_string): s = "01234567890123456789" # string of length 20 @@ -48,15 +46,14 @@ def test_shorten_separator(self, f=output.shorten_string): self.assertEqual(f(s, 10, "|---|"), "01|---|789") self.assertEqual(f(s, 19, "..."), "01234567...23456789") - self.assertEqual(f(s, 19, "..") , "01234567..123456789") - self.assertEqual(f(s, 19, ".") , "012345678.123456789") - self.assertEqual(f(s, 19, "") , "0123456780123456789") + self.assertEqual(f(s, 19, ".."), "01234567..123456789") + self.assertEqual(f(s, 19, "."), "012345678.123456789") + self.assertEqual(f(s, 19, ""), "0123456780123456789") class TestShortenEAW(unittest.TestCase): - def test_shorten_eaw_noop(self, f=output.shorten_string_eaw): - self.assertEqual(f("" , 10), "") + self.assertEqual(f("", 10), "") self.assertEqual(f("foobar", 10), "foobar") def test_shorten_eaw(self, f=output.shorten_string_eaw): @@ -74,9 +71,9 @@ def test_shorten_eaw(self, f=output.shorten_string_eaw): self.assertEqual(f(s, 12), "01234…456789") self.assertEqual(f(s, 11), "01234…56789") self.assertEqual(f(s, 10), "0123…56789") - self.assertEqual(f(s, 9) , "0123…6789") - self.assertEqual(f(s, 3) , "0…9") - self.assertEqual(f(s, 2) , "…9") + self.assertEqual(f(s, 9), "0123…6789") + self.assertEqual(f(s, 3), "0…9") + self.assertEqual(f(s, 2), "…9") def test_shorten_eaw_wide(self, f=output.shorten_string_eaw): s = "幻想郷幻想郷幻想郷幻想郷" # 12 wide characters @@ -93,8 +90,8 @@ def test_shorten_eaw_wide(self, f=output.shorten_string_eaw): self.assertEqual(f(s, 12), "幻想…幻想郷") self.assertEqual(f(s, 11), "幻想…幻想郷") self.assertEqual(f(s, 10), "幻想…想郷") - self.assertEqual(f(s, 9) , "幻想…想郷") - self.assertEqual(f(s, 3) , "…郷") + self.assertEqual(f(s, 9), "幻想…想郷") + self.assertEqual(f(s, 3), "…郷") def test_shorten_eaw_mix(self, f=output.shorten_string_eaw): s = "幻-想-郷##幻-想-郷##幻-想-郷" # mixed characters @@ -112,8 +109,8 @@ def test_shorten_eaw_mix(self, f=output.shorten_string_eaw): self.assertEqual(f(s, 12), "幻-想…-想-郷") self.assertEqual(f(s, 11), "幻-想…想-郷") self.assertEqual(f(s, 10), "幻-…-想-郷") - self.assertEqual(f(s, 9) , "幻-…想-郷") - self.assertEqual(f(s, 3) , "…郷") + self.assertEqual(f(s, 9), "幻-…想-郷") + self.assertEqual(f(s, 3), "…郷") def test_shorten_eaw_separator(self, f=output.shorten_string_eaw): s = "01234567890123456789" # 20 ascii characters @@ -123,9 +120,9 @@ def test_shorten_eaw_separator(self, f=output.shorten_string_eaw): self.assertEqual(f(s, 10, "|---|"), "01|---|789") self.assertEqual(f(s, 19, "..."), "01234567...23456789") - self.assertEqual(f(s, 19, "..") , "01234567..123456789") - self.assertEqual(f(s, 19, ".") , "012345678.123456789") - self.assertEqual(f(s, 19, "") , "0123456780123456789") + self.assertEqual(f(s, 19, ".."), "01234567..123456789") + self.assertEqual(f(s, 19, "."), "012345678.123456789") + self.assertEqual(f(s, 19, ""), "0123456780123456789") def test_shorten_eaw_separator_wide(self, f=output.shorten_string_eaw): s = "幻想郷幻想郷幻想郷幻想郷" # 12 wide characters @@ -135,9 +132,9 @@ def test_shorten_eaw_separator_wide(self, f=output.shorten_string_eaw): self.assertEqual(f(s, 10, "|---|"), "幻|---|郷") self.assertEqual(f(s, 19, "..."), "幻想郷幻...郷幻想郷") - self.assertEqual(f(s, 19, "..") , "幻想郷幻..郷幻想郷") - self.assertEqual(f(s, 19, ".") , "幻想郷幻.想郷幻想郷") - self.assertEqual(f(s, 19, "") , "幻想郷幻想郷幻想郷") + self.assertEqual(f(s, 19, ".."), "幻想郷幻..郷幻想郷") + self.assertEqual(f(s, 19, "."), "幻想郷幻.想郷幻想郷") + self.assertEqual(f(s, 19, ""), "幻想郷幻想郷幻想郷") def test_shorten_eaw_separator_mix_(self, f=output.shorten_string_eaw): s = "幻-想-郷##幻-想-郷##幻-想-郷" # mixed characters @@ -147,9 +144,9 @@ def test_shorten_eaw_separator_mix_(self, f=output.shorten_string_eaw): self.assertEqual(f(s, 10, "|---|"), "幻|---|-郷") self.assertEqual(f(s, 19, "..."), "幻-想-郷...幻-想-郷") - self.assertEqual(f(s, 19, "..") , "幻-想-郷..#幻-想-郷") - self.assertEqual(f(s, 19, ".") , "幻-想-郷#.#幻-想-郷") - self.assertEqual(f(s, 19, "") , "幻-想-郷###幻-想-郷") + self.assertEqual(f(s, 19, ".."), "幻-想-郷..#幻-想-郷") + self.assertEqual(f(s, 19, "."), "幻-想-郷#.#幻-想-郷") + self.assertEqual(f(s, 19, ""), "幻-想-郷###幻-想-郷") if __name__ == "__main__": diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index dd53803fe0..13c78f452c 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2019-2023 Mike Fährmann # @@ -7,30 +6,28 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import collections +import logging import os +import shutil import sys +import tempfile import unittest -from unittest.mock import Mock, mock_open, patch - -import shutil -import logging import zipfile -import tempfile -import collections from datetime import datetime +from pathlib import Path +from unittest.mock import Mock, mock_open, patch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import extractor, output, path # noqa E402 -from gallery_dl import postprocessor, config # noqa E402 -from gallery_dl.postprocessor.common import PostProcessor # noqa E402 +from gallery_dl import config, extractor, output, path, postprocessor # noqa E402 +from gallery_dl.postprocessor.common import PostProcessor class MockPostprocessorModule(Mock): __postprocessor__ = "mock" -class FakeJob(): - +class FakeJob: def __init__(self, extr=extractor.find("generic:https://example.org/")): extr.directory_fmt = ("{category}",) self.extractor = extr @@ -45,36 +42,34 @@ def register_hooks(self, hooks, options): class TestPostprocessorModule(unittest.TestCase): - def setUp(self): postprocessor._cache.clear() def test_find(self): - for name in (postprocessor.modules): + for name in postprocessor.modules: cls = postprocessor.find(name) self.assertEqual(cls.__name__, name.capitalize() + "PP") self.assertIs(cls.__base__, PostProcessor) self.assertEqual(postprocessor.find("foo"), None) - self.assertEqual(postprocessor.find(1234) , None) - self.assertEqual(postprocessor.find(None) , None) + self.assertEqual(postprocessor.find(1234), None) + self.assertEqual(postprocessor.find(None), None) @patch("builtins.__import__") def test_cache(self, import_module): import_module.return_value = MockPostprocessorModule() - for name in (postprocessor.modules): + for name in postprocessor.modules: postprocessor.find(name) self.assertEqual(import_module.call_count, len(postprocessor.modules)) # no new calls to import_module - for name in (postprocessor.modules): + for name in postprocessor.modules: postprocessor.find(name) self.assertEqual(import_module.call_count, len(postprocessor.modules)) class BasePostprocessorTest(unittest.TestCase): - @classmethod def setUpClass(cls): cls.dir = tempfile.TemporaryDirectory() @@ -105,21 +100,19 @@ def _create(self, options=None, data=None): return pp(self.job, options) def _trigger(self, events=None): - for event in (events or ("prepare", "file")): + for event in events or ("prepare", "file"): for callback in self.job.hooks[event]: callback(self.pathfmt) class ClassifyTest(BasePostprocessorTest): - def test_classify_default(self): pp = self._create() - self.assertEqual(pp.mapping, { - ext: directory - for directory, exts in pp.DEFAULT_MAPPING.items() - for ext in exts - }) + self.assertEqual( + pp.mapping, + {ext: directory for directory, exts in pp.DEFAULT_MAPPING.items() for ext in exts}, + ) self.pathfmt.set_extension("jpg") self.pathfmt.build_path() @@ -145,33 +138,41 @@ def test_classify_noop(self): self.assertEqual(mkdirs.call_count, 0) def test_classify_custom(self): - pp = self._create({"mapping": { - "foo/bar": ["foo", "bar"], - }}) - - self.assertEqual(pp.mapping, { - "foo": "foo/bar", - "bar": "foo/bar", - }) + pp = self._create( + { + "mapping": { + "foo/bar": ["foo", "bar"], + } + } + ) + + self.assertEqual( + pp.mapping, + { + "foo": "foo/bar", + "bar": "foo/bar", + }, + ) self.pathfmt.set_extension("foo") self.pathfmt.build_path() pp.prepare(self.pathfmt) - path = os.path.join(self.dir.name, "test", "foo", "bar") - self.assertEqual(self.pathfmt.path, path + "/file.foo") - self.assertEqual(self.pathfmt.realpath, path + "/file.foo") + path = Path(self.dir.name) / "test" / "foo" / "bar" + self.assertEqual(self.pathfmt.path, str(path / "file.foo")) + self.assertEqual(self.pathfmt.realpath, str(path / "file.foo")) with patch("os.makedirs") as mkdirs: self._trigger() - mkdirs.assert_called_once_with(path, exist_ok=True) + mkdirs.assert_called_once_with(str(path), exist_ok=True) class ExecTest(BasePostprocessorTest): - def test_command_string(self): - self._create({ - "command": "echo {} {_path} {_directory} {_filename} && rm {};", - }) + self._create( + { + "command": "echo {} {_path} {_directory} {_filename} && rm {};", + } + ) with patch("gallery_dl.util.Popen") as p: i = Mock() @@ -180,18 +181,17 @@ def test_command_string(self): self._trigger(("after",)) p.assert_called_once_with( - "echo {0} {0} {1} {2} && rm {0};".format( - self.pathfmt.realpath, - self.pathfmt.realdirectory, - self.pathfmt.filename), - shell=True) + f"echo {self.pathfmt.realpath} {self.pathfmt.realpath} {self.pathfmt.realdirectory} {self.pathfmt.filename} && rm {self.pathfmt.realpath};", + shell=True, + ) i.wait.assert_called_once_with() def test_command_list(self): - self._create({ - "command": ["~/script.sh", "{category}", - "\fE _directory.upper()"], - }) + self._create( + { + "command": ["~/script.sh", "{category}", "\fE _directory.upper()"], + } + ) with patch("gallery_dl.util.Popen") as p: i = Mock() @@ -209,9 +209,11 @@ def test_command_list(self): ) def test_command_returncode(self): - self._create({ - "command": "echo {}", - }) + self._create( + { + "command": "echo {}", + } + ) with patch("gallery_dl.util.Popen") as p: i = Mock() @@ -221,15 +223,19 @@ def test_command_returncode(self): with self.assertLogs() as log: self._trigger(("after",)) - msg = ("WARNING:postprocessor.exec:'echo {}' returned with " - "non-zero exit status (123)".format(self.pathfmt.realpath)) + msg = ( + f"WARNING:postprocessor.exec:'echo {self.pathfmt.realpath}' returned with " + "non-zero exit status (123)" + ) self.assertEqual(log.output[0], msg) def test_async(self): - self._create({ - "async" : True, - "command": "echo {}", - }) + self._create( + { + "async": True, + "command": "echo {}", + } + ) with patch("gallery_dl.util.Popen") as p: i = Mock() @@ -241,7 +247,6 @@ def test_async(self): class HashTest(BasePostprocessorTest): - def test_default(self): self._create({}) @@ -251,10 +256,8 @@ def test_default(self): self._trigger() kwdict = self.pathfmt.kwdict - self.assertEqual( - "35c9c9c7c90ad764bae9e2623f522c24", kwdict["md5"], "md5") - self.assertEqual( - "14d3d804494ef4e57d72de63e4cfee761240471a", kwdict["sha1"], "sha1") + self.assertEqual("35c9c9c7c90ad764bae9e2623f522c24", kwdict["md5"], "md5") + self.assertEqual("14d3d804494ef4e57d72de63e4cfee761240471a", kwdict["sha1"], "sha1") def test_custom_hashes(self): self._create({"hashes": "sha256:a,sha512:b"}) @@ -267,11 +270,15 @@ def test_custom_hashes(self): kwdict = self.pathfmt.kwdict self.assertEqual( "4775b55be17206445d7015a5fc7656f38a74b880670523c3b175455f885f2395", - kwdict["a"], "sha256") + kwdict["a"], + "sha256", + ) self.assertEqual( "6028f9e6957f4ca929941318c4bba6258713fd5162f9e33bd10e1c456d252700" "3e1095b50736c4fd1e2deea152e3c8ecd5993462a747208e4d842659935a1c62", - kwdict["b"], "sha512") + kwdict["b"], + "sha512", + ) def test_custom_hashes_dict(self): self._create({"hashes": {"a": "sha256", "b": "sha512"}}) @@ -284,33 +291,39 @@ def test_custom_hashes_dict(self): kwdict = self.pathfmt.kwdict self.assertEqual( "4775b55be17206445d7015a5fc7656f38a74b880670523c3b175455f885f2395", - kwdict["a"], "sha256") + kwdict["a"], + "sha256", + ) self.assertEqual( "6028f9e6957f4ca929941318c4bba6258713fd5162f9e33bd10e1c456d252700" "3e1095b50736c4fd1e2deea152e3c8ecd5993462a747208e4d842659935a1c62", - kwdict["b"], "sha512") + kwdict["b"], + "sha512", + ) class MetadataTest(BasePostprocessorTest): - def test_metadata_default(self): pp = self._create() # default arguments - self.assertEqual(pp.write , pp._write_json) + self.assertEqual(pp.write, pp._write_json) self.assertEqual(pp.extension, "json") self.assertTrue(callable(pp._json_encode)) def test_metadata_json(self): - pp = self._create({ - "mode" : "json", - "extension" : "JSON", - }, { - "public" : "hello ワールド", - "_private" : "foo バー", - }) - - self.assertEqual(pp.write , pp._write_json) + pp = self._create( + { + "mode": "json", + "extension": "JSON", + }, + { + "public": "hello ワールド", + "_private": "foo バー", + }, + ) + + self.assertEqual(pp.write, pp._write_json) self.assertEqual(pp.extension, "JSON") self.assertTrue(callable(pp._json_encode)) @@ -322,31 +335,37 @@ def test_metadata_json(self): if sys.hexversion >= 0x3060000: # python 3.4 & 3.5 have random order without 'sort: True' - self.assertEqual(self._output(m), """{ + self.assertEqual( + self._output(m), + """{ "category": "test", "filename": "file", "extension": "ext", "public": "hello ワールド" } -""") +""", + ) def test_metadata_json_options(self): - pp = self._create({ - "mode" : "json", - "ascii" : True, - "sort" : True, - "separators": [",", " : "], - "private" : True, - "indent" : None, - "open" : "a", - "encoding" : "UTF-8", - "extension" : "JSON", - }, { - "public" : "hello ワールド", - "_private" : "foo バー", - }) - - self.assertEqual(pp.write , pp._write_json) + pp = self._create( + { + "mode": "json", + "ascii": True, + "sort": True, + "separators": [",", " : "], + "private": True, + "indent": None, + "open": "a", + "encoding": "UTF-8", + "extension": "JSON", + }, + { + "public": "hello ワールド", + "_private": "foo バー", + }, + ) + + self.assertEqual(pp.write, pp._write_json) self.assertEqual(pp.extension, "JSON") self.assertTrue(callable(pp._json_encode)) @@ -355,13 +374,16 @@ def test_metadata_json_options(self): path = self.pathfmt.realpath + ".JSON" m.assert_called_once_with(path, "a", encoding="UTF-8") - self.assertEqual(self._output(m), """{\ + self.assertEqual( + self._output(m), + """{\ "_private" : "foo \\u30d0\\u30fc",\ "category" : "test",\ "extension" : "ext",\ "filename" : "file",\ "public" : "hello \\u30ef\\u30fc\\u30eb\\u30c9"} -""") +""", + ) def test_metadata_tags(self): pp = self._create( @@ -417,10 +439,12 @@ def test_metadata_tags_dict(self): def test_metadata_tags_list_of_dict(self): self._create( {"mode": "tags"}, - {"tags": [ - {"g": "foobar1", "m": "foobar2", "u": True}, - {"g": None, "m": "foobarbaz", "u": [3, 4]}, - ]}, + { + "tags": [ + {"g": "foobar1", "m": "foobar2", "u": True}, + {"g": None, "m": "foobarbaz", "u": [3, 4]}, + ] + }, ) with patch("builtins.open", mock_open()) as m: self._trigger() @@ -444,10 +468,12 @@ def test(pp_info): test({"format": "{foo}\n{missing}\n"}) def test_metadata_extfmt(self): - pp = self._create({ - "extension" : "ignored", - "extension-format": "json", - }) + pp = self._create( + { + "extension": "ignored", + "extension-format": "json", + } + ) self.assertEqual(pp._filename, pp._filename_extfmt) @@ -458,9 +484,11 @@ def test_metadata_extfmt(self): m.assert_called_once_with(path, "w", encoding="utf-8") def test_metadata_extfmt_2(self): - self._create({ - "extension-format": "{extension!u}-data:{category:Res/ES/}", - }) + self._create( + { + "extension-format": "{extension!u}-data:{category:Res/ES/}", + } + ) self.pathfmt.prefix = "2." with patch("builtins.open", mock_open()) as m: @@ -470,9 +498,11 @@ def test_metadata_extfmt_2(self): m.assert_called_once_with(path, "w", encoding="utf-8") def test_metadata_directory(self): - self._create({ - "directory": "metadata", - }) + self._create( + { + "directory": "metadata", + } + ) with patch("builtins.open", mock_open()) as m: self._trigger() @@ -481,10 +511,12 @@ def test_metadata_directory(self): m.assert_called_once_with(path, "w", encoding="utf-8") def test_metadata_directory_2(self): - self._create({ - "directory" : "metadata////", - "extension-format": "json", - }) + self._create( + { + "directory": "metadata////", + "extension-format": "json", + } + ) with patch("builtins.open", mock_open()) as m: self._trigger() @@ -514,10 +546,12 @@ def test_metadata_basedirectory(self): m.assert_called_once_with(path, "w", encoding="utf-8") def test_metadata_basedirectory_custom(self): - self._create({ - "base-directory": "/home/test", - "directory": "meta", - }) + self._create( + { + "base-directory": "/home/test", + "directory": "meta", + } + ) with patch("builtins.open", mock_open()) as m: self._trigger() @@ -526,10 +560,12 @@ def test_metadata_basedirectory_custom(self): m.assert_called_once_with(path, "w", encoding="utf-8") def test_metadata_filename(self): - self._create({ - "filename" : "{category}_{filename}_/meta/\n\r.data", - "extension-format": "json", - }) + self._create( + { + "filename": "{category}_{filename}_/meta/\n\r.data", + "extension-format": "json", + } + ) with patch("builtins.open", mock_open()) as m: self._trigger() @@ -543,21 +579,27 @@ def test_metadata_stdout(self): with patch("sys.stdout", Mock()) as m: self._trigger() - self.assertEqual(self._output(m), """\ + self.assertEqual( + self._output(m), + """\ {"category": "test", "extension": "ext", "filename": "file"} -""") +""", + ) def test_metadata_modify(self): kwdict = {"foo": 0, "bar": {"bax": 1, "bay": 2, "baz": 3, "ba2": {}}} - self._create({ - "mode": "modify", - "fields": { - "foo" : "{filename}-{foo!s}", - "foo2" : "\fE bar['bax'] + 122", - "bar[\"baz\"]" : "{_now}", - "bar['ba2'][a]": "test", + self._create( + { + "mode": "modify", + "fields": { + "foo": "{filename}-{foo!s}", + "foo2": "\fE bar['bax'] + 122", + 'bar["baz"]': "{_now}", + "bar['ba2'][a]": "test", + }, }, - }, kwdict) + kwdict, + ) pdict = self.pathfmt.kwdict self.assertIsNot(kwdict, pdict) @@ -566,7 +608,7 @@ def test_metadata_modify(self): self._trigger() - self.assertEqual(pdict["foo"] , "file-0") + self.assertEqual(pdict["foo"], "file-0") self.assertEqual(pdict["foo2"], 123) self.assertEqual(pdict["bar"]["ba2"]["a"], "test") self.assertIsInstance(pdict["bar"]["baz"], datetime) @@ -580,10 +622,13 @@ def test_metadata_delete(self): "baz": {"a": 3, "b": 4}, }, } - self._create({ - "mode": "delete", - "fields": ["foo", "bar['bax']", "bar[\"baz\"][a]"], - }, kwdict) + self._create( + { + "mode": "delete", + "fields": ["foo", "bar['bax']", 'bar["baz"][a]'], + }, + kwdict, + ) pdict = self.pathfmt.kwdict self.assertIsNot(kwdict, pdict) @@ -606,8 +651,7 @@ def test_metadata_delete(self): def test_metadata_option_skip(self): self._create({"skip": True}) - with patch("builtins.open", mock_open()) as m, \ - patch("os.path.exists") as e: + with patch("builtins.open", mock_open()) as m, patch("os.path.exists") as e: e.return_value = True self._trigger() @@ -615,8 +659,7 @@ def test_metadata_option_skip(self): self.assertTrue(not m.called) self.assertTrue(not len(self._output(m))) - with patch("builtins.open", mock_open()) as m, \ - patch("os.path.exists") as e: + with patch("builtins.open", mock_open()) as m, patch("os.path.exists") as e: e.return_value = False self._trigger() @@ -630,8 +673,7 @@ def test_metadata_option_skip(self): def test_metadata_option_skip_false(self): self._create({"skip": False}) - with patch("builtins.open", mock_open()) as m, \ - patch("os.path.exists") as e: + with patch("builtins.open", mock_open()) as m, patch("os.path.exists") as e: self._trigger() self.assertTrue(not e.called) @@ -646,11 +688,14 @@ def test_metadata_option_include(self): with patch("builtins.open", mock_open()) as m: self._trigger() - self.assertEqual(self._output(m), """{ + self.assertEqual( + self._output(m), + """{ "_private": "foo バー", "filename": "file" } -""") +""", + ) def test_metadata_option_exclude(self): self._create( @@ -661,23 +706,21 @@ def test_metadata_option_exclude(self): with patch("builtins.open", mock_open()) as m: self._trigger() - self.assertEqual(self._output(m), """{ + self.assertEqual( + self._output(m), + """{ "extension": "ext", "public": "hello ワールド" } -""") +""", + ) @staticmethod def _output(mock): - return "".join( - call[1][0] - for call in mock.mock_calls - if call[0].endswith("write") - ) + return "".join(call[1][0] for call in mock.mock_calls if call[0].endswith("write")) class MtimeTest(BasePostprocessorTest): - def test_mtime_datetime(self): self._create(None, {"date": datetime(1980, 1, 1)}) self._trigger() @@ -710,7 +753,6 @@ def test_mtime_value(self): class PythonTest(BasePostprocessorTest): - def test_module(self): path = os.path.join(self.dir.name, "module.py") self._write_module(path) @@ -744,7 +786,6 @@ def calc(kwdict): class RenameTest(BasePostprocessorTest): - def _prepare(self, filename): path = self.pathfmt.realdirectory shutil.rmtree(path, ignore_errors=True) @@ -791,22 +832,29 @@ def test_rename_skip(self): with self.assertLogs("postprocessor.rename", level="WARNING") as cm: self._trigger() - self.assertTrue(cm.output[0].startswith( - "WARNING:postprocessor.rename:Not renaming " - "'12345.ext' to 'file.ext'")) + self.assertTrue( + cm.output[0].startswith( + "WARNING:postprocessor.rename:Not renaming " "'12345.ext' to 'file.ext'" + ) + ) self.assertEqual(sorted(os.listdir(path)), ["12345.ext", "file.ext"]) class ZipTest(BasePostprocessorTest): - def test_zip_default(self): pp = self._create() self.assertEqual(self.job.hooks["file"][0], pp.write_fast) self.assertEqual(pp.path, self.pathfmt.realdirectory[:-1]) self.assertEqual(pp.delete, True) - self.assertEqual(pp.args, ( - pp.path + ".zip", "a", zipfile.ZIP_STORED, True, - )) + self.assertEqual( + pp.args, + ( + pp.path + ".zip", + "a", + zipfile.ZIP_STORED, + True, + ), + ) self.assertTrue(pp.args[0].endswith("/test.zip")) def test_zip_safe(self): @@ -814,41 +862,54 @@ def test_zip_safe(self): self.assertEqual(self.job.hooks["file"][0], pp.write_safe) self.assertEqual(pp.path, self.pathfmt.realdirectory[:-1]) self.assertEqual(pp.delete, True) - self.assertEqual(pp.args, ( - pp.path + ".zip", "a", zipfile.ZIP_STORED, True, - )) + self.assertEqual( + pp.args, + ( + pp.path + ".zip", + "a", + zipfile.ZIP_STORED, + True, + ), + ) self.assertTrue(pp.args[0].endswith("/test.zip")) def test_zip_options(self): - pp = self._create({ - "keep-files": True, - "compression": "zip", - "extension": "cbz", - }) + pp = self._create( + { + "keep-files": True, + "compression": "zip", + "extension": "cbz", + } + ) self.assertEqual(pp.delete, False) - self.assertEqual(pp.args, ( - pp.path + ".cbz", "a", zipfile.ZIP_DEFLATED, True, - )) + self.assertEqual( + pp.args, + ( + pp.path + ".cbz", + "a", + zipfile.ZIP_DEFLATED, + True, + ), + ) self.assertTrue(pp.args[0].endswith("/test.cbz")) def test_zip_write(self): with tempfile.NamedTemporaryFile("w", dir=self.dir.name) as file: - pp = self._create({"files": [file.name, "_info_.json"], - "keep-files": True}) + pp = self._create({"files": [file.name, "_info_.json"], "keep-files": True}) filename = os.path.basename(file.name) file.write("foobar\n") # write dummy file with 3 different names for i in range(3): - name = "file{}.ext".format(i) + name = f"file{i}.ext" self.pathfmt.temppath = file.name self.pathfmt.filename = name self._trigger() nti = pp.zfile.NameToInfo - self.assertEqual(len(nti), i+2) + self.assertEqual(len(nti), i + 2) self.assertIn(name, nti) # check file contents @@ -877,7 +938,6 @@ def test_zip_write(self): os.unlink(pp.zfile.filename) def test_zip_write_mock(self): - def side_effect(_, name): pp.zfile.NameToInfo.add(name) @@ -889,7 +949,7 @@ def side_effect(_, name): # write 3 files for i in range(3): self.pathfmt.temppath = self.pathfmt.realdirectory + "file.ext" - self.pathfmt.filename = "file{}.ext".format(i) + self.pathfmt.filename = f"file{i}.ext" self._trigger() # write the last file a second time (should be skipped) diff --git a/test/test_results.py b/test/test_results.py index f36f7984a5..ec7f70ded0 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2015-2023 Mike Fährmann # @@ -7,19 +6,17 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import collections +import datetime +import hashlib +import json import os +import re import sys import unittest -import re -import json -import hashlib -import datetime -import collections - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import \ - extractor, util, job, config, exception, formatter # noqa E402 +from gallery_dl import extractor, util, job, config, exception, formatter # noqa E402 RESULTS = os.environ.get("GDL_TEST_RESULTS") @@ -30,8 +27,7 @@ # temporary issues, etc. -BROKEN = { -} +BROKEN = {} CONFIG = { "cache": { @@ -66,7 +62,6 @@ class TestExtractorResults(unittest.TestCase): - def setUp(self): setup_test_config() @@ -82,7 +77,7 @@ def tearDownClass(cls): if cls._skipped: print("\n\nSkipped tests:") for url, exc in cls._skipped: - print('- {} ("{}")'.format(url, exc)) + print(f'- {url} ("{exc}")') def assertRange(self, value, range, msg=None): if range.step > 1: @@ -104,10 +99,10 @@ def _run_test(self, result): self.assertIs(extr_url.__class__, extr_cls.__class__) if len(result) <= 2: - return # only matching + return None # only matching if auth is None: - auth = (cat in AUTH_REQUIRED) + auth = cat in AUTH_REQUIRED elif not auth: # auth explicitly disabled for key in AUTH_KEYS: @@ -121,7 +116,7 @@ def _run_test(self, result): key = key.split(".") config.set(key[:-1], key[-1], value) if "#range" in result: - config.set((), "image-range" , result["#range"]) + config.set((), "image-range", result["#range"]) config.set((), "chapter-range", result["#range"]) tjob = ResultJob(extr, content=("#sha1_content" in result)) @@ -129,7 +124,7 @@ def _run_test(self, result): if "#exception" in result: with self.assertRaises(result["#exception"], msg="#exception"): tjob.run() - return + return None try: tjob.run() @@ -137,17 +132,15 @@ def _run_test(self, result): pass except exception.HttpError as exc: exc = str(exc) - if re.match(r"'5\d\d ", exc) or \ - re.search(r"\bRead timed out\b", exc): + if re.match(r"'5\d\d ", exc) or re.search(r"\bRead timed out\b", exc): self._skipped.append((result["#url"], exc)) self.skipTest(exc) raise if result.get("#archive", True): self.assertEqual( - len(set(tjob.archive_list)), - len(tjob.archive_list), - msg="archive-id uniqueness") + len(set(tjob.archive_list)), len(tjob.archive_list), msg="archive-id uniqueness" + ) if tjob.queue: # test '_extractor' entries @@ -165,10 +158,7 @@ def _run_test(self, result): # test extraction results if "#sha1_url" in result: - self.assertEqual( - result["#sha1_url"], - tjob.url_hash.hexdigest(), - msg="#sha1_url") + self.assertEqual(result["#sha1_url"], tjob.url_hash.hexdigest(), msg="#sha1_url") if "#sha1_content" in result: expected = result["#sha1_content"] @@ -180,17 +170,15 @@ def _run_test(self, result): if "#sha1_metadata" in result: self.assertEqual( - result["#sha1_metadata"], - tjob.kwdict_hash.hexdigest(), - "#sha1_metadata") + result["#sha1_metadata"], tjob.kwdict_hash.hexdigest(), "#sha1_metadata" + ) if "#count" in result: count = result["#count"] len_urls = len(tjob.url_list) if isinstance(count, str): - self.assertRegex( - count, r"^ *(==|!=|<|<=|>|>=) *\d+ *$", msg="#count") - expr = "{} {}".format(len_urls, count) + self.assertRegex(count, r"^ *(==|!=|<|<=|>|>=) *\d+ *$", msg="#count") + expr = f"{len_urls} {count}" self.assertTrue(eval(expr), msg=expr) elif isinstance(count, range): self.assertRange(len_urls, count, msg="#count") @@ -226,7 +214,7 @@ def _test_kwdict(self, kwdict, tests, parent=None): key = key[1:] if key not in kwdict: continue - path = "{}.{}".format(parent, key) if parent else key + path = f"{parent}.{key}" if parent else key self.assertIn(key, kwdict, msg=path) value = kwdict[key] @@ -243,7 +231,7 @@ def _test_kwdict(self, kwdict, tests, parent=None): for idx, item in enumerate(test): if isinstance(item, dict): subtest = True - subpath = "{}[{}]".format(path, idx) + subpath = f"{path}[{idx}]" self._test_kwdict(value[idx], item, subpath) if not subtest: self.assertEqual(test, value, msg=path) @@ -285,12 +273,9 @@ def __init__(self, url, parent=None, content=False): else: self._update_content = lambda url, kwdict: None - self.format_directory = TestFormatter( - "".join(self.extractor.directory_fmt)).format_map - self.format_filename = TestFormatter( - self.extractor.filename_fmt).format_map - self.format_archive = TestFormatter( - self.extractor.archive_fmt).format_map + self.format_directory = TestFormatter("".join(self.extractor.directory_fmt)).format_map + self.format_filename = TestFormatter(self.extractor.filename_fmt).format_map + self.format_archive = TestFormatter(self.extractor.archive_fmt).format_map def run(self): self._init() @@ -324,8 +309,7 @@ def _update_kwdict(self, kwdict, to_list=True): if to_list: self.kwdict_list.append(kwdict.copy()) kwdict = util.filter_dict(kwdict) - self.kwdict_hash.update( - json.dumps(kwdict, sort_keys=True, default=str).encode()) + self.kwdict_hash.update(json.dumps(kwdict, sort_keys=True, default=str).encode()) def _update_archive(self, kwdict): archive_id = self.format_archive(kwdict) @@ -346,8 +330,7 @@ def _update_content(self, url, kwdict): return -class TestPathfmt(): - +class TestPathfmt: def __init__(self, hashobj): self.hashobj = hashobj self.path = "" @@ -378,7 +361,6 @@ def part_size(self): class TestFormatter(formatter.StringFormatter): - @staticmethod def _noop(_): return "" @@ -389,6 +371,7 @@ def _apply_simple(self, key, fmt): def wrap(obj): return fmt(obj[key]) + return wrap def _apply(self, key, funcs, fmt): @@ -400,6 +383,7 @@ def wrap(obj): for func in funcs: obj = func(obj) return fmt(obj) + return wrap @@ -409,16 +393,13 @@ def setup_test_config(): def load_test_config(): try: - path = os.path.join( - os.path.dirname(os.path.dirname(__file__)), - "archive", "config.json") + path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "archive", "config.json") with open(path) as fp: CONFIG.update(json.loads(fp.read())) except FileNotFoundError: pass except Exception as exc: - sys.exit("Error when loading {}: {}: {}".format( - path, exc.__class__.__name__, exc)) + sys.exit(f"Error when loading {path}: {exc.__class__.__name__}: {exc}") def result_categories(result): @@ -432,6 +413,7 @@ def result_categories(result): def generate_tests(): """Dynamically generate extractor unittests""" + def _generate_method(result): def test(self): print("\n" + result["#url"]) @@ -446,6 +428,7 @@ def test(self): else: self._skipped.append((result["#url"], "manual skip")) self.skipTest(exc) + return test # enable selective testing for direct calls @@ -455,8 +438,7 @@ def test(self): if category.startswith("+"): basecategory = category[1:].lower() - tests = [t for t in results.all() - if result_categories(t)[0].lower() == basecategory] + tests = [t for t in results.all() if result_categories(t)[0].lower() == basecategory] else: tests = results.category(category) @@ -466,11 +448,9 @@ def test(self): tests = [t for t in tests if url in t["#url"]] elif subcategory.startswith("~"): com = subcategory[1:] - tests = [t for t in tests - if "#comment" in t and com in t["#comment"].lower()] + tests = [t for t in tests if "#comment" in t and com in t["#comment"].lower()] else: - tests = [t for t in tests - if result_categories(t)[-1] == subcategory] + tests = [t for t in tests if result_categories(t)[-1] == subcategory] else: tests = results.all() @@ -478,12 +458,12 @@ def test(self): enum = collections.defaultdict(int) for result in tests: base, cat, sub = result_categories(result) - name = "{}_{}".format(cat, sub) + name = f"{cat}_{sub}" enum[name] += 1 method = _generate_method(result) method.__doc__ = result["#url"] - method.__name__ = "test_{}_{}".format(name, enum[name]) + method.__name__ = f"test_{name}_{enum[name]}" setattr(TestExtractorResults, method.__name__, method) diff --git a/test/test_text.py b/test/test_text.py index 1b19c4742a..490dbec69d 100644 --- a/test/test_text.py +++ b/test/test_text.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2015-2022 Mike Fährmann # @@ -7,22 +6,19 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import datetime import os import sys import unittest -import datetime - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import text, util # noqa E402 - INVALID = ((), [], {}, None, 1, 2.3) INVALID_ALT = ((), [], {}, None, "") class TestText(unittest.TestCase): - def test_remove_html(self, f=text.remove_html): result = "Hello World." @@ -31,8 +27,7 @@ def test_remove_html(self, f=text.remove_html): self.assertEqual(f("Hello World."), result) self.assertEqual(f(" Hello World. "), result) self.assertEqual(f("Hello
          World."), result) - self.assertEqual( - f("
          HelloWorld.
          "), result) + self.assertEqual(f("
          HelloWorld.
          "), result) # empty HTML self.assertEqual(f("
          "), "") @@ -56,12 +51,10 @@ def test_split_html(self, f=text.split_html): self.assertEqual(f(" Hello World. "), ["Hello World."]) self.assertEqual(f("Hello
          World."), result) self.assertEqual(f(" Hello
          World. "), result) - self.assertEqual( - f("
          HelloWorld.
          "), result) + self.assertEqual(f("
          HelloWorld.
          "), result) # escaped HTML entities - self.assertEqual( - f("<foo> <bar> "), ["", ""]) + self.assertEqual(f("<foo> <bar> "), ["", ""]) # empty HTML self.assertEqual(f("
          "), empty) @@ -121,17 +114,17 @@ def test_ensure_http_scheme(self, f=text.ensure_http_scheme): def test_root_from_url(self, f=text.root_from_url): result = "https://example.org" - self.assertEqual(f("https://example.org") , result) - self.assertEqual(f("https://example.org/") , result) + self.assertEqual(f("https://example.org"), result) + self.assertEqual(f("https://example.org/"), result) self.assertEqual(f("https://example.org/path"), result) - self.assertEqual(f("example.org/") , result) - self.assertEqual(f("example.org/path/") , result) + self.assertEqual(f("example.org/"), result) + self.assertEqual(f("example.org/path/"), result) result = "http://example.org" - self.assertEqual(f("http://example.org") , result) - self.assertEqual(f("http://example.org/") , result) + self.assertEqual(f("http://example.org"), result) + self.assertEqual(f("http://example.org/"), result) self.assertEqual(f("http://example.org/path/"), result) - self.assertEqual(f("example.org/", "http://") , result) + self.assertEqual(f("example.org/", "http://"), result) def test_filename_from_url(self, f=text.filename_from_url): result = "filename.ext" @@ -142,8 +135,7 @@ def test_filename_from_url(self, f=text.filename_from_url): self.assertEqual(f("/filename.ext"), result) self.assertEqual(f("example.org/filename.ext"), result) self.assertEqual(f("http://example.org/v2/filename.ext"), result) - self.assertEqual( - f("http://example.org/v2/filename.ext?param=value#frag"), result) + self.assertEqual(f("http://example.org/v2/filename.ext?param=value#frag"), result) # invalid arguments for value in INVALID: @@ -159,8 +151,7 @@ def test_ext_from_url(self, f=text.ext_from_url): self.assertEqual(f("/filename.ExT"), result) self.assertEqual(f("example.org/filename.ext"), result) self.assertEqual(f("http://example.org/v2/filename.ext"), result) - self.assertEqual( - f("http://example.org/v2/filename.ext?param=value#frag"), result) + self.assertEqual(f("http://example.org/v2/filename.ext?param=value#frag"), result) # invalid arguments for value in INVALID: @@ -176,8 +167,7 @@ def test_nameext_from_url(self, f=text.nameext_from_url): self.assertEqual(f("/filename.ExT"), result) self.assertEqual(f("example.org/filename.ext"), result) self.assertEqual(f("http://example.org/v2/filename.ext"), result) - self.assertEqual( - f("http://example.org/v2/filename.ext?param=value#frag"), result) + self.assertEqual(f("http://example.org/v2/filename.ext?param=value#frag"), result) # long "extension" fn = "httpswww.example.orgpath-path-path-path-path-path-path-path" @@ -189,7 +179,7 @@ def test_nameext_from_url(self, f=text.nameext_from_url): def test_extract(self, f=text.extract): txt = "" - self.assertEqual(f(txt, "<", ">"), ("a" , 3)) + self.assertEqual(f(txt, "<", ">"), ("a", 3)) self.assertEqual(f(txt, "X", ">"), (None, 0)) self.assertEqual(f(txt, "<", "X"), (None, 0)) @@ -201,9 +191,9 @@ def test_extract(self, f=text.extract): # invalid arguments for value in INVALID: - self.assertEqual(f(value, "<" , ">") , (None, 0)) - self.assertEqual(f(txt , value, ">") , (None, 0)) - self.assertEqual(f(txt , "<" , value), (None, 0)) + self.assertEqual(f(value, "<", ">"), (None, 0)) + self.assertEqual(f(txt, value, ">"), (None, 0)) + self.assertEqual(f(txt, "<", value), (None, 0)) def test_extr(self, f=text.extr): txt = "" @@ -219,13 +209,13 @@ def test_extr(self, f=text.extr): # invalid arguments for value in INVALID: - self.assertEqual(f(value, "<" , ">") , "") - self.assertEqual(f(txt , value, ">") , "") - self.assertEqual(f(txt , "<" , value), "") + self.assertEqual(f(value, "<", ">"), "") + self.assertEqual(f(txt, value, ">"), "") + self.assertEqual(f(txt, "<", value), "") def test_rextract(self, f=text.rextract): txt = "" - self.assertEqual(f(txt, "<", ">"), ("b" , 3)) + self.assertEqual(f(txt, "<", ">"), ("b", 3)) self.assertEqual(f(txt, "X", ">"), (None, -1)) self.assertEqual(f(txt, "<", "X"), (None, -1)) @@ -237,15 +227,14 @@ def test_rextract(self, f=text.rextract): # invalid arguments for value in INVALID: - self.assertEqual(f(value, "<" , ">") , (None, -1)) - self.assertEqual(f(txt , value, ">") , (None, -1)) - self.assertEqual(f(txt , "<" , value), (None, -1)) + self.assertEqual(f(value, "<", ">"), (None, -1)) + self.assertEqual(f(txt, value, ">"), (None, -1)) + self.assertEqual(f(txt, "<", value), (None, -1)) def test_extract_all(self, f=text.extract_all): txt = "[c][b][a]: xyz! [d][e" - self.assertEqual( - f(txt, ()), ({}, 0)) + self.assertEqual(f(txt, ()), ({}, 0)) self.assertEqual( f(txt, (("C", "[", "]"), ("B", "[", "]"), ("A", "[", "]"))), ({"A": "a", "B": "b", "C": "c"}, 9), @@ -289,16 +278,11 @@ def test_extract_iter(self, f=text.extract_iter): def g(*args): return list(f(*args)) - self.assertEqual( - g("", "[", "]"), []) - self.assertEqual( - g("[a]", "[", "]"), ["a"]) - self.assertEqual( - g(txt, "[", "]"), ["c", "b", "a", "d"]) - self.assertEqual( - g(txt, "X", "X"), []) - self.assertEqual( - g(txt, "[", "]", 6), ["a", "d"]) + self.assertEqual(g("", "[", "]"), []) + self.assertEqual(g("[a]", "[", "]"), ["a"]) + self.assertEqual(g(txt, "[", "]"), ["c", "b", "a", "d"]) + self.assertEqual(g(txt, "X", "X"), []) + self.assertEqual(g(txt, "[", "]", 6), ["a", "d"]) def test_extract_from(self, f=text.extract_from): txt = "[c][b][a]: xyz! [d][e" @@ -439,9 +423,9 @@ def test_parse_timestamp(self, f=text.parse_timestamp): null = util.datetime_utcfromtimestamp(0) value = util.datetime_utcfromtimestamp(1555816235) - self.assertEqual(f(0) , null) - self.assertEqual(f("0") , null) - self.assertEqual(f(1555816235) , value) + self.assertEqual(f(0), null) + self.assertEqual(f("0"), null) + self.assertEqual(f(1555816235), value) self.assertEqual(f("1555816235"), value) for value in INVALID_ALT: @@ -452,8 +436,8 @@ def test_parse_datetime(self, f=text.parse_datetime): null = util.datetime_utcfromtimestamp(0) self.assertEqual(f("1970-01-01T00:00:00+00:00"), null) - self.assertEqual(f("1970-01-01T00:00:00+0000") , null) - self.assertEqual(f("1970.01.01", "%Y.%m.%d") , null) + self.assertEqual(f("1970-01-01T00:00:00+0000"), null) + self.assertEqual(f("1970.01.01", "%Y.%m.%d"), null) self.assertEqual( f("2019-05-07T21:25:02+09:00"), diff --git a/test/test_util.py b/test/test_util.py index 0a9ff423d4..30cb849f77 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2015-2023 Mike Fährmann # @@ -7,26 +6,24 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. -import os -import sys -import unittest - +import datetime +import http.cookiejar import io -import time +import itertools +import os +import platform import random import string -import datetime -import platform +import sys import tempfile -import itertools -import http.cookiejar +import time +import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from gallery_dl import util, text, exception # noqa E402 class TestRange(unittest.TestCase): - def test_parse_empty(self, f=util.RangePredicate._parse): self.assertEqual(f(""), []) self.assertEqual(f([]), []) @@ -36,9 +33,7 @@ def test_parse_digit(self, f=util.RangePredicate._parse): self.assertEqual( f("2, 3, 4"), - [range(2, 3), - range(3, 4), - range(4, 5)], + [range(2, 3), range(3, 4), range(4, 5)], ) def test_parse_range(self, f=util.RangePredicate._parse): @@ -49,44 +44,38 @@ def test_parse_range(self, f=util.RangePredicate._parse): self.assertEqual( f("-2,4,6-8,10-"), - [range(1, 3), - range(4, 5), - range(6, 9), - range(10, sys.maxsize)], + [range(1, 3), range(4, 5), range(6, 9), range(10, sys.maxsize)], ) self.assertEqual( f(" - 3 , 4- 4, 2-6"), - [range(1, 4), - range(4, 5), - range(2, 7)], + [range(1, 4), range(4, 5), range(2, 7)], ) def test_parse_slice(self, f=util.RangePredicate._parse): - self.assertEqual(f("2:4") , [range(2, 4)]) - self.assertEqual(f("3::") , [range(3, sys.maxsize)]) - self.assertEqual(f(":4:") , [range(1, 4)]) - self.assertEqual(f("::5") , [range(1, sys.maxsize, 5)]) - self.assertEqual(f("::") , [range(1, sys.maxsize)]) + self.assertEqual(f("2:4"), [range(2, 4)]) + self.assertEqual(f("3::"), [range(3, sys.maxsize)]) + self.assertEqual(f(":4:"), [range(1, 4)]) + self.assertEqual(f("::5"), [range(1, sys.maxsize, 5)]) + self.assertEqual(f("::"), [range(1, sys.maxsize)]) self.assertEqual(f("2:3:4"), [range(2, 3, 4)]) self.assertEqual( f("2:4, 4:, :4, :4:, ::4"), - [range(2, 4), - range(4, sys.maxsize), - range(1, 4), - range(1, 4), - range(1, sys.maxsize, 4)], + [ + range(2, 4), + range(4, sys.maxsize), + range(1, 4), + range(1, 4), + range(1, sys.maxsize, 4), + ], ) self.assertEqual( f(" : 3 , 4: 4, 2:6"), - [range(1, 3), - range(4, 4), - range(2, 6)], + [range(1, 3), range(4, 4), range(2, 6)], ) class TestPredicate(unittest.TestCase): - def test_range_predicate(self): dummy = None @@ -136,10 +125,8 @@ def test_filter_predicate(self): with self.assertRaises(SyntaxError): util.FilterPredicate("(") - self.assertFalse( - util.FilterPredicate("a > 1")(url, {"a": None})) - self.assertFalse( - util.FilterPredicate("b > 1")(url, {"a": 2})) + self.assertFalse(util.FilterPredicate("a > 1")(url, {"a": None})) + self.assertFalse(util.FilterPredicate("b > 1")(url, {"a": 2})) pred = util.FilterPredicate(["a < 3", "b < 4", "c < 5"]) self.assertTrue(pred(url, {"a": 2, "b": 3, "c": 4})) @@ -156,44 +143,48 @@ def test_build_predicate(self): pred = util.build_predicate([util.UniquePredicate()]) self.assertIsInstance(pred, util.UniquePredicate) - pred = util.build_predicate([util.UniquePredicate(), - util.UniquePredicate()]) + pred = util.build_predicate([util.UniquePredicate(), util.UniquePredicate()]) self.assertIs(pred.func, util.chain_predicates) class TestISO639_1(unittest.TestCase): - def test_code_to_language(self): d = "default" - self._run_test(util.code_to_language, { - ("en",): "English", - ("FR",): "French", - ("ja",): "Japanese", - ("xx",): None, - ("" ,): None, - (None,): None, - ("en", d): "English", - ("FR", d): "French", - ("xx", d): d, - ("" , d): d, - (None, d): d, - }) + self._run_test( + util.code_to_language, + { + ("en",): "English", + ("FR",): "French", + ("ja",): "Japanese", + ("xx",): None, + ("",): None, + (None,): None, + ("en", d): "English", + ("FR", d): "French", + ("xx", d): d, + ("", d): d, + (None, d): d, + }, + ) def test_language_to_code(self): d = "default" - self._run_test(util.language_to_code, { - ("English",): "en", - ("fRENch",): "fr", - ("Japanese",): "ja", - ("xx",): None, - ("" ,): None, - (None,): None, - ("English", d): "en", - ("fRENch", d): "fr", - ("xx", d): d, - ("" , d): d, - (None, d): d, - }) + self._run_test( + util.language_to_code, + { + ("English",): "en", + ("fRENch",): "fr", + ("Japanese",): "ja", + ("xx",): None, + ("",): None, + (None,): None, + ("English", d): "en", + ("fRENch", d): "fr", + ("xx", d): d, + ("", d): d, + (None, d): d, + }, + ) def _run_test(self, func, tests): for args, result in tests.items(): @@ -201,9 +192,7 @@ def _run_test(self, func, tests): class TestCookiesTxt(unittest.TestCase): - def test_cookiestxt_load(self): - def _assert(content, expected): cookies = util.cookiestxt_load(io.StringIO(content, None)) for c, e in zip(cookies, expected): @@ -238,16 +227,11 @@ def _assert(content, expected): "www.example.org FALSE / FALSE n4 \n" "www.example.org FALSE /path FALSE 100 n5 v5\n", [ - self._cookie( - "n1", "v1", ".example.org", True, "/", False), - self._cookie( - "n2", "v2", ".example.org", True, "/", True, 2145945600), - self._cookie( - "n3", None, ".example.org", True, "/path", False), - self._cookie( - "n4", "" , "www.example.org", False, "/", False), - self._cookie( - "n5", "v5", "www.example.org", False, "/path", False, 100), + self._cookie("n1", "v1", ".example.org", True, "/", False), + self._cookie("n2", "v2", ".example.org", True, "/", True, 2145945600), + self._cookie("n3", None, ".example.org", True, "/path", False), + self._cookie("n4", "", "www.example.org", False, "/", False), + self._cookie("n5", "v5", "www.example.org", False, "/path", False, 100), ], ) @@ -255,7 +239,6 @@ def _assert(content, expected): util.cookiestxt_load("example.org\tTRUE\t/\tTRUE\t0\tname") def test_cookiestxt_store(self): - def _assert(cookies, expected): fp = io.StringIO(newline=None) util.cookiestxt_store(fp, cookies) @@ -264,23 +247,16 @@ def _assert(cookies, expected): _assert([], "# Netscape HTTP Cookie File\n\n") _assert( [self._cookie("name", "value", ".example.org")], - "# Netscape HTTP Cookie File\n\n" - ".example.org\tTRUE\t/\tTRUE\t0\tname\tvalue\n", + "# Netscape HTTP Cookie File\n\n" ".example.org\tTRUE\t/\tTRUE\t0\tname\tvalue\n", ) _assert( [ - self._cookie( - "n1", "v1", ".example.org", True, "/", False), - self._cookie( - "n2", "v2", ".example.org", True, "/", True, 2145945600), - self._cookie( - "n3", None, ".example.org", True, "/path", False), - self._cookie( - "n4", "" , "www.example.org", False, "/", False), - self._cookie( - "n5", "v5", "www.example.org", False, "/path", False, 100), - self._cookie( - "n6", "v6", "", False), + self._cookie("n1", "v1", ".example.org", True, "/", False), + self._cookie("n2", "v2", ".example.org", True, "/", True, 2145945600), + self._cookie("n3", None, ".example.org", True, "/path", False), + self._cookie("n4", "", "www.example.org", False, "/", False), + self._cookie("n5", "v5", "www.example.org", False, "/path", False, 100), + self._cookie("n6", "v6", "", False), ], "# Netscape HTTP Cookie File\n" "\n" @@ -291,17 +267,30 @@ def _assert(cookies, expected): "www.example.org FALSE /path FALSE 100 n5 v5\n", ) - def _cookie(self, name, value, domain, domain_specified=True, - path="/", secure=True, expires=None): + def _cookie( + self, name, value, domain, domain_specified=True, path="/", secure=True, expires=None + ): return http.cookiejar.Cookie( - 0, name, value, None, False, - domain, domain_specified, domain.startswith("."), - path, False, secure, expires, False, None, None, {}, + 0, + name, + value, + None, + False, + domain, + domain_specified, + domain.startswith("."), + path, + False, + secure, + expires, + False, + None, + None, + {}, ) class TestCompileExpression(unittest.TestCase): - def test_compile_expression(self): expr = util.compile_expression("1 + 2 * 3") self.assertEqual(expr(), 7) @@ -391,7 +380,6 @@ def hash(value): class TestOther(unittest.TestCase): - def test_bencode(self): self.assertEqual(util.bencode(0), "") self.assertEqual(util.bencode(123), "123") @@ -414,34 +402,22 @@ def test_bencode_bdecode(self): def test_advance(self): items = range(5) - self.assertCountEqual( - util.advance(items, 0), items) - self.assertCountEqual( - util.advance(items, 3), range(3, 5)) - self.assertCountEqual( - util.advance(items, 9), []) - self.assertCountEqual( - util.advance(util.advance(items, 1), 2), range(3, 5)) + self.assertCountEqual(util.advance(items, 0), items) + self.assertCountEqual(util.advance(items, 3), range(3, 5)) + self.assertCountEqual(util.advance(items, 9), []) + self.assertCountEqual(util.advance(util.advance(items, 1), 2), range(3, 5)) def test_unique(self): - self.assertSequenceEqual( - list(util.unique("")), "") - self.assertSequenceEqual( - list(util.unique("AABBCC")), "ABC") - self.assertSequenceEqual( - list(util.unique("ABABABCAABBCC")), "ABC") - self.assertSequenceEqual( - list(util.unique([1, 2, 1, 3, 2, 1])), [1, 2, 3]) + self.assertSequenceEqual(list(util.unique("")), "") + self.assertSequenceEqual(list(util.unique("AABBCC")), "ABC") + self.assertSequenceEqual(list(util.unique("ABABABCAABBCC")), "ABC") + self.assertSequenceEqual(list(util.unique([1, 2, 1, 3, 2, 1])), [1, 2, 3]) def test_unique_sequence(self): - self.assertSequenceEqual( - list(util.unique_sequence("")), "") - self.assertSequenceEqual( - list(util.unique_sequence("AABBCC")), "ABC") - self.assertSequenceEqual( - list(util.unique_sequence("ABABABCAABBCC")), "ABABABCABC") - self.assertSequenceEqual( - list(util.unique_sequence([1, 2, 1, 3, 2, 1])), [1, 2, 1, 3, 2, 1]) + self.assertSequenceEqual(list(util.unique_sequence("")), "") + self.assertSequenceEqual(list(util.unique_sequence("AABBCC")), "ABC") + self.assertSequenceEqual(list(util.unique_sequence("ABABABCAABBCC")), "ABABABCABC") + self.assertSequenceEqual(list(util.unique_sequence([1, 2, 1, 3, 2, 1])), [1, 2, 1, 3, 2, 1]) def test_contains(self): c = [1, "2", 3, 4, "5", "foo"] @@ -485,44 +461,28 @@ def test_noop(self): self.assertEqual(util.noop(), None) def test_md5(self): - self.assertEqual(util.md5(b""), - "d41d8cd98f00b204e9800998ecf8427e") - self.assertEqual(util.md5(b"hello"), - "5d41402abc4b2a76b9719d911017c592") - - self.assertEqual(util.md5(""), - "d41d8cd98f00b204e9800998ecf8427e") - self.assertEqual(util.md5("hello"), - "5d41402abc4b2a76b9719d911017c592") - self.assertEqual(util.md5("ワルド"), - "051f29cd6c942cf110a0ccc5729871d2") - - self.assertEqual(util.md5(0), - "d41d8cd98f00b204e9800998ecf8427e") - self.assertEqual(util.md5(()), - "d41d8cd98f00b204e9800998ecf8427e") - self.assertEqual(util.md5(None), - "d41d8cd98f00b204e9800998ecf8427e") + self.assertEqual(util.md5(b""), "d41d8cd98f00b204e9800998ecf8427e") + self.assertEqual(util.md5(b"hello"), "5d41402abc4b2a76b9719d911017c592") + + self.assertEqual(util.md5(""), "d41d8cd98f00b204e9800998ecf8427e") + self.assertEqual(util.md5("hello"), "5d41402abc4b2a76b9719d911017c592") + self.assertEqual(util.md5("ワルド"), "051f29cd6c942cf110a0ccc5729871d2") + + self.assertEqual(util.md5(0), "d41d8cd98f00b204e9800998ecf8427e") + self.assertEqual(util.md5(()), "d41d8cd98f00b204e9800998ecf8427e") + self.assertEqual(util.md5(None), "d41d8cd98f00b204e9800998ecf8427e") def test_sha1(self): - self.assertEqual(util.sha1(b""), - "da39a3ee5e6b4b0d3255bfef95601890afd80709") - self.assertEqual(util.sha1(b"hello"), - "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d") - - self.assertEqual(util.sha1(""), - "da39a3ee5e6b4b0d3255bfef95601890afd80709") - self.assertEqual(util.sha1("hello"), - "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d") - self.assertEqual(util.sha1("ワルド"), - "0cbe319081aa0e9298448ec2bb16df8c494aa04e") - - self.assertEqual(util.sha1(0), - "da39a3ee5e6b4b0d3255bfef95601890afd80709") - self.assertEqual(util.sha1(()), - "da39a3ee5e6b4b0d3255bfef95601890afd80709") - self.assertEqual(util.sha1(None), - "da39a3ee5e6b4b0d3255bfef95601890afd80709") + self.assertEqual(util.sha1(b""), "da39a3ee5e6b4b0d3255bfef95601890afd80709") + self.assertEqual(util.sha1(b"hello"), "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d") + + self.assertEqual(util.sha1(""), "da39a3ee5e6b4b0d3255bfef95601890afd80709") + self.assertEqual(util.sha1("hello"), "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d") + self.assertEqual(util.sha1("ワルド"), "0cbe319081aa0e9298448ec2bb16df8c494aa04e") + + self.assertEqual(util.sha1(0), "da39a3ee5e6b4b0d3255bfef95601890afd80709") + self.assertEqual(util.sha1(()), "da39a3ee5e6b4b0d3255bfef95601890afd80709") + self.assertEqual(util.sha1(None), "da39a3ee5e6b4b0d3255bfef95601890afd80709") def test_import_file(self): module = util.import_file("datetime") @@ -544,7 +504,6 @@ def test_import_file(self): self.assertIs(module.datetime, datetime) def test_build_duration_func(self, f=util.build_duration_func): - def test_single(df, v): for _ in range(10): self.assertEqual(df(), v) @@ -575,54 +534,52 @@ def test_range(df, lower, upper): def test_extractor_filter(self): # empty func = util.build_extractor_filter("") - self.assertEqual(func(TestExtractor) , True) + self.assertEqual(func(TestExtractor), True) self.assertEqual(func(TestExtractorParent), True) - self.assertEqual(func(TestExtractorAlt) , True) + self.assertEqual(func(TestExtractorAlt), True) # category func = util.build_extractor_filter("test_category") - self.assertEqual(func(TestExtractor) , False) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), False) - self.assertEqual(func(TestExtractorAlt) , True) + self.assertEqual(func(TestExtractorAlt), True) # subcategory func = util.build_extractor_filter("*:test_subcategory") - self.assertEqual(func(TestExtractor) , False) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), True) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) # basecategory func = util.build_extractor_filter("test_basecategory") - self.assertEqual(func(TestExtractor) , False) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), False) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) # category-subcategory pair func = util.build_extractor_filter("test_category:test_subcategory") - self.assertEqual(func(TestExtractor) , False) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), True) - self.assertEqual(func(TestExtractorAlt) , True) + self.assertEqual(func(TestExtractorAlt), True) # combination - func = util.build_extractor_filter( - ["test_category", "*:test_subcategory"]) - self.assertEqual(func(TestExtractor) , False) + func = util.build_extractor_filter(["test_category", "*:test_subcategory"]) + self.assertEqual(func(TestExtractor), False) self.assertEqual(func(TestExtractorParent), False) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) # whitelist - func = util.build_extractor_filter( - "test_category:test_subcategory", negate=False) - self.assertEqual(func(TestExtractor) , True) + func = util.build_extractor_filter("test_category:test_subcategory", negate=False) + self.assertEqual(func(TestExtractor), True) self.assertEqual(func(TestExtractorParent), False) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) func = util.build_extractor_filter( - ["test_category:test_subcategory", "*:test_subcategory_parent"], - negate=False) - self.assertEqual(func(TestExtractor) , True) + ["test_category:test_subcategory", "*:test_subcategory_parent"], negate=False + ) + self.assertEqual(func(TestExtractor), True) self.assertEqual(func(TestExtractorParent), True) - self.assertEqual(func(TestExtractorAlt) , False) + self.assertEqual(func(TestExtractorAlt), False) def test_generate_token(self): tokens = set() @@ -638,35 +595,33 @@ def test_generate_token(self): self.assertRegex(token, r"^[0-9a-f]+$") def test_format_value(self): - self.assertEqual(util.format_value(0) , "0") - self.assertEqual(util.format_value(1) , "1") - self.assertEqual(util.format_value(12) , "12") - self.assertEqual(util.format_value(123) , "123") - self.assertEqual(util.format_value(1234) , "1.23k") - self.assertEqual(util.format_value(12345) , "12.34k") - self.assertEqual(util.format_value(123456) , "123.45k") - self.assertEqual(util.format_value(1234567) , "1.23M") - self.assertEqual(util.format_value(12345678) , "12.34M") - self.assertEqual(util.format_value(123456789) , "123.45M") + self.assertEqual(util.format_value(0), "0") + self.assertEqual(util.format_value(1), "1") + self.assertEqual(util.format_value(12), "12") + self.assertEqual(util.format_value(123), "123") + self.assertEqual(util.format_value(1234), "1.23k") + self.assertEqual(util.format_value(12345), "12.34k") + self.assertEqual(util.format_value(123456), "123.45k") + self.assertEqual(util.format_value(1234567), "1.23M") + self.assertEqual(util.format_value(12345678), "12.34M") + self.assertEqual(util.format_value(123456789), "123.45M") self.assertEqual(util.format_value(1234567890), "1.23G") def test_combine_dict(self): - self.assertEqual( - util.combine_dict({}, {}), - {}) - self.assertEqual( - util.combine_dict({1: 1, 2: 2}, {2: 4, 4: 8}), - {1: 1, 2: 4, 4: 8}) + self.assertEqual(util.combine_dict({}, {}), {}) + self.assertEqual(util.combine_dict({1: 1, 2: 2}, {2: 4, 4: 8}), {1: 1, 2: 4, 4: 8}) self.assertEqual( util.combine_dict( - {1: {11: 22, 12: 24}, 2: {13: 26, 14: 28}}, - {1: {11: 33, 13: 39}, 2: "str"}), - {1: {11: 33, 12: 24, 13: 39}, 2: "str"}) + {1: {11: 22, 12: 24}, 2: {13: 26, 14: 28}}, {1: {11: 33, 13: 39}, 2: "str"} + ), + {1: {11: 33, 12: 24, 13: 39}, 2: "str"}, + ) self.assertEqual( util.combine_dict( - {1: {2: {3: {4: {"1": "a", "2": "b"}}}}}, - {1: {2: {3: {4: {"1": "A", "3": "C"}}}}}), - {1: {2: {3: {4: {"1": "A", "2": "b", "3": "C"}}}}}) + {1: {2: {3: {4: {"1": "a", "2": "b"}}}}}, {1: {2: {3: {4: {"1": "A", "3": "C"}}}}} + ), + {1: {2: {3: {4: {"1": "A", "2": "b", "3": "C"}}}}}, + ) def test_transform_dict(self): d = {} @@ -675,13 +630,11 @@ def test_transform_dict(self): d = {1: 123, 2: "123", 3: True, 4: None} util.transform_dict(d, str) - self.assertEqual( - d, {1: "123", 2: "123", 3: "True", 4: "None"}) + self.assertEqual(d, {1: "123", 2: "123", 3: "True", 4: "None"}) d = {1: 123, 2: "123", 3: "foo", 4: {11: 321, 12: "321", 13: "bar"}} util.transform_dict(d, text.parse_int) - self.assertEqual( - d, {1: 123, 2: 123, 3: 0, 4: {11: 321, 12: 321, 13: 0}}) + self.assertEqual(d, {1: 123, 2: 123, 3: 0, 4: {11: 321, 12: 321, 13: 0}}) def test_filter_dict(self): d = {} @@ -699,7 +652,6 @@ def test_filter_dict(self): self.assertEqual(r, {"foo": 123}) def test_enumerate_reversed(self): - seq = [11, 22, 33] result = [(3, 33), (2, 22), (1, 11)] @@ -715,75 +667,62 @@ def assertEqual(it1, it2): for i1, i2 in itertools.zip_longest(it1, it2): ae(i1, i2) - assertEqual( - util.enumerate_reversed(seq), [(2, 33), (1, 22), (0, 11)]) - assertEqual( - util.enumerate_reversed(seq, 1), result) - assertEqual( - util.enumerate_reversed(seq, 2), [(4, 33), (3, 22), (2, 11)]) - - assertEqual( - util.enumerate_reversed(gen(), 0, len(seq)), - [(2, 33), (1, 22), (0, 11)]) - assertEqual( - util.enumerate_reversed(gen(), 1, len(seq)), result) - assertEqual( - util.enumerate_reversed(gen_2(), 1, len(seq)), result) - assertEqual( - util.enumerate_reversed(gen_2(), 2, len(seq)), - [(4, 33), (3, 22), (2, 11)]) + assertEqual(util.enumerate_reversed(seq), [(2, 33), (1, 22), (0, 11)]) + assertEqual(util.enumerate_reversed(seq, 1), result) + assertEqual(util.enumerate_reversed(seq, 2), [(4, 33), (3, 22), (2, 11)]) + + assertEqual(util.enumerate_reversed(gen(), 0, len(seq)), [(2, 33), (1, 22), (0, 11)]) + assertEqual(util.enumerate_reversed(gen(), 1, len(seq)), result) + assertEqual(util.enumerate_reversed(gen_2(), 1, len(seq)), result) + assertEqual(util.enumerate_reversed(gen_2(), 2, len(seq)), [(4, 33), (3, 22), (2, 11)]) def test_number_to_string(self, f=util.number_to_string): - self.assertEqual(f(1) , "1") - self.assertEqual(f(1.0) , "1.0") - self.assertEqual(f("1.0") , "1.0") - self.assertEqual(f([1]) , [1]) + self.assertEqual(f(1), "1") + self.assertEqual(f(1.0), "1.0") + self.assertEqual(f("1.0"), "1.0") + self.assertEqual(f([1]), [1]) self.assertEqual(f({1: 2}), {1: 2}) - self.assertEqual(f(True) , True) - self.assertEqual(f(None) , None) + self.assertEqual(f(True), True) + self.assertEqual(f(None), None) def test_to_string(self, f=util.to_string): - self.assertEqual(f(1) , "1") - self.assertEqual(f(1.0) , "1.0") + self.assertEqual(f(1), "1") + self.assertEqual(f(1.0), "1.0") self.assertEqual(f("1.0"), "1.0") - self.assertEqual(f("") , "") - self.assertEqual(f(None) , "") - self.assertEqual(f(0) , "") + self.assertEqual(f(""), "") + self.assertEqual(f(None), "") + self.assertEqual(f(0), "") self.assertEqual(f(["a"]), "a") - self.assertEqual(f([1]) , "1") + self.assertEqual(f([1]), "1") self.assertEqual(f(["a", "b", "c"]), "a, b, c") self.assertEqual(f([1, 2, 3]), "1, 2, 3") def test_datetime_to_timestamp(self, f=util.datetime_to_timestamp): self.assertEqual(f(util.EPOCH), 0.0) self.assertEqual(f(datetime.datetime(2010, 1, 1)), 1262304000.0) - self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)), - 1262304000.128000) + self.assertEqual(f(datetime.datetime(2010, 1, 1, 0, 0, 0, 128000)), 1262304000.128000) with self.assertRaises(TypeError): f(None) - def test_datetime_to_timestamp_string( - self, f=util.datetime_to_timestamp_string): + def test_datetime_to_timestamp_string(self, f=util.datetime_to_timestamp_string): self.assertEqual(f(util.EPOCH), "0") self.assertEqual(f(datetime.datetime(2010, 1, 1)), "1262304000") self.assertEqual(f(None), "") - def test_datetime_from_timestamp( - self, f=util.datetime_from_timestamp): + def test_datetime_from_timestamp(self, f=util.datetime_from_timestamp): self.assertEqual(f(0.0), util.EPOCH) self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) - self.assertEqual(f(1262304000.128000).replace(microsecond=0), - datetime.datetime(2010, 1, 1, 0, 0, 0)) + self.assertEqual( + f(1262304000.128000).replace(microsecond=0), datetime.datetime(2010, 1, 1, 0, 0, 0) + ) - def test_datetime_utcfromtimestamp( - self, f=util.datetime_utcfromtimestamp): + def test_datetime_utcfromtimestamp(self, f=util.datetime_utcfromtimestamp): self.assertEqual(f(0.0), util.EPOCH) self.assertEqual(f(1262304000.0), datetime.datetime(2010, 1, 1)) - def test_datetime_utcnow( - self, f=util.datetime_utcnow): + def test_datetime_utcnow(self, f=util.datetime_utcnow): self.assertIsInstance(f(), datetime.datetime) def test_universal_none(self): @@ -893,7 +832,7 @@ def test_null_context(self): self.assertIs(exc, exc_orig) -class TestExtractor(): +class TestExtractor: category = "test_category" subcategory = "test_subcategory" basecategory = "test_basecategory" diff --git a/test/test_ytdl.py b/test/test_ytdl.py index f7eb671276..20c3af6668 100644 --- a/test/test_ytdl.py +++ b/test/test_ytdl.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # Copyright 2022-2023 Mike Fährmann # @@ -23,37 +22,35 @@ def setUpClass(cls): try: cls.module = __import__(cls.module_name) except (ImportError, SyntaxError): - raise unittest.SkipTest("cannot import module '{}'".format( - cls.module_name)) + raise unittest.SkipTest(f"cannot import module '{cls.module_name}'") cls.default = ytdl.parse_command_line(cls.module, []) def test_ignore_errors(self): - self._("--ignore-errors" , "ignoreerrors", True) + self._("--ignore-errors", "ignoreerrors", True) self._("--abort-on-error", "ignoreerrors", False) def test_default_search(self): - self._(["--default-search", "foo"] , "default_search", "foo") + self._(["--default-search", "foo"], "default_search", "foo") def test_mark_watched(self): - self._("--mark-watched" , "mark_watched", True) + self._("--mark-watched", "mark_watched", True) self._("--no-mark-watched", "mark_watched", False) def test_proxy(self): - self._(["--proxy", "socks5://127.0.0.1:1080/"], - "proxy", "socks5://127.0.0.1:1080/") - self._(["--cn-verification-proxy", "https://127.0.0.1"], - "cn_verification_proxy", "https://127.0.0.1") - self._(["--geo-verification-proxy", "127.0.0.1"], - "geo_verification_proxy", "127.0.0.1") + self._(["--proxy", "socks5://127.0.0.1:1080/"], "proxy", "socks5://127.0.0.1:1080/") + self._( + ["--cn-verification-proxy", "https://127.0.0.1"], + "cn_verification_proxy", + "https://127.0.0.1", + ) + self._(["--geo-verification-proxy", "127.0.0.1"], "geo_verification_proxy", "127.0.0.1") def test_network_options(self): - self._(["--socket-timeout", "3.5"], - "socket_timeout", 3.5) - self._(["--source-address", "127.0.0.1"], - "source_address", "127.0.0.1") - self._("-4" , "source_address", "0.0.0.0") + self._(["--socket-timeout", "3.5"], "socket_timeout", 3.5) + self._(["--source-address", "127.0.0.1"], "source_address", "127.0.0.1") + self._("-4", "source_address", "0.0.0.0") self._("--force-ipv4", "source_address", "0.0.0.0") - self._("-6" , "source_address", "::") + self._("-6", "source_address", "::") self._("--force-ipv6", "source_address", "::") def test_thumbnail_options(self): @@ -61,26 +58,26 @@ def test_thumbnail_options(self): self._("--write-all-thumbnails", "write_all_thumbnails", True) def test_authentication_options(self): - self._(["-u" , "foo"], "username", "foo") + self._(["-u", "foo"], "username", "foo") self._(["--username", "foo"], "username", "foo") - self._(["-p" , "bar"], "password", "bar") + self._(["-p", "bar"], "password", "bar") self._(["--password", "bar"], "password", "bar") - self._(["--ap-mso" , "mso"], "ap_mso", "mso") + self._(["--ap-mso", "mso"], "ap_mso", "mso") self._(["--ap-username", "foo"], "ap_username", "foo") self._(["--ap-password", "bar"], "ap_password", "bar") - self._(["-2" , "pass"], "twofactor", "pass") + self._(["-2", "pass"], "twofactor", "pass") self._(["--twofactor", "pass"], "twofactor", "pass") self._(["--video-password", "pass"], "videopassword", "pass") - self._("-n" , "usenetrc", True) + self._("-n", "usenetrc", True) self._("--netrc", "usenetrc", True) def test_subtitle_options(self): - self._("--write-sub" , "writesubtitles" , True) + self._("--write-sub", "writesubtitles", True) self._("--write-auto-sub", "writeautomaticsub", True) self._(["--sub-format", "best"], "subtitlesformat", "best") @@ -100,8 +97,9 @@ def test_geo_bypass(self): self._("--geo-bypass", "geo_bypass", True) self._("--no-geo-bypass", "geo_bypass", False) self._(["--geo-bypass-country", "EN"], "geo_bypass_country", "EN") - self._(["--geo-bypass-ip-block", "198.51.100.14/24"], - "geo_bypass_ip_block", "198.51.100.14/24") + self._( + ["--geo-bypass-ip-block", "198.51.100.14/24"], "geo_bypass_ip_block", "198.51.100.14/24" + ) def test_headers(self): headers = self.module.std_headers @@ -116,41 +114,58 @@ def test_headers(self): self.assertNotEqual(headers["Accept"], "*/*") self.assertNotIn("DNT", headers) - self._([ - "--add-header", "accept:*/*", - "--add-header", "dnt:1", - ]) + self._( + [ + "--add-header", + "accept:*/*", + "--add-header", + "dnt:1", + ] + ) self.assertEqual(headers["accept"], "*/*") self.assertEqual(headers["dnt"], "1") def test_extract_audio(self): opts = self._(["--extract-audio"]) - self.assertEqual(opts["postprocessors"][0], { - "key": "FFmpegExtractAudio", - "preferredcodec": "best", - "preferredquality": "5", - "nopostoverwrites": False, - }) - - opts = self._([ - "--extract-audio", - "--audio-format", "opus", - "--audio-quality", "9", - "--no-post-overwrites", - ]) - self.assertEqual(opts["postprocessors"][0], { - "key": "FFmpegExtractAudio", - "preferredcodec": "opus", - "preferredquality": "9", - "nopostoverwrites": True, - }) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "FFmpegExtractAudio", + "preferredcodec": "best", + "preferredquality": "5", + "nopostoverwrites": False, + }, + ) + + opts = self._( + [ + "--extract-audio", + "--audio-format", + "opus", + "--audio-quality", + "9", + "--no-post-overwrites", + ] + ) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "FFmpegExtractAudio", + "preferredcodec": "opus", + "preferredquality": "9", + "nopostoverwrites": True, + }, + ) def test_recode_video(self): opts = self._(["--recode-video", " mkv "]) - self.assertEqual(opts["postprocessors"][0], { - "key": "FFmpegVideoConvertor", - "preferedformat": "mkv", - }) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "FFmpegVideoConvertor", + "preferedformat": "mkv", + }, + ) def test_subs(self): opts = self._(["--convert-subs", "srt"]) @@ -173,12 +188,14 @@ def test_embed(self): subs["already_have_subtitle"] = True thumb["already_have_thumbnail"] = "all" - opts = self._([ - "--embed-thumbnail", - "--embed-subs", - "--write-sub", - "--write-all-thumbnails", - ]) + opts = self._( + [ + "--embed-thumbnail", + "--embed-subs", + "--write-sub", + "--write-all-thumbnails", + ] + ) self.assertEqual(opts["postprocessors"][:2], [subs, thumb]) def test_metadata(self): @@ -187,10 +204,13 @@ def test_metadata(self): def test_metadata_from_title(self): opts = self._(["--metadata-from-title", "%(artist)s - %(title)s"]) - self.assertEqual(opts["postprocessors"][0], { - "key": "MetadataFromTitle", - "titleformat": "%(artist)s - %(title)s", - }) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "MetadataFromTitle", + "titleformat": "%(artist)s - %(title)s", + }, + ) def test_xattr(self): self._("--xattr-set-filesize", "xattr_set_filesize", True) @@ -213,11 +233,14 @@ def test_noop(self): ] if self.module_name != "yt_dlp": - cmdline.extend(( - "--dump-json", - "--dump-single-json", - "--config-location", "~", - )) + cmdline.extend( + ( + "--dump-json", + "--dump-single-json", + "--config-location", + "~", + ) + ) result = self._(cmdline) result["daterange"] = self.default["daterange"] @@ -244,30 +267,38 @@ def test_retries_extractor(self): def test_remuxs_video(self): opts = self._(["--remux-video", " mkv "]) - self.assertEqual(opts["postprocessors"][0], { - "key": "FFmpegVideoRemuxer", - "preferedformat": "mkv", - }) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "FFmpegVideoRemuxer", + "preferedformat": "mkv", + }, + ) def test_metadata(self): - opts = self._(["--embed-metadata", - "--no-embed-chapters", - "--embed-info-json"]) - self.assertEqual(opts["postprocessors"][0], { - "key": "FFmpegMetadata", - "add_chapters": False, - "add_metadata": True, - "add_infojson": True, - }) + opts = self._(["--embed-metadata", "--no-embed-chapters", "--embed-info-json"]) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "FFmpegMetadata", + "add_chapters": False, + "add_metadata": True, + "add_infojson": True, + }, + ) def test_metadata_from_title(self): opts = self._(["--metadata-from-title", "%(artist)s - %(title)s"]) - self.assertEqual(opts["postprocessors"][0], { - "key" : "MetadataParser", - "when" : "pre_process", - "actions": [self.module.MetadataFromFieldPP.to_action( - "title:%(artist)s - %(title)s")], - }) + self.assertEqual( + opts["postprocessors"][0], + { + "key": "MetadataParser", + "when": "pre_process", + "actions": [ + self.module.MetadataFromFieldPP.to_action("title:%(artist)s - %(title)s") + ], + }, + ) def test_geo_bypass(self): try: @@ -276,37 +307,42 @@ def test_geo_bypass(self): # before --xff (c16644642) return Test_CommandlineArguments.test_geo_bypass(self) - self._(["--xff", "default"], - "geo_bypass", "default") - self._(["--xff", "never"], - "geo_bypass", "never") - self._(["--xff", "EN"], - "geo_bypass", "EN") - self._(["--xff", "198.51.100.14/24"], - "geo_bypass", "198.51.100.14/24") - - self._("--geo-bypass", - "geo_bypass", "default") - self._("--no-geo-bypass", - "geo_bypass", "never") - self._(["--geo-bypass-country", "EN"], - "geo_bypass", "EN") - self._(["--geo-bypass-ip-block", "198.51.100.14/24"], - "geo_bypass", "198.51.100.14/24") + self._(["--xff", "default"], "geo_bypass", "default") + self._(["--xff", "never"], "geo_bypass", "never") + self._(["--xff", "EN"], "geo_bypass", "EN") + self._(["--xff", "198.51.100.14/24"], "geo_bypass", "198.51.100.14/24") + + self._("--geo-bypass", "geo_bypass", "default") + self._("--no-geo-bypass", "geo_bypass", "never") + self._(["--geo-bypass-country", "EN"], "geo_bypass", "EN") + self._(["--geo-bypass-ip-block", "198.51.100.14/24"], "geo_bypass", "198.51.100.14/24") def test_cookiesfrombrowser(self): - self._(["--cookies-from-browser", "firefox"], - "cookiesfrombrowser", ("firefox", None, None, None)) - self._(["--cookies-from-browser", "firefox:profile"], - "cookiesfrombrowser", ("firefox", "profile", None, None)) - self._(["--cookies-from-browser", "firefox+keyring"], - "cookiesfrombrowser", ("firefox", None, "KEYRING", None)) - self._(["--cookies-from-browser", "firefox::container"], - "cookiesfrombrowser", ("firefox", None, None, "container")) - self._(["--cookies-from-browser", - "firefox+keyring:profile::container"], - "cookiesfrombrowser", - ("firefox", "profile", "KEYRING", "container")) + self._( + ["--cookies-from-browser", "firefox"], + "cookiesfrombrowser", + ("firefox", None, None, None), + ) + self._( + ["--cookies-from-browser", "firefox:profile"], + "cookiesfrombrowser", + ("firefox", "profile", None, None), + ) + self._( + ["--cookies-from-browser", "firefox+keyring"], + "cookiesfrombrowser", + ("firefox", None, "KEYRING", None), + ) + self._( + ["--cookies-from-browser", "firefox::container"], + "cookiesfrombrowser", + ("firefox", None, None, "container"), + ) + self._( + ["--cookies-from-browser", "firefox+keyring:profile::container"], + "cookiesfrombrowser", + ("firefox", "profile", "KEYRING", "container"), + ) if __name__ == "__main__": From 0a14b9e449a02ec863378223cef5e22f4ea2ded5 Mon Sep 17 00:00:00 2001 From: breakid Date: Sat, 16 Nov 2024 14:56:59 -0500 Subject: [PATCH 2/9] Removed linting exceptions for tests --- pyproject.toml | 22 +--------------------- test/results/2chen.py | 2 +- test/results/4archive.py | 3 ++- test/results/8muses.py | 2 +- test/results/bilibili.py | 3 --- test/results/bluesky.py | 2 -- test/results/boosty.py | 2 -- test/results/everia.py | 3 --- test/results/flickr.py | 2 -- test/results/rule34xyz.py | 3 --- test/results/tumblr.py | 2 -- test/results/weibo.py | 2 -- test/test_cache.py | 4 ++-- test/test_config.py | 3 ++- test/test_cookies.py | 3 ++- test/test_downloader.py | 8 ++++++-- test/test_extractor.py | 7 ++++--- test/test_formatter.py | 4 +++- test/test_job.py | 7 +++++-- test/test_oauth.py | 5 +++-- test/test_postprocessor.py | 10 ++++++++-- test/test_results.py | 8 ++++++-- test/test_text.py | 3 ++- test/test_util.py | 7 ++++--- test/test_ytdl.py | 3 ++- 25 files changed, 54 insertions(+), 66 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a5a4d84b20..cba6eb75f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,7 +87,6 @@ force-single-line = true known-first-party = ["gallery_dl"] -# Ignore `E402` (import violations) in all `__init__.py` files, and in select subdirectories. [tool.ruff.lint.per-file-ignores] "gallery_dl/extractor/500px.py" = [ "E501", # Line too long; file uses large query strings @@ -96,26 +95,7 @@ "E501", # Line too long; file uses large query strings ] "test/**" = [ - "C4", # flake8-comprehensions - "COM", # flake8-commas - "E", # pycodestyle (errors) - "EXE", # flake8-executable - "F", # Pyflakes - "FA", # flake8-future-annotations - "FLY", # flynt - "FURB", # refurb - "G", # flake8-logging-format - "I", # isort - "ICN", # flake8-import-conventions - "ISC", # flake8-implicit-str-concat - "PGH", # pygrep-hooks - "PIE", # flake8-pie - "PYI", # flake8-pyi - "Q", # flake8-quotes - "SIM", # flake8-simplify - "T10", # flake8-debugger - "TCH", # flake8-type-checking - "UP", # pyupgrade + "E501", # Line too long ] [tool.ruff.lint.pydocstyle] diff --git a/test/results/2chen.py b/test/results/2chen.py index cd864c8b37..adce745f88 100644 --- a/test/results/2chen.py +++ b/test/results/2chen.py @@ -1,10 +1,10 @@ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import datetime gallery_dl = __import__("gallery_dl.extractor.2chen") _2chen = getattr(gallery_dl.extractor, "2chen") -import datetime __tests__ = ( { diff --git a/test/results/4archive.py b/test/results/4archive.py index b9883e12b5..404ed37d60 100644 --- a/test/results/4archive.py +++ b/test/results/4archive.py @@ -2,9 +2,10 @@ # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +import datetime + gallery_dl = __import__("gallery_dl.extractor.4archive") _4archive = getattr(gallery_dl.extractor, "4archive") -import datetime __tests__ = ( { diff --git a/test/results/8muses.py b/test/results/8muses.py index 7c18b90a9d..b1038b2b41 100644 --- a/test/results/8muses.py +++ b/test/results/8muses.py @@ -1,10 +1,10 @@ # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. +from gallery_dl import exception gallery_dl = __import__("gallery_dl.extractor.8muses") _8muses = getattr(gallery_dl.extractor, "8muses") -from gallery_dl import exception __tests__ = ( { diff --git a/test/results/bilibili.py b/test/results/bilibili.py index 727ad5984a..4f1908096b 100644 --- a/test/results/bilibili.py +++ b/test/results/bilibili.py @@ -1,12 +1,9 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import bilibili - __tests__ = ( { "#url": "https://www.bilibili.com/opus/988425412565532689", diff --git a/test/results/bluesky.py b/test/results/bluesky.py index adcddc1262..591cc26e7b 100644 --- a/test/results/bluesky.py +++ b/test/results/bluesky.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. diff --git a/test/results/boosty.py b/test/results/boosty.py index 00d2020469..3957166c7b 100644 --- a/test/results/boosty.py +++ b/test/results/boosty.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. diff --git a/test/results/everia.py b/test/results/everia.py index 8d3a6d689d..77fa1b802e 100644 --- a/test/results/everia.py +++ b/test/results/everia.py @@ -1,12 +1,9 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import everia - __tests__ = ( { "#url": "https://everia.club/2024/09/23/mikacho-조미카-joapictures-someday/", diff --git a/test/results/flickr.py b/test/results/flickr.py index 983a2f7991..092acfc19f 100644 --- a/test/results/flickr.py +++ b/test/results/flickr.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. diff --git a/test/results/rule34xyz.py b/test/results/rule34xyz.py index 58fe9bd5b8..288df66c72 100644 --- a/test/results/rule34xyz.py +++ b/test/results/rule34xyz.py @@ -1,12 +1,9 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. from gallery_dl.extractor import rule34xyz - __tests__ = ( { "#url": "https://rule34.xyz/sfw", diff --git a/test/results/tumblr.py b/test/results/tumblr.py index 1e6e6f95fd..72ac5d8e0c 100644 --- a/test/results/tumblr.py +++ b/test/results/tumblr.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. diff --git a/test/results/weibo.py b/test/results/weibo.py index 6b62ebafc8..9c78ee6d22 100644 --- a/test/results/weibo.py +++ b/test/results/weibo.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. diff --git a/test/test_cache.py b/test/test_cache.py index f3d5dd6568..dd639e324c 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -13,11 +13,11 @@ from unittest.mock import patch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import config, util # noqa E402 +from gallery_dl import config # noqa: E402 dbpath = tempfile.mkstemp()[1] config.set(("cache",), "file", dbpath) -from gallery_dl import cache +from gallery_dl import cache # noqa: E402 cache._init() diff --git a/test/test_config.py b/test/test_config.py index 0c25c1526f..14fc705be2 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -13,7 +13,8 @@ ROOTDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOTDIR) -from gallery_dl import config, util # noqa E402 +from gallery_dl import config # noqa: E402 +from gallery_dl import util # noqa: E402 class TestConfig(unittest.TestCase): diff --git a/test/test_cookies.py b/test/test_cookies.py index 5443a36c27..591b15161e 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -16,7 +16,8 @@ from unittest import mock sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import config, extractor # noqa E402 +from gallery_dl import config # noqa: E402 +from gallery_dl import extractor # noqa: E402 class TestCookiejar(unittest.TestCase): diff --git a/test/test_downloader.py b/test/test_downloader.py index 490e87f191..6b05630119 100644 --- a/test/test_downloader.py +++ b/test/test_downloader.py @@ -21,8 +21,12 @@ from unittest.mock import patch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import downloader, extractor, output, config, path # noqa E402 -from gallery_dl.downloader.http import MIME_TYPES, SIGNATURE_CHECKS # noqa E402 +from gallery_dl import config # noqa: E402 +from gallery_dl import downloader # noqa: E402 +from gallery_dl import extractor # noqa: E402 +from gallery_dl import output # noqa: E402 +from gallery_dl import path # noqa: E402 +from gallery_dl.downloader.http import MIME_TYPES # noqa: E402 class MockDownloaderModule(Mock): diff --git a/test/test_extractor.py b/test/test_extractor.py index bfa18cbb8f..1818610919 100644 --- a/test/test_extractor.py +++ b/test/test_extractor.py @@ -16,9 +16,10 @@ from unittest.mock import patch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import extractor, util # noqa E402 -from gallery_dl.extractor import mastodon # noqa E402 -from gallery_dl.extractor.common import Extractor, Message +from gallery_dl import extractor # noqa: E402 +from gallery_dl import util # noqa: E402 +from gallery_dl.extractor.common import Extractor +from gallery_dl.extractor.common import Message from gallery_dl.extractor.directlink import DirectlinkExtractor _list_classes = extractor._list_classes diff --git a/test/test_formatter.py b/test/test_formatter.py index 19a619aba1..587f4f136a 100644 --- a/test/test_formatter.py +++ b/test/test_formatter.py @@ -15,7 +15,9 @@ from time import sleep sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import formatter, text, util # noqa E402 +from gallery_dl import formatter # noqa: E402 +from gallery_dl import text # noqa: E402 +from gallery_dl import util # noqa: E402 class TestFormatter(unittest.TestCase): diff --git a/test/test_job.py b/test/test_job.py index f206026617..b987b2c1cf 100644 --- a/test/test_job.py +++ b/test/test_job.py @@ -13,8 +13,11 @@ from unittest.mock import patch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import job, config, text # noqa E402 -from gallery_dl.extractor.common import Extractor, Message +from gallery_dl import config # noqa: E402 +from gallery_dl import job # noqa: E402 +from gallery_dl import text # noqa: E402 +from gallery_dl.extractor.common import Extractor +from gallery_dl.extractor.common import Message class TestJob(unittest.TestCase): diff --git a/test/test_oauth.py b/test/test_oauth.py index f85b0a6273..65a1772d43 100644 --- a/test/test_oauth.py +++ b/test/test_oauth.py @@ -12,7 +12,8 @@ from unittest.mock import patch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import oauth, text # noqa E402 +from gallery_dl import oauth # noqa: E402 +from gallery_dl import text # noqa: E402 TESTSERVER = "http://term.ie/oauth/example" CONSUMER_KEY = "key" @@ -38,7 +39,7 @@ def test_concat(self): ) def test_nonce(self, size=16): - nonce_values = set(oauth.nonce(size) for _ in range(size)) + nonce_values = {oauth.nonce(size) for _ in range(size)} # uniqueness self.assertEqual(len(nonce_values), size) diff --git a/test/test_postprocessor.py b/test/test_postprocessor.py index 13c78f452c..02ee5dcd7f 100644 --- a/test/test_postprocessor.py +++ b/test/test_postprocessor.py @@ -16,10 +16,16 @@ import zipfile from datetime import datetime from pathlib import Path -from unittest.mock import Mock, mock_open, patch +from unittest.mock import Mock +from unittest.mock import mock_open +from unittest.mock import patch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import config, extractor, output, path, postprocessor # noqa E402 +from gallery_dl import config # noqa: E402 +from gallery_dl import extractor # noqa: E402 +from gallery_dl import output # noqa: E402 +from gallery_dl import path # noqa: E402 +from gallery_dl import postprocessor # noqa: E402 from gallery_dl.postprocessor.common import PostProcessor diff --git a/test/test_results.py b/test/test_results.py index ec7f70ded0..b2bb401092 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -16,8 +16,12 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import extractor, util, job, config, exception, formatter # noqa E402 - +from gallery_dl import config # noqa: E402 +from gallery_dl import exception # noqa: E402 +from gallery_dl import extractor # noqa: E402 +from gallery_dl import formatter # noqa: E402 +from gallery_dl import job # noqa: E402 +from gallery_dl import util # noqa: E402 RESULTS = os.environ.get("GDL_TEST_RESULTS") if RESULTS: diff --git a/test/test_text.py b/test/test_text.py index 490dbec69d..cee7c0cf2c 100644 --- a/test/test_text.py +++ b/test/test_text.py @@ -12,7 +12,8 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import text, util # noqa E402 +from gallery_dl import text # noqa: E402 +from gallery_dl import util # noqa: E402 INVALID = ((), [], {}, None, 1, 2.3) INVALID_ALT = ((), [], {}, None, "") diff --git a/test/test_util.py b/test/test_util.py index 30cb849f77..d7e90d3e72 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -20,7 +20,9 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import util, text, exception # noqa E402 +from gallery_dl import exception # noqa: E402 +from gallery_dl import text # noqa: E402 +from gallery_dl import util # noqa: E402 class TestRange(unittest.TestCase): @@ -656,8 +658,7 @@ def test_enumerate_reversed(self): result = [(3, 33), (2, 22), (1, 11)] def gen(): - for i in seq: - yield i + yield from seq def gen_2(): yield from seq diff --git a/test/test_ytdl.py b/test/test_ytdl.py index 20c3af6668..ebdb307ee2 100644 --- a/test/test_ytdl.py +++ b/test/test_ytdl.py @@ -11,7 +11,8 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from gallery_dl import ytdl, util, config # noqa E402 +from gallery_dl import util # noqa: E402 +from gallery_dl import ytdl # noqa: E402 class Test_CommandlineArguments(unittest.TestCase): From 25f56537ff2973818c42de1a35c14d465f3d99de Mon Sep 17 00:00:00 2001 From: breakid Date: Sat, 16 Nov 2024 15:36:16 -0500 Subject: [PATCH 3/9] Updated line-length and supported versions in 'tests' GitHub Action --- .github/workflows/tests.yml | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c88dde73e7..8f3a845955 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,10 +17,6 @@ jobs: fail-fast: false matrix: python-version: - - "3.5" - - "3.6" - - "3.7" - - "3.8" - "3.9" - "3.10" - "3.11" @@ -49,30 +45,12 @@ jobs: - name: Install yt-dlp run: | - case "${{ matrix.python-version }}" in - 3.4|3.5) - # don't install yt-dlp - ;; - 3.6|3.7|3.8) - # install from PyPI - pip install yt-dlp - ;; - *) - # install from master - pip install https://github.com/yt-dlp/yt-dlp/archive/refs/heads/master.tar.gz - ;; - esac + # install from master + pip install https://github.com/yt-dlp/yt-dlp/archive/refs/heads/master.tar.gz - name: Lint with flake8 run: | - case "${{ matrix.python-version }}" in - 3.4|3.5|3.6|3.7) - flake8 --extend-exclude scripts/export_tests.py,scripts/pyprint.py . - ;; - *) - flake8 . - ;; - esac + flake8 --max-line-length=100 . - name: Run tests run: | From d4950f9b6a79e4391e11cde3e22d067e25a07fa1 Mon Sep 17 00:00:00 2001 From: breakid Date: Sat, 16 Nov 2024 15:56:28 -0500 Subject: [PATCH 4/9] Updated GitHub Actions to run Ruff instead of flake8 --- .github/workflows/tests.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8f3a845955..ee4e7cf295 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,12 +11,15 @@ on: jobs: test: - runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: python-version: + - "3.5" + - "3.6" + - "3.7" + - "3.8" - "3.9" - "3.10" - "3.11" @@ -41,16 +44,20 @@ jobs: - name: Install dependencies run: | pip install -r requirements.txt - pip install flake8 youtube-dl + pip install ruff youtube-dl - name: Install yt-dlp run: | # install from master pip install https://github.com/yt-dlp/yt-dlp/archive/refs/heads/master.tar.gz - - name: Lint with flake8 + - name: Lint with Ruff + run: | + ruff check + + - name: Check formatting with Ruff run: | - flake8 --max-line-length=100 . + ruff format --check - name: Run tests run: | From b1bad2e1dc3de36c6c7512a1ed92c7b8b98d36fe Mon Sep 17 00:00:00 2001 From: breakid Date: Sat, 16 Nov 2024 15:59:58 -0500 Subject: [PATCH 5/9] Removed flake8-executable check --- .github/workflows/tests.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ee4e7cf295..2a44fee438 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,8 +48,19 @@ jobs: - name: Install yt-dlp run: | - # install from master - pip install https://github.com/yt-dlp/yt-dlp/archive/refs/heads/master.tar.gz + case "${{ matrix.python-version }}" in + 3.4|3.5) + # don't install yt-dlp + ;; + 3.6|3.7|3.8) + # install from PyPI + pip install yt-dlp + ;; + *) + # install from master + pip install https://github.com/yt-dlp/yt-dlp/archive/refs/heads/master.tar.gz + ;; + esac - name: Lint with Ruff run: | From 74948ef2e01119f6d2193990b6a92cb0adb174f4 Mon Sep 17 00:00:00 2001 From: breakid Date: Sat, 16 Nov 2024 16:04:01 -0500 Subject: [PATCH 6/9] Actually removed flake8-executable check --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cba6eb75f4..ea2947a6ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,6 @@ "C4", # flake8-comprehensions "COM", # flake8-commas "E", # pycodestyle (errors) - "EXE", # flake8-executable "F", # Pyflakes "FA", # flake8-future-annotations "FLY", # flynt @@ -57,6 +56,7 @@ # "DTZ", # flake8-datetimez # "EM", # flake8-errmsg # "ERA", # eradicate + # "EXE", # flake8-executable # "FBT", # flake8-boolean-trap # "INP", # flake8-no-pep420 # "N", # pep8-naming From 7a9f174e41f3169ba4e6f78aed052e74ce0bcef4 Mon Sep 17 00:00:00 2001 From: breakid Date: Sat, 16 Nov 2024 16:31:29 -0500 Subject: [PATCH 7/9] Fixed a typo in run_tests.py --- .github/workflows/tests.yml | 2 -- scripts/run_tests.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2a44fee438..38cb281337 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,8 +16,6 @@ jobs: fail-fast: false matrix: python-version: - - "3.5" - - "3.6" - "3.7" - "3.8" - "3.9" diff --git a/scripts/run_tests.py b/scripts/run_tests.py index ff4bc643d1..d867a35e66 100755 --- a/scripts/run_tests.py +++ b/scripts/run_tests.py @@ -1,4 +1,4 @@ -# !/usr/bin/env python3 +#!/usr/bin/env python3 # Copyright 2021 Mike Fährmann # From 5297cbf945f39b81ea219d2ff0bc9cb92c881996 Mon Sep 17 00:00:00 2001 From: breakid Date: Sat, 16 Nov 2024 17:23:42 -0500 Subject: [PATCH 8/9] Removed type hint that broke in python3.9 --- gallery_dl/extractor/common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gallery_dl/extractor/common.py b/gallery_dl/extractor/common.py index d385196b5c..a324968d6a 100644 --- a/gallery_dl/extractor/common.py +++ b/gallery_dl/extractor/common.py @@ -24,7 +24,6 @@ from typing import Any from typing import AnyStr from typing import Literal -from typing import Self import requests from requests.adapters import HTTPAdapter @@ -72,7 +71,7 @@ def __init__(self, match: re.Match[AnyStr]) -> None: self._parentdir = "" @classmethod - def from_url(cls, url) -> Self | None: + def from_url(cls, url): if isinstance(cls.pattern, str): cls.pattern = re.compile(cls.pattern) match = cls.pattern.match(url) From 4838adf8897e4d0880e6d473c2f6b251900633f7 Mon Sep 17 00:00:00 2001 From: breakid Date: Sat, 16 Nov 2024 17:31:02 -0500 Subject: [PATCH 9/9] Replaced pypy3.9 with pypy3.10, the currently supported version --- .github/workflows/tests.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 38cb281337..0a6310522d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,14 +16,12 @@ jobs: fail-fast: false matrix: python-version: - - "3.7" - - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - - "pypy3.9" + - "pypy3.10" steps: - uses: actions/checkout@v4