-
Notifications
You must be signed in to change notification settings - Fork 0
/
scryfaller.py
314 lines (260 loc) · 9.33 KB
/
scryfaller.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
"""
Copyright (C) 2021 Marvin Lenk
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""
import requests as req
import urllib.parse
import shutil
import tempfile
import threading
import os
class picLoadThread(threading.Thread):
"""Thread for downloading a picture."""
def __init__(self, threadID, workarr, func):
threading.Thread.__init__(self)
self.threadID = threadID
self.workarr = workarr
self.func = func
def run(self):
self.func(self.workarr)
def isdf(apireq):
"""Checks if card is double faced by searching for 'dfc' in the layout.
Returns array of bools for each card individually"""
keywords = ['dfc', 'transform']
return [any(x in apireq['data'][i]['layout'] for x in keywords) for i in range(0, apireq['total_cards'])]
def searchapi(card_name, scryconf, strict=None):
"""Generate request url from card name (with additional infos)"""
# First, look for additional infos in front
cname, set, cnum = stripinfos(card_name)
# Set flag - from card name
set_str = ''
if set != '':
set_str = '+set=' + set
# Collector number flag - from card name
cnum_str = ''
if cnum != -1:
cnum_str = '+cn=' + str(cnum)
# Uniqueness flag
unique_str = scryconf.get_searchflag('unique')
if unique_str != '':
unique_str = '&unique=' + unique_str
# Paper printing flag
game_str = scryconf.get_searchflag('game')
if game_str != '':
game_str = '+game=' + game_str
# Order flag
order_str = scryconf.get_searchflag('order')
if order_str != '':
order_str = '&order=' + order_str
# Strict name
card_str = urllib.parse.quote(str(cname))
if strict is None:
strict = scryconf.get_searchflag('strict')
if strict:
card_str = '!"' + card_str + '"'
# Language flag
lang_str = scryconf.get_searchflag('lang')
if lang_str != '':
lang_str = '+lang=' + lang_str
# Promo cards - None allows for promos, True forces promos, False disallows
promo_str = scryconf.get_searchflag('promo')
if promo_str != '':
promo_str = '+' if promo_str == 'True' else '+-'
promo_str += 'is=promo'
out = 'https://api.scryfall.com/cards/search?q=' + card_str
out += game_str + set_str + cnum_str + lang_str + promo_str + order_str + unique_str
return out
def getjson(url):
"""Files 'url' request to the server and returns json (in dict form)."""
r = req.get(url)
if r.status_code != 200:
print("Error code " + str(r.status_code))
return None
if 'warnings' in r.json().keys():
print(r.json()['warnings'])
return r.json()
def cardreq(card_name, scryconf, strict=None):
"""Generates json (in dict form) from requested card name (with additional infos).
The flag 'strict' overrides the conf file entry."""
# key 'total_cards' is the length of the data array of individual cards
# key 'data' holds an array with length 'total_cards' of cards
# every entry of the list ist another dictionary
# Pic urls are listed in 'image_uris'
r = getjson(searchapi(card_name, scryconf, strict))
if r is None:
return None
# If more than one page, append data until everything is fetched
while r['has_more']:
rt = getjson(r['next_page'])
if rt['has_more']:
r['next_page'] = rt['next_page']
else:
del r['next_page']
r['has_more'] = False
r['data'].extend(rt['data'])
return r
def loadpic(url, path):
"""Download picture form url to path"""
rpic = req.get(url, stream=True)
if rpic.status_code != 200:
print("Error code " + str(rpic.status_code))
del rpic
return False
with open(path, 'wb') as out_file:
shutil.copyfileobj(rpic.raw, out_file)
del rpic
return True
def loadpicarr(urlpatharr):
"""Downloads all urls to corresponding paths given in an array."""
# array structure is [[url, path], [url, path], ...]
for el in urlpatharr:
loadpic(el[0], el[1])
return True
def tmppics(apireq, dir, tp='normal'):
"""Downloads card previews of the json 'apireq' to a folder 'dir' in multiple threads.
Picture types 'tp' can be 'small', 'normal', 'large', 'png', 'art_crop' and 'border_crop'"""
data = apireq['data']
df = isdf(apireq)
if dir[-1] == os.sep:
pathsep = dir
else:
pathsep = dir + os.sep
ending = '.jpg'
if tp == 'png':
ending = '.png'
ncards = apireq['total_cards']
# Limit number of threads to 20 (40)
nthreads = ncards if ncards < 20 else 20
# Double the number of threads for double faced cards
nthreads *= 2 if df else 1
urlpatharr = []
for i in range(0, apireq['total_cards']):
data_i = data[i]
if df[i]:
urlpatharr.append([data_i['card_faces'][0]['image_uris'][tp], pathsep + str(i) + ending])
urlpatharr.append([data_i['card_faces'][1]['image_uris'][tp], pathsep + str(i) + 'b' + ending])
else:
urlpatharr.append([data_i['image_uris'][tp], pathsep + str(i) + ending])
threadarr = []
sllen = len(urlpatharr) // nthreads
slrest = len(urlpatharr) % nthreads
sllo = 0
slhi = sllen
for i in range(0, nthreads):
# distribute rest as evenly as possible
if slrest > 0:
slhi += 1
slrest -= 1
threadarr.append(picLoadThread(i, urlpatharr[sllo:slhi], loadpicarr))
threadarr[-1].start()
sllo = slhi
slhi = sllo + sllen
for t in threadarr:
t.join()
return True
def getfirstname(text):
"""Extract first valid card name from array of strings."""
name = ''
line = 0
while name == '' and line < len(text):
txt = text[line]
if lineskipcheck(txt):
line += 1
continue
name = stripall(txt)
return line, name
def stripall(text):
"""Strips as many unnecessary infos from a card name as possible."""
# dfc is not stripped here due to the bad implementation
txt = stripnumber(text)
txt = stripcommander(txt)
txt = stripfoil(txt)
return txt
def stripnumber(text):
"""Strips a leading number from a string (including the character after the number)."""
txt = text.strip()
i = 1
check = False
while i < len(txt):
if txt[0:i].isnumeric():
check = True
i += 1
continue
elif check == False:
return txt
else:
# For any reasonable input, expect whitespace or colon after number, so strip this as well
return txt[i:].strip()
return ''
def stripcommander(text):
"""Strips the commander identification from a string."""
txt = text.strip()
# Deckstats format
if txt.find('#!Commander') != -1:
txt = txt[:txt.find('#!Commander')].strip()
# Archidekt format
if txt.find('[Commander') != -1:
txt = txt[:txt.find('[Commander')].strip()
return txt
def stripdfc(text):
"""Returns only the first part of a double-faced card name."""
txt = text.strip()
txtar = txt.split('//')
return txtar[0].strip()
def stripfoil(text):
"""Strips the foil identification from a string."""
txt = text.strip()
# Deckstats format
if txt.find('#!Foil') != -1:
txt = txt[:txt.find('#!Foil')].strip()
# Archidekt format
if txt.find('*F*') != -1:
txt = txt[:txt.find('*F*')].strip()
return txt
def stripinfos(text):
"""Extracts set and collector number info from a string and returns the stripped name,
the set and the collector number."""
txt = text.strip()
name = txt
set = ''
cnum = -1
# check if additional infos are given
# Deckstats format
# Typical inputs will be '[PZNR] Turntimber Symbiosis // ...' or '[AKH#247] Scattered Groves'
if txt[0] == '[':
cpos = txt.find(']')
hpos = txt.find('#')
# if hashtag is provided, collector num is set
if hpos != -1:
i = hpos + 1
while txt[hpos+1:i+1].isnumeric():
i += 1
cnum = int(txt[hpos+1:i])
else:
hpos = cpos
# check if set is actually set or rather collector number
if txt[1:hpos].isnumeric() and cnum == -1:
cnum = int(txt[1:hpos])
else:
set = txt[1:hpos]
name = txt[cpos+1:].strip()
# Archidekt format
# Typical input will be 'Scattered Grove (akh)'
if txt[-1] == ')':
ipos = len(txt) - txt[::-1].find('(')
set = txt[ipos:-1]
name = txt[:ipos-1].strip()
name = stripdfc(name)
return name, set, cnum
def lineskipcheck(text):
"""Checks if the given text is a comment or to short and should be skipped."""
return text[0] == '#' or text[0:2] == '//' or len(text) < 3