-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmain.py
34 lines (32 loc) · 866 Bytes
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
from scrapy.crawler import CrawlerProcess
from quotes_spyder.quotes_spyder.spiders.quotes import QuotesSpider
from quotes_spyder.quotes_spyder.spiders.authors import AuthorsSpider
process_au = CrawlerProcess(settings={
'FEEDS': {
'authors.json': {
'format': 'json',
'encoding': 'utf8',
'store_empty': False,
'fields': None,
'indent': 4
}
}
})
process_qu = CrawlerProcess(settings={
'FEEDS': {
'quotes.json': {
'format': 'json',
'encoding': 'utf8',
'store_empty': False,
'fields': None,
'indent': 4,
'item_export_kwargs': {
'ensure_ascii': False
},
}
}
})
process_qu.crawl(QuotesSpider)
process_au.crawl(AuthorsSpider)
process_au.start()
process_qu.start()